drivers/net/ionic/ionic_rxtx.c

   1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
   2  * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <string.h>
   9 #include <errno.h>
  10 #include <stdint.h>
  11 #include <stdarg.h>
  12 #include <unistd.h>
  13 #include <inttypes.h>
  14
  15 #include <rte_byteorder.h>
  16 #include <rte_common.h>
  17 #include <rte_cycles.h>
  18 #include <rte_log.h>
  19 #include <rte_debug.h>
  20 #include <rte_interrupts.h>
  21 #include <rte_pci.h>
  22 #include <rte_memory.h>
  23 #include <rte_memzone.h>
  24 #include <rte_launch.h>
  25 #include <rte_eal.h>
  26 #include <rte_per_lcore.h>
  27 #include <rte_lcore.h>
  28 #include <rte_atomic.h>
  29 #include <rte_branch_prediction.h>
  30 #include <rte_mempool.h>
  31 #include <rte_malloc.h>
  32 #include <rte_mbuf.h>
  33 #include <rte_ether.h>
  34 #include <rte_ethdev_driver.h>
  35 #include <rte_prefetch.h>
  36 #include <rte_udp.h>
  37 #include <rte_tcp.h>
  38 #include <rte_sctp.h>
  39 #include <rte_string_fns.h>
  40 #include <rte_errno.h>
  41 #include <rte_ip.h>
  42 #include <rte_net.h>
  43
  44 #include "ionic_logs.h"
  45 #include "ionic_mac_api.h"
  46 #include "ionic_ethdev.h"
  47 #include "ionic_lif.h"
  48 #include "ionic_rxtx.h"
  49
  50 #define IONIC_RX_RING_DOORBELL_STRIDE           (32 - 1)
  51
  52 /*********************************************************************
  53  *
  54  *  TX functions
  55  *
  56  **********************************************************************/
  57
  58 void
  59 ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
  60                 struct rte_eth_txq_info *qinfo)
  61 {
  62         struct ionic_qcq *txq = dev->data->tx_queues[queue_id];
  63         struct ionic_queue *q = &txq->q;
  64
  65         qinfo->nb_desc = q->num_descs;
  66         qinfo->conf.offloads = txq->offloads;
  67         qinfo->conf.tx_deferred_start = txq->flags & IONIC_QCQ_F_DEFERRED;
  68 }
  69
  70 static inline void __rte_cold
  71 ionic_tx_flush(struct ionic_cq *cq)
  72 {
  73         struct ionic_queue *q = cq->bound_q;
  74         struct ionic_desc_info *q_desc_info;
  75         struct rte_mbuf *txm, *next;
  76         struct ionic_txq_comp *cq_desc_base = cq->base;
  77         struct ionic_txq_comp *cq_desc;
  78         u_int32_t comp_index = (u_int32_t)-1;
  79
  80         cq_desc = &cq_desc_base[cq->tail_idx];
  81         while (color_match(cq_desc->color, cq->done_color)) {
  82                 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
  83
  84                 /* Prefetch the next 4 descriptors (not really useful here) */
  85                 if ((cq->tail_idx & 0x3) == 0)
  86                         rte_prefetch0(&cq_desc_base[cq->tail_idx]);
  87
  88                 if (cq->tail_idx == 0)
  89                         cq->done_color = !cq->done_color;
  90
  91                 comp_index = cq_desc->comp_index;
  92
  93                 cq_desc = &cq_desc_base[cq->tail_idx];
  94         }
  95
  96         if (comp_index != (u_int32_t)-1) {
  97                 while (q->tail_idx != comp_index) {
  98                         q_desc_info = &q->info[q->tail_idx];
  99
 100                         q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
 101
 102                         /* Prefetch the next 4 descriptors */
 103                         if ((q->tail_idx & 0x3) == 0)
 104                                 /* q desc info */
 105                                 rte_prefetch0(&q->info[q->tail_idx]);
 106
 107                         /*
 108                          * Note: you can just use rte_pktmbuf_free,
 109                          * but this loop is faster
 110                          */
 111                         txm = q_desc_info->cb_arg;
 112                         while (txm != NULL) {
 113                                 next = txm->next;
 114                                 rte_pktmbuf_free_seg(txm);
 115                                 txm = next;
 116                         }
 117                 }
 118         }
 119 }
 120
 121 void __rte_cold
 122 ionic_dev_tx_queue_release(void *tx_queue)
 123 {
 124         struct ionic_qcq *txq = (struct ionic_qcq *)tx_queue;
 125
 126         IONIC_PRINT_CALL();
 127
 128         ionic_qcq_free(txq);
 129 }
 130
 131 int __rte_cold
 132 ionic_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id)
 133 {
 134         struct ionic_qcq *txq;
 135
 136         IONIC_PRINT_CALL();
 137
 138         txq = eth_dev->data->tx_queues[tx_queue_id];
 139
 140         /*
 141          * Note: we should better post NOP Tx desc and wait for its completion
 142          * before disabling Tx queue
 143          */
 144
 145         ionic_qcq_disable(txq);
 146
 147         ionic_tx_flush(&txq->cq);
 148
 149         ionic_lif_txq_deinit(txq);
 150
 151         eth_dev->data->tx_queue_state[tx_queue_id] =
 152                 RTE_ETH_QUEUE_STATE_STOPPED;
 153
 154         return 0;
 155 }
 156
 157 int __rte_cold
 158 ionic_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id,
 159                 uint16_t nb_desc, uint32_t socket_id __rte_unused,
 160                 const struct rte_eth_txconf *tx_conf)
 161 {
 162         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 163         struct ionic_qcq *txq;
 164         uint64_t offloads;
 165         int err;
 166
 167         IONIC_PRINT_CALL();
 168
 169         IONIC_PRINT(DEBUG, "Configuring TX queue %u with %u buffers",
 170                 tx_queue_id, nb_desc);
 171
 172         if (tx_queue_id >= lif->ntxqcqs) {
 173                 IONIC_PRINT(DEBUG, "Queue index %u not available "
 174                         "(max %u queues)",
 175                         tx_queue_id, lif->ntxqcqs);
 176                 return -EINVAL;
 177         }
 178
 179         offloads = tx_conf->offloads | eth_dev->data->dev_conf.txmode.offloads;
 180
 181         /* Validate number of receive descriptors */
 182         if (!rte_is_power_of_2(nb_desc) || nb_desc < IONIC_MIN_RING_DESC)
 183                 return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */
 184
 185         /* Free memory prior to re-allocation if needed... */
 186         if (eth_dev->data->tx_queues[tx_queue_id] != NULL) {
 187                 void *tx_queue = eth_dev->data->tx_queues[tx_queue_id];
 188                 ionic_dev_tx_queue_release(tx_queue);
 189                 eth_dev->data->tx_queues[tx_queue_id] = NULL;
 190         }
 191
 192         err = ionic_tx_qcq_alloc(lif, tx_queue_id, nb_desc, &txq);
 193         if (err) {
 194                 IONIC_PRINT(DEBUG, "Queue allocation failure");
 195                 return -EINVAL;
 196         }
 197
 198         /* Do not start queue with rte_eth_dev_start() */
 199         if (tx_conf->tx_deferred_start)
 200                 txq->flags |= IONIC_QCQ_F_DEFERRED;
 201
 202         txq->offloads = offloads;
 203
 204         eth_dev->data->tx_queues[tx_queue_id] = txq;
 205
 206         return 0;
 207 }
 208
 209 /*
 210  * Start Transmit Units for specified queue.
 211  */
 212 int __rte_cold
 213 ionic_dev_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id)
 214 {
 215         struct ionic_qcq *txq;
 216         int err;
 217
 218         IONIC_PRINT_CALL();
 219
 220         txq = eth_dev->data->tx_queues[tx_queue_id];
 221
 222         err = ionic_lif_txq_init(txq);
 223         if (err)
 224                 return err;
 225
 226         ionic_qcq_enable(txq);
 227
 228         eth_dev->data->tx_queue_state[tx_queue_id] =
 229                 RTE_ETH_QUEUE_STATE_STARTED;
 230
 231         return 0;
 232 }
 233
 234 static void
 235 ionic_tx_tcp_pseudo_csum(struct rte_mbuf *txm)
 236 {
 237         struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(txm, struct ether_hdr *);
 238         char *l3_hdr = ((char *)eth_hdr) + txm->l2_len;
 239         struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *)
 240                 (l3_hdr + txm->l3_len);
 241
 242         if (txm->ol_flags & PKT_TX_IP_CKSUM) {
 243                 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr;
 244                 ipv4_hdr->hdr_checksum = 0;
 245                 tcp_hdr->cksum = 0;
 246                 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, tcp_hdr);
 247         } else {
 248                 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr;
 249                 tcp_hdr->cksum = 0;
 250                 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, tcp_hdr);
 251         }
 252 }
 253
 254 static void
 255 ionic_tx_tcp_inner_pseudo_csum(struct rte_mbuf *txm)
 256 {
 257         struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(txm, struct ether_hdr *);
 258         char *l3_hdr = ((char *)eth_hdr) + txm->outer_l2_len +
 259                 txm->outer_l3_len + txm->l2_len;
 260         struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *)
 261                 (l3_hdr + txm->l3_len);
 262
 263         if (txm->ol_flags & PKT_TX_IPV4) {
 264                 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr;
 265                 ipv4_hdr->hdr_checksum = 0;
 266                 tcp_hdr->cksum = 0;
 267                 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, tcp_hdr);
 268         } else {
 269                 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr;
 270                 tcp_hdr->cksum = 0;
 271                 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, tcp_hdr);
 272         }
 273 }
 274
 275 static void
 276 ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc,
 277                 struct rte_mbuf *txm,
 278                 rte_iova_t addr, uint8_t nsge, uint16_t len,
 279                 uint32_t hdrlen, uint32_t mss,
 280                 bool encap,
 281                 uint16_t vlan_tci, bool has_vlan,
 282                 bool start, bool done)
 283 {
 284         uint8_t flags = 0;
 285         flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
 286         flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
 287         flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0;
 288         flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0;
 289
 290         desc->cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO,
 291                 flags, nsge, addr);
 292         desc->len = len;
 293         desc->vlan_tci = vlan_tci;
 294         desc->hdr_len = hdrlen;
 295         desc->mss = mss;
 296
 297         ionic_q_post(q, done, NULL, done ? txm : NULL);
 298 }
 299
 300 static struct ionic_txq_desc *
 301 ionic_tx_tso_next(struct ionic_queue *q, struct ionic_txq_sg_elem **elem)
 302 {
 303         struct ionic_txq_desc *desc_base = q->base;
 304         struct ionic_txq_sg_desc *sg_desc_base = q->sg_base;
 305         struct ionic_txq_desc *desc = &desc_base[q->head_idx];
 306         struct ionic_txq_sg_desc *sg_desc = &sg_desc_base[q->head_idx];
 307
 308         *elem = sg_desc->elems;
 309         return desc;
 310 }
 311
 312 static int
 313 ionic_tx_tso(struct ionic_queue *q, struct rte_mbuf *txm,
 314                 uint64_t offloads __rte_unused, bool not_xmit_more)
 315 {
 316         struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q);
 317         struct ionic_txq_desc *desc;
 318         struct ionic_txq_sg_elem *elem;
 319         struct rte_mbuf *txm_seg;
 320         uint64_t desc_addr = 0;
 321         uint16_t desc_len = 0;
 322         uint8_t desc_nsge;
 323         uint32_t hdrlen;
 324         uint32_t mss = txm->tso_segsz;
 325         uint32_t frag_left = 0;
 326         uint32_t left;
 327         uint32_t seglen;
 328         uint32_t len;
 329         uint32_t offset = 0;
 330         bool start, done;
 331         bool encap;
 332         bool has_vlan = !!(txm->ol_flags & PKT_TX_VLAN_PKT);
 333         uint16_t vlan_tci = txm->vlan_tci;
 334         uint64_t ol_flags = txm->ol_flags;
 335
 336         encap = ((ol_flags & PKT_TX_OUTER_IP_CKSUM) ||
 337                 (ol_flags & PKT_TX_OUTER_UDP_CKSUM)) &&
 338                 ((ol_flags & PKT_TX_OUTER_IPV4) ||
 339                 (ol_flags & PKT_TX_OUTER_IPV6));
 340
 341         /* Preload inner-most TCP csum field with IP pseudo hdr
 342          * calculated with IP length set to zero.  HW will later
 343          * add in length to each TCP segment resulting from the TSO.
 344          */
 345
 346         if (encap) {
 347                 ionic_tx_tcp_inner_pseudo_csum(txm);
 348                 hdrlen = txm->outer_l2_len + txm->outer_l3_len +
 349                         txm->l2_len + txm->l3_len + txm->l4_len;
 350         } else {
 351                 ionic_tx_tcp_pseudo_csum(txm);
 352                 hdrlen = txm->l2_len + txm->l3_len + txm->l4_len;
 353         }
 354
 355         seglen = hdrlen + mss;
 356         left = txm->data_len;
 357
 358         desc = ionic_tx_tso_next(q, &elem);
 359         start = true;
 360
 361         /* Chop data up into desc segments */
 362
 363         while (left > 0) {
 364                 len = RTE_MIN(seglen, left);
 365                 frag_left = seglen - len;
 366                 desc_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(txm));
 367                 desc_len = len;
 368                 desc_nsge = 0;
 369                 left -= len;
 370                 offset += len;
 371                 if (txm->nb_segs > 1 && frag_left > 0)
 372                         continue;
 373                 done = (txm->nb_segs == 1 && left == 0);
 374                 ionic_tx_tso_post(q, desc, txm,
 375                         desc_addr, desc_nsge, desc_len,
 376                         hdrlen, mss,
 377                         encap,
 378                         vlan_tci, has_vlan,
 379                         start, done && not_xmit_more);
 380                 desc = ionic_tx_tso_next(q, &elem);
 381                 start = false;
 382                 seglen = mss;
 383         }
 384
 385         /* Chop frags into desc segments */
 386
 387         txm_seg = txm->next;
 388         while (txm_seg != NULL) {
 389                 offset = 0;
 390                 left = txm_seg->data_len;
 391                 stats->frags++;
 392
 393                 while (left > 0) {
 394                         rte_iova_t data_iova;
 395                         data_iova = rte_mbuf_data_iova(txm_seg);
 396                         elem->addr = rte_cpu_to_le_64(data_iova) + offset;
 397                         if (frag_left > 0) {
 398                                 len = RTE_MIN(frag_left, left);
 399                                 frag_left -= len;
 400                                 elem->len = len;
 401                                 elem++;
 402                                 desc_nsge++;
 403                         } else {
 404                                 len = RTE_MIN(mss, left);
 405                                 frag_left = mss - len;
 406                                 data_iova = rte_mbuf_data_iova(txm_seg);
 407                                 desc_addr = rte_cpu_to_le_64(data_iova);
 408                                 desc_len = len;
 409                                 desc_nsge = 0;
 410                         }
 411                         left -= len;
 412                         offset += len;
 413                         if (txm_seg->next != NULL && frag_left > 0)
 414                                 continue;
 415                         done = (txm_seg->next == NULL && left == 0);
 416                         ionic_tx_tso_post(q, desc, txm_seg,
 417                                 desc_addr, desc_nsge, desc_len,
 418                                 hdrlen, mss,
 419                                 encap,
 420                                 vlan_tci, has_vlan,
 421                                 start, done && not_xmit_more);
 422                         desc = ionic_tx_tso_next(q, &elem);
 423                         start = false;
 424                 }
 425
 426                 txm_seg = txm_seg->next;
 427         }
 428
 429         stats->tso++;
 430
 431         return 0;
 432 }
 433
 434 static int
 435 ionic_tx(struct ionic_queue *q, struct rte_mbuf *txm,
 436                 uint64_t offloads, bool not_xmit_more)
 437 {
 438         struct ionic_txq_desc *desc_base = q->base;
 439         struct ionic_txq_sg_desc *sg_desc_base = q->sg_base;
 440         struct ionic_txq_desc *desc = &desc_base[q->head_idx];
 441         struct ionic_txq_sg_desc *sg_desc = &sg_desc_base[q->head_idx];
 442         struct ionic_txq_sg_elem *elem = sg_desc->elems;
 443         struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q);
 444         struct rte_mbuf *txm_seg;
 445         bool encap;
 446         bool has_vlan;
 447         uint64_t ol_flags = txm->ol_flags;
 448         uint64_t addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(txm));
 449         uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE;
 450         uint8_t flags = 0;
 451
 452         if ((ol_flags & PKT_TX_IP_CKSUM) &&
 453                         (offloads & DEV_TX_OFFLOAD_IPV4_CKSUM)) {
 454                 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW;
 455                 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3;
 456                 if (((ol_flags & PKT_TX_TCP_CKSUM) &&
 457                                 (offloads & DEV_TX_OFFLOAD_TCP_CKSUM)) ||
 458                                 ((ol_flags & PKT_TX_UDP_CKSUM) &&
 459                                 (offloads & DEV_TX_OFFLOAD_UDP_CKSUM)))
 460                         flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4;
 461         } else {
 462                 stats->no_csum++;
 463         }
 464
 465         has_vlan = (ol_flags & PKT_TX_VLAN_PKT);
 466         encap = ((ol_flags & PKT_TX_OUTER_IP_CKSUM) ||
 467                         (ol_flags & PKT_TX_OUTER_UDP_CKSUM)) &&
 468                         ((ol_flags & PKT_TX_OUTER_IPV4) ||
 469                         (ol_flags & PKT_TX_OUTER_IPV6));
 470
 471         flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
 472         flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
 473
 474         desc->cmd = encode_txq_desc_cmd(opcode, flags, txm->nb_segs - 1, addr);
 475         desc->len = txm->data_len;
 476         desc->vlan_tci = txm->vlan_tci;
 477
 478         txm_seg = txm->next;
 479         while (txm_seg != NULL) {
 480                 elem->len = txm_seg->data_len;
 481                 elem->addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm_seg));
 482                 stats->frags++;
 483                 elem++;
 484                 txm_seg = txm_seg->next;
 485         }
 486
 487         ionic_q_post(q, not_xmit_more, NULL, txm);
 488
 489         return 0;
 490 }
 491
 492 uint16_t
 493 ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 494                 uint16_t nb_pkts)
 495 {
 496         struct ionic_qcq *txq = (struct ionic_qcq *)tx_queue;
 497         struct ionic_queue *q = &txq->q;
 498         struct ionic_cq *cq = &txq->cq;
 499         struct ionic_tx_stats *stats = IONIC_Q_TO_TX_STATS(q);
 500         uint32_t next_q_head_idx;
 501         uint32_t bytes_tx = 0;
 502         uint16_t nb_tx = 0;
 503         int err;
 504         bool last;
 505
 506         /* Cleaning old buffers */
 507         ionic_tx_flush(cq);
 508
 509         if (unlikely(ionic_q_space_avail(q) < nb_pkts)) {
 510                 stats->stop += nb_pkts;
 511                 return 0;
 512         }
 513
 514         while (nb_tx < nb_pkts) {
 515                 last = (nb_tx == (nb_pkts - 1));
 516
 517                 next_q_head_idx = (q->head_idx + 1) & (q->num_descs - 1);
 518                 if ((next_q_head_idx & 0x3) == 0) {
 519                         struct ionic_txq_desc *desc_base = q->base;
 520                         rte_prefetch0(&desc_base[next_q_head_idx]);
 521                         rte_prefetch0(&q->info[next_q_head_idx]);
 522                 }
 523
 524                 if (tx_pkts[nb_tx]->ol_flags & PKT_TX_TCP_SEG)
 525                         err = ionic_tx_tso(q, tx_pkts[nb_tx], txq->offloads,
 526                                 last);
 527                 else
 528                         err = ionic_tx(q, tx_pkts[nb_tx], txq->offloads, last);
 529                 if (err) {
 530                         stats->drop += nb_pkts - nb_tx;
 531                         if (nb_tx > 0)
 532                                 ionic_q_flush(q);
 533                         break;
 534                 }
 535
 536                 bytes_tx += tx_pkts[nb_tx]->pkt_len;
 537                 nb_tx++;
 538         }
 539
 540         stats->packets += nb_tx;
 541         stats->bytes += bytes_tx;
 542
 543         return nb_tx;
 544 }
 545
 546 /*********************************************************************
 547  *
 548  *  TX prep functions
 549  *
 550  **********************************************************************/
 551
 552 #define IONIC_TX_OFFLOAD_MASK ( \
 553         PKT_TX_IPV4 |           \
 554         PKT_TX_IPV6 |           \
 555         PKT_TX_VLAN |           \
 556         PKT_TX_IP_CKSUM |       \
 557         PKT_TX_TCP_SEG |        \
 558         PKT_TX_L4_MASK)
 559
 560 #define IONIC_TX_OFFLOAD_NOTSUP_MASK \
 561         (PKT_TX_OFFLOAD_MASK ^ IONIC_TX_OFFLOAD_MASK)
 562
 563 uint16_t
 564 ionic_prep_pkts(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts,
 565                 uint16_t nb_pkts)
 566 {
 567         struct rte_mbuf *txm;
 568         uint64_t offloads;
 569         int i = 0;
 570
 571         for (i = 0; i < nb_pkts; i++) {
 572                 txm = tx_pkts[i];
 573
 574                 if (txm->nb_segs > IONIC_TX_MAX_SG_ELEMS) {
 575                         rte_errno = -EINVAL;
 576                         break;
 577                 }
 578
 579                 offloads = txm->ol_flags;
 580
 581                 if (offloads & IONIC_TX_OFFLOAD_NOTSUP_MASK) {
 582                         rte_errno = -ENOTSUP;
 583                         break;
 584                 }
 585         }
 586
 587         return i;
 588 }
 589
 590 /*********************************************************************
 591  *
 592  *  RX functions
 593  *
 594  **********************************************************************/
 595
 596 static void ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index,
 597                 struct rte_mbuf *mbuf);
 598
 599 void
 600 ionic_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
 601                 struct rte_eth_rxq_info *qinfo)
 602 {
 603         struct ionic_qcq *rxq = dev->data->rx_queues[queue_id];
 604         struct ionic_queue *q = &rxq->q;
 605
 606         qinfo->mp = rxq->mb_pool;
 607         qinfo->scattered_rx = dev->data->scattered_rx;
 608         qinfo->nb_desc = q->num_descs;
 609         qinfo->conf.rx_deferred_start = rxq->flags & IONIC_QCQ_F_DEFERRED;
 610         qinfo->conf.offloads = rxq->offloads;
 611 }
 612
 613 static void __rte_cold
 614 ionic_rx_empty(struct ionic_queue *q)
 615 {
 616         struct ionic_qcq *rxq = IONIC_Q_TO_QCQ(q);
 617         struct ionic_desc_info *cur;
 618         struct rte_mbuf *mbuf;
 619
 620         while (q->tail_idx != q->head_idx) {
 621                 cur = &q->info[q->tail_idx];
 622                 mbuf = cur->cb_arg;
 623                 rte_mempool_put(rxq->mb_pool, mbuf);
 624
 625                 q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
 626         }
 627 }
 628
 629 void __rte_cold
 630 ionic_dev_rx_queue_release(void *rx_queue)
 631 {
 632         struct ionic_qcq *rxq = (struct ionic_qcq *)rx_queue;
 633
 634         IONIC_PRINT_CALL();
 635
 636         ionic_rx_empty(&rxq->q);
 637
 638         ionic_qcq_free(rxq);
 639 }
 640
 641 int __rte_cold
 642 ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
 643                 uint16_t rx_queue_id,
 644                 uint16_t nb_desc,
 645                 uint32_t socket_id __rte_unused,
 646                 const struct rte_eth_rxconf *rx_conf,
 647                 struct rte_mempool *mp)
 648 {
 649         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 650         struct ionic_qcq *rxq;
 651         uint64_t offloads;
 652         int err;
 653
 654         IONIC_PRINT_CALL();
 655
 656         IONIC_PRINT(DEBUG, "Configuring RX queue %u with %u buffers",
 657                 rx_queue_id, nb_desc);
 658
 659         if (rx_queue_id >= lif->nrxqcqs) {
 660                 IONIC_PRINT(ERR,
 661                         "Queue index %u not available (max %u queues)",
 662                         rx_queue_id, lif->nrxqcqs);
 663                 return -EINVAL;
 664         }
 665
 666         offloads = rx_conf->offloads | eth_dev->data->dev_conf.rxmode.offloads;
 667
 668         /* Validate number of receive descriptors */
 669         if (!rte_is_power_of_2(nb_desc) ||
 670                         nb_desc < IONIC_MIN_RING_DESC ||
 671                         nb_desc > IONIC_MAX_RING_DESC) {
 672                 IONIC_PRINT(ERR,
 673                         "Bad number of descriptors (%u) for queue %u (min: %u)",
 674                         nb_desc, rx_queue_id, IONIC_MIN_RING_DESC);
 675                 return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */
 676         }
 677
 678         if (rx_conf->offloads & DEV_RX_OFFLOAD_SCATTER)
 679                 eth_dev->data->scattered_rx = 1;
 680
 681         /* Free memory prior to re-allocation if needed... */
 682         if (eth_dev->data->rx_queues[rx_queue_id] != NULL) {
 683                 void *rx_queue = eth_dev->data->rx_queues[rx_queue_id];
 684                 ionic_dev_rx_queue_release(rx_queue);
 685                 eth_dev->data->rx_queues[rx_queue_id] = NULL;
 686         }
 687
 688         err = ionic_rx_qcq_alloc(lif, rx_queue_id, nb_desc, &rxq);
 689         if (err) {
 690                 IONIC_PRINT(ERR, "Queue allocation failure");
 691                 return -EINVAL;
 692         }
 693
 694         rxq->mb_pool = mp;
 695
 696         /*
 697          * Note: the interface does not currently support
 698          * DEV_RX_OFFLOAD_KEEP_CRC, please also consider ETHER_CRC_LEN
 699          * when the adapter will be able to keep the CRC and subtract
 700          * it to the length for all received packets:
 701          * if (eth_dev->data->dev_conf.rxmode.offloads &
 702          *     DEV_RX_OFFLOAD_KEEP_CRC)
 703          *   rxq->crc_len = ETHER_CRC_LEN;
 704          */
 705
 706         /* Do not start queue with rte_eth_dev_start() */
 707         if (rx_conf->rx_deferred_start)
 708                 rxq->flags |= IONIC_QCQ_F_DEFERRED;
 709
 710         rxq->offloads = offloads;
 711
 712         eth_dev->data->rx_queues[rx_queue_id] = rxq;
 713
 714         return 0;
 715 }
 716
 717 static void
 718 ionic_rx_clean(struct ionic_queue *q,
 719                 uint32_t q_desc_index, uint32_t cq_desc_index,
 720                 void *cb_arg, void *service_cb_arg)
 721 {
 722         struct ionic_rxq_comp *cq_desc_base = q->bound_cq->base;
 723         struct ionic_rxq_comp *cq_desc = &cq_desc_base[cq_desc_index];
 724         struct rte_mbuf *rxm = cb_arg;
 725         struct rte_mbuf *rxm_seg;
 726         struct ionic_qcq *rxq = IONIC_Q_TO_QCQ(q);
 727         uint32_t max_frame_size =
 728                 rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 729         uint64_t pkt_flags = 0;
 730         uint32_t pkt_type;
 731         struct ionic_rx_stats *stats = IONIC_Q_TO_RX_STATS(q);
 732         struct ionic_rx_service *recv_args = (struct ionic_rx_service *)
 733                 service_cb_arg;
 734         uint32_t buf_size = (uint16_t)
 735                 (rte_pktmbuf_data_room_size(rxq->mb_pool) -
 736                 RTE_PKTMBUF_HEADROOM);
 737         uint32_t left;
 738
 739         if (!recv_args) {
 740                 stats->no_cb_arg++;
 741                 /* Flush */
 742                 rte_pktmbuf_free(rxm);
 743                 /*
 744                  * Note: rte_mempool_put is faster with no segs
 745                  * rte_mempool_put(rxq->mb_pool, rxm);
 746                  */
 747                 return;
 748         }
 749
 750         if (cq_desc->status) {
 751                 stats->bad_cq_status++;
 752                 ionic_rx_recycle(q, q_desc_index, rxm);
 753                 return;
 754         }
 755
 756         if (recv_args->nb_rx >= recv_args->nb_pkts) {
 757                 stats->no_room++;
 758                 ionic_rx_recycle(q, q_desc_index, rxm);
 759                 return;
 760         }
 761
 762         if (cq_desc->len > max_frame_size ||
 763                         cq_desc->len == 0) {
 764                 stats->bad_len++;
 765                 ionic_rx_recycle(q, q_desc_index, rxm);
 766                 return;
 767         }
 768
 769         rxm->data_off = RTE_PKTMBUF_HEADROOM;
 770         rte_prefetch1((char *)rxm->buf_addr + rxm->data_off);
 771         rxm->nb_segs = 1; /* cq_desc->num_sg_elems */
 772         rxm->pkt_len = cq_desc->len;
 773         rxm->port = rxq->lif->port_id;
 774
 775         left = cq_desc->len;
 776
 777         rxm->data_len = RTE_MIN(buf_size, left);
 778         left -= rxm->data_len;
 779
 780         rxm_seg = rxm->next;
 781         while (rxm_seg && left) {
 782                 rxm_seg->data_len = RTE_MIN(buf_size, left);
 783                 left -= rxm_seg->data_len;
 784
 785                 rxm_seg = rxm_seg->next;
 786                 rxm->nb_segs++;
 787         }
 788
 789         /* RSS */
 790         pkt_flags |= PKT_RX_RSS_HASH;
 791         rxm->hash.rss = cq_desc->rss_hash;
 792
 793         /* Vlan Strip */
 794         if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) {
 795                 pkt_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
 796                 rxm->vlan_tci = cq_desc->vlan_tci;
 797         }
 798
 799         /* Checksum */
 800         if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) {
 801                 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_OK)
 802                         pkt_flags |= PKT_RX_IP_CKSUM_GOOD;
 803                 else if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD)
 804                         pkt_flags |= PKT_RX_IP_CKSUM_BAD;
 805
 806                 if ((cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_OK) ||
 807                         (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_OK))
 808                         pkt_flags |= PKT_RX_L4_CKSUM_GOOD;
 809                 else if ((cq_desc->csum_flags &
 810                                 IONIC_RXQ_COMP_CSUM_F_TCP_BAD) ||
 811                                 (cq_desc->csum_flags &
 812                                 IONIC_RXQ_COMP_CSUM_F_UDP_BAD))
 813                         pkt_flags |= PKT_RX_L4_CKSUM_BAD;
 814         }
 815
 816         rxm->ol_flags = pkt_flags;
 817
 818         /* Packet Type */
 819         switch (cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) {
 820         case IONIC_PKT_TYPE_IPV4:
 821                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4;
 822                 break;
 823         case IONIC_PKT_TYPE_IPV6:
 824                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6;
 825                 break;
 826         case IONIC_PKT_TYPE_IPV4_TCP:
 827                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
 828                         RTE_PTYPE_L4_TCP;
 829                 break;
 830         case IONIC_PKT_TYPE_IPV6_TCP:
 831                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
 832                         RTE_PTYPE_L4_TCP;
 833                 break;
 834         case IONIC_PKT_TYPE_IPV4_UDP:
 835                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
 836                         RTE_PTYPE_L4_UDP;
 837                 break;
 838         case IONIC_PKT_TYPE_IPV6_UDP:
 839                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
 840                         RTE_PTYPE_L4_UDP;
 841                 break;
 842         default:
 843                 {
 844                         struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm,
 845                                 struct rte_ether_hdr *);
 846                         uint16_t ether_type = eth_h->ether_type;
 847                         if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP))
 848                                 pkt_type = RTE_PTYPE_L2_ETHER_ARP;
 849                         else
 850                                 pkt_type = RTE_PTYPE_UNKNOWN;
 851                         break;
 852                 }
 853         }
 854
 855         rxm->packet_type = pkt_type;
 856
 857         recv_args->rx_pkts[recv_args->nb_rx] = rxm;
 858         recv_args->nb_rx++;
 859
 860         stats->packets++;
 861         stats->bytes += rxm->pkt_len;
 862 }
 863
 864 static void
 865 ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index,
 866                  struct rte_mbuf *mbuf)
 867 {
 868         struct ionic_rxq_desc *desc_base = q->base;
 869         struct ionic_rxq_desc *old = &desc_base[q_desc_index];
 870         struct ionic_rxq_desc *new = &desc_base[q->head_idx];
 871
 872         new->addr = old->addr;
 873         new->len = old->len;
 874
 875         ionic_q_post(q, true, ionic_rx_clean, mbuf);
 876 }
 877
 878 static int __rte_cold
 879 ionic_rx_fill(struct ionic_qcq *rxq, uint32_t len)
 880 {
 881         struct ionic_queue *q = &rxq->q;
 882         struct ionic_rxq_desc *desc_base = q->base;
 883         struct ionic_rxq_sg_desc *sg_desc_base = q->sg_base;
 884         struct ionic_rxq_desc *desc;
 885         struct ionic_rxq_sg_desc *sg_desc;
 886         struct ionic_rxq_sg_elem *elem;
 887         rte_iova_t dma_addr;
 888         uint32_t i, j, nsegs, buf_size, size;
 889         bool ring_doorbell;
 890
 891         buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
 892                 RTE_PKTMBUF_HEADROOM);
 893
 894         /* Initialize software ring entries */
 895         for (i = ionic_q_space_avail(q); i; i--) {
 896                 struct rte_mbuf *rxm = rte_mbuf_raw_alloc(rxq->mb_pool);
 897                 struct rte_mbuf *prev_rxm_seg;
 898
 899                 if (rxm == NULL) {
 900                         IONIC_PRINT(ERR, "RX mbuf alloc failed");
 901                         return -ENOMEM;
 902                 }
 903
 904                 nsegs = (len + buf_size - 1) / buf_size;
 905
 906                 desc = &desc_base[q->head_idx];
 907                 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(rxm));
 908                 desc->addr = dma_addr;
 909                 desc->len = buf_size;
 910                 size = buf_size;
 911                 desc->opcode = (nsegs > 1) ? IONIC_RXQ_DESC_OPCODE_SG :
 912                         IONIC_RXQ_DESC_OPCODE_SIMPLE;
 913                 rxm->next = NULL;
 914
 915                 prev_rxm_seg = rxm;
 916                 sg_desc = &sg_desc_base[q->head_idx];
 917                 elem = sg_desc->elems;
 918                 for (j = 0; j < nsegs - 1 && j < IONIC_RX_MAX_SG_ELEMS; j++) {
 919                         struct rte_mbuf *rxm_seg;
 920                         rte_iova_t data_iova;
 921
 922                         rxm_seg = rte_mbuf_raw_alloc(rxq->mb_pool);
 923                         if (rxm_seg == NULL) {
 924                                 IONIC_PRINT(ERR, "RX mbuf alloc failed");
 925                                 return -ENOMEM;
 926                         }
 927
 928                         data_iova = rte_mbuf_data_iova(rxm_seg);
 929                         dma_addr = rte_cpu_to_le_64(data_iova);
 930                         elem->addr = dma_addr;
 931                         elem->len = buf_size;
 932                         size += buf_size;
 933                         elem++;
 934                         rxm_seg->next = NULL;
 935                         prev_rxm_seg->next = rxm_seg;
 936                         prev_rxm_seg = rxm_seg;
 937                 }
 938
 939                 if (size < len)
 940                         IONIC_PRINT(ERR, "Rx SG size is not sufficient (%d < %d)",
 941                                 size, len);
 942
 943                 ring_doorbell = ((q->head_idx + 1) &
 944                         IONIC_RX_RING_DOORBELL_STRIDE) == 0;
 945
 946                 ionic_q_post(q, ring_doorbell, ionic_rx_clean, rxm);
 947         }
 948
 949         return 0;
 950 }
 951
 952 /*
 953  * Start Receive Units for specified queue.
 954  */
 955 int __rte_cold
 956 ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
 957 {
 958         uint32_t frame_size = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 959         struct ionic_qcq *rxq;
 960         int err;
 961
 962         IONIC_PRINT_CALL();
 963
 964         IONIC_PRINT(DEBUG, "Allocating RX queue buffers (size: %u)",
 965                 frame_size);
 966
 967         rxq = eth_dev->data->rx_queues[rx_queue_id];
 968
 969         err = ionic_lif_rxq_init(rxq);
 970         if (err)
 971                 return err;
 972
 973         /* Allocate buffers for descriptor rings */
 974         if (ionic_rx_fill(rxq, frame_size) != 0) {
 975                 IONIC_PRINT(ERR, "Could not alloc mbuf for queue:%d",
 976                         rx_queue_id);
 977                 return -1;
 978         }
 979
 980         ionic_qcq_enable(rxq);
 981
 982         eth_dev->data->rx_queue_state[rx_queue_id] =
 983                 RTE_ETH_QUEUE_STATE_STARTED;
 984
 985         return 0;
 986 }
 987
 988 static inline void __rte_cold
 989 ionic_rxq_service(struct ionic_cq *cq, uint32_t work_to_do,
 990                 void *service_cb_arg)
 991 {
 992         struct ionic_queue *q = cq->bound_q;
 993         struct ionic_desc_info *q_desc_info;
 994         struct ionic_rxq_comp *cq_desc_base = cq->base;
 995         struct ionic_rxq_comp *cq_desc;
 996         bool more;
 997         uint32_t curr_q_tail_idx, curr_cq_tail_idx;
 998         uint32_t work_done = 0;
 999
1000         if (work_to_do == 0)
1001                 return;
1002
1003         cq_desc = &cq_desc_base[cq->tail_idx];
1004         while (color_match(cq_desc->pkt_type_color, cq->done_color)) {
1005                 curr_cq_tail_idx = cq->tail_idx;
1006                 cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1);
1007
1008                 if (cq->tail_idx == 0)
1009                         cq->done_color = !cq->done_color;
1010
1011                 /* Prefetch the next 4 descriptors */
1012                 if ((cq->tail_idx & 0x3) == 0)
1013                         rte_prefetch0(&cq_desc_base[cq->tail_idx]);
1014
1015                 do {
1016                         more = (q->tail_idx != cq_desc->comp_index);
1017
1018                         q_desc_info = &q->info[q->tail_idx];
1019
1020                         curr_q_tail_idx = q->tail_idx;
1021                         q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1);
1022
1023                         /* Prefetch the next 4 descriptors */
1024                         if ((q->tail_idx & 0x3) == 0)
1025                                 /* q desc info */
1026                                 rte_prefetch0(&q->info[q->tail_idx]);
1027
1028                         ionic_rx_clean(q, curr_q_tail_idx, curr_cq_tail_idx,
1029                                 q_desc_info->cb_arg, service_cb_arg);
1030
1031                 } while (more);
1032
1033                 if (++work_done == work_to_do)
1034                         break;
1035
1036                 cq_desc = &cq_desc_base[cq->tail_idx];
1037         }
1038 }
1039
1040 /*
1041  * Stop Receive Units for specified queue.
1042  */
1043 int __rte_cold
1044 ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
1045 {
1046         struct ionic_qcq *rxq;
1047
1048         IONIC_PRINT_CALL();
1049
1050         rxq = eth_dev->data->rx_queues[rx_queue_id];
1051
1052         ionic_qcq_disable(rxq);
1053
1054         /* Flush */
1055         ionic_rxq_service(&rxq->cq, -1, NULL);
1056
1057         ionic_lif_rxq_deinit(rxq);
1058
1059         eth_dev->data->rx_queue_state[rx_queue_id] =
1060                 RTE_ETH_QUEUE_STATE_STOPPED;
1061
1062         return 0;
1063 }
1064
1065 uint16_t
1066 ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1067                 uint16_t nb_pkts)
1068 {
1069         struct ionic_qcq *rxq = (struct ionic_qcq *)rx_queue;
1070         uint32_t frame_size =
1071                 rxq->lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
1072         struct ionic_cq *cq = &rxq->cq;
1073         struct ionic_rx_service service_cb_arg;
1074
1075         service_cb_arg.rx_pkts = rx_pkts;
1076         service_cb_arg.nb_pkts = nb_pkts;
1077         service_cb_arg.nb_rx = 0;
1078
1079         ionic_rxq_service(cq, nb_pkts, &service_cb_arg);
1080
1081         ionic_rx_fill(rxq, frame_size);
1082
1083         return service_cb_arg.nb_rx;
1084 }