drivers/net/ionic/ionic_rxtx.c

   1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
   2  * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdio.h>
   7 #include <stdlib.h>
   8 #include <string.h>
   9 #include <errno.h>
  10 #include <stdint.h>
  11 #include <stdarg.h>
  12 #include <unistd.h>
  13 #include <inttypes.h>
  14
  15 #include <rte_byteorder.h>
  16 #include <rte_common.h>
  17 #include <rte_cycles.h>
  18 #include <rte_log.h>
  19 #include <rte_debug.h>
  20 #include <rte_interrupts.h>
  21 #include <rte_pci.h>
  22 #include <rte_memory.h>
  23 #include <rte_memzone.h>
  24 #include <rte_launch.h>
  25 #include <rte_eal.h>
  26 #include <rte_per_lcore.h>
  27 #include <rte_lcore.h>
  28 #include <rte_atomic.h>
  29 #include <rte_branch_prediction.h>
  30 #include <rte_mempool.h>
  31 #include <rte_malloc.h>
  32 #include <rte_mbuf.h>
  33 #include <rte_ether.h>
  34 #include <ethdev_driver.h>
  35 #include <rte_prefetch.h>
  36 #include <rte_udp.h>
  37 #include <rte_tcp.h>
  38 #include <rte_sctp.h>
  39 #include <rte_string_fns.h>
  40 #include <rte_errno.h>
  41 #include <rte_ip.h>
  42 #include <rte_net.h>
  43
  44 #include "ionic_logs.h"
  45 #include "ionic_mac_api.h"
  46 #include "ionic_ethdev.h"
  47 #include "ionic_lif.h"
  48 #include "ionic_rxtx.h"
  49
  50 #define IONIC_RX_RING_DOORBELL_STRIDE           (32 - 1)
  51
  52 /*********************************************************************
  53  *
  54  *  TX functions
  55  *
  56  **********************************************************************/
  57
  58 void
  59 ionic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
  60                 struct rte_eth_txq_info *qinfo)
  61 {
  62         struct ionic_tx_qcq *txq = dev->data->tx_queues[queue_id];
  63         struct ionic_queue *q = &txq->qcq.q;
  64
  65         qinfo->nb_desc = q->num_descs;
  66         qinfo->conf.offloads = dev->data->dev_conf.txmode.offloads;
  67         qinfo->conf.tx_deferred_start = txq->flags & IONIC_QCQ_F_DEFERRED;
  68 }
  69
  70 static __rte_always_inline void
  71 ionic_tx_flush(struct ionic_tx_qcq *txq)
  72 {
  73         struct ionic_cq *cq = &txq->qcq.cq;
  74         struct ionic_queue *q = &txq->qcq.q;
  75         struct rte_mbuf *txm, *next;
  76         struct ionic_txq_comp *cq_desc_base = cq->base;
  77         struct ionic_txq_comp *cq_desc;
  78         void **info;
  79         u_int32_t comp_index = (u_int32_t)-1;
  80
  81         cq_desc = &cq_desc_base[cq->tail_idx];
  82         while (color_match(cq_desc->color, cq->done_color)) {
  83                 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);
  84
  85                 /* Prefetch the next 4 descriptors (not really useful here) */
  86                 if ((cq->tail_idx & 0x3) == 0)
  87                         rte_prefetch0(&cq_desc_base[cq->tail_idx]);
  88
  89                 if (cq->tail_idx == 0)
  90                         cq->done_color = !cq->done_color;
  91
  92                 comp_index = cq_desc->comp_index;
  93
  94                 cq_desc = &cq_desc_base[cq->tail_idx];
  95         }
  96
  97         if (comp_index != (u_int32_t)-1) {
  98                 while (q->tail_idx != comp_index) {
  99                         info = IONIC_INFO_PTR(q, q->tail_idx);
 100
 101                         q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
 102
 103                         /* Prefetch the next 4 descriptors */
 104                         if ((q->tail_idx & 0x3) == 0)
 105                                 /* q desc info */
 106                                 rte_prefetch0(&q->info[q->tail_idx]);
 107
 108                         /*
 109                          * Note: you can just use rte_pktmbuf_free,
 110                          * but this loop is faster
 111                          */
 112                         txm = info[0];
 113                         while (txm != NULL) {
 114                                 next = txm->next;
 115                                 rte_pktmbuf_free_seg(txm);
 116                                 txm = next;
 117                         }
 118                 }
 119         }
 120 }
 121
 122 void __rte_cold
 123 ionic_dev_tx_queue_release(void *tx_queue)
 124 {
 125         struct ionic_tx_qcq *txq = tx_queue;
 126         struct ionic_tx_stats *stats = &txq->stats;
 127
 128         IONIC_PRINT_CALL();
 129
 130         IONIC_PRINT(DEBUG, "TX queue %u pkts %ju tso %ju",
 131                 txq->qcq.q.index, stats->packets, stats->tso);
 132
 133         ionic_lif_txq_deinit(txq);
 134
 135         ionic_qcq_free(&txq->qcq);
 136 }
 137
 138 int __rte_cold
 139 ionic_dev_tx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id)
 140 {
 141         struct ionic_tx_qcq *txq;
 142
 143         IONIC_PRINT(DEBUG, "Stopping TX queue %u", tx_queue_id);
 144
 145         txq = eth_dev->data->tx_queues[tx_queue_id];
 146
 147         eth_dev->data->tx_queue_state[tx_queue_id] =
 148                 RTE_ETH_QUEUE_STATE_STOPPED;
 149
 150         /*
 151          * Note: we should better post NOP Tx desc and wait for its completion
 152          * before disabling Tx queue
 153          */
 154
 155         ionic_qcq_disable(&txq->qcq);
 156
 157         ionic_tx_flush(txq);
 158
 159         return 0;
 160 }
 161
 162 int __rte_cold
 163 ionic_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id,
 164                 uint16_t nb_desc, uint32_t socket_id,
 165                 const struct rte_eth_txconf *tx_conf)
 166 {
 167         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 168         struct ionic_tx_qcq *txq;
 169         uint64_t offloads;
 170         int err;
 171
 172         if (tx_queue_id >= lif->ntxqcqs) {
 173                 IONIC_PRINT(DEBUG, "Queue index %u not available "
 174                         "(max %u queues)",
 175                         tx_queue_id, lif->ntxqcqs);
 176                 return -EINVAL;
 177         }
 178
 179         offloads = tx_conf->offloads | eth_dev->data->dev_conf.txmode.offloads;
 180         IONIC_PRINT(DEBUG,
 181                 "Configuring skt %u TX queue %u with %u buffers, offloads %jx",
 182                 socket_id, tx_queue_id, nb_desc, offloads);
 183
 184         /* Validate number of receive descriptors */
 185         if (!rte_is_power_of_2(nb_desc) || nb_desc < IONIC_MIN_RING_DESC)
 186                 return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */
 187
 188         /* Free memory prior to re-allocation if needed... */
 189         if (eth_dev->data->tx_queues[tx_queue_id] != NULL) {
 190                 void *tx_queue = eth_dev->data->tx_queues[tx_queue_id];
 191                 ionic_dev_tx_queue_release(tx_queue);
 192                 eth_dev->data->tx_queues[tx_queue_id] = NULL;
 193         }
 194
 195         eth_dev->data->tx_queue_state[tx_queue_id] =
 196                 RTE_ETH_QUEUE_STATE_STOPPED;
 197
 198         err = ionic_tx_qcq_alloc(lif, socket_id, tx_queue_id, nb_desc, &txq);
 199         if (err) {
 200                 IONIC_PRINT(DEBUG, "Queue allocation failure");
 201                 return -EINVAL;
 202         }
 203
 204         /* Do not start queue with rte_eth_dev_start() */
 205         if (tx_conf->tx_deferred_start)
 206                 txq->flags |= IONIC_QCQ_F_DEFERRED;
 207
 208         /* Convert the offload flags into queue flags */
 209         if (offloads & DEV_TX_OFFLOAD_IPV4_CKSUM)
 210                 txq->flags |= IONIC_QCQ_F_CSUM_L3;
 211         if (offloads & DEV_TX_OFFLOAD_TCP_CKSUM)
 212                 txq->flags |= IONIC_QCQ_F_CSUM_TCP;
 213         if (offloads & DEV_TX_OFFLOAD_UDP_CKSUM)
 214                 txq->flags |= IONIC_QCQ_F_CSUM_UDP;
 215
 216         eth_dev->data->tx_queues[tx_queue_id] = txq;
 217
 218         return 0;
 219 }
 220
 221 /*
 222  * Start Transmit Units for specified queue.
 223  */
 224 int __rte_cold
 225 ionic_dev_tx_queue_start(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id)
 226 {
 227         uint8_t *tx_queue_state = eth_dev->data->tx_queue_state;
 228         struct ionic_tx_qcq *txq;
 229         int err;
 230
 231         if (tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) {
 232                 IONIC_PRINT(DEBUG, "TX queue %u already started",
 233                         tx_queue_id);
 234                 return 0;
 235         }
 236
 237         txq = eth_dev->data->tx_queues[tx_queue_id];
 238
 239         IONIC_PRINT(DEBUG, "Starting TX queue %u, %u descs",
 240                 tx_queue_id, txq->qcq.q.num_descs);
 241
 242         if (!(txq->flags & IONIC_QCQ_F_INITED)) {
 243                 err = ionic_lif_txq_init(txq);
 244                 if (err)
 245                         return err;
 246         } else {
 247                 ionic_qcq_enable(&txq->qcq);
 248         }
 249
 250         tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 251
 252         return 0;
 253 }
 254
 255 static void
 256 ionic_tx_tcp_pseudo_csum(struct rte_mbuf *txm)
 257 {
 258         struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(txm, struct ether_hdr *);
 259         char *l3_hdr = ((char *)eth_hdr) + txm->l2_len;
 260         struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *)
 261                 (l3_hdr + txm->l3_len);
 262
 263         if (txm->ol_flags & PKT_TX_IP_CKSUM) {
 264                 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr;
 265                 ipv4_hdr->hdr_checksum = 0;
 266                 tcp_hdr->cksum = 0;
 267                 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, tcp_hdr);
 268         } else {
 269                 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr;
 270                 tcp_hdr->cksum = 0;
 271                 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, tcp_hdr);
 272         }
 273 }
 274
 275 static void
 276 ionic_tx_tcp_inner_pseudo_csum(struct rte_mbuf *txm)
 277 {
 278         struct ether_hdr *eth_hdr = rte_pktmbuf_mtod(txm, struct ether_hdr *);
 279         char *l3_hdr = ((char *)eth_hdr) + txm->outer_l2_len +
 280                 txm->outer_l3_len + txm->l2_len;
 281         struct rte_tcp_hdr *tcp_hdr = (struct rte_tcp_hdr *)
 282                 (l3_hdr + txm->l3_len);
 283
 284         if (txm->ol_flags & PKT_TX_IPV4) {
 285                 struct rte_ipv4_hdr *ipv4_hdr = (struct rte_ipv4_hdr *)l3_hdr;
 286                 ipv4_hdr->hdr_checksum = 0;
 287                 tcp_hdr->cksum = 0;
 288                 tcp_hdr->cksum = rte_ipv4_udptcp_cksum(ipv4_hdr, tcp_hdr);
 289         } else {
 290                 struct rte_ipv6_hdr *ipv6_hdr = (struct rte_ipv6_hdr *)l3_hdr;
 291                 tcp_hdr->cksum = 0;
 292                 tcp_hdr->cksum = rte_ipv6_udptcp_cksum(ipv6_hdr, tcp_hdr);
 293         }
 294 }
 295
 296 static void
 297 ionic_tx_tso_post(struct ionic_queue *q, struct ionic_txq_desc *desc,
 298                 struct rte_mbuf *txm,
 299                 rte_iova_t addr, uint8_t nsge, uint16_t len,
 300                 uint32_t hdrlen, uint32_t mss,
 301                 bool encap,
 302                 uint16_t vlan_tci, bool has_vlan,
 303                 bool start, bool done)
 304 {
 305         void **info;
 306         uint8_t flags = 0;
 307         flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
 308         flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
 309         flags |= start ? IONIC_TXQ_DESC_FLAG_TSO_SOT : 0;
 310         flags |= done ? IONIC_TXQ_DESC_FLAG_TSO_EOT : 0;
 311
 312         desc->cmd = encode_txq_desc_cmd(IONIC_TXQ_DESC_OPCODE_TSO,
 313                 flags, nsge, addr);
 314         desc->len = len;
 315         desc->vlan_tci = vlan_tci;
 316         desc->hdr_len = hdrlen;
 317         desc->mss = mss;
 318
 319         if (done) {
 320                 info = IONIC_INFO_PTR(q, q->head_idx);
 321                 info[0] = txm;
 322         }
 323
 324         q->head_idx = Q_NEXT_TO_POST(q, 1);
 325
 326         if (done)
 327                 ionic_q_flush(q);
 328 }
 329
 330 static struct ionic_txq_desc *
 331 ionic_tx_tso_next(struct ionic_tx_qcq *txq, struct ionic_txq_sg_elem **elem)
 332 {
 333         struct ionic_queue *q = &txq->qcq.q;
 334         struct ionic_txq_desc *desc_base = q->base;
 335         struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base;
 336         struct ionic_txq_desc *desc = &desc_base[q->head_idx];
 337         struct ionic_txq_sg_desc_v1 *sg_desc = &sg_desc_base[q->head_idx];
 338
 339         *elem = sg_desc->elems;
 340         return desc;
 341 }
 342
 343 static int
 344 ionic_tx_tso(struct ionic_tx_qcq *txq, struct rte_mbuf *txm,
 345                 bool not_xmit_more)
 346 {
 347         struct ionic_queue *q = &txq->qcq.q;
 348         struct ionic_tx_stats *stats = &txq->stats;
 349         struct ionic_txq_desc *desc;
 350         struct ionic_txq_sg_elem *elem;
 351         struct rte_mbuf *txm_seg;
 352         rte_iova_t data_iova;
 353         uint64_t desc_addr = 0, next_addr;
 354         uint16_t desc_len = 0;
 355         uint8_t desc_nsge;
 356         uint32_t hdrlen;
 357         uint32_t mss = txm->tso_segsz;
 358         uint32_t frag_left = 0;
 359         uint32_t left;
 360         uint32_t seglen;
 361         uint32_t len;
 362         uint32_t offset = 0;
 363         bool start, done;
 364         bool encap;
 365         bool has_vlan = !!(txm->ol_flags & PKT_TX_VLAN_PKT);
 366         uint16_t vlan_tci = txm->vlan_tci;
 367         uint64_t ol_flags = txm->ol_flags;
 368
 369         encap = ((ol_flags & PKT_TX_OUTER_IP_CKSUM) ||
 370                 (ol_flags & PKT_TX_OUTER_UDP_CKSUM)) &&
 371                 ((ol_flags & PKT_TX_OUTER_IPV4) ||
 372                 (ol_flags & PKT_TX_OUTER_IPV6));
 373
 374         /* Preload inner-most TCP csum field with IP pseudo hdr
 375          * calculated with IP length set to zero.  HW will later
 376          * add in length to each TCP segment resulting from the TSO.
 377          */
 378
 379         if (encap) {
 380                 ionic_tx_tcp_inner_pseudo_csum(txm);
 381                 hdrlen = txm->outer_l2_len + txm->outer_l3_len +
 382                         txm->l2_len + txm->l3_len + txm->l4_len;
 383         } else {
 384                 ionic_tx_tcp_pseudo_csum(txm);
 385                 hdrlen = txm->l2_len + txm->l3_len + txm->l4_len;
 386         }
 387
 388         seglen = hdrlen + mss;
 389         left = txm->data_len;
 390         data_iova = rte_mbuf_data_iova(txm);
 391
 392         desc = ionic_tx_tso_next(txq, &elem);
 393         start = true;
 394
 395         /* Chop data up into desc segments */
 396
 397         while (left > 0) {
 398                 len = RTE_MIN(seglen, left);
 399                 frag_left = seglen - len;
 400                 desc_addr = rte_cpu_to_le_64(data_iova + offset);
 401                 desc_len = len;
 402                 desc_nsge = 0;
 403                 left -= len;
 404                 offset += len;
 405                 if (txm->nb_segs > 1 && frag_left > 0)
 406                         continue;
 407                 done = (txm->nb_segs == 1 && left == 0);
 408                 ionic_tx_tso_post(q, desc, txm,
 409                         desc_addr, desc_nsge, desc_len,
 410                         hdrlen, mss,
 411                         encap,
 412                         vlan_tci, has_vlan,
 413                         start, done && not_xmit_more);
 414                 desc = ionic_tx_tso_next(txq, &elem);
 415                 start = false;
 416                 seglen = mss;
 417         }
 418
 419         /* Chop frags into desc segments */
 420
 421         txm_seg = txm->next;
 422         while (txm_seg != NULL) {
 423                 offset = 0;
 424                 data_iova = rte_mbuf_data_iova(txm_seg);
 425                 left = txm_seg->data_len;
 426
 427                 while (left > 0) {
 428                         next_addr = rte_cpu_to_le_64(data_iova + offset);
 429                         if (frag_left > 0) {
 430                                 len = RTE_MIN(frag_left, left);
 431                                 frag_left -= len;
 432                                 elem->addr = next_addr;
 433                                 elem->len = len;
 434                                 elem++;
 435                                 desc_nsge++;
 436                         } else {
 437                                 len = RTE_MIN(mss, left);
 438                                 frag_left = mss - len;
 439                                 desc_addr = next_addr;
 440                                 desc_len = len;
 441                                 desc_nsge = 0;
 442                         }
 443                         left -= len;
 444                         offset += len;
 445                         if (txm_seg->next != NULL && frag_left > 0)
 446                                 continue;
 447
 448                         done = (txm_seg->next == NULL && left == 0);
 449                         ionic_tx_tso_post(q, desc, txm_seg,
 450                                 desc_addr, desc_nsge, desc_len,
 451                                 hdrlen, mss,
 452                                 encap,
 453                                 vlan_tci, has_vlan,
 454                                 start, done && not_xmit_more);
 455                         desc = ionic_tx_tso_next(txq, &elem);
 456                         start = false;
 457                 }
 458
 459                 txm_seg = txm_seg->next;
 460         }
 461
 462         stats->tso++;
 463
 464         return 0;
 465 }
 466
 467 static __rte_always_inline int
 468 ionic_tx(struct ionic_tx_qcq *txq, struct rte_mbuf *txm,
 469                 bool not_xmit_more)
 470 {
 471         struct ionic_queue *q = &txq->qcq.q;
 472         struct ionic_txq_desc *desc, *desc_base = q->base;
 473         struct ionic_txq_sg_desc_v1 *sg_desc_base = q->sg_base;
 474         struct ionic_txq_sg_elem *elem;
 475         struct ionic_tx_stats *stats = &txq->stats;
 476         struct rte_mbuf *txm_seg;
 477         void **info;
 478         bool encap;
 479         bool has_vlan;
 480         uint64_t ol_flags = txm->ol_flags;
 481         uint64_t addr;
 482         uint8_t opcode = IONIC_TXQ_DESC_OPCODE_CSUM_NONE;
 483         uint8_t flags = 0;
 484
 485         desc = &desc_base[q->head_idx];
 486         info = IONIC_INFO_PTR(q, q->head_idx);
 487
 488         if ((ol_flags & PKT_TX_IP_CKSUM) &&
 489             (txq->flags & IONIC_QCQ_F_CSUM_L3)) {
 490                 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW;
 491                 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L3;
 492         }
 493
 494         if (((ol_flags & PKT_TX_TCP_CKSUM) &&
 495              (txq->flags & IONIC_QCQ_F_CSUM_TCP)) ||
 496             ((ol_flags & PKT_TX_UDP_CKSUM) &&
 497              (txq->flags & IONIC_QCQ_F_CSUM_UDP))) {
 498                 opcode = IONIC_TXQ_DESC_OPCODE_CSUM_HW;
 499                 flags |= IONIC_TXQ_DESC_FLAG_CSUM_L4;
 500         }
 501
 502         if (opcode == IONIC_TXQ_DESC_OPCODE_CSUM_NONE)
 503                 stats->no_csum++;
 504
 505         has_vlan = (ol_flags & PKT_TX_VLAN_PKT);
 506         encap = ((ol_flags & PKT_TX_OUTER_IP_CKSUM) ||
 507                         (ol_flags & PKT_TX_OUTER_UDP_CKSUM)) &&
 508                         ((ol_flags & PKT_TX_OUTER_IPV4) ||
 509                         (ol_flags & PKT_TX_OUTER_IPV6));
 510
 511         flags |= has_vlan ? IONIC_TXQ_DESC_FLAG_VLAN : 0;
 512         flags |= encap ? IONIC_TXQ_DESC_FLAG_ENCAP : 0;
 513
 514         addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm));
 515
 516         desc->cmd = encode_txq_desc_cmd(opcode, flags, txm->nb_segs - 1, addr);
 517         desc->len = txm->data_len;
 518         desc->vlan_tci = txm->vlan_tci;
 519
 520         info[0] = txm;
 521
 522         elem = sg_desc_base[q->head_idx].elems;
 523
 524         txm_seg = txm->next;
 525         while (txm_seg != NULL) {
 526                 elem->len = txm_seg->data_len;
 527                 elem->addr = rte_cpu_to_le_64(rte_mbuf_data_iova(txm_seg));
 528                 elem++;
 529                 txm_seg = txm_seg->next;
 530         }
 531
 532         q->head_idx = Q_NEXT_TO_POST(q, 1);
 533
 534         if (not_xmit_more)
 535                 ionic_q_flush(q);
 536
 537         return 0;
 538 }
 539
 540 uint16_t
 541 ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 542                 uint16_t nb_pkts)
 543 {
 544         struct ionic_tx_qcq *txq = tx_queue;
 545         struct ionic_queue *q = &txq->qcq.q;
 546         struct ionic_tx_stats *stats = &txq->stats;
 547         uint32_t next_q_head_idx;
 548         uint32_t bytes_tx = 0;
 549         uint16_t nb_tx = 0;
 550         int err;
 551         bool last;
 552
 553         /* Cleaning old buffers */
 554         ionic_tx_flush(txq);
 555
 556         if (unlikely(ionic_q_space_avail(q) < nb_pkts)) {
 557                 stats->stop += nb_pkts;
 558                 return 0;
 559         }
 560
 561         while (nb_tx < nb_pkts) {
 562                 last = (nb_tx == (nb_pkts - 1));
 563
 564                 next_q_head_idx = Q_NEXT_TO_POST(q, 1);
 565                 if ((next_q_head_idx & 0x3) == 0) {
 566                         struct ionic_txq_desc *desc_base = q->base;
 567                         rte_prefetch0(&desc_base[next_q_head_idx]);
 568                         rte_prefetch0(&q->info[next_q_head_idx]);
 569                 }
 570
 571                 if (tx_pkts[nb_tx]->ol_flags & PKT_TX_TCP_SEG)
 572                         err = ionic_tx_tso(txq, tx_pkts[nb_tx], last);
 573                 else
 574                         err = ionic_tx(txq, tx_pkts[nb_tx], last);
 575                 if (err) {
 576                         stats->drop += nb_pkts - nb_tx;
 577                         if (nb_tx > 0)
 578                                 ionic_q_flush(q);
 579                         break;
 580                 }
 581
 582                 bytes_tx += tx_pkts[nb_tx]->pkt_len;
 583                 nb_tx++;
 584         }
 585
 586         stats->packets += nb_tx;
 587         stats->bytes += bytes_tx;
 588
 589         return nb_tx;
 590 }
 591
 592 /*********************************************************************
 593  *
 594  *  TX prep functions
 595  *
 596  **********************************************************************/
 597
 598 #define IONIC_TX_OFFLOAD_MASK ( \
 599         PKT_TX_IPV4 |           \
 600         PKT_TX_IPV6 |           \
 601         PKT_TX_VLAN |           \
 602         PKT_TX_IP_CKSUM |       \
 603         PKT_TX_TCP_SEG |        \
 604         PKT_TX_L4_MASK)
 605
 606 #define IONIC_TX_OFFLOAD_NOTSUP_MASK \
 607         (PKT_TX_OFFLOAD_MASK ^ IONIC_TX_OFFLOAD_MASK)
 608
 609 uint16_t
 610 ionic_prep_pkts(void *tx_queue __rte_unused, struct rte_mbuf **tx_pkts,
 611                 uint16_t nb_pkts)
 612 {
 613         struct rte_mbuf *txm;
 614         uint64_t offloads;
 615         int i = 0;
 616
 617         for (i = 0; i < nb_pkts; i++) {
 618                 txm = tx_pkts[i];
 619
 620                 if (txm->nb_segs > IONIC_TX_MAX_SG_ELEMS_V1 + 1) {
 621                         rte_errno = -EINVAL;
 622                         break;
 623                 }
 624
 625                 offloads = txm->ol_flags;
 626
 627                 if (offloads & IONIC_TX_OFFLOAD_NOTSUP_MASK) {
 628                         rte_errno = -ENOTSUP;
 629                         break;
 630                 }
 631         }
 632
 633         return i;
 634 }
 635
 636 /*********************************************************************
 637  *
 638  *  RX functions
 639  *
 640  **********************************************************************/
 641
 642 static void ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index,
 643                 struct rte_mbuf *mbuf);
 644
 645 void
 646 ionic_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
 647                 struct rte_eth_rxq_info *qinfo)
 648 {
 649         struct ionic_rx_qcq *rxq = dev->data->rx_queues[queue_id];
 650         struct ionic_queue *q = &rxq->qcq.q;
 651
 652         qinfo->mp = rxq->mb_pool;
 653         qinfo->scattered_rx = dev->data->scattered_rx;
 654         qinfo->nb_desc = q->num_descs;
 655         qinfo->conf.rx_deferred_start = rxq->flags & IONIC_QCQ_F_DEFERRED;
 656         qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads;
 657 }
 658
 659 static void __rte_cold
 660 ionic_rx_empty(struct ionic_rx_qcq *rxq)
 661 {
 662         struct ionic_queue *q = &rxq->qcq.q;
 663         struct rte_mbuf *mbuf;
 664         void **info;
 665
 666         while (q->tail_idx != q->head_idx) {
 667                 info = IONIC_INFO_PTR(q, q->tail_idx);
 668                 mbuf = info[0];
 669                 rte_mempool_put(rxq->mb_pool, mbuf);
 670
 671                 q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
 672         }
 673 }
 674
 675 void __rte_cold
 676 ionic_dev_rx_queue_release(void *rx_queue)
 677 {
 678         struct ionic_rx_qcq *rxq = rx_queue;
 679         struct ionic_rx_stats *stats;
 680
 681         if (!rxq)
 682                 return;
 683
 684         IONIC_PRINT_CALL();
 685
 686         stats = &rxq->stats;
 687
 688         IONIC_PRINT(DEBUG, "RX queue %u pkts %ju mtod %ju",
 689                 rxq->qcq.q.index, stats->packets, stats->mtods);
 690
 691         ionic_rx_empty(rxq);
 692
 693         ionic_lif_rxq_deinit(rxq);
 694
 695         ionic_qcq_free(&rxq->qcq);
 696 }
 697
 698 int __rte_cold
 699 ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
 700                 uint16_t rx_queue_id,
 701                 uint16_t nb_desc,
 702                 uint32_t socket_id,
 703                 const struct rte_eth_rxconf *rx_conf,
 704                 struct rte_mempool *mp)
 705 {
 706         struct ionic_lif *lif = IONIC_ETH_DEV_TO_LIF(eth_dev);
 707         struct ionic_rx_qcq *rxq;
 708         uint64_t offloads;
 709         int err;
 710
 711         if (rx_queue_id >= lif->nrxqcqs) {
 712                 IONIC_PRINT(ERR,
 713                         "Queue index %u not available (max %u queues)",
 714                         rx_queue_id, lif->nrxqcqs);
 715                 return -EINVAL;
 716         }
 717
 718         offloads = rx_conf->offloads | eth_dev->data->dev_conf.rxmode.offloads;
 719         IONIC_PRINT(DEBUG,
 720                 "Configuring skt %u RX queue %u with %u buffers, offloads %jx",
 721                 socket_id, rx_queue_id, nb_desc, offloads);
 722
 723         if (!rx_conf->rx_drop_en)
 724                 IONIC_PRINT(WARNING, "No-drop mode is not supported");
 725
 726         /* Validate number of receive descriptors */
 727         if (!rte_is_power_of_2(nb_desc) ||
 728                         nb_desc < IONIC_MIN_RING_DESC ||
 729                         nb_desc > IONIC_MAX_RING_DESC) {
 730                 IONIC_PRINT(ERR,
 731                         "Bad descriptor count (%u) for queue %u (min: %u)",
 732                         nb_desc, rx_queue_id, IONIC_MIN_RING_DESC);
 733                 return -EINVAL; /* or use IONIC_DEFAULT_RING_DESC */
 734         }
 735
 736         /* Free memory prior to re-allocation if needed... */
 737         if (eth_dev->data->rx_queues[rx_queue_id] != NULL) {
 738                 void *rx_queue = eth_dev->data->rx_queues[rx_queue_id];
 739                 ionic_dev_rx_queue_release(rx_queue);
 740                 eth_dev->data->rx_queues[rx_queue_id] = NULL;
 741         }
 742
 743         eth_dev->data->rx_queue_state[rx_queue_id] =
 744                 RTE_ETH_QUEUE_STATE_STOPPED;
 745
 746         err = ionic_rx_qcq_alloc(lif, socket_id, rx_queue_id, nb_desc,
 747                         &rxq);
 748         if (err) {
 749                 IONIC_PRINT(ERR, "Queue %d allocation failure", rx_queue_id);
 750                 return -EINVAL;
 751         }
 752
 753         rxq->mb_pool = mp;
 754
 755         /*
 756          * Note: the interface does not currently support
 757          * DEV_RX_OFFLOAD_KEEP_CRC, please also consider ETHER_CRC_LEN
 758          * when the adapter will be able to keep the CRC and subtract
 759          * it to the length for all received packets:
 760          * if (eth_dev->data->dev_conf.rxmode.offloads &
 761          *     DEV_RX_OFFLOAD_KEEP_CRC)
 762          *   rxq->crc_len = ETHER_CRC_LEN;
 763          */
 764
 765         /* Do not start queue with rte_eth_dev_start() */
 766         if (rx_conf->rx_deferred_start)
 767                 rxq->flags |= IONIC_QCQ_F_DEFERRED;
 768
 769         eth_dev->data->rx_queues[rx_queue_id] = rxq;
 770
 771         return 0;
 772 }
 773
 774 static __rte_always_inline void
 775 ionic_rx_clean(struct ionic_rx_qcq *rxq,
 776                 uint32_t q_desc_index, uint32_t cq_desc_index,
 777                 void *service_cb_arg)
 778 {
 779         struct ionic_queue *q = &rxq->qcq.q;
 780         struct ionic_cq *cq = &rxq->qcq.cq;
 781         struct ionic_rxq_comp *cq_desc_base = cq->base;
 782         struct ionic_rxq_comp *cq_desc = &cq_desc_base[cq_desc_index];
 783         struct rte_mbuf *rxm, *rxm_seg;
 784         uint32_t max_frame_size =
 785                 rxq->qcq.lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
 786         uint64_t pkt_flags = 0;
 787         uint32_t pkt_type;
 788         struct ionic_rx_stats *stats = &rxq->stats;
 789         struct ionic_rx_service *recv_args = (struct ionic_rx_service *)
 790                 service_cb_arg;
 791         uint32_t buf_size = (uint16_t)
 792                 (rte_pktmbuf_data_room_size(rxq->mb_pool) -
 793                 RTE_PKTMBUF_HEADROOM);
 794         uint32_t left;
 795         void **info;
 796
 797         assert(q_desc_index == cq_desc->comp_index);
 798
 799         info = IONIC_INFO_PTR(q, cq_desc->comp_index);
 800
 801         rxm = info[0];
 802
 803         if (!recv_args) {
 804                 stats->no_cb_arg++;
 805                 /* Flush */
 806                 rte_pktmbuf_free(rxm);
 807                 /*
 808                  * Note: rte_mempool_put is faster with no segs
 809                  * rte_mempool_put(rxq->mb_pool, rxm);
 810                  */
 811                 return;
 812         }
 813
 814         if (cq_desc->status) {
 815                 stats->bad_cq_status++;
 816                 ionic_rx_recycle(q, q_desc_index, rxm);
 817                 return;
 818         }
 819
 820         if (recv_args->nb_rx >= recv_args->nb_pkts) {
 821                 stats->no_room++;
 822                 ionic_rx_recycle(q, q_desc_index, rxm);
 823                 return;
 824         }
 825
 826         if (cq_desc->len > max_frame_size ||
 827                         cq_desc->len == 0) {
 828                 stats->bad_len++;
 829                 ionic_rx_recycle(q, q_desc_index, rxm);
 830                 return;
 831         }
 832
 833         rxm->data_off = RTE_PKTMBUF_HEADROOM;
 834         rte_prefetch1((char *)rxm->buf_addr + rxm->data_off);
 835         rxm->nb_segs = 1; /* cq_desc->num_sg_elems */
 836         rxm->pkt_len = cq_desc->len;
 837         rxm->port = rxq->qcq.lif->port_id;
 838
 839         left = cq_desc->len;
 840
 841         rxm->data_len = RTE_MIN(buf_size, left);
 842         left -= rxm->data_len;
 843
 844         rxm_seg = rxm->next;
 845         while (rxm_seg && left) {
 846                 rxm_seg->data_len = RTE_MIN(buf_size, left);
 847                 left -= rxm_seg->data_len;
 848
 849                 rxm_seg = rxm_seg->next;
 850                 rxm->nb_segs++;
 851         }
 852
 853         /* RSS */
 854         pkt_flags |= PKT_RX_RSS_HASH;
 855         rxm->hash.rss = cq_desc->rss_hash;
 856
 857         /* Vlan Strip */
 858         if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_VLAN) {
 859                 pkt_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;
 860                 rxm->vlan_tci = cq_desc->vlan_tci;
 861         }
 862
 863         /* Checksum */
 864         if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_CALC) {
 865                 if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_OK)
 866                         pkt_flags |= PKT_RX_IP_CKSUM_GOOD;
 867                 else if (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_IP_BAD)
 868                         pkt_flags |= PKT_RX_IP_CKSUM_BAD;
 869
 870                 if ((cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_TCP_OK) ||
 871                         (cq_desc->csum_flags & IONIC_RXQ_COMP_CSUM_F_UDP_OK))
 872                         pkt_flags |= PKT_RX_L4_CKSUM_GOOD;
 873                 else if ((cq_desc->csum_flags &
 874                                 IONIC_RXQ_COMP_CSUM_F_TCP_BAD) ||
 875                                 (cq_desc->csum_flags &
 876                                 IONIC_RXQ_COMP_CSUM_F_UDP_BAD))
 877                         pkt_flags |= PKT_RX_L4_CKSUM_BAD;
 878         }
 879
 880         rxm->ol_flags = pkt_flags;
 881
 882         /* Packet Type */
 883         switch (cq_desc->pkt_type_color & IONIC_RXQ_COMP_PKT_TYPE_MASK) {
 884         case IONIC_PKT_TYPE_IPV4:
 885                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4;
 886                 break;
 887         case IONIC_PKT_TYPE_IPV6:
 888                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6;
 889                 break;
 890         case IONIC_PKT_TYPE_IPV4_TCP:
 891                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
 892                         RTE_PTYPE_L4_TCP;
 893                 break;
 894         case IONIC_PKT_TYPE_IPV6_TCP:
 895                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
 896                         RTE_PTYPE_L4_TCP;
 897                 break;
 898         case IONIC_PKT_TYPE_IPV4_UDP:
 899                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
 900                         RTE_PTYPE_L4_UDP;
 901                 break;
 902         case IONIC_PKT_TYPE_IPV6_UDP:
 903                 pkt_type = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
 904                         RTE_PTYPE_L4_UDP;
 905                 break;
 906         default:
 907                 {
 908                         struct rte_ether_hdr *eth_h = rte_pktmbuf_mtod(rxm,
 909                                 struct rte_ether_hdr *);
 910                         uint16_t ether_type = eth_h->ether_type;
 911                         if (ether_type == rte_cpu_to_be_16(RTE_ETHER_TYPE_ARP))
 912                                 pkt_type = RTE_PTYPE_L2_ETHER_ARP;
 913                         else
 914                                 pkt_type = RTE_PTYPE_UNKNOWN;
 915                         stats->mtods++;
 916                         break;
 917                 }
 918         }
 919
 920         rxm->packet_type = pkt_type;
 921
 922         recv_args->rx_pkts[recv_args->nb_rx] = rxm;
 923         recv_args->nb_rx++;
 924
 925         stats->packets++;
 926         stats->bytes += rxm->pkt_len;
 927 }
 928
 929 static void
 930 ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index,
 931                  struct rte_mbuf *mbuf)
 932 {
 933         struct ionic_rxq_desc *desc_base = q->base;
 934         struct ionic_rxq_desc *old = &desc_base[q_desc_index];
 935         struct ionic_rxq_desc *new = &desc_base[q->head_idx];
 936
 937         new->addr = old->addr;
 938         new->len = old->len;
 939
 940         q->info[q->head_idx] = mbuf;
 941
 942         q->head_idx = Q_NEXT_TO_POST(q, 1);
 943
 944         ionic_q_flush(q);
 945 }
 946
 947 static __rte_always_inline int
 948 ionic_rx_fill(struct ionic_rx_qcq *rxq, uint32_t len)
 949 {
 950         struct ionic_queue *q = &rxq->qcq.q;
 951         struct ionic_rxq_desc *desc, *desc_base = q->base;
 952         struct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base;
 953         struct ionic_rxq_sg_elem *elem;
 954         void **info;
 955         rte_iova_t dma_addr;
 956         uint32_t i, j, nsegs, buf_size, size;
 957         bool ring_doorbell;
 958
 959         buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
 960                 RTE_PKTMBUF_HEADROOM);
 961
 962         /* Initialize software ring entries */
 963         for (i = ionic_q_space_avail(q); i; i--) {
 964                 struct rte_mbuf *rxm = rte_mbuf_raw_alloc(rxq->mb_pool);
 965                 struct rte_mbuf *prev_rxm_seg;
 966
 967                 if (rxm == NULL) {
 968                         IONIC_PRINT(ERR, "RX mbuf alloc failed");
 969                         return -ENOMEM;
 970                 }
 971
 972                 info = IONIC_INFO_PTR(q, q->head_idx);
 973
 974                 nsegs = (len + buf_size - 1) / buf_size;
 975
 976                 desc = &desc_base[q->head_idx];
 977                 dma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(rxm));
 978                 desc->addr = dma_addr;
 979                 desc->len = buf_size;
 980                 size = buf_size;
 981                 desc->opcode = (nsegs > 1) ? IONIC_RXQ_DESC_OPCODE_SG :
 982                         IONIC_RXQ_DESC_OPCODE_SIMPLE;
 983                 rxm->next = NULL;
 984
 985                 prev_rxm_seg = rxm;
 986                 sg_desc = &sg_desc_base[q->head_idx];
 987                 elem = sg_desc->elems;
 988                 for (j = 0; j < nsegs - 1 && j < IONIC_RX_MAX_SG_ELEMS; j++) {
 989                         struct rte_mbuf *rxm_seg;
 990                         rte_iova_t data_iova;
 991
 992                         rxm_seg = rte_mbuf_raw_alloc(rxq->mb_pool);
 993                         if (rxm_seg == NULL) {
 994                                 IONIC_PRINT(ERR, "RX mbuf alloc failed");
 995                                 return -ENOMEM;
 996                         }
 997
 998                         data_iova = rte_mbuf_data_iova(rxm_seg);
 999                         dma_addr = rte_cpu_to_le_64(data_iova);
1000                         elem->addr = dma_addr;
1001                         elem->len = buf_size;
1002                         size += buf_size;
1003                         elem++;
1004                         rxm_seg->next = NULL;
1005                         prev_rxm_seg->next = rxm_seg;
1006                         prev_rxm_seg = rxm_seg;
1007                 }
1008
1009                 if (size < len)
1010                         IONIC_PRINT(ERR, "Rx SG size is not sufficient (%d < %d)",
1011                                 size, len);
1012
1013                 ring_doorbell = ((q->head_idx + 1) &
1014                         IONIC_RX_RING_DOORBELL_STRIDE) == 0;
1015
1016                 info[0] = rxm;
1017
1018                 q->head_idx = Q_NEXT_TO_POST(q, 1);
1019
1020                 if (ring_doorbell)
1021                         ionic_q_flush(q);
1022         }
1023
1024         return 0;
1025 }
1026
1027 /*
1028  * Start Receive Units for specified queue.
1029  */
1030 int __rte_cold
1031 ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
1032 {
1033         uint32_t frame_size = eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
1034         uint8_t *rx_queue_state = eth_dev->data->rx_queue_state;
1035         struct ionic_rx_qcq *rxq;
1036         int err;
1037
1038         if (rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STARTED) {
1039                 IONIC_PRINT(DEBUG, "RX queue %u already started",
1040                         rx_queue_id);
1041                 return 0;
1042         }
1043
1044         rxq = eth_dev->data->rx_queues[rx_queue_id];
1045
1046         IONIC_PRINT(DEBUG, "Starting RX queue %u, %u descs (size: %u)",
1047                 rx_queue_id, rxq->qcq.q.num_descs, frame_size);
1048
1049         if (!(rxq->flags & IONIC_QCQ_F_INITED)) {
1050                 err = ionic_lif_rxq_init(rxq);
1051                 if (err)
1052                         return err;
1053         } else {
1054                 ionic_qcq_enable(&rxq->qcq);
1055         }
1056
1057         /* Allocate buffers for descriptor rings */
1058         if (ionic_rx_fill(rxq, frame_size) != 0) {
1059                 IONIC_PRINT(ERR, "Could not alloc mbuf for queue:%d",
1060                         rx_queue_id);
1061                 return -1;
1062         }
1063
1064         rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
1065
1066         return 0;
1067 }
1068
1069 static __rte_always_inline void
1070 ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
1071                 void *service_cb_arg)
1072 {
1073         struct ionic_cq *cq = &rxq->qcq.cq;
1074         struct ionic_queue *q = &rxq->qcq.q;
1075         struct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base;
1076         bool more;
1077         uint32_t curr_q_tail_idx, curr_cq_tail_idx;
1078         uint32_t work_done = 0;
1079
1080         if (work_to_do == 0)
1081                 return;
1082
1083         cq_desc = &cq_desc_base[cq->tail_idx];
1084         while (color_match(cq_desc->pkt_type_color, cq->done_color)) {
1085                 curr_cq_tail_idx = cq->tail_idx;
1086                 cq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);
1087
1088                 if (cq->tail_idx == 0)
1089                         cq->done_color = !cq->done_color;
1090
1091                 /* Prefetch the next 4 descriptors */
1092                 if ((cq->tail_idx & 0x3) == 0)
1093                         rte_prefetch0(&cq_desc_base[cq->tail_idx]);
1094
1095                 do {
1096                         more = (q->tail_idx != cq_desc->comp_index);
1097
1098                         curr_q_tail_idx = q->tail_idx;
1099                         q->tail_idx = Q_NEXT_TO_SRVC(q, 1);
1100
1101                         /* Prefetch the next 4 descriptors */
1102                         if ((q->tail_idx & 0x3) == 0)
1103                                 /* q desc info */
1104                                 rte_prefetch0(&q->info[q->tail_idx]);
1105
1106                         ionic_rx_clean(rxq, curr_q_tail_idx, curr_cq_tail_idx,
1107                                 service_cb_arg);
1108
1109                 } while (more);
1110
1111                 if (++work_done == work_to_do)
1112                         break;
1113
1114                 cq_desc = &cq_desc_base[cq->tail_idx];
1115         }
1116 }
1117
1118 /*
1119  * Stop Receive Units for specified queue.
1120  */
1121 int __rte_cold
1122 ionic_dev_rx_queue_stop(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)
1123 {
1124         struct ionic_rx_qcq *rxq;
1125
1126         IONIC_PRINT(DEBUG, "Stopping RX queue %u", rx_queue_id);
1127
1128         rxq = eth_dev->data->rx_queues[rx_queue_id];
1129
1130         eth_dev->data->rx_queue_state[rx_queue_id] =
1131                 RTE_ETH_QUEUE_STATE_STOPPED;
1132
1133         ionic_qcq_disable(&rxq->qcq);
1134
1135         /* Flush */
1136         ionic_rxq_service(rxq, -1, NULL);
1137
1138         return 0;
1139 }
1140
1141 uint16_t
1142 ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
1143                 uint16_t nb_pkts)
1144 {
1145         struct ionic_rx_qcq *rxq = rx_queue;
1146         uint32_t frame_size =
1147                 rxq->qcq.lif->eth_dev->data->dev_conf.rxmode.max_rx_pkt_len;
1148         struct ionic_rx_service service_cb_arg;
1149
1150         service_cb_arg.rx_pkts = rx_pkts;
1151         service_cb_arg.nb_pkts = nb_pkts;
1152         service_cb_arg.nb_rx = 0;
1153
1154         ionic_rxq_service(rxq, nb_pkts, &service_cb_arg);
1155
1156         ionic_rx_fill(rxq, frame_size);
1157
1158         return service_cb_arg.nb_rx;
1159 }