drivers/net/dpaa2/dpaa2_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
   4  *   Copyright 2016-2019 NXP
   5  *
   6  */
   7
   8 #include <time.h>
   9 #include <net/if.h>
  10
  11 #include <rte_mbuf.h>
  12 #include <rte_ethdev_driver.h>
  13 #include <rte_malloc.h>
  14 #include <rte_memcpy.h>
  15 #include <rte_string_fns.h>
  16 #include <rte_dev.h>
  17
  18 #include <rte_fslmc.h>
  19 #include <fslmc_vfio.h>
  20 #include <dpaa2_hw_pvt.h>
  21 #include <dpaa2_hw_dpio.h>
  22 #include <dpaa2_hw_mempool.h>
  23
  24 #include "dpaa2_pmd_logs.h"
  25 #include "dpaa2_ethdev.h"
  26 #include "base/dpaa2_hw_dpni_annot.h"
  27
  28 static inline uint32_t __attribute__((hot))
  29 dpaa2_dev_rx_parse_slow(struct rte_mbuf *mbuf,
  30                         struct dpaa2_annot_hdr *annotation);
  31
  32 #define DPAA2_MBUF_TO_CONTIG_FD(_mbuf, _fd, _bpid)  do { \
  33         DPAA2_SET_FD_ADDR(_fd, DPAA2_MBUF_VADDR_TO_IOVA(_mbuf)); \
  34         DPAA2_SET_FD_LEN(_fd, _mbuf->data_len); \
  35         DPAA2_SET_ONLY_FD_BPID(_fd, _bpid); \
  36         DPAA2_SET_FD_OFFSET(_fd, _mbuf->data_off); \
  37         DPAA2_SET_FD_FRC(_fd, 0);               \
  38         DPAA2_RESET_FD_CTRL(_fd);               \
  39         DPAA2_RESET_FD_FLC(_fd);                \
  40 } while (0)
  41
  42 static inline void __attribute__((hot))
  43 dpaa2_dev_rx_parse_new(struct rte_mbuf *m, const struct qbman_fd *fd)
  44 {
  45         struct dpaa2_annot_hdr *annotation;
  46         uint16_t frc = DPAA2_GET_FD_FRC_PARSE_SUM(fd);
  47
  48         m->packet_type = RTE_PTYPE_UNKNOWN;
  49         switch (frc) {
  50         case DPAA2_PKT_TYPE_ETHER:
  51                 m->packet_type = RTE_PTYPE_L2_ETHER;
  52                 break;
  53         case DPAA2_PKT_TYPE_IPV4:
  54                 m->packet_type = RTE_PTYPE_L2_ETHER |
  55                         RTE_PTYPE_L3_IPV4;
  56                 break;
  57         case DPAA2_PKT_TYPE_IPV6:
  58                 m->packet_type = RTE_PTYPE_L2_ETHER |
  59                         RTE_PTYPE_L3_IPV6;
  60                 break;
  61         case DPAA2_PKT_TYPE_IPV4_EXT:
  62                 m->packet_type = RTE_PTYPE_L2_ETHER |
  63                         RTE_PTYPE_L3_IPV4_EXT;
  64                 break;
  65         case DPAA2_PKT_TYPE_IPV6_EXT:
  66                 m->packet_type = RTE_PTYPE_L2_ETHER |
  67                         RTE_PTYPE_L3_IPV6_EXT;
  68                 break;
  69         case DPAA2_PKT_TYPE_IPV4_TCP:
  70                 m->packet_type = RTE_PTYPE_L2_ETHER |
  71                         RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP;
  72                 break;
  73         case DPAA2_PKT_TYPE_IPV6_TCP:
  74                 m->packet_type = RTE_PTYPE_L2_ETHER |
  75                         RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP;
  76                 break;
  77         case DPAA2_PKT_TYPE_IPV4_UDP:
  78                 m->packet_type = RTE_PTYPE_L2_ETHER |
  79                         RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP;
  80                 break;
  81         case DPAA2_PKT_TYPE_IPV6_UDP:
  82                 m->packet_type = RTE_PTYPE_L2_ETHER |
  83                         RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP;
  84                 break;
  85         case DPAA2_PKT_TYPE_IPV4_SCTP:
  86                 m->packet_type = RTE_PTYPE_L2_ETHER |
  87                         RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP;
  88                 break;
  89         case DPAA2_PKT_TYPE_IPV6_SCTP:
  90                 m->packet_type = RTE_PTYPE_L2_ETHER |
  91                         RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP;
  92                 break;
  93         case DPAA2_PKT_TYPE_IPV4_ICMP:
  94                 m->packet_type = RTE_PTYPE_L2_ETHER |
  95                         RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_ICMP;
  96                 break;
  97         case DPAA2_PKT_TYPE_IPV6_ICMP:
  98                 m->packet_type = RTE_PTYPE_L2_ETHER |
  99                         RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_ICMP;
 100                 break;
 101         default:
 102                 m->packet_type = dpaa2_dev_rx_parse_slow(m,
 103                   (void *)((size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd))
 104                          + DPAA2_FD_PTA_SIZE));
 105         }
 106         m->hash.rss = fd->simple.flc_hi;
 107         m->ol_flags |= PKT_RX_RSS_HASH;
 108
 109         if (dpaa2_enable_ts == PMD_DPAA2_ENABLE_TS) {
 110                 annotation = (struct dpaa2_annot_hdr *)
 111                         ((size_t)DPAA2_IOVA_TO_VADDR(
 112                         DPAA2_GET_FD_ADDR(fd)) + DPAA2_FD_PTA_SIZE);
 113                 m->timestamp = annotation->word2;
 114                 m->ol_flags |= PKT_RX_TIMESTAMP;
 115                 DPAA2_PMD_DP_DEBUG("pkt timestamp:0x%" PRIx64 "", m->timestamp);
 116         }
 117
 118         DPAA2_PMD_DP_DEBUG("HW frc = 0x%x\t packet type =0x%x "
 119                 "ol_flags =0x%" PRIx64 "",
 120                 frc, m->packet_type, m->ol_flags);
 121 }
 122
 123 static inline uint32_t __attribute__((hot))
 124 dpaa2_dev_rx_parse_slow(struct rte_mbuf *mbuf,
 125                         struct dpaa2_annot_hdr *annotation)
 126 {
 127         uint32_t pkt_type = RTE_PTYPE_UNKNOWN;
 128         uint16_t *vlan_tci;
 129
 130         DPAA2_PMD_DP_DEBUG("(slow parse)annotation(3)=0x%" PRIx64 "\t"
 131                         "(4)=0x%" PRIx64 "\t",
 132                         annotation->word3, annotation->word4);
 133
 134         if (BIT_ISSET_AT_POS(annotation->word3, L2_VLAN_1_PRESENT)) {
 135                 vlan_tci = rte_pktmbuf_mtod_offset(mbuf, uint16_t *,
 136                         (VLAN_TCI_OFFSET_1(annotation->word5) >> 16));
 137                 mbuf->vlan_tci = rte_be_to_cpu_16(*vlan_tci);
 138                 mbuf->ol_flags |= PKT_RX_VLAN;
 139                 pkt_type |= RTE_PTYPE_L2_ETHER_VLAN;
 140         } else if (BIT_ISSET_AT_POS(annotation->word3, L2_VLAN_N_PRESENT)) {
 141                 vlan_tci = rte_pktmbuf_mtod_offset(mbuf, uint16_t *,
 142                         (VLAN_TCI_OFFSET_1(annotation->word5) >> 16));
 143                 mbuf->vlan_tci = rte_be_to_cpu_16(*vlan_tci);
 144                 mbuf->ol_flags |= PKT_RX_VLAN | PKT_RX_QINQ;
 145                 pkt_type |= RTE_PTYPE_L2_ETHER_QINQ;
 146         }
 147
 148         if (BIT_ISSET_AT_POS(annotation->word3, L2_ARP_PRESENT)) {
 149                 pkt_type |= RTE_PTYPE_L2_ETHER_ARP;
 150                 goto parse_done;
 151         } else if (BIT_ISSET_AT_POS(annotation->word3, L2_ETH_MAC_PRESENT)) {
 152                 pkt_type |= RTE_PTYPE_L2_ETHER;
 153         } else {
 154                 goto parse_done;
 155         }
 156
 157         if (BIT_ISSET_AT_POS(annotation->word4, L3_IPV4_1_PRESENT |
 158                              L3_IPV4_N_PRESENT)) {
 159                 pkt_type |= RTE_PTYPE_L3_IPV4;
 160                 if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_OPT_PRESENT |
 161                         L3_IP_N_OPT_PRESENT))
 162                         pkt_type |= RTE_PTYPE_L3_IPV4_EXT;
 163
 164         } else if (BIT_ISSET_AT_POS(annotation->word4, L3_IPV6_1_PRESENT |
 165                   L3_IPV6_N_PRESENT)) {
 166                 pkt_type |= RTE_PTYPE_L3_IPV6;
 167                 if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_OPT_PRESENT |
 168                     L3_IP_N_OPT_PRESENT))
 169                         pkt_type |= RTE_PTYPE_L3_IPV6_EXT;
 170         } else {
 171                 goto parse_done;
 172         }
 173
 174         if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L3CE))
 175                 mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD;
 176         else if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE))
 177                 mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD;
 178
 179         if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_FIRST_FRAGMENT |
 180             L3_IP_1_MORE_FRAGMENT |
 181             L3_IP_N_FIRST_FRAGMENT |
 182             L3_IP_N_MORE_FRAGMENT)) {
 183                 pkt_type |= RTE_PTYPE_L4_FRAG;
 184                 goto parse_done;
 185         } else {
 186                 pkt_type |= RTE_PTYPE_L4_NONFRAG;
 187         }
 188
 189         if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_UDP_PRESENT))
 190                 pkt_type |= RTE_PTYPE_L4_UDP;
 191
 192         else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_TCP_PRESENT))
 193                 pkt_type |= RTE_PTYPE_L4_TCP;
 194
 195         else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_SCTP_PRESENT))
 196                 pkt_type |= RTE_PTYPE_L4_SCTP;
 197
 198         else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_ICMP_PRESENT))
 199                 pkt_type |= RTE_PTYPE_L4_ICMP;
 200
 201         else if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_UNKNOWN_PROTOCOL))
 202                 pkt_type |= RTE_PTYPE_UNKNOWN;
 203
 204 parse_done:
 205         return pkt_type;
 206 }
 207
 208 static inline uint32_t __attribute__((hot))
 209 dpaa2_dev_rx_parse(struct rte_mbuf *mbuf, void *hw_annot_addr)
 210 {
 211         struct dpaa2_annot_hdr *annotation =
 212                         (struct dpaa2_annot_hdr *)hw_annot_addr;
 213
 214         DPAA2_PMD_DP_DEBUG("(fast parse) Annotation = 0x%" PRIx64 "\t",
 215                            annotation->word4);
 216
 217         if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L3CE))
 218                 mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD;
 219         else if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE))
 220                 mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD;
 221
 222         mbuf->ol_flags |= PKT_RX_TIMESTAMP;
 223         mbuf->timestamp = annotation->word2;
 224         DPAA2_PMD_DP_DEBUG("pkt timestamp: 0x%" PRIx64 "", mbuf->timestamp);
 225
 226         /* Check detailed parsing requirement */
 227         if (annotation->word3 & 0x7FFFFC3FFFF)
 228                 return dpaa2_dev_rx_parse_slow(mbuf, annotation);
 229
 230         /* Return some common types from parse processing */
 231         switch (annotation->word4) {
 232         case DPAA2_L3_IPv4:
 233                 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4;
 234         case DPAA2_L3_IPv6:
 235                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6;
 236         case DPAA2_L3_IPv4_TCP:
 237                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
 238                                 RTE_PTYPE_L4_TCP;
 239         case DPAA2_L3_IPv4_UDP:
 240                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
 241                                 RTE_PTYPE_L4_UDP;
 242         case DPAA2_L3_IPv6_TCP:
 243                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
 244                                 RTE_PTYPE_L4_TCP;
 245         case DPAA2_L3_IPv6_UDP:
 246                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
 247                                 RTE_PTYPE_L4_UDP;
 248         default:
 249                 break;
 250         }
 251
 252         return dpaa2_dev_rx_parse_slow(mbuf, annotation);
 253 }
 254
 255 static inline struct rte_mbuf *__attribute__((hot))
 256 eth_sg_fd_to_mbuf(const struct qbman_fd *fd)
 257 {
 258         struct qbman_sge *sgt, *sge;
 259         size_t sg_addr, fd_addr;
 260         int i = 0;
 261         struct rte_mbuf *first_seg, *next_seg, *cur_seg, *temp;
 262
 263         fd_addr = (size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
 264
 265         /* Get Scatter gather table address */
 266         sgt = (struct qbman_sge *)(fd_addr + DPAA2_GET_FD_OFFSET(fd));
 267
 268         sge = &sgt[i++];
 269         sg_addr = (size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FLE_ADDR(sge));
 270
 271         /* First Scatter gather entry */
 272         first_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr,
 273                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
 274         /* Prepare all the metadata for first segment */
 275         first_seg->buf_addr = (uint8_t *)sg_addr;
 276         first_seg->ol_flags = 0;
 277         first_seg->data_off = DPAA2_GET_FLE_OFFSET(sge);
 278         first_seg->data_len = sge->length  & 0x1FFFF;
 279         first_seg->pkt_len = DPAA2_GET_FD_LEN(fd);
 280         first_seg->nb_segs = 1;
 281         first_seg->next = NULL;
 282         if (dpaa2_svr_family == SVR_LX2160A)
 283                 dpaa2_dev_rx_parse_new(first_seg, fd);
 284         else
 285                 first_seg->packet_type = dpaa2_dev_rx_parse(first_seg,
 286                         (void *)((size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd))
 287                          + DPAA2_FD_PTA_SIZE));
 288
 289         rte_mbuf_refcnt_set(first_seg, 1);
 290         cur_seg = first_seg;
 291         while (!DPAA2_SG_IS_FINAL(sge)) {
 292                 sge = &sgt[i++];
 293                 sg_addr = (size_t)DPAA2_IOVA_TO_VADDR(
 294                                 DPAA2_GET_FLE_ADDR(sge));
 295                 next_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr,
 296                         rte_dpaa2_bpid_info[DPAA2_GET_FLE_BPID(sge)].meta_data_size);
 297                 next_seg->buf_addr  = (uint8_t *)sg_addr;
 298                 next_seg->data_off  = DPAA2_GET_FLE_OFFSET(sge);
 299                 next_seg->data_len  = sge->length  & 0x1FFFF;
 300                 first_seg->nb_segs += 1;
 301                 rte_mbuf_refcnt_set(next_seg, 1);
 302                 cur_seg->next = next_seg;
 303                 next_seg->next = NULL;
 304                 cur_seg = next_seg;
 305         }
 306         temp = DPAA2_INLINE_MBUF_FROM_BUF(fd_addr,
 307                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
 308         rte_mbuf_refcnt_set(temp, 1);
 309         rte_pktmbuf_free_seg(temp);
 310
 311         return (void *)first_seg;
 312 }
 313
 314 static inline struct rte_mbuf *__attribute__((hot))
 315 eth_fd_to_mbuf(const struct qbman_fd *fd)
 316 {
 317         struct rte_mbuf *mbuf = DPAA2_INLINE_MBUF_FROM_BUF(
 318                 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)),
 319                      rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
 320
 321         /* need to repopulated some of the fields,
 322          * as they may have changed in last transmission
 323          */
 324         mbuf->nb_segs = 1;
 325         mbuf->ol_flags = 0;
 326         mbuf->data_off = DPAA2_GET_FD_OFFSET(fd);
 327         mbuf->data_len = DPAA2_GET_FD_LEN(fd);
 328         mbuf->pkt_len = mbuf->data_len;
 329         mbuf->next = NULL;
 330         rte_mbuf_refcnt_set(mbuf, 1);
 331
 332         /* Parse the packet */
 333         /* parse results for LX2 are there in FRC field of FD.
 334          * For other DPAA2 platforms , parse results are after
 335          * the private - sw annotation area
 336          */
 337
 338         if (dpaa2_svr_family == SVR_LX2160A)
 339                 dpaa2_dev_rx_parse_new(mbuf, fd);
 340         else
 341                 mbuf->packet_type = dpaa2_dev_rx_parse(mbuf,
 342                         (void *)((size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd))
 343                          + DPAA2_FD_PTA_SIZE));
 344
 345         DPAA2_PMD_DP_DEBUG("to mbuf - mbuf =%p, mbuf->buf_addr =%p, off = %d,"
 346                 "fd_off=%d fd =%" PRIx64 ", meta = %d  bpid =%d, len=%d\n",
 347                 mbuf, mbuf->buf_addr, mbuf->data_off,
 348                 DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ADDR(fd),
 349                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size,
 350                 DPAA2_GET_FD_BPID(fd), DPAA2_GET_FD_LEN(fd));
 351
 352         return mbuf;
 353 }
 354
 355 static int __attribute__ ((noinline)) __attribute__((hot))
 356 eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf,
 357                   struct qbman_fd *fd, uint16_t bpid)
 358 {
 359         struct rte_mbuf *cur_seg = mbuf, *prev_seg, *mi, *temp;
 360         struct qbman_sge *sgt, *sge = NULL;
 361         int i;
 362
 363         temp = rte_pktmbuf_alloc(mbuf->pool);
 364         if (temp == NULL) {
 365                 DPAA2_PMD_DP_DEBUG("No memory to allocate S/G table\n");
 366                 return -ENOMEM;
 367         }
 368
 369         DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(temp));
 370         DPAA2_SET_FD_LEN(fd, mbuf->pkt_len);
 371         DPAA2_SET_ONLY_FD_BPID(fd, bpid);
 372         DPAA2_SET_FD_OFFSET(fd, temp->data_off);
 373         DPAA2_FD_SET_FORMAT(fd, qbman_fd_sg);
 374         DPAA2_RESET_FD_FRC(fd);
 375         DPAA2_RESET_FD_CTRL(fd);
 376         /*Set Scatter gather table and Scatter gather entries*/
 377         sgt = (struct qbman_sge *)(
 378                         (size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd))
 379                         + DPAA2_GET_FD_OFFSET(fd));
 380
 381         for (i = 0; i < mbuf->nb_segs; i++) {
 382                 sge = &sgt[i];
 383                 /*Resetting the buffer pool id and offset field*/
 384                 sge->fin_bpid_offset = 0;
 385                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(cur_seg));
 386                 DPAA2_SET_FLE_OFFSET(sge, cur_seg->data_off);
 387                 sge->length = cur_seg->data_len;
 388                 if (RTE_MBUF_DIRECT(cur_seg)) {
 389                         if (rte_mbuf_refcnt_read(cur_seg) > 1) {
 390                                 /* If refcnt > 1, invalid bpid is set to ensure
 391                                  * buffer is not freed by HW
 392                                  */
 393                                 DPAA2_SET_FLE_IVP(sge);
 394                                 rte_mbuf_refcnt_update(cur_seg, -1);
 395                         } else
 396                                 DPAA2_SET_FLE_BPID(sge,
 397                                                 mempool_to_bpid(cur_seg->pool));
 398                         cur_seg = cur_seg->next;
 399                 } else {
 400                         /* Get owner MBUF from indirect buffer */
 401                         mi = rte_mbuf_from_indirect(cur_seg);
 402                         if (rte_mbuf_refcnt_read(mi) > 1) {
 403                                 /* If refcnt > 1, invalid bpid is set to ensure
 404                                  * owner buffer is not freed by HW
 405                                  */
 406                                 DPAA2_SET_FLE_IVP(sge);
 407                         } else {
 408                                 DPAA2_SET_FLE_BPID(sge,
 409                                                    mempool_to_bpid(mi->pool));
 410                                 rte_mbuf_refcnt_update(mi, 1);
 411                         }
 412                         prev_seg = cur_seg;
 413                         cur_seg = cur_seg->next;
 414                         prev_seg->next = NULL;
 415                         rte_pktmbuf_free(prev_seg);
 416                 }
 417         }
 418         DPAA2_SG_SET_FINAL(sge, true);
 419         return 0;
 420 }
 421
 422 static void
 423 eth_mbuf_to_fd(struct rte_mbuf *mbuf,
 424                struct qbman_fd *fd, uint16_t bpid) __attribute__((unused));
 425
 426 static void __attribute__ ((noinline)) __attribute__((hot))
 427 eth_mbuf_to_fd(struct rte_mbuf *mbuf,
 428                struct qbman_fd *fd, uint16_t bpid)
 429 {
 430         DPAA2_MBUF_TO_CONTIG_FD(mbuf, fd, bpid);
 431
 432         DPAA2_PMD_DP_DEBUG("mbuf =%p, mbuf->buf_addr =%p, off = %d,"
 433                 "fd_off=%d fd =%" PRIx64 ", meta = %d  bpid =%d, len=%d\n",
 434                 mbuf, mbuf->buf_addr, mbuf->data_off,
 435                 DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ADDR(fd),
 436                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size,
 437                 DPAA2_GET_FD_BPID(fd), DPAA2_GET_FD_LEN(fd));
 438         if (RTE_MBUF_DIRECT(mbuf)) {
 439                 if (rte_mbuf_refcnt_read(mbuf) > 1) {
 440                         DPAA2_SET_FD_IVP(fd);
 441                         rte_mbuf_refcnt_update(mbuf, -1);
 442                 }
 443         } else {
 444                 struct rte_mbuf *mi;
 445
 446                 mi = rte_mbuf_from_indirect(mbuf);
 447                 if (rte_mbuf_refcnt_read(mi) > 1)
 448                         DPAA2_SET_FD_IVP(fd);
 449                 else
 450                         rte_mbuf_refcnt_update(mi, 1);
 451                 rte_pktmbuf_free(mbuf);
 452         }
 453 }
 454
 455 static inline int __attribute__((hot))
 456 eth_copy_mbuf_to_fd(struct rte_mbuf *mbuf,
 457                     struct qbman_fd *fd, uint16_t bpid)
 458 {
 459         struct rte_mbuf *m;
 460         void *mb = NULL;
 461
 462         if (rte_dpaa2_mbuf_alloc_bulk(
 463                 rte_dpaa2_bpid_info[bpid].bp_list->mp, &mb, 1)) {
 464                 DPAA2_PMD_DP_DEBUG("Unable to allocated DPAA2 buffer\n");
 465                 return -1;
 466         }
 467         m = (struct rte_mbuf *)mb;
 468         memcpy((char *)m->buf_addr + mbuf->data_off,
 469                (void *)((char *)mbuf->buf_addr + mbuf->data_off),
 470                 mbuf->pkt_len);
 471
 472         /* Copy required fields */
 473         m->data_off = mbuf->data_off;
 474         m->ol_flags = mbuf->ol_flags;
 475         m->packet_type = mbuf->packet_type;
 476         m->tx_offload = mbuf->tx_offload;
 477
 478         DPAA2_MBUF_TO_CONTIG_FD(m, fd, bpid);
 479
 480         DPAA2_PMD_DP_DEBUG(
 481                 "mbuf: %p, BMAN buf addr: %p, fdaddr: %" PRIx64 ", bpid: %d,"
 482                 " meta: %d, off: %d, len: %d\n",
 483                 (void *)mbuf,
 484                 mbuf->buf_addr,
 485                 DPAA2_GET_FD_ADDR(fd),
 486                 DPAA2_GET_FD_BPID(fd),
 487                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size,
 488                 DPAA2_GET_FD_OFFSET(fd),
 489                 DPAA2_GET_FD_LEN(fd));
 490
 491 return 0;
 492 }
 493
 494 /* This function assumes that caller will be keep the same value for nb_pkts
 495  * across calls per queue, if that is not the case, better use non-prefetch
 496  * version of rx call.
 497  * It will return the packets as requested in previous call without honoring
 498  * the current nb_pkts or bufs space.
 499  */
 500 uint16_t
 501 dpaa2_dev_prefetch_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 502 {
 503         /* Function receive frames for a given device and VQ*/
 504         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
 505         struct qbman_result *dq_storage, *dq_storage1 = NULL;
 506         uint32_t fqid = dpaa2_q->fqid;
 507         int ret, num_rx = 0, pull_size;
 508         uint8_t pending, status;
 509         struct qbman_swp *swp;
 510         const struct qbman_fd *fd, *next_fd;
 511         struct qbman_pull_desc pulldesc;
 512         struct queue_storage_info_t *q_storage = dpaa2_q->q_storage;
 513         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
 514
 515         if (unlikely(!DPAA2_PER_LCORE_ETHRX_DPIO)) {
 516                 ret = dpaa2_affine_qbman_ethrx_swp();
 517                 if (ret) {
 518                         DPAA2_PMD_ERR("Failure in affining portal");
 519                         return 0;
 520                 }
 521         }
 522
 523         if (unlikely(!rte_dpaa2_bpid_info &&
 524                      rte_eal_process_type() == RTE_PROC_SECONDARY))
 525                 rte_dpaa2_bpid_info = dpaa2_q->bp_array;
 526
 527         swp = DPAA2_PER_LCORE_ETHRX_PORTAL;
 528         pull_size = (nb_pkts > dpaa2_dqrr_size) ? dpaa2_dqrr_size : nb_pkts;
 529         if (unlikely(!q_storage->active_dqs)) {
 530                 q_storage->toggle = 0;
 531                 dq_storage = q_storage->dq_storage[q_storage->toggle];
 532                 q_storage->last_num_pkts = pull_size;
 533                 qbman_pull_desc_clear(&pulldesc);
 534                 qbman_pull_desc_set_numframes(&pulldesc,
 535                                               q_storage->last_num_pkts);
 536                 qbman_pull_desc_set_fq(&pulldesc, fqid);
 537                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
 538                         (uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
 539                 if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) {
 540                         while (!qbman_check_command_complete(
 541                                get_swp_active_dqs(
 542                                DPAA2_PER_LCORE_ETHRX_DPIO->index)))
 543                                 ;
 544                         clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index);
 545                 }
 546                 while (1) {
 547                         if (qbman_swp_pull(swp, &pulldesc)) {
 548                                 DPAA2_PMD_DP_DEBUG("VDQ command is not issued."
 549                                                   " QBMAN is busy (1)\n");
 550                                 /* Portal was busy, try again */
 551                                 continue;
 552                         }
 553                         break;
 554                 }
 555                 q_storage->active_dqs = dq_storage;
 556                 q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index;
 557                 set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index,
 558                                    dq_storage);
 559         }
 560
 561         dq_storage = q_storage->active_dqs;
 562         rte_prefetch0((void *)(size_t)(dq_storage));
 563         rte_prefetch0((void *)(size_t)(dq_storage + 1));
 564
 565         /* Prepare next pull descriptor. This will give space for the
 566          * prefething done on DQRR entries
 567          */
 568         q_storage->toggle ^= 1;
 569         dq_storage1 = q_storage->dq_storage[q_storage->toggle];
 570         qbman_pull_desc_clear(&pulldesc);
 571         qbman_pull_desc_set_numframes(&pulldesc, pull_size);
 572         qbman_pull_desc_set_fq(&pulldesc, fqid);
 573         qbman_pull_desc_set_storage(&pulldesc, dq_storage1,
 574                 (uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage1)), 1);
 575
 576         /* Check if the previous issued command is completed.
 577          * Also seems like the SWP is shared between the Ethernet Driver
 578          * and the SEC driver.
 579          */
 580         while (!qbman_check_command_complete(dq_storage))
 581                 ;
 582         if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id))
 583                 clear_swp_active_dqs(q_storage->active_dpio_id);
 584
 585         pending = 1;
 586
 587         do {
 588                 /* Loop until the dq_storage is updated with
 589                  * new token by QBMAN
 590                  */
 591                 while (!qbman_check_new_result(dq_storage))
 592                         ;
 593                 rte_prefetch0((void *)((size_t)(dq_storage + 2)));
 594                 /* Check whether Last Pull command is Expired and
 595                  * setting Condition for Loop termination
 596                  */
 597                 if (qbman_result_DQ_is_pull_complete(dq_storage)) {
 598                         pending = 0;
 599                         /* Check for valid frame. */
 600                         status = qbman_result_DQ_flags(dq_storage);
 601                         if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0))
 602                                 continue;
 603                 }
 604                 fd = qbman_result_DQ_fd(dq_storage);
 605
 606                 if (dpaa2_svr_family != SVR_LX2160A) {
 607                         next_fd = qbman_result_DQ_fd(dq_storage + 1);
 608                         /* Prefetch Annotation address for the parse results */
 609                         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(
 610                                       next_fd) + DPAA2_FD_PTA_SIZE + 16));
 611                 }
 612
 613                 if (unlikely(DPAA2_FD_GET_FORMAT(fd) == qbman_fd_sg))
 614                         bufs[num_rx] = eth_sg_fd_to_mbuf(fd);
 615                 else
 616                         bufs[num_rx] = eth_fd_to_mbuf(fd);
 617                 bufs[num_rx]->port = eth_data->port_id;
 618
 619                 if (eth_data->dev_conf.rxmode.offloads &
 620                                 DEV_RX_OFFLOAD_VLAN_STRIP)
 621                         rte_vlan_strip(bufs[num_rx]);
 622
 623                 dq_storage++;
 624                 num_rx++;
 625         } while (pending);
 626
 627         if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) {
 628                 while (!qbman_check_command_complete(
 629                        get_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)))
 630                         ;
 631                 clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index);
 632         }
 633         /* issue a volatile dequeue command for next pull */
 634         while (1) {
 635                 if (qbman_swp_pull(swp, &pulldesc)) {
 636                         DPAA2_PMD_DP_DEBUG("VDQ command is not issued."
 637                                           "QBMAN is busy (2)\n");
 638                         continue;
 639                 }
 640                 break;
 641         }
 642         q_storage->active_dqs = dq_storage1;
 643         q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index;
 644         set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index, dq_storage1);
 645
 646         dpaa2_q->rx_pkts += num_rx;
 647
 648         return num_rx;
 649 }
 650
 651 void __attribute__((hot))
 652 dpaa2_dev_process_parallel_event(struct qbman_swp *swp,
 653                                  const struct qbman_fd *fd,
 654                                  const struct qbman_result *dq,
 655                                  struct dpaa2_queue *rxq,
 656                                  struct rte_event *ev)
 657 {
 658         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd) +
 659                 DPAA2_FD_PTA_SIZE + 16));
 660
 661         ev->flow_id = rxq->ev.flow_id;
 662         ev->sub_event_type = rxq->ev.sub_event_type;
 663         ev->event_type = RTE_EVENT_TYPE_ETHDEV;
 664         ev->op = RTE_EVENT_OP_NEW;
 665         ev->sched_type = rxq->ev.sched_type;
 666         ev->queue_id = rxq->ev.queue_id;
 667         ev->priority = rxq->ev.priority;
 668
 669         ev->mbuf = eth_fd_to_mbuf(fd);
 670
 671         qbman_swp_dqrr_consume(swp, dq);
 672 }
 673
 674 void __attribute__((hot))
 675 dpaa2_dev_process_atomic_event(struct qbman_swp *swp __attribute__((unused)),
 676                                const struct qbman_fd *fd,
 677                                const struct qbman_result *dq,
 678                                struct dpaa2_queue *rxq,
 679                                struct rte_event *ev)
 680 {
 681         uint8_t dqrr_index;
 682
 683         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd) +
 684                 DPAA2_FD_PTA_SIZE + 16));
 685
 686         ev->flow_id = rxq->ev.flow_id;
 687         ev->sub_event_type = rxq->ev.sub_event_type;
 688         ev->event_type = RTE_EVENT_TYPE_ETHDEV;
 689         ev->op = RTE_EVENT_OP_NEW;
 690         ev->sched_type = rxq->ev.sched_type;
 691         ev->queue_id = rxq->ev.queue_id;
 692         ev->priority = rxq->ev.priority;
 693
 694         ev->mbuf = eth_fd_to_mbuf(fd);
 695
 696         dqrr_index = qbman_get_dqrr_idx(dq);
 697         ev->mbuf->seqn = dqrr_index + 1;
 698         DPAA2_PER_LCORE_DQRR_SIZE++;
 699         DPAA2_PER_LCORE_DQRR_HELD |= 1 << dqrr_index;
 700         DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = ev->mbuf;
 701 }
 702
 703 void __attribute__((hot))
 704 dpaa2_dev_process_ordered_event(struct qbman_swp *swp,
 705                                 const struct qbman_fd *fd,
 706                                 const struct qbman_result *dq,
 707                                 struct dpaa2_queue *rxq,
 708                                 struct rte_event *ev)
 709 {
 710         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd) +
 711                 DPAA2_FD_PTA_SIZE + 16));
 712
 713         ev->flow_id = rxq->ev.flow_id;
 714         ev->sub_event_type = rxq->ev.sub_event_type;
 715         ev->event_type = RTE_EVENT_TYPE_ETHDEV;
 716         ev->op = RTE_EVENT_OP_NEW;
 717         ev->sched_type = rxq->ev.sched_type;
 718         ev->queue_id = rxq->ev.queue_id;
 719         ev->priority = rxq->ev.priority;
 720
 721         ev->mbuf = eth_fd_to_mbuf(fd);
 722
 723         ev->mbuf->seqn = DPAA2_ENQUEUE_FLAG_ORP;
 724         ev->mbuf->seqn |= qbman_result_DQ_odpid(dq) << DPAA2_EQCR_OPRID_SHIFT;
 725         ev->mbuf->seqn |= qbman_result_DQ_seqnum(dq) << DPAA2_EQCR_SEQNUM_SHIFT;
 726
 727         qbman_swp_dqrr_consume(swp, dq);
 728 }
 729
 730 uint16_t
 731 dpaa2_dev_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 732 {
 733         /* Function receive frames for a given device and VQ */
 734         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
 735         struct qbman_result *dq_storage;
 736         uint32_t fqid = dpaa2_q->fqid;
 737         int ret, num_rx = 0, next_pull = nb_pkts, num_pulled;
 738         uint8_t pending, status;
 739         struct qbman_swp *swp;
 740         const struct qbman_fd *fd, *next_fd;
 741         struct qbman_pull_desc pulldesc;
 742         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
 743
 744         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 745                 ret = dpaa2_affine_qbman_swp();
 746                 if (ret) {
 747                         DPAA2_PMD_ERR("Failure in affining portal\n");
 748                         return 0;
 749                 }
 750         }
 751         swp = DPAA2_PER_LCORE_PORTAL;
 752
 753         do {
 754                 dq_storage = dpaa2_q->q_storage->dq_storage[0];
 755                 qbman_pull_desc_clear(&pulldesc);
 756                 qbman_pull_desc_set_fq(&pulldesc, fqid);
 757                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
 758                                 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
 759
 760                 if (next_pull > dpaa2_dqrr_size) {
 761                         qbman_pull_desc_set_numframes(&pulldesc,
 762                                 dpaa2_dqrr_size);
 763                         next_pull -= dpaa2_dqrr_size;
 764                 } else {
 765                         qbman_pull_desc_set_numframes(&pulldesc, next_pull);
 766                         next_pull = 0;
 767                 }
 768
 769                 while (1) {
 770                         if (qbman_swp_pull(swp, &pulldesc)) {
 771                                 DPAA2_PMD_DP_DEBUG(
 772                                         "VDQ command is not issued.QBMAN is busy\n");
 773                                 /* Portal was busy, try again */
 774                                 continue;
 775                         }
 776                         break;
 777                 }
 778
 779                 rte_prefetch0((void *)((size_t)(dq_storage + 1)));
 780                 /* Check if the previous issued command is completed. */
 781                 while (!qbman_check_command_complete(dq_storage))
 782                         ;
 783
 784                 num_pulled = 0;
 785                 pending = 1;
 786                 do {
 787                         /* Loop until the dq_storage is updated with
 788                          * new token by QBMAN
 789                          */
 790                         while (!qbman_check_new_result(dq_storage))
 791                                 ;
 792                         rte_prefetch0((void *)((size_t)(dq_storage + 2)));
 793                         /* Check whether Last Pull command is Expired and
 794                          * setting Condition for Loop termination
 795                          */
 796                         if (qbman_result_DQ_is_pull_complete(dq_storage)) {
 797                                 pending = 0;
 798                                 /* Check for valid frame. */
 799                                 status = qbman_result_DQ_flags(dq_storage);
 800                                 if (unlikely((status &
 801                                         QBMAN_DQ_STAT_VALIDFRAME) == 0))
 802                                         continue;
 803                         }
 804                         fd = qbman_result_DQ_fd(dq_storage);
 805
 806                         next_fd = qbman_result_DQ_fd(dq_storage + 1);
 807                         /* Prefetch Annotation address for the parse results */
 808                         rte_prefetch0(
 809                                 (void *)(size_t)(DPAA2_GET_FD_ADDR(next_fd)
 810                                         + DPAA2_FD_PTA_SIZE + 16));
 811
 812                         if (unlikely(DPAA2_FD_GET_FORMAT(fd) == qbman_fd_sg))
 813                                 bufs[num_rx] = eth_sg_fd_to_mbuf(fd);
 814                         else
 815                                 bufs[num_rx] = eth_fd_to_mbuf(fd);
 816                         bufs[num_rx]->port = eth_data->port_id;
 817
 818                 if (eth_data->dev_conf.rxmode.offloads &
 819                                 DEV_RX_OFFLOAD_VLAN_STRIP) {
 820                         rte_vlan_strip(bufs[num_rx]);
 821                 }
 822
 823                         dq_storage++;
 824                         num_rx++;
 825                         num_pulled++;
 826                 } while (pending);
 827         /* Last VDQ provided all packets and more packets are requested */
 828         } while (next_pull && num_pulled == dpaa2_dqrr_size);
 829
 830         dpaa2_q->rx_pkts += num_rx;
 831
 832         return num_rx;
 833 }
 834
 835 uint16_t dpaa2_dev_tx_conf(void *queue)
 836 {
 837         /* Function receive frames for a given device and VQ */
 838         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
 839         struct qbman_result *dq_storage;
 840         uint32_t fqid = dpaa2_q->fqid;
 841         int ret, num_tx_conf = 0, num_pulled;
 842         uint8_t pending, status;
 843         struct qbman_swp *swp;
 844         const struct qbman_fd *fd, *next_fd;
 845         struct qbman_pull_desc pulldesc;
 846         struct qbman_release_desc releasedesc;
 847         uint32_t bpid;
 848         uint64_t buf;
 849
 850         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 851                 ret = dpaa2_affine_qbman_swp();
 852                 if (ret) {
 853                         DPAA2_PMD_ERR("Failure in affining portal\n");
 854                         return 0;
 855                 }
 856         }
 857         swp = DPAA2_PER_LCORE_PORTAL;
 858
 859         do {
 860                 dq_storage = dpaa2_q->q_storage->dq_storage[0];
 861                 qbman_pull_desc_clear(&pulldesc);
 862                 qbman_pull_desc_set_fq(&pulldesc, fqid);
 863                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
 864                                 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
 865
 866                 qbman_pull_desc_set_numframes(&pulldesc, dpaa2_dqrr_size);
 867
 868                 while (1) {
 869                         if (qbman_swp_pull(swp, &pulldesc)) {
 870                                 DPAA2_PMD_DP_DEBUG("VDQ command is not issued."
 871                                                    "QBMAN is busy\n");
 872                                 /* Portal was busy, try again */
 873                                 continue;
 874                         }
 875                         break;
 876                 }
 877
 878                 rte_prefetch0((void *)((size_t)(dq_storage + 1)));
 879                 /* Check if the previous issued command is completed. */
 880                 while (!qbman_check_command_complete(dq_storage))
 881                         ;
 882
 883                 num_pulled = 0;
 884                 pending = 1;
 885                 do {
 886                         /* Loop until the dq_storage is updated with
 887                          * new token by QBMAN
 888                          */
 889                         while (!qbman_check_new_result(dq_storage))
 890                                 ;
 891                         rte_prefetch0((void *)((size_t)(dq_storage + 2)));
 892                         /* Check whether Last Pull command is Expired and
 893                          * setting Condition for Loop termination
 894                          */
 895                         if (qbman_result_DQ_is_pull_complete(dq_storage)) {
 896                                 pending = 0;
 897                                 /* Check for valid frame. */
 898                                 status = qbman_result_DQ_flags(dq_storage);
 899                                 if (unlikely((status &
 900                                         QBMAN_DQ_STAT_VALIDFRAME) == 0))
 901                                         continue;
 902                         }
 903                         fd = qbman_result_DQ_fd(dq_storage);
 904
 905                         next_fd = qbman_result_DQ_fd(dq_storage + 1);
 906                         /* Prefetch Annotation address for the parse results */
 907                         rte_prefetch0((void *)(size_t)
 908                                 (DPAA2_GET_FD_ADDR(next_fd) +
 909                                  DPAA2_FD_PTA_SIZE + 16));
 910
 911                         bpid = DPAA2_GET_FD_BPID(fd);
 912
 913                         /* Create a release descriptor required for releasing
 914                          * buffers into QBMAN
 915                          */
 916                         qbman_release_desc_clear(&releasedesc);
 917                         qbman_release_desc_set_bpid(&releasedesc, bpid);
 918
 919                         buf = DPAA2_GET_FD_ADDR(fd);
 920                         /* feed them to bman */
 921                         do {
 922                                 ret = qbman_swp_release(swp, &releasedesc,
 923                                                         &buf, 1);
 924                         } while (ret == -EBUSY);
 925
 926                         dq_storage++;
 927                         num_tx_conf++;
 928                         num_pulled++;
 929                 } while (pending);
 930
 931         /* Last VDQ provided all packets and more packets are requested */
 932         } while (num_pulled == dpaa2_dqrr_size);
 933
 934         dpaa2_q->rx_pkts += num_tx_conf;
 935
 936         return num_tx_conf;
 937 }
 938
 939 /*
 940  * Callback to handle sending packets through WRIOP based interface
 941  */
 942 uint16_t
 943 dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 944 {
 945         /* Function to transmit the frames to given device and VQ*/
 946         uint32_t loop, retry_count;
 947         int32_t ret;
 948         struct qbman_fd fd_arr[MAX_TX_RING_SLOTS];
 949         struct rte_mbuf *mi;
 950         uint32_t frames_to_send;
 951         struct rte_mempool *mp;
 952         struct qbman_eq_desc eqdesc;
 953         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
 954         struct qbman_swp *swp;
 955         uint16_t num_tx = 0;
 956         uint16_t bpid;
 957         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
 958         struct dpaa2_dev_priv *priv = eth_data->dev_private;
 959         uint32_t flags[MAX_TX_RING_SLOTS] = {0};
 960
 961         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 962                 ret = dpaa2_affine_qbman_swp();
 963                 if (ret) {
 964                         DPAA2_PMD_ERR("Failure in affining portal");
 965                         return 0;
 966                 }
 967         }
 968         swp = DPAA2_PER_LCORE_PORTAL;
 969
 970         DPAA2_PMD_DP_DEBUG("===> eth_data =%p, fqid =%d\n",
 971                         eth_data, dpaa2_q->fqid);
 972
 973         /*Prepare enqueue descriptor*/
 974         qbman_eq_desc_clear(&eqdesc);
 975         qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ);
 976         qbman_eq_desc_set_fq(&eqdesc, dpaa2_q->fqid);
 977
 978         /*Clear the unused FD fields before sending*/
 979         while (nb_pkts) {
 980                 /*Check if the queue is congested*/
 981                 retry_count = 0;
 982                 while (qbman_result_SCN_state(dpaa2_q->cscn)) {
 983                         retry_count++;
 984                         /* Retry for some time before giving up */
 985                         if (retry_count > CONG_RETRY_COUNT)
 986                                 goto skip_tx;
 987                 }
 988
 989                 frames_to_send = (nb_pkts > dpaa2_eqcr_size) ?
 990                         dpaa2_eqcr_size : nb_pkts;
 991
 992                 for (loop = 0; loop < frames_to_send; loop++) {
 993                         if ((*bufs)->seqn) {
 994                                 uint8_t dqrr_index = (*bufs)->seqn - 1;
 995
 996                                 flags[loop] = QBMAN_ENQUEUE_FLAG_DCA |
 997                                                 dqrr_index;
 998                                 DPAA2_PER_LCORE_DQRR_SIZE--;
 999                                 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dqrr_index);
1000                                 (*bufs)->seqn = DPAA2_INVALID_MBUF_SEQN;
1001                         }
1002
1003                         if (likely(RTE_MBUF_DIRECT(*bufs))) {
1004                                 mp = (*bufs)->pool;
1005                                 /* Check the basic scenario and set
1006                                  * the FD appropriately here itself.
1007                                  */
1008                                 if (likely(mp && mp->ops_index ==
1009                                     priv->bp_list->dpaa2_ops_index &&
1010                                     (*bufs)->nb_segs == 1 &&
1011                                     rte_mbuf_refcnt_read((*bufs)) == 1)) {
1012                                         if (unlikely(((*bufs)->ol_flags
1013                                                 & PKT_TX_VLAN_PKT) ||
1014                                                 (eth_data->dev_conf.txmode.offloads
1015                                                 & DEV_TX_OFFLOAD_VLAN_INSERT))) {
1016                                                 ret = rte_vlan_insert(bufs);
1017                                                 if (ret)
1018                                                         goto send_n_return;
1019                                         }
1020                                         DPAA2_MBUF_TO_CONTIG_FD((*bufs),
1021                                         &fd_arr[loop], mempool_to_bpid(mp));
1022                                         bufs++;
1023                                         continue;
1024                                 }
1025                         } else {
1026                                 mi = rte_mbuf_from_indirect(*bufs);
1027                                 mp = mi->pool;
1028                         }
1029                         /* Not a hw_pkt pool allocated frame */
1030                         if (unlikely(!mp || !priv->bp_list)) {
1031                                 DPAA2_PMD_ERR("Err: No buffer pool attached");
1032                                 goto send_n_return;
1033                         }
1034
1035                         if (unlikely(((*bufs)->ol_flags & PKT_TX_VLAN_PKT) ||
1036                                 (eth_data->dev_conf.txmode.offloads
1037                                 & DEV_TX_OFFLOAD_VLAN_INSERT))) {
1038                                 int ret = rte_vlan_insert(bufs);
1039                                 if (ret)
1040                                         goto send_n_return;
1041                         }
1042                         if (mp->ops_index != priv->bp_list->dpaa2_ops_index) {
1043                                 DPAA2_PMD_WARN("Non DPAA2 buffer pool");
1044                                 /* alloc should be from the default buffer pool
1045                                  * attached to this interface
1046                                  */
1047                                 bpid = priv->bp_list->buf_pool.bpid;
1048
1049                                 if (unlikely((*bufs)->nb_segs > 1)) {
1050                                         DPAA2_PMD_ERR("S/G support not added"
1051                                                 " for non hw offload buffer");
1052                                         goto send_n_return;
1053                                 }
1054                                 if (eth_copy_mbuf_to_fd(*bufs,
1055                                                         &fd_arr[loop], bpid)) {
1056                                         goto send_n_return;
1057                                 }
1058                                 /* free the original packet */
1059                                 rte_pktmbuf_free(*bufs);
1060                         } else {
1061                                 bpid = mempool_to_bpid(mp);
1062                                 if (unlikely((*bufs)->nb_segs > 1)) {
1063                                         if (eth_mbuf_to_sg_fd(*bufs,
1064                                                         &fd_arr[loop], bpid))
1065                                                 goto send_n_return;
1066                                 } else {
1067                                         eth_mbuf_to_fd(*bufs,
1068                                                        &fd_arr[loop], bpid);
1069                                 }
1070                         }
1071                         bufs++;
1072                 }
1073                 loop = 0;
1074                 while (loop < frames_to_send) {
1075                         loop += qbman_swp_enqueue_multiple(swp, &eqdesc,
1076                                         &fd_arr[loop], &flags[loop],
1077                                         frames_to_send - loop);
1078                 }
1079
1080                 num_tx += frames_to_send;
1081                 nb_pkts -= frames_to_send;
1082         }
1083         dpaa2_q->tx_pkts += num_tx;
1084         return num_tx;
1085
1086 send_n_return:
1087         /* send any already prepared fd */
1088         if (loop) {
1089                 unsigned int i = 0;
1090
1091                 while (i < loop) {
1092                         i += qbman_swp_enqueue_multiple(swp, &eqdesc,
1093                                                         &fd_arr[i],
1094                                                         &flags[loop],
1095                                                         loop - i);
1096                 }
1097                 num_tx += loop;
1098         }
1099 skip_tx:
1100         dpaa2_q->tx_pkts += num_tx;
1101         return num_tx;
1102 }
1103
1104 void
1105 dpaa2_dev_free_eqresp_buf(uint16_t eqresp_ci)
1106 {
1107         struct dpaa2_dpio_dev *dpio_dev = DPAA2_PER_LCORE_DPIO;
1108         struct qbman_fd *fd;
1109         struct rte_mbuf *m;
1110
1111         fd = qbman_result_eqresp_fd(&dpio_dev->eqresp[eqresp_ci]);
1112         m = eth_fd_to_mbuf(fd);
1113         rte_pktmbuf_free(m);
1114 }
1115
1116 static void
1117 dpaa2_set_enqueue_descriptor(struct dpaa2_queue *dpaa2_q,
1118                              struct rte_mbuf *m,
1119                              struct qbman_eq_desc *eqdesc)
1120 {
1121         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
1122         struct dpaa2_dev_priv *priv = eth_data->dev_private;
1123         struct dpaa2_dpio_dev *dpio_dev = DPAA2_PER_LCORE_DPIO;
1124         struct eqresp_metadata *eqresp_meta;
1125         uint16_t orpid, seqnum;
1126         uint8_t dq_idx;
1127
1128         qbman_eq_desc_set_fq(eqdesc, dpaa2_q->fqid);
1129
1130         if (m->seqn & DPAA2_ENQUEUE_FLAG_ORP) {
1131                 orpid = (m->seqn & DPAA2_EQCR_OPRID_MASK) >>
1132                         DPAA2_EQCR_OPRID_SHIFT;
1133                 seqnum = (m->seqn & DPAA2_EQCR_SEQNUM_MASK) >>
1134                         DPAA2_EQCR_SEQNUM_SHIFT;
1135
1136                 if (!priv->en_loose_ordered) {
1137                         qbman_eq_desc_set_orp(eqdesc, 1, orpid, seqnum, 0);
1138                         qbman_eq_desc_set_response(eqdesc, (uint64_t)
1139                                 DPAA2_VADDR_TO_IOVA(&dpio_dev->eqresp[
1140                                 dpio_dev->eqresp_pi]), 1);
1141                         qbman_eq_desc_set_token(eqdesc, 1);
1142
1143                         eqresp_meta = &dpio_dev->eqresp_meta[
1144                                 dpio_dev->eqresp_pi];
1145                         eqresp_meta->dpaa2_q = dpaa2_q;
1146                         eqresp_meta->mp = m->pool;
1147
1148                         dpio_dev->eqresp_pi + 1 < MAX_EQ_RESP_ENTRIES ?
1149                                 dpio_dev->eqresp_pi++ :
1150                                 (dpio_dev->eqresp_pi = 0);
1151                 } else {
1152                         qbman_eq_desc_set_orp(eqdesc, 0, orpid, seqnum, 0);
1153                 }
1154         } else {
1155                 dq_idx = m->seqn - 1;
1156                 qbman_eq_desc_set_dca(eqdesc, 1, dq_idx, 0);
1157                 DPAA2_PER_LCORE_DQRR_SIZE--;
1158                 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dq_idx);
1159         }
1160         m->seqn = DPAA2_INVALID_MBUF_SEQN;
1161 }
1162
1163 /* Callback to handle sending ordered packets through WRIOP based interface */
1164 uint16_t
1165 dpaa2_dev_tx_ordered(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
1166 {
1167         /* Function to transmit the frames to given device and VQ*/
1168         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
1169         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
1170         struct dpaa2_dev_priv *priv = eth_data->dev_private;
1171         struct dpaa2_queue *order_sendq = (struct dpaa2_queue *)priv->tx_vq[0];
1172         struct qbman_fd fd_arr[MAX_TX_RING_SLOTS];
1173         struct rte_mbuf *mi;
1174         struct rte_mempool *mp;
1175         struct qbman_eq_desc eqdesc[MAX_TX_RING_SLOTS];
1176         struct qbman_swp *swp;
1177         uint32_t frames_to_send, num_free_eq_desc;
1178         uint32_t loop, retry_count;
1179         int32_t ret;
1180         uint16_t num_tx = 0;
1181         uint16_t bpid;
1182
1183         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
1184                 ret = dpaa2_affine_qbman_swp();
1185                 if (ret) {
1186                         DPAA2_PMD_ERR("Failure in affining portal");
1187                         return 0;
1188                 }
1189         }
1190         swp = DPAA2_PER_LCORE_PORTAL;
1191
1192         DPAA2_PMD_DP_DEBUG("===> eth_data =%p, fqid =%d\n",
1193                            eth_data, dpaa2_q->fqid);
1194
1195         /* This would also handle normal and atomic queues as any type
1196          * of packet can be enqueued when ordered queues are being used.
1197          */
1198         while (nb_pkts) {
1199                 /*Check if the queue is congested*/
1200                 retry_count = 0;
1201                 while (qbman_result_SCN_state(dpaa2_q->cscn)) {
1202                         retry_count++;
1203                         /* Retry for some time before giving up */
1204                         if (retry_count > CONG_RETRY_COUNT)
1205                                 goto skip_tx;
1206                 }
1207
1208                 frames_to_send = (nb_pkts > dpaa2_eqcr_size) ?
1209                         dpaa2_eqcr_size : nb_pkts;
1210
1211                 if (!priv->en_loose_ordered) {
1212                         if ((*bufs)->seqn & DPAA2_ENQUEUE_FLAG_ORP) {
1213                                 num_free_eq_desc = dpaa2_free_eq_descriptors();
1214                                 if (num_free_eq_desc < frames_to_send)
1215                                         frames_to_send = num_free_eq_desc;
1216                         }
1217                 }
1218
1219                 for (loop = 0; loop < frames_to_send; loop++) {
1220                         /*Prepare enqueue descriptor*/
1221                         qbman_eq_desc_clear(&eqdesc[loop]);
1222
1223                         if ((*bufs)->seqn) {
1224                                 /* Use only queue 0 for Tx in case of atomic/
1225                                  * ordered packets as packets can get unordered
1226                                  * when being tranmitted out from the interface
1227                                  */
1228                                 dpaa2_set_enqueue_descriptor(order_sendq,
1229                                                              (*bufs),
1230                                                              &eqdesc[loop]);
1231                         } else {
1232                                 qbman_eq_desc_set_no_orp(&eqdesc[loop],
1233                                                          DPAA2_EQ_RESP_ERR_FQ);
1234                                 qbman_eq_desc_set_fq(&eqdesc[loop],
1235                                                      dpaa2_q->fqid);
1236                         }
1237
1238                         if (likely(RTE_MBUF_DIRECT(*bufs))) {
1239                                 mp = (*bufs)->pool;
1240                                 /* Check the basic scenario and set
1241                                  * the FD appropriately here itself.
1242                                  */
1243                                 if (likely(mp && mp->ops_index ==
1244                                     priv->bp_list->dpaa2_ops_index &&
1245                                     (*bufs)->nb_segs == 1 &&
1246                                     rte_mbuf_refcnt_read((*bufs)) == 1)) {
1247                                         if (unlikely((*bufs)->ol_flags
1248                                                 & PKT_TX_VLAN_PKT)) {
1249                                           ret = rte_vlan_insert(bufs);
1250                                           if (ret)
1251                                                 goto send_n_return;
1252                                         }
1253                                         DPAA2_MBUF_TO_CONTIG_FD((*bufs),
1254                                                 &fd_arr[loop],
1255                                                 mempool_to_bpid(mp));
1256                                         bufs++;
1257                                         continue;
1258                                 }
1259                         } else {
1260                                 mi = rte_mbuf_from_indirect(*bufs);
1261                                 mp = mi->pool;
1262                         }
1263                         /* Not a hw_pkt pool allocated frame */
1264                         if (unlikely(!mp || !priv->bp_list)) {
1265                                 DPAA2_PMD_ERR("Err: No buffer pool attached");
1266                                 goto send_n_return;
1267                         }
1268
1269                         if (mp->ops_index != priv->bp_list->dpaa2_ops_index) {
1270                                 DPAA2_PMD_WARN("Non DPAA2 buffer pool");
1271                                 /* alloc should be from the default buffer pool
1272                                  * attached to this interface
1273                                  */
1274                                 bpid = priv->bp_list->buf_pool.bpid;
1275
1276                                 if (unlikely((*bufs)->nb_segs > 1)) {
1277                                         DPAA2_PMD_ERR(
1278                                                 "S/G not supp for non hw offload buffer");
1279                                         goto send_n_return;
1280                                 }
1281                                 if (eth_copy_mbuf_to_fd(*bufs,
1282                                                         &fd_arr[loop], bpid)) {
1283                                         goto send_n_return;
1284                                 }
1285                                 /* free the original packet */
1286                                 rte_pktmbuf_free(*bufs);
1287                         } else {
1288                                 bpid = mempool_to_bpid(mp);
1289                                 if (unlikely((*bufs)->nb_segs > 1)) {
1290                                         if (eth_mbuf_to_sg_fd(*bufs,
1291                                                               &fd_arr[loop],
1292                                                               bpid))
1293                                                 goto send_n_return;
1294                                 } else {
1295                                         eth_mbuf_to_fd(*bufs,
1296                                                        &fd_arr[loop], bpid);
1297                                 }
1298                         }
1299                         bufs++;
1300                 }
1301                 loop = 0;
1302                 while (loop < frames_to_send) {
1303                         loop += qbman_swp_enqueue_multiple_desc(swp,
1304                                         &eqdesc[loop], &fd_arr[loop],
1305                                         frames_to_send - loop);
1306                 }
1307
1308                 num_tx += frames_to_send;
1309                 nb_pkts -= frames_to_send;
1310         }
1311         dpaa2_q->tx_pkts += num_tx;
1312         return num_tx;
1313
1314 send_n_return:
1315         /* send any already prepared fd */
1316         if (loop) {
1317                 unsigned int i = 0;
1318
1319                 while (i < loop) {
1320                         i += qbman_swp_enqueue_multiple_desc(swp, &eqdesc[loop],
1321                                                         &fd_arr[i], loop - i);
1322                 }
1323                 num_tx += loop;
1324         }
1325 skip_tx:
1326         dpaa2_q->tx_pkts += num_tx;
1327         return num_tx;
1328 }
1329
1330 /**
1331  * Dummy DPDK callback for TX.
1332  *
1333  * This function is used to temporarily replace the real callback during
1334  * unsafe control operations on the queue, or in case of error.
1335  *
1336  * @param dpdk_txq
1337  *   Generic pointer to TX queue structure.
1338  * @param[in] pkts
1339  *   Packets to transmit.
1340  * @param pkts_n
1341  *   Number of packets in array.
1342  *
1343  * @return
1344  *   Number of packets successfully transmitted (<= pkts_n).
1345  */
1346 uint16_t
1347 dummy_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
1348 {
1349         (void)queue;
1350         (void)bufs;
1351         (void)nb_pkts;
1352         return 0;
1353 }
1354
1355 #if defined(RTE_TOOLCHAIN_GCC)
1356 #pragma GCC diagnostic push
1357 #pragma GCC diagnostic ignored "-Wcast-qual"
1358 #elif defined(RTE_TOOLCHAIN_CLANG)
1359 #pragma clang diagnostic push
1360 #pragma clang diagnostic ignored "-Wcast-qual"
1361 #endif
1362
1363 /* This function loopbacks all the received packets.*/
1364 uint16_t
1365 dpaa2_dev_loopback_rx(void *queue,
1366                       struct rte_mbuf **bufs __rte_unused,
1367                       uint16_t nb_pkts)
1368 {
1369         /* Function receive frames for a given device and VQ*/
1370         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
1371         struct qbman_result *dq_storage, *dq_storage1 = NULL;
1372         uint32_t fqid = dpaa2_q->fqid;
1373         int ret, num_rx = 0, num_tx = 0, pull_size;
1374         uint8_t pending, status;
1375         struct qbman_swp *swp;
1376         struct qbman_fd *fd[DPAA2_LX2_DQRR_RING_SIZE];
1377         struct qbman_pull_desc pulldesc;
1378         struct qbman_eq_desc eqdesc;
1379         struct queue_storage_info_t *q_storage = dpaa2_q->q_storage;
1380         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
1381         struct dpaa2_dev_priv *priv = eth_data->dev_private;
1382         struct dpaa2_queue *tx_q = priv->tx_vq[0];
1383         /* todo - currently we are using 1st TX queue only for loopback*/
1384
1385         if (unlikely(!DPAA2_PER_LCORE_ETHRX_DPIO)) {
1386                 ret = dpaa2_affine_qbman_ethrx_swp();
1387                 if (ret) {
1388                         DPAA2_PMD_ERR("Failure in affining portal");
1389                         return 0;
1390                 }
1391         }
1392         swp = DPAA2_PER_LCORE_ETHRX_PORTAL;
1393         pull_size = (nb_pkts > dpaa2_dqrr_size) ? dpaa2_dqrr_size : nb_pkts;
1394         if (unlikely(!q_storage->active_dqs)) {
1395                 q_storage->toggle = 0;
1396                 dq_storage = q_storage->dq_storage[q_storage->toggle];
1397                 q_storage->last_num_pkts = pull_size;
1398                 qbman_pull_desc_clear(&pulldesc);
1399                 qbman_pull_desc_set_numframes(&pulldesc,
1400                                               q_storage->last_num_pkts);
1401                 qbman_pull_desc_set_fq(&pulldesc, fqid);
1402                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
1403                         (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
1404                 if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) {
1405                         while (!qbman_check_command_complete(
1406                                get_swp_active_dqs(
1407                                DPAA2_PER_LCORE_ETHRX_DPIO->index)))
1408                                 ;
1409                         clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index);
1410                 }
1411                 while (1) {
1412                         if (qbman_swp_pull(swp, &pulldesc)) {
1413                                 DPAA2_PMD_DP_DEBUG(
1414                                         "VDQ command not issued.QBMAN busy\n");
1415                                 /* Portal was busy, try again */
1416                                 continue;
1417                         }
1418                         break;
1419                 }
1420                 q_storage->active_dqs = dq_storage;
1421                 q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index;
1422                 set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index,
1423                                    dq_storage);
1424         }
1425
1426         dq_storage = q_storage->active_dqs;
1427         rte_prefetch0((void *)(size_t)(dq_storage));
1428         rte_prefetch0((void *)(size_t)(dq_storage + 1));
1429
1430         /* Prepare next pull descriptor. This will give space for the
1431          * prefething done on DQRR entries
1432          */
1433         q_storage->toggle ^= 1;
1434         dq_storage1 = q_storage->dq_storage[q_storage->toggle];
1435         qbman_pull_desc_clear(&pulldesc);
1436         qbman_pull_desc_set_numframes(&pulldesc, pull_size);
1437         qbman_pull_desc_set_fq(&pulldesc, fqid);
1438         qbman_pull_desc_set_storage(&pulldesc, dq_storage1,
1439                 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage1)), 1);
1440
1441         /*Prepare enqueue descriptor*/
1442         qbman_eq_desc_clear(&eqdesc);
1443         qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ);
1444         qbman_eq_desc_set_response(&eqdesc, 0, 0);
1445         qbman_eq_desc_set_fq(&eqdesc, tx_q->fqid);
1446
1447         /* Check if the previous issued command is completed.
1448          * Also seems like the SWP is shared between the Ethernet Driver
1449          * and the SEC driver.
1450          */
1451         while (!qbman_check_command_complete(dq_storage))
1452                 ;
1453         if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id))
1454                 clear_swp_active_dqs(q_storage->active_dpio_id);
1455
1456         pending = 1;
1457
1458         do {
1459                 /* Loop until the dq_storage is updated with
1460                  * new token by QBMAN
1461                  */
1462                 while (!qbman_check_new_result(dq_storage))
1463                         ;
1464                 rte_prefetch0((void *)((size_t)(dq_storage + 2)));
1465                 /* Check whether Last Pull command is Expired and
1466                  * setting Condition for Loop termination
1467                  */
1468                 if (qbman_result_DQ_is_pull_complete(dq_storage)) {
1469                         pending = 0;
1470                         /* Check for valid frame. */
1471                         status = qbman_result_DQ_flags(dq_storage);
1472                         if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0))
1473                                 continue;
1474                 }
1475                 fd[num_rx] = (struct qbman_fd *)qbman_result_DQ_fd(dq_storage);
1476
1477                 dq_storage++;
1478                 num_rx++;
1479         } while (pending);
1480
1481         while (num_tx < num_rx) {
1482                 num_tx += qbman_swp_enqueue_multiple_fd(swp, &eqdesc,
1483                                 &fd[num_tx], 0, num_rx - num_tx);
1484         }
1485
1486         if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) {
1487                 while (!qbman_check_command_complete(
1488                        get_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)))
1489                         ;
1490                 clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index);
1491         }
1492         /* issue a volatile dequeue command for next pull */
1493         while (1) {
1494                 if (qbman_swp_pull(swp, &pulldesc)) {
1495                         DPAA2_PMD_DP_DEBUG("VDQ command is not issued."
1496                                           "QBMAN is busy (2)\n");
1497                         continue;
1498                 }
1499                 break;
1500         }
1501         q_storage->active_dqs = dq_storage1;
1502         q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index;
1503         set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index, dq_storage1);
1504
1505         dpaa2_q->rx_pkts += num_rx;
1506         dpaa2_q->tx_pkts += num_tx;
1507
1508         return 0;
1509 }
1510 #if defined(RTE_TOOLCHAIN_GCC)
1511 #pragma GCC diagnostic pop
1512 #elif defined(RTE_TOOLCHAIN_CLANG)
1513 #pragma clang diagnostic pop
1514 #endif