drivers/net/dpaa2/dpaa2_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
   4  *   Copyright 2016-2020 NXP
   5  *
   6  */
   7
   8 #include <time.h>
   9 #include <net/if.h>
  10
  11 #include <rte_mbuf.h>
  12 #include <rte_ethdev_driver.h>
  13 #include <rte_malloc.h>
  14 #include <rte_memcpy.h>
  15 #include <rte_string_fns.h>
  16 #include <rte_dev.h>
  17
  18 #include <rte_fslmc.h>
  19 #include <fslmc_vfio.h>
  20 #include <dpaa2_hw_pvt.h>
  21 #include <dpaa2_hw_dpio.h>
  22 #include <dpaa2_hw_mempool.h>
  23
  24 #include "dpaa2_pmd_logs.h"
  25 #include "dpaa2_ethdev.h"
  26 #include "base/dpaa2_hw_dpni_annot.h"
  27
  28 static inline uint32_t __rte_hot
  29 dpaa2_dev_rx_parse_slow(struct rte_mbuf *mbuf,
  30                         struct dpaa2_annot_hdr *annotation);
  31
  32 static void enable_tx_tstamp(struct qbman_fd *fd) __rte_unused;
  33
  34 #define DPAA2_MBUF_TO_CONTIG_FD(_mbuf, _fd, _bpid)  do { \
  35         DPAA2_SET_FD_ADDR(_fd, DPAA2_MBUF_VADDR_TO_IOVA(_mbuf)); \
  36         DPAA2_SET_FD_LEN(_fd, _mbuf->data_len); \
  37         DPAA2_SET_ONLY_FD_BPID(_fd, _bpid); \
  38         DPAA2_SET_FD_OFFSET(_fd, _mbuf->data_off); \
  39         DPAA2_SET_FD_FRC(_fd, 0);               \
  40         DPAA2_RESET_FD_CTRL(_fd);               \
  41         DPAA2_RESET_FD_FLC(_fd);                \
  42 } while (0)
  43
  44 static inline void __rte_hot
  45 dpaa2_dev_rx_parse_new(struct rte_mbuf *m, const struct qbman_fd *fd,
  46                        void *hw_annot_addr)
  47 {
  48         uint16_t frc = DPAA2_GET_FD_FRC_PARSE_SUM(fd);
  49         struct dpaa2_annot_hdr *annotation =
  50                         (struct dpaa2_annot_hdr *)hw_annot_addr;
  51
  52         m->packet_type = RTE_PTYPE_UNKNOWN;
  53         switch (frc) {
  54         case DPAA2_PKT_TYPE_ETHER:
  55                 m->packet_type = RTE_PTYPE_L2_ETHER;
  56                 break;
  57         case DPAA2_PKT_TYPE_IPV4:
  58                 m->packet_type = RTE_PTYPE_L2_ETHER |
  59                         RTE_PTYPE_L3_IPV4;
  60                 break;
  61         case DPAA2_PKT_TYPE_IPV6:
  62                 m->packet_type = RTE_PTYPE_L2_ETHER |
  63                         RTE_PTYPE_L3_IPV6;
  64                 break;
  65         case DPAA2_PKT_TYPE_IPV4_EXT:
  66                 m->packet_type = RTE_PTYPE_L2_ETHER |
  67                         RTE_PTYPE_L3_IPV4_EXT;
  68                 break;
  69         case DPAA2_PKT_TYPE_IPV6_EXT:
  70                 m->packet_type = RTE_PTYPE_L2_ETHER |
  71                         RTE_PTYPE_L3_IPV6_EXT;
  72                 break;
  73         case DPAA2_PKT_TYPE_IPV4_TCP:
  74                 m->packet_type = RTE_PTYPE_L2_ETHER |
  75                         RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP;
  76                 break;
  77         case DPAA2_PKT_TYPE_IPV6_TCP:
  78                 m->packet_type = RTE_PTYPE_L2_ETHER |
  79                         RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP;
  80                 break;
  81         case DPAA2_PKT_TYPE_IPV4_UDP:
  82                 m->packet_type = RTE_PTYPE_L2_ETHER |
  83                         RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP;
  84                 break;
  85         case DPAA2_PKT_TYPE_IPV6_UDP:
  86                 m->packet_type = RTE_PTYPE_L2_ETHER |
  87                         RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP;
  88                 break;
  89         case DPAA2_PKT_TYPE_IPV4_SCTP:
  90                 m->packet_type = RTE_PTYPE_L2_ETHER |
  91                         RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP;
  92                 break;
  93         case DPAA2_PKT_TYPE_IPV6_SCTP:
  94                 m->packet_type = RTE_PTYPE_L2_ETHER |
  95                         RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP;
  96                 break;
  97         case DPAA2_PKT_TYPE_IPV4_ICMP:
  98                 m->packet_type = RTE_PTYPE_L2_ETHER |
  99                         RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_ICMP;
 100                 break;
 101         case DPAA2_PKT_TYPE_IPV6_ICMP:
 102                 m->packet_type = RTE_PTYPE_L2_ETHER |
 103                         RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_ICMP;
 104                 break;
 105         default:
 106                 m->packet_type = dpaa2_dev_rx_parse_slow(m, annotation);
 107         }
 108         m->hash.rss = fd->simple.flc_hi;
 109         m->ol_flags |= PKT_RX_RSS_HASH;
 110
 111         if (dpaa2_enable_ts[m->port]) {
 112                 m->timestamp = annotation->word2;
 113                 m->ol_flags |= PKT_RX_TIMESTAMP;
 114                 DPAA2_PMD_DP_DEBUG("pkt timestamp:0x%" PRIx64 "", m->timestamp);
 115         }
 116
 117         DPAA2_PMD_DP_DEBUG("HW frc = 0x%x\t packet type =0x%x "
 118                 "ol_flags =0x%" PRIx64 "",
 119                 frc, m->packet_type, m->ol_flags);
 120 }
 121
 122 static inline uint32_t __rte_hot
 123 dpaa2_dev_rx_parse_slow(struct rte_mbuf *mbuf,
 124                         struct dpaa2_annot_hdr *annotation)
 125 {
 126         uint32_t pkt_type = RTE_PTYPE_UNKNOWN;
 127         uint16_t *vlan_tci;
 128
 129         DPAA2_PMD_DP_DEBUG("(slow parse)annotation(3)=0x%" PRIx64 "\t"
 130                         "(4)=0x%" PRIx64 "\t",
 131                         annotation->word3, annotation->word4);
 132
 133 #if defined(RTE_LIBRTE_IEEE1588)
 134         if (BIT_ISSET_AT_POS(annotation->word1, DPAA2_ETH_FAS_PTP))
 135                 mbuf->ol_flags |= PKT_RX_IEEE1588_PTP;
 136 #endif
 137
 138         if (BIT_ISSET_AT_POS(annotation->word3, L2_VLAN_1_PRESENT)) {
 139                 vlan_tci = rte_pktmbuf_mtod_offset(mbuf, uint16_t *,
 140                         (VLAN_TCI_OFFSET_1(annotation->word5) >> 16));
 141                 mbuf->vlan_tci = rte_be_to_cpu_16(*vlan_tci);
 142                 mbuf->ol_flags |= PKT_RX_VLAN;
 143                 pkt_type |= RTE_PTYPE_L2_ETHER_VLAN;
 144         } else if (BIT_ISSET_AT_POS(annotation->word3, L2_VLAN_N_PRESENT)) {
 145                 vlan_tci = rte_pktmbuf_mtod_offset(mbuf, uint16_t *,
 146                         (VLAN_TCI_OFFSET_1(annotation->word5) >> 16));
 147                 mbuf->vlan_tci = rte_be_to_cpu_16(*vlan_tci);
 148                 mbuf->ol_flags |= PKT_RX_VLAN | PKT_RX_QINQ;
 149                 pkt_type |= RTE_PTYPE_L2_ETHER_QINQ;
 150         }
 151
 152         if (BIT_ISSET_AT_POS(annotation->word3, L2_ARP_PRESENT)) {
 153                 pkt_type |= RTE_PTYPE_L2_ETHER_ARP;
 154                 goto parse_done;
 155         } else if (BIT_ISSET_AT_POS(annotation->word3, L2_ETH_MAC_PRESENT)) {
 156                 pkt_type |= RTE_PTYPE_L2_ETHER;
 157         } else {
 158                 goto parse_done;
 159         }
 160
 161         if (BIT_ISSET_AT_POS(annotation->word4, L3_IPV4_1_PRESENT |
 162                              L3_IPV4_N_PRESENT)) {
 163                 pkt_type |= RTE_PTYPE_L3_IPV4;
 164                 if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_OPT_PRESENT |
 165                         L3_IP_N_OPT_PRESENT))
 166                         pkt_type |= RTE_PTYPE_L3_IPV4_EXT;
 167
 168         } else if (BIT_ISSET_AT_POS(annotation->word4, L3_IPV6_1_PRESENT |
 169                   L3_IPV6_N_PRESENT)) {
 170                 pkt_type |= RTE_PTYPE_L3_IPV6;
 171                 if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_OPT_PRESENT |
 172                     L3_IP_N_OPT_PRESENT))
 173                         pkt_type |= RTE_PTYPE_L3_IPV6_EXT;
 174         } else {
 175                 goto parse_done;
 176         }
 177
 178         if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L3CE))
 179                 mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD;
 180         else if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE))
 181                 mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD;
 182
 183         if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_FIRST_FRAGMENT |
 184             L3_IP_1_MORE_FRAGMENT |
 185             L3_IP_N_FIRST_FRAGMENT |
 186             L3_IP_N_MORE_FRAGMENT)) {
 187                 pkt_type |= RTE_PTYPE_L4_FRAG;
 188                 goto parse_done;
 189         } else {
 190                 pkt_type |= RTE_PTYPE_L4_NONFRAG;
 191         }
 192
 193         if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_UDP_PRESENT))
 194                 pkt_type |= RTE_PTYPE_L4_UDP;
 195
 196         else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_TCP_PRESENT))
 197                 pkt_type |= RTE_PTYPE_L4_TCP;
 198
 199         else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_SCTP_PRESENT))
 200                 pkt_type |= RTE_PTYPE_L4_SCTP;
 201
 202         else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_ICMP_PRESENT))
 203                 pkt_type |= RTE_PTYPE_L4_ICMP;
 204
 205         else if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_UNKNOWN_PROTOCOL))
 206                 pkt_type |= RTE_PTYPE_UNKNOWN;
 207
 208 parse_done:
 209         return pkt_type;
 210 }
 211
 212 static inline uint32_t __rte_hot
 213 dpaa2_dev_rx_parse(struct rte_mbuf *mbuf, void *hw_annot_addr)
 214 {
 215         struct dpaa2_annot_hdr *annotation =
 216                         (struct dpaa2_annot_hdr *)hw_annot_addr;
 217
 218         DPAA2_PMD_DP_DEBUG("(fast parse) Annotation = 0x%" PRIx64 "\t",
 219                            annotation->word4);
 220
 221         if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L3CE))
 222                 mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD;
 223         else if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE))
 224                 mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD;
 225
 226         mbuf->ol_flags |= PKT_RX_TIMESTAMP;
 227         mbuf->timestamp = annotation->word2;
 228         DPAA2_PMD_DP_DEBUG("pkt timestamp: 0x%" PRIx64 "", mbuf->timestamp);
 229
 230         /* Check detailed parsing requirement */
 231         if (annotation->word3 & 0x7FFFFC3FFFF)
 232                 return dpaa2_dev_rx_parse_slow(mbuf, annotation);
 233
 234         /* Return some common types from parse processing */
 235         switch (annotation->word4) {
 236         case DPAA2_L3_IPv4:
 237                 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4;
 238         case DPAA2_L3_IPv6:
 239                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6;
 240         case DPAA2_L3_IPv4_TCP:
 241                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
 242                                 RTE_PTYPE_L4_TCP;
 243         case DPAA2_L3_IPv4_UDP:
 244                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
 245                                 RTE_PTYPE_L4_UDP;
 246         case DPAA2_L3_IPv6_TCP:
 247                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
 248                                 RTE_PTYPE_L4_TCP;
 249         case DPAA2_L3_IPv6_UDP:
 250                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
 251                                 RTE_PTYPE_L4_UDP;
 252         default:
 253                 break;
 254         }
 255
 256         return dpaa2_dev_rx_parse_slow(mbuf, annotation);
 257 }
 258
 259 static inline struct rte_mbuf *__rte_hot
 260 eth_sg_fd_to_mbuf(const struct qbman_fd *fd,
 261                   int port_id)
 262 {
 263         struct qbman_sge *sgt, *sge;
 264         size_t sg_addr, fd_addr;
 265         int i = 0;
 266         void *hw_annot_addr;
 267         struct rte_mbuf *first_seg, *next_seg, *cur_seg, *temp;
 268
 269         fd_addr = (size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
 270         hw_annot_addr = (void *)(fd_addr + DPAA2_FD_PTA_SIZE);
 271
 272         /* Get Scatter gather table address */
 273         sgt = (struct qbman_sge *)(fd_addr + DPAA2_GET_FD_OFFSET(fd));
 274
 275         sge = &sgt[i++];
 276         sg_addr = (size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FLE_ADDR(sge));
 277
 278         /* First Scatter gather entry */
 279         first_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr,
 280                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
 281         /* Prepare all the metadata for first segment */
 282         first_seg->buf_addr = (uint8_t *)sg_addr;
 283         first_seg->ol_flags = 0;
 284         first_seg->data_off = DPAA2_GET_FLE_OFFSET(sge);
 285         first_seg->data_len = sge->length  & 0x1FFFF;
 286         first_seg->pkt_len = DPAA2_GET_FD_LEN(fd);
 287         first_seg->nb_segs = 1;
 288         first_seg->next = NULL;
 289         first_seg->port = port_id;
 290         if (dpaa2_svr_family == SVR_LX2160A)
 291                 dpaa2_dev_rx_parse_new(first_seg, fd, hw_annot_addr);
 292         else
 293                 first_seg->packet_type =
 294                         dpaa2_dev_rx_parse(first_seg, hw_annot_addr);
 295
 296         rte_mbuf_refcnt_set(first_seg, 1);
 297         cur_seg = first_seg;
 298         while (!DPAA2_SG_IS_FINAL(sge)) {
 299                 sge = &sgt[i++];
 300                 sg_addr = (size_t)DPAA2_IOVA_TO_VADDR(
 301                                 DPAA2_GET_FLE_ADDR(sge));
 302                 next_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr,
 303                         rte_dpaa2_bpid_info[DPAA2_GET_FLE_BPID(sge)].meta_data_size);
 304                 next_seg->buf_addr  = (uint8_t *)sg_addr;
 305                 next_seg->data_off  = DPAA2_GET_FLE_OFFSET(sge);
 306                 next_seg->data_len  = sge->length  & 0x1FFFF;
 307                 first_seg->nb_segs += 1;
 308                 rte_mbuf_refcnt_set(next_seg, 1);
 309                 cur_seg->next = next_seg;
 310                 next_seg->next = NULL;
 311                 cur_seg = next_seg;
 312         }
 313         temp = DPAA2_INLINE_MBUF_FROM_BUF(fd_addr,
 314                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
 315         rte_mbuf_refcnt_set(temp, 1);
 316         rte_pktmbuf_free_seg(temp);
 317
 318         return (void *)first_seg;
 319 }
 320
 321 static inline struct rte_mbuf *__rte_hot
 322 eth_fd_to_mbuf(const struct qbman_fd *fd,
 323                int port_id)
 324 {
 325         void *v_addr = DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
 326         void *hw_annot_addr = (void *)((size_t)v_addr + DPAA2_FD_PTA_SIZE);
 327         struct rte_mbuf *mbuf = DPAA2_INLINE_MBUF_FROM_BUF(v_addr,
 328                      rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
 329
 330         /* need to repopulated some of the fields,
 331          * as they may have changed in last transmission
 332          */
 333         mbuf->nb_segs = 1;
 334         mbuf->ol_flags = 0;
 335         mbuf->data_off = DPAA2_GET_FD_OFFSET(fd);
 336         mbuf->data_len = DPAA2_GET_FD_LEN(fd);
 337         mbuf->pkt_len = mbuf->data_len;
 338         mbuf->port = port_id;
 339         mbuf->next = NULL;
 340         rte_mbuf_refcnt_set(mbuf, 1);
 341
 342         /* Parse the packet */
 343         /* parse results for LX2 are there in FRC field of FD.
 344          * For other DPAA2 platforms , parse results are after
 345          * the private - sw annotation area
 346          */
 347
 348         if (dpaa2_svr_family == SVR_LX2160A)
 349                 dpaa2_dev_rx_parse_new(mbuf, fd, hw_annot_addr);
 350         else
 351                 mbuf->packet_type = dpaa2_dev_rx_parse(mbuf, hw_annot_addr);
 352
 353         DPAA2_PMD_DP_DEBUG("to mbuf - mbuf =%p, mbuf->buf_addr =%p, off = %d,"
 354                 "fd_off=%d fd =%" PRIx64 ", meta = %d  bpid =%d, len=%d\n",
 355                 mbuf, mbuf->buf_addr, mbuf->data_off,
 356                 DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ADDR(fd),
 357                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size,
 358                 DPAA2_GET_FD_BPID(fd), DPAA2_GET_FD_LEN(fd));
 359
 360         return mbuf;
 361 }
 362
 363 static int __rte_noinline __rte_hot
 364 eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf,
 365                   struct qbman_fd *fd, uint16_t bpid)
 366 {
 367         struct rte_mbuf *cur_seg = mbuf, *prev_seg, *mi, *temp;
 368         struct qbman_sge *sgt, *sge = NULL;
 369         int i;
 370
 371         temp = rte_pktmbuf_alloc(mbuf->pool);
 372         if (temp == NULL) {
 373                 DPAA2_PMD_DP_DEBUG("No memory to allocate S/G table\n");
 374                 return -ENOMEM;
 375         }
 376
 377         DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(temp));
 378         DPAA2_SET_FD_LEN(fd, mbuf->pkt_len);
 379         DPAA2_SET_ONLY_FD_BPID(fd, bpid);
 380         DPAA2_SET_FD_OFFSET(fd, temp->data_off);
 381         DPAA2_FD_SET_FORMAT(fd, qbman_fd_sg);
 382         DPAA2_RESET_FD_FRC(fd);
 383         DPAA2_RESET_FD_CTRL(fd);
 384         /*Set Scatter gather table and Scatter gather entries*/
 385         sgt = (struct qbman_sge *)(
 386                         (size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd))
 387                         + DPAA2_GET_FD_OFFSET(fd));
 388
 389         for (i = 0; i < mbuf->nb_segs; i++) {
 390                 sge = &sgt[i];
 391                 /*Resetting the buffer pool id and offset field*/
 392                 sge->fin_bpid_offset = 0;
 393                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(cur_seg));
 394                 DPAA2_SET_FLE_OFFSET(sge, cur_seg->data_off);
 395                 sge->length = cur_seg->data_len;
 396                 if (RTE_MBUF_DIRECT(cur_seg)) {
 397                         if (rte_mbuf_refcnt_read(cur_seg) > 1) {
 398                                 /* If refcnt > 1, invalid bpid is set to ensure
 399                                  * buffer is not freed by HW
 400                                  */
 401                                 DPAA2_SET_FLE_IVP(sge);
 402                                 rte_mbuf_refcnt_update(cur_seg, -1);
 403                         } else
 404                                 DPAA2_SET_FLE_BPID(sge,
 405                                                 mempool_to_bpid(cur_seg->pool));
 406                         cur_seg = cur_seg->next;
 407                 } else {
 408                         /* Get owner MBUF from indirect buffer */
 409                         mi = rte_mbuf_from_indirect(cur_seg);
 410                         if (rte_mbuf_refcnt_read(mi) > 1) {
 411                                 /* If refcnt > 1, invalid bpid is set to ensure
 412                                  * owner buffer is not freed by HW
 413                                  */
 414                                 DPAA2_SET_FLE_IVP(sge);
 415                         } else {
 416                                 DPAA2_SET_FLE_BPID(sge,
 417                                                    mempool_to_bpid(mi->pool));
 418                                 rte_mbuf_refcnt_update(mi, 1);
 419                         }
 420                         prev_seg = cur_seg;
 421                         cur_seg = cur_seg->next;
 422                         prev_seg->next = NULL;
 423                         rte_pktmbuf_free(prev_seg);
 424                 }
 425         }
 426         DPAA2_SG_SET_FINAL(sge, true);
 427         return 0;
 428 }
 429
 430 static void
 431 eth_mbuf_to_fd(struct rte_mbuf *mbuf,
 432                struct qbman_fd *fd, uint16_t bpid) __rte_unused;
 433
 434 static void __rte_noinline __rte_hot
 435 eth_mbuf_to_fd(struct rte_mbuf *mbuf,
 436                struct qbman_fd *fd, uint16_t bpid)
 437 {
 438         DPAA2_MBUF_TO_CONTIG_FD(mbuf, fd, bpid);
 439
 440         DPAA2_PMD_DP_DEBUG("mbuf =%p, mbuf->buf_addr =%p, off = %d,"
 441                 "fd_off=%d fd =%" PRIx64 ", meta = %d  bpid =%d, len=%d\n",
 442                 mbuf, mbuf->buf_addr, mbuf->data_off,
 443                 DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ADDR(fd),
 444                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size,
 445                 DPAA2_GET_FD_BPID(fd), DPAA2_GET_FD_LEN(fd));
 446         if (RTE_MBUF_DIRECT(mbuf)) {
 447                 if (rte_mbuf_refcnt_read(mbuf) > 1) {
 448                         DPAA2_SET_FD_IVP(fd);
 449                         rte_mbuf_refcnt_update(mbuf, -1);
 450                 }
 451         } else {
 452                 struct rte_mbuf *mi;
 453
 454                 mi = rte_mbuf_from_indirect(mbuf);
 455                 if (rte_mbuf_refcnt_read(mi) > 1)
 456                         DPAA2_SET_FD_IVP(fd);
 457                 else
 458                         rte_mbuf_refcnt_update(mi, 1);
 459                 rte_pktmbuf_free(mbuf);
 460         }
 461 }
 462
 463 static inline int __rte_hot
 464 eth_copy_mbuf_to_fd(struct rte_mbuf *mbuf,
 465                     struct qbman_fd *fd, uint16_t bpid)
 466 {
 467         struct rte_mbuf *m;
 468         void *mb = NULL;
 469
 470         if (rte_dpaa2_mbuf_alloc_bulk(
 471                 rte_dpaa2_bpid_info[bpid].bp_list->mp, &mb, 1)) {
 472                 DPAA2_PMD_DP_DEBUG("Unable to allocated DPAA2 buffer\n");
 473                 return -1;
 474         }
 475         m = (struct rte_mbuf *)mb;
 476         memcpy((char *)m->buf_addr + mbuf->data_off,
 477                (void *)((char *)mbuf->buf_addr + mbuf->data_off),
 478                 mbuf->pkt_len);
 479
 480         /* Copy required fields */
 481         m->data_off = mbuf->data_off;
 482         m->ol_flags = mbuf->ol_flags;
 483         m->packet_type = mbuf->packet_type;
 484         m->tx_offload = mbuf->tx_offload;
 485
 486         DPAA2_MBUF_TO_CONTIG_FD(m, fd, bpid);
 487
 488         DPAA2_PMD_DP_DEBUG(
 489                 "mbuf: %p, BMAN buf addr: %p, fdaddr: %" PRIx64 ", bpid: %d,"
 490                 " meta: %d, off: %d, len: %d\n",
 491                 (void *)mbuf,
 492                 mbuf->buf_addr,
 493                 DPAA2_GET_FD_ADDR(fd),
 494                 DPAA2_GET_FD_BPID(fd),
 495                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size,
 496                 DPAA2_GET_FD_OFFSET(fd),
 497                 DPAA2_GET_FD_LEN(fd));
 498
 499 return 0;
 500 }
 501
 502 /* This function assumes that caller will be keep the same value for nb_pkts
 503  * across calls per queue, if that is not the case, better use non-prefetch
 504  * version of rx call.
 505  * It will return the packets as requested in previous call without honoring
 506  * the current nb_pkts or bufs space.
 507  */
 508 uint16_t
 509 dpaa2_dev_prefetch_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 510 {
 511         /* Function receive frames for a given device and VQ*/
 512         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
 513         struct qbman_result *dq_storage, *dq_storage1 = NULL;
 514         uint32_t fqid = dpaa2_q->fqid;
 515         int ret, num_rx = 0, pull_size;
 516         uint8_t pending, status;
 517         struct qbman_swp *swp;
 518         const struct qbman_fd *fd;
 519         struct qbman_pull_desc pulldesc;
 520         struct queue_storage_info_t *q_storage = dpaa2_q->q_storage;
 521         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
 522 #if defined(RTE_LIBRTE_IEEE1588)
 523         struct dpaa2_dev_priv *priv = eth_data->dev_private;
 524 #endif
 525
 526         if (unlikely(!DPAA2_PER_LCORE_ETHRX_DPIO)) {
 527                 ret = dpaa2_affine_qbman_ethrx_swp();
 528                 if (ret) {
 529                         DPAA2_PMD_ERR("Failure in affining portal");
 530                         return 0;
 531                 }
 532         }
 533
 534         if (unlikely(!rte_dpaa2_bpid_info &&
 535                      rte_eal_process_type() == RTE_PROC_SECONDARY))
 536                 rte_dpaa2_bpid_info = dpaa2_q->bp_array;
 537
 538         swp = DPAA2_PER_LCORE_ETHRX_PORTAL;
 539         pull_size = (nb_pkts > dpaa2_dqrr_size) ? dpaa2_dqrr_size : nb_pkts;
 540         if (unlikely(!q_storage->active_dqs)) {
 541                 q_storage->toggle = 0;
 542                 dq_storage = q_storage->dq_storage[q_storage->toggle];
 543                 q_storage->last_num_pkts = pull_size;
 544                 qbman_pull_desc_clear(&pulldesc);
 545                 qbman_pull_desc_set_numframes(&pulldesc,
 546                                               q_storage->last_num_pkts);
 547                 qbman_pull_desc_set_fq(&pulldesc, fqid);
 548                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
 549                         (uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
 550                 if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) {
 551                         while (!qbman_check_command_complete(
 552                                get_swp_active_dqs(
 553                                DPAA2_PER_LCORE_ETHRX_DPIO->index)))
 554                                 ;
 555                         clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index);
 556                 }
 557                 while (1) {
 558                         if (qbman_swp_pull(swp, &pulldesc)) {
 559                                 DPAA2_PMD_DP_DEBUG("VDQ command is not issued."
 560                                                   " QBMAN is busy (1)\n");
 561                                 /* Portal was busy, try again */
 562                                 continue;
 563                         }
 564                         break;
 565                 }
 566                 q_storage->active_dqs = dq_storage;
 567                 q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index;
 568                 set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index,
 569                                    dq_storage);
 570         }
 571
 572         dq_storage = q_storage->active_dqs;
 573         rte_prefetch0((void *)(size_t)(dq_storage));
 574         rte_prefetch0((void *)(size_t)(dq_storage + 1));
 575
 576         /* Prepare next pull descriptor. This will give space for the
 577          * prefething done on DQRR entries
 578          */
 579         q_storage->toggle ^= 1;
 580         dq_storage1 = q_storage->dq_storage[q_storage->toggle];
 581         qbman_pull_desc_clear(&pulldesc);
 582         qbman_pull_desc_set_numframes(&pulldesc, pull_size);
 583         qbman_pull_desc_set_fq(&pulldesc, fqid);
 584         qbman_pull_desc_set_storage(&pulldesc, dq_storage1,
 585                 (uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage1)), 1);
 586
 587         /* Check if the previous issued command is completed.
 588          * Also seems like the SWP is shared between the Ethernet Driver
 589          * and the SEC driver.
 590          */
 591         while (!qbman_check_command_complete(dq_storage))
 592                 ;
 593         if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id))
 594                 clear_swp_active_dqs(q_storage->active_dpio_id);
 595
 596         pending = 1;
 597
 598         do {
 599                 /* Loop until the dq_storage is updated with
 600                  * new token by QBMAN
 601                  */
 602                 while (!qbman_check_new_result(dq_storage))
 603                         ;
 604                 rte_prefetch0((void *)((size_t)(dq_storage + 2)));
 605                 /* Check whether Last Pull command is Expired and
 606                  * setting Condition for Loop termination
 607                  */
 608                 if (qbman_result_DQ_is_pull_complete(dq_storage)) {
 609                         pending = 0;
 610                         /* Check for valid frame. */
 611                         status = qbman_result_DQ_flags(dq_storage);
 612                         if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0))
 613                                 continue;
 614                 }
 615                 fd = qbman_result_DQ_fd(dq_storage);
 616
 617 #ifndef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
 618                 if (dpaa2_svr_family != SVR_LX2160A) {
 619                         const struct qbman_fd *next_fd =
 620                                 qbman_result_DQ_fd(dq_storage + 1);
 621                         /* Prefetch Annotation address for the parse results */
 622                         rte_prefetch0(DPAA2_IOVA_TO_VADDR((DPAA2_GET_FD_ADDR(
 623                                 next_fd) + DPAA2_FD_PTA_SIZE + 16)));
 624                 }
 625 #endif
 626
 627                 if (unlikely(DPAA2_FD_GET_FORMAT(fd) == qbman_fd_sg))
 628                         bufs[num_rx] = eth_sg_fd_to_mbuf(fd, eth_data->port_id);
 629                 else
 630                         bufs[num_rx] = eth_fd_to_mbuf(fd, eth_data->port_id);
 631 #if defined(RTE_LIBRTE_IEEE1588)
 632                 priv->rx_timestamp = bufs[num_rx]->timestamp;
 633 #endif
 634
 635                 if (eth_data->dev_conf.rxmode.offloads &
 636                                 DEV_RX_OFFLOAD_VLAN_STRIP)
 637                         rte_vlan_strip(bufs[num_rx]);
 638
 639                 dq_storage++;
 640                 num_rx++;
 641         } while (pending);
 642
 643         if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) {
 644                 while (!qbman_check_command_complete(
 645                        get_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)))
 646                         ;
 647                 clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index);
 648         }
 649         /* issue a volatile dequeue command for next pull */
 650         while (1) {
 651                 if (qbman_swp_pull(swp, &pulldesc)) {
 652                         DPAA2_PMD_DP_DEBUG("VDQ command is not issued."
 653                                           "QBMAN is busy (2)\n");
 654                         continue;
 655                 }
 656                 break;
 657         }
 658         q_storage->active_dqs = dq_storage1;
 659         q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index;
 660         set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index, dq_storage1);
 661
 662         dpaa2_q->rx_pkts += num_rx;
 663
 664         return num_rx;
 665 }
 666
 667 void __rte_hot
 668 dpaa2_dev_process_parallel_event(struct qbman_swp *swp,
 669                                  const struct qbman_fd *fd,
 670                                  const struct qbman_result *dq,
 671                                  struct dpaa2_queue *rxq,
 672                                  struct rte_event *ev)
 673 {
 674         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd) +
 675                 DPAA2_FD_PTA_SIZE + 16));
 676
 677         ev->flow_id = rxq->ev.flow_id;
 678         ev->sub_event_type = rxq->ev.sub_event_type;
 679         ev->event_type = RTE_EVENT_TYPE_ETHDEV;
 680         ev->op = RTE_EVENT_OP_NEW;
 681         ev->sched_type = rxq->ev.sched_type;
 682         ev->queue_id = rxq->ev.queue_id;
 683         ev->priority = rxq->ev.priority;
 684
 685         ev->mbuf = eth_fd_to_mbuf(fd, rxq->eth_data->port_id);
 686
 687         qbman_swp_dqrr_consume(swp, dq);
 688 }
 689
 690 void __rte_hot
 691 dpaa2_dev_process_atomic_event(struct qbman_swp *swp __rte_unused,
 692                                const struct qbman_fd *fd,
 693                                const struct qbman_result *dq,
 694                                struct dpaa2_queue *rxq,
 695                                struct rte_event *ev)
 696 {
 697         uint8_t dqrr_index;
 698
 699         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd) +
 700                 DPAA2_FD_PTA_SIZE + 16));
 701
 702         ev->flow_id = rxq->ev.flow_id;
 703         ev->sub_event_type = rxq->ev.sub_event_type;
 704         ev->event_type = RTE_EVENT_TYPE_ETHDEV;
 705         ev->op = RTE_EVENT_OP_NEW;
 706         ev->sched_type = rxq->ev.sched_type;
 707         ev->queue_id = rxq->ev.queue_id;
 708         ev->priority = rxq->ev.priority;
 709
 710         ev->mbuf = eth_fd_to_mbuf(fd, rxq->eth_data->port_id);
 711
 712         dqrr_index = qbman_get_dqrr_idx(dq);
 713         ev->mbuf->seqn = dqrr_index + 1;
 714         DPAA2_PER_LCORE_DQRR_SIZE++;
 715         DPAA2_PER_LCORE_DQRR_HELD |= 1 << dqrr_index;
 716         DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = ev->mbuf;
 717 }
 718
 719 void __rte_hot
 720 dpaa2_dev_process_ordered_event(struct qbman_swp *swp,
 721                                 const struct qbman_fd *fd,
 722                                 const struct qbman_result *dq,
 723                                 struct dpaa2_queue *rxq,
 724                                 struct rte_event *ev)
 725 {
 726         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd) +
 727                 DPAA2_FD_PTA_SIZE + 16));
 728
 729         ev->flow_id = rxq->ev.flow_id;
 730         ev->sub_event_type = rxq->ev.sub_event_type;
 731         ev->event_type = RTE_EVENT_TYPE_ETHDEV;
 732         ev->op = RTE_EVENT_OP_NEW;
 733         ev->sched_type = rxq->ev.sched_type;
 734         ev->queue_id = rxq->ev.queue_id;
 735         ev->priority = rxq->ev.priority;
 736
 737         ev->mbuf = eth_fd_to_mbuf(fd, rxq->eth_data->port_id);
 738
 739         ev->mbuf->seqn = DPAA2_ENQUEUE_FLAG_ORP;
 740         ev->mbuf->seqn |= qbman_result_DQ_odpid(dq) << DPAA2_EQCR_OPRID_SHIFT;
 741         ev->mbuf->seqn |= qbman_result_DQ_seqnum(dq) << DPAA2_EQCR_SEQNUM_SHIFT;
 742
 743         qbman_swp_dqrr_consume(swp, dq);
 744 }
 745
 746 uint16_t
 747 dpaa2_dev_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 748 {
 749         /* Function receive frames for a given device and VQ */
 750         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
 751         struct qbman_result *dq_storage;
 752         uint32_t fqid = dpaa2_q->fqid;
 753         int ret, num_rx = 0, next_pull = nb_pkts, num_pulled;
 754         uint8_t pending, status;
 755         struct qbman_swp *swp;
 756         const struct qbman_fd *fd;
 757         struct qbman_pull_desc pulldesc;
 758         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
 759
 760         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 761                 ret = dpaa2_affine_qbman_swp();
 762                 if (ret) {
 763                         DPAA2_PMD_ERR(
 764                                 "Failed to allocate IO portal, tid: %d\n",
 765                                 rte_gettid());
 766                         return 0;
 767                 }
 768         }
 769         swp = DPAA2_PER_LCORE_PORTAL;
 770
 771         do {
 772                 dq_storage = dpaa2_q->q_storage->dq_storage[0];
 773                 qbman_pull_desc_clear(&pulldesc);
 774                 qbman_pull_desc_set_fq(&pulldesc, fqid);
 775                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
 776                                 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
 777
 778                 if (next_pull > dpaa2_dqrr_size) {
 779                         qbman_pull_desc_set_numframes(&pulldesc,
 780                                 dpaa2_dqrr_size);
 781                         next_pull -= dpaa2_dqrr_size;
 782                 } else {
 783                         qbman_pull_desc_set_numframes(&pulldesc, next_pull);
 784                         next_pull = 0;
 785                 }
 786
 787                 while (1) {
 788                         if (qbman_swp_pull(swp, &pulldesc)) {
 789                                 DPAA2_PMD_DP_DEBUG(
 790                                         "VDQ command is not issued.QBMAN is busy\n");
 791                                 /* Portal was busy, try again */
 792                                 continue;
 793                         }
 794                         break;
 795                 }
 796
 797                 rte_prefetch0((void *)((size_t)(dq_storage + 1)));
 798                 /* Check if the previous issued command is completed. */
 799                 while (!qbman_check_command_complete(dq_storage))
 800                         ;
 801
 802                 num_pulled = 0;
 803                 pending = 1;
 804                 do {
 805                         /* Loop until the dq_storage is updated with
 806                          * new token by QBMAN
 807                          */
 808                         while (!qbman_check_new_result(dq_storage))
 809                                 ;
 810                         rte_prefetch0((void *)((size_t)(dq_storage + 2)));
 811                         /* Check whether Last Pull command is Expired and
 812                          * setting Condition for Loop termination
 813                          */
 814                         if (qbman_result_DQ_is_pull_complete(dq_storage)) {
 815                                 pending = 0;
 816                                 /* Check for valid frame. */
 817                                 status = qbman_result_DQ_flags(dq_storage);
 818                                 if (unlikely((status &
 819                                         QBMAN_DQ_STAT_VALIDFRAME) == 0))
 820                                         continue;
 821                         }
 822                         fd = qbman_result_DQ_fd(dq_storage);
 823
 824 #ifndef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
 825                         if (dpaa2_svr_family != SVR_LX2160A) {
 826                                 const struct qbman_fd *next_fd =
 827                                         qbman_result_DQ_fd(dq_storage + 1);
 828
 829                                 /* Prefetch Annotation address for the parse
 830                                  * results.
 831                                  */
 832                                 rte_prefetch0((DPAA2_IOVA_TO_VADDR(
 833                                         DPAA2_GET_FD_ADDR(next_fd) +
 834                                         DPAA2_FD_PTA_SIZE + 16)));
 835                         }
 836 #endif
 837
 838                         if (unlikely(DPAA2_FD_GET_FORMAT(fd) == qbman_fd_sg))
 839                                 bufs[num_rx] = eth_sg_fd_to_mbuf(fd,
 840                                                         eth_data->port_id);
 841                         else
 842                                 bufs[num_rx] = eth_fd_to_mbuf(fd,
 843                                                         eth_data->port_id);
 844
 845                 if (eth_data->dev_conf.rxmode.offloads &
 846                                 DEV_RX_OFFLOAD_VLAN_STRIP) {
 847                         rte_vlan_strip(bufs[num_rx]);
 848                 }
 849
 850                         dq_storage++;
 851                         num_rx++;
 852                         num_pulled++;
 853                 } while (pending);
 854         /* Last VDQ provided all packets and more packets are requested */
 855         } while (next_pull && num_pulled == dpaa2_dqrr_size);
 856
 857         dpaa2_q->rx_pkts += num_rx;
 858
 859         return num_rx;
 860 }
 861
 862 uint16_t dpaa2_dev_tx_conf(void *queue)
 863 {
 864         /* Function receive frames for a given device and VQ */
 865         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
 866         struct qbman_result *dq_storage;
 867         uint32_t fqid = dpaa2_q->fqid;
 868         int ret, num_tx_conf = 0, num_pulled;
 869         uint8_t pending, status;
 870         struct qbman_swp *swp;
 871         const struct qbman_fd *fd, *next_fd;
 872         struct qbman_pull_desc pulldesc;
 873         struct qbman_release_desc releasedesc;
 874         uint32_t bpid;
 875         uint64_t buf;
 876 #if defined(RTE_LIBRTE_IEEE1588)
 877         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
 878         struct dpaa2_dev_priv *priv = eth_data->dev_private;
 879         struct dpaa2_annot_hdr *annotation;
 880 #endif
 881
 882         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 883                 ret = dpaa2_affine_qbman_swp();
 884                 if (ret) {
 885                         DPAA2_PMD_ERR(
 886                                 "Failed to allocate IO portal, tid: %d\n",
 887                                 rte_gettid());
 888                         return 0;
 889                 }
 890         }
 891         swp = DPAA2_PER_LCORE_PORTAL;
 892
 893         do {
 894                 dq_storage = dpaa2_q->q_storage->dq_storage[0];
 895                 qbman_pull_desc_clear(&pulldesc);
 896                 qbman_pull_desc_set_fq(&pulldesc, fqid);
 897                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
 898                                 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
 899
 900                 qbman_pull_desc_set_numframes(&pulldesc, dpaa2_dqrr_size);
 901
 902                 while (1) {
 903                         if (qbman_swp_pull(swp, &pulldesc)) {
 904                                 DPAA2_PMD_DP_DEBUG("VDQ command is not issued."
 905                                                    "QBMAN is busy\n");
 906                                 /* Portal was busy, try again */
 907                                 continue;
 908                         }
 909                         break;
 910                 }
 911
 912                 rte_prefetch0((void *)((size_t)(dq_storage + 1)));
 913                 /* Check if the previous issued command is completed. */
 914                 while (!qbman_check_command_complete(dq_storage))
 915                         ;
 916
 917                 num_pulled = 0;
 918                 pending = 1;
 919                 do {
 920                         /* Loop until the dq_storage is updated with
 921                          * new token by QBMAN
 922                          */
 923                         while (!qbman_check_new_result(dq_storage))
 924                                 ;
 925                         rte_prefetch0((void *)((size_t)(dq_storage + 2)));
 926                         /* Check whether Last Pull command is Expired and
 927                          * setting Condition for Loop termination
 928                          */
 929                         if (qbman_result_DQ_is_pull_complete(dq_storage)) {
 930                                 pending = 0;
 931                                 /* Check for valid frame. */
 932                                 status = qbman_result_DQ_flags(dq_storage);
 933                                 if (unlikely((status &
 934                                         QBMAN_DQ_STAT_VALIDFRAME) == 0))
 935                                         continue;
 936                         }
 937                         fd = qbman_result_DQ_fd(dq_storage);
 938
 939                         next_fd = qbman_result_DQ_fd(dq_storage + 1);
 940                         /* Prefetch Annotation address for the parse results */
 941                         rte_prefetch0((void *)(size_t)
 942                                 (DPAA2_GET_FD_ADDR(next_fd) +
 943                                  DPAA2_FD_PTA_SIZE + 16));
 944
 945                         bpid = DPAA2_GET_FD_BPID(fd);
 946
 947                         /* Create a release descriptor required for releasing
 948                          * buffers into QBMAN
 949                          */
 950                         qbman_release_desc_clear(&releasedesc);
 951                         qbman_release_desc_set_bpid(&releasedesc, bpid);
 952
 953                         buf = DPAA2_GET_FD_ADDR(fd);
 954                         /* feed them to bman */
 955                         do {
 956                                 ret = qbman_swp_release(swp, &releasedesc,
 957                                                         &buf, 1);
 958                         } while (ret == -EBUSY);
 959
 960                         dq_storage++;
 961                         num_tx_conf++;
 962                         num_pulled++;
 963 #if defined(RTE_LIBRTE_IEEE1588)
 964                         annotation = (struct dpaa2_annot_hdr *)((size_t)
 965                                 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) +
 966                                 DPAA2_FD_PTA_SIZE);
 967                         priv->tx_timestamp = annotation->word2;
 968 #endif
 969                 } while (pending);
 970
 971         /* Last VDQ provided all packets and more packets are requested */
 972         } while (num_pulled == dpaa2_dqrr_size);
 973
 974         dpaa2_q->rx_pkts += num_tx_conf;
 975
 976         return num_tx_conf;
 977 }
 978
 979 /* Configure the egress frame annotation for timestamp update */
 980 static void enable_tx_tstamp(struct qbman_fd *fd)
 981 {
 982         struct dpaa2_faead *fd_faead;
 983
 984         /* Set frame annotation status field as valid */
 985         (fd)->simple.frc |= DPAA2_FD_FRC_FASV;
 986
 987         /* Set frame annotation egress action descriptor as valid */
 988         (fd)->simple.frc |= DPAA2_FD_FRC_FAEADV;
 989
 990         /* Set Annotation Length as 128B */
 991         (fd)->simple.ctrl |= DPAA2_FD_CTRL_ASAL;
 992
 993         /* enable update of confirmation frame annotation */
 994         fd_faead = (struct dpaa2_faead *)((size_t)
 995                         DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) +
 996                         DPAA2_FD_PTA_SIZE + DPAA2_FD_HW_ANNOT_FAEAD_OFFSET);
 997         fd_faead->ctrl = DPAA2_ANNOT_FAEAD_A2V | DPAA2_ANNOT_FAEAD_UPDV |
 998                                 DPAA2_ANNOT_FAEAD_UPD;
 999 }
1000
1001 /*
1002  * Callback to handle sending packets through WRIOP based interface
1003  */
1004 uint16_t
1005 dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
1006 {
1007         /* Function to transmit the frames to given device and VQ*/
1008         uint32_t loop, retry_count;
1009         int32_t ret;
1010         struct qbman_fd fd_arr[MAX_TX_RING_SLOTS];
1011         struct rte_mbuf *mi;
1012         uint32_t frames_to_send;
1013         struct rte_mempool *mp;
1014         struct qbman_eq_desc eqdesc;
1015         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
1016         struct qbman_swp *swp;
1017         uint16_t num_tx = 0;
1018         uint16_t bpid;
1019         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
1020         struct dpaa2_dev_priv *priv = eth_data->dev_private;
1021         uint32_t flags[MAX_TX_RING_SLOTS] = {0};
1022
1023         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
1024                 ret = dpaa2_affine_qbman_swp();
1025                 if (ret) {
1026                         DPAA2_PMD_ERR(
1027                                 "Failed to allocate IO portal, tid: %d\n",
1028                                 rte_gettid());
1029                         return 0;
1030                 }
1031         }
1032         swp = DPAA2_PER_LCORE_PORTAL;
1033
1034         DPAA2_PMD_DP_DEBUG("===> eth_data =%p, fqid =%d\n",
1035                         eth_data, dpaa2_q->fqid);
1036
1037 #ifdef RTE_LIBRTE_IEEE1588
1038         /* IEEE1588 driver need pointer to tx confirmation queue
1039          * corresponding to last packet transmitted for reading
1040          * the timestamp
1041          */
1042         priv->next_tx_conf_queue = dpaa2_q->tx_conf_queue;
1043         dpaa2_dev_tx_conf(dpaa2_q->tx_conf_queue);
1044 #endif
1045
1046         /*Prepare enqueue descriptor*/
1047         qbman_eq_desc_clear(&eqdesc);
1048         qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ);
1049         qbman_eq_desc_set_fq(&eqdesc, dpaa2_q->fqid);
1050
1051         /*Clear the unused FD fields before sending*/
1052         while (nb_pkts) {
1053                 /*Check if the queue is congested*/
1054                 retry_count = 0;
1055                 while (qbman_result_SCN_state(dpaa2_q->cscn)) {
1056                         retry_count++;
1057                         /* Retry for some time before giving up */
1058                         if (retry_count > CONG_RETRY_COUNT)
1059                                 goto skip_tx;
1060                 }
1061
1062                 frames_to_send = (nb_pkts > dpaa2_eqcr_size) ?
1063                         dpaa2_eqcr_size : nb_pkts;
1064
1065                 for (loop = 0; loop < frames_to_send; loop++) {
1066                         if ((*bufs)->seqn) {
1067                                 uint8_t dqrr_index = (*bufs)->seqn - 1;
1068
1069                                 flags[loop] = QBMAN_ENQUEUE_FLAG_DCA |
1070                                                 dqrr_index;
1071                                 DPAA2_PER_LCORE_DQRR_SIZE--;
1072                                 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dqrr_index);
1073                                 (*bufs)->seqn = DPAA2_INVALID_MBUF_SEQN;
1074                         }
1075
1076                         if (likely(RTE_MBUF_DIRECT(*bufs))) {
1077                                 mp = (*bufs)->pool;
1078                                 /* Check the basic scenario and set
1079                                  * the FD appropriately here itself.
1080                                  */
1081                                 if (likely(mp && mp->ops_index ==
1082                                     priv->bp_list->dpaa2_ops_index &&
1083                                     (*bufs)->nb_segs == 1 &&
1084                                     rte_mbuf_refcnt_read((*bufs)) == 1)) {
1085                                         if (unlikely(((*bufs)->ol_flags
1086                                                 & PKT_TX_VLAN_PKT) ||
1087                                                 (eth_data->dev_conf.txmode.offloads
1088                                                 & DEV_TX_OFFLOAD_VLAN_INSERT))) {
1089                                                 ret = rte_vlan_insert(bufs);
1090                                                 if (ret)
1091                                                         goto send_n_return;
1092                                         }
1093                                         DPAA2_MBUF_TO_CONTIG_FD((*bufs),
1094                                         &fd_arr[loop], mempool_to_bpid(mp));
1095                                         bufs++;
1096 #ifdef RTE_LIBRTE_IEEE1588
1097                                         enable_tx_tstamp(&fd_arr[loop]);
1098 #endif
1099                                         continue;
1100                                 }
1101                         } else {
1102                                 mi = rte_mbuf_from_indirect(*bufs);
1103                                 mp = mi->pool;
1104                         }
1105                         /* Not a hw_pkt pool allocated frame */
1106                         if (unlikely(!mp || !priv->bp_list)) {
1107                                 DPAA2_PMD_ERR("Err: No buffer pool attached");
1108                                 goto send_n_return;
1109                         }
1110
1111                         if (unlikely(((*bufs)->ol_flags & PKT_TX_VLAN_PKT) ||
1112                                 (eth_data->dev_conf.txmode.offloads
1113                                 & DEV_TX_OFFLOAD_VLAN_INSERT))) {
1114                                 int ret = rte_vlan_insert(bufs);
1115                                 if (ret)
1116                                         goto send_n_return;
1117                         }
1118                         if (mp->ops_index != priv->bp_list->dpaa2_ops_index) {
1119                                 DPAA2_PMD_WARN("Non DPAA2 buffer pool");
1120                                 /* alloc should be from the default buffer pool
1121                                  * attached to this interface
1122                                  */
1123                                 bpid = priv->bp_list->buf_pool.bpid;
1124
1125                                 if (unlikely((*bufs)->nb_segs > 1)) {
1126                                         DPAA2_PMD_ERR("S/G support not added"
1127                                                 " for non hw offload buffer");
1128                                         goto send_n_return;
1129                                 }
1130                                 if (eth_copy_mbuf_to_fd(*bufs,
1131                                                         &fd_arr[loop], bpid)) {
1132                                         goto send_n_return;
1133                                 }
1134                                 /* free the original packet */
1135                                 rte_pktmbuf_free(*bufs);
1136                         } else {
1137                                 bpid = mempool_to_bpid(mp);
1138                                 if (unlikely((*bufs)->nb_segs > 1)) {
1139                                         if (eth_mbuf_to_sg_fd(*bufs,
1140                                                         &fd_arr[loop], bpid))
1141                                                 goto send_n_return;
1142                                 } else {
1143                                         eth_mbuf_to_fd(*bufs,
1144                                                        &fd_arr[loop], bpid);
1145                                 }
1146                         }
1147 #ifdef RTE_LIBRTE_IEEE1588
1148                         enable_tx_tstamp(&fd_arr[loop]);
1149 #endif
1150                         bufs++;
1151                 }
1152
1153                 loop = 0;
1154                 retry_count = 0;
1155                 while (loop < frames_to_send) {
1156                         ret = qbman_swp_enqueue_multiple(swp, &eqdesc,
1157                                         &fd_arr[loop], &flags[loop],
1158                                         frames_to_send - loop);
1159                         if (unlikely(ret < 0)) {
1160                                 retry_count++;
1161                                 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) {
1162                                         num_tx += loop;
1163                                         nb_pkts -= loop;
1164                                         goto send_n_return;
1165                                 }
1166                         } else {
1167                                 loop += ret;
1168                                 retry_count = 0;
1169                         }
1170                 }
1171
1172                 num_tx += loop;
1173                 nb_pkts -= loop;
1174         }
1175         dpaa2_q->tx_pkts += num_tx;
1176         return num_tx;
1177
1178 send_n_return:
1179         /* send any already prepared fd */
1180         if (loop) {
1181                 unsigned int i = 0;
1182
1183                 retry_count = 0;
1184                 while (i < loop) {
1185                         ret = qbman_swp_enqueue_multiple(swp, &eqdesc,
1186                                                          &fd_arr[i],
1187                                                          &flags[i],
1188                                                          loop - i);
1189                         if (unlikely(ret < 0)) {
1190                                 retry_count++;
1191                                 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT)
1192                                         break;
1193                         } else {
1194                                 i += ret;
1195                                 retry_count = 0;
1196                         }
1197                 }
1198                 num_tx += i;
1199         }
1200 skip_tx:
1201         dpaa2_q->tx_pkts += num_tx;
1202         return num_tx;
1203 }
1204
1205 void
1206 dpaa2_dev_free_eqresp_buf(uint16_t eqresp_ci)
1207 {
1208         struct dpaa2_dpio_dev *dpio_dev = DPAA2_PER_LCORE_DPIO;
1209         struct qbman_fd *fd;
1210         struct rte_mbuf *m;
1211
1212         fd = qbman_result_eqresp_fd(&dpio_dev->eqresp[eqresp_ci]);
1213
1214         /* Setting port id does not matter as we are to free the mbuf */
1215         m = eth_fd_to_mbuf(fd, 0);
1216         rte_pktmbuf_free(m);
1217 }
1218
1219 static void
1220 dpaa2_set_enqueue_descriptor(struct dpaa2_queue *dpaa2_q,
1221                              struct rte_mbuf *m,
1222                              struct qbman_eq_desc *eqdesc)
1223 {
1224         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
1225         struct dpaa2_dev_priv *priv = eth_data->dev_private;
1226         struct dpaa2_dpio_dev *dpio_dev = DPAA2_PER_LCORE_DPIO;
1227         struct eqresp_metadata *eqresp_meta;
1228         uint16_t orpid, seqnum;
1229         uint8_t dq_idx;
1230
1231         qbman_eq_desc_set_fq(eqdesc, dpaa2_q->fqid);
1232
1233         if (m->seqn & DPAA2_ENQUEUE_FLAG_ORP) {
1234                 orpid = (m->seqn & DPAA2_EQCR_OPRID_MASK) >>
1235                         DPAA2_EQCR_OPRID_SHIFT;
1236                 seqnum = (m->seqn & DPAA2_EQCR_SEQNUM_MASK) >>
1237                         DPAA2_EQCR_SEQNUM_SHIFT;
1238
1239                 if (!priv->en_loose_ordered) {
1240                         qbman_eq_desc_set_orp(eqdesc, 1, orpid, seqnum, 0);
1241                         qbman_eq_desc_set_response(eqdesc, (uint64_t)
1242                                 DPAA2_VADDR_TO_IOVA(&dpio_dev->eqresp[
1243                                 dpio_dev->eqresp_pi]), 1);
1244                         qbman_eq_desc_set_token(eqdesc, 1);
1245
1246                         eqresp_meta = &dpio_dev->eqresp_meta[
1247                                 dpio_dev->eqresp_pi];
1248                         eqresp_meta->dpaa2_q = dpaa2_q;
1249                         eqresp_meta->mp = m->pool;
1250
1251                         dpio_dev->eqresp_pi + 1 < MAX_EQ_RESP_ENTRIES ?
1252                                 dpio_dev->eqresp_pi++ :
1253                                 (dpio_dev->eqresp_pi = 0);
1254                 } else {
1255                         qbman_eq_desc_set_orp(eqdesc, 0, orpid, seqnum, 0);
1256                 }
1257         } else {
1258                 dq_idx = m->seqn - 1;
1259                 qbman_eq_desc_set_dca(eqdesc, 1, dq_idx, 0);
1260                 DPAA2_PER_LCORE_DQRR_SIZE--;
1261                 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dq_idx);
1262         }
1263         m->seqn = DPAA2_INVALID_MBUF_SEQN;
1264 }
1265
1266 /* Callback to handle sending ordered packets through WRIOP based interface */
1267 uint16_t
1268 dpaa2_dev_tx_ordered(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
1269 {
1270         /* Function to transmit the frames to given device and VQ*/
1271         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
1272         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
1273         struct dpaa2_dev_priv *priv = eth_data->dev_private;
1274         struct dpaa2_queue *order_sendq = (struct dpaa2_queue *)priv->tx_vq[0];
1275         struct qbman_fd fd_arr[MAX_TX_RING_SLOTS];
1276         struct rte_mbuf *mi;
1277         struct rte_mempool *mp;
1278         struct qbman_eq_desc eqdesc[MAX_TX_RING_SLOTS];
1279         struct qbman_swp *swp;
1280         uint32_t frames_to_send, num_free_eq_desc;
1281         uint32_t loop, retry_count;
1282         int32_t ret;
1283         uint16_t num_tx = 0;
1284         uint16_t bpid;
1285
1286         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
1287                 ret = dpaa2_affine_qbman_swp();
1288                 if (ret) {
1289                         DPAA2_PMD_ERR(
1290                                 "Failed to allocate IO portal, tid: %d\n",
1291                                 rte_gettid());
1292                         return 0;
1293                 }
1294         }
1295         swp = DPAA2_PER_LCORE_PORTAL;
1296
1297         DPAA2_PMD_DP_DEBUG("===> eth_data =%p, fqid =%d\n",
1298                            eth_data, dpaa2_q->fqid);
1299
1300         /* This would also handle normal and atomic queues as any type
1301          * of packet can be enqueued when ordered queues are being used.
1302          */
1303         while (nb_pkts) {
1304                 /*Check if the queue is congested*/
1305                 retry_count = 0;
1306                 while (qbman_result_SCN_state(dpaa2_q->cscn)) {
1307                         retry_count++;
1308                         /* Retry for some time before giving up */
1309                         if (retry_count > CONG_RETRY_COUNT)
1310                                 goto skip_tx;
1311                 }
1312
1313                 frames_to_send = (nb_pkts > dpaa2_eqcr_size) ?
1314                         dpaa2_eqcr_size : nb_pkts;
1315
1316                 if (!priv->en_loose_ordered) {
1317                         if ((*bufs)->seqn & DPAA2_ENQUEUE_FLAG_ORP) {
1318                                 num_free_eq_desc = dpaa2_free_eq_descriptors();
1319                                 if (num_free_eq_desc < frames_to_send)
1320                                         frames_to_send = num_free_eq_desc;
1321                         }
1322                 }
1323
1324                 for (loop = 0; loop < frames_to_send; loop++) {
1325                         /*Prepare enqueue descriptor*/
1326                         qbman_eq_desc_clear(&eqdesc[loop]);
1327
1328                         if ((*bufs)->seqn) {
1329                                 /* Use only queue 0 for Tx in case of atomic/
1330                                  * ordered packets as packets can get unordered
1331                                  * when being tranmitted out from the interface
1332                                  */
1333                                 dpaa2_set_enqueue_descriptor(order_sendq,
1334                                                              (*bufs),
1335                                                              &eqdesc[loop]);
1336                         } else {
1337                                 qbman_eq_desc_set_no_orp(&eqdesc[loop],
1338                                                          DPAA2_EQ_RESP_ERR_FQ);
1339                                 qbman_eq_desc_set_fq(&eqdesc[loop],
1340                                                      dpaa2_q->fqid);
1341                         }
1342
1343                         if (likely(RTE_MBUF_DIRECT(*bufs))) {
1344                                 mp = (*bufs)->pool;
1345                                 /* Check the basic scenario and set
1346                                  * the FD appropriately here itself.
1347                                  */
1348                                 if (likely(mp && mp->ops_index ==
1349                                     priv->bp_list->dpaa2_ops_index &&
1350                                     (*bufs)->nb_segs == 1 &&
1351                                     rte_mbuf_refcnt_read((*bufs)) == 1)) {
1352                                         if (unlikely((*bufs)->ol_flags
1353                                                 & PKT_TX_VLAN_PKT)) {
1354                                           ret = rte_vlan_insert(bufs);
1355                                           if (ret)
1356                                                 goto send_n_return;
1357                                         }
1358                                         DPAA2_MBUF_TO_CONTIG_FD((*bufs),
1359                                                 &fd_arr[loop],
1360                                                 mempool_to_bpid(mp));
1361                                         bufs++;
1362                                         continue;
1363                                 }
1364                         } else {
1365                                 mi = rte_mbuf_from_indirect(*bufs);
1366                                 mp = mi->pool;
1367                         }
1368                         /* Not a hw_pkt pool allocated frame */
1369                         if (unlikely(!mp || !priv->bp_list)) {
1370                                 DPAA2_PMD_ERR("Err: No buffer pool attached");
1371                                 goto send_n_return;
1372                         }
1373
1374                         if (mp->ops_index != priv->bp_list->dpaa2_ops_index) {
1375                                 DPAA2_PMD_WARN("Non DPAA2 buffer pool");
1376                                 /* alloc should be from the default buffer pool
1377                                  * attached to this interface
1378                                  */
1379                                 bpid = priv->bp_list->buf_pool.bpid;
1380
1381                                 if (unlikely((*bufs)->nb_segs > 1)) {
1382                                         DPAA2_PMD_ERR(
1383                                                 "S/G not supp for non hw offload buffer");
1384                                         goto send_n_return;
1385                                 }
1386                                 if (eth_copy_mbuf_to_fd(*bufs,
1387                                                         &fd_arr[loop], bpid)) {
1388                                         goto send_n_return;
1389                                 }
1390                                 /* free the original packet */
1391                                 rte_pktmbuf_free(*bufs);
1392                         } else {
1393                                 bpid = mempool_to_bpid(mp);
1394                                 if (unlikely((*bufs)->nb_segs > 1)) {
1395                                         if (eth_mbuf_to_sg_fd(*bufs,
1396                                                               &fd_arr[loop],
1397                                                               bpid))
1398                                                 goto send_n_return;
1399                                 } else {
1400                                         eth_mbuf_to_fd(*bufs,
1401                                                        &fd_arr[loop], bpid);
1402                                 }
1403                         }
1404                         bufs++;
1405                 }
1406
1407                 loop = 0;
1408                 retry_count = 0;
1409                 while (loop < frames_to_send) {
1410                         ret = qbman_swp_enqueue_multiple_desc(swp,
1411                                         &eqdesc[loop], &fd_arr[loop],
1412                                         frames_to_send - loop);
1413                         if (unlikely(ret < 0)) {
1414                                 retry_count++;
1415                                 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) {
1416                                         num_tx += loop;
1417                                         nb_pkts -= loop;
1418                                         goto send_n_return;
1419                                 }
1420                         } else {
1421                                 loop += ret;
1422                                 retry_count = 0;
1423                         }
1424                 }
1425
1426                 num_tx += loop;
1427                 nb_pkts -= loop;
1428         }
1429         dpaa2_q->tx_pkts += num_tx;
1430         return num_tx;
1431
1432 send_n_return:
1433         /* send any already prepared fd */
1434         if (loop) {
1435                 unsigned int i = 0;
1436
1437                 retry_count = 0;
1438                 while (i < loop) {
1439                         ret = qbman_swp_enqueue_multiple_desc(swp,
1440                                        &eqdesc[loop], &fd_arr[i], loop - i);
1441                         if (unlikely(ret < 0)) {
1442                                 retry_count++;
1443                                 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT)
1444                                         break;
1445                         } else {
1446                                 i += ret;
1447                                 retry_count = 0;
1448                         }
1449                 }
1450                 num_tx += i;
1451         }
1452 skip_tx:
1453         dpaa2_q->tx_pkts += num_tx;
1454         return num_tx;
1455 }
1456
1457 /**
1458  * Dummy DPDK callback for TX.
1459  *
1460  * This function is used to temporarily replace the real callback during
1461  * unsafe control operations on the queue, or in case of error.
1462  *
1463  * @param dpdk_txq
1464  *   Generic pointer to TX queue structure.
1465  * @param[in] pkts
1466  *   Packets to transmit.
1467  * @param pkts_n
1468  *   Number of packets in array.
1469  *
1470  * @return
1471  *   Number of packets successfully transmitted (<= pkts_n).
1472  */
1473 uint16_t
1474 dummy_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
1475 {
1476         (void)queue;
1477         (void)bufs;
1478         (void)nb_pkts;
1479         return 0;
1480 }
1481
1482 #if defined(RTE_TOOLCHAIN_GCC)
1483 #pragma GCC diagnostic push
1484 #pragma GCC diagnostic ignored "-Wcast-qual"
1485 #elif defined(RTE_TOOLCHAIN_CLANG)
1486 #pragma clang diagnostic push
1487 #pragma clang diagnostic ignored "-Wcast-qual"
1488 #endif
1489
1490 /* This function loopbacks all the received packets.*/
1491 uint16_t
1492 dpaa2_dev_loopback_rx(void *queue,
1493                       struct rte_mbuf **bufs __rte_unused,
1494                       uint16_t nb_pkts)
1495 {
1496         /* Function receive frames for a given device and VQ*/
1497         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
1498         struct qbman_result *dq_storage, *dq_storage1 = NULL;
1499         uint32_t fqid = dpaa2_q->fqid;
1500         int ret, num_rx = 0, num_tx = 0, pull_size;
1501         uint8_t pending, status;
1502         struct qbman_swp *swp;
1503         struct qbman_fd *fd[DPAA2_LX2_DQRR_RING_SIZE];
1504         struct qbman_pull_desc pulldesc;
1505         struct qbman_eq_desc eqdesc;
1506         struct queue_storage_info_t *q_storage = dpaa2_q->q_storage;
1507         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
1508         struct dpaa2_dev_priv *priv = eth_data->dev_private;
1509         struct dpaa2_queue *tx_q = priv->tx_vq[0];
1510         /* todo - currently we are using 1st TX queue only for loopback*/
1511
1512         if (unlikely(!DPAA2_PER_LCORE_ETHRX_DPIO)) {
1513                 ret = dpaa2_affine_qbman_ethrx_swp();
1514                 if (ret) {
1515                         DPAA2_PMD_ERR("Failure in affining portal");
1516                         return 0;
1517                 }
1518         }
1519         swp = DPAA2_PER_LCORE_ETHRX_PORTAL;
1520         pull_size = (nb_pkts > dpaa2_dqrr_size) ? dpaa2_dqrr_size : nb_pkts;
1521         if (unlikely(!q_storage->active_dqs)) {
1522                 q_storage->toggle = 0;
1523                 dq_storage = q_storage->dq_storage[q_storage->toggle];
1524                 q_storage->last_num_pkts = pull_size;
1525                 qbman_pull_desc_clear(&pulldesc);
1526                 qbman_pull_desc_set_numframes(&pulldesc,
1527                                               q_storage->last_num_pkts);
1528                 qbman_pull_desc_set_fq(&pulldesc, fqid);
1529                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
1530                         (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
1531                 if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) {
1532                         while (!qbman_check_command_complete(
1533                                get_swp_active_dqs(
1534                                DPAA2_PER_LCORE_ETHRX_DPIO->index)))
1535                                 ;
1536                         clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index);
1537                 }
1538                 while (1) {
1539                         if (qbman_swp_pull(swp, &pulldesc)) {
1540                                 DPAA2_PMD_DP_DEBUG(
1541                                         "VDQ command not issued.QBMAN busy\n");
1542                                 /* Portal was busy, try again */
1543                                 continue;
1544                         }
1545                         break;
1546                 }
1547                 q_storage->active_dqs = dq_storage;
1548                 q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index;
1549                 set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index,
1550                                    dq_storage);
1551         }
1552
1553         dq_storage = q_storage->active_dqs;
1554         rte_prefetch0((void *)(size_t)(dq_storage));
1555         rte_prefetch0((void *)(size_t)(dq_storage + 1));
1556
1557         /* Prepare next pull descriptor. This will give space for the
1558          * prefething done on DQRR entries
1559          */
1560         q_storage->toggle ^= 1;
1561         dq_storage1 = q_storage->dq_storage[q_storage->toggle];
1562         qbman_pull_desc_clear(&pulldesc);
1563         qbman_pull_desc_set_numframes(&pulldesc, pull_size);
1564         qbman_pull_desc_set_fq(&pulldesc, fqid);
1565         qbman_pull_desc_set_storage(&pulldesc, dq_storage1,
1566                 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage1)), 1);
1567
1568         /*Prepare enqueue descriptor*/
1569         qbman_eq_desc_clear(&eqdesc);
1570         qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ);
1571         qbman_eq_desc_set_response(&eqdesc, 0, 0);
1572         qbman_eq_desc_set_fq(&eqdesc, tx_q->fqid);
1573
1574         /* Check if the previous issued command is completed.
1575          * Also seems like the SWP is shared between the Ethernet Driver
1576          * and the SEC driver.
1577          */
1578         while (!qbman_check_command_complete(dq_storage))
1579                 ;
1580         if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id))
1581                 clear_swp_active_dqs(q_storage->active_dpio_id);
1582
1583         pending = 1;
1584
1585         do {
1586                 /* Loop until the dq_storage is updated with
1587                  * new token by QBMAN
1588                  */
1589                 while (!qbman_check_new_result(dq_storage))
1590                         ;
1591                 rte_prefetch0((void *)((size_t)(dq_storage + 2)));
1592                 /* Check whether Last Pull command is Expired and
1593                  * setting Condition for Loop termination
1594                  */
1595                 if (qbman_result_DQ_is_pull_complete(dq_storage)) {
1596                         pending = 0;
1597                         /* Check for valid frame. */
1598                         status = qbman_result_DQ_flags(dq_storage);
1599                         if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0))
1600                                 continue;
1601                 }
1602                 fd[num_rx] = (struct qbman_fd *)qbman_result_DQ_fd(dq_storage);
1603
1604                 dq_storage++;
1605                 num_rx++;
1606         } while (pending);
1607
1608         while (num_tx < num_rx) {
1609                 num_tx += qbman_swp_enqueue_multiple_fd(swp, &eqdesc,
1610                                 &fd[num_tx], 0, num_rx - num_tx);
1611         }
1612
1613         if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) {
1614                 while (!qbman_check_command_complete(
1615                        get_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)))
1616                         ;
1617                 clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index);
1618         }
1619         /* issue a volatile dequeue command for next pull */
1620         while (1) {
1621                 if (qbman_swp_pull(swp, &pulldesc)) {
1622                         DPAA2_PMD_DP_DEBUG("VDQ command is not issued."
1623                                           "QBMAN is busy (2)\n");
1624                         continue;
1625                 }
1626                 break;
1627         }
1628         q_storage->active_dqs = dq_storage1;
1629         q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index;
1630         set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index, dq_storage1);
1631
1632         dpaa2_q->rx_pkts += num_rx;
1633         dpaa2_q->tx_pkts += num_tx;
1634
1635         return 0;
1636 }
1637 #if defined(RTE_TOOLCHAIN_GCC)
1638 #pragma GCC diagnostic pop
1639 #elif defined(RTE_TOOLCHAIN_CLANG)
1640 #pragma clang diagnostic pop
1641 #endif