drivers/net/dpaa2/dpaa2_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
   4  *   Copyright 2016-2019 NXP
   5  *
   6  */
   7
   8 #include <time.h>
   9 #include <net/if.h>
  10
  11 #include <rte_mbuf.h>
  12 #include <rte_ethdev_driver.h>
  13 #include <rte_malloc.h>
  14 #include <rte_memcpy.h>
  15 #include <rte_string_fns.h>
  16 #include <rte_dev.h>
  17
  18 #include <rte_fslmc.h>
  19 #include <fslmc_vfio.h>
  20 #include <dpaa2_hw_pvt.h>
  21 #include <dpaa2_hw_dpio.h>
  22 #include <dpaa2_hw_mempool.h>
  23
  24 #include "dpaa2_pmd_logs.h"
  25 #include "dpaa2_ethdev.h"
  26 #include "base/dpaa2_hw_dpni_annot.h"
  27
  28 static inline uint32_t __attribute__((hot))
  29 dpaa2_dev_rx_parse_slow(struct rte_mbuf *mbuf,
  30                         struct dpaa2_annot_hdr *annotation);
  31
  32 static void enable_tx_tstamp(struct qbman_fd *fd) __attribute__((unused));
  33
  34 #define DPAA2_MBUF_TO_CONTIG_FD(_mbuf, _fd, _bpid)  do { \
  35         DPAA2_SET_FD_ADDR(_fd, DPAA2_MBUF_VADDR_TO_IOVA(_mbuf)); \
  36         DPAA2_SET_FD_LEN(_fd, _mbuf->data_len); \
  37         DPAA2_SET_ONLY_FD_BPID(_fd, _bpid); \
  38         DPAA2_SET_FD_OFFSET(_fd, _mbuf->data_off); \
  39         DPAA2_SET_FD_FRC(_fd, 0);               \
  40         DPAA2_RESET_FD_CTRL(_fd);               \
  41         DPAA2_RESET_FD_FLC(_fd);                \
  42 } while (0)
  43
  44 static inline void __attribute__((hot))
  45 dpaa2_dev_rx_parse_new(struct rte_mbuf *m, const struct qbman_fd *fd)
  46 {
  47         struct dpaa2_annot_hdr *annotation;
  48         uint16_t frc = DPAA2_GET_FD_FRC_PARSE_SUM(fd);
  49
  50         m->packet_type = RTE_PTYPE_UNKNOWN;
  51         switch (frc) {
  52         case DPAA2_PKT_TYPE_ETHER:
  53                 m->packet_type = RTE_PTYPE_L2_ETHER;
  54                 break;
  55         case DPAA2_PKT_TYPE_IPV4:
  56                 m->packet_type = RTE_PTYPE_L2_ETHER |
  57                         RTE_PTYPE_L3_IPV4;
  58                 break;
  59         case DPAA2_PKT_TYPE_IPV6:
  60                 m->packet_type = RTE_PTYPE_L2_ETHER |
  61                         RTE_PTYPE_L3_IPV6;
  62                 break;
  63         case DPAA2_PKT_TYPE_IPV4_EXT:
  64                 m->packet_type = RTE_PTYPE_L2_ETHER |
  65                         RTE_PTYPE_L3_IPV4_EXT;
  66                 break;
  67         case DPAA2_PKT_TYPE_IPV6_EXT:
  68                 m->packet_type = RTE_PTYPE_L2_ETHER |
  69                         RTE_PTYPE_L3_IPV6_EXT;
  70                 break;
  71         case DPAA2_PKT_TYPE_IPV4_TCP:
  72                 m->packet_type = RTE_PTYPE_L2_ETHER |
  73                         RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP;
  74                 break;
  75         case DPAA2_PKT_TYPE_IPV6_TCP:
  76                 m->packet_type = RTE_PTYPE_L2_ETHER |
  77                         RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP;
  78                 break;
  79         case DPAA2_PKT_TYPE_IPV4_UDP:
  80                 m->packet_type = RTE_PTYPE_L2_ETHER |
  81                         RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP;
  82                 break;
  83         case DPAA2_PKT_TYPE_IPV6_UDP:
  84                 m->packet_type = RTE_PTYPE_L2_ETHER |
  85                         RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP;
  86                 break;
  87         case DPAA2_PKT_TYPE_IPV4_SCTP:
  88                 m->packet_type = RTE_PTYPE_L2_ETHER |
  89                         RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP;
  90                 break;
  91         case DPAA2_PKT_TYPE_IPV6_SCTP:
  92                 m->packet_type = RTE_PTYPE_L2_ETHER |
  93                         RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP;
  94                 break;
  95         case DPAA2_PKT_TYPE_IPV4_ICMP:
  96                 m->packet_type = RTE_PTYPE_L2_ETHER |
  97                         RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_ICMP;
  98                 break;
  99         case DPAA2_PKT_TYPE_IPV6_ICMP:
 100                 m->packet_type = RTE_PTYPE_L2_ETHER |
 101                         RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_ICMP;
 102                 break;
 103         default:
 104                 m->packet_type = dpaa2_dev_rx_parse_slow(m,
 105                   (void *)((size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd))
 106                          + DPAA2_FD_PTA_SIZE));
 107         }
 108         m->hash.rss = fd->simple.flc_hi;
 109         m->ol_flags |= PKT_RX_RSS_HASH;
 110
 111         if (dpaa2_enable_ts == PMD_DPAA2_ENABLE_TS) {
 112                 annotation = (struct dpaa2_annot_hdr *)
 113                         ((size_t)DPAA2_IOVA_TO_VADDR(
 114                         DPAA2_GET_FD_ADDR(fd)) + DPAA2_FD_PTA_SIZE);
 115                 m->timestamp = annotation->word2;
 116                 m->ol_flags |= PKT_RX_TIMESTAMP;
 117                 DPAA2_PMD_DP_DEBUG("pkt timestamp:0x%" PRIx64 "", m->timestamp);
 118         }
 119
 120         DPAA2_PMD_DP_DEBUG("HW frc = 0x%x\t packet type =0x%x "
 121                 "ol_flags =0x%" PRIx64 "",
 122                 frc, m->packet_type, m->ol_flags);
 123 }
 124
 125 static inline uint32_t __attribute__((hot))
 126 dpaa2_dev_rx_parse_slow(struct rte_mbuf *mbuf,
 127                         struct dpaa2_annot_hdr *annotation)
 128 {
 129         uint32_t pkt_type = RTE_PTYPE_UNKNOWN;
 130         uint16_t *vlan_tci;
 131
 132         DPAA2_PMD_DP_DEBUG("(slow parse)annotation(3)=0x%" PRIx64 "\t"
 133                         "(4)=0x%" PRIx64 "\t",
 134                         annotation->word3, annotation->word4);
 135
 136 #if defined(RTE_LIBRTE_IEEE1588)
 137         if (BIT_ISSET_AT_POS(annotation->word1, DPAA2_ETH_FAS_PTP))
 138                 mbuf->ol_flags |= PKT_RX_IEEE1588_PTP;
 139 #endif
 140
 141         if (BIT_ISSET_AT_POS(annotation->word3, L2_VLAN_1_PRESENT)) {
 142                 vlan_tci = rte_pktmbuf_mtod_offset(mbuf, uint16_t *,
 143                         (VLAN_TCI_OFFSET_1(annotation->word5) >> 16));
 144                 mbuf->vlan_tci = rte_be_to_cpu_16(*vlan_tci);
 145                 mbuf->ol_flags |= PKT_RX_VLAN;
 146                 pkt_type |= RTE_PTYPE_L2_ETHER_VLAN;
 147         } else if (BIT_ISSET_AT_POS(annotation->word3, L2_VLAN_N_PRESENT)) {
 148                 vlan_tci = rte_pktmbuf_mtod_offset(mbuf, uint16_t *,
 149                         (VLAN_TCI_OFFSET_1(annotation->word5) >> 16));
 150                 mbuf->vlan_tci = rte_be_to_cpu_16(*vlan_tci);
 151                 mbuf->ol_flags |= PKT_RX_VLAN | PKT_RX_QINQ;
 152                 pkt_type |= RTE_PTYPE_L2_ETHER_QINQ;
 153         }
 154
 155         if (BIT_ISSET_AT_POS(annotation->word3, L2_ARP_PRESENT)) {
 156                 pkt_type |= RTE_PTYPE_L2_ETHER_ARP;
 157                 goto parse_done;
 158         } else if (BIT_ISSET_AT_POS(annotation->word3, L2_ETH_MAC_PRESENT)) {
 159                 pkt_type |= RTE_PTYPE_L2_ETHER;
 160         } else {
 161                 goto parse_done;
 162         }
 163
 164         if (BIT_ISSET_AT_POS(annotation->word4, L3_IPV4_1_PRESENT |
 165                              L3_IPV4_N_PRESENT)) {
 166                 pkt_type |= RTE_PTYPE_L3_IPV4;
 167                 if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_OPT_PRESENT |
 168                         L3_IP_N_OPT_PRESENT))
 169                         pkt_type |= RTE_PTYPE_L3_IPV4_EXT;
 170
 171         } else if (BIT_ISSET_AT_POS(annotation->word4, L3_IPV6_1_PRESENT |
 172                   L3_IPV6_N_PRESENT)) {
 173                 pkt_type |= RTE_PTYPE_L3_IPV6;
 174                 if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_OPT_PRESENT |
 175                     L3_IP_N_OPT_PRESENT))
 176                         pkt_type |= RTE_PTYPE_L3_IPV6_EXT;
 177         } else {
 178                 goto parse_done;
 179         }
 180
 181         if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L3CE))
 182                 mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD;
 183         else if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE))
 184                 mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD;
 185
 186         if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_FIRST_FRAGMENT |
 187             L3_IP_1_MORE_FRAGMENT |
 188             L3_IP_N_FIRST_FRAGMENT |
 189             L3_IP_N_MORE_FRAGMENT)) {
 190                 pkt_type |= RTE_PTYPE_L4_FRAG;
 191                 goto parse_done;
 192         } else {
 193                 pkt_type |= RTE_PTYPE_L4_NONFRAG;
 194         }
 195
 196         if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_UDP_PRESENT))
 197                 pkt_type |= RTE_PTYPE_L4_UDP;
 198
 199         else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_TCP_PRESENT))
 200                 pkt_type |= RTE_PTYPE_L4_TCP;
 201
 202         else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_SCTP_PRESENT))
 203                 pkt_type |= RTE_PTYPE_L4_SCTP;
 204
 205         else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_ICMP_PRESENT))
 206                 pkt_type |= RTE_PTYPE_L4_ICMP;
 207
 208         else if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_UNKNOWN_PROTOCOL))
 209                 pkt_type |= RTE_PTYPE_UNKNOWN;
 210
 211 parse_done:
 212         return pkt_type;
 213 }
 214
 215 static inline uint32_t __attribute__((hot))
 216 dpaa2_dev_rx_parse(struct rte_mbuf *mbuf, void *hw_annot_addr)
 217 {
 218         struct dpaa2_annot_hdr *annotation =
 219                         (struct dpaa2_annot_hdr *)hw_annot_addr;
 220
 221         DPAA2_PMD_DP_DEBUG("(fast parse) Annotation = 0x%" PRIx64 "\t",
 222                            annotation->word4);
 223
 224         if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L3CE))
 225                 mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD;
 226         else if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE))
 227                 mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD;
 228
 229         mbuf->ol_flags |= PKT_RX_TIMESTAMP;
 230         mbuf->timestamp = annotation->word2;
 231         DPAA2_PMD_DP_DEBUG("pkt timestamp: 0x%" PRIx64 "", mbuf->timestamp);
 232
 233         /* Check detailed parsing requirement */
 234         if (annotation->word3 & 0x7FFFFC3FFFF)
 235                 return dpaa2_dev_rx_parse_slow(mbuf, annotation);
 236
 237         /* Return some common types from parse processing */
 238         switch (annotation->word4) {
 239         case DPAA2_L3_IPv4:
 240                 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4;
 241         case DPAA2_L3_IPv6:
 242                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6;
 243         case DPAA2_L3_IPv4_TCP:
 244                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
 245                                 RTE_PTYPE_L4_TCP;
 246         case DPAA2_L3_IPv4_UDP:
 247                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
 248                                 RTE_PTYPE_L4_UDP;
 249         case DPAA2_L3_IPv6_TCP:
 250                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
 251                                 RTE_PTYPE_L4_TCP;
 252         case DPAA2_L3_IPv6_UDP:
 253                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
 254                                 RTE_PTYPE_L4_UDP;
 255         default:
 256                 break;
 257         }
 258
 259         return dpaa2_dev_rx_parse_slow(mbuf, annotation);
 260 }
 261
 262 static inline struct rte_mbuf *__attribute__((hot))
 263 eth_sg_fd_to_mbuf(const struct qbman_fd *fd)
 264 {
 265         struct qbman_sge *sgt, *sge;
 266         size_t sg_addr, fd_addr;
 267         int i = 0;
 268         struct rte_mbuf *first_seg, *next_seg, *cur_seg, *temp;
 269
 270         fd_addr = (size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
 271
 272         /* Get Scatter gather table address */
 273         sgt = (struct qbman_sge *)(fd_addr + DPAA2_GET_FD_OFFSET(fd));
 274
 275         sge = &sgt[i++];
 276         sg_addr = (size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FLE_ADDR(sge));
 277
 278         /* First Scatter gather entry */
 279         first_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr,
 280                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
 281         /* Prepare all the metadata for first segment */
 282         first_seg->buf_addr = (uint8_t *)sg_addr;
 283         first_seg->ol_flags = 0;
 284         first_seg->data_off = DPAA2_GET_FLE_OFFSET(sge);
 285         first_seg->data_len = sge->length  & 0x1FFFF;
 286         first_seg->pkt_len = DPAA2_GET_FD_LEN(fd);
 287         first_seg->nb_segs = 1;
 288         first_seg->next = NULL;
 289         if (dpaa2_svr_family == SVR_LX2160A)
 290                 dpaa2_dev_rx_parse_new(first_seg, fd);
 291         else
 292                 first_seg->packet_type = dpaa2_dev_rx_parse(first_seg,
 293                         (void *)((size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd))
 294                          + DPAA2_FD_PTA_SIZE));
 295
 296         rte_mbuf_refcnt_set(first_seg, 1);
 297         cur_seg = first_seg;
 298         while (!DPAA2_SG_IS_FINAL(sge)) {
 299                 sge = &sgt[i++];
 300                 sg_addr = (size_t)DPAA2_IOVA_TO_VADDR(
 301                                 DPAA2_GET_FLE_ADDR(sge));
 302                 next_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr,
 303                         rte_dpaa2_bpid_info[DPAA2_GET_FLE_BPID(sge)].meta_data_size);
 304                 next_seg->buf_addr  = (uint8_t *)sg_addr;
 305                 next_seg->data_off  = DPAA2_GET_FLE_OFFSET(sge);
 306                 next_seg->data_len  = sge->length  & 0x1FFFF;
 307                 first_seg->nb_segs += 1;
 308                 rte_mbuf_refcnt_set(next_seg, 1);
 309                 cur_seg->next = next_seg;
 310                 next_seg->next = NULL;
 311                 cur_seg = next_seg;
 312         }
 313         temp = DPAA2_INLINE_MBUF_FROM_BUF(fd_addr,
 314                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
 315         rte_mbuf_refcnt_set(temp, 1);
 316         rte_pktmbuf_free_seg(temp);
 317
 318         return (void *)first_seg;
 319 }
 320
 321 static inline struct rte_mbuf *__attribute__((hot))
 322 eth_fd_to_mbuf(const struct qbman_fd *fd)
 323 {
 324         struct rte_mbuf *mbuf = DPAA2_INLINE_MBUF_FROM_BUF(
 325                 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)),
 326                      rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
 327
 328         /* need to repopulated some of the fields,
 329          * as they may have changed in last transmission
 330          */
 331         mbuf->nb_segs = 1;
 332         mbuf->ol_flags = 0;
 333         mbuf->data_off = DPAA2_GET_FD_OFFSET(fd);
 334         mbuf->data_len = DPAA2_GET_FD_LEN(fd);
 335         mbuf->pkt_len = mbuf->data_len;
 336         mbuf->next = NULL;
 337         rte_mbuf_refcnt_set(mbuf, 1);
 338
 339         /* Parse the packet */
 340         /* parse results for LX2 are there in FRC field of FD.
 341          * For other DPAA2 platforms , parse results are after
 342          * the private - sw annotation area
 343          */
 344
 345         if (dpaa2_svr_family == SVR_LX2160A)
 346                 dpaa2_dev_rx_parse_new(mbuf, fd);
 347         else
 348                 mbuf->packet_type = dpaa2_dev_rx_parse(mbuf,
 349                         (void *)((size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd))
 350                          + DPAA2_FD_PTA_SIZE));
 351
 352         DPAA2_PMD_DP_DEBUG("to mbuf - mbuf =%p, mbuf->buf_addr =%p, off = %d,"
 353                 "fd_off=%d fd =%" PRIx64 ", meta = %d  bpid =%d, len=%d\n",
 354                 mbuf, mbuf->buf_addr, mbuf->data_off,
 355                 DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ADDR(fd),
 356                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size,
 357                 DPAA2_GET_FD_BPID(fd), DPAA2_GET_FD_LEN(fd));
 358
 359         return mbuf;
 360 }
 361
 362 static int __attribute__ ((noinline)) __attribute__((hot))
 363 eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf,
 364                   struct qbman_fd *fd, uint16_t bpid)
 365 {
 366         struct rte_mbuf *cur_seg = mbuf, *prev_seg, *mi, *temp;
 367         struct qbman_sge *sgt, *sge = NULL;
 368         int i;
 369
 370         temp = rte_pktmbuf_alloc(mbuf->pool);
 371         if (temp == NULL) {
 372                 DPAA2_PMD_DP_DEBUG("No memory to allocate S/G table\n");
 373                 return -ENOMEM;
 374         }
 375
 376         DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(temp));
 377         DPAA2_SET_FD_LEN(fd, mbuf->pkt_len);
 378         DPAA2_SET_ONLY_FD_BPID(fd, bpid);
 379         DPAA2_SET_FD_OFFSET(fd, temp->data_off);
 380         DPAA2_FD_SET_FORMAT(fd, qbman_fd_sg);
 381         DPAA2_RESET_FD_FRC(fd);
 382         DPAA2_RESET_FD_CTRL(fd);
 383         /*Set Scatter gather table and Scatter gather entries*/
 384         sgt = (struct qbman_sge *)(
 385                         (size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd))
 386                         + DPAA2_GET_FD_OFFSET(fd));
 387
 388         for (i = 0; i < mbuf->nb_segs; i++) {
 389                 sge = &sgt[i];
 390                 /*Resetting the buffer pool id and offset field*/
 391                 sge->fin_bpid_offset = 0;
 392                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(cur_seg));
 393                 DPAA2_SET_FLE_OFFSET(sge, cur_seg->data_off);
 394                 sge->length = cur_seg->data_len;
 395                 if (RTE_MBUF_DIRECT(cur_seg)) {
 396                         if (rte_mbuf_refcnt_read(cur_seg) > 1) {
 397                                 /* If refcnt > 1, invalid bpid is set to ensure
 398                                  * buffer is not freed by HW
 399                                  */
 400                                 DPAA2_SET_FLE_IVP(sge);
 401                                 rte_mbuf_refcnt_update(cur_seg, -1);
 402                         } else
 403                                 DPAA2_SET_FLE_BPID(sge,
 404                                                 mempool_to_bpid(cur_seg->pool));
 405                         cur_seg = cur_seg->next;
 406                 } else {
 407                         /* Get owner MBUF from indirect buffer */
 408                         mi = rte_mbuf_from_indirect(cur_seg);
 409                         if (rte_mbuf_refcnt_read(mi) > 1) {
 410                                 /* If refcnt > 1, invalid bpid is set to ensure
 411                                  * owner buffer is not freed by HW
 412                                  */
 413                                 DPAA2_SET_FLE_IVP(sge);
 414                         } else {
 415                                 DPAA2_SET_FLE_BPID(sge,
 416                                                    mempool_to_bpid(mi->pool));
 417                                 rte_mbuf_refcnt_update(mi, 1);
 418                         }
 419                         prev_seg = cur_seg;
 420                         cur_seg = cur_seg->next;
 421                         prev_seg->next = NULL;
 422                         rte_pktmbuf_free(prev_seg);
 423                 }
 424         }
 425         DPAA2_SG_SET_FINAL(sge, true);
 426         return 0;
 427 }
 428
 429 static void
 430 eth_mbuf_to_fd(struct rte_mbuf *mbuf,
 431                struct qbman_fd *fd, uint16_t bpid) __attribute__((unused));
 432
 433 static void __attribute__ ((noinline)) __attribute__((hot))
 434 eth_mbuf_to_fd(struct rte_mbuf *mbuf,
 435                struct qbman_fd *fd, uint16_t bpid)
 436 {
 437         DPAA2_MBUF_TO_CONTIG_FD(mbuf, fd, bpid);
 438
 439         DPAA2_PMD_DP_DEBUG("mbuf =%p, mbuf->buf_addr =%p, off = %d,"
 440                 "fd_off=%d fd =%" PRIx64 ", meta = %d  bpid =%d, len=%d\n",
 441                 mbuf, mbuf->buf_addr, mbuf->data_off,
 442                 DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ADDR(fd),
 443                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size,
 444                 DPAA2_GET_FD_BPID(fd), DPAA2_GET_FD_LEN(fd));
 445         if (RTE_MBUF_DIRECT(mbuf)) {
 446                 if (rte_mbuf_refcnt_read(mbuf) > 1) {
 447                         DPAA2_SET_FD_IVP(fd);
 448                         rte_mbuf_refcnt_update(mbuf, -1);
 449                 }
 450         } else {
 451                 struct rte_mbuf *mi;
 452
 453                 mi = rte_mbuf_from_indirect(mbuf);
 454                 if (rte_mbuf_refcnt_read(mi) > 1)
 455                         DPAA2_SET_FD_IVP(fd);
 456                 else
 457                         rte_mbuf_refcnt_update(mi, 1);
 458                 rte_pktmbuf_free(mbuf);
 459         }
 460 }
 461
 462 static inline int __attribute__((hot))
 463 eth_copy_mbuf_to_fd(struct rte_mbuf *mbuf,
 464                     struct qbman_fd *fd, uint16_t bpid)
 465 {
 466         struct rte_mbuf *m;
 467         void *mb = NULL;
 468
 469         if (rte_dpaa2_mbuf_alloc_bulk(
 470                 rte_dpaa2_bpid_info[bpid].bp_list->mp, &mb, 1)) {
 471                 DPAA2_PMD_DP_DEBUG("Unable to allocated DPAA2 buffer\n");
 472                 return -1;
 473         }
 474         m = (struct rte_mbuf *)mb;
 475         memcpy((char *)m->buf_addr + mbuf->data_off,
 476                (void *)((char *)mbuf->buf_addr + mbuf->data_off),
 477                 mbuf->pkt_len);
 478
 479         /* Copy required fields */
 480         m->data_off = mbuf->data_off;
 481         m->ol_flags = mbuf->ol_flags;
 482         m->packet_type = mbuf->packet_type;
 483         m->tx_offload = mbuf->tx_offload;
 484
 485         DPAA2_MBUF_TO_CONTIG_FD(m, fd, bpid);
 486
 487         DPAA2_PMD_DP_DEBUG(
 488                 "mbuf: %p, BMAN buf addr: %p, fdaddr: %" PRIx64 ", bpid: %d,"
 489                 " meta: %d, off: %d, len: %d\n",
 490                 (void *)mbuf,
 491                 mbuf->buf_addr,
 492                 DPAA2_GET_FD_ADDR(fd),
 493                 DPAA2_GET_FD_BPID(fd),
 494                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size,
 495                 DPAA2_GET_FD_OFFSET(fd),
 496                 DPAA2_GET_FD_LEN(fd));
 497
 498 return 0;
 499 }
 500
 501 /* This function assumes that caller will be keep the same value for nb_pkts
 502  * across calls per queue, if that is not the case, better use non-prefetch
 503  * version of rx call.
 504  * It will return the packets as requested in previous call without honoring
 505  * the current nb_pkts or bufs space.
 506  */
 507 uint16_t
 508 dpaa2_dev_prefetch_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 509 {
 510         /* Function receive frames for a given device and VQ*/
 511         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
 512         struct qbman_result *dq_storage, *dq_storage1 = NULL;
 513         uint32_t fqid = dpaa2_q->fqid;
 514         int ret, num_rx = 0, pull_size;
 515         uint8_t pending, status;
 516         struct qbman_swp *swp;
 517         const struct qbman_fd *fd, *next_fd;
 518         struct qbman_pull_desc pulldesc;
 519         struct queue_storage_info_t *q_storage = dpaa2_q->q_storage;
 520         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
 521 #if defined(RTE_LIBRTE_IEEE1588)
 522         struct dpaa2_dev_priv *priv = eth_data->dev_private;
 523 #endif
 524
 525         if (unlikely(!DPAA2_PER_LCORE_ETHRX_DPIO)) {
 526                 ret = dpaa2_affine_qbman_ethrx_swp();
 527                 if (ret) {
 528                         DPAA2_PMD_ERR("Failure in affining portal");
 529                         return 0;
 530                 }
 531         }
 532
 533         if (unlikely(!rte_dpaa2_bpid_info &&
 534                      rte_eal_process_type() == RTE_PROC_SECONDARY))
 535                 rte_dpaa2_bpid_info = dpaa2_q->bp_array;
 536
 537         swp = DPAA2_PER_LCORE_ETHRX_PORTAL;
 538         pull_size = (nb_pkts > dpaa2_dqrr_size) ? dpaa2_dqrr_size : nb_pkts;
 539         if (unlikely(!q_storage->active_dqs)) {
 540                 q_storage->toggle = 0;
 541                 dq_storage = q_storage->dq_storage[q_storage->toggle];
 542                 q_storage->last_num_pkts = pull_size;
 543                 qbman_pull_desc_clear(&pulldesc);
 544                 qbman_pull_desc_set_numframes(&pulldesc,
 545                                               q_storage->last_num_pkts);
 546                 qbman_pull_desc_set_fq(&pulldesc, fqid);
 547                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
 548                         (uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
 549                 if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) {
 550                         while (!qbman_check_command_complete(
 551                                get_swp_active_dqs(
 552                                DPAA2_PER_LCORE_ETHRX_DPIO->index)))
 553                                 ;
 554                         clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index);
 555                 }
 556                 while (1) {
 557                         if (qbman_swp_pull(swp, &pulldesc)) {
 558                                 DPAA2_PMD_DP_DEBUG("VDQ command is not issued."
 559                                                   " QBMAN is busy (1)\n");
 560                                 /* Portal was busy, try again */
 561                                 continue;
 562                         }
 563                         break;
 564                 }
 565                 q_storage->active_dqs = dq_storage;
 566                 q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index;
 567                 set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index,
 568                                    dq_storage);
 569         }
 570
 571         dq_storage = q_storage->active_dqs;
 572         rte_prefetch0((void *)(size_t)(dq_storage));
 573         rte_prefetch0((void *)(size_t)(dq_storage + 1));
 574
 575         /* Prepare next pull descriptor. This will give space for the
 576          * prefething done on DQRR entries
 577          */
 578         q_storage->toggle ^= 1;
 579         dq_storage1 = q_storage->dq_storage[q_storage->toggle];
 580         qbman_pull_desc_clear(&pulldesc);
 581         qbman_pull_desc_set_numframes(&pulldesc, pull_size);
 582         qbman_pull_desc_set_fq(&pulldesc, fqid);
 583         qbman_pull_desc_set_storage(&pulldesc, dq_storage1,
 584                 (uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage1)), 1);
 585
 586         /* Check if the previous issued command is completed.
 587          * Also seems like the SWP is shared between the Ethernet Driver
 588          * and the SEC driver.
 589          */
 590         while (!qbman_check_command_complete(dq_storage))
 591                 ;
 592         if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id))
 593                 clear_swp_active_dqs(q_storage->active_dpio_id);
 594
 595         pending = 1;
 596
 597         do {
 598                 /* Loop until the dq_storage is updated with
 599                  * new token by QBMAN
 600                  */
 601                 while (!qbman_check_new_result(dq_storage))
 602                         ;
 603                 rte_prefetch0((void *)((size_t)(dq_storage + 2)));
 604                 /* Check whether Last Pull command is Expired and
 605                  * setting Condition for Loop termination
 606                  */
 607                 if (qbman_result_DQ_is_pull_complete(dq_storage)) {
 608                         pending = 0;
 609                         /* Check for valid frame. */
 610                         status = qbman_result_DQ_flags(dq_storage);
 611                         if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0))
 612                                 continue;
 613                 }
 614                 fd = qbman_result_DQ_fd(dq_storage);
 615
 616                 if (dpaa2_svr_family != SVR_LX2160A) {
 617                         next_fd = qbman_result_DQ_fd(dq_storage + 1);
 618                         /* Prefetch Annotation address for the parse results */
 619                         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(
 620                                       next_fd) + DPAA2_FD_PTA_SIZE + 16));
 621                 }
 622
 623                 if (unlikely(DPAA2_FD_GET_FORMAT(fd) == qbman_fd_sg))
 624                         bufs[num_rx] = eth_sg_fd_to_mbuf(fd);
 625                 else
 626                         bufs[num_rx] = eth_fd_to_mbuf(fd);
 627                 bufs[num_rx]->port = eth_data->port_id;
 628 #if defined(RTE_LIBRTE_IEEE1588)
 629                 priv->rx_timestamp = bufs[num_rx]->timestamp;
 630 #endif
 631
 632                 if (eth_data->dev_conf.rxmode.offloads &
 633                                 DEV_RX_OFFLOAD_VLAN_STRIP)
 634                         rte_vlan_strip(bufs[num_rx]);
 635
 636                 dq_storage++;
 637                 num_rx++;
 638         } while (pending);
 639
 640         if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) {
 641                 while (!qbman_check_command_complete(
 642                        get_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)))
 643                         ;
 644                 clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index);
 645         }
 646         /* issue a volatile dequeue command for next pull */
 647         while (1) {
 648                 if (qbman_swp_pull(swp, &pulldesc)) {
 649                         DPAA2_PMD_DP_DEBUG("VDQ command is not issued."
 650                                           "QBMAN is busy (2)\n");
 651                         continue;
 652                 }
 653                 break;
 654         }
 655         q_storage->active_dqs = dq_storage1;
 656         q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index;
 657         set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index, dq_storage1);
 658
 659         dpaa2_q->rx_pkts += num_rx;
 660
 661         return num_rx;
 662 }
 663
 664 void __attribute__((hot))
 665 dpaa2_dev_process_parallel_event(struct qbman_swp *swp,
 666                                  const struct qbman_fd *fd,
 667                                  const struct qbman_result *dq,
 668                                  struct dpaa2_queue *rxq,
 669                                  struct rte_event *ev)
 670 {
 671         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd) +
 672                 DPAA2_FD_PTA_SIZE + 16));
 673
 674         ev->flow_id = rxq->ev.flow_id;
 675         ev->sub_event_type = rxq->ev.sub_event_type;
 676         ev->event_type = RTE_EVENT_TYPE_ETHDEV;
 677         ev->op = RTE_EVENT_OP_NEW;
 678         ev->sched_type = rxq->ev.sched_type;
 679         ev->queue_id = rxq->ev.queue_id;
 680         ev->priority = rxq->ev.priority;
 681
 682         ev->mbuf = eth_fd_to_mbuf(fd);
 683
 684         qbman_swp_dqrr_consume(swp, dq);
 685 }
 686
 687 void __attribute__((hot))
 688 dpaa2_dev_process_atomic_event(struct qbman_swp *swp __attribute__((unused)),
 689                                const struct qbman_fd *fd,
 690                                const struct qbman_result *dq,
 691                                struct dpaa2_queue *rxq,
 692                                struct rte_event *ev)
 693 {
 694         uint8_t dqrr_index;
 695
 696         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd) +
 697                 DPAA2_FD_PTA_SIZE + 16));
 698
 699         ev->flow_id = rxq->ev.flow_id;
 700         ev->sub_event_type = rxq->ev.sub_event_type;
 701         ev->event_type = RTE_EVENT_TYPE_ETHDEV;
 702         ev->op = RTE_EVENT_OP_NEW;
 703         ev->sched_type = rxq->ev.sched_type;
 704         ev->queue_id = rxq->ev.queue_id;
 705         ev->priority = rxq->ev.priority;
 706
 707         ev->mbuf = eth_fd_to_mbuf(fd);
 708
 709         dqrr_index = qbman_get_dqrr_idx(dq);
 710         ev->mbuf->seqn = dqrr_index + 1;
 711         DPAA2_PER_LCORE_DQRR_SIZE++;
 712         DPAA2_PER_LCORE_DQRR_HELD |= 1 << dqrr_index;
 713         DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = ev->mbuf;
 714 }
 715
 716 void __attribute__((hot))
 717 dpaa2_dev_process_ordered_event(struct qbman_swp *swp,
 718                                 const struct qbman_fd *fd,
 719                                 const struct qbman_result *dq,
 720                                 struct dpaa2_queue *rxq,
 721                                 struct rte_event *ev)
 722 {
 723         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd) +
 724                 DPAA2_FD_PTA_SIZE + 16));
 725
 726         ev->flow_id = rxq->ev.flow_id;
 727         ev->sub_event_type = rxq->ev.sub_event_type;
 728         ev->event_type = RTE_EVENT_TYPE_ETHDEV;
 729         ev->op = RTE_EVENT_OP_NEW;
 730         ev->sched_type = rxq->ev.sched_type;
 731         ev->queue_id = rxq->ev.queue_id;
 732         ev->priority = rxq->ev.priority;
 733
 734         ev->mbuf = eth_fd_to_mbuf(fd);
 735
 736         ev->mbuf->seqn = DPAA2_ENQUEUE_FLAG_ORP;
 737         ev->mbuf->seqn |= qbman_result_DQ_odpid(dq) << DPAA2_EQCR_OPRID_SHIFT;
 738         ev->mbuf->seqn |= qbman_result_DQ_seqnum(dq) << DPAA2_EQCR_SEQNUM_SHIFT;
 739
 740         qbman_swp_dqrr_consume(swp, dq);
 741 }
 742
 743 uint16_t
 744 dpaa2_dev_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 745 {
 746         /* Function receive frames for a given device and VQ */
 747         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
 748         struct qbman_result *dq_storage;
 749         uint32_t fqid = dpaa2_q->fqid;
 750         int ret, num_rx = 0, next_pull = nb_pkts, num_pulled;
 751         uint8_t pending, status;
 752         struct qbman_swp *swp;
 753         const struct qbman_fd *fd, *next_fd;
 754         struct qbman_pull_desc pulldesc;
 755         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
 756
 757         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 758                 ret = dpaa2_affine_qbman_swp();
 759                 if (ret) {
 760                         DPAA2_PMD_ERR("Failure in affining portal\n");
 761                         return 0;
 762                 }
 763         }
 764         swp = DPAA2_PER_LCORE_PORTAL;
 765
 766         do {
 767                 dq_storage = dpaa2_q->q_storage->dq_storage[0];
 768                 qbman_pull_desc_clear(&pulldesc);
 769                 qbman_pull_desc_set_fq(&pulldesc, fqid);
 770                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
 771                                 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
 772
 773                 if (next_pull > dpaa2_dqrr_size) {
 774                         qbman_pull_desc_set_numframes(&pulldesc,
 775                                 dpaa2_dqrr_size);
 776                         next_pull -= dpaa2_dqrr_size;
 777                 } else {
 778                         qbman_pull_desc_set_numframes(&pulldesc, next_pull);
 779                         next_pull = 0;
 780                 }
 781
 782                 while (1) {
 783                         if (qbman_swp_pull(swp, &pulldesc)) {
 784                                 DPAA2_PMD_DP_DEBUG(
 785                                         "VDQ command is not issued.QBMAN is busy\n");
 786                                 /* Portal was busy, try again */
 787                                 continue;
 788                         }
 789                         break;
 790                 }
 791
 792                 rte_prefetch0((void *)((size_t)(dq_storage + 1)));
 793                 /* Check if the previous issued command is completed. */
 794                 while (!qbman_check_command_complete(dq_storage))
 795                         ;
 796
 797                 num_pulled = 0;
 798                 pending = 1;
 799                 do {
 800                         /* Loop until the dq_storage is updated with
 801                          * new token by QBMAN
 802                          */
 803                         while (!qbman_check_new_result(dq_storage))
 804                                 ;
 805                         rte_prefetch0((void *)((size_t)(dq_storage + 2)));
 806                         /* Check whether Last Pull command is Expired and
 807                          * setting Condition for Loop termination
 808                          */
 809                         if (qbman_result_DQ_is_pull_complete(dq_storage)) {
 810                                 pending = 0;
 811                                 /* Check for valid frame. */
 812                                 status = qbman_result_DQ_flags(dq_storage);
 813                                 if (unlikely((status &
 814                                         QBMAN_DQ_STAT_VALIDFRAME) == 0))
 815                                         continue;
 816                         }
 817                         fd = qbman_result_DQ_fd(dq_storage);
 818
 819                         next_fd = qbman_result_DQ_fd(dq_storage + 1);
 820                         /* Prefetch Annotation address for the parse results */
 821                         rte_prefetch0(
 822                                 (void *)(size_t)(DPAA2_GET_FD_ADDR(next_fd)
 823                                         + DPAA2_FD_PTA_SIZE + 16));
 824
 825                         if (unlikely(DPAA2_FD_GET_FORMAT(fd) == qbman_fd_sg))
 826                                 bufs[num_rx] = eth_sg_fd_to_mbuf(fd);
 827                         else
 828                                 bufs[num_rx] = eth_fd_to_mbuf(fd);
 829                         bufs[num_rx]->port = eth_data->port_id;
 830
 831                 if (eth_data->dev_conf.rxmode.offloads &
 832                                 DEV_RX_OFFLOAD_VLAN_STRIP) {
 833                         rte_vlan_strip(bufs[num_rx]);
 834                 }
 835
 836                         dq_storage++;
 837                         num_rx++;
 838                         num_pulled++;
 839                 } while (pending);
 840         /* Last VDQ provided all packets and more packets are requested */
 841         } while (next_pull && num_pulled == dpaa2_dqrr_size);
 842
 843         dpaa2_q->rx_pkts += num_rx;
 844
 845         return num_rx;
 846 }
 847
 848 uint16_t dpaa2_dev_tx_conf(void *queue)
 849 {
 850         /* Function receive frames for a given device and VQ */
 851         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
 852         struct qbman_result *dq_storage;
 853         uint32_t fqid = dpaa2_q->fqid;
 854         int ret, num_tx_conf = 0, num_pulled;
 855         uint8_t pending, status;
 856         struct qbman_swp *swp;
 857         const struct qbman_fd *fd, *next_fd;
 858         struct qbman_pull_desc pulldesc;
 859         struct qbman_release_desc releasedesc;
 860         uint32_t bpid;
 861         uint64_t buf;
 862 #if defined(RTE_LIBRTE_IEEE1588)
 863         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
 864         struct dpaa2_dev_priv *priv = eth_data->dev_private;
 865         struct dpaa2_annot_hdr *annotation;
 866 #endif
 867
 868         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 869                 ret = dpaa2_affine_qbman_swp();
 870                 if (ret) {
 871                         DPAA2_PMD_ERR("Failure in affining portal\n");
 872                         return 0;
 873                 }
 874         }
 875         swp = DPAA2_PER_LCORE_PORTAL;
 876
 877         do {
 878                 dq_storage = dpaa2_q->q_storage->dq_storage[0];
 879                 qbman_pull_desc_clear(&pulldesc);
 880                 qbman_pull_desc_set_fq(&pulldesc, fqid);
 881                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
 882                                 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
 883
 884                 qbman_pull_desc_set_numframes(&pulldesc, dpaa2_dqrr_size);
 885
 886                 while (1) {
 887                         if (qbman_swp_pull(swp, &pulldesc)) {
 888                                 DPAA2_PMD_DP_DEBUG("VDQ command is not issued."
 889                                                    "QBMAN is busy\n");
 890                                 /* Portal was busy, try again */
 891                                 continue;
 892                         }
 893                         break;
 894                 }
 895
 896                 rte_prefetch0((void *)((size_t)(dq_storage + 1)));
 897                 /* Check if the previous issued command is completed. */
 898                 while (!qbman_check_command_complete(dq_storage))
 899                         ;
 900
 901                 num_pulled = 0;
 902                 pending = 1;
 903                 do {
 904                         /* Loop until the dq_storage is updated with
 905                          * new token by QBMAN
 906                          */
 907                         while (!qbman_check_new_result(dq_storage))
 908                                 ;
 909                         rte_prefetch0((void *)((size_t)(dq_storage + 2)));
 910                         /* Check whether Last Pull command is Expired and
 911                          * setting Condition for Loop termination
 912                          */
 913                         if (qbman_result_DQ_is_pull_complete(dq_storage)) {
 914                                 pending = 0;
 915                                 /* Check for valid frame. */
 916                                 status = qbman_result_DQ_flags(dq_storage);
 917                                 if (unlikely((status &
 918                                         QBMAN_DQ_STAT_VALIDFRAME) == 0))
 919                                         continue;
 920                         }
 921                         fd = qbman_result_DQ_fd(dq_storage);
 922
 923                         next_fd = qbman_result_DQ_fd(dq_storage + 1);
 924                         /* Prefetch Annotation address for the parse results */
 925                         rte_prefetch0((void *)(size_t)
 926                                 (DPAA2_GET_FD_ADDR(next_fd) +
 927                                  DPAA2_FD_PTA_SIZE + 16));
 928
 929                         bpid = DPAA2_GET_FD_BPID(fd);
 930
 931                         /* Create a release descriptor required for releasing
 932                          * buffers into QBMAN
 933                          */
 934                         qbman_release_desc_clear(&releasedesc);
 935                         qbman_release_desc_set_bpid(&releasedesc, bpid);
 936
 937                         buf = DPAA2_GET_FD_ADDR(fd);
 938                         /* feed them to bman */
 939                         do {
 940                                 ret = qbman_swp_release(swp, &releasedesc,
 941                                                         &buf, 1);
 942                         } while (ret == -EBUSY);
 943
 944                         dq_storage++;
 945                         num_tx_conf++;
 946                         num_pulled++;
 947 #if defined(RTE_LIBRTE_IEEE1588)
 948                         annotation = (struct dpaa2_annot_hdr *)((size_t)
 949                                 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) +
 950                                 DPAA2_FD_PTA_SIZE);
 951                         priv->tx_timestamp = annotation->word2;
 952 #endif
 953                 } while (pending);
 954
 955         /* Last VDQ provided all packets and more packets are requested */
 956         } while (num_pulled == dpaa2_dqrr_size);
 957
 958         dpaa2_q->rx_pkts += num_tx_conf;
 959
 960         return num_tx_conf;
 961 }
 962
 963 /* Configure the egress frame annotation for timestamp update */
 964 static void enable_tx_tstamp(struct qbman_fd *fd)
 965 {
 966         struct dpaa2_faead *fd_faead;
 967
 968         /* Set frame annotation status field as valid */
 969         (fd)->simple.frc |= DPAA2_FD_FRC_FASV;
 970
 971         /* Set frame annotation egress action descriptor as valid */
 972         (fd)->simple.frc |= DPAA2_FD_FRC_FAEADV;
 973
 974         /* Set Annotation Length as 128B */
 975         (fd)->simple.ctrl |= DPAA2_FD_CTRL_ASAL;
 976
 977         /* enable update of confirmation frame annotation */
 978         fd_faead = (struct dpaa2_faead *)((size_t)
 979                         DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) +
 980                         DPAA2_FD_PTA_SIZE + DPAA2_FD_HW_ANNOT_FAEAD_OFFSET);
 981         fd_faead->ctrl = DPAA2_ANNOT_FAEAD_A2V | DPAA2_ANNOT_FAEAD_UPDV |
 982                                 DPAA2_ANNOT_FAEAD_UPD;
 983 }
 984
 985 /*
 986  * Callback to handle sending packets through WRIOP based interface
 987  */
 988 uint16_t
 989 dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 990 {
 991         /* Function to transmit the frames to given device and VQ*/
 992         uint32_t loop, retry_count;
 993         int32_t ret;
 994         struct qbman_fd fd_arr[MAX_TX_RING_SLOTS];
 995         struct rte_mbuf *mi;
 996         uint32_t frames_to_send;
 997         struct rte_mempool *mp;
 998         struct qbman_eq_desc eqdesc;
 999         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
1000         struct qbman_swp *swp;
1001         uint16_t num_tx = 0;
1002         uint16_t bpid;
1003         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
1004         struct dpaa2_dev_priv *priv = eth_data->dev_private;
1005         uint32_t flags[MAX_TX_RING_SLOTS] = {0};
1006
1007         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
1008                 ret = dpaa2_affine_qbman_swp();
1009                 if (ret) {
1010                         DPAA2_PMD_ERR("Failure in affining portal");
1011                         return 0;
1012                 }
1013         }
1014         swp = DPAA2_PER_LCORE_PORTAL;
1015
1016         DPAA2_PMD_DP_DEBUG("===> eth_data =%p, fqid =%d\n",
1017                         eth_data, dpaa2_q->fqid);
1018
1019 #ifdef RTE_LIBRTE_IEEE1588
1020         /* IEEE1588 driver need pointer to tx confirmation queue
1021          * corresponding to last packet transmitted for reading
1022          * the timestamp
1023          */
1024         priv->next_tx_conf_queue = dpaa2_q->tx_conf_queue;
1025         dpaa2_dev_tx_conf(dpaa2_q->tx_conf_queue);
1026 #endif
1027
1028         /*Prepare enqueue descriptor*/
1029         qbman_eq_desc_clear(&eqdesc);
1030         qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ);
1031         qbman_eq_desc_set_fq(&eqdesc, dpaa2_q->fqid);
1032
1033         /*Clear the unused FD fields before sending*/
1034         while (nb_pkts) {
1035                 /*Check if the queue is congested*/
1036                 retry_count = 0;
1037                 while (qbman_result_SCN_state(dpaa2_q->cscn)) {
1038                         retry_count++;
1039                         /* Retry for some time before giving up */
1040                         if (retry_count > CONG_RETRY_COUNT)
1041                                 goto skip_tx;
1042                 }
1043
1044                 frames_to_send = (nb_pkts > dpaa2_eqcr_size) ?
1045                         dpaa2_eqcr_size : nb_pkts;
1046
1047                 for (loop = 0; loop < frames_to_send; loop++) {
1048                         if ((*bufs)->seqn) {
1049                                 uint8_t dqrr_index = (*bufs)->seqn - 1;
1050
1051                                 flags[loop] = QBMAN_ENQUEUE_FLAG_DCA |
1052                                                 dqrr_index;
1053                                 DPAA2_PER_LCORE_DQRR_SIZE--;
1054                                 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dqrr_index);
1055                                 (*bufs)->seqn = DPAA2_INVALID_MBUF_SEQN;
1056                         }
1057
1058                         if (likely(RTE_MBUF_DIRECT(*bufs))) {
1059                                 mp = (*bufs)->pool;
1060                                 /* Check the basic scenario and set
1061                                  * the FD appropriately here itself.
1062                                  */
1063                                 if (likely(mp && mp->ops_index ==
1064                                     priv->bp_list->dpaa2_ops_index &&
1065                                     (*bufs)->nb_segs == 1 &&
1066                                     rte_mbuf_refcnt_read((*bufs)) == 1)) {
1067                                         if (unlikely(((*bufs)->ol_flags
1068                                                 & PKT_TX_VLAN_PKT) ||
1069                                                 (eth_data->dev_conf.txmode.offloads
1070                                                 & DEV_TX_OFFLOAD_VLAN_INSERT))) {
1071                                                 ret = rte_vlan_insert(bufs);
1072                                                 if (ret)
1073                                                         goto send_n_return;
1074                                         }
1075                                         DPAA2_MBUF_TO_CONTIG_FD((*bufs),
1076                                         &fd_arr[loop], mempool_to_bpid(mp));
1077                                         bufs++;
1078 #ifdef RTE_LIBRTE_IEEE1588
1079                                         enable_tx_tstamp(&fd_arr[loop]);
1080 #endif
1081                                         continue;
1082                                 }
1083                         } else {
1084                                 mi = rte_mbuf_from_indirect(*bufs);
1085                                 mp = mi->pool;
1086                         }
1087                         /* Not a hw_pkt pool allocated frame */
1088                         if (unlikely(!mp || !priv->bp_list)) {
1089                                 DPAA2_PMD_ERR("Err: No buffer pool attached");
1090                                 goto send_n_return;
1091                         }
1092
1093                         if (unlikely(((*bufs)->ol_flags & PKT_TX_VLAN_PKT) ||
1094                                 (eth_data->dev_conf.txmode.offloads
1095                                 & DEV_TX_OFFLOAD_VLAN_INSERT))) {
1096                                 int ret = rte_vlan_insert(bufs);
1097                                 if (ret)
1098                                         goto send_n_return;
1099                         }
1100                         if (mp->ops_index != priv->bp_list->dpaa2_ops_index) {
1101                                 DPAA2_PMD_WARN("Non DPAA2 buffer pool");
1102                                 /* alloc should be from the default buffer pool
1103                                  * attached to this interface
1104                                  */
1105                                 bpid = priv->bp_list->buf_pool.bpid;
1106
1107                                 if (unlikely((*bufs)->nb_segs > 1)) {
1108                                         DPAA2_PMD_ERR("S/G support not added"
1109                                                 " for non hw offload buffer");
1110                                         goto send_n_return;
1111                                 }
1112                                 if (eth_copy_mbuf_to_fd(*bufs,
1113                                                         &fd_arr[loop], bpid)) {
1114                                         goto send_n_return;
1115                                 }
1116                                 /* free the original packet */
1117                                 rte_pktmbuf_free(*bufs);
1118                         } else {
1119                                 bpid = mempool_to_bpid(mp);
1120                                 if (unlikely((*bufs)->nb_segs > 1)) {
1121                                         if (eth_mbuf_to_sg_fd(*bufs,
1122                                                         &fd_arr[loop], bpid))
1123                                                 goto send_n_return;
1124                                 } else {
1125                                         eth_mbuf_to_fd(*bufs,
1126                                                        &fd_arr[loop], bpid);
1127                                 }
1128                         }
1129 #ifdef RTE_LIBRTE_IEEE1588
1130                         enable_tx_tstamp(&fd_arr[loop]);
1131 #endif
1132                         bufs++;
1133                 }
1134                 loop = 0;
1135                 while (loop < frames_to_send) {
1136                         loop += qbman_swp_enqueue_multiple(swp, &eqdesc,
1137                                         &fd_arr[loop], &flags[loop],
1138                                         frames_to_send - loop);
1139                 }
1140
1141                 num_tx += frames_to_send;
1142                 nb_pkts -= frames_to_send;
1143         }
1144         dpaa2_q->tx_pkts += num_tx;
1145         return num_tx;
1146
1147 send_n_return:
1148         /* send any already prepared fd */
1149         if (loop) {
1150                 unsigned int i = 0;
1151
1152                 while (i < loop) {
1153                         i += qbman_swp_enqueue_multiple(swp, &eqdesc,
1154                                                         &fd_arr[i],
1155                                                         &flags[loop],
1156                                                         loop - i);
1157                 }
1158                 num_tx += loop;
1159         }
1160 skip_tx:
1161         dpaa2_q->tx_pkts += num_tx;
1162         return num_tx;
1163 }
1164
1165 void
1166 dpaa2_dev_free_eqresp_buf(uint16_t eqresp_ci)
1167 {
1168         struct dpaa2_dpio_dev *dpio_dev = DPAA2_PER_LCORE_DPIO;
1169         struct qbman_fd *fd;
1170         struct rte_mbuf *m;
1171
1172         fd = qbman_result_eqresp_fd(&dpio_dev->eqresp[eqresp_ci]);
1173         m = eth_fd_to_mbuf(fd);
1174         rte_pktmbuf_free(m);
1175 }
1176
1177 static void
1178 dpaa2_set_enqueue_descriptor(struct dpaa2_queue *dpaa2_q,
1179                              struct rte_mbuf *m,
1180                              struct qbman_eq_desc *eqdesc)
1181 {
1182         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
1183         struct dpaa2_dev_priv *priv = eth_data->dev_private;
1184         struct dpaa2_dpio_dev *dpio_dev = DPAA2_PER_LCORE_DPIO;
1185         struct eqresp_metadata *eqresp_meta;
1186         uint16_t orpid, seqnum;
1187         uint8_t dq_idx;
1188
1189         qbman_eq_desc_set_fq(eqdesc, dpaa2_q->fqid);
1190
1191         if (m->seqn & DPAA2_ENQUEUE_FLAG_ORP) {
1192                 orpid = (m->seqn & DPAA2_EQCR_OPRID_MASK) >>
1193                         DPAA2_EQCR_OPRID_SHIFT;
1194                 seqnum = (m->seqn & DPAA2_EQCR_SEQNUM_MASK) >>
1195                         DPAA2_EQCR_SEQNUM_SHIFT;
1196
1197                 if (!priv->en_loose_ordered) {
1198                         qbman_eq_desc_set_orp(eqdesc, 1, orpid, seqnum, 0);
1199                         qbman_eq_desc_set_response(eqdesc, (uint64_t)
1200                                 DPAA2_VADDR_TO_IOVA(&dpio_dev->eqresp[
1201                                 dpio_dev->eqresp_pi]), 1);
1202                         qbman_eq_desc_set_token(eqdesc, 1);
1203
1204                         eqresp_meta = &dpio_dev->eqresp_meta[
1205                                 dpio_dev->eqresp_pi];
1206                         eqresp_meta->dpaa2_q = dpaa2_q;
1207                         eqresp_meta->mp = m->pool;
1208
1209                         dpio_dev->eqresp_pi + 1 < MAX_EQ_RESP_ENTRIES ?
1210                                 dpio_dev->eqresp_pi++ :
1211                                 (dpio_dev->eqresp_pi = 0);
1212                 } else {
1213                         qbman_eq_desc_set_orp(eqdesc, 0, orpid, seqnum, 0);
1214                 }
1215         } else {
1216                 dq_idx = m->seqn - 1;
1217                 qbman_eq_desc_set_dca(eqdesc, 1, dq_idx, 0);
1218                 DPAA2_PER_LCORE_DQRR_SIZE--;
1219                 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dq_idx);
1220         }
1221         m->seqn = DPAA2_INVALID_MBUF_SEQN;
1222 }
1223
1224 /* Callback to handle sending ordered packets through WRIOP based interface */
1225 uint16_t
1226 dpaa2_dev_tx_ordered(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
1227 {
1228         /* Function to transmit the frames to given device and VQ*/
1229         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
1230         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
1231         struct dpaa2_dev_priv *priv = eth_data->dev_private;
1232         struct dpaa2_queue *order_sendq = (struct dpaa2_queue *)priv->tx_vq[0];
1233         struct qbman_fd fd_arr[MAX_TX_RING_SLOTS];
1234         struct rte_mbuf *mi;
1235         struct rte_mempool *mp;
1236         struct qbman_eq_desc eqdesc[MAX_TX_RING_SLOTS];
1237         struct qbman_swp *swp;
1238         uint32_t frames_to_send, num_free_eq_desc;
1239         uint32_t loop, retry_count;
1240         int32_t ret;
1241         uint16_t num_tx = 0;
1242         uint16_t bpid;
1243
1244         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
1245                 ret = dpaa2_affine_qbman_swp();
1246                 if (ret) {
1247                         DPAA2_PMD_ERR("Failure in affining portal");
1248                         return 0;
1249                 }
1250         }
1251         swp = DPAA2_PER_LCORE_PORTAL;
1252
1253         DPAA2_PMD_DP_DEBUG("===> eth_data =%p, fqid =%d\n",
1254                            eth_data, dpaa2_q->fqid);
1255
1256         /* This would also handle normal and atomic queues as any type
1257          * of packet can be enqueued when ordered queues are being used.
1258          */
1259         while (nb_pkts) {
1260                 /*Check if the queue is congested*/
1261                 retry_count = 0;
1262                 while (qbman_result_SCN_state(dpaa2_q->cscn)) {
1263                         retry_count++;
1264                         /* Retry for some time before giving up */
1265                         if (retry_count > CONG_RETRY_COUNT)
1266                                 goto skip_tx;
1267                 }
1268
1269                 frames_to_send = (nb_pkts > dpaa2_eqcr_size) ?
1270                         dpaa2_eqcr_size : nb_pkts;
1271
1272                 if (!priv->en_loose_ordered) {
1273                         if ((*bufs)->seqn & DPAA2_ENQUEUE_FLAG_ORP) {
1274                                 num_free_eq_desc = dpaa2_free_eq_descriptors();
1275                                 if (num_free_eq_desc < frames_to_send)
1276                                         frames_to_send = num_free_eq_desc;
1277                         }
1278                 }
1279
1280                 for (loop = 0; loop < frames_to_send; loop++) {
1281                         /*Prepare enqueue descriptor*/
1282                         qbman_eq_desc_clear(&eqdesc[loop]);
1283
1284                         if ((*bufs)->seqn) {
1285                                 /* Use only queue 0 for Tx in case of atomic/
1286                                  * ordered packets as packets can get unordered
1287                                  * when being tranmitted out from the interface
1288                                  */
1289                                 dpaa2_set_enqueue_descriptor(order_sendq,
1290                                                              (*bufs),
1291                                                              &eqdesc[loop]);
1292                         } else {
1293                                 qbman_eq_desc_set_no_orp(&eqdesc[loop],
1294                                                          DPAA2_EQ_RESP_ERR_FQ);
1295                                 qbman_eq_desc_set_fq(&eqdesc[loop],
1296                                                      dpaa2_q->fqid);
1297                         }
1298
1299                         if (likely(RTE_MBUF_DIRECT(*bufs))) {
1300                                 mp = (*bufs)->pool;
1301                                 /* Check the basic scenario and set
1302                                  * the FD appropriately here itself.
1303                                  */
1304                                 if (likely(mp && mp->ops_index ==
1305                                     priv->bp_list->dpaa2_ops_index &&
1306                                     (*bufs)->nb_segs == 1 &&
1307                                     rte_mbuf_refcnt_read((*bufs)) == 1)) {
1308                                         if (unlikely((*bufs)->ol_flags
1309                                                 & PKT_TX_VLAN_PKT)) {
1310                                           ret = rte_vlan_insert(bufs);
1311                                           if (ret)
1312                                                 goto send_n_return;
1313                                         }
1314                                         DPAA2_MBUF_TO_CONTIG_FD((*bufs),
1315                                                 &fd_arr[loop],
1316                                                 mempool_to_bpid(mp));
1317                                         bufs++;
1318                                         continue;
1319                                 }
1320                         } else {
1321                                 mi = rte_mbuf_from_indirect(*bufs);
1322                                 mp = mi->pool;
1323                         }
1324                         /* Not a hw_pkt pool allocated frame */
1325                         if (unlikely(!mp || !priv->bp_list)) {
1326                                 DPAA2_PMD_ERR("Err: No buffer pool attached");
1327                                 goto send_n_return;
1328                         }
1329
1330                         if (mp->ops_index != priv->bp_list->dpaa2_ops_index) {
1331                                 DPAA2_PMD_WARN("Non DPAA2 buffer pool");
1332                                 /* alloc should be from the default buffer pool
1333                                  * attached to this interface
1334                                  */
1335                                 bpid = priv->bp_list->buf_pool.bpid;
1336
1337                                 if (unlikely((*bufs)->nb_segs > 1)) {
1338                                         DPAA2_PMD_ERR(
1339                                                 "S/G not supp for non hw offload buffer");
1340                                         goto send_n_return;
1341                                 }
1342                                 if (eth_copy_mbuf_to_fd(*bufs,
1343                                                         &fd_arr[loop], bpid)) {
1344                                         goto send_n_return;
1345                                 }
1346                                 /* free the original packet */
1347                                 rte_pktmbuf_free(*bufs);
1348                         } else {
1349                                 bpid = mempool_to_bpid(mp);
1350                                 if (unlikely((*bufs)->nb_segs > 1)) {
1351                                         if (eth_mbuf_to_sg_fd(*bufs,
1352                                                               &fd_arr[loop],
1353                                                               bpid))
1354                                                 goto send_n_return;
1355                                 } else {
1356                                         eth_mbuf_to_fd(*bufs,
1357                                                        &fd_arr[loop], bpid);
1358                                 }
1359                         }
1360                         bufs++;
1361                 }
1362                 loop = 0;
1363                 while (loop < frames_to_send) {
1364                         loop += qbman_swp_enqueue_multiple_desc(swp,
1365                                         &eqdesc[loop], &fd_arr[loop],
1366                                         frames_to_send - loop);
1367                 }
1368
1369                 num_tx += frames_to_send;
1370                 nb_pkts -= frames_to_send;
1371         }
1372         dpaa2_q->tx_pkts += num_tx;
1373         return num_tx;
1374
1375 send_n_return:
1376         /* send any already prepared fd */
1377         if (loop) {
1378                 unsigned int i = 0;
1379
1380                 while (i < loop) {
1381                         i += qbman_swp_enqueue_multiple_desc(swp, &eqdesc[loop],
1382                                                         &fd_arr[i], loop - i);
1383                 }
1384                 num_tx += loop;
1385         }
1386 skip_tx:
1387         dpaa2_q->tx_pkts += num_tx;
1388         return num_tx;
1389 }
1390
1391 /**
1392  * Dummy DPDK callback for TX.
1393  *
1394  * This function is used to temporarily replace the real callback during
1395  * unsafe control operations on the queue, or in case of error.
1396  *
1397  * @param dpdk_txq
1398  *   Generic pointer to TX queue structure.
1399  * @param[in] pkts
1400  *   Packets to transmit.
1401  * @param pkts_n
1402  *   Number of packets in array.
1403  *
1404  * @return
1405  *   Number of packets successfully transmitted (<= pkts_n).
1406  */
1407 uint16_t
1408 dummy_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
1409 {
1410         (void)queue;
1411         (void)bufs;
1412         (void)nb_pkts;
1413         return 0;
1414 }
1415
1416 #if defined(RTE_TOOLCHAIN_GCC)
1417 #pragma GCC diagnostic push
1418 #pragma GCC diagnostic ignored "-Wcast-qual"
1419 #elif defined(RTE_TOOLCHAIN_CLANG)
1420 #pragma clang diagnostic push
1421 #pragma clang diagnostic ignored "-Wcast-qual"
1422 #endif
1423
1424 /* This function loopbacks all the received packets.*/
1425 uint16_t
1426 dpaa2_dev_loopback_rx(void *queue,
1427                       struct rte_mbuf **bufs __rte_unused,
1428                       uint16_t nb_pkts)
1429 {
1430         /* Function receive frames for a given device and VQ*/
1431         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
1432         struct qbman_result *dq_storage, *dq_storage1 = NULL;
1433         uint32_t fqid = dpaa2_q->fqid;
1434         int ret, num_rx = 0, num_tx = 0, pull_size;
1435         uint8_t pending, status;
1436         struct qbman_swp *swp;
1437         struct qbman_fd *fd[DPAA2_LX2_DQRR_RING_SIZE];
1438         struct qbman_pull_desc pulldesc;
1439         struct qbman_eq_desc eqdesc;
1440         struct queue_storage_info_t *q_storage = dpaa2_q->q_storage;
1441         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
1442         struct dpaa2_dev_priv *priv = eth_data->dev_private;
1443         struct dpaa2_queue *tx_q = priv->tx_vq[0];
1444         /* todo - currently we are using 1st TX queue only for loopback*/
1445
1446         if (unlikely(!DPAA2_PER_LCORE_ETHRX_DPIO)) {
1447                 ret = dpaa2_affine_qbman_ethrx_swp();
1448                 if (ret) {
1449                         DPAA2_PMD_ERR("Failure in affining portal");
1450                         return 0;
1451                 }
1452         }
1453         swp = DPAA2_PER_LCORE_ETHRX_PORTAL;
1454         pull_size = (nb_pkts > dpaa2_dqrr_size) ? dpaa2_dqrr_size : nb_pkts;
1455         if (unlikely(!q_storage->active_dqs)) {
1456                 q_storage->toggle = 0;
1457                 dq_storage = q_storage->dq_storage[q_storage->toggle];
1458                 q_storage->last_num_pkts = pull_size;
1459                 qbman_pull_desc_clear(&pulldesc);
1460                 qbman_pull_desc_set_numframes(&pulldesc,
1461                                               q_storage->last_num_pkts);
1462                 qbman_pull_desc_set_fq(&pulldesc, fqid);
1463                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
1464                         (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
1465                 if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) {
1466                         while (!qbman_check_command_complete(
1467                                get_swp_active_dqs(
1468                                DPAA2_PER_LCORE_ETHRX_DPIO->index)))
1469                                 ;
1470                         clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index);
1471                 }
1472                 while (1) {
1473                         if (qbman_swp_pull(swp, &pulldesc)) {
1474                                 DPAA2_PMD_DP_DEBUG(
1475                                         "VDQ command not issued.QBMAN busy\n");
1476                                 /* Portal was busy, try again */
1477                                 continue;
1478                         }
1479                         break;
1480                 }
1481                 q_storage->active_dqs = dq_storage;
1482                 q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index;
1483                 set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index,
1484                                    dq_storage);
1485         }
1486
1487         dq_storage = q_storage->active_dqs;
1488         rte_prefetch0((void *)(size_t)(dq_storage));
1489         rte_prefetch0((void *)(size_t)(dq_storage + 1));
1490
1491         /* Prepare next pull descriptor. This will give space for the
1492          * prefething done on DQRR entries
1493          */
1494         q_storage->toggle ^= 1;
1495         dq_storage1 = q_storage->dq_storage[q_storage->toggle];
1496         qbman_pull_desc_clear(&pulldesc);
1497         qbman_pull_desc_set_numframes(&pulldesc, pull_size);
1498         qbman_pull_desc_set_fq(&pulldesc, fqid);
1499         qbman_pull_desc_set_storage(&pulldesc, dq_storage1,
1500                 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage1)), 1);
1501
1502         /*Prepare enqueue descriptor*/
1503         qbman_eq_desc_clear(&eqdesc);
1504         qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ);
1505         qbman_eq_desc_set_response(&eqdesc, 0, 0);
1506         qbman_eq_desc_set_fq(&eqdesc, tx_q->fqid);
1507
1508         /* Check if the previous issued command is completed.
1509          * Also seems like the SWP is shared between the Ethernet Driver
1510          * and the SEC driver.
1511          */
1512         while (!qbman_check_command_complete(dq_storage))
1513                 ;
1514         if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id))
1515                 clear_swp_active_dqs(q_storage->active_dpio_id);
1516
1517         pending = 1;
1518
1519         do {
1520                 /* Loop until the dq_storage is updated with
1521                  * new token by QBMAN
1522                  */
1523                 while (!qbman_check_new_result(dq_storage))
1524                         ;
1525                 rte_prefetch0((void *)((size_t)(dq_storage + 2)));
1526                 /* Check whether Last Pull command is Expired and
1527                  * setting Condition for Loop termination
1528                  */
1529                 if (qbman_result_DQ_is_pull_complete(dq_storage)) {
1530                         pending = 0;
1531                         /* Check for valid frame. */
1532                         status = qbman_result_DQ_flags(dq_storage);
1533                         if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0))
1534                                 continue;
1535                 }
1536                 fd[num_rx] = (struct qbman_fd *)qbman_result_DQ_fd(dq_storage);
1537
1538                 dq_storage++;
1539                 num_rx++;
1540         } while (pending);
1541
1542         while (num_tx < num_rx) {
1543                 num_tx += qbman_swp_enqueue_multiple_fd(swp, &eqdesc,
1544                                 &fd[num_tx], 0, num_rx - num_tx);
1545         }
1546
1547         if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) {
1548                 while (!qbman_check_command_complete(
1549                        get_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)))
1550                         ;
1551                 clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index);
1552         }
1553         /* issue a volatile dequeue command for next pull */
1554         while (1) {
1555                 if (qbman_swp_pull(swp, &pulldesc)) {
1556                         DPAA2_PMD_DP_DEBUG("VDQ command is not issued."
1557                                           "QBMAN is busy (2)\n");
1558                         continue;
1559                 }
1560                 break;
1561         }
1562         q_storage->active_dqs = dq_storage1;
1563         q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index;
1564         set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index, dq_storage1);
1565
1566         dpaa2_q->rx_pkts += num_rx;
1567         dpaa2_q->tx_pkts += num_tx;
1568
1569         return 0;
1570 }
1571 #if defined(RTE_TOOLCHAIN_GCC)
1572 #pragma GCC diagnostic pop
1573 #elif defined(RTE_TOOLCHAIN_CLANG)
1574 #pragma clang diagnostic pop
1575 #endif