drivers/net/dpaa2/dpaa2_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
   4  *   Copyright 2016-2020 NXP
   5  *
   6  */
   7
   8 #include <time.h>
   9 #include <net/if.h>
  10
  11 #include <rte_mbuf.h>
  12 #include <rte_ethdev_driver.h>
  13 #include <rte_malloc.h>
  14 #include <rte_memcpy.h>
  15 #include <rte_string_fns.h>
  16 #include <rte_dev.h>
  17
  18 #include <rte_fslmc.h>
  19 #include <fslmc_vfio.h>
  20 #include <dpaa2_hw_pvt.h>
  21 #include <dpaa2_hw_dpio.h>
  22 #include <dpaa2_hw_mempool.h>
  23
  24 #include "dpaa2_pmd_logs.h"
  25 #include "dpaa2_ethdev.h"
  26 #include "base/dpaa2_hw_dpni_annot.h"
  27
  28 static inline uint32_t __rte_hot
  29 dpaa2_dev_rx_parse_slow(struct rte_mbuf *mbuf,
  30                         struct dpaa2_annot_hdr *annotation);
  31
  32 static void enable_tx_tstamp(struct qbman_fd *fd) __rte_unused;
  33
  34 #define DPAA2_MBUF_TO_CONTIG_FD(_mbuf, _fd, _bpid)  do { \
  35         DPAA2_SET_FD_ADDR(_fd, DPAA2_MBUF_VADDR_TO_IOVA(_mbuf)); \
  36         DPAA2_SET_FD_LEN(_fd, _mbuf->data_len); \
  37         DPAA2_SET_ONLY_FD_BPID(_fd, _bpid); \
  38         DPAA2_SET_FD_OFFSET(_fd, _mbuf->data_off); \
  39         DPAA2_SET_FD_FRC(_fd, 0);               \
  40         DPAA2_RESET_FD_CTRL(_fd);               \
  41         DPAA2_RESET_FD_FLC(_fd);                \
  42 } while (0)
  43
  44 static inline void __rte_hot
  45 dpaa2_dev_rx_parse_new(struct rte_mbuf *m, const struct qbman_fd *fd)
  46 {
  47         struct dpaa2_annot_hdr *annotation;
  48         uint16_t frc = DPAA2_GET_FD_FRC_PARSE_SUM(fd);
  49
  50         m->packet_type = RTE_PTYPE_UNKNOWN;
  51         switch (frc) {
  52         case DPAA2_PKT_TYPE_ETHER:
  53                 m->packet_type = RTE_PTYPE_L2_ETHER;
  54                 break;
  55         case DPAA2_PKT_TYPE_IPV4:
  56                 m->packet_type = RTE_PTYPE_L2_ETHER |
  57                         RTE_PTYPE_L3_IPV4;
  58                 break;
  59         case DPAA2_PKT_TYPE_IPV6:
  60                 m->packet_type = RTE_PTYPE_L2_ETHER |
  61                         RTE_PTYPE_L3_IPV6;
  62                 break;
  63         case DPAA2_PKT_TYPE_IPV4_EXT:
  64                 m->packet_type = RTE_PTYPE_L2_ETHER |
  65                         RTE_PTYPE_L3_IPV4_EXT;
  66                 break;
  67         case DPAA2_PKT_TYPE_IPV6_EXT:
  68                 m->packet_type = RTE_PTYPE_L2_ETHER |
  69                         RTE_PTYPE_L3_IPV6_EXT;
  70                 break;
  71         case DPAA2_PKT_TYPE_IPV4_TCP:
  72                 m->packet_type = RTE_PTYPE_L2_ETHER |
  73                         RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP;
  74                 break;
  75         case DPAA2_PKT_TYPE_IPV6_TCP:
  76                 m->packet_type = RTE_PTYPE_L2_ETHER |
  77                         RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP;
  78                 break;
  79         case DPAA2_PKT_TYPE_IPV4_UDP:
  80                 m->packet_type = RTE_PTYPE_L2_ETHER |
  81                         RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP;
  82                 break;
  83         case DPAA2_PKT_TYPE_IPV6_UDP:
  84                 m->packet_type = RTE_PTYPE_L2_ETHER |
  85                         RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP;
  86                 break;
  87         case DPAA2_PKT_TYPE_IPV4_SCTP:
  88                 m->packet_type = RTE_PTYPE_L2_ETHER |
  89                         RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP;
  90                 break;
  91         case DPAA2_PKT_TYPE_IPV6_SCTP:
  92                 m->packet_type = RTE_PTYPE_L2_ETHER |
  93                         RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP;
  94                 break;
  95         case DPAA2_PKT_TYPE_IPV4_ICMP:
  96                 m->packet_type = RTE_PTYPE_L2_ETHER |
  97                         RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_ICMP;
  98                 break;
  99         case DPAA2_PKT_TYPE_IPV6_ICMP:
 100                 m->packet_type = RTE_PTYPE_L2_ETHER |
 101                         RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_ICMP;
 102                 break;
 103         default:
 104                 m->packet_type = dpaa2_dev_rx_parse_slow(m,
 105                   (void *)((size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd))
 106                          + DPAA2_FD_PTA_SIZE));
 107         }
 108         m->hash.rss = fd->simple.flc_hi;
 109         m->ol_flags |= PKT_RX_RSS_HASH;
 110
 111         if (dpaa2_enable_ts == PMD_DPAA2_ENABLE_TS) {
 112                 annotation = (struct dpaa2_annot_hdr *)
 113                         ((size_t)DPAA2_IOVA_TO_VADDR(
 114                         DPAA2_GET_FD_ADDR(fd)) + DPAA2_FD_PTA_SIZE);
 115                 m->timestamp = annotation->word2;
 116                 m->ol_flags |= PKT_RX_TIMESTAMP;
 117                 DPAA2_PMD_DP_DEBUG("pkt timestamp:0x%" PRIx64 "", m->timestamp);
 118         }
 119
 120         DPAA2_PMD_DP_DEBUG("HW frc = 0x%x\t packet type =0x%x "
 121                 "ol_flags =0x%" PRIx64 "",
 122                 frc, m->packet_type, m->ol_flags);
 123 }
 124
 125 static inline uint32_t __rte_hot
 126 dpaa2_dev_rx_parse_slow(struct rte_mbuf *mbuf,
 127                         struct dpaa2_annot_hdr *annotation)
 128 {
 129         uint32_t pkt_type = RTE_PTYPE_UNKNOWN;
 130         uint16_t *vlan_tci;
 131
 132         DPAA2_PMD_DP_DEBUG("(slow parse)annotation(3)=0x%" PRIx64 "\t"
 133                         "(4)=0x%" PRIx64 "\t",
 134                         annotation->word3, annotation->word4);
 135
 136 #if defined(RTE_LIBRTE_IEEE1588)
 137         if (BIT_ISSET_AT_POS(annotation->word1, DPAA2_ETH_FAS_PTP))
 138                 mbuf->ol_flags |= PKT_RX_IEEE1588_PTP;
 139 #endif
 140
 141         if (BIT_ISSET_AT_POS(annotation->word3, L2_VLAN_1_PRESENT)) {
 142                 vlan_tci = rte_pktmbuf_mtod_offset(mbuf, uint16_t *,
 143                         (VLAN_TCI_OFFSET_1(annotation->word5) >> 16));
 144                 mbuf->vlan_tci = rte_be_to_cpu_16(*vlan_tci);
 145                 mbuf->ol_flags |= PKT_RX_VLAN;
 146                 pkt_type |= RTE_PTYPE_L2_ETHER_VLAN;
 147         } else if (BIT_ISSET_AT_POS(annotation->word3, L2_VLAN_N_PRESENT)) {
 148                 vlan_tci = rte_pktmbuf_mtod_offset(mbuf, uint16_t *,
 149                         (VLAN_TCI_OFFSET_1(annotation->word5) >> 16));
 150                 mbuf->vlan_tci = rte_be_to_cpu_16(*vlan_tci);
 151                 mbuf->ol_flags |= PKT_RX_VLAN | PKT_RX_QINQ;
 152                 pkt_type |= RTE_PTYPE_L2_ETHER_QINQ;
 153         }
 154
 155         if (BIT_ISSET_AT_POS(annotation->word3, L2_ARP_PRESENT)) {
 156                 pkt_type |= RTE_PTYPE_L2_ETHER_ARP;
 157                 goto parse_done;
 158         } else if (BIT_ISSET_AT_POS(annotation->word3, L2_ETH_MAC_PRESENT)) {
 159                 pkt_type |= RTE_PTYPE_L2_ETHER;
 160         } else {
 161                 goto parse_done;
 162         }
 163
 164         if (BIT_ISSET_AT_POS(annotation->word4, L3_IPV4_1_PRESENT |
 165                              L3_IPV4_N_PRESENT)) {
 166                 pkt_type |= RTE_PTYPE_L3_IPV4;
 167                 if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_OPT_PRESENT |
 168                         L3_IP_N_OPT_PRESENT))
 169                         pkt_type |= RTE_PTYPE_L3_IPV4_EXT;
 170
 171         } else if (BIT_ISSET_AT_POS(annotation->word4, L3_IPV6_1_PRESENT |
 172                   L3_IPV6_N_PRESENT)) {
 173                 pkt_type |= RTE_PTYPE_L3_IPV6;
 174                 if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_OPT_PRESENT |
 175                     L3_IP_N_OPT_PRESENT))
 176                         pkt_type |= RTE_PTYPE_L3_IPV6_EXT;
 177         } else {
 178                 goto parse_done;
 179         }
 180
 181         if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L3CE))
 182                 mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD;
 183         else if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE))
 184                 mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD;
 185
 186         if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_FIRST_FRAGMENT |
 187             L3_IP_1_MORE_FRAGMENT |
 188             L3_IP_N_FIRST_FRAGMENT |
 189             L3_IP_N_MORE_FRAGMENT)) {
 190                 pkt_type |= RTE_PTYPE_L4_FRAG;
 191                 goto parse_done;
 192         } else {
 193                 pkt_type |= RTE_PTYPE_L4_NONFRAG;
 194         }
 195
 196         if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_UDP_PRESENT))
 197                 pkt_type |= RTE_PTYPE_L4_UDP;
 198
 199         else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_TCP_PRESENT))
 200                 pkt_type |= RTE_PTYPE_L4_TCP;
 201
 202         else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_SCTP_PRESENT))
 203                 pkt_type |= RTE_PTYPE_L4_SCTP;
 204
 205         else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_ICMP_PRESENT))
 206                 pkt_type |= RTE_PTYPE_L4_ICMP;
 207
 208         else if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_UNKNOWN_PROTOCOL))
 209                 pkt_type |= RTE_PTYPE_UNKNOWN;
 210
 211 parse_done:
 212         return pkt_type;
 213 }
 214
 215 static inline uint32_t __rte_hot
 216 dpaa2_dev_rx_parse(struct rte_mbuf *mbuf, void *hw_annot_addr)
 217 {
 218         struct dpaa2_annot_hdr *annotation =
 219                         (struct dpaa2_annot_hdr *)hw_annot_addr;
 220
 221         DPAA2_PMD_DP_DEBUG("(fast parse) Annotation = 0x%" PRIx64 "\t",
 222                            annotation->word4);
 223
 224         if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L3CE))
 225                 mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD;
 226         else if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE))
 227                 mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD;
 228
 229         mbuf->ol_flags |= PKT_RX_TIMESTAMP;
 230         mbuf->timestamp = annotation->word2;
 231         DPAA2_PMD_DP_DEBUG("pkt timestamp: 0x%" PRIx64 "", mbuf->timestamp);
 232
 233         /* Check detailed parsing requirement */
 234         if (annotation->word3 & 0x7FFFFC3FFFF)
 235                 return dpaa2_dev_rx_parse_slow(mbuf, annotation);
 236
 237         /* Return some common types from parse processing */
 238         switch (annotation->word4) {
 239         case DPAA2_L3_IPv4:
 240                 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4;
 241         case DPAA2_L3_IPv6:
 242                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6;
 243         case DPAA2_L3_IPv4_TCP:
 244                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
 245                                 RTE_PTYPE_L4_TCP;
 246         case DPAA2_L3_IPv4_UDP:
 247                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
 248                                 RTE_PTYPE_L4_UDP;
 249         case DPAA2_L3_IPv6_TCP:
 250                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
 251                                 RTE_PTYPE_L4_TCP;
 252         case DPAA2_L3_IPv6_UDP:
 253                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
 254                                 RTE_PTYPE_L4_UDP;
 255         default:
 256                 break;
 257         }
 258
 259         return dpaa2_dev_rx_parse_slow(mbuf, annotation);
 260 }
 261
 262 static inline struct rte_mbuf *__rte_hot
 263 eth_sg_fd_to_mbuf(const struct qbman_fd *fd,
 264                   int port_id)
 265 {
 266         struct qbman_sge *sgt, *sge;
 267         size_t sg_addr, fd_addr;
 268         int i = 0;
 269         struct rte_mbuf *first_seg, *next_seg, *cur_seg, *temp;
 270
 271         fd_addr = (size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
 272
 273         /* Get Scatter gather table address */
 274         sgt = (struct qbman_sge *)(fd_addr + DPAA2_GET_FD_OFFSET(fd));
 275
 276         sge = &sgt[i++];
 277         sg_addr = (size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FLE_ADDR(sge));
 278
 279         /* First Scatter gather entry */
 280         first_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr,
 281                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
 282         /* Prepare all the metadata for first segment */
 283         first_seg->buf_addr = (uint8_t *)sg_addr;
 284         first_seg->ol_flags = 0;
 285         first_seg->data_off = DPAA2_GET_FLE_OFFSET(sge);
 286         first_seg->data_len = sge->length  & 0x1FFFF;
 287         first_seg->pkt_len = DPAA2_GET_FD_LEN(fd);
 288         first_seg->nb_segs = 1;
 289         first_seg->next = NULL;
 290         first_seg->port = port_id;
 291         if (dpaa2_svr_family == SVR_LX2160A)
 292                 dpaa2_dev_rx_parse_new(first_seg, fd);
 293         else
 294                 first_seg->packet_type = dpaa2_dev_rx_parse(first_seg,
 295                         (void *)((size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd))
 296                          + DPAA2_FD_PTA_SIZE));
 297
 298         rte_mbuf_refcnt_set(first_seg, 1);
 299         cur_seg = first_seg;
 300         while (!DPAA2_SG_IS_FINAL(sge)) {
 301                 sge = &sgt[i++];
 302                 sg_addr = (size_t)DPAA2_IOVA_TO_VADDR(
 303                                 DPAA2_GET_FLE_ADDR(sge));
 304                 next_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr,
 305                         rte_dpaa2_bpid_info[DPAA2_GET_FLE_BPID(sge)].meta_data_size);
 306                 next_seg->buf_addr  = (uint8_t *)sg_addr;
 307                 next_seg->data_off  = DPAA2_GET_FLE_OFFSET(sge);
 308                 next_seg->data_len  = sge->length  & 0x1FFFF;
 309                 first_seg->nb_segs += 1;
 310                 rte_mbuf_refcnt_set(next_seg, 1);
 311                 cur_seg->next = next_seg;
 312                 next_seg->next = NULL;
 313                 cur_seg = next_seg;
 314         }
 315         temp = DPAA2_INLINE_MBUF_FROM_BUF(fd_addr,
 316                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
 317         rte_mbuf_refcnt_set(temp, 1);
 318         rte_pktmbuf_free_seg(temp);
 319
 320         return (void *)first_seg;
 321 }
 322
 323 static inline struct rte_mbuf *__rte_hot
 324 eth_fd_to_mbuf(const struct qbman_fd *fd,
 325                int port_id)
 326 {
 327         void *iova_addr = DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
 328         struct rte_mbuf *mbuf = DPAA2_INLINE_MBUF_FROM_BUF(iova_addr,
 329                      rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
 330
 331         /* need to repopulated some of the fields,
 332          * as they may have changed in last transmission
 333          */
 334         mbuf->nb_segs = 1;
 335         mbuf->ol_flags = 0;
 336         mbuf->data_off = DPAA2_GET_FD_OFFSET(fd);
 337         mbuf->data_len = DPAA2_GET_FD_LEN(fd);
 338         mbuf->pkt_len = mbuf->data_len;
 339         mbuf->port = port_id;
 340         mbuf->next = NULL;
 341         rte_mbuf_refcnt_set(mbuf, 1);
 342
 343         /* Parse the packet */
 344         /* parse results for LX2 are there in FRC field of FD.
 345          * For other DPAA2 platforms , parse results are after
 346          * the private - sw annotation area
 347          */
 348
 349         if (dpaa2_svr_family == SVR_LX2160A)
 350                 dpaa2_dev_rx_parse_new(mbuf, fd);
 351         else
 352                 mbuf->packet_type = dpaa2_dev_rx_parse(mbuf,
 353                         (void *)((size_t)iova_addr + DPAA2_FD_PTA_SIZE));
 354
 355         DPAA2_PMD_DP_DEBUG("to mbuf - mbuf =%p, mbuf->buf_addr =%p, off = %d,"
 356                 "fd_off=%d fd =%" PRIx64 ", meta = %d  bpid =%d, len=%d\n",
 357                 mbuf, mbuf->buf_addr, mbuf->data_off,
 358                 DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ADDR(fd),
 359                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size,
 360                 DPAA2_GET_FD_BPID(fd), DPAA2_GET_FD_LEN(fd));
 361
 362         return mbuf;
 363 }
 364
 365 static int __rte_noinline __rte_hot
 366 eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf,
 367                   struct qbman_fd *fd, uint16_t bpid)
 368 {
 369         struct rte_mbuf *cur_seg = mbuf, *prev_seg, *mi, *temp;
 370         struct qbman_sge *sgt, *sge = NULL;
 371         int i;
 372
 373         temp = rte_pktmbuf_alloc(mbuf->pool);
 374         if (temp == NULL) {
 375                 DPAA2_PMD_DP_DEBUG("No memory to allocate S/G table\n");
 376                 return -ENOMEM;
 377         }
 378
 379         DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(temp));
 380         DPAA2_SET_FD_LEN(fd, mbuf->pkt_len);
 381         DPAA2_SET_ONLY_FD_BPID(fd, bpid);
 382         DPAA2_SET_FD_OFFSET(fd, temp->data_off);
 383         DPAA2_FD_SET_FORMAT(fd, qbman_fd_sg);
 384         DPAA2_RESET_FD_FRC(fd);
 385         DPAA2_RESET_FD_CTRL(fd);
 386         /*Set Scatter gather table and Scatter gather entries*/
 387         sgt = (struct qbman_sge *)(
 388                         (size_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd))
 389                         + DPAA2_GET_FD_OFFSET(fd));
 390
 391         for (i = 0; i < mbuf->nb_segs; i++) {
 392                 sge = &sgt[i];
 393                 /*Resetting the buffer pool id and offset field*/
 394                 sge->fin_bpid_offset = 0;
 395                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(cur_seg));
 396                 DPAA2_SET_FLE_OFFSET(sge, cur_seg->data_off);
 397                 sge->length = cur_seg->data_len;
 398                 if (RTE_MBUF_DIRECT(cur_seg)) {
 399                         if (rte_mbuf_refcnt_read(cur_seg) > 1) {
 400                                 /* If refcnt > 1, invalid bpid is set to ensure
 401                                  * buffer is not freed by HW
 402                                  */
 403                                 DPAA2_SET_FLE_IVP(sge);
 404                                 rte_mbuf_refcnt_update(cur_seg, -1);
 405                         } else
 406                                 DPAA2_SET_FLE_BPID(sge,
 407                                                 mempool_to_bpid(cur_seg->pool));
 408                         cur_seg = cur_seg->next;
 409                 } else {
 410                         /* Get owner MBUF from indirect buffer */
 411                         mi = rte_mbuf_from_indirect(cur_seg);
 412                         if (rte_mbuf_refcnt_read(mi) > 1) {
 413                                 /* If refcnt > 1, invalid bpid is set to ensure
 414                                  * owner buffer is not freed by HW
 415                                  */
 416                                 DPAA2_SET_FLE_IVP(sge);
 417                         } else {
 418                                 DPAA2_SET_FLE_BPID(sge,
 419                                                    mempool_to_bpid(mi->pool));
 420                                 rte_mbuf_refcnt_update(mi, 1);
 421                         }
 422                         prev_seg = cur_seg;
 423                         cur_seg = cur_seg->next;
 424                         prev_seg->next = NULL;
 425                         rte_pktmbuf_free(prev_seg);
 426                 }
 427         }
 428         DPAA2_SG_SET_FINAL(sge, true);
 429         return 0;
 430 }
 431
 432 static void
 433 eth_mbuf_to_fd(struct rte_mbuf *mbuf,
 434                struct qbman_fd *fd, uint16_t bpid) __rte_unused;
 435
 436 static void __rte_noinline __rte_hot
 437 eth_mbuf_to_fd(struct rte_mbuf *mbuf,
 438                struct qbman_fd *fd, uint16_t bpid)
 439 {
 440         DPAA2_MBUF_TO_CONTIG_FD(mbuf, fd, bpid);
 441
 442         DPAA2_PMD_DP_DEBUG("mbuf =%p, mbuf->buf_addr =%p, off = %d,"
 443                 "fd_off=%d fd =%" PRIx64 ", meta = %d  bpid =%d, len=%d\n",
 444                 mbuf, mbuf->buf_addr, mbuf->data_off,
 445                 DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ADDR(fd),
 446                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size,
 447                 DPAA2_GET_FD_BPID(fd), DPAA2_GET_FD_LEN(fd));
 448         if (RTE_MBUF_DIRECT(mbuf)) {
 449                 if (rte_mbuf_refcnt_read(mbuf) > 1) {
 450                         DPAA2_SET_FD_IVP(fd);
 451                         rte_mbuf_refcnt_update(mbuf, -1);
 452                 }
 453         } else {
 454                 struct rte_mbuf *mi;
 455
 456                 mi = rte_mbuf_from_indirect(mbuf);
 457                 if (rte_mbuf_refcnt_read(mi) > 1)
 458                         DPAA2_SET_FD_IVP(fd);
 459                 else
 460                         rte_mbuf_refcnt_update(mi, 1);
 461                 rte_pktmbuf_free(mbuf);
 462         }
 463 }
 464
 465 static inline int __rte_hot
 466 eth_copy_mbuf_to_fd(struct rte_mbuf *mbuf,
 467                     struct qbman_fd *fd, uint16_t bpid)
 468 {
 469         struct rte_mbuf *m;
 470         void *mb = NULL;
 471
 472         if (rte_dpaa2_mbuf_alloc_bulk(
 473                 rte_dpaa2_bpid_info[bpid].bp_list->mp, &mb, 1)) {
 474                 DPAA2_PMD_DP_DEBUG("Unable to allocated DPAA2 buffer\n");
 475                 return -1;
 476         }
 477         m = (struct rte_mbuf *)mb;
 478         memcpy((char *)m->buf_addr + mbuf->data_off,
 479                (void *)((char *)mbuf->buf_addr + mbuf->data_off),
 480                 mbuf->pkt_len);
 481
 482         /* Copy required fields */
 483         m->data_off = mbuf->data_off;
 484         m->ol_flags = mbuf->ol_flags;
 485         m->packet_type = mbuf->packet_type;
 486         m->tx_offload = mbuf->tx_offload;
 487
 488         DPAA2_MBUF_TO_CONTIG_FD(m, fd, bpid);
 489
 490         DPAA2_PMD_DP_DEBUG(
 491                 "mbuf: %p, BMAN buf addr: %p, fdaddr: %" PRIx64 ", bpid: %d,"
 492                 " meta: %d, off: %d, len: %d\n",
 493                 (void *)mbuf,
 494                 mbuf->buf_addr,
 495                 DPAA2_GET_FD_ADDR(fd),
 496                 DPAA2_GET_FD_BPID(fd),
 497                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size,
 498                 DPAA2_GET_FD_OFFSET(fd),
 499                 DPAA2_GET_FD_LEN(fd));
 500
 501 return 0;
 502 }
 503
 504 /* This function assumes that caller will be keep the same value for nb_pkts
 505  * across calls per queue, if that is not the case, better use non-prefetch
 506  * version of rx call.
 507  * It will return the packets as requested in previous call without honoring
 508  * the current nb_pkts or bufs space.
 509  */
 510 uint16_t
 511 dpaa2_dev_prefetch_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 512 {
 513         /* Function receive frames for a given device and VQ*/
 514         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
 515         struct qbman_result *dq_storage, *dq_storage1 = NULL;
 516         uint32_t fqid = dpaa2_q->fqid;
 517         int ret, num_rx = 0, pull_size;
 518         uint8_t pending, status;
 519         struct qbman_swp *swp;
 520         const struct qbman_fd *fd;
 521         struct qbman_pull_desc pulldesc;
 522         struct queue_storage_info_t *q_storage = dpaa2_q->q_storage;
 523         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
 524 #if defined(RTE_LIBRTE_IEEE1588)
 525         struct dpaa2_dev_priv *priv = eth_data->dev_private;
 526 #endif
 527
 528         if (unlikely(!DPAA2_PER_LCORE_ETHRX_DPIO)) {
 529                 ret = dpaa2_affine_qbman_ethrx_swp();
 530                 if (ret) {
 531                         DPAA2_PMD_ERR("Failure in affining portal");
 532                         return 0;
 533                 }
 534         }
 535
 536         if (unlikely(!rte_dpaa2_bpid_info &&
 537                      rte_eal_process_type() == RTE_PROC_SECONDARY))
 538                 rte_dpaa2_bpid_info = dpaa2_q->bp_array;
 539
 540         swp = DPAA2_PER_LCORE_ETHRX_PORTAL;
 541         pull_size = (nb_pkts > dpaa2_dqrr_size) ? dpaa2_dqrr_size : nb_pkts;
 542         if (unlikely(!q_storage->active_dqs)) {
 543                 q_storage->toggle = 0;
 544                 dq_storage = q_storage->dq_storage[q_storage->toggle];
 545                 q_storage->last_num_pkts = pull_size;
 546                 qbman_pull_desc_clear(&pulldesc);
 547                 qbman_pull_desc_set_numframes(&pulldesc,
 548                                               q_storage->last_num_pkts);
 549                 qbman_pull_desc_set_fq(&pulldesc, fqid);
 550                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
 551                         (uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
 552                 if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) {
 553                         while (!qbman_check_command_complete(
 554                                get_swp_active_dqs(
 555                                DPAA2_PER_LCORE_ETHRX_DPIO->index)))
 556                                 ;
 557                         clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index);
 558                 }
 559                 while (1) {
 560                         if (qbman_swp_pull(swp, &pulldesc)) {
 561                                 DPAA2_PMD_DP_DEBUG("VDQ command is not issued."
 562                                                   " QBMAN is busy (1)\n");
 563                                 /* Portal was busy, try again */
 564                                 continue;
 565                         }
 566                         break;
 567                 }
 568                 q_storage->active_dqs = dq_storage;
 569                 q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index;
 570                 set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index,
 571                                    dq_storage);
 572         }
 573
 574         dq_storage = q_storage->active_dqs;
 575         rte_prefetch0((void *)(size_t)(dq_storage));
 576         rte_prefetch0((void *)(size_t)(dq_storage + 1));
 577
 578         /* Prepare next pull descriptor. This will give space for the
 579          * prefething done on DQRR entries
 580          */
 581         q_storage->toggle ^= 1;
 582         dq_storage1 = q_storage->dq_storage[q_storage->toggle];
 583         qbman_pull_desc_clear(&pulldesc);
 584         qbman_pull_desc_set_numframes(&pulldesc, pull_size);
 585         qbman_pull_desc_set_fq(&pulldesc, fqid);
 586         qbman_pull_desc_set_storage(&pulldesc, dq_storage1,
 587                 (uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage1)), 1);
 588
 589         /* Check if the previous issued command is completed.
 590          * Also seems like the SWP is shared between the Ethernet Driver
 591          * and the SEC driver.
 592          */
 593         while (!qbman_check_command_complete(dq_storage))
 594                 ;
 595         if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id))
 596                 clear_swp_active_dqs(q_storage->active_dpio_id);
 597
 598         pending = 1;
 599
 600         do {
 601                 /* Loop until the dq_storage is updated with
 602                  * new token by QBMAN
 603                  */
 604                 while (!qbman_check_new_result(dq_storage))
 605                         ;
 606                 rte_prefetch0((void *)((size_t)(dq_storage + 2)));
 607                 /* Check whether Last Pull command is Expired and
 608                  * setting Condition for Loop termination
 609                  */
 610                 if (qbman_result_DQ_is_pull_complete(dq_storage)) {
 611                         pending = 0;
 612                         /* Check for valid frame. */
 613                         status = qbman_result_DQ_flags(dq_storage);
 614                         if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0))
 615                                 continue;
 616                 }
 617                 fd = qbman_result_DQ_fd(dq_storage);
 618
 619 #ifndef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
 620                 if (dpaa2_svr_family != SVR_LX2160A) {
 621                         const struct qbman_fd *next_fd =
 622                                 qbman_result_DQ_fd(dq_storage + 1);
 623                         /* Prefetch Annotation address for the parse results */
 624                         rte_prefetch0(DPAA2_IOVA_TO_VADDR((DPAA2_GET_FD_ADDR(
 625                                 next_fd) + DPAA2_FD_PTA_SIZE + 16)));
 626                 }
 627 #endif
 628
 629                 if (unlikely(DPAA2_FD_GET_FORMAT(fd) == qbman_fd_sg))
 630                         bufs[num_rx] = eth_sg_fd_to_mbuf(fd, eth_data->port_id);
 631                 else
 632                         bufs[num_rx] = eth_fd_to_mbuf(fd, eth_data->port_id);
 633 #if defined(RTE_LIBRTE_IEEE1588)
 634                 priv->rx_timestamp = bufs[num_rx]->timestamp;
 635 #endif
 636
 637                 if (eth_data->dev_conf.rxmode.offloads &
 638                                 DEV_RX_OFFLOAD_VLAN_STRIP)
 639                         rte_vlan_strip(bufs[num_rx]);
 640
 641                 dq_storage++;
 642                 num_rx++;
 643         } while (pending);
 644
 645         if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) {
 646                 while (!qbman_check_command_complete(
 647                        get_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)))
 648                         ;
 649                 clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index);
 650         }
 651         /* issue a volatile dequeue command for next pull */
 652         while (1) {
 653                 if (qbman_swp_pull(swp, &pulldesc)) {
 654                         DPAA2_PMD_DP_DEBUG("VDQ command is not issued."
 655                                           "QBMAN is busy (2)\n");
 656                         continue;
 657                 }
 658                 break;
 659         }
 660         q_storage->active_dqs = dq_storage1;
 661         q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index;
 662         set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index, dq_storage1);
 663
 664         dpaa2_q->rx_pkts += num_rx;
 665
 666         return num_rx;
 667 }
 668
 669 void __rte_hot
 670 dpaa2_dev_process_parallel_event(struct qbman_swp *swp,
 671                                  const struct qbman_fd *fd,
 672                                  const struct qbman_result *dq,
 673                                  struct dpaa2_queue *rxq,
 674                                  struct rte_event *ev)
 675 {
 676         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd) +
 677                 DPAA2_FD_PTA_SIZE + 16));
 678
 679         ev->flow_id = rxq->ev.flow_id;
 680         ev->sub_event_type = rxq->ev.sub_event_type;
 681         ev->event_type = RTE_EVENT_TYPE_ETHDEV;
 682         ev->op = RTE_EVENT_OP_NEW;
 683         ev->sched_type = rxq->ev.sched_type;
 684         ev->queue_id = rxq->ev.queue_id;
 685         ev->priority = rxq->ev.priority;
 686
 687         ev->mbuf = eth_fd_to_mbuf(fd, rxq->eth_data->port_id);
 688
 689         qbman_swp_dqrr_consume(swp, dq);
 690 }
 691
 692 void __rte_hot
 693 dpaa2_dev_process_atomic_event(struct qbman_swp *swp __rte_unused,
 694                                const struct qbman_fd *fd,
 695                                const struct qbman_result *dq,
 696                                struct dpaa2_queue *rxq,
 697                                struct rte_event *ev)
 698 {
 699         uint8_t dqrr_index;
 700
 701         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd) +
 702                 DPAA2_FD_PTA_SIZE + 16));
 703
 704         ev->flow_id = rxq->ev.flow_id;
 705         ev->sub_event_type = rxq->ev.sub_event_type;
 706         ev->event_type = RTE_EVENT_TYPE_ETHDEV;
 707         ev->op = RTE_EVENT_OP_NEW;
 708         ev->sched_type = rxq->ev.sched_type;
 709         ev->queue_id = rxq->ev.queue_id;
 710         ev->priority = rxq->ev.priority;
 711
 712         ev->mbuf = eth_fd_to_mbuf(fd, rxq->eth_data->port_id);
 713
 714         dqrr_index = qbman_get_dqrr_idx(dq);
 715         ev->mbuf->seqn = dqrr_index + 1;
 716         DPAA2_PER_LCORE_DQRR_SIZE++;
 717         DPAA2_PER_LCORE_DQRR_HELD |= 1 << dqrr_index;
 718         DPAA2_PER_LCORE_DQRR_MBUF(dqrr_index) = ev->mbuf;
 719 }
 720
 721 void __rte_hot
 722 dpaa2_dev_process_ordered_event(struct qbman_swp *swp,
 723                                 const struct qbman_fd *fd,
 724                                 const struct qbman_result *dq,
 725                                 struct dpaa2_queue *rxq,
 726                                 struct rte_event *ev)
 727 {
 728         rte_prefetch0((void *)(size_t)(DPAA2_GET_FD_ADDR(fd) +
 729                 DPAA2_FD_PTA_SIZE + 16));
 730
 731         ev->flow_id = rxq->ev.flow_id;
 732         ev->sub_event_type = rxq->ev.sub_event_type;
 733         ev->event_type = RTE_EVENT_TYPE_ETHDEV;
 734         ev->op = RTE_EVENT_OP_NEW;
 735         ev->sched_type = rxq->ev.sched_type;
 736         ev->queue_id = rxq->ev.queue_id;
 737         ev->priority = rxq->ev.priority;
 738
 739         ev->mbuf = eth_fd_to_mbuf(fd, rxq->eth_data->port_id);
 740
 741         ev->mbuf->seqn = DPAA2_ENQUEUE_FLAG_ORP;
 742         ev->mbuf->seqn |= qbman_result_DQ_odpid(dq) << DPAA2_EQCR_OPRID_SHIFT;
 743         ev->mbuf->seqn |= qbman_result_DQ_seqnum(dq) << DPAA2_EQCR_SEQNUM_SHIFT;
 744
 745         qbman_swp_dqrr_consume(swp, dq);
 746 }
 747
 748 uint16_t
 749 dpaa2_dev_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 750 {
 751         /* Function receive frames for a given device and VQ */
 752         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
 753         struct qbman_result *dq_storage;
 754         uint32_t fqid = dpaa2_q->fqid;
 755         int ret, num_rx = 0, next_pull = nb_pkts, num_pulled;
 756         uint8_t pending, status;
 757         struct qbman_swp *swp;
 758         const struct qbman_fd *fd;
 759         struct qbman_pull_desc pulldesc;
 760         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
 761
 762         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 763                 ret = dpaa2_affine_qbman_swp();
 764                 if (ret) {
 765                         DPAA2_PMD_ERR(
 766                                 "Failed to allocate IO portal, tid: %d\n",
 767                                 rte_gettid());
 768                         return 0;
 769                 }
 770         }
 771         swp = DPAA2_PER_LCORE_PORTAL;
 772
 773         do {
 774                 dq_storage = dpaa2_q->q_storage->dq_storage[0];
 775                 qbman_pull_desc_clear(&pulldesc);
 776                 qbman_pull_desc_set_fq(&pulldesc, fqid);
 777                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
 778                                 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
 779
 780                 if (next_pull > dpaa2_dqrr_size) {
 781                         qbman_pull_desc_set_numframes(&pulldesc,
 782                                 dpaa2_dqrr_size);
 783                         next_pull -= dpaa2_dqrr_size;
 784                 } else {
 785                         qbman_pull_desc_set_numframes(&pulldesc, next_pull);
 786                         next_pull = 0;
 787                 }
 788
 789                 while (1) {
 790                         if (qbman_swp_pull(swp, &pulldesc)) {
 791                                 DPAA2_PMD_DP_DEBUG(
 792                                         "VDQ command is not issued.QBMAN is busy\n");
 793                                 /* Portal was busy, try again */
 794                                 continue;
 795                         }
 796                         break;
 797                 }
 798
 799                 rte_prefetch0((void *)((size_t)(dq_storage + 1)));
 800                 /* Check if the previous issued command is completed. */
 801                 while (!qbman_check_command_complete(dq_storage))
 802                         ;
 803
 804                 num_pulled = 0;
 805                 pending = 1;
 806                 do {
 807                         /* Loop until the dq_storage is updated with
 808                          * new token by QBMAN
 809                          */
 810                         while (!qbman_check_new_result(dq_storage))
 811                                 ;
 812                         rte_prefetch0((void *)((size_t)(dq_storage + 2)));
 813                         /* Check whether Last Pull command is Expired and
 814                          * setting Condition for Loop termination
 815                          */
 816                         if (qbman_result_DQ_is_pull_complete(dq_storage)) {
 817                                 pending = 0;
 818                                 /* Check for valid frame. */
 819                                 status = qbman_result_DQ_flags(dq_storage);
 820                                 if (unlikely((status &
 821                                         QBMAN_DQ_STAT_VALIDFRAME) == 0))
 822                                         continue;
 823                         }
 824                         fd = qbman_result_DQ_fd(dq_storage);
 825
 826 #ifndef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
 827                         if (dpaa2_svr_family != SVR_LX2160A) {
 828                                 const struct qbman_fd *next_fd =
 829                                         qbman_result_DQ_fd(dq_storage + 1);
 830
 831                                 /* Prefetch Annotation address for the parse
 832                                  * results.
 833                                  */
 834                                 rte_prefetch0((DPAA2_IOVA_TO_VADDR(
 835                                         DPAA2_GET_FD_ADDR(next_fd) +
 836                                         DPAA2_FD_PTA_SIZE + 16)));
 837                         }
 838 #endif
 839
 840                         if (unlikely(DPAA2_FD_GET_FORMAT(fd) == qbman_fd_sg))
 841                                 bufs[num_rx] = eth_sg_fd_to_mbuf(fd,
 842                                                         eth_data->port_id);
 843                         else
 844                                 bufs[num_rx] = eth_fd_to_mbuf(fd,
 845                                                         eth_data->port_id);
 846
 847                 if (eth_data->dev_conf.rxmode.offloads &
 848                                 DEV_RX_OFFLOAD_VLAN_STRIP) {
 849                         rte_vlan_strip(bufs[num_rx]);
 850                 }
 851
 852                         dq_storage++;
 853                         num_rx++;
 854                         num_pulled++;
 855                 } while (pending);
 856         /* Last VDQ provided all packets and more packets are requested */
 857         } while (next_pull && num_pulled == dpaa2_dqrr_size);
 858
 859         dpaa2_q->rx_pkts += num_rx;
 860
 861         return num_rx;
 862 }
 863
 864 uint16_t dpaa2_dev_tx_conf(void *queue)
 865 {
 866         /* Function receive frames for a given device and VQ */
 867         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
 868         struct qbman_result *dq_storage;
 869         uint32_t fqid = dpaa2_q->fqid;
 870         int ret, num_tx_conf = 0, num_pulled;
 871         uint8_t pending, status;
 872         struct qbman_swp *swp;
 873         const struct qbman_fd *fd, *next_fd;
 874         struct qbman_pull_desc pulldesc;
 875         struct qbman_release_desc releasedesc;
 876         uint32_t bpid;
 877         uint64_t buf;
 878 #if defined(RTE_LIBRTE_IEEE1588)
 879         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
 880         struct dpaa2_dev_priv *priv = eth_data->dev_private;
 881         struct dpaa2_annot_hdr *annotation;
 882 #endif
 883
 884         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 885                 ret = dpaa2_affine_qbman_swp();
 886                 if (ret) {
 887                         DPAA2_PMD_ERR(
 888                                 "Failed to allocate IO portal, tid: %d\n",
 889                                 rte_gettid());
 890                         return 0;
 891                 }
 892         }
 893         swp = DPAA2_PER_LCORE_PORTAL;
 894
 895         do {
 896                 dq_storage = dpaa2_q->q_storage->dq_storage[0];
 897                 qbman_pull_desc_clear(&pulldesc);
 898                 qbman_pull_desc_set_fq(&pulldesc, fqid);
 899                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
 900                                 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
 901
 902                 qbman_pull_desc_set_numframes(&pulldesc, dpaa2_dqrr_size);
 903
 904                 while (1) {
 905                         if (qbman_swp_pull(swp, &pulldesc)) {
 906                                 DPAA2_PMD_DP_DEBUG("VDQ command is not issued."
 907                                                    "QBMAN is busy\n");
 908                                 /* Portal was busy, try again */
 909                                 continue;
 910                         }
 911                         break;
 912                 }
 913
 914                 rte_prefetch0((void *)((size_t)(dq_storage + 1)));
 915                 /* Check if the previous issued command is completed. */
 916                 while (!qbman_check_command_complete(dq_storage))
 917                         ;
 918
 919                 num_pulled = 0;
 920                 pending = 1;
 921                 do {
 922                         /* Loop until the dq_storage is updated with
 923                          * new token by QBMAN
 924                          */
 925                         while (!qbman_check_new_result(dq_storage))
 926                                 ;
 927                         rte_prefetch0((void *)((size_t)(dq_storage + 2)));
 928                         /* Check whether Last Pull command is Expired and
 929                          * setting Condition for Loop termination
 930                          */
 931                         if (qbman_result_DQ_is_pull_complete(dq_storage)) {
 932                                 pending = 0;
 933                                 /* Check for valid frame. */
 934                                 status = qbman_result_DQ_flags(dq_storage);
 935                                 if (unlikely((status &
 936                                         QBMAN_DQ_STAT_VALIDFRAME) == 0))
 937                                         continue;
 938                         }
 939                         fd = qbman_result_DQ_fd(dq_storage);
 940
 941                         next_fd = qbman_result_DQ_fd(dq_storage + 1);
 942                         /* Prefetch Annotation address for the parse results */
 943                         rte_prefetch0((void *)(size_t)
 944                                 (DPAA2_GET_FD_ADDR(next_fd) +
 945                                  DPAA2_FD_PTA_SIZE + 16));
 946
 947                         bpid = DPAA2_GET_FD_BPID(fd);
 948
 949                         /* Create a release descriptor required for releasing
 950                          * buffers into QBMAN
 951                          */
 952                         qbman_release_desc_clear(&releasedesc);
 953                         qbman_release_desc_set_bpid(&releasedesc, bpid);
 954
 955                         buf = DPAA2_GET_FD_ADDR(fd);
 956                         /* feed them to bman */
 957                         do {
 958                                 ret = qbman_swp_release(swp, &releasedesc,
 959                                                         &buf, 1);
 960                         } while (ret == -EBUSY);
 961
 962                         dq_storage++;
 963                         num_tx_conf++;
 964                         num_pulled++;
 965 #if defined(RTE_LIBRTE_IEEE1588)
 966                         annotation = (struct dpaa2_annot_hdr *)((size_t)
 967                                 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) +
 968                                 DPAA2_FD_PTA_SIZE);
 969                         priv->tx_timestamp = annotation->word2;
 970 #endif
 971                 } while (pending);
 972
 973         /* Last VDQ provided all packets and more packets are requested */
 974         } while (num_pulled == dpaa2_dqrr_size);
 975
 976         dpaa2_q->rx_pkts += num_tx_conf;
 977
 978         return num_tx_conf;
 979 }
 980
 981 /* Configure the egress frame annotation for timestamp update */
 982 static void enable_tx_tstamp(struct qbman_fd *fd)
 983 {
 984         struct dpaa2_faead *fd_faead;
 985
 986         /* Set frame annotation status field as valid */
 987         (fd)->simple.frc |= DPAA2_FD_FRC_FASV;
 988
 989         /* Set frame annotation egress action descriptor as valid */
 990         (fd)->simple.frc |= DPAA2_FD_FRC_FAEADV;
 991
 992         /* Set Annotation Length as 128B */
 993         (fd)->simple.ctrl |= DPAA2_FD_CTRL_ASAL;
 994
 995         /* enable update of confirmation frame annotation */
 996         fd_faead = (struct dpaa2_faead *)((size_t)
 997                         DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)) +
 998                         DPAA2_FD_PTA_SIZE + DPAA2_FD_HW_ANNOT_FAEAD_OFFSET);
 999         fd_faead->ctrl = DPAA2_ANNOT_FAEAD_A2V | DPAA2_ANNOT_FAEAD_UPDV |
1000                                 DPAA2_ANNOT_FAEAD_UPD;
1001 }
1002
1003 /*
1004  * Callback to handle sending packets through WRIOP based interface
1005  */
1006 uint16_t
1007 dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
1008 {
1009         /* Function to transmit the frames to given device and VQ*/
1010         uint32_t loop, retry_count;
1011         int32_t ret;
1012         struct qbman_fd fd_arr[MAX_TX_RING_SLOTS];
1013         struct rte_mbuf *mi;
1014         uint32_t frames_to_send;
1015         struct rte_mempool *mp;
1016         struct qbman_eq_desc eqdesc;
1017         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
1018         struct qbman_swp *swp;
1019         uint16_t num_tx = 0;
1020         uint16_t bpid;
1021         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
1022         struct dpaa2_dev_priv *priv = eth_data->dev_private;
1023         uint32_t flags[MAX_TX_RING_SLOTS] = {0};
1024
1025         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
1026                 ret = dpaa2_affine_qbman_swp();
1027                 if (ret) {
1028                         DPAA2_PMD_ERR(
1029                                 "Failed to allocate IO portal, tid: %d\n",
1030                                 rte_gettid());
1031                         return 0;
1032                 }
1033         }
1034         swp = DPAA2_PER_LCORE_PORTAL;
1035
1036         DPAA2_PMD_DP_DEBUG("===> eth_data =%p, fqid =%d\n",
1037                         eth_data, dpaa2_q->fqid);
1038
1039 #ifdef RTE_LIBRTE_IEEE1588
1040         /* IEEE1588 driver need pointer to tx confirmation queue
1041          * corresponding to last packet transmitted for reading
1042          * the timestamp
1043          */
1044         priv->next_tx_conf_queue = dpaa2_q->tx_conf_queue;
1045         dpaa2_dev_tx_conf(dpaa2_q->tx_conf_queue);
1046 #endif
1047
1048         /*Prepare enqueue descriptor*/
1049         qbman_eq_desc_clear(&eqdesc);
1050         qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ);
1051         qbman_eq_desc_set_fq(&eqdesc, dpaa2_q->fqid);
1052
1053         /*Clear the unused FD fields before sending*/
1054         while (nb_pkts) {
1055                 /*Check if the queue is congested*/
1056                 retry_count = 0;
1057                 while (qbman_result_SCN_state(dpaa2_q->cscn)) {
1058                         retry_count++;
1059                         /* Retry for some time before giving up */
1060                         if (retry_count > CONG_RETRY_COUNT)
1061                                 goto skip_tx;
1062                 }
1063
1064                 frames_to_send = (nb_pkts > dpaa2_eqcr_size) ?
1065                         dpaa2_eqcr_size : nb_pkts;
1066
1067                 for (loop = 0; loop < frames_to_send; loop++) {
1068                         if ((*bufs)->seqn) {
1069                                 uint8_t dqrr_index = (*bufs)->seqn - 1;
1070
1071                                 flags[loop] = QBMAN_ENQUEUE_FLAG_DCA |
1072                                                 dqrr_index;
1073                                 DPAA2_PER_LCORE_DQRR_SIZE--;
1074                                 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dqrr_index);
1075                                 (*bufs)->seqn = DPAA2_INVALID_MBUF_SEQN;
1076                         }
1077
1078                         if (likely(RTE_MBUF_DIRECT(*bufs))) {
1079                                 mp = (*bufs)->pool;
1080                                 /* Check the basic scenario and set
1081                                  * the FD appropriately here itself.
1082                                  */
1083                                 if (likely(mp && mp->ops_index ==
1084                                     priv->bp_list->dpaa2_ops_index &&
1085                                     (*bufs)->nb_segs == 1 &&
1086                                     rte_mbuf_refcnt_read((*bufs)) == 1)) {
1087                                         if (unlikely(((*bufs)->ol_flags
1088                                                 & PKT_TX_VLAN_PKT) ||
1089                                                 (eth_data->dev_conf.txmode.offloads
1090                                                 & DEV_TX_OFFLOAD_VLAN_INSERT))) {
1091                                                 ret = rte_vlan_insert(bufs);
1092                                                 if (ret)
1093                                                         goto send_n_return;
1094                                         }
1095                                         DPAA2_MBUF_TO_CONTIG_FD((*bufs),
1096                                         &fd_arr[loop], mempool_to_bpid(mp));
1097                                         bufs++;
1098 #ifdef RTE_LIBRTE_IEEE1588
1099                                         enable_tx_tstamp(&fd_arr[loop]);
1100 #endif
1101                                         continue;
1102                                 }
1103                         } else {
1104                                 mi = rte_mbuf_from_indirect(*bufs);
1105                                 mp = mi->pool;
1106                         }
1107                         /* Not a hw_pkt pool allocated frame */
1108                         if (unlikely(!mp || !priv->bp_list)) {
1109                                 DPAA2_PMD_ERR("Err: No buffer pool attached");
1110                                 goto send_n_return;
1111                         }
1112
1113                         if (unlikely(((*bufs)->ol_flags & PKT_TX_VLAN_PKT) ||
1114                                 (eth_data->dev_conf.txmode.offloads
1115                                 & DEV_TX_OFFLOAD_VLAN_INSERT))) {
1116                                 int ret = rte_vlan_insert(bufs);
1117                                 if (ret)
1118                                         goto send_n_return;
1119                         }
1120                         if (mp->ops_index != priv->bp_list->dpaa2_ops_index) {
1121                                 DPAA2_PMD_WARN("Non DPAA2 buffer pool");
1122                                 /* alloc should be from the default buffer pool
1123                                  * attached to this interface
1124                                  */
1125                                 bpid = priv->bp_list->buf_pool.bpid;
1126
1127                                 if (unlikely((*bufs)->nb_segs > 1)) {
1128                                         DPAA2_PMD_ERR("S/G support not added"
1129                                                 " for non hw offload buffer");
1130                                         goto send_n_return;
1131                                 }
1132                                 if (eth_copy_mbuf_to_fd(*bufs,
1133                                                         &fd_arr[loop], bpid)) {
1134                                         goto send_n_return;
1135                                 }
1136                                 /* free the original packet */
1137                                 rte_pktmbuf_free(*bufs);
1138                         } else {
1139                                 bpid = mempool_to_bpid(mp);
1140                                 if (unlikely((*bufs)->nb_segs > 1)) {
1141                                         if (eth_mbuf_to_sg_fd(*bufs,
1142                                                         &fd_arr[loop], bpid))
1143                                                 goto send_n_return;
1144                                 } else {
1145                                         eth_mbuf_to_fd(*bufs,
1146                                                        &fd_arr[loop], bpid);
1147                                 }
1148                         }
1149 #ifdef RTE_LIBRTE_IEEE1588
1150                         enable_tx_tstamp(&fd_arr[loop]);
1151 #endif
1152                         bufs++;
1153                 }
1154
1155                 loop = 0;
1156                 retry_count = 0;
1157                 while (loop < frames_to_send) {
1158                         ret = qbman_swp_enqueue_multiple(swp, &eqdesc,
1159                                         &fd_arr[loop], &flags[loop],
1160                                         frames_to_send - loop);
1161                         if (unlikely(ret < 0)) {
1162                                 retry_count++;
1163                                 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) {
1164                                         num_tx += loop;
1165                                         nb_pkts -= loop;
1166                                         goto send_n_return;
1167                                 }
1168                         } else {
1169                                 loop += ret;
1170                                 retry_count = 0;
1171                         }
1172                 }
1173
1174                 num_tx += loop;
1175                 nb_pkts -= loop;
1176         }
1177         dpaa2_q->tx_pkts += num_tx;
1178         return num_tx;
1179
1180 send_n_return:
1181         /* send any already prepared fd */
1182         if (loop) {
1183                 unsigned int i = 0;
1184
1185                 retry_count = 0;
1186                 while (i < loop) {
1187                         ret = qbman_swp_enqueue_multiple(swp, &eqdesc,
1188                                                          &fd_arr[i],
1189                                                          &flags[i],
1190                                                          loop - i);
1191                         if (unlikely(ret < 0)) {
1192                                 retry_count++;
1193                                 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT)
1194                                         break;
1195                         } else {
1196                                 i += ret;
1197                                 retry_count = 0;
1198                         }
1199                 }
1200                 num_tx += i;
1201         }
1202 skip_tx:
1203         dpaa2_q->tx_pkts += num_tx;
1204         return num_tx;
1205 }
1206
1207 void
1208 dpaa2_dev_free_eqresp_buf(uint16_t eqresp_ci)
1209 {
1210         struct dpaa2_dpio_dev *dpio_dev = DPAA2_PER_LCORE_DPIO;
1211         struct qbman_fd *fd;
1212         struct rte_mbuf *m;
1213
1214         fd = qbman_result_eqresp_fd(&dpio_dev->eqresp[eqresp_ci]);
1215
1216         /* Setting port id does not matter as we are to free the mbuf */
1217         m = eth_fd_to_mbuf(fd, 0);
1218         rte_pktmbuf_free(m);
1219 }
1220
1221 static void
1222 dpaa2_set_enqueue_descriptor(struct dpaa2_queue *dpaa2_q,
1223                              struct rte_mbuf *m,
1224                              struct qbman_eq_desc *eqdesc)
1225 {
1226         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
1227         struct dpaa2_dev_priv *priv = eth_data->dev_private;
1228         struct dpaa2_dpio_dev *dpio_dev = DPAA2_PER_LCORE_DPIO;
1229         struct eqresp_metadata *eqresp_meta;
1230         uint16_t orpid, seqnum;
1231         uint8_t dq_idx;
1232
1233         qbman_eq_desc_set_fq(eqdesc, dpaa2_q->fqid);
1234
1235         if (m->seqn & DPAA2_ENQUEUE_FLAG_ORP) {
1236                 orpid = (m->seqn & DPAA2_EQCR_OPRID_MASK) >>
1237                         DPAA2_EQCR_OPRID_SHIFT;
1238                 seqnum = (m->seqn & DPAA2_EQCR_SEQNUM_MASK) >>
1239                         DPAA2_EQCR_SEQNUM_SHIFT;
1240
1241                 if (!priv->en_loose_ordered) {
1242                         qbman_eq_desc_set_orp(eqdesc, 1, orpid, seqnum, 0);
1243                         qbman_eq_desc_set_response(eqdesc, (uint64_t)
1244                                 DPAA2_VADDR_TO_IOVA(&dpio_dev->eqresp[
1245                                 dpio_dev->eqresp_pi]), 1);
1246                         qbman_eq_desc_set_token(eqdesc, 1);
1247
1248                         eqresp_meta = &dpio_dev->eqresp_meta[
1249                                 dpio_dev->eqresp_pi];
1250                         eqresp_meta->dpaa2_q = dpaa2_q;
1251                         eqresp_meta->mp = m->pool;
1252
1253                         dpio_dev->eqresp_pi + 1 < MAX_EQ_RESP_ENTRIES ?
1254                                 dpio_dev->eqresp_pi++ :
1255                                 (dpio_dev->eqresp_pi = 0);
1256                 } else {
1257                         qbman_eq_desc_set_orp(eqdesc, 0, orpid, seqnum, 0);
1258                 }
1259         } else {
1260                 dq_idx = m->seqn - 1;
1261                 qbman_eq_desc_set_dca(eqdesc, 1, dq_idx, 0);
1262                 DPAA2_PER_LCORE_DQRR_SIZE--;
1263                 DPAA2_PER_LCORE_DQRR_HELD &= ~(1 << dq_idx);
1264         }
1265         m->seqn = DPAA2_INVALID_MBUF_SEQN;
1266 }
1267
1268 /* Callback to handle sending ordered packets through WRIOP based interface */
1269 uint16_t
1270 dpaa2_dev_tx_ordered(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
1271 {
1272         /* Function to transmit the frames to given device and VQ*/
1273         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
1274         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
1275         struct dpaa2_dev_priv *priv = eth_data->dev_private;
1276         struct dpaa2_queue *order_sendq = (struct dpaa2_queue *)priv->tx_vq[0];
1277         struct qbman_fd fd_arr[MAX_TX_RING_SLOTS];
1278         struct rte_mbuf *mi;
1279         struct rte_mempool *mp;
1280         struct qbman_eq_desc eqdesc[MAX_TX_RING_SLOTS];
1281         struct qbman_swp *swp;
1282         uint32_t frames_to_send, num_free_eq_desc;
1283         uint32_t loop, retry_count;
1284         int32_t ret;
1285         uint16_t num_tx = 0;
1286         uint16_t bpid;
1287
1288         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
1289                 ret = dpaa2_affine_qbman_swp();
1290                 if (ret) {
1291                         DPAA2_PMD_ERR(
1292                                 "Failed to allocate IO portal, tid: %d\n",
1293                                 rte_gettid());
1294                         return 0;
1295                 }
1296         }
1297         swp = DPAA2_PER_LCORE_PORTAL;
1298
1299         DPAA2_PMD_DP_DEBUG("===> eth_data =%p, fqid =%d\n",
1300                            eth_data, dpaa2_q->fqid);
1301
1302         /* This would also handle normal and atomic queues as any type
1303          * of packet can be enqueued when ordered queues are being used.
1304          */
1305         while (nb_pkts) {
1306                 /*Check if the queue is congested*/
1307                 retry_count = 0;
1308                 while (qbman_result_SCN_state(dpaa2_q->cscn)) {
1309                         retry_count++;
1310                         /* Retry for some time before giving up */
1311                         if (retry_count > CONG_RETRY_COUNT)
1312                                 goto skip_tx;
1313                 }
1314
1315                 frames_to_send = (nb_pkts > dpaa2_eqcr_size) ?
1316                         dpaa2_eqcr_size : nb_pkts;
1317
1318                 if (!priv->en_loose_ordered) {
1319                         if ((*bufs)->seqn & DPAA2_ENQUEUE_FLAG_ORP) {
1320                                 num_free_eq_desc = dpaa2_free_eq_descriptors();
1321                                 if (num_free_eq_desc < frames_to_send)
1322                                         frames_to_send = num_free_eq_desc;
1323                         }
1324                 }
1325
1326                 for (loop = 0; loop < frames_to_send; loop++) {
1327                         /*Prepare enqueue descriptor*/
1328                         qbman_eq_desc_clear(&eqdesc[loop]);
1329
1330                         if ((*bufs)->seqn) {
1331                                 /* Use only queue 0 for Tx in case of atomic/
1332                                  * ordered packets as packets can get unordered
1333                                  * when being tranmitted out from the interface
1334                                  */
1335                                 dpaa2_set_enqueue_descriptor(order_sendq,
1336                                                              (*bufs),
1337                                                              &eqdesc[loop]);
1338                         } else {
1339                                 qbman_eq_desc_set_no_orp(&eqdesc[loop],
1340                                                          DPAA2_EQ_RESP_ERR_FQ);
1341                                 qbman_eq_desc_set_fq(&eqdesc[loop],
1342                                                      dpaa2_q->fqid);
1343                         }
1344
1345                         if (likely(RTE_MBUF_DIRECT(*bufs))) {
1346                                 mp = (*bufs)->pool;
1347                                 /* Check the basic scenario and set
1348                                  * the FD appropriately here itself.
1349                                  */
1350                                 if (likely(mp && mp->ops_index ==
1351                                     priv->bp_list->dpaa2_ops_index &&
1352                                     (*bufs)->nb_segs == 1 &&
1353                                     rte_mbuf_refcnt_read((*bufs)) == 1)) {
1354                                         if (unlikely((*bufs)->ol_flags
1355                                                 & PKT_TX_VLAN_PKT)) {
1356                                           ret = rte_vlan_insert(bufs);
1357                                           if (ret)
1358                                                 goto send_n_return;
1359                                         }
1360                                         DPAA2_MBUF_TO_CONTIG_FD((*bufs),
1361                                                 &fd_arr[loop],
1362                                                 mempool_to_bpid(mp));
1363                                         bufs++;
1364                                         continue;
1365                                 }
1366                         } else {
1367                                 mi = rte_mbuf_from_indirect(*bufs);
1368                                 mp = mi->pool;
1369                         }
1370                         /* Not a hw_pkt pool allocated frame */
1371                         if (unlikely(!mp || !priv->bp_list)) {
1372                                 DPAA2_PMD_ERR("Err: No buffer pool attached");
1373                                 goto send_n_return;
1374                         }
1375
1376                         if (mp->ops_index != priv->bp_list->dpaa2_ops_index) {
1377                                 DPAA2_PMD_WARN("Non DPAA2 buffer pool");
1378                                 /* alloc should be from the default buffer pool
1379                                  * attached to this interface
1380                                  */
1381                                 bpid = priv->bp_list->buf_pool.bpid;
1382
1383                                 if (unlikely((*bufs)->nb_segs > 1)) {
1384                                         DPAA2_PMD_ERR(
1385                                                 "S/G not supp for non hw offload buffer");
1386                                         goto send_n_return;
1387                                 }
1388                                 if (eth_copy_mbuf_to_fd(*bufs,
1389                                                         &fd_arr[loop], bpid)) {
1390                                         goto send_n_return;
1391                                 }
1392                                 /* free the original packet */
1393                                 rte_pktmbuf_free(*bufs);
1394                         } else {
1395                                 bpid = mempool_to_bpid(mp);
1396                                 if (unlikely((*bufs)->nb_segs > 1)) {
1397                                         if (eth_mbuf_to_sg_fd(*bufs,
1398                                                               &fd_arr[loop],
1399                                                               bpid))
1400                                                 goto send_n_return;
1401                                 } else {
1402                                         eth_mbuf_to_fd(*bufs,
1403                                                        &fd_arr[loop], bpid);
1404                                 }
1405                         }
1406                         bufs++;
1407                 }
1408
1409                 loop = 0;
1410                 retry_count = 0;
1411                 while (loop < frames_to_send) {
1412                         ret = qbman_swp_enqueue_multiple_desc(swp,
1413                                         &eqdesc[loop], &fd_arr[loop],
1414                                         frames_to_send - loop);
1415                         if (unlikely(ret < 0)) {
1416                                 retry_count++;
1417                                 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT) {
1418                                         num_tx += loop;
1419                                         nb_pkts -= loop;
1420                                         goto send_n_return;
1421                                 }
1422                         } else {
1423                                 loop += ret;
1424                                 retry_count = 0;
1425                         }
1426                 }
1427
1428                 num_tx += loop;
1429                 nb_pkts -= loop;
1430         }
1431         dpaa2_q->tx_pkts += num_tx;
1432         return num_tx;
1433
1434 send_n_return:
1435         /* send any already prepared fd */
1436         if (loop) {
1437                 unsigned int i = 0;
1438
1439                 retry_count = 0;
1440                 while (i < loop) {
1441                         ret = qbman_swp_enqueue_multiple_desc(swp,
1442                                        &eqdesc[loop], &fd_arr[i], loop - i);
1443                         if (unlikely(ret < 0)) {
1444                                 retry_count++;
1445                                 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT)
1446                                         break;
1447                         } else {
1448                                 i += ret;
1449                                 retry_count = 0;
1450                         }
1451                 }
1452                 num_tx += i;
1453         }
1454 skip_tx:
1455         dpaa2_q->tx_pkts += num_tx;
1456         return num_tx;
1457 }
1458
1459 /**
1460  * Dummy DPDK callback for TX.
1461  *
1462  * This function is used to temporarily replace the real callback during
1463  * unsafe control operations on the queue, or in case of error.
1464  *
1465  * @param dpdk_txq
1466  *   Generic pointer to TX queue structure.
1467  * @param[in] pkts
1468  *   Packets to transmit.
1469  * @param pkts_n
1470  *   Number of packets in array.
1471  *
1472  * @return
1473  *   Number of packets successfully transmitted (<= pkts_n).
1474  */
1475 uint16_t
1476 dummy_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
1477 {
1478         (void)queue;
1479         (void)bufs;
1480         (void)nb_pkts;
1481         return 0;
1482 }
1483
1484 #if defined(RTE_TOOLCHAIN_GCC)
1485 #pragma GCC diagnostic push
1486 #pragma GCC diagnostic ignored "-Wcast-qual"
1487 #elif defined(RTE_TOOLCHAIN_CLANG)
1488 #pragma clang diagnostic push
1489 #pragma clang diagnostic ignored "-Wcast-qual"
1490 #endif
1491
1492 /* This function loopbacks all the received packets.*/
1493 uint16_t
1494 dpaa2_dev_loopback_rx(void *queue,
1495                       struct rte_mbuf **bufs __rte_unused,
1496                       uint16_t nb_pkts)
1497 {
1498         /* Function receive frames for a given device and VQ*/
1499         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
1500         struct qbman_result *dq_storage, *dq_storage1 = NULL;
1501         uint32_t fqid = dpaa2_q->fqid;
1502         int ret, num_rx = 0, num_tx = 0, pull_size;
1503         uint8_t pending, status;
1504         struct qbman_swp *swp;
1505         struct qbman_fd *fd[DPAA2_LX2_DQRR_RING_SIZE];
1506         struct qbman_pull_desc pulldesc;
1507         struct qbman_eq_desc eqdesc;
1508         struct queue_storage_info_t *q_storage = dpaa2_q->q_storage;
1509         struct rte_eth_dev_data *eth_data = dpaa2_q->eth_data;
1510         struct dpaa2_dev_priv *priv = eth_data->dev_private;
1511         struct dpaa2_queue *tx_q = priv->tx_vq[0];
1512         /* todo - currently we are using 1st TX queue only for loopback*/
1513
1514         if (unlikely(!DPAA2_PER_LCORE_ETHRX_DPIO)) {
1515                 ret = dpaa2_affine_qbman_ethrx_swp();
1516                 if (ret) {
1517                         DPAA2_PMD_ERR("Failure in affining portal");
1518                         return 0;
1519                 }
1520         }
1521         swp = DPAA2_PER_LCORE_ETHRX_PORTAL;
1522         pull_size = (nb_pkts > dpaa2_dqrr_size) ? dpaa2_dqrr_size : nb_pkts;
1523         if (unlikely(!q_storage->active_dqs)) {
1524                 q_storage->toggle = 0;
1525                 dq_storage = q_storage->dq_storage[q_storage->toggle];
1526                 q_storage->last_num_pkts = pull_size;
1527                 qbman_pull_desc_clear(&pulldesc);
1528                 qbman_pull_desc_set_numframes(&pulldesc,
1529                                               q_storage->last_num_pkts);
1530                 qbman_pull_desc_set_fq(&pulldesc, fqid);
1531                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
1532                         (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
1533                 if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) {
1534                         while (!qbman_check_command_complete(
1535                                get_swp_active_dqs(
1536                                DPAA2_PER_LCORE_ETHRX_DPIO->index)))
1537                                 ;
1538                         clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index);
1539                 }
1540                 while (1) {
1541                         if (qbman_swp_pull(swp, &pulldesc)) {
1542                                 DPAA2_PMD_DP_DEBUG(
1543                                         "VDQ command not issued.QBMAN busy\n");
1544                                 /* Portal was busy, try again */
1545                                 continue;
1546                         }
1547                         break;
1548                 }
1549                 q_storage->active_dqs = dq_storage;
1550                 q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index;
1551                 set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index,
1552                                    dq_storage);
1553         }
1554
1555         dq_storage = q_storage->active_dqs;
1556         rte_prefetch0((void *)(size_t)(dq_storage));
1557         rte_prefetch0((void *)(size_t)(dq_storage + 1));
1558
1559         /* Prepare next pull descriptor. This will give space for the
1560          * prefething done on DQRR entries
1561          */
1562         q_storage->toggle ^= 1;
1563         dq_storage1 = q_storage->dq_storage[q_storage->toggle];
1564         qbman_pull_desc_clear(&pulldesc);
1565         qbman_pull_desc_set_numframes(&pulldesc, pull_size);
1566         qbman_pull_desc_set_fq(&pulldesc, fqid);
1567         qbman_pull_desc_set_storage(&pulldesc, dq_storage1,
1568                 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage1)), 1);
1569
1570         /*Prepare enqueue descriptor*/
1571         qbman_eq_desc_clear(&eqdesc);
1572         qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ);
1573         qbman_eq_desc_set_response(&eqdesc, 0, 0);
1574         qbman_eq_desc_set_fq(&eqdesc, tx_q->fqid);
1575
1576         /* Check if the previous issued command is completed.
1577          * Also seems like the SWP is shared between the Ethernet Driver
1578          * and the SEC driver.
1579          */
1580         while (!qbman_check_command_complete(dq_storage))
1581                 ;
1582         if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id))
1583                 clear_swp_active_dqs(q_storage->active_dpio_id);
1584
1585         pending = 1;
1586
1587         do {
1588                 /* Loop until the dq_storage is updated with
1589                  * new token by QBMAN
1590                  */
1591                 while (!qbman_check_new_result(dq_storage))
1592                         ;
1593                 rte_prefetch0((void *)((size_t)(dq_storage + 2)));
1594                 /* Check whether Last Pull command is Expired and
1595                  * setting Condition for Loop termination
1596                  */
1597                 if (qbman_result_DQ_is_pull_complete(dq_storage)) {
1598                         pending = 0;
1599                         /* Check for valid frame. */
1600                         status = qbman_result_DQ_flags(dq_storage);
1601                         if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0))
1602                                 continue;
1603                 }
1604                 fd[num_rx] = (struct qbman_fd *)qbman_result_DQ_fd(dq_storage);
1605
1606                 dq_storage++;
1607                 num_rx++;
1608         } while (pending);
1609
1610         while (num_tx < num_rx) {
1611                 num_tx += qbman_swp_enqueue_multiple_fd(swp, &eqdesc,
1612                                 &fd[num_tx], 0, num_rx - num_tx);
1613         }
1614
1615         if (check_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)) {
1616                 while (!qbman_check_command_complete(
1617                        get_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index)))
1618                         ;
1619                 clear_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index);
1620         }
1621         /* issue a volatile dequeue command for next pull */
1622         while (1) {
1623                 if (qbman_swp_pull(swp, &pulldesc)) {
1624                         DPAA2_PMD_DP_DEBUG("VDQ command is not issued."
1625                                           "QBMAN is busy (2)\n");
1626                         continue;
1627                 }
1628                 break;
1629         }
1630         q_storage->active_dqs = dq_storage1;
1631         q_storage->active_dpio_id = DPAA2_PER_LCORE_ETHRX_DPIO->index;
1632         set_swp_active_dqs(DPAA2_PER_LCORE_ETHRX_DPIO->index, dq_storage1);
1633
1634         dpaa2_q->rx_pkts += num_rx;
1635         dpaa2_q->tx_pkts += num_tx;
1636
1637         return 0;
1638 }
1639 #if defined(RTE_TOOLCHAIN_GCC)
1640 #pragma GCC diagnostic pop
1641 #elif defined(RTE_TOOLCHAIN_CLANG)
1642 #pragma clang diagnostic pop
1643 #endif