drivers/net/dpaa2/dpaa2_rxtx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
   4  *   Copyright 2016 NXP
   5  *
   6  */
   7
   8 #include <time.h>
   9 #include <net/if.h>
  10
  11 #include <rte_mbuf.h>
  12 #include <rte_ethdev.h>
  13 #include <rte_malloc.h>
  14 #include <rte_memcpy.h>
  15 #include <rte_string_fns.h>
  16 #include <rte_dev.h>
  17
  18 #include <fslmc_logs.h>
  19 #include <fslmc_vfio.h>
  20 #include <dpaa2_hw_pvt.h>
  21 #include <dpaa2_hw_dpio.h>
  22 #include <dpaa2_hw_mempool.h>
  23
  24 #include "dpaa2_ethdev.h"
  25 #include "base/dpaa2_hw_dpni_annot.h"
  26
  27 static inline void __attribute__((hot))
  28 dpaa2_dev_rx_parse_frc(struct rte_mbuf *m, uint16_t frc)
  29 {
  30         PMD_RX_LOG(DEBUG, "frc = 0x%x   ", frc);
  31
  32         m->packet_type = RTE_PTYPE_UNKNOWN;
  33         switch (frc) {
  34         case DPAA2_PKT_TYPE_ETHER:
  35                 m->packet_type = RTE_PTYPE_L2_ETHER;
  36                 break;
  37         case DPAA2_PKT_TYPE_IPV4:
  38                 m->packet_type = RTE_PTYPE_L2_ETHER |
  39                         RTE_PTYPE_L3_IPV4;
  40                 break;
  41         case DPAA2_PKT_TYPE_IPV6:
  42                 m->packet_type = RTE_PTYPE_L2_ETHER |
  43                         RTE_PTYPE_L3_IPV6;
  44                 break;
  45         case DPAA2_PKT_TYPE_IPV4_EXT:
  46                 m->packet_type = RTE_PTYPE_L2_ETHER |
  47                         RTE_PTYPE_L3_IPV4_EXT;
  48                 break;
  49         case DPAA2_PKT_TYPE_IPV6_EXT:
  50                 m->packet_type = RTE_PTYPE_L2_ETHER |
  51                         RTE_PTYPE_L3_IPV6_EXT;
  52                 break;
  53         case DPAA2_PKT_TYPE_IPV4_TCP:
  54                 m->packet_type = RTE_PTYPE_L2_ETHER |
  55                         RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_TCP;
  56                 break;
  57         case DPAA2_PKT_TYPE_IPV6_TCP:
  58                 m->packet_type = RTE_PTYPE_L2_ETHER |
  59                         RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_TCP;
  60                 break;
  61         case DPAA2_PKT_TYPE_IPV4_UDP:
  62                 m->packet_type = RTE_PTYPE_L2_ETHER |
  63                         RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_UDP;
  64                 break;
  65         case DPAA2_PKT_TYPE_IPV6_UDP:
  66                 m->packet_type = RTE_PTYPE_L2_ETHER |
  67                         RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_UDP;
  68                 break;
  69         case DPAA2_PKT_TYPE_IPV4_SCTP:
  70                 m->packet_type = RTE_PTYPE_L2_ETHER |
  71                         RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_SCTP;
  72                 break;
  73         case DPAA2_PKT_TYPE_IPV6_SCTP:
  74                 m->packet_type = RTE_PTYPE_L2_ETHER |
  75                         RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_SCTP;
  76                 break;
  77         case DPAA2_PKT_TYPE_IPV4_ICMP:
  78                 m->packet_type = RTE_PTYPE_L2_ETHER |
  79                         RTE_PTYPE_L3_IPV4 | RTE_PTYPE_L4_ICMP;
  80                 break;
  81         case DPAA2_PKT_TYPE_IPV6_ICMP:
  82                 m->packet_type = RTE_PTYPE_L2_ETHER |
  83                         RTE_PTYPE_L3_IPV6 | RTE_PTYPE_L4_ICMP;
  84                 break;
  85         case DPAA2_PKT_TYPE_VLAN_1:
  86         case DPAA2_PKT_TYPE_VLAN_2:
  87                 m->ol_flags |= PKT_RX_VLAN;
  88                 break;
  89         /* More switch cases can be added */
  90         /* TODO: Add handling for checksum error check from FRC */
  91         default:
  92                 m->packet_type = RTE_PTYPE_UNKNOWN;
  93         }
  94 }
  95
  96 static inline uint32_t __attribute__((hot))
  97 dpaa2_dev_rx_parse_slow(uint64_t hw_annot_addr)
  98 {
  99         uint32_t pkt_type = RTE_PTYPE_UNKNOWN;
 100         struct dpaa2_annot_hdr *annotation =
 101                         (struct dpaa2_annot_hdr *)hw_annot_addr;
 102
 103         PMD_RX_LOG(DEBUG, "annotation = 0x%lx   ", annotation->word4);
 104         if (BIT_ISSET_AT_POS(annotation->word3, L2_ARP_PRESENT)) {
 105                 pkt_type = RTE_PTYPE_L2_ETHER_ARP;
 106                 goto parse_done;
 107         } else if (BIT_ISSET_AT_POS(annotation->word3, L2_ETH_MAC_PRESENT)) {
 108                 pkt_type = RTE_PTYPE_L2_ETHER;
 109         } else {
 110                 goto parse_done;
 111         }
 112
 113         if (BIT_ISSET_AT_POS(annotation->word4, L3_IPV4_1_PRESENT |
 114                              L3_IPV4_N_PRESENT)) {
 115                 pkt_type |= RTE_PTYPE_L3_IPV4;
 116                 if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_OPT_PRESENT |
 117                         L3_IP_N_OPT_PRESENT))
 118                         pkt_type |= RTE_PTYPE_L3_IPV4_EXT;
 119
 120         } else if (BIT_ISSET_AT_POS(annotation->word4, L3_IPV6_1_PRESENT |
 121                   L3_IPV6_N_PRESENT)) {
 122                 pkt_type |= RTE_PTYPE_L3_IPV6;
 123                 if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_OPT_PRESENT |
 124                     L3_IP_N_OPT_PRESENT))
 125                         pkt_type |= RTE_PTYPE_L3_IPV6_EXT;
 126         } else {
 127                 goto parse_done;
 128         }
 129
 130         if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_1_FIRST_FRAGMENT |
 131             L3_IP_1_MORE_FRAGMENT |
 132             L3_IP_N_FIRST_FRAGMENT |
 133             L3_IP_N_MORE_FRAGMENT)) {
 134                 pkt_type |= RTE_PTYPE_L4_FRAG;
 135                 goto parse_done;
 136         } else {
 137                 pkt_type |= RTE_PTYPE_L4_NONFRAG;
 138         }
 139
 140         if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_UDP_PRESENT))
 141                 pkt_type |= RTE_PTYPE_L4_UDP;
 142
 143         else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_TCP_PRESENT))
 144                 pkt_type |= RTE_PTYPE_L4_TCP;
 145
 146         else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_SCTP_PRESENT))
 147                 pkt_type |= RTE_PTYPE_L4_SCTP;
 148
 149         else if (BIT_ISSET_AT_POS(annotation->word4, L3_PROTO_ICMP_PRESENT))
 150                 pkt_type |= RTE_PTYPE_L4_ICMP;
 151
 152         else if (BIT_ISSET_AT_POS(annotation->word4, L3_IP_UNKNOWN_PROTOCOL))
 153                 pkt_type |= RTE_PTYPE_UNKNOWN;
 154
 155 parse_done:
 156         return pkt_type;
 157 }
 158
 159
 160 static inline uint32_t __attribute__((hot))
 161 dpaa2_dev_rx_parse(uint64_t hw_annot_addr)
 162 {
 163         struct dpaa2_annot_hdr *annotation =
 164                         (struct dpaa2_annot_hdr *)hw_annot_addr;
 165
 166         PMD_RX_LOG(DEBUG, "annotation = 0x%lx   ", annotation->word4);
 167
 168         /* Return some common types from parse processing */
 169         switch (annotation->word4) {
 170         case DPAA2_L3_IPv4:
 171                 return RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4;
 172         case DPAA2_L3_IPv6:
 173                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6;
 174         case DPAA2_L3_IPv4_TCP:
 175                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
 176                                 RTE_PTYPE_L4_TCP;
 177         case DPAA2_L3_IPv4_UDP:
 178                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
 179                                 RTE_PTYPE_L4_UDP;
 180         case DPAA2_L3_IPv6_TCP:
 181                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
 182                                 RTE_PTYPE_L4_TCP;
 183         case DPAA2_L3_IPv6_UDP:
 184                 return  RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
 185                                 RTE_PTYPE_L4_UDP;
 186         default:
 187                 PMD_RX_LOG(DEBUG, "Slow parse the parsing results\n");
 188                 break;
 189         }
 190
 191         return dpaa2_dev_rx_parse_slow(hw_annot_addr);
 192 }
 193
 194 static inline void __attribute__((hot))
 195 dpaa2_dev_rx_offload(uint64_t hw_annot_addr, struct rte_mbuf *mbuf)
 196 {
 197         struct dpaa2_annot_hdr *annotation =
 198                 (struct dpaa2_annot_hdr *)hw_annot_addr;
 199
 200         if (BIT_ISSET_AT_POS(annotation->word3,
 201                              L2_VLAN_1_PRESENT | L2_VLAN_N_PRESENT))
 202                 mbuf->ol_flags |= PKT_RX_VLAN;
 203
 204         if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L3CE))
 205                 mbuf->ol_flags |= PKT_RX_IP_CKSUM_BAD;
 206
 207         if (BIT_ISSET_AT_POS(annotation->word8, DPAA2_ETH_FAS_L4CE))
 208                 mbuf->ol_flags |= PKT_RX_L4_CKSUM_BAD;
 209 }
 210
 211 static inline struct rte_mbuf *__attribute__((hot))
 212 eth_sg_fd_to_mbuf(const struct qbman_fd *fd)
 213 {
 214         struct qbman_sge *sgt, *sge;
 215         dma_addr_t sg_addr;
 216         int i = 0;
 217         uint64_t fd_addr;
 218         struct rte_mbuf *first_seg, *next_seg, *cur_seg, *temp;
 219
 220         fd_addr = (uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
 221
 222         /* Get Scatter gather table address */
 223         sgt = (struct qbman_sge *)(fd_addr + DPAA2_GET_FD_OFFSET(fd));
 224
 225         sge = &sgt[i++];
 226         sg_addr = (uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FLE_ADDR(sge));
 227
 228         /* First Scatter gather entry */
 229         first_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr,
 230                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
 231         /* Prepare all the metadata for first segment */
 232         first_seg->buf_addr = (uint8_t *)sg_addr;
 233         first_seg->ol_flags = 0;
 234         first_seg->data_off = DPAA2_GET_FLE_OFFSET(sge);
 235         first_seg->data_len = sge->length  & 0x1FFFF;
 236         first_seg->pkt_len = DPAA2_GET_FD_LEN(fd);
 237         first_seg->nb_segs = 1;
 238         first_seg->next = NULL;
 239         if (dpaa2_svr_family == SVR_LX2160A)
 240                 dpaa2_dev_rx_parse_frc(first_seg,
 241                                 DPAA2_GET_FD_FRC_PARSE_SUM(fd));
 242         else {
 243                 first_seg->packet_type = dpaa2_dev_rx_parse(
 244                          (uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd))
 245                          + DPAA2_FD_PTA_SIZE);
 246                 dpaa2_dev_rx_offload((uint64_t)DPAA2_IOVA_TO_VADDR(
 247                         DPAA2_GET_FD_ADDR(fd)) +
 248                         DPAA2_FD_PTA_SIZE, first_seg);
 249         }
 250         rte_mbuf_refcnt_set(first_seg, 1);
 251         cur_seg = first_seg;
 252         while (!DPAA2_SG_IS_FINAL(sge)) {
 253                 sge = &sgt[i++];
 254                 sg_addr = (uint64_t)DPAA2_IOVA_TO_VADDR(
 255                                 DPAA2_GET_FLE_ADDR(sge));
 256                 next_seg = DPAA2_INLINE_MBUF_FROM_BUF(sg_addr,
 257                         rte_dpaa2_bpid_info[DPAA2_GET_FLE_BPID(sge)].meta_data_size);
 258                 next_seg->buf_addr  = (uint8_t *)sg_addr;
 259                 next_seg->data_off  = DPAA2_GET_FLE_OFFSET(sge);
 260                 next_seg->data_len  = sge->length  & 0x1FFFF;
 261                 first_seg->nb_segs += 1;
 262                 rte_mbuf_refcnt_set(next_seg, 1);
 263                 cur_seg->next = next_seg;
 264                 next_seg->next = NULL;
 265                 cur_seg = next_seg;
 266         }
 267         temp = DPAA2_INLINE_MBUF_FROM_BUF(fd_addr,
 268                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
 269         rte_mbuf_refcnt_set(temp, 1);
 270         rte_pktmbuf_free_seg(temp);
 271
 272         return (void *)first_seg;
 273 }
 274
 275 static inline struct rte_mbuf *__attribute__((hot))
 276 eth_fd_to_mbuf(const struct qbman_fd *fd)
 277 {
 278         struct rte_mbuf *mbuf = DPAA2_INLINE_MBUF_FROM_BUF(
 279                 DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd)),
 280                      rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size);
 281
 282         /* need to repopulated some of the fields,
 283          * as they may have changed in last transmission
 284          */
 285         mbuf->nb_segs = 1;
 286         mbuf->ol_flags = 0;
 287         mbuf->data_off = DPAA2_GET_FD_OFFSET(fd);
 288         mbuf->data_len = DPAA2_GET_FD_LEN(fd);
 289         mbuf->pkt_len = mbuf->data_len;
 290         mbuf->next = NULL;
 291         rte_mbuf_refcnt_set(mbuf, 1);
 292
 293         /* Parse the packet */
 294         /* parse results for LX2 are there in FRC field of FD.
 295          * For other DPAA2 platforms , parse results are after
 296          * the private - sw annotation area
 297          */
 298
 299         if (dpaa2_svr_family == SVR_LX2160A)
 300                 dpaa2_dev_rx_parse_frc(mbuf, DPAA2_GET_FD_FRC_PARSE_SUM(fd));
 301         else {
 302                 mbuf->packet_type = dpaa2_dev_rx_parse(
 303                         (uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd))
 304                          + DPAA2_FD_PTA_SIZE);
 305                 dpaa2_dev_rx_offload((uint64_t)DPAA2_IOVA_TO_VADDR(
 306                              DPAA2_GET_FD_ADDR(fd)) +
 307                              DPAA2_FD_PTA_SIZE, mbuf);
 308         }
 309
 310         PMD_RX_LOG(DEBUG, "to mbuf - mbuf =%p, mbuf->buf_addr =%p, off = %d,"
 311                 "fd_off=%d fd =%lx, meta = %d  bpid =%d, len=%d\n",
 312                 mbuf, mbuf->buf_addr, mbuf->data_off,
 313                 DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ADDR(fd),
 314                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size,
 315                 DPAA2_GET_FD_BPID(fd), DPAA2_GET_FD_LEN(fd));
 316
 317         return mbuf;
 318 }
 319
 320 static int __attribute__ ((noinline)) __attribute__((hot))
 321 eth_mbuf_to_sg_fd(struct rte_mbuf *mbuf,
 322                   struct qbman_fd *fd, uint16_t bpid)
 323 {
 324         struct rte_mbuf *cur_seg = mbuf, *prev_seg, *mi, *temp;
 325         struct qbman_sge *sgt, *sge = NULL;
 326         int i;
 327
 328         /* First Prepare FD to be transmited*/
 329         /* Resetting the buffer pool id and offset field*/
 330         fd->simple.bpid_offset = 0;
 331
 332         if (unlikely(mbuf->ol_flags & PKT_TX_VLAN_PKT)) {
 333                 int ret = rte_vlan_insert(&mbuf);
 334                 if (ret)
 335                         return ret;
 336         }
 337
 338         temp = rte_pktmbuf_alloc(mbuf->pool);
 339         if (temp == NULL) {
 340                 PMD_TX_LOG(ERR, "No memory to allocate S/G table");
 341                 return -ENOMEM;
 342         }
 343
 344         DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(temp));
 345         DPAA2_SET_FD_LEN(fd, mbuf->pkt_len);
 346         DPAA2_SET_FD_OFFSET(fd, temp->data_off);
 347         DPAA2_SET_FD_BPID(fd, bpid);
 348         DPAA2_SET_FD_ASAL(fd, DPAA2_ASAL_VAL);
 349         DPAA2_FD_SET_FORMAT(fd, qbman_fd_sg);
 350         /*Set Scatter gather table and Scatter gather entries*/
 351         sgt = (struct qbman_sge *)(
 352                         (uint64_t)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd))
 353                         + DPAA2_GET_FD_OFFSET(fd));
 354
 355         for (i = 0; i < mbuf->nb_segs; i++) {
 356                 sge = &sgt[i];
 357                 /*Resetting the buffer pool id and offset field*/
 358                 sge->fin_bpid_offset = 0;
 359                 DPAA2_SET_FLE_ADDR(sge, DPAA2_MBUF_VADDR_TO_IOVA(cur_seg));
 360                 DPAA2_SET_FLE_OFFSET(sge, cur_seg->data_off);
 361                 sge->length = cur_seg->data_len;
 362                 if (RTE_MBUF_DIRECT(cur_seg)) {
 363                         if (rte_mbuf_refcnt_read(cur_seg) > 1) {
 364                                 /* If refcnt > 1, invalid bpid is set to ensure
 365                                  * buffer is not freed by HW
 366                                  */
 367                                 DPAA2_SET_FLE_IVP(sge);
 368                                 rte_mbuf_refcnt_update(cur_seg, -1);
 369                         } else
 370                                 DPAA2_SET_FLE_BPID(sge,
 371                                                 mempool_to_bpid(cur_seg->pool));
 372                         cur_seg = cur_seg->next;
 373                 } else {
 374                         /* Get owner MBUF from indirect buffer */
 375                         mi = rte_mbuf_from_indirect(cur_seg);
 376                         if (rte_mbuf_refcnt_read(mi) > 1) {
 377                                 /* If refcnt > 1, invalid bpid is set to ensure
 378                                  * owner buffer is not freed by HW
 379                                  */
 380                                 DPAA2_SET_FLE_IVP(sge);
 381                         } else {
 382                                 DPAA2_SET_FLE_BPID(sge,
 383                                                    mempool_to_bpid(mi->pool));
 384                                 rte_mbuf_refcnt_update(mi, 1);
 385                         }
 386                         prev_seg = cur_seg;
 387                         cur_seg = cur_seg->next;
 388                         prev_seg->next = NULL;
 389                         rte_pktmbuf_free(prev_seg);
 390                 }
 391         }
 392         DPAA2_SG_SET_FINAL(sge, true);
 393         return 0;
 394 }
 395
 396 static void
 397 eth_mbuf_to_fd(struct rte_mbuf *mbuf,
 398                struct qbman_fd *fd, uint16_t bpid) __attribute__((unused));
 399
 400 static void __attribute__ ((noinline)) __attribute__((hot))
 401 eth_mbuf_to_fd(struct rte_mbuf *mbuf,
 402                struct qbman_fd *fd, uint16_t bpid)
 403 {
 404         if (unlikely(mbuf->ol_flags & PKT_TX_VLAN_PKT)) {
 405                 if (rte_vlan_insert(&mbuf)) {
 406                         rte_pktmbuf_free(mbuf);
 407                         return;
 408                 }
 409         }
 410         /*Resetting the buffer pool id and offset field*/
 411         fd->simple.bpid_offset = 0;
 412
 413         DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(mbuf));
 414         DPAA2_SET_FD_LEN(fd, mbuf->data_len);
 415         DPAA2_SET_FD_BPID(fd, bpid);
 416         DPAA2_SET_FD_OFFSET(fd, mbuf->data_off);
 417         DPAA2_SET_FD_ASAL(fd, DPAA2_ASAL_VAL);
 418
 419         PMD_TX_LOG(DEBUG, "mbuf =%p, mbuf->buf_addr =%p, off = %d,"
 420                 "fd_off=%d fd =%lx, meta = %d  bpid =%d, len=%d\n",
 421                 mbuf, mbuf->buf_addr, mbuf->data_off,
 422                 DPAA2_GET_FD_OFFSET(fd), DPAA2_GET_FD_ADDR(fd),
 423                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size,
 424                 DPAA2_GET_FD_BPID(fd), DPAA2_GET_FD_LEN(fd));
 425         if (RTE_MBUF_DIRECT(mbuf)) {
 426                 if (rte_mbuf_refcnt_read(mbuf) > 1) {
 427                         DPAA2_SET_FD_IVP(fd);
 428                         rte_mbuf_refcnt_update(mbuf, -1);
 429                 }
 430         } else {
 431                 struct rte_mbuf *mi;
 432
 433                 mi = rte_mbuf_from_indirect(mbuf);
 434                 if (rte_mbuf_refcnt_read(mi) > 1)
 435                         DPAA2_SET_FD_IVP(fd);
 436                 else
 437                         rte_mbuf_refcnt_update(mi, 1);
 438                 rte_pktmbuf_free(mbuf);
 439         }
 440 }
 441
 442 static inline int __attribute__((hot))
 443 eth_copy_mbuf_to_fd(struct rte_mbuf *mbuf,
 444                     struct qbman_fd *fd, uint16_t bpid)
 445 {
 446         struct rte_mbuf *m;
 447         void *mb = NULL;
 448
 449         if (unlikely(mbuf->ol_flags & PKT_TX_VLAN_PKT)) {
 450                 int ret = rte_vlan_insert(&mbuf);
 451                 if (ret)
 452                         return ret;
 453         }
 454
 455         if (rte_dpaa2_mbuf_alloc_bulk(
 456                 rte_dpaa2_bpid_info[bpid].bp_list->mp, &mb, 1)) {
 457                 PMD_TX_LOG(WARNING, "Unable to allocated DPAA2 buffer");
 458                 return -1;
 459         }
 460         m = (struct rte_mbuf *)mb;
 461         memcpy((char *)m->buf_addr + mbuf->data_off,
 462                (void *)((char *)mbuf->buf_addr + mbuf->data_off),
 463                 mbuf->pkt_len);
 464
 465         /* Copy required fields */
 466         m->data_off = mbuf->data_off;
 467         m->ol_flags = mbuf->ol_flags;
 468         m->packet_type = mbuf->packet_type;
 469         m->tx_offload = mbuf->tx_offload;
 470
 471         /*Resetting the buffer pool id and offset field*/
 472         fd->simple.bpid_offset = 0;
 473
 474         DPAA2_SET_FD_ADDR(fd, DPAA2_MBUF_VADDR_TO_IOVA(m));
 475         DPAA2_SET_FD_LEN(fd, mbuf->data_len);
 476         DPAA2_SET_FD_BPID(fd, bpid);
 477         DPAA2_SET_FD_OFFSET(fd, mbuf->data_off);
 478         DPAA2_SET_FD_ASAL(fd, DPAA2_ASAL_VAL);
 479
 480         PMD_TX_LOG(DEBUG, " mbuf %p BMAN buf addr %p",
 481                    (void *)mbuf, mbuf->buf_addr);
 482
 483         PMD_TX_LOG(DEBUG, " fdaddr =%lx bpid =%d meta =%d off =%d, len =%d",
 484                    DPAA2_GET_FD_ADDR(fd),
 485                 DPAA2_GET_FD_BPID(fd),
 486                 rte_dpaa2_bpid_info[DPAA2_GET_FD_BPID(fd)].meta_data_size,
 487                 DPAA2_GET_FD_OFFSET(fd),
 488                 DPAA2_GET_FD_LEN(fd));
 489
 490         return 0;
 491 }
 492
 493 uint16_t
 494 dpaa2_dev_prefetch_rx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 495 {
 496         /* Function receive frames for a given device and VQ*/
 497         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
 498         struct qbman_result *dq_storage;
 499         uint32_t fqid = dpaa2_q->fqid;
 500         int ret, num_rx = 0;
 501         uint8_t is_last = 0, status;
 502         struct qbman_swp *swp;
 503         const struct qbman_fd *fd[DPAA2_DQRR_RING_SIZE];
 504         struct qbman_pull_desc pulldesc;
 505         struct queue_storage_info_t *q_storage = dpaa2_q->q_storage;
 506         struct rte_eth_dev *dev = dpaa2_q->dev;
 507
 508         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 509                 ret = dpaa2_affine_qbman_swp();
 510                 if (ret) {
 511                         RTE_LOG(ERR, PMD, "Failure in affining portal\n");
 512                         return 0;
 513                 }
 514         }
 515         swp = DPAA2_PER_LCORE_PORTAL;
 516         if (!q_storage->active_dqs) {
 517                 q_storage->toggle = 0;
 518                 dq_storage = q_storage->dq_storage[q_storage->toggle];
 519                 qbman_pull_desc_clear(&pulldesc);
 520                 qbman_pull_desc_set_numframes(&pulldesc,
 521                                               (nb_pkts > DPAA2_DQRR_RING_SIZE) ?
 522                                                DPAA2_DQRR_RING_SIZE : nb_pkts);
 523                 qbman_pull_desc_set_fq(&pulldesc, fqid);
 524                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
 525                         (dma_addr_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
 526                 if (check_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)) {
 527                         while (!qbman_check_command_complete(
 528                                get_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)))
 529                                 ;
 530                         clear_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index);
 531                 }
 532                 while (1) {
 533                         if (qbman_swp_pull(swp, &pulldesc)) {
 534                                 PMD_RX_LOG(WARNING, "VDQ command is not issued."
 535                                            "QBMAN is busy\n");
 536                                 /* Portal was busy, try again */
 537                                 continue;
 538                         }
 539                         break;
 540                 }
 541                 q_storage->active_dqs = dq_storage;
 542                 q_storage->active_dpio_id = DPAA2_PER_LCORE_DPIO->index;
 543                 set_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index, dq_storage);
 544         }
 545         dq_storage = q_storage->active_dqs;
 546         /* Check if the previous issued command is completed.
 547          * Also seems like the SWP is shared between the Ethernet Driver
 548          * and the SEC driver.
 549          */
 550         while (!qbman_check_command_complete(dq_storage))
 551                 ;
 552         if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id))
 553                 clear_swp_active_dqs(q_storage->active_dpio_id);
 554         while (!is_last) {
 555                 /* Loop until the dq_storage is updated with
 556                  * new token by QBMAN
 557                  */
 558                 while (!qbman_check_new_result(dq_storage))
 559                         ;
 560                 rte_prefetch0((void *)((uint64_t)(dq_storage + 1)));
 561                 /* Check whether Last Pull command is Expired and
 562                  * setting Condition for Loop termination
 563                  */
 564                 if (qbman_result_DQ_is_pull_complete(dq_storage)) {
 565                         is_last = 1;
 566                         /* Check for valid frame. */
 567                         status = (uint8_t)qbman_result_DQ_flags(dq_storage);
 568                         if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0))
 569                                 continue;
 570                 }
 571                 fd[num_rx] = qbman_result_DQ_fd(dq_storage);
 572
 573                 /* Prefetch Annotation address for the parse results */
 574                 rte_prefetch0((void *)((uint64_t)DPAA2_GET_FD_ADDR(fd[num_rx])
 575                                 + DPAA2_FD_PTA_SIZE + 16));
 576
 577                 if (unlikely(DPAA2_FD_GET_FORMAT(fd[num_rx]) == qbman_fd_sg))
 578                         bufs[num_rx] = eth_sg_fd_to_mbuf(fd[num_rx]);
 579                 else
 580                         bufs[num_rx] = eth_fd_to_mbuf(fd[num_rx]);
 581                 bufs[num_rx]->port = dev->data->port_id;
 582
 583                 if (dev->data->dev_conf.rxmode.hw_vlan_strip)
 584                         rte_vlan_strip(bufs[num_rx]);
 585
 586                 dq_storage++;
 587                 num_rx++;
 588         }
 589
 590         if (check_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)) {
 591                 while (!qbman_check_command_complete(
 592                        get_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)))
 593                         ;
 594                 clear_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index);
 595         }
 596         q_storage->toggle ^= 1;
 597         dq_storage = q_storage->dq_storage[q_storage->toggle];
 598         qbman_pull_desc_clear(&pulldesc);
 599         qbman_pull_desc_set_numframes(&pulldesc, DPAA2_DQRR_RING_SIZE);
 600         qbman_pull_desc_set_fq(&pulldesc, fqid);
 601         qbman_pull_desc_set_storage(&pulldesc, dq_storage,
 602                         (dma_addr_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
 603         /* Issue a volatile dequeue command. */
 604         while (1) {
 605                 if (qbman_swp_pull(swp, &pulldesc)) {
 606                         PMD_RX_LOG(WARNING, "VDQ command is not issued."
 607                                    "QBMAN is busy\n");
 608                         continue;
 609                 }
 610                 break;
 611         }
 612         q_storage->active_dqs = dq_storage;
 613         q_storage->active_dpio_id = DPAA2_PER_LCORE_DPIO->index;
 614         set_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index, dq_storage);
 615
 616         dpaa2_q->rx_pkts += num_rx;
 617
 618         /* Return the total number of packets received to DPAA2 app */
 619         return num_rx;
 620 }
 621
 622 void __attribute__((hot))
 623 dpaa2_dev_process_parallel_event(struct qbman_swp *swp,
 624                                  const struct qbman_fd *fd,
 625                                  const struct qbman_result *dq,
 626                                  struct dpaa2_queue *rxq,
 627                                  struct rte_event *ev)
 628 {
 629         ev->mbuf = eth_fd_to_mbuf(fd);
 630
 631         ev->flow_id = rxq->ev.flow_id;
 632         ev->sub_event_type = rxq->ev.sub_event_type;
 633         ev->event_type = RTE_EVENT_TYPE_ETHDEV;
 634         ev->op = RTE_EVENT_OP_NEW;
 635         ev->sched_type = rxq->ev.sched_type;
 636         ev->queue_id = rxq->ev.queue_id;
 637         ev->priority = rxq->ev.priority;
 638
 639         qbman_swp_dqrr_consume(swp, dq);
 640 }
 641
 642 /*
 643  * Callback to handle sending packets through WRIOP based interface
 644  */
 645 uint16_t
 646 dpaa2_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 647 {
 648         /* Function to transmit the frames to given device and VQ*/
 649         uint32_t loop, retry_count;
 650         int32_t ret;
 651         struct qbman_fd fd_arr[MAX_TX_RING_SLOTS];
 652         struct rte_mbuf *mi;
 653         uint32_t frames_to_send;
 654         struct rte_mempool *mp;
 655         struct qbman_eq_desc eqdesc;
 656         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)queue;
 657         struct qbman_swp *swp;
 658         uint16_t num_tx = 0;
 659         uint16_t bpid;
 660         struct rte_eth_dev *dev = dpaa2_q->dev;
 661         struct dpaa2_dev_priv *priv = dev->data->dev_private;
 662
 663         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 664                 ret = dpaa2_affine_qbman_swp();
 665                 if (ret) {
 666                         RTE_LOG(ERR, PMD, "Failure in affining portal\n");
 667                         return 0;
 668                 }
 669         }
 670         swp = DPAA2_PER_LCORE_PORTAL;
 671
 672         PMD_TX_LOG(DEBUG, "===> dev =%p, fqid =%d", dev, dpaa2_q->fqid);
 673
 674         /*Prepare enqueue descriptor*/
 675         qbman_eq_desc_clear(&eqdesc);
 676         qbman_eq_desc_set_no_orp(&eqdesc, DPAA2_EQ_RESP_ERR_FQ);
 677         qbman_eq_desc_set_response(&eqdesc, 0, 0);
 678         qbman_eq_desc_set_qd(&eqdesc, priv->qdid,
 679                              dpaa2_q->flow_id, dpaa2_q->tc_index);
 680
 681         /*Clear the unused FD fields before sending*/
 682         while (nb_pkts) {
 683                 /*Check if the queue is congested*/
 684                 retry_count = 0;
 685                 while (qbman_result_SCN_state(dpaa2_q->cscn)) {
 686                         retry_count++;
 687                         /* Retry for some time before giving up */
 688                         if (retry_count > CONG_RETRY_COUNT)
 689                                 goto skip_tx;
 690                 }
 691
 692                 frames_to_send = (nb_pkts >> 3) ? MAX_TX_RING_SLOTS : nb_pkts;
 693
 694                 for (loop = 0; loop < frames_to_send; loop++) {
 695                         fd_arr[loop].simple.frc = 0;
 696                         DPAA2_RESET_FD_CTRL((&fd_arr[loop]));
 697                         DPAA2_SET_FD_FLC((&fd_arr[loop]), NULL);
 698                         if (RTE_MBUF_DIRECT(*bufs)) {
 699                                 mp = (*bufs)->pool;
 700                         } else {
 701                                 mi = rte_mbuf_from_indirect(*bufs);
 702                                 mp = mi->pool;
 703                         }
 704                         /* Not a hw_pkt pool allocated frame */
 705                         if (unlikely(!mp || !priv->bp_list)) {
 706                                 PMD_TX_LOG(ERR, "err: no bpool attached");
 707                                 goto send_n_return;
 708                         }
 709
 710                         if (mp->ops_index != priv->bp_list->dpaa2_ops_index) {
 711                                 PMD_TX_LOG(ERR, "non hw offload bufffer ");
 712                                 /* alloc should be from the default buffer pool
 713                                  * attached to this interface
 714                                  */
 715                                 bpid = priv->bp_list->buf_pool.bpid;
 716
 717                                 if (unlikely((*bufs)->nb_segs > 1)) {
 718                                         PMD_TX_LOG(ERR, "S/G support not added"
 719                                                 " for non hw offload buffer");
 720                                         goto send_n_return;
 721                                 }
 722                                 if (eth_copy_mbuf_to_fd(*bufs,
 723                                                         &fd_arr[loop], bpid)) {
 724                                         goto send_n_return;
 725                                 }
 726                                 /* free the original packet */
 727                                 rte_pktmbuf_free(*bufs);
 728                         } else {
 729                                 bpid = mempool_to_bpid(mp);
 730                                 if (unlikely((*bufs)->nb_segs > 1)) {
 731                                         if (eth_mbuf_to_sg_fd(*bufs,
 732                                                         &fd_arr[loop], bpid))
 733                                                 goto send_n_return;
 734                                 } else {
 735                                         eth_mbuf_to_fd(*bufs,
 736                                                        &fd_arr[loop], bpid);
 737                                 }
 738                         }
 739                         bufs++;
 740                 }
 741                 loop = 0;
 742                 while (loop < frames_to_send) {
 743                         loop += qbman_swp_enqueue_multiple(swp, &eqdesc,
 744                                         &fd_arr[loop], frames_to_send - loop);
 745                 }
 746
 747                 num_tx += frames_to_send;
 748                 dpaa2_q->tx_pkts += frames_to_send;
 749                 nb_pkts -= frames_to_send;
 750         }
 751         return num_tx;
 752
 753 send_n_return:
 754         /* send any already prepared fd */
 755         if (loop) {
 756                 unsigned int i = 0;
 757
 758                 while (i < loop) {
 759                         i += qbman_swp_enqueue_multiple(swp, &eqdesc,
 760                                                         &fd_arr[i], loop - i);
 761                 }
 762                 num_tx += loop;
 763                 dpaa2_q->tx_pkts += loop;
 764         }
 765 skip_tx:
 766         return num_tx;
 767 }
 768
 769 /**
 770  * Dummy DPDK callback for TX.
 771  *
 772  * This function is used to temporarily replace the real callback during
 773  * unsafe control operations on the queue, or in case of error.
 774  *
 775  * @param dpdk_txq
 776  *   Generic pointer to TX queue structure.
 777  * @param[in] pkts
 778  *   Packets to transmit.
 779  * @param pkts_n
 780  *   Number of packets in array.
 781  *
 782  * @return
 783  *   Number of packets successfully transmitted (<= pkts_n).
 784  */
 785 uint16_t
 786 dummy_dev_tx(void *queue, struct rte_mbuf **bufs, uint16_t nb_pkts)
 787 {
 788         (void)queue;
 789         (void)bufs;
 790         (void)nb_pkts;
 791         return 0;
 792 }