drivers/raw/dpaa2_qdma/dpaa2_qdma.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018-2020 NXP
   3  */
   4
   5 #include <string.h>
   6
   7 #include <rte_eal.h>
   8 #include <rte_fslmc.h>
   9 #include <rte_atomic.h>
  10 #include <rte_lcore.h>
  11 #include <rte_rawdev.h>
  12 #include <rte_rawdev_pmd.h>
  13 #include <rte_malloc.h>
  14 #include <rte_ring.h>
  15 #include <rte_mempool.h>
  16 #include <rte_prefetch.h>
  17 #include <rte_kvargs.h>
  18
  19 #include <mc/fsl_dpdmai.h>
  20 #include <portal/dpaa2_hw_pvt.h>
  21 #include <portal/dpaa2_hw_dpio.h>
  22
  23 #include "rte_pmd_dpaa2_qdma.h"
  24 #include "dpaa2_qdma.h"
  25 #include "dpaa2_qdma_logs.h"
  26
  27 #define DPAA2_QDMA_NO_PREFETCH "no_prefetch"
  28
  29 /* Dynamic log type identifier */
  30 int dpaa2_qdma_logtype;
  31
  32 uint32_t dpaa2_coherent_no_alloc_cache;
  33 uint32_t dpaa2_coherent_alloc_cache;
  34
  35 /* QDMA device */
  36 static struct qdma_device q_dev;
  37
  38 /* QDMA H/W queues list */
  39 TAILQ_HEAD(qdma_hw_queue_list, qdma_hw_queue);
  40 static struct qdma_hw_queue_list qdma_queue_list
  41         = TAILQ_HEAD_INITIALIZER(qdma_queue_list);
  42
  43 /* QDMA per core data */
  44 static struct qdma_per_core_info qdma_core_info[RTE_MAX_LCORE];
  45
  46 static inline int
  47 qdma_populate_fd_pci(phys_addr_t src, phys_addr_t dest,
  48                         uint32_t len, struct qbman_fd *fd,
  49                         struct rte_qdma_rbp *rbp)
  50 {
  51         fd->simple_pci.saddr_lo = lower_32_bits((uint64_t) (src));
  52         fd->simple_pci.saddr_hi = upper_32_bits((uint64_t) (src));
  53
  54         fd->simple_pci.len_sl = len;
  55
  56         fd->simple_pci.bmt = 1;
  57         fd->simple_pci.fmt = 3;
  58         fd->simple_pci.sl = 1;
  59         fd->simple_pci.ser = 1;
  60
  61         fd->simple_pci.sportid = rbp->sportid;  /*pcie 3 */
  62         fd->simple_pci.srbp = rbp->srbp;
  63         if (rbp->srbp)
  64                 fd->simple_pci.rdttype = 0;
  65         else
  66                 fd->simple_pci.rdttype = dpaa2_coherent_alloc_cache;
  67
  68         /*dest is pcie memory */
  69         fd->simple_pci.dportid = rbp->dportid;  /*pcie 3 */
  70         fd->simple_pci.drbp = rbp->drbp;
  71         if (rbp->drbp)
  72                 fd->simple_pci.wrttype = 0;
  73         else
  74                 fd->simple_pci.wrttype = dpaa2_coherent_no_alloc_cache;
  75
  76         fd->simple_pci.daddr_lo = lower_32_bits((uint64_t) (dest));
  77         fd->simple_pci.daddr_hi = upper_32_bits((uint64_t) (dest));
  78
  79         return 0;
  80 }
  81
  82 static inline int
  83 qdma_populate_fd_ddr(phys_addr_t src, phys_addr_t dest,
  84                         uint32_t len, struct qbman_fd *fd)
  85 {
  86         fd->simple_ddr.saddr_lo = lower_32_bits((uint64_t) (src));
  87         fd->simple_ddr.saddr_hi = upper_32_bits((uint64_t) (src));
  88
  89         fd->simple_ddr.len = len;
  90
  91         fd->simple_ddr.bmt = 1;
  92         fd->simple_ddr.fmt = 3;
  93         fd->simple_ddr.sl = 1;
  94         fd->simple_ddr.ser = 1;
  95         /**
  96          * src If RBP=0 {NS,RDTTYPE[3:0]}: 0_1011
  97          * Coherent copy of cacheable memory,
  98         * lookup in downstream cache, no allocate
  99          * on miss
 100          */
 101         fd->simple_ddr.rns = 0;
 102         fd->simple_ddr.rdttype = dpaa2_coherent_alloc_cache;
 103         /**
 104          * dest If RBP=0 {NS,WRTTYPE[3:0]}: 0_0111
 105          * Coherent write of cacheable memory,
 106          * lookup in downstream cache, no allocate on miss
 107          */
 108         fd->simple_ddr.wns = 0;
 109         fd->simple_ddr.wrttype = dpaa2_coherent_no_alloc_cache;
 110
 111         fd->simple_ddr.daddr_lo = lower_32_bits((uint64_t) (dest));
 112         fd->simple_ddr.daddr_hi = upper_32_bits((uint64_t) (dest));
 113
 114         return 0;
 115 }
 116
 117 static void
 118 dpaa2_qdma_populate_fle(struct qbman_fle *fle,
 119                         uint64_t fle_iova,
 120                         struct rte_qdma_rbp *rbp,
 121                         uint64_t src, uint64_t dest,
 122                         size_t len, uint32_t flags, uint32_t fmt)
 123 {
 124         struct qdma_sdd *sdd;
 125         uint64_t sdd_iova;
 126
 127         sdd = (struct qdma_sdd *)
 128                         ((uintptr_t)(uint64_t)fle - QDMA_FLE_FLE_OFFSET +
 129                         QDMA_FLE_SDD_OFFSET);
 130         sdd_iova = fle_iova - QDMA_FLE_FLE_OFFSET + QDMA_FLE_SDD_OFFSET;
 131
 132         /* first frame list to source descriptor */
 133         DPAA2_SET_FLE_ADDR(fle, sdd_iova);
 134         DPAA2_SET_FLE_LEN(fle, (2 * (sizeof(struct qdma_sdd))));
 135
 136         /* source and destination descriptor */
 137         if (rbp && rbp->enable) {
 138                 /* source */
 139                 sdd->read_cmd.portid = rbp->sportid;
 140                 sdd->rbpcmd_simple.pfid = rbp->spfid;
 141                 sdd->rbpcmd_simple.vfid = rbp->svfid;
 142
 143                 if (rbp->srbp) {
 144                         sdd->read_cmd.rbp = rbp->srbp;
 145                         sdd->read_cmd.rdtype = DPAA2_RBP_MEM_RW;
 146                 } else {
 147                         sdd->read_cmd.rdtype = dpaa2_coherent_no_alloc_cache;
 148                 }
 149                 sdd++;
 150                 /* destination */
 151                 sdd->write_cmd.portid = rbp->dportid;
 152                 sdd->rbpcmd_simple.pfid = rbp->dpfid;
 153                 sdd->rbpcmd_simple.vfid = rbp->dvfid;
 154
 155                 if (rbp->drbp) {
 156                         sdd->write_cmd.rbp = rbp->drbp;
 157                         sdd->write_cmd.wrttype = DPAA2_RBP_MEM_RW;
 158                 } else {
 159                         sdd->write_cmd.wrttype = dpaa2_coherent_alloc_cache;
 160                 }
 161
 162         } else {
 163                 sdd->read_cmd.rdtype = dpaa2_coherent_no_alloc_cache;
 164                 sdd++;
 165                 sdd->write_cmd.wrttype = dpaa2_coherent_alloc_cache;
 166         }
 167         fle++;
 168         /* source frame list to source buffer */
 169         if (flags & RTE_QDMA_JOB_SRC_PHY) {
 170                 DPAA2_SET_FLE_ADDR(fle, src);
 171 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
 172                 DPAA2_SET_FLE_BMT(fle);
 173 #endif
 174         } else {
 175                 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(src));
 176         }
 177         fle->word4.fmt = fmt;
 178         DPAA2_SET_FLE_LEN(fle, len);
 179
 180         fle++;
 181         /* destination frame list to destination buffer */
 182         if (flags & RTE_QDMA_JOB_DEST_PHY) {
 183 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
 184                 DPAA2_SET_FLE_BMT(fle);
 185 #endif
 186                 DPAA2_SET_FLE_ADDR(fle, dest);
 187         } else {
 188                 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(dest));
 189         }
 190         fle->word4.fmt = fmt;
 191         DPAA2_SET_FLE_LEN(fle, len);
 192
 193         /* Final bit: 1, for last frame list */
 194         DPAA2_SET_FLE_FIN(fle);
 195 }
 196
 197 static inline int dpdmai_dev_set_fd_us(
 198                 struct qdma_virt_queue *qdma_vq,
 199                 struct qbman_fd *fd,
 200                 struct rte_qdma_job **job,
 201                 uint16_t nb_jobs)
 202 {
 203         struct rte_qdma_rbp *rbp = &qdma_vq->rbp;
 204         struct rte_qdma_job **ppjob;
 205         size_t iova;
 206         int ret = 0, loop;
 207
 208         for (loop = 0; loop < nb_jobs; loop++) {
 209                 if (job[loop]->src & QDMA_RBP_UPPER_ADDRESS_MASK)
 210                         iova = (size_t)job[loop]->dest;
 211                 else
 212                         iova = (size_t)job[loop]->src;
 213
 214                 /* Set the metadata */
 215                 job[loop]->vq_id = qdma_vq->vq_id;
 216                 ppjob = (struct rte_qdma_job **)DPAA2_IOVA_TO_VADDR(iova) - 1;
 217                 *ppjob = job[loop];
 218
 219                 if ((rbp->drbp == 1) || (rbp->srbp == 1))
 220                         ret = qdma_populate_fd_pci((phys_addr_t)job[loop]->src,
 221                                                 (phys_addr_t)job[loop]->dest,
 222                                                 job[loop]->len, &fd[loop], rbp);
 223                 else
 224                         ret = qdma_populate_fd_ddr((phys_addr_t)job[loop]->src,
 225                                                 (phys_addr_t)job[loop]->dest,
 226                                                 job[loop]->len, &fd[loop]);
 227         }
 228
 229         return ret;
 230 }
 231
 232 static uint32_t qdma_populate_sg_entry(
 233                 struct rte_qdma_job **jobs,
 234                 struct qdma_sg_entry *src_sge,
 235                 struct qdma_sg_entry *dst_sge,
 236                 uint16_t nb_jobs)
 237 {
 238         uint16_t i;
 239         uint32_t total_len = 0;
 240         uint64_t iova;
 241
 242         for (i = 0; i < nb_jobs; i++) {
 243                 /* source SG */
 244                 if (likely(jobs[i]->flags & RTE_QDMA_JOB_SRC_PHY)) {
 245                         src_sge->addr_lo = (uint32_t)jobs[i]->src;
 246                         src_sge->addr_hi = (jobs[i]->src >> 32);
 247                 } else {
 248                         iova = DPAA2_VADDR_TO_IOVA(jobs[i]->src);
 249                         src_sge->addr_lo = (uint32_t)iova;
 250                         src_sge->addr_hi = iova >> 32;
 251                 }
 252                 src_sge->data_len.data_len_sl0 = jobs[i]->len;
 253                 src_sge->ctrl.sl = QDMA_SG_SL_LONG;
 254                 src_sge->ctrl.fmt = QDMA_SG_FMT_SDB;
 255 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
 256                 src_sge->ctrl.bmt = QDMA_SG_BMT_ENABLE;
 257 #else
 258                 src_sge->ctrl.bmt = QDMA_SG_BMT_DISABLE;
 259 #endif
 260                 /* destination SG */
 261                 if (likely(jobs[i]->flags & RTE_QDMA_JOB_DEST_PHY)) {
 262                         dst_sge->addr_lo = (uint32_t)jobs[i]->dest;
 263                         dst_sge->addr_hi = (jobs[i]->dest >> 32);
 264                 } else {
 265                         iova = DPAA2_VADDR_TO_IOVA(jobs[i]->dest);
 266                         dst_sge->addr_lo = (uint32_t)iova;
 267                         dst_sge->addr_hi = iova >> 32;
 268                 }
 269                 dst_sge->data_len.data_len_sl0 = jobs[i]->len;
 270                 dst_sge->ctrl.sl = QDMA_SG_SL_LONG;
 271                 dst_sge->ctrl.fmt = QDMA_SG_FMT_SDB;
 272 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
 273                 dst_sge->ctrl.bmt = QDMA_SG_BMT_ENABLE;
 274 #else
 275                 dst_sge->ctrl.bmt = QDMA_SG_BMT_DISABLE;
 276 #endif
 277                 total_len += jobs[i]->len;
 278
 279                 if (i == (nb_jobs - 1)) {
 280                         src_sge->ctrl.f = QDMA_SG_F;
 281                         dst_sge->ctrl.f = QDMA_SG_F;
 282                 } else {
 283                         src_sge->ctrl.f = 0;
 284                         dst_sge->ctrl.f = 0;
 285                 }
 286                 src_sge++;
 287                 dst_sge++;
 288         }
 289
 290         return total_len;
 291 }
 292
 293 static inline int dpdmai_dev_set_multi_fd_lf(
 294                 struct qdma_virt_queue *qdma_vq,
 295                 struct qbman_fd *fd,
 296                 struct rte_qdma_job **job,
 297                 uint16_t nb_jobs)
 298 {
 299         struct rte_qdma_rbp *rbp = &qdma_vq->rbp;
 300         struct rte_qdma_job **ppjob;
 301         uint16_t i;
 302         int ret;
 303         struct qdma_device *qdma_dev = QDMA_DEV_OF_VQ(qdma_vq);
 304         void *elem[RTE_QDMA_BURST_NB_MAX];
 305         struct qbman_fle *fle;
 306         uint64_t elem_iova, fle_iova;
 307
 308         ret = rte_mempool_get_bulk(qdma_dev->fle_pool, elem, nb_jobs);
 309         if (ret) {
 310                 DPAA2_QDMA_DP_DEBUG("Memory alloc failed for FLE");
 311                 return ret;
 312         }
 313
 314         for (i = 0; i < nb_jobs; i++) {
 315 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
 316                 elem_iova = rte_mempool_virt2iova(elem[i]);
 317 #else
 318                 elem_iova = DPAA2_VADDR_TO_IOVA(elem[i]);
 319 #endif
 320
 321                 *((uint16_t *)
 322                 ((uintptr_t)(uint64_t)elem[i] + QDMA_FLE_JOB_NB_OFFSET)) = 1;
 323
 324                 ppjob = (struct rte_qdma_job **)
 325                         ((uintptr_t)(uint64_t)elem[i] + QDMA_FLE_JOBS_OFFSET);
 326                 *ppjob = job[i];
 327
 328                 job[i]->vq_id = qdma_vq->vq_id;
 329
 330                 fle = (struct qbman_fle *)
 331                         ((uintptr_t)(uint64_t)elem[i] + QDMA_FLE_FLE_OFFSET);
 332                 fle_iova = elem_iova + QDMA_FLE_FLE_OFFSET;
 333
 334                 DPAA2_SET_FD_ADDR(&fd[i], fle_iova);
 335                 DPAA2_SET_FD_COMPOUND_FMT(&fd[i]);
 336                 DPAA2_SET_FD_FRC(&fd[i], QDMA_SER_CTX);
 337
 338                 memset(fle, 0, DPAA2_QDMA_MAX_FLE * sizeof(struct qbman_fle) +
 339                         DPAA2_QDMA_MAX_SDD * sizeof(struct qdma_sdd));
 340
 341                 dpaa2_qdma_populate_fle(fle, fle_iova, rbp,
 342                                 job[i]->src, job[i]->dest, job[i]->len,
 343                                 job[i]->flags, QBMAN_FLE_WORD4_FMT_SBF);
 344         }
 345
 346         return 0;
 347 }
 348
 349 static inline int dpdmai_dev_set_sg_fd_lf(
 350                 struct qdma_virt_queue *qdma_vq,
 351                 struct qbman_fd *fd,
 352                 struct rte_qdma_job **job,
 353                 uint16_t nb_jobs)
 354 {
 355         struct rte_qdma_rbp *rbp = &qdma_vq->rbp;
 356         struct rte_qdma_job **ppjob;
 357         void *elem;
 358         struct qbman_fle *fle;
 359         uint64_t elem_iova, fle_iova, src, dst;
 360         int ret = 0, i;
 361         struct qdma_sg_entry *src_sge, *dst_sge;
 362         uint32_t len, fmt, flags;
 363         struct qdma_device *qdma_dev = QDMA_DEV_OF_VQ(qdma_vq);
 364
 365         /*
 366          * Get an FLE/SDD from FLE pool.
 367          * Note: IO metadata is before the FLE and SDD memory.
 368          */
 369         ret = rte_mempool_get(qdma_dev->fle_pool, (void **)(&elem));
 370         if (ret) {
 371                 DPAA2_QDMA_DP_DEBUG("Memory alloc failed for FLE");
 372                 return ret;
 373         }
 374
 375 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
 376         elem_iova = rte_mempool_virt2iova(elem);
 377 #else
 378         elem_iova = DPAA2_VADDR_TO_IOVA(elem);
 379 #endif
 380
 381         /* Set the metadata */
 382         /* Save job context. */
 383         *((uint16_t *)
 384         ((uintptr_t)(uint64_t)elem + QDMA_FLE_JOB_NB_OFFSET)) = nb_jobs;
 385         ppjob = (struct rte_qdma_job **)
 386                 ((uintptr_t)(uint64_t)elem + QDMA_FLE_JOBS_OFFSET);
 387         for (i = 0; i < nb_jobs; i++)
 388                 ppjob[i] = job[i];
 389
 390         ppjob[0]->vq_id = qdma_vq->vq_id;
 391
 392         fle = (struct qbman_fle *)
 393                 ((uintptr_t)(uint64_t)elem + QDMA_FLE_FLE_OFFSET);
 394         fle_iova = elem_iova + QDMA_FLE_FLE_OFFSET;
 395
 396         DPAA2_SET_FD_ADDR(fd, fle_iova);
 397         DPAA2_SET_FD_COMPOUND_FMT(fd);
 398         DPAA2_SET_FD_FRC(fd, QDMA_SER_CTX);
 399
 400         /* Populate FLE */
 401         if (likely(nb_jobs > 1)) {
 402                 src_sge = (struct qdma_sg_entry *)
 403                         ((uintptr_t)(uint64_t)elem + QDMA_FLE_SG_ENTRY_OFFSET);
 404                 dst_sge = src_sge + DPAA2_QDMA_MAX_SG_NB;
 405                 src = elem_iova + QDMA_FLE_SG_ENTRY_OFFSET;
 406                 dst = src +
 407                         DPAA2_QDMA_MAX_SG_NB * sizeof(struct qdma_sg_entry);
 408                 len = qdma_populate_sg_entry(job, src_sge, dst_sge, nb_jobs);
 409                 fmt = QBMAN_FLE_WORD4_FMT_SGE;
 410                 flags = RTE_QDMA_JOB_SRC_PHY | RTE_QDMA_JOB_DEST_PHY;
 411         } else {
 412                 src = job[0]->src;
 413                 dst = job[0]->dest;
 414                 len = job[0]->len;
 415                 fmt = QBMAN_FLE_WORD4_FMT_SBF;
 416                 flags = job[0]->flags;
 417         }
 418
 419         memset(fle, 0, DPAA2_QDMA_MAX_FLE * sizeof(struct qbman_fle) +
 420                         DPAA2_QDMA_MAX_SDD * sizeof(struct qdma_sdd));
 421
 422         dpaa2_qdma_populate_fle(fle, fle_iova, rbp,
 423                                         src, dst, len, flags, fmt);
 424
 425         return 0;
 426 }
 427
 428 static inline uint16_t dpdmai_dev_get_job_us(
 429                                 struct qdma_virt_queue *qdma_vq __rte_unused,
 430                                 const struct qbman_fd *fd,
 431                                 struct rte_qdma_job **job, uint16_t *nb_jobs)
 432 {
 433         uint16_t vqid;
 434         size_t iova;
 435         struct rte_qdma_job **ppjob;
 436
 437         if (fd->simple_pci.saddr_hi & (QDMA_RBP_UPPER_ADDRESS_MASK >> 32))
 438                 iova = (size_t)(((uint64_t)fd->simple_pci.daddr_hi) << 32
 439                                 | (uint64_t)fd->simple_pci.daddr_lo);
 440         else
 441                 iova = (size_t)(((uint64_t)fd->simple_pci.saddr_hi) << 32
 442                                 | (uint64_t)fd->simple_pci.saddr_lo);
 443
 444         ppjob = (struct rte_qdma_job **)DPAA2_IOVA_TO_VADDR(iova) - 1;
 445         *job = (struct rte_qdma_job *)*ppjob;
 446         (*job)->status = (fd->simple_pci.acc_err << 8) |
 447                                         (fd->simple_pci.error);
 448         vqid = (*job)->vq_id;
 449         *nb_jobs = 1;
 450
 451         return vqid;
 452 }
 453
 454 static inline uint16_t dpdmai_dev_get_job_lf(
 455                                                 struct qdma_virt_queue *qdma_vq,
 456                                                 const struct qbman_fd *fd,
 457                                                 struct rte_qdma_job **job,
 458                                                 uint16_t *nb_jobs)
 459 {
 460         struct qbman_fle *fle;
 461         struct rte_qdma_job **ppjob = NULL;
 462         uint16_t i, status;
 463         struct qdma_device *qdma_dev = QDMA_DEV_OF_VQ(qdma_vq);
 464
 465         /*
 466          * Fetch metadata from FLE. job and vq_id were set
 467          * in metadata in the enqueue operation.
 468          */
 469         fle = (struct qbman_fle *)
 470                         DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
 471         *nb_jobs = *((uint16_t *)((uintptr_t)(uint64_t)fle -
 472                                 QDMA_FLE_FLE_OFFSET + QDMA_FLE_JOB_NB_OFFSET));
 473         status = (DPAA2_GET_FD_ERR(fd) << 8) | (DPAA2_GET_FD_FRC(fd) & 0xFF);
 474
 475         ppjob = (struct rte_qdma_job **)((uintptr_t)(uint64_t)fle -
 476                                 QDMA_FLE_FLE_OFFSET + QDMA_FLE_JOBS_OFFSET);
 477
 478         for (i = 0; i < (*nb_jobs); i++) {
 479                 job[i] = ppjob[i];
 480                 job[i]->status = status;
 481         }
 482
 483         /* Free FLE to the pool */
 484         rte_mempool_put(qdma_dev->fle_pool,
 485                         (void *)
 486                         ((uintptr_t)(uint64_t)fle - QDMA_FLE_FLE_OFFSET));
 487
 488         return job[0]->vq_id;
 489 }
 490
 491 /* Function to receive a QDMA job for a given device and queue*/
 492 static int
 493 dpdmai_dev_dequeue_multijob_prefetch(
 494                         struct qdma_virt_queue *qdma_vq,
 495                         uint16_t *vq_id,
 496                         struct rte_qdma_job **job,
 497                         uint16_t nb_jobs)
 498 {
 499         struct qdma_hw_queue *qdma_pq = qdma_vq->hw_queue;
 500         struct dpaa2_dpdmai_dev *dpdmai_dev = qdma_pq->dpdmai_dev;
 501         uint16_t rxq_id = qdma_pq->queue_id;
 502
 503         struct dpaa2_queue *rxq;
 504         struct qbman_result *dq_storage, *dq_storage1 = NULL;
 505         struct qbman_pull_desc pulldesc;
 506         struct qbman_swp *swp;
 507         struct queue_storage_info_t *q_storage;
 508         uint32_t fqid;
 509         uint8_t status, pending;
 510         uint8_t num_rx = 0;
 511         const struct qbman_fd *fd;
 512         uint16_t vqid, num_rx_ret;
 513         int ret, pull_size;
 514
 515         if (qdma_vq->flags & RTE_QDMA_VQ_FD_SG_FORMAT) {
 516                 /** Make sure there are enough space to get jobs.*/
 517                 if (unlikely(nb_jobs < DPAA2_QDMA_MAX_SG_NB))
 518                         return -EINVAL;
 519                 nb_jobs = 1;
 520         }
 521
 522         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 523                 ret = dpaa2_affine_qbman_swp();
 524                 if (ret) {
 525                         DPAA2_QDMA_ERR(
 526                                 "Failed to allocate IO portal, tid: %d\n",
 527                                 rte_gettid());
 528                         return 0;
 529                 }
 530         }
 531         swp = DPAA2_PER_LCORE_PORTAL;
 532
 533         pull_size = (nb_jobs > dpaa2_dqrr_size) ? dpaa2_dqrr_size : nb_jobs;
 534         rxq = &(dpdmai_dev->rx_queue[rxq_id]);
 535         fqid = rxq->fqid;
 536         q_storage = rxq->q_storage;
 537
 538         if (unlikely(!q_storage->active_dqs)) {
 539                 q_storage->toggle = 0;
 540                 dq_storage = q_storage->dq_storage[q_storage->toggle];
 541                 q_storage->last_num_pkts = pull_size;
 542                 qbman_pull_desc_clear(&pulldesc);
 543                 qbman_pull_desc_set_numframes(&pulldesc,
 544                                               q_storage->last_num_pkts);
 545                 qbman_pull_desc_set_fq(&pulldesc, fqid);
 546                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
 547                                 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
 548                 if (check_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)) {
 549                         while (!qbman_check_command_complete(
 550                                 get_swp_active_dqs(
 551                                 DPAA2_PER_LCORE_DPIO->index)))
 552                                 ;
 553                         clear_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index);
 554                 }
 555                 while (1) {
 556                         if (qbman_swp_pull(swp, &pulldesc)) {
 557                                 DPAA2_QDMA_DP_WARN(
 558                                         "VDQ command not issued.QBMAN busy\n");
 559                                         /* Portal was busy, try again */
 560                                 continue;
 561                         }
 562                         break;
 563                 }
 564                 q_storage->active_dqs = dq_storage;
 565                 q_storage->active_dpio_id = DPAA2_PER_LCORE_DPIO->index;
 566                 set_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index,
 567                                    dq_storage);
 568         }
 569
 570         dq_storage = q_storage->active_dqs;
 571         rte_prefetch0((void *)(size_t)(dq_storage));
 572         rte_prefetch0((void *)(size_t)(dq_storage + 1));
 573
 574         /* Prepare next pull descriptor. This will give space for the
 575          * prefething done on DQRR entries
 576          */
 577         q_storage->toggle ^= 1;
 578         dq_storage1 = q_storage->dq_storage[q_storage->toggle];
 579         qbman_pull_desc_clear(&pulldesc);
 580         qbman_pull_desc_set_numframes(&pulldesc, pull_size);
 581         qbman_pull_desc_set_fq(&pulldesc, fqid);
 582         qbman_pull_desc_set_storage(&pulldesc, dq_storage1,
 583                 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage1)), 1);
 584
 585         /* Check if the previous issued command is completed.
 586          * Also seems like the SWP is shared between the Ethernet Driver
 587          * and the SEC driver.
 588          */
 589         while (!qbman_check_command_complete(dq_storage))
 590                 ;
 591         if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id))
 592                 clear_swp_active_dqs(q_storage->active_dpio_id);
 593
 594         pending = 1;
 595
 596         do {
 597                 /* Loop until the dq_storage is updated with
 598                  * new token by QBMAN
 599                  */
 600                 while (!qbman_check_new_result(dq_storage))
 601                         ;
 602                 rte_prefetch0((void *)((size_t)(dq_storage + 2)));
 603                 /* Check whether Last Pull command is Expired and
 604                  * setting Condition for Loop termination
 605                  */
 606                 if (qbman_result_DQ_is_pull_complete(dq_storage)) {
 607                         pending = 0;
 608                         /* Check for valid frame. */
 609                         status = qbman_result_DQ_flags(dq_storage);
 610                         if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0))
 611                                 continue;
 612                 }
 613                 fd = qbman_result_DQ_fd(dq_storage);
 614
 615                 vqid = qdma_vq->get_job(qdma_vq, fd, &job[num_rx],
 616                                                                 &num_rx_ret);
 617                 if (vq_id)
 618                         vq_id[num_rx] = vqid;
 619
 620                 dq_storage++;
 621                 num_rx += num_rx_ret;
 622         } while (pending);
 623
 624         if (check_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)) {
 625                 while (!qbman_check_command_complete(
 626                         get_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)))
 627                         ;
 628                 clear_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index);
 629         }
 630         /* issue a volatile dequeue command for next pull */
 631         while (1) {
 632                 if (qbman_swp_pull(swp, &pulldesc)) {
 633                         DPAA2_QDMA_DP_WARN(
 634                                 "VDQ command is not issued. QBMAN is busy (2)\n");
 635                         continue;
 636                 }
 637                 break;
 638         }
 639
 640         q_storage->active_dqs = dq_storage1;
 641         q_storage->active_dpio_id = DPAA2_PER_LCORE_DPIO->index;
 642         set_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index, dq_storage1);
 643
 644         return num_rx;
 645 }
 646
 647 static int
 648 dpdmai_dev_dequeue_multijob_no_prefetch(
 649                 struct qdma_virt_queue *qdma_vq,
 650                 uint16_t *vq_id,
 651                 struct rte_qdma_job **job,
 652                 uint16_t nb_jobs)
 653 {
 654         struct qdma_hw_queue *qdma_pq = qdma_vq->hw_queue;
 655         struct dpaa2_dpdmai_dev *dpdmai_dev = qdma_pq->dpdmai_dev;
 656         uint16_t rxq_id = qdma_pq->queue_id;
 657
 658         struct dpaa2_queue *rxq;
 659         struct qbman_result *dq_storage;
 660         struct qbman_pull_desc pulldesc;
 661         struct qbman_swp *swp;
 662         uint32_t fqid;
 663         uint8_t status, pending;
 664         uint8_t num_rx = 0;
 665         const struct qbman_fd *fd;
 666         uint16_t vqid, num_rx_ret;
 667         int ret, next_pull, num_pulled = 0;
 668
 669         if (qdma_vq->flags & RTE_QDMA_VQ_FD_SG_FORMAT) {
 670                 /** Make sure there are enough space to get jobs.*/
 671                 if (unlikely(nb_jobs < DPAA2_QDMA_MAX_SG_NB))
 672                         return -EINVAL;
 673                 nb_jobs = 1;
 674         }
 675
 676         next_pull = nb_jobs;
 677
 678         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 679                 ret = dpaa2_affine_qbman_swp();
 680                 if (ret) {
 681                         DPAA2_QDMA_ERR(
 682                                 "Failed to allocate IO portal, tid: %d\n",
 683                                 rte_gettid());
 684                         return 0;
 685                 }
 686         }
 687         swp = DPAA2_PER_LCORE_PORTAL;
 688
 689         rxq = &(dpdmai_dev->rx_queue[rxq_id]);
 690         fqid = rxq->fqid;
 691
 692         do {
 693                 dq_storage = rxq->q_storage->dq_storage[0];
 694                 /* Prepare dequeue descriptor */
 695                 qbman_pull_desc_clear(&pulldesc);
 696                 qbman_pull_desc_set_fq(&pulldesc, fqid);
 697                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
 698                         (uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
 699
 700                 if (next_pull > dpaa2_dqrr_size) {
 701                         qbman_pull_desc_set_numframes(&pulldesc,
 702                                         dpaa2_dqrr_size);
 703                         next_pull -= dpaa2_dqrr_size;
 704                 } else {
 705                         qbman_pull_desc_set_numframes(&pulldesc, next_pull);
 706                         next_pull = 0;
 707                 }
 708
 709                 while (1) {
 710                         if (qbman_swp_pull(swp, &pulldesc)) {
 711                                 DPAA2_QDMA_DP_WARN(
 712                                         "VDQ command not issued. QBMAN busy");
 713                                 /* Portal was busy, try again */
 714                                 continue;
 715                         }
 716                         break;
 717                 }
 718
 719                 rte_prefetch0((void *)((size_t)(dq_storage + 1)));
 720                 /* Check if the previous issued command is completed. */
 721                 while (!qbman_check_command_complete(dq_storage))
 722                         ;
 723
 724                 num_pulled = 0;
 725                 pending = 1;
 726
 727                 do {
 728                         /* Loop until dq_storage is updated
 729                          * with new token by QBMAN
 730                          */
 731                         while (!qbman_check_new_result(dq_storage))
 732                                 ;
 733                         rte_prefetch0((void *)((size_t)(dq_storage + 2)));
 734
 735                         if (qbman_result_DQ_is_pull_complete(dq_storage)) {
 736                                 pending = 0;
 737                                 /* Check for valid frame. */
 738                                 status = qbman_result_DQ_flags(dq_storage);
 739                                 if (unlikely((status &
 740                                         QBMAN_DQ_STAT_VALIDFRAME) == 0))
 741                                         continue;
 742                         }
 743                         fd = qbman_result_DQ_fd(dq_storage);
 744
 745                         vqid = qdma_vq->get_job(qdma_vq, fd,
 746                                                 &job[num_rx], &num_rx_ret);
 747                         if (vq_id)
 748                                 vq_id[num_rx] = vqid;
 749
 750                         dq_storage++;
 751                         num_rx += num_rx_ret;
 752                         num_pulled++;
 753
 754                 } while (pending);
 755         /* Last VDQ provided all packets and more packets are requested */
 756         } while (next_pull && num_pulled == dpaa2_dqrr_size);
 757
 758         return num_rx;
 759 }
 760
 761 static int
 762 dpdmai_dev_enqueue_multi(
 763                         struct qdma_virt_queue *qdma_vq,
 764                         struct rte_qdma_job **job,
 765                         uint16_t nb_jobs)
 766 {
 767         struct qdma_hw_queue *qdma_pq = qdma_vq->hw_queue;
 768         struct dpaa2_dpdmai_dev *dpdmai_dev = qdma_pq->dpdmai_dev;
 769         uint16_t txq_id = qdma_pq->queue_id;
 770
 771         struct qbman_fd fd[RTE_QDMA_BURST_NB_MAX];
 772         struct dpaa2_queue *txq;
 773         struct qbman_eq_desc eqdesc;
 774         struct qbman_swp *swp;
 775         int ret;
 776         uint32_t num_to_send = 0;
 777         uint16_t num_tx = 0;
 778         uint32_t enqueue_loop, retry_count, loop;
 779
 780         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 781                 ret = dpaa2_affine_qbman_swp();
 782                 if (ret) {
 783                         DPAA2_QDMA_ERR(
 784                                 "Failed to allocate IO portal, tid: %d\n",
 785                                 rte_gettid());
 786                         return 0;
 787                 }
 788         }
 789         swp = DPAA2_PER_LCORE_PORTAL;
 790
 791         txq = &(dpdmai_dev->tx_queue[txq_id]);
 792
 793         /* Prepare enqueue descriptor */
 794         qbman_eq_desc_clear(&eqdesc);
 795         qbman_eq_desc_set_fq(&eqdesc, txq->fqid);
 796         qbman_eq_desc_set_no_orp(&eqdesc, 0);
 797         qbman_eq_desc_set_response(&eqdesc, 0, 0);
 798
 799         if (qdma_vq->flags & RTE_QDMA_VQ_FD_SG_FORMAT) {
 800                 uint16_t fd_nb;
 801                 uint16_t sg_entry_nb = nb_jobs > DPAA2_QDMA_MAX_SG_NB ?
 802                                                 DPAA2_QDMA_MAX_SG_NB : nb_jobs;
 803                 uint16_t job_idx = 0;
 804                 uint16_t fd_sg_nb[8];
 805                 uint16_t nb_jobs_ret = 0;
 806
 807                 if (nb_jobs % DPAA2_QDMA_MAX_SG_NB)
 808                         fd_nb = nb_jobs / DPAA2_QDMA_MAX_SG_NB + 1;
 809                 else
 810                         fd_nb = nb_jobs / DPAA2_QDMA_MAX_SG_NB;
 811
 812                 memset(&fd[0], 0, sizeof(struct qbman_fd) * fd_nb);
 813
 814                 for (loop = 0; loop < fd_nb; loop++) {
 815                         ret = qdma_vq->set_fd(qdma_vq, &fd[loop], &job[job_idx],
 816                                               sg_entry_nb);
 817                         if (unlikely(ret < 0))
 818                                 return 0;
 819                         fd_sg_nb[loop] = sg_entry_nb;
 820                         nb_jobs -= sg_entry_nb;
 821                         job_idx += sg_entry_nb;
 822                         sg_entry_nb = nb_jobs > DPAA2_QDMA_MAX_SG_NB ?
 823                                                 DPAA2_QDMA_MAX_SG_NB : nb_jobs;
 824                 }
 825
 826                 /* Enqueue the packet to the QBMAN */
 827                 enqueue_loop = 0; retry_count = 0;
 828
 829                 while (enqueue_loop < fd_nb) {
 830                         ret = qbman_swp_enqueue_multiple(swp,
 831                                         &eqdesc, &fd[enqueue_loop],
 832                                         NULL, fd_nb - enqueue_loop);
 833                         if (unlikely(ret < 0)) {
 834                                 retry_count++;
 835                                 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT)
 836                                         return nb_jobs_ret;
 837                         } else {
 838                                 for (loop = 0; loop < (uint32_t)ret; loop++)
 839                                         nb_jobs_ret +=
 840                                                 fd_sg_nb[enqueue_loop + loop];
 841                                 enqueue_loop += ret;
 842                                 retry_count = 0;
 843                         }
 844                 }
 845
 846                 return nb_jobs_ret;
 847         }
 848
 849         memset(fd, 0, nb_jobs * sizeof(struct qbman_fd));
 850
 851         while (nb_jobs > 0) {
 852                 num_to_send = (nb_jobs > dpaa2_eqcr_size) ?
 853                         dpaa2_eqcr_size : nb_jobs;
 854
 855                 ret = qdma_vq->set_fd(qdma_vq, &fd[num_tx],
 856                                                 &job[num_tx], num_to_send);
 857                 if (unlikely(ret < 0))
 858                         break;
 859
 860                 /* Enqueue the packet to the QBMAN */
 861                 enqueue_loop = 0; retry_count = 0;
 862                 loop = num_to_send;
 863
 864                 while (enqueue_loop < loop) {
 865                         ret = qbman_swp_enqueue_multiple(swp,
 866                                                 &eqdesc,
 867                                                 &fd[num_tx + enqueue_loop],
 868                                                 NULL,
 869                                                 loop - enqueue_loop);
 870                         if (unlikely(ret < 0)) {
 871                                 retry_count++;
 872                                 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT)
 873                                         return num_tx;
 874                         } else {
 875                                 enqueue_loop += ret;
 876                                 retry_count = 0;
 877                         }
 878                 }
 879                 num_tx += num_to_send;
 880                 nb_jobs -= loop;
 881         }
 882         return num_tx;
 883 }
 884
 885 static struct qdma_hw_queue *
 886 alloc_hw_queue(uint32_t lcore_id)
 887 {
 888         struct qdma_hw_queue *queue = NULL;
 889
 890         DPAA2_QDMA_FUNC_TRACE();
 891
 892         /* Get a free queue from the list */
 893         TAILQ_FOREACH(queue, &qdma_queue_list, next) {
 894                 if (queue->num_users == 0) {
 895                         queue->lcore_id = lcore_id;
 896                         queue->num_users++;
 897                         break;
 898                 }
 899         }
 900
 901         return queue;
 902 }
 903
 904 static void
 905 free_hw_queue(struct qdma_hw_queue *queue)
 906 {
 907         DPAA2_QDMA_FUNC_TRACE();
 908
 909         queue->num_users--;
 910 }
 911
 912
 913 static struct qdma_hw_queue *
 914 get_hw_queue(struct qdma_device *qdma_dev, uint32_t lcore_id)
 915 {
 916         struct qdma_per_core_info *core_info;
 917         struct qdma_hw_queue *queue, *temp;
 918         uint32_t least_num_users;
 919         int num_hw_queues, i;
 920
 921         DPAA2_QDMA_FUNC_TRACE();
 922
 923         core_info = &qdma_core_info[lcore_id];
 924         num_hw_queues = core_info->num_hw_queues;
 925
 926         /*
 927          * Allocate a HW queue if there are less queues
 928          * than maximum per core queues configured
 929          */
 930         if (num_hw_queues < qdma_dev->max_hw_queues_per_core) {
 931                 queue = alloc_hw_queue(lcore_id);
 932                 if (queue) {
 933                         core_info->hw_queues[num_hw_queues] = queue;
 934                         core_info->num_hw_queues++;
 935                         return queue;
 936                 }
 937         }
 938
 939         queue = core_info->hw_queues[0];
 940         /* In case there is no queue associated with the core return NULL */
 941         if (!queue)
 942                 return NULL;
 943
 944         /* Fetch the least loaded H/W queue */
 945         least_num_users = core_info->hw_queues[0]->num_users;
 946         for (i = 0; i < num_hw_queues; i++) {
 947                 temp = core_info->hw_queues[i];
 948                 if (temp->num_users < least_num_users)
 949                         queue = temp;
 950         }
 951
 952         if (queue)
 953                 queue->num_users++;
 954
 955         return queue;
 956 }
 957
 958 static void
 959 put_hw_queue(struct qdma_hw_queue *queue)
 960 {
 961         struct qdma_per_core_info *core_info;
 962         int lcore_id, num_hw_queues, i;
 963
 964         DPAA2_QDMA_FUNC_TRACE();
 965
 966         /*
 967          * If this is the last user of the queue free it.
 968          * Also remove it from QDMA core info.
 969          */
 970         if (queue->num_users == 1) {
 971                 free_hw_queue(queue);
 972
 973                 /* Remove the physical queue from core info */
 974                 lcore_id = queue->lcore_id;
 975                 core_info = &qdma_core_info[lcore_id];
 976                 num_hw_queues = core_info->num_hw_queues;
 977                 for (i = 0; i < num_hw_queues; i++) {
 978                         if (queue == core_info->hw_queues[i])
 979                                 break;
 980                 }
 981                 for (; i < num_hw_queues - 1; i++)
 982                         core_info->hw_queues[i] = core_info->hw_queues[i + 1];
 983                 core_info->hw_queues[i] = NULL;
 984         } else {
 985                 queue->num_users--;
 986         }
 987 }
 988
 989 static int
 990 dpaa2_qdma_attr_get(struct rte_rawdev *rawdev,
 991                     __rte_unused const char *attr_name,
 992                     uint64_t *attr_value)
 993 {
 994         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
 995         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
 996         struct rte_qdma_attr *qdma_attr = (struct rte_qdma_attr *)attr_value;
 997
 998         DPAA2_QDMA_FUNC_TRACE();
 999
1000         qdma_attr->num_hw_queues = qdma_dev->num_hw_queues;
1001
1002         return 0;
1003 }
1004
1005 static int
1006 dpaa2_qdma_reset(struct rte_rawdev *rawdev)
1007 {
1008         struct qdma_hw_queue *queue;
1009         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1010         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1011         int i;
1012
1013         DPAA2_QDMA_FUNC_TRACE();
1014
1015         /* In case QDMA device is not in stopped state, return -EBUSY */
1016         if (qdma_dev->state == 1) {
1017                 DPAA2_QDMA_ERR(
1018                         "Device is in running state. Stop before reset.");
1019                 return -EBUSY;
1020         }
1021
1022         /* In case there are pending jobs on any VQ, return -EBUSY */
1023         for (i = 0; i < qdma_dev->max_vqs; i++) {
1024                 if (qdma_dev->vqs[i].in_use && (qdma_dev->vqs[i].num_enqueues !=
1025                     qdma_dev->vqs[i].num_dequeues)) {
1026                         DPAA2_QDMA_ERR("Jobs are still pending on VQ: %d", i);
1027                         return -EBUSY;
1028                 }
1029         }
1030
1031         /* Reset HW queues */
1032         TAILQ_FOREACH(queue, &qdma_queue_list, next)
1033                 queue->num_users = 0;
1034
1035         /* Reset and free virtual queues */
1036         for (i = 0; i < qdma_dev->max_vqs; i++) {
1037                 if (qdma_dev->vqs[i].status_ring)
1038                         rte_ring_free(qdma_dev->vqs[i].status_ring);
1039         }
1040         if (qdma_dev->vqs)
1041                 rte_free(qdma_dev->vqs);
1042         qdma_dev->vqs = NULL;
1043
1044         /* Reset per core info */
1045         memset(&qdma_core_info, 0,
1046                 sizeof(struct qdma_per_core_info) * RTE_MAX_LCORE);
1047
1048         /* Free the FLE pool */
1049         if (qdma_dev->fle_pool)
1050                 rte_mempool_free(qdma_dev->fle_pool);
1051
1052         /* Reset QDMA device structure */
1053         qdma_dev->max_hw_queues_per_core = 0;
1054         qdma_dev->fle_pool = NULL;
1055         qdma_dev->fle_pool_count = 0;
1056         qdma_dev->max_vqs = 0;
1057
1058         return 0;
1059 }
1060
1061 static int
1062 dpaa2_qdma_configure(const struct rte_rawdev *rawdev,
1063                          rte_rawdev_obj_t config,
1064                          size_t config_size)
1065 {
1066         char name[32]; /* RTE_MEMZONE_NAMESIZE = 32 */
1067         struct rte_qdma_config *qdma_config = (struct rte_qdma_config *)config;
1068         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1069         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1070
1071         DPAA2_QDMA_FUNC_TRACE();
1072
1073         if (config_size != sizeof(*qdma_config))
1074                 return -EINVAL;
1075
1076         /* In case QDMA device is not in stopped state, return -EBUSY */
1077         if (qdma_dev->state == 1) {
1078                 DPAA2_QDMA_ERR(
1079                         "Device is in running state. Stop before config.");
1080                 return -1;
1081         }
1082
1083         /* Set max HW queue per core */
1084         if (qdma_config->max_hw_queues_per_core > MAX_HW_QUEUE_PER_CORE) {
1085                 DPAA2_QDMA_ERR("H/W queues per core is more than: %d",
1086                                MAX_HW_QUEUE_PER_CORE);
1087                 return -EINVAL;
1088         }
1089         qdma_dev->max_hw_queues_per_core =
1090                 qdma_config->max_hw_queues_per_core;
1091
1092         /* Allocate Virtual Queues */
1093         sprintf(name, "qdma_%d_vq", rawdev->dev_id);
1094         qdma_dev->vqs = rte_malloc(name,
1095                         (sizeof(struct qdma_virt_queue) * qdma_config->max_vqs),
1096                         RTE_CACHE_LINE_SIZE);
1097         if (!qdma_dev->vqs) {
1098                 DPAA2_QDMA_ERR("qdma_virtual_queues allocation failed");
1099                 return -ENOMEM;
1100         }
1101         qdma_dev->max_vqs = qdma_config->max_vqs;
1102
1103         /* Allocate FLE pool; just append PID so that in case of
1104          * multiprocess, the pool's don't collide.
1105          */
1106         snprintf(name, sizeof(name), "qdma_fle_pool%u",
1107                  getpid());
1108         qdma_dev->fle_pool = rte_mempool_create(name,
1109                         qdma_config->fle_pool_count, QDMA_FLE_POOL_SIZE,
1110                         QDMA_FLE_CACHE_SIZE(qdma_config->fle_pool_count), 0,
1111                         NULL, NULL, NULL, NULL, SOCKET_ID_ANY, 0);
1112         if (!qdma_dev->fle_pool) {
1113                 DPAA2_QDMA_ERR("qdma_fle_pool create failed");
1114                 rte_free(qdma_dev->vqs);
1115                 qdma_dev->vqs = NULL;
1116                 return -ENOMEM;
1117         }
1118         qdma_dev->fle_pool_count = qdma_config->fle_pool_count;
1119
1120         return 0;
1121 }
1122
1123 static int
1124 dpaa2_qdma_start(struct rte_rawdev *rawdev)
1125 {
1126         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1127         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1128
1129         DPAA2_QDMA_FUNC_TRACE();
1130
1131         qdma_dev->state = 1;
1132
1133         return 0;
1134 }
1135
1136 static int
1137 check_devargs_handler(__rte_unused const char *key, const char *value,
1138                       __rte_unused void *opaque)
1139 {
1140         if (strcmp(value, "1"))
1141                 return -1;
1142
1143         return 0;
1144 }
1145
1146 static int
1147 dpaa2_get_devargs(struct rte_devargs *devargs, const char *key)
1148 {
1149         struct rte_kvargs *kvlist;
1150
1151         if (!devargs)
1152                 return 0;
1153
1154         kvlist = rte_kvargs_parse(devargs->args, NULL);
1155         if (!kvlist)
1156                 return 0;
1157
1158         if (!rte_kvargs_count(kvlist, key)) {
1159                 rte_kvargs_free(kvlist);
1160                 return 0;
1161         }
1162
1163         if (rte_kvargs_process(kvlist, key,
1164                                check_devargs_handler, NULL) < 0) {
1165                 rte_kvargs_free(kvlist);
1166                 return 0;
1167         }
1168         rte_kvargs_free(kvlist);
1169
1170         return 1;
1171 }
1172
1173 static int
1174 dpaa2_qdma_queue_setup(struct rte_rawdev *rawdev,
1175                           __rte_unused uint16_t queue_id,
1176                           rte_rawdev_obj_t queue_conf,
1177                           size_t conf_size)
1178 {
1179         char ring_name[32];
1180         int i;
1181         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1182         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1183         struct rte_qdma_queue_config *q_config =
1184                 (struct rte_qdma_queue_config *)queue_conf;
1185
1186         DPAA2_QDMA_FUNC_TRACE();
1187
1188         if (conf_size != sizeof(*q_config))
1189                 return -EINVAL;
1190
1191         rte_spinlock_lock(&qdma_dev->lock);
1192
1193         /* Get a free Virtual Queue */
1194         for (i = 0; i < qdma_dev->max_vqs; i++) {
1195                 if (qdma_dev->vqs[i].in_use == 0)
1196                         break;
1197         }
1198
1199         /* Return in case no VQ is free */
1200         if (i == qdma_dev->max_vqs) {
1201                 rte_spinlock_unlock(&qdma_dev->lock);
1202                 DPAA2_QDMA_ERR("Unable to get lock on QDMA device");
1203                 return -ENODEV;
1204         }
1205
1206         if (q_config->flags & RTE_QDMA_VQ_FD_SG_FORMAT) {
1207                 if (!(q_config->flags & RTE_QDMA_VQ_EXCLUSIVE_PQ)) {
1208                         DPAA2_QDMA_ERR(
1209                                 "qDMA SG format only supports physical queue!");
1210                         rte_spinlock_unlock(&qdma_dev->lock);
1211                         return -ENODEV;
1212                 }
1213                 if (!(q_config->flags & RTE_QDMA_VQ_FD_LONG_FORMAT)) {
1214                         DPAA2_QDMA_ERR(
1215                                 "qDMA SG format only supports long FD format!");
1216                         rte_spinlock_unlock(&qdma_dev->lock);
1217                         return -ENODEV;
1218                 }
1219         }
1220
1221         if (q_config->flags & RTE_QDMA_VQ_EXCLUSIVE_PQ) {
1222                 /* Allocate HW queue for a VQ */
1223                 qdma_dev->vqs[i].hw_queue = alloc_hw_queue(q_config->lcore_id);
1224                 qdma_dev->vqs[i].exclusive_hw_queue = 1;
1225         } else {
1226                 /* Allocate a Ring for Virtual Queue in VQ mode */
1227                 snprintf(ring_name, sizeof(ring_name), "status ring %d", i);
1228                 qdma_dev->vqs[i].status_ring = rte_ring_create(ring_name,
1229                         qdma_dev->fle_pool_count, rte_socket_id(), 0);
1230                 if (!qdma_dev->vqs[i].status_ring) {
1231                         DPAA2_QDMA_ERR("Status ring creation failed for vq");
1232                         rte_spinlock_unlock(&qdma_dev->lock);
1233                         return rte_errno;
1234                 }
1235
1236                 /* Get a HW queue (shared) for a VQ */
1237                 qdma_dev->vqs[i].hw_queue = get_hw_queue(qdma_dev,
1238                                                     q_config->lcore_id);
1239                 qdma_dev->vqs[i].exclusive_hw_queue = 0;
1240         }
1241
1242         if (qdma_dev->vqs[i].hw_queue == NULL) {
1243                 DPAA2_QDMA_ERR("No H/W queue available for VQ");
1244                 if (qdma_dev->vqs[i].status_ring)
1245                         rte_ring_free(qdma_dev->vqs[i].status_ring);
1246                 qdma_dev->vqs[i].status_ring = NULL;
1247                 rte_spinlock_unlock(&qdma_dev->lock);
1248                 return -ENODEV;
1249         }
1250
1251         qdma_dev->vqs[i].flags = q_config->flags;
1252         qdma_dev->vqs[i].in_use = 1;
1253         qdma_dev->vqs[i].lcore_id = q_config->lcore_id;
1254         memset(&qdma_dev->vqs[i].rbp, 0, sizeof(struct rte_qdma_rbp));
1255
1256         if (q_config->flags & RTE_QDMA_VQ_FD_LONG_FORMAT) {
1257                 if (q_config->flags & RTE_QDMA_VQ_FD_SG_FORMAT)
1258                         qdma_dev->vqs[i].set_fd = dpdmai_dev_set_sg_fd_lf;
1259                 else
1260                         qdma_dev->vqs[i].set_fd = dpdmai_dev_set_multi_fd_lf;
1261                 qdma_dev->vqs[i].get_job = dpdmai_dev_get_job_lf;
1262         } else {
1263                 qdma_dev->vqs[i].set_fd = dpdmai_dev_set_fd_us;
1264                 qdma_dev->vqs[i].get_job = dpdmai_dev_get_job_us;
1265         }
1266         if (dpaa2_get_devargs(rawdev->device->devargs,
1267                         DPAA2_QDMA_NO_PREFETCH) ||
1268                         (getenv("DPAA2_NO_QDMA_PREFETCH_RX"))) {
1269                 /* If no prefetch is configured. */
1270                 qdma_dev->vqs[i].dequeue_job =
1271                                 dpdmai_dev_dequeue_multijob_no_prefetch;
1272                 DPAA2_QDMA_INFO("No Prefetch RX Mode enabled");
1273         } else {
1274                 qdma_dev->vqs[i].dequeue_job =
1275                         dpdmai_dev_dequeue_multijob_prefetch;
1276         }
1277
1278         qdma_dev->vqs[i].enqueue_job = dpdmai_dev_enqueue_multi;
1279
1280         if (q_config->rbp != NULL)
1281                 memcpy(&qdma_dev->vqs[i].rbp, q_config->rbp,
1282                                 sizeof(struct rte_qdma_rbp));
1283
1284         rte_spinlock_unlock(&qdma_dev->lock);
1285
1286         return i;
1287 }
1288
1289 static int
1290 dpaa2_qdma_enqueue(struct rte_rawdev *rawdev,
1291                   __rte_unused struct rte_rawdev_buf **buffers,
1292                   unsigned int nb_jobs,
1293                   rte_rawdev_obj_t context)
1294 {
1295         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1296         struct rte_qdma_enqdeq *e_context =
1297                 (struct rte_qdma_enqdeq *)context;
1298         struct qdma_virt_queue *qdma_vq =
1299                 &dpdmai_dev->qdma_dev->vqs[e_context->vq_id];
1300         int ret;
1301
1302         /* Return error in case of wrong lcore_id */
1303         if (rte_lcore_id() != qdma_vq->lcore_id) {
1304                 DPAA2_QDMA_ERR("QDMA enqueue for vqid %d on wrong core",
1305                                 e_context->vq_id);
1306                 return -EINVAL;
1307         }
1308
1309         ret = qdma_vq->enqueue_job(qdma_vq, e_context->job, nb_jobs);
1310         if (ret < 0) {
1311                 DPAA2_QDMA_ERR("DPDMAI device enqueue failed: %d", ret);
1312                 return ret;
1313         }
1314
1315         qdma_vq->num_enqueues += ret;
1316
1317         return ret;
1318 }
1319
1320 static int
1321 dpaa2_qdma_dequeue(struct rte_rawdev *rawdev,
1322                    __rte_unused struct rte_rawdev_buf **buffers,
1323                    unsigned int nb_jobs,
1324                    rte_rawdev_obj_t cntxt)
1325 {
1326         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1327         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1328         struct rte_qdma_enqdeq *context =
1329                 (struct rte_qdma_enqdeq *)cntxt;
1330         struct qdma_virt_queue *qdma_vq = &qdma_dev->vqs[context->vq_id];
1331         struct qdma_virt_queue *temp_qdma_vq;
1332         int ret = 0, i;
1333         unsigned int ring_count;
1334
1335         if (qdma_vq->flags & RTE_QDMA_VQ_FD_SG_FORMAT) {
1336                 /** Make sure there are enough space to get jobs.*/
1337                 if (unlikely(nb_jobs < DPAA2_QDMA_MAX_SG_NB))
1338                         return -EINVAL;
1339         }
1340
1341         /* Return error in case of wrong lcore_id */
1342         if (rte_lcore_id() != (unsigned int)(qdma_vq->lcore_id)) {
1343                 DPAA2_QDMA_WARN("QDMA dequeue for vqid %d on wrong core",
1344                                 context->vq_id);
1345                 return -1;
1346         }
1347
1348         /* Only dequeue when there are pending jobs on VQ */
1349         if (qdma_vq->num_enqueues == qdma_vq->num_dequeues)
1350                 return 0;
1351
1352         if (!(qdma_vq->flags & RTE_QDMA_VQ_FD_SG_FORMAT) &&
1353                 qdma_vq->num_enqueues < (qdma_vq->num_dequeues + nb_jobs))
1354                 nb_jobs = (qdma_vq->num_enqueues - qdma_vq->num_dequeues);
1355
1356         if (qdma_vq->exclusive_hw_queue) {
1357                 /* In case of exclusive queue directly fetch from HW queue */
1358                 ret = qdma_vq->dequeue_job(qdma_vq, NULL,
1359                                         context->job, nb_jobs);
1360                 if (ret < 0) {
1361                         DPAA2_QDMA_ERR(
1362                                 "Dequeue from DPDMAI device failed: %d", ret);
1363                         return ret;
1364                 }
1365                 qdma_vq->num_dequeues += ret;
1366         } else {
1367                 uint16_t temp_vq_id[RTE_QDMA_BURST_NB_MAX];
1368                 /*
1369                  * Get the QDMA completed jobs from the software ring.
1370                  * In case they are not available on the ring poke the HW
1371                  * to fetch completed jobs from corresponding HW queues
1372                  */
1373                 ring_count = rte_ring_count(qdma_vq->status_ring);
1374                 if (ring_count < nb_jobs) {
1375                         /* TODO - How to have right budget */
1376                         ret = qdma_vq->dequeue_job(qdma_vq,
1377                                         temp_vq_id, context->job, nb_jobs);
1378                         for (i = 0; i < ret; i++) {
1379                                 temp_qdma_vq = &qdma_dev->vqs[temp_vq_id[i]];
1380                                 rte_ring_enqueue(temp_qdma_vq->status_ring,
1381                                         (void *)(context->job[i]));
1382                         }
1383                         ring_count = rte_ring_count(
1384                                         qdma_vq->status_ring);
1385                 }
1386
1387                 if (ring_count) {
1388                         /* Dequeue job from the software ring
1389                          * to provide to the user
1390                          */
1391                         ret = rte_ring_dequeue_bulk(qdma_vq->status_ring,
1392                                                     (void **)context->job,
1393                                                     ring_count, NULL);
1394                         if (ret)
1395                                 qdma_vq->num_dequeues += ret;
1396                 }
1397         }
1398
1399         return ret;
1400 }
1401
1402 void
1403 rte_qdma_vq_stats(struct rte_rawdev *rawdev,
1404                 uint16_t vq_id,
1405                 struct rte_qdma_vq_stats *vq_status)
1406 {
1407         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1408         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1409         struct qdma_virt_queue *qdma_vq = &qdma_dev->vqs[vq_id];
1410
1411         if (qdma_vq->in_use) {
1412                 vq_status->exclusive_hw_queue = qdma_vq->exclusive_hw_queue;
1413                 vq_status->lcore_id = qdma_vq->lcore_id;
1414                 vq_status->num_enqueues = qdma_vq->num_enqueues;
1415                 vq_status->num_dequeues = qdma_vq->num_dequeues;
1416                 vq_status->num_pending_jobs = vq_status->num_enqueues -
1417                                 vq_status->num_dequeues;
1418         }
1419 }
1420
1421 static int
1422 dpaa2_qdma_queue_release(struct rte_rawdev *rawdev,
1423                          uint16_t vq_id)
1424 {
1425         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1426         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1427
1428         struct qdma_virt_queue *qdma_vq = &qdma_dev->vqs[vq_id];
1429
1430         DPAA2_QDMA_FUNC_TRACE();
1431
1432         /* In case there are pending jobs on any VQ, return -EBUSY */
1433         if (qdma_vq->num_enqueues != qdma_vq->num_dequeues)
1434                 return -EBUSY;
1435
1436         rte_spinlock_lock(&qdma_dev->lock);
1437
1438         if (qdma_vq->exclusive_hw_queue)
1439                 free_hw_queue(qdma_vq->hw_queue);
1440         else {
1441                 if (qdma_vq->status_ring)
1442                         rte_ring_free(qdma_vq->status_ring);
1443
1444                 put_hw_queue(qdma_vq->hw_queue);
1445         }
1446
1447         memset(qdma_vq, 0, sizeof(struct qdma_virt_queue));
1448
1449         rte_spinlock_unlock(&qdma_dev->lock);
1450
1451         return 0;
1452 }
1453
1454 static void
1455 dpaa2_qdma_stop(struct rte_rawdev *rawdev)
1456 {
1457         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1458         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1459
1460         DPAA2_QDMA_FUNC_TRACE();
1461
1462         qdma_dev->state = 0;
1463 }
1464
1465 static int
1466 dpaa2_qdma_close(struct rte_rawdev *rawdev)
1467 {
1468         DPAA2_QDMA_FUNC_TRACE();
1469
1470         dpaa2_qdma_reset(rawdev);
1471
1472         return 0;
1473 }
1474
1475 static struct rte_rawdev_ops dpaa2_qdma_ops = {
1476         .dev_configure            = dpaa2_qdma_configure,
1477         .dev_start                = dpaa2_qdma_start,
1478         .dev_stop                 = dpaa2_qdma_stop,
1479         .dev_reset                = dpaa2_qdma_reset,
1480         .dev_close                = dpaa2_qdma_close,
1481         .queue_setup              = dpaa2_qdma_queue_setup,
1482         .queue_release            = dpaa2_qdma_queue_release,
1483         .attr_get                 = dpaa2_qdma_attr_get,
1484         .enqueue_bufs             = dpaa2_qdma_enqueue,
1485         .dequeue_bufs             = dpaa2_qdma_dequeue,
1486 };
1487
1488 static int
1489 add_hw_queues_to_list(struct dpaa2_dpdmai_dev *dpdmai_dev)
1490 {
1491         struct qdma_hw_queue *queue;
1492         int i;
1493
1494         DPAA2_QDMA_FUNC_TRACE();
1495
1496         for (i = 0; i < dpdmai_dev->num_queues; i++) {
1497                 queue = rte_zmalloc(NULL, sizeof(struct qdma_hw_queue), 0);
1498                 if (!queue) {
1499                         DPAA2_QDMA_ERR(
1500                                 "Memory allocation failed for QDMA queue");
1501                         return -ENOMEM;
1502                 }
1503
1504                 queue->dpdmai_dev = dpdmai_dev;
1505                 queue->queue_id = i;
1506
1507                 TAILQ_INSERT_TAIL(&qdma_queue_list, queue, next);
1508                 dpdmai_dev->qdma_dev->num_hw_queues++;
1509         }
1510
1511         return 0;
1512 }
1513
1514 static void
1515 remove_hw_queues_from_list(struct dpaa2_dpdmai_dev *dpdmai_dev)
1516 {
1517         struct qdma_hw_queue *queue = NULL;
1518         struct qdma_hw_queue *tqueue = NULL;
1519
1520         DPAA2_QDMA_FUNC_TRACE();
1521
1522         TAILQ_FOREACH_SAFE(queue, &qdma_queue_list, next, tqueue) {
1523                 if (queue->dpdmai_dev == dpdmai_dev) {
1524                         TAILQ_REMOVE(&qdma_queue_list, queue, next);
1525                         rte_free(queue);
1526                         queue = NULL;
1527                 }
1528         }
1529 }
1530
1531 static int
1532 dpaa2_dpdmai_dev_uninit(struct rte_rawdev *rawdev)
1533 {
1534         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1535         int ret, i;
1536
1537         DPAA2_QDMA_FUNC_TRACE();
1538
1539         /* Remove HW queues from global list */
1540         remove_hw_queues_from_list(dpdmai_dev);
1541
1542         ret = dpdmai_disable(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1543                              dpdmai_dev->token);
1544         if (ret)
1545                 DPAA2_QDMA_ERR("dmdmai disable failed");
1546
1547         /* Set up the DQRR storage for Rx */
1548         for (i = 0; i < dpdmai_dev->num_queues; i++) {
1549                 struct dpaa2_queue *rxq = &(dpdmai_dev->rx_queue[i]);
1550
1551                 if (rxq->q_storage) {
1552                         dpaa2_free_dq_storage(rxq->q_storage);
1553                         rte_free(rxq->q_storage);
1554                 }
1555         }
1556
1557         /* Close the device at underlying layer*/
1558         ret = dpdmai_close(&dpdmai_dev->dpdmai, CMD_PRI_LOW, dpdmai_dev->token);
1559         if (ret)
1560                 DPAA2_QDMA_ERR("Failure closing dpdmai device");
1561
1562         return 0;
1563 }
1564
1565 static int
1566 dpaa2_dpdmai_dev_init(struct rte_rawdev *rawdev, int dpdmai_id)
1567 {
1568         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1569         struct dpdmai_rx_queue_cfg rx_queue_cfg;
1570         struct dpdmai_attr attr;
1571         struct dpdmai_rx_queue_attr rx_attr;
1572         struct dpdmai_tx_queue_attr tx_attr;
1573         int ret, i;
1574
1575         DPAA2_QDMA_FUNC_TRACE();
1576
1577         /* Open DPDMAI device */
1578         dpdmai_dev->dpdmai_id = dpdmai_id;
1579         dpdmai_dev->dpdmai.regs = dpaa2_get_mcp_ptr(MC_PORTAL_INDEX);
1580         dpdmai_dev->qdma_dev = &q_dev;
1581         ret = dpdmai_open(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1582                           dpdmai_dev->dpdmai_id, &dpdmai_dev->token);
1583         if (ret) {
1584                 DPAA2_QDMA_ERR("dpdmai_open() failed with err: %d", ret);
1585                 return ret;
1586         }
1587
1588         /* Get DPDMAI attributes */
1589         ret = dpdmai_get_attributes(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1590                                     dpdmai_dev->token, &attr);
1591         if (ret) {
1592                 DPAA2_QDMA_ERR("dpdmai get attributes failed with err: %d",
1593                                ret);
1594                 goto init_err;
1595         }
1596         dpdmai_dev->num_queues = attr.num_of_queues;
1597
1598         /* Set up Rx Queues */
1599         for (i = 0; i < dpdmai_dev->num_queues; i++) {
1600                 struct dpaa2_queue *rxq;
1601
1602                 memset(&rx_queue_cfg, 0, sizeof(struct dpdmai_rx_queue_cfg));
1603                 ret = dpdmai_set_rx_queue(&dpdmai_dev->dpdmai,
1604                                           CMD_PRI_LOW,
1605                                           dpdmai_dev->token,
1606                                           i, 0, &rx_queue_cfg);
1607                 if (ret) {
1608                         DPAA2_QDMA_ERR("Setting Rx queue failed with err: %d",
1609                                        ret);
1610                         goto init_err;
1611                 }
1612
1613                 /* Allocate DQ storage for the DPDMAI Rx queues */
1614                 rxq = &(dpdmai_dev->rx_queue[i]);
1615                 rxq->q_storage = rte_malloc("dq_storage",
1616                                             sizeof(struct queue_storage_info_t),
1617                                             RTE_CACHE_LINE_SIZE);
1618                 if (!rxq->q_storage) {
1619                         DPAA2_QDMA_ERR("q_storage allocation failed");
1620                         ret = -ENOMEM;
1621                         goto init_err;
1622                 }
1623
1624                 memset(rxq->q_storage, 0, sizeof(struct queue_storage_info_t));
1625                 ret = dpaa2_alloc_dq_storage(rxq->q_storage);
1626                 if (ret) {
1627                         DPAA2_QDMA_ERR("dpaa2_alloc_dq_storage failed");
1628                         goto init_err;
1629                 }
1630         }
1631
1632         /* Get Rx and Tx queues FQID's */
1633         for (i = 0; i < dpdmai_dev->num_queues; i++) {
1634                 ret = dpdmai_get_rx_queue(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1635                                           dpdmai_dev->token, i, 0, &rx_attr);
1636                 if (ret) {
1637                         DPAA2_QDMA_ERR("Reading device failed with err: %d",
1638                                        ret);
1639                         goto init_err;
1640                 }
1641                 dpdmai_dev->rx_queue[i].fqid = rx_attr.fqid;
1642
1643                 ret = dpdmai_get_tx_queue(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1644                                           dpdmai_dev->token, i, 0, &tx_attr);
1645                 if (ret) {
1646                         DPAA2_QDMA_ERR("Reading device failed with err: %d",
1647                                        ret);
1648                         goto init_err;
1649                 }
1650                 dpdmai_dev->tx_queue[i].fqid = tx_attr.fqid;
1651         }
1652
1653         /* Enable the device */
1654         ret = dpdmai_enable(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1655                             dpdmai_dev->token);
1656         if (ret) {
1657                 DPAA2_QDMA_ERR("Enabling device failed with err: %d", ret);
1658                 goto init_err;
1659         }
1660
1661         /* Add the HW queue to the global list */
1662         ret = add_hw_queues_to_list(dpdmai_dev);
1663         if (ret) {
1664                 DPAA2_QDMA_ERR("Adding H/W queue to list failed");
1665                 goto init_err;
1666         }
1667
1668         if (!dpaa2_coherent_no_alloc_cache) {
1669                 if (dpaa2_svr_family == SVR_LX2160A) {
1670                         dpaa2_coherent_no_alloc_cache =
1671                                 DPAA2_LX2_COHERENT_NO_ALLOCATE_CACHE;
1672                         dpaa2_coherent_alloc_cache =
1673                                 DPAA2_LX2_COHERENT_ALLOCATE_CACHE;
1674                 } else {
1675                         dpaa2_coherent_no_alloc_cache =
1676                                 DPAA2_COHERENT_NO_ALLOCATE_CACHE;
1677                         dpaa2_coherent_alloc_cache =
1678                                 DPAA2_COHERENT_ALLOCATE_CACHE;
1679                 }
1680         }
1681
1682         DPAA2_QDMA_DEBUG("Initialized dpdmai object successfully");
1683
1684         rte_spinlock_init(&dpdmai_dev->qdma_dev->lock);
1685
1686         return 0;
1687 init_err:
1688         dpaa2_dpdmai_dev_uninit(rawdev);
1689         return ret;
1690 }
1691
1692 static int
1693 rte_dpaa2_qdma_probe(struct rte_dpaa2_driver *dpaa2_drv,
1694                      struct rte_dpaa2_device *dpaa2_dev)
1695 {
1696         struct rte_rawdev *rawdev;
1697         int ret;
1698
1699         DPAA2_QDMA_FUNC_TRACE();
1700
1701         rawdev = rte_rawdev_pmd_allocate(dpaa2_dev->device.name,
1702                         sizeof(struct dpaa2_dpdmai_dev),
1703                         rte_socket_id());
1704         if (!rawdev) {
1705                 DPAA2_QDMA_ERR("Unable to allocate rawdevice");
1706                 return -EINVAL;
1707         }
1708
1709         dpaa2_dev->rawdev = rawdev;
1710         rawdev->dev_ops = &dpaa2_qdma_ops;
1711         rawdev->device = &dpaa2_dev->device;
1712         rawdev->driver_name = dpaa2_drv->driver.name;
1713
1714         /* Invoke PMD device initialization function */
1715         ret = dpaa2_dpdmai_dev_init(rawdev, dpaa2_dev->object_id);
1716         if (ret) {
1717                 rte_rawdev_pmd_release(rawdev);
1718                 return ret;
1719         }
1720
1721         /* Reset the QDMA device */
1722         ret = dpaa2_qdma_reset(rawdev);
1723         if (ret) {
1724                 DPAA2_QDMA_ERR("Resetting QDMA failed");
1725                 return ret;
1726         }
1727
1728         return 0;
1729 }
1730
1731 static int
1732 rte_dpaa2_qdma_remove(struct rte_dpaa2_device *dpaa2_dev)
1733 {
1734         struct rte_rawdev *rawdev = dpaa2_dev->rawdev;
1735         int ret;
1736
1737         DPAA2_QDMA_FUNC_TRACE();
1738
1739         dpaa2_dpdmai_dev_uninit(rawdev);
1740
1741         ret = rte_rawdev_pmd_release(rawdev);
1742         if (ret)
1743                 DPAA2_QDMA_ERR("Device cleanup failed");
1744
1745         return 0;
1746 }
1747
1748 static struct rte_dpaa2_driver rte_dpaa2_qdma_pmd = {
1749         .drv_flags = RTE_DPAA2_DRV_IOVA_AS_VA,
1750         .drv_type = DPAA2_QDMA,
1751         .probe = rte_dpaa2_qdma_probe,
1752         .remove = rte_dpaa2_qdma_remove,
1753 };
1754
1755 RTE_PMD_REGISTER_DPAA2(dpaa2_qdma, rte_dpaa2_qdma_pmd);
1756 RTE_PMD_REGISTER_PARAM_STRING(dpaa2_qdma,
1757         "no_prefetch=<int> ");
1758 RTE_LOG_REGISTER(dpaa2_qdma_logtype, pmd.raw.dpaa2.qdma, INFO);