drivers/raw/dpaa2_qdma/dpaa2_qdma.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018-2020 NXP
   3  */
   4
   5 #include <string.h>
   6
   7 #include <rte_eal.h>
   8 #include <rte_fslmc.h>
   9 #include <rte_atomic.h>
  10 #include <rte_lcore.h>
  11 #include <rte_rawdev.h>
  12 #include <rte_rawdev_pmd.h>
  13 #include <rte_malloc.h>
  14 #include <rte_ring.h>
  15 #include <rte_mempool.h>
  16 #include <rte_prefetch.h>
  17 #include <rte_kvargs.h>
  18
  19 #include <mc/fsl_dpdmai.h>
  20 #include <portal/dpaa2_hw_pvt.h>
  21 #include <portal/dpaa2_hw_dpio.h>
  22
  23 #include "rte_pmd_dpaa2_qdma.h"
  24 #include "dpaa2_qdma.h"
  25 #include "dpaa2_qdma_logs.h"
  26
  27 #define DPAA2_QDMA_NO_PREFETCH "no_prefetch"
  28
  29 /* Dynamic log type identifier */
  30 int dpaa2_qdma_logtype;
  31
  32 uint32_t dpaa2_coherent_no_alloc_cache;
  33 uint32_t dpaa2_coherent_alloc_cache;
  34
  35 /* QDMA device */
  36 static struct qdma_device q_dev;
  37
  38 /* QDMA H/W queues list */
  39 TAILQ_HEAD(qdma_hw_queue_list, qdma_hw_queue);
  40 static struct qdma_hw_queue_list qdma_queue_list
  41         = TAILQ_HEAD_INITIALIZER(qdma_queue_list);
  42
  43 /* QDMA per core data */
  44 static struct qdma_per_core_info qdma_core_info[RTE_MAX_LCORE];
  45
  46 static inline int
  47 qdma_populate_fd_pci(phys_addr_t src, phys_addr_t dest,
  48                         uint32_t len, struct qbman_fd *fd,
  49                         struct rte_qdma_rbp *rbp, int ser)
  50 {
  51         fd->simple_pci.saddr_lo = lower_32_bits((uint64_t) (src));
  52         fd->simple_pci.saddr_hi = upper_32_bits((uint64_t) (src));
  53
  54         fd->simple_pci.len_sl = len;
  55
  56         fd->simple_pci.bmt = 1;
  57         fd->simple_pci.fmt = 3;
  58         fd->simple_pci.sl = 1;
  59         fd->simple_pci.ser = ser;
  60
  61         fd->simple_pci.sportid = rbp->sportid;  /*pcie 3 */
  62         fd->simple_pci.srbp = rbp->srbp;
  63         if (rbp->srbp)
  64                 fd->simple_pci.rdttype = 0;
  65         else
  66                 fd->simple_pci.rdttype = dpaa2_coherent_alloc_cache;
  67
  68         /*dest is pcie memory */
  69         fd->simple_pci.dportid = rbp->dportid;  /*pcie 3 */
  70         fd->simple_pci.drbp = rbp->drbp;
  71         if (rbp->drbp)
  72                 fd->simple_pci.wrttype = 0;
  73         else
  74                 fd->simple_pci.wrttype = dpaa2_coherent_no_alloc_cache;
  75
  76         fd->simple_pci.daddr_lo = lower_32_bits((uint64_t) (dest));
  77         fd->simple_pci.daddr_hi = upper_32_bits((uint64_t) (dest));
  78
  79         return 0;
  80 }
  81
  82 static inline int
  83 qdma_populate_fd_ddr(phys_addr_t src, phys_addr_t dest,
  84                         uint32_t len, struct qbman_fd *fd, int ser)
  85 {
  86         fd->simple_ddr.saddr_lo = lower_32_bits((uint64_t) (src));
  87         fd->simple_ddr.saddr_hi = upper_32_bits((uint64_t) (src));
  88
  89         fd->simple_ddr.len = len;
  90
  91         fd->simple_ddr.bmt = 1;
  92         fd->simple_ddr.fmt = 3;
  93         fd->simple_ddr.sl = 1;
  94         fd->simple_ddr.ser = ser;
  95         /**
  96          * src If RBP=0 {NS,RDTTYPE[3:0]}: 0_1011
  97          * Coherent copy of cacheable memory,
  98         * lookup in downstream cache, no allocate
  99          * on miss
 100          */
 101         fd->simple_ddr.rns = 0;
 102         fd->simple_ddr.rdttype = dpaa2_coherent_alloc_cache;
 103         /**
 104          * dest If RBP=0 {NS,WRTTYPE[3:0]}: 0_0111
 105          * Coherent write of cacheable memory,
 106          * lookup in downstream cache, no allocate on miss
 107          */
 108         fd->simple_ddr.wns = 0;
 109         fd->simple_ddr.wrttype = dpaa2_coherent_no_alloc_cache;
 110
 111         fd->simple_ddr.daddr_lo = lower_32_bits((uint64_t) (dest));
 112         fd->simple_ddr.daddr_hi = upper_32_bits((uint64_t) (dest));
 113
 114         return 0;
 115 }
 116
 117 static void
 118 dpaa2_qdma_populate_fle(struct qbman_fle *fle,
 119                         uint64_t fle_iova,
 120                         struct rte_qdma_rbp *rbp,
 121                         uint64_t src, uint64_t dest,
 122                         size_t len, uint32_t flags, uint32_t fmt)
 123 {
 124         struct qdma_sdd *sdd;
 125         uint64_t sdd_iova;
 126
 127         sdd = (struct qdma_sdd *)
 128                         ((uintptr_t)(uint64_t)fle - QDMA_FLE_FLE_OFFSET +
 129                         QDMA_FLE_SDD_OFFSET);
 130         sdd_iova = fle_iova - QDMA_FLE_FLE_OFFSET + QDMA_FLE_SDD_OFFSET;
 131
 132         /* first frame list to source descriptor */
 133         DPAA2_SET_FLE_ADDR(fle, sdd_iova);
 134         DPAA2_SET_FLE_LEN(fle, (2 * (sizeof(struct qdma_sdd))));
 135
 136         /* source and destination descriptor */
 137         if (rbp && rbp->enable) {
 138                 /* source */
 139                 sdd->read_cmd.portid = rbp->sportid;
 140                 sdd->rbpcmd_simple.pfid = rbp->spfid;
 141                 sdd->rbpcmd_simple.vfid = rbp->svfid;
 142
 143                 if (rbp->srbp) {
 144                         sdd->read_cmd.rbp = rbp->srbp;
 145                         sdd->read_cmd.rdtype = DPAA2_RBP_MEM_RW;
 146                 } else {
 147                         sdd->read_cmd.rdtype = dpaa2_coherent_no_alloc_cache;
 148                 }
 149                 sdd++;
 150                 /* destination */
 151                 sdd->write_cmd.portid = rbp->dportid;
 152                 sdd->rbpcmd_simple.pfid = rbp->dpfid;
 153                 sdd->rbpcmd_simple.vfid = rbp->dvfid;
 154
 155                 if (rbp->drbp) {
 156                         sdd->write_cmd.rbp = rbp->drbp;
 157                         sdd->write_cmd.wrttype = DPAA2_RBP_MEM_RW;
 158                 } else {
 159                         sdd->write_cmd.wrttype = dpaa2_coherent_alloc_cache;
 160                 }
 161
 162         } else {
 163                 sdd->read_cmd.rdtype = dpaa2_coherent_no_alloc_cache;
 164                 sdd++;
 165                 sdd->write_cmd.wrttype = dpaa2_coherent_alloc_cache;
 166         }
 167         fle++;
 168         /* source frame list to source buffer */
 169         if (flags & RTE_QDMA_JOB_SRC_PHY) {
 170                 DPAA2_SET_FLE_ADDR(fle, src);
 171 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
 172                 DPAA2_SET_FLE_BMT(fle);
 173 #endif
 174         } else {
 175                 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(src));
 176         }
 177         fle->word4.fmt = fmt;
 178         DPAA2_SET_FLE_LEN(fle, len);
 179
 180         fle++;
 181         /* destination frame list to destination buffer */
 182         if (flags & RTE_QDMA_JOB_DEST_PHY) {
 183 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
 184                 DPAA2_SET_FLE_BMT(fle);
 185 #endif
 186                 DPAA2_SET_FLE_ADDR(fle, dest);
 187         } else {
 188                 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(dest));
 189         }
 190         fle->word4.fmt = fmt;
 191         DPAA2_SET_FLE_LEN(fle, len);
 192
 193         /* Final bit: 1, for last frame list */
 194         DPAA2_SET_FLE_FIN(fle);
 195 }
 196
 197 static inline int dpdmai_dev_set_fd_us(
 198                 struct qdma_virt_queue *qdma_vq,
 199                 struct qbman_fd *fd,
 200                 struct rte_qdma_job **job,
 201                 uint16_t nb_jobs)
 202 {
 203         struct rte_qdma_rbp *rbp = &qdma_vq->rbp;
 204         struct rte_qdma_job **ppjob;
 205         size_t iova;
 206         int ret = 0, loop;
 207         int ser = (qdma_vq->flags & RTE_QDMA_VQ_NO_RESPONSE) ?
 208                                 0 : 1;
 209
 210         for (loop = 0; loop < nb_jobs; loop++) {
 211                 if (job[loop]->src & QDMA_RBP_UPPER_ADDRESS_MASK)
 212                         iova = (size_t)job[loop]->dest;
 213                 else
 214                         iova = (size_t)job[loop]->src;
 215
 216                 /* Set the metadata */
 217                 job[loop]->vq_id = qdma_vq->vq_id;
 218                 ppjob = (struct rte_qdma_job **)DPAA2_IOVA_TO_VADDR(iova) - 1;
 219                 *ppjob = job[loop];
 220
 221                 if ((rbp->drbp == 1) || (rbp->srbp == 1))
 222                         ret = qdma_populate_fd_pci((phys_addr_t)job[loop]->src,
 223                                         (phys_addr_t)job[loop]->dest,
 224                                         job[loop]->len, &fd[loop], rbp, ser);
 225                 else
 226                         ret = qdma_populate_fd_ddr((phys_addr_t)job[loop]->src,
 227                                         (phys_addr_t)job[loop]->dest,
 228                                         job[loop]->len, &fd[loop], ser);
 229         }
 230
 231         return ret;
 232 }
 233
 234 static uint32_t qdma_populate_sg_entry(
 235                 struct rte_qdma_job **jobs,
 236                 struct qdma_sg_entry *src_sge,
 237                 struct qdma_sg_entry *dst_sge,
 238                 uint16_t nb_jobs)
 239 {
 240         uint16_t i;
 241         uint32_t total_len = 0;
 242         uint64_t iova;
 243
 244         for (i = 0; i < nb_jobs; i++) {
 245                 /* source SG */
 246                 if (likely(jobs[i]->flags & RTE_QDMA_JOB_SRC_PHY)) {
 247                         src_sge->addr_lo = (uint32_t)jobs[i]->src;
 248                         src_sge->addr_hi = (jobs[i]->src >> 32);
 249                 } else {
 250                         iova = DPAA2_VADDR_TO_IOVA(jobs[i]->src);
 251                         src_sge->addr_lo = (uint32_t)iova;
 252                         src_sge->addr_hi = iova >> 32;
 253                 }
 254                 src_sge->data_len.data_len_sl0 = jobs[i]->len;
 255                 src_sge->ctrl.sl = QDMA_SG_SL_LONG;
 256                 src_sge->ctrl.fmt = QDMA_SG_FMT_SDB;
 257 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
 258                 src_sge->ctrl.bmt = QDMA_SG_BMT_ENABLE;
 259 #else
 260                 src_sge->ctrl.bmt = QDMA_SG_BMT_DISABLE;
 261 #endif
 262                 /* destination SG */
 263                 if (likely(jobs[i]->flags & RTE_QDMA_JOB_DEST_PHY)) {
 264                         dst_sge->addr_lo = (uint32_t)jobs[i]->dest;
 265                         dst_sge->addr_hi = (jobs[i]->dest >> 32);
 266                 } else {
 267                         iova = DPAA2_VADDR_TO_IOVA(jobs[i]->dest);
 268                         dst_sge->addr_lo = (uint32_t)iova;
 269                         dst_sge->addr_hi = iova >> 32;
 270                 }
 271                 dst_sge->data_len.data_len_sl0 = jobs[i]->len;
 272                 dst_sge->ctrl.sl = QDMA_SG_SL_LONG;
 273                 dst_sge->ctrl.fmt = QDMA_SG_FMT_SDB;
 274 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
 275                 dst_sge->ctrl.bmt = QDMA_SG_BMT_ENABLE;
 276 #else
 277                 dst_sge->ctrl.bmt = QDMA_SG_BMT_DISABLE;
 278 #endif
 279                 total_len += jobs[i]->len;
 280
 281                 if (i == (nb_jobs - 1)) {
 282                         src_sge->ctrl.f = QDMA_SG_F;
 283                         dst_sge->ctrl.f = QDMA_SG_F;
 284                 } else {
 285                         src_sge->ctrl.f = 0;
 286                         dst_sge->ctrl.f = 0;
 287                 }
 288                 src_sge++;
 289                 dst_sge++;
 290         }
 291
 292         return total_len;
 293 }
 294
 295 static inline int dpdmai_dev_set_multi_fd_lf_no_rsp(
 296                 struct qdma_virt_queue *qdma_vq,
 297                 struct qbman_fd *fd,
 298                 struct rte_qdma_job **job,
 299                 uint16_t nb_jobs)
 300 {
 301         struct rte_qdma_rbp *rbp = &qdma_vq->rbp;
 302         struct rte_qdma_job **ppjob;
 303         uint16_t i;
 304         void *elem;
 305         struct qbman_fle *fle;
 306         uint64_t elem_iova, fle_iova;
 307
 308         for (i = 0; i < nb_jobs; i++) {
 309                 elem = job[i]->usr_elem;
 310 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
 311                 elem_iova = rte_mempool_virt2iova(elem);
 312 #else
 313                 elem_iova = DPAA2_VADDR_TO_IOVA(elem);
 314 #endif
 315
 316                 ppjob = (struct rte_qdma_job **)
 317                         ((uintptr_t)(uint64_t)elem +
 318                          QDMA_FLE_SINGLE_JOB_OFFSET);
 319                 *ppjob = job[i];
 320
 321                 job[i]->vq_id = qdma_vq->vq_id;
 322
 323                 fle = (struct qbman_fle *)
 324                         ((uintptr_t)(uint64_t)elem + QDMA_FLE_FLE_OFFSET);
 325                 fle_iova = elem_iova + QDMA_FLE_FLE_OFFSET;
 326
 327                 DPAA2_SET_FD_ADDR(&fd[i], fle_iova);
 328                 DPAA2_SET_FD_COMPOUND_FMT(&fd[i]);
 329
 330                 memset(fle, 0, DPAA2_QDMA_MAX_FLE * sizeof(struct qbman_fle) +
 331                                 DPAA2_QDMA_MAX_SDD * sizeof(struct qdma_sdd));
 332
 333                 dpaa2_qdma_populate_fle(fle, fle_iova, rbp,
 334                         job[i]->src, job[i]->dest, job[i]->len,
 335                         job[i]->flags, QBMAN_FLE_WORD4_FMT_SBF);
 336         }
 337
 338         return 0;
 339 }
 340
 341 static inline int dpdmai_dev_set_multi_fd_lf(
 342                 struct qdma_virt_queue *qdma_vq,
 343                 struct qbman_fd *fd,
 344                 struct rte_qdma_job **job,
 345                 uint16_t nb_jobs)
 346 {
 347         struct rte_qdma_rbp *rbp = &qdma_vq->rbp;
 348         struct rte_qdma_job **ppjob;
 349         uint16_t i;
 350         int ret;
 351         void *elem[RTE_QDMA_BURST_NB_MAX];
 352         struct qbman_fle *fle;
 353         uint64_t elem_iova, fle_iova;
 354
 355         ret = rte_mempool_get_bulk(qdma_vq->fle_pool, elem, nb_jobs);
 356         if (ret) {
 357                 DPAA2_QDMA_DP_DEBUG("Memory alloc failed for FLE");
 358                 return ret;
 359         }
 360
 361         for (i = 0; i < nb_jobs; i++) {
 362 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
 363                 elem_iova = rte_mempool_virt2iova(elem[i]);
 364 #else
 365                 elem_iova = DPAA2_VADDR_TO_IOVA(elem[i]);
 366 #endif
 367
 368                 ppjob = (struct rte_qdma_job **)
 369                         ((uintptr_t)(uint64_t)elem[i] +
 370                          QDMA_FLE_SINGLE_JOB_OFFSET);
 371                 *ppjob = job[i];
 372
 373                 job[i]->vq_id = qdma_vq->vq_id;
 374
 375                 fle = (struct qbman_fle *)
 376                         ((uintptr_t)(uint64_t)elem[i] + QDMA_FLE_FLE_OFFSET);
 377                 fle_iova = elem_iova + QDMA_FLE_FLE_OFFSET;
 378
 379                 DPAA2_SET_FD_ADDR(&fd[i], fle_iova);
 380                 DPAA2_SET_FD_COMPOUND_FMT(&fd[i]);
 381                 DPAA2_SET_FD_FRC(&fd[i], QDMA_SER_CTX);
 382
 383                 memset(fle, 0, DPAA2_QDMA_MAX_FLE * sizeof(struct qbman_fle) +
 384                         DPAA2_QDMA_MAX_SDD * sizeof(struct qdma_sdd));
 385
 386                 dpaa2_qdma_populate_fle(fle, fle_iova, rbp,
 387                                 job[i]->src, job[i]->dest, job[i]->len,
 388                                 job[i]->flags, QBMAN_FLE_WORD4_FMT_SBF);
 389         }
 390
 391         return 0;
 392 }
 393
 394 static inline int dpdmai_dev_set_sg_fd_lf(
 395                 struct qdma_virt_queue *qdma_vq,
 396                 struct qbman_fd *fd,
 397                 struct rte_qdma_job **job,
 398                 uint16_t nb_jobs)
 399 {
 400         struct rte_qdma_rbp *rbp = &qdma_vq->rbp;
 401         struct rte_qdma_job **ppjob;
 402         void *elem;
 403         struct qbman_fle *fle;
 404         uint64_t elem_iova, fle_iova, src, dst;
 405         int ret = 0, i;
 406         struct qdma_sg_entry *src_sge, *dst_sge;
 407         uint32_t len, fmt, flags;
 408
 409         /*
 410          * Get an FLE/SDD from FLE pool.
 411          * Note: IO metadata is before the FLE and SDD memory.
 412          */
 413         if (qdma_vq->flags & RTE_QDMA_VQ_NO_RESPONSE) {
 414                 elem = job[0]->usr_elem;
 415         } else {
 416                 ret = rte_mempool_get(qdma_vq->fle_pool, &elem);
 417                 if (ret) {
 418                         DPAA2_QDMA_DP_DEBUG("Memory alloc failed for FLE");
 419                         return ret;
 420                 }
 421         }
 422
 423 #ifdef RTE_LIBRTE_DPAA2_USE_PHYS_IOVA
 424         elem_iova = rte_mempool_virt2iova(elem);
 425 #else
 426         elem_iova = DPAA2_VADDR_TO_IOVA(elem);
 427 #endif
 428
 429         /* Set the metadata */
 430         /* Save job context. */
 431         *((uint16_t *)
 432         ((uintptr_t)(uint64_t)elem + QDMA_FLE_JOB_NB_OFFSET)) = nb_jobs;
 433         ppjob = (struct rte_qdma_job **)
 434                 ((uintptr_t)(uint64_t)elem + QDMA_FLE_SG_JOBS_OFFSET);
 435         for (i = 0; i < nb_jobs; i++)
 436                 ppjob[i] = job[i];
 437
 438         ppjob[0]->vq_id = qdma_vq->vq_id;
 439
 440         fle = (struct qbman_fle *)
 441                 ((uintptr_t)(uint64_t)elem + QDMA_FLE_FLE_OFFSET);
 442         fle_iova = elem_iova + QDMA_FLE_FLE_OFFSET;
 443
 444         DPAA2_SET_FD_ADDR(fd, fle_iova);
 445         DPAA2_SET_FD_COMPOUND_FMT(fd);
 446         if (!(qdma_vq->flags & RTE_QDMA_VQ_NO_RESPONSE))
 447                 DPAA2_SET_FD_FRC(fd, QDMA_SER_CTX);
 448
 449         /* Populate FLE */
 450         if (likely(nb_jobs > 1)) {
 451                 src_sge = (struct qdma_sg_entry *)
 452                         ((uintptr_t)(uint64_t)elem + QDMA_FLE_SG_ENTRY_OFFSET);
 453                 dst_sge = src_sge + DPAA2_QDMA_MAX_SG_NB;
 454                 src = elem_iova + QDMA_FLE_SG_ENTRY_OFFSET;
 455                 dst = src +
 456                         DPAA2_QDMA_MAX_SG_NB * sizeof(struct qdma_sg_entry);
 457                 len = qdma_populate_sg_entry(job, src_sge, dst_sge, nb_jobs);
 458                 fmt = QBMAN_FLE_WORD4_FMT_SGE;
 459                 flags = RTE_QDMA_JOB_SRC_PHY | RTE_QDMA_JOB_DEST_PHY;
 460         } else {
 461                 src = job[0]->src;
 462                 dst = job[0]->dest;
 463                 len = job[0]->len;
 464                 fmt = QBMAN_FLE_WORD4_FMT_SBF;
 465                 flags = job[0]->flags;
 466         }
 467
 468         memset(fle, 0, DPAA2_QDMA_MAX_FLE * sizeof(struct qbman_fle) +
 469                         DPAA2_QDMA_MAX_SDD * sizeof(struct qdma_sdd));
 470
 471         dpaa2_qdma_populate_fle(fle, fle_iova, rbp,
 472                                         src, dst, len, flags, fmt);
 473
 474         return 0;
 475 }
 476
 477 static inline uint16_t dpdmai_dev_get_job_us(
 478                                 struct qdma_virt_queue *qdma_vq __rte_unused,
 479                                 const struct qbman_fd *fd,
 480                                 struct rte_qdma_job **job, uint16_t *nb_jobs)
 481 {
 482         uint16_t vqid;
 483         size_t iova;
 484         struct rte_qdma_job **ppjob;
 485
 486         if (fd->simple_pci.saddr_hi & (QDMA_RBP_UPPER_ADDRESS_MASK >> 32))
 487                 iova = (size_t)(((uint64_t)fd->simple_pci.daddr_hi) << 32
 488                                 | (uint64_t)fd->simple_pci.daddr_lo);
 489         else
 490                 iova = (size_t)(((uint64_t)fd->simple_pci.saddr_hi) << 32
 491                                 | (uint64_t)fd->simple_pci.saddr_lo);
 492
 493         ppjob = (struct rte_qdma_job **)DPAA2_IOVA_TO_VADDR(iova) - 1;
 494         *job = (struct rte_qdma_job *)*ppjob;
 495         (*job)->status = (fd->simple_pci.acc_err << 8) |
 496                                         (fd->simple_pci.error);
 497         vqid = (*job)->vq_id;
 498         *nb_jobs = 1;
 499
 500         return vqid;
 501 }
 502
 503 static inline uint16_t dpdmai_dev_get_single_job_lf(
 504                                                 struct qdma_virt_queue *qdma_vq,
 505                                                 const struct qbman_fd *fd,
 506                                                 struct rte_qdma_job **job,
 507                                                 uint16_t *nb_jobs)
 508 {
 509         struct qbman_fle *fle;
 510         struct rte_qdma_job **ppjob = NULL;
 511         uint16_t status;
 512
 513         /*
 514          * Fetch metadata from FLE. job and vq_id were set
 515          * in metadata in the enqueue operation.
 516          */
 517         fle = (struct qbman_fle *)
 518                         DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
 519
 520         *nb_jobs = 1;
 521         ppjob = (struct rte_qdma_job **)((uintptr_t)(uint64_t)fle -
 522                         QDMA_FLE_FLE_OFFSET + QDMA_FLE_SINGLE_JOB_OFFSET);
 523
 524         status = (DPAA2_GET_FD_ERR(fd) << 8) | (DPAA2_GET_FD_FRC(fd) & 0xFF);
 525
 526         *job = *ppjob;
 527         (*job)->status = status;
 528
 529         /* Free FLE to the pool */
 530         rte_mempool_put(qdma_vq->fle_pool,
 531                         (void *)
 532                         ((uintptr_t)(uint64_t)fle - QDMA_FLE_FLE_OFFSET));
 533
 534         return (*job)->vq_id;
 535 }
 536
 537 static inline uint16_t dpdmai_dev_get_sg_job_lf(
 538                                                 struct qdma_virt_queue *qdma_vq,
 539                                                 const struct qbman_fd *fd,
 540                                                 struct rte_qdma_job **job,
 541                                                 uint16_t *nb_jobs)
 542 {
 543         struct qbman_fle *fle;
 544         struct rte_qdma_job **ppjob = NULL;
 545         uint16_t i, status;
 546
 547         /*
 548          * Fetch metadata from FLE. job and vq_id were set
 549          * in metadata in the enqueue operation.
 550          */
 551         fle = (struct qbman_fle *)
 552                         DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
 553         *nb_jobs = *((uint16_t *)((uintptr_t)(uint64_t)fle -
 554                                 QDMA_FLE_FLE_OFFSET + QDMA_FLE_JOB_NB_OFFSET));
 555         ppjob = (struct rte_qdma_job **)((uintptr_t)(uint64_t)fle -
 556                                 QDMA_FLE_FLE_OFFSET + QDMA_FLE_SG_JOBS_OFFSET);
 557         status = (DPAA2_GET_FD_ERR(fd) << 8) | (DPAA2_GET_FD_FRC(fd) & 0xFF);
 558
 559         for (i = 0; i < (*nb_jobs); i++) {
 560                 job[i] = ppjob[i];
 561                 job[i]->status = status;
 562         }
 563
 564         /* Free FLE to the pool */
 565         rte_mempool_put(qdma_vq->fle_pool,
 566                         (void *)
 567                         ((uintptr_t)(uint64_t)fle - QDMA_FLE_FLE_OFFSET));
 568
 569         return job[0]->vq_id;
 570 }
 571
 572 /* Function to receive a QDMA job for a given device and queue*/
 573 static int
 574 dpdmai_dev_dequeue_multijob_prefetch(
 575                         struct qdma_virt_queue *qdma_vq,
 576                         uint16_t *vq_id,
 577                         struct rte_qdma_job **job,
 578                         uint16_t nb_jobs)
 579 {
 580         struct qdma_hw_queue *qdma_pq = qdma_vq->hw_queue;
 581         struct dpaa2_dpdmai_dev *dpdmai_dev = qdma_pq->dpdmai_dev;
 582         uint16_t rxq_id = qdma_pq->queue_id;
 583
 584         struct dpaa2_queue *rxq;
 585         struct qbman_result *dq_storage, *dq_storage1 = NULL;
 586         struct qbman_pull_desc pulldesc;
 587         struct qbman_swp *swp;
 588         struct queue_storage_info_t *q_storage;
 589         uint32_t fqid;
 590         uint8_t status, pending;
 591         uint8_t num_rx = 0;
 592         const struct qbman_fd *fd;
 593         uint16_t vqid, num_rx_ret;
 594         int ret, pull_size;
 595
 596         if (qdma_vq->flags & RTE_QDMA_VQ_FD_SG_FORMAT) {
 597                 /** Make sure there are enough space to get jobs.*/
 598                 if (unlikely(nb_jobs < DPAA2_QDMA_MAX_SG_NB))
 599                         return -EINVAL;
 600                 nb_jobs = 1;
 601         }
 602
 603         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 604                 ret = dpaa2_affine_qbman_swp();
 605                 if (ret) {
 606                         DPAA2_QDMA_ERR(
 607                                 "Failed to allocate IO portal, tid: %d\n",
 608                                 rte_gettid());
 609                         return 0;
 610                 }
 611         }
 612         swp = DPAA2_PER_LCORE_PORTAL;
 613
 614         pull_size = (nb_jobs > dpaa2_dqrr_size) ? dpaa2_dqrr_size : nb_jobs;
 615         rxq = &(dpdmai_dev->rx_queue[rxq_id]);
 616         fqid = rxq->fqid;
 617         q_storage = rxq->q_storage;
 618
 619         if (unlikely(!q_storage->active_dqs)) {
 620                 q_storage->toggle = 0;
 621                 dq_storage = q_storage->dq_storage[q_storage->toggle];
 622                 q_storage->last_num_pkts = pull_size;
 623                 qbman_pull_desc_clear(&pulldesc);
 624                 qbman_pull_desc_set_numframes(&pulldesc,
 625                                               q_storage->last_num_pkts);
 626                 qbman_pull_desc_set_fq(&pulldesc, fqid);
 627                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
 628                                 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
 629                 if (check_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)) {
 630                         while (!qbman_check_command_complete(
 631                                 get_swp_active_dqs(
 632                                 DPAA2_PER_LCORE_DPIO->index)))
 633                                 ;
 634                         clear_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index);
 635                 }
 636                 while (1) {
 637                         if (qbman_swp_pull(swp, &pulldesc)) {
 638                                 DPAA2_QDMA_DP_WARN(
 639                                         "VDQ command not issued.QBMAN busy\n");
 640                                         /* Portal was busy, try again */
 641                                 continue;
 642                         }
 643                         break;
 644                 }
 645                 q_storage->active_dqs = dq_storage;
 646                 q_storage->active_dpio_id = DPAA2_PER_LCORE_DPIO->index;
 647                 set_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index,
 648                                    dq_storage);
 649         }
 650
 651         dq_storage = q_storage->active_dqs;
 652         rte_prefetch0((void *)(size_t)(dq_storage));
 653         rte_prefetch0((void *)(size_t)(dq_storage + 1));
 654
 655         /* Prepare next pull descriptor. This will give space for the
 656          * prefething done on DQRR entries
 657          */
 658         q_storage->toggle ^= 1;
 659         dq_storage1 = q_storage->dq_storage[q_storage->toggle];
 660         qbman_pull_desc_clear(&pulldesc);
 661         qbman_pull_desc_set_numframes(&pulldesc, pull_size);
 662         qbman_pull_desc_set_fq(&pulldesc, fqid);
 663         qbman_pull_desc_set_storage(&pulldesc, dq_storage1,
 664                 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage1)), 1);
 665
 666         /* Check if the previous issued command is completed.
 667          * Also seems like the SWP is shared between the Ethernet Driver
 668          * and the SEC driver.
 669          */
 670         while (!qbman_check_command_complete(dq_storage))
 671                 ;
 672         if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id))
 673                 clear_swp_active_dqs(q_storage->active_dpio_id);
 674
 675         pending = 1;
 676
 677         do {
 678                 /* Loop until the dq_storage is updated with
 679                  * new token by QBMAN
 680                  */
 681                 while (!qbman_check_new_result(dq_storage))
 682                         ;
 683                 rte_prefetch0((void *)((size_t)(dq_storage + 2)));
 684                 /* Check whether Last Pull command is Expired and
 685                  * setting Condition for Loop termination
 686                  */
 687                 if (qbman_result_DQ_is_pull_complete(dq_storage)) {
 688                         pending = 0;
 689                         /* Check for valid frame. */
 690                         status = qbman_result_DQ_flags(dq_storage);
 691                         if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0))
 692                                 continue;
 693                 }
 694                 fd = qbman_result_DQ_fd(dq_storage);
 695
 696                 vqid = qdma_vq->get_job(qdma_vq, fd, &job[num_rx],
 697                                                                 &num_rx_ret);
 698                 if (vq_id)
 699                         vq_id[num_rx] = vqid;
 700
 701                 dq_storage++;
 702                 num_rx += num_rx_ret;
 703         } while (pending);
 704
 705         if (check_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)) {
 706                 while (!qbman_check_command_complete(
 707                         get_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)))
 708                         ;
 709                 clear_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index);
 710         }
 711         /* issue a volatile dequeue command for next pull */
 712         while (1) {
 713                 if (qbman_swp_pull(swp, &pulldesc)) {
 714                         DPAA2_QDMA_DP_WARN(
 715                                 "VDQ command is not issued. QBMAN is busy (2)\n");
 716                         continue;
 717                 }
 718                 break;
 719         }
 720
 721         q_storage->active_dqs = dq_storage1;
 722         q_storage->active_dpio_id = DPAA2_PER_LCORE_DPIO->index;
 723         set_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index, dq_storage1);
 724
 725         return num_rx;
 726 }
 727
 728 static int
 729 dpdmai_dev_dequeue_multijob_no_prefetch(
 730                 struct qdma_virt_queue *qdma_vq,
 731                 uint16_t *vq_id,
 732                 struct rte_qdma_job **job,
 733                 uint16_t nb_jobs)
 734 {
 735         struct qdma_hw_queue *qdma_pq = qdma_vq->hw_queue;
 736         struct dpaa2_dpdmai_dev *dpdmai_dev = qdma_pq->dpdmai_dev;
 737         uint16_t rxq_id = qdma_pq->queue_id;
 738
 739         struct dpaa2_queue *rxq;
 740         struct qbman_result *dq_storage;
 741         struct qbman_pull_desc pulldesc;
 742         struct qbman_swp *swp;
 743         uint32_t fqid;
 744         uint8_t status, pending;
 745         uint8_t num_rx = 0;
 746         const struct qbman_fd *fd;
 747         uint16_t vqid, num_rx_ret;
 748         int ret, next_pull, num_pulled = 0;
 749
 750         if (qdma_vq->flags & RTE_QDMA_VQ_FD_SG_FORMAT) {
 751                 /** Make sure there are enough space to get jobs.*/
 752                 if (unlikely(nb_jobs < DPAA2_QDMA_MAX_SG_NB))
 753                         return -EINVAL;
 754                 nb_jobs = 1;
 755         }
 756
 757         next_pull = nb_jobs;
 758
 759         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 760                 ret = dpaa2_affine_qbman_swp();
 761                 if (ret) {
 762                         DPAA2_QDMA_ERR(
 763                                 "Failed to allocate IO portal, tid: %d\n",
 764                                 rte_gettid());
 765                         return 0;
 766                 }
 767         }
 768         swp = DPAA2_PER_LCORE_PORTAL;
 769
 770         rxq = &(dpdmai_dev->rx_queue[rxq_id]);
 771         fqid = rxq->fqid;
 772
 773         do {
 774                 dq_storage = rxq->q_storage->dq_storage[0];
 775                 /* Prepare dequeue descriptor */
 776                 qbman_pull_desc_clear(&pulldesc);
 777                 qbman_pull_desc_set_fq(&pulldesc, fqid);
 778                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
 779                         (uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
 780
 781                 if (next_pull > dpaa2_dqrr_size) {
 782                         qbman_pull_desc_set_numframes(&pulldesc,
 783                                         dpaa2_dqrr_size);
 784                         next_pull -= dpaa2_dqrr_size;
 785                 } else {
 786                         qbman_pull_desc_set_numframes(&pulldesc, next_pull);
 787                         next_pull = 0;
 788                 }
 789
 790                 while (1) {
 791                         if (qbman_swp_pull(swp, &pulldesc)) {
 792                                 DPAA2_QDMA_DP_WARN(
 793                                         "VDQ command not issued. QBMAN busy");
 794                                 /* Portal was busy, try again */
 795                                 continue;
 796                         }
 797                         break;
 798                 }
 799
 800                 rte_prefetch0((void *)((size_t)(dq_storage + 1)));
 801                 /* Check if the previous issued command is completed. */
 802                 while (!qbman_check_command_complete(dq_storage))
 803                         ;
 804
 805                 num_pulled = 0;
 806                 pending = 1;
 807
 808                 do {
 809                         /* Loop until dq_storage is updated
 810                          * with new token by QBMAN
 811                          */
 812                         while (!qbman_check_new_result(dq_storage))
 813                                 ;
 814                         rte_prefetch0((void *)((size_t)(dq_storage + 2)));
 815
 816                         if (qbman_result_DQ_is_pull_complete(dq_storage)) {
 817                                 pending = 0;
 818                                 /* Check for valid frame. */
 819                                 status = qbman_result_DQ_flags(dq_storage);
 820                                 if (unlikely((status &
 821                                         QBMAN_DQ_STAT_VALIDFRAME) == 0))
 822                                         continue;
 823                         }
 824                         fd = qbman_result_DQ_fd(dq_storage);
 825
 826                         vqid = qdma_vq->get_job(qdma_vq, fd,
 827                                                 &job[num_rx], &num_rx_ret);
 828                         if (vq_id)
 829                                 vq_id[num_rx] = vqid;
 830
 831                         dq_storage++;
 832                         num_rx += num_rx_ret;
 833                         num_pulled++;
 834
 835                 } while (pending);
 836         /* Last VDQ provided all packets and more packets are requested */
 837         } while (next_pull && num_pulled == dpaa2_dqrr_size);
 838
 839         return num_rx;
 840 }
 841
 842 static int
 843 dpdmai_dev_enqueue_multi(
 844                         struct qdma_virt_queue *qdma_vq,
 845                         struct rte_qdma_job **job,
 846                         uint16_t nb_jobs)
 847 {
 848         struct qdma_hw_queue *qdma_pq = qdma_vq->hw_queue;
 849         struct dpaa2_dpdmai_dev *dpdmai_dev = qdma_pq->dpdmai_dev;
 850         uint16_t txq_id = qdma_pq->queue_id;
 851
 852         struct qbman_fd fd[RTE_QDMA_BURST_NB_MAX];
 853         struct dpaa2_queue *txq;
 854         struct qbman_eq_desc eqdesc;
 855         struct qbman_swp *swp;
 856         int ret;
 857         uint32_t num_to_send = 0;
 858         uint16_t num_tx = 0;
 859         uint32_t enqueue_loop, retry_count, loop;
 860
 861         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 862                 ret = dpaa2_affine_qbman_swp();
 863                 if (ret) {
 864                         DPAA2_QDMA_ERR(
 865                                 "Failed to allocate IO portal, tid: %d\n",
 866                                 rte_gettid());
 867                         return 0;
 868                 }
 869         }
 870         swp = DPAA2_PER_LCORE_PORTAL;
 871
 872         txq = &(dpdmai_dev->tx_queue[txq_id]);
 873
 874         /* Prepare enqueue descriptor */
 875         qbman_eq_desc_clear(&eqdesc);
 876         qbman_eq_desc_set_fq(&eqdesc, txq->fqid);
 877         qbman_eq_desc_set_no_orp(&eqdesc, 0);
 878         qbman_eq_desc_set_response(&eqdesc, 0, 0);
 879
 880         if (qdma_vq->flags & RTE_QDMA_VQ_FD_SG_FORMAT) {
 881                 uint16_t fd_nb;
 882                 uint16_t sg_entry_nb = nb_jobs > DPAA2_QDMA_MAX_SG_NB ?
 883                                                 DPAA2_QDMA_MAX_SG_NB : nb_jobs;
 884                 uint16_t job_idx = 0;
 885                 uint16_t fd_sg_nb[8];
 886                 uint16_t nb_jobs_ret = 0;
 887
 888                 if (nb_jobs % DPAA2_QDMA_MAX_SG_NB)
 889                         fd_nb = nb_jobs / DPAA2_QDMA_MAX_SG_NB + 1;
 890                 else
 891                         fd_nb = nb_jobs / DPAA2_QDMA_MAX_SG_NB;
 892
 893                 memset(&fd[0], 0, sizeof(struct qbman_fd) * fd_nb);
 894
 895                 for (loop = 0; loop < fd_nb; loop++) {
 896                         ret = qdma_vq->set_fd(qdma_vq, &fd[loop], &job[job_idx],
 897                                               sg_entry_nb);
 898                         if (unlikely(ret < 0))
 899                                 return 0;
 900                         fd_sg_nb[loop] = sg_entry_nb;
 901                         nb_jobs -= sg_entry_nb;
 902                         job_idx += sg_entry_nb;
 903                         sg_entry_nb = nb_jobs > DPAA2_QDMA_MAX_SG_NB ?
 904                                                 DPAA2_QDMA_MAX_SG_NB : nb_jobs;
 905                 }
 906
 907                 /* Enqueue the packet to the QBMAN */
 908                 enqueue_loop = 0; retry_count = 0;
 909
 910                 while (enqueue_loop < fd_nb) {
 911                         ret = qbman_swp_enqueue_multiple(swp,
 912                                         &eqdesc, &fd[enqueue_loop],
 913                                         NULL, fd_nb - enqueue_loop);
 914                         if (unlikely(ret < 0)) {
 915                                 retry_count++;
 916                                 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT)
 917                                         return nb_jobs_ret;
 918                         } else {
 919                                 for (loop = 0; loop < (uint32_t)ret; loop++)
 920                                         nb_jobs_ret +=
 921                                                 fd_sg_nb[enqueue_loop + loop];
 922                                 enqueue_loop += ret;
 923                                 retry_count = 0;
 924                         }
 925                 }
 926
 927                 return nb_jobs_ret;
 928         }
 929
 930         memset(fd, 0, nb_jobs * sizeof(struct qbman_fd));
 931
 932         while (nb_jobs > 0) {
 933                 num_to_send = (nb_jobs > dpaa2_eqcr_size) ?
 934                         dpaa2_eqcr_size : nb_jobs;
 935
 936                 ret = qdma_vq->set_fd(qdma_vq, &fd[num_tx],
 937                                                 &job[num_tx], num_to_send);
 938                 if (unlikely(ret < 0))
 939                         break;
 940
 941                 /* Enqueue the packet to the QBMAN */
 942                 enqueue_loop = 0; retry_count = 0;
 943                 loop = num_to_send;
 944
 945                 while (enqueue_loop < loop) {
 946                         ret = qbman_swp_enqueue_multiple(swp,
 947                                                 &eqdesc,
 948                                                 &fd[num_tx + enqueue_loop],
 949                                                 NULL,
 950                                                 loop - enqueue_loop);
 951                         if (unlikely(ret < 0)) {
 952                                 retry_count++;
 953                                 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT)
 954                                         return num_tx;
 955                         } else {
 956                                 enqueue_loop += ret;
 957                                 retry_count = 0;
 958                         }
 959                 }
 960                 num_tx += num_to_send;
 961                 nb_jobs -= loop;
 962         }
 963         return num_tx;
 964 }
 965
 966 static struct qdma_hw_queue *
 967 alloc_hw_queue(uint32_t lcore_id)
 968 {
 969         struct qdma_hw_queue *queue = NULL;
 970
 971         DPAA2_QDMA_FUNC_TRACE();
 972
 973         /* Get a free queue from the list */
 974         TAILQ_FOREACH(queue, &qdma_queue_list, next) {
 975                 if (queue->num_users == 0) {
 976                         queue->lcore_id = lcore_id;
 977                         queue->num_users++;
 978                         break;
 979                 }
 980         }
 981
 982         return queue;
 983 }
 984
 985 static void
 986 free_hw_queue(struct qdma_hw_queue *queue)
 987 {
 988         DPAA2_QDMA_FUNC_TRACE();
 989
 990         queue->num_users--;
 991 }
 992
 993
 994 static struct qdma_hw_queue *
 995 get_hw_queue(struct qdma_device *qdma_dev, uint32_t lcore_id)
 996 {
 997         struct qdma_per_core_info *core_info;
 998         struct qdma_hw_queue *queue, *temp;
 999         uint32_t least_num_users;
1000         int num_hw_queues, i;
1001
1002         DPAA2_QDMA_FUNC_TRACE();
1003
1004         core_info = &qdma_core_info[lcore_id];
1005         num_hw_queues = core_info->num_hw_queues;
1006
1007         /*
1008          * Allocate a HW queue if there are less queues
1009          * than maximum per core queues configured
1010          */
1011         if (num_hw_queues < qdma_dev->max_hw_queues_per_core) {
1012                 queue = alloc_hw_queue(lcore_id);
1013                 if (queue) {
1014                         core_info->hw_queues[num_hw_queues] = queue;
1015                         core_info->num_hw_queues++;
1016                         return queue;
1017                 }
1018         }
1019
1020         queue = core_info->hw_queues[0];
1021         /* In case there is no queue associated with the core return NULL */
1022         if (!queue)
1023                 return NULL;
1024
1025         /* Fetch the least loaded H/W queue */
1026         least_num_users = core_info->hw_queues[0]->num_users;
1027         for (i = 0; i < num_hw_queues; i++) {
1028                 temp = core_info->hw_queues[i];
1029                 if (temp->num_users < least_num_users)
1030                         queue = temp;
1031         }
1032
1033         if (queue)
1034                 queue->num_users++;
1035
1036         return queue;
1037 }
1038
1039 static void
1040 put_hw_queue(struct qdma_hw_queue *queue)
1041 {
1042         struct qdma_per_core_info *core_info;
1043         int lcore_id, num_hw_queues, i;
1044
1045         DPAA2_QDMA_FUNC_TRACE();
1046
1047         /*
1048          * If this is the last user of the queue free it.
1049          * Also remove it from QDMA core info.
1050          */
1051         if (queue->num_users == 1) {
1052                 free_hw_queue(queue);
1053
1054                 /* Remove the physical queue from core info */
1055                 lcore_id = queue->lcore_id;
1056                 core_info = &qdma_core_info[lcore_id];
1057                 num_hw_queues = core_info->num_hw_queues;
1058                 for (i = 0; i < num_hw_queues; i++) {
1059                         if (queue == core_info->hw_queues[i])
1060                                 break;
1061                 }
1062                 for (; i < num_hw_queues - 1; i++)
1063                         core_info->hw_queues[i] = core_info->hw_queues[i + 1];
1064                 core_info->hw_queues[i] = NULL;
1065         } else {
1066                 queue->num_users--;
1067         }
1068 }
1069
1070 static int
1071 dpaa2_qdma_attr_get(struct rte_rawdev *rawdev,
1072                     __rte_unused const char *attr_name,
1073                     uint64_t *attr_value)
1074 {
1075         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1076         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1077         struct rte_qdma_attr *qdma_attr = (struct rte_qdma_attr *)attr_value;
1078
1079         DPAA2_QDMA_FUNC_TRACE();
1080
1081         qdma_attr->num_hw_queues = qdma_dev->num_hw_queues;
1082
1083         return 0;
1084 }
1085
1086 static int
1087 dpaa2_qdma_reset(struct rte_rawdev *rawdev)
1088 {
1089         struct qdma_hw_queue *queue;
1090         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1091         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1092         int i;
1093
1094         DPAA2_QDMA_FUNC_TRACE();
1095
1096         /* In case QDMA device is not in stopped state, return -EBUSY */
1097         if (qdma_dev->state == 1) {
1098                 DPAA2_QDMA_ERR(
1099                         "Device is in running state. Stop before reset.");
1100                 return -EBUSY;
1101         }
1102
1103         /* In case there are pending jobs on any VQ, return -EBUSY */
1104         for (i = 0; i < qdma_dev->max_vqs; i++) {
1105                 if (qdma_dev->vqs[i].in_use && (qdma_dev->vqs[i].num_enqueues !=
1106                     qdma_dev->vqs[i].num_dequeues)) {
1107                         DPAA2_QDMA_ERR("Jobs are still pending on VQ: %d", i);
1108                         return -EBUSY;
1109                 }
1110         }
1111
1112         /* Reset HW queues */
1113         TAILQ_FOREACH(queue, &qdma_queue_list, next)
1114                 queue->num_users = 0;
1115
1116         /* Reset and free virtual queues */
1117         for (i = 0; i < qdma_dev->max_vqs; i++) {
1118                 if (qdma_dev->vqs[i].status_ring)
1119                         rte_ring_free(qdma_dev->vqs[i].status_ring);
1120         }
1121         if (qdma_dev->vqs)
1122                 rte_free(qdma_dev->vqs);
1123         qdma_dev->vqs = NULL;
1124
1125         /* Reset per core info */
1126         memset(&qdma_core_info, 0,
1127                 sizeof(struct qdma_per_core_info) * RTE_MAX_LCORE);
1128
1129         /* Reset QDMA device structure */
1130         qdma_dev->max_hw_queues_per_core = 0;
1131         qdma_dev->fle_queue_pool_cnt = 0;
1132         qdma_dev->max_vqs = 0;
1133
1134         return 0;
1135 }
1136
1137 static int
1138 dpaa2_qdma_configure(const struct rte_rawdev *rawdev,
1139                          rte_rawdev_obj_t config,
1140                          size_t config_size)
1141 {
1142         char name[32]; /* RTE_MEMZONE_NAMESIZE = 32 */
1143         struct rte_qdma_config *qdma_config = (struct rte_qdma_config *)config;
1144         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1145         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1146
1147         DPAA2_QDMA_FUNC_TRACE();
1148
1149         if (config_size != sizeof(*qdma_config))
1150                 return -EINVAL;
1151
1152         /* In case QDMA device is not in stopped state, return -EBUSY */
1153         if (qdma_dev->state == 1) {
1154                 DPAA2_QDMA_ERR(
1155                         "Device is in running state. Stop before config.");
1156                 return -1;
1157         }
1158
1159         /* Set max HW queue per core */
1160         if (qdma_config->max_hw_queues_per_core > MAX_HW_QUEUE_PER_CORE) {
1161                 DPAA2_QDMA_ERR("H/W queues per core is more than: %d",
1162                                MAX_HW_QUEUE_PER_CORE);
1163                 return -EINVAL;
1164         }
1165         qdma_dev->max_hw_queues_per_core =
1166                 qdma_config->max_hw_queues_per_core;
1167
1168         /* Allocate Virtual Queues */
1169         sprintf(name, "qdma_%d_vq", rawdev->dev_id);
1170         qdma_dev->vqs = rte_malloc(name,
1171                         (sizeof(struct qdma_virt_queue) * qdma_config->max_vqs),
1172                         RTE_CACHE_LINE_SIZE);
1173         if (!qdma_dev->vqs) {
1174                 DPAA2_QDMA_ERR("qdma_virtual_queues allocation failed");
1175                 return -ENOMEM;
1176         }
1177         qdma_dev->max_vqs = qdma_config->max_vqs;
1178         qdma_dev->fle_queue_pool_cnt = qdma_config->fle_queue_pool_cnt;
1179
1180         return 0;
1181 }
1182
1183 static int
1184 dpaa2_qdma_start(struct rte_rawdev *rawdev)
1185 {
1186         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1187         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1188
1189         DPAA2_QDMA_FUNC_TRACE();
1190
1191         qdma_dev->state = 1;
1192
1193         return 0;
1194 }
1195
1196 static int
1197 check_devargs_handler(__rte_unused const char *key, const char *value,
1198                       __rte_unused void *opaque)
1199 {
1200         if (strcmp(value, "1"))
1201                 return -1;
1202
1203         return 0;
1204 }
1205
1206 static int
1207 dpaa2_get_devargs(struct rte_devargs *devargs, const char *key)
1208 {
1209         struct rte_kvargs *kvlist;
1210
1211         if (!devargs)
1212                 return 0;
1213
1214         kvlist = rte_kvargs_parse(devargs->args, NULL);
1215         if (!kvlist)
1216                 return 0;
1217
1218         if (!rte_kvargs_count(kvlist, key)) {
1219                 rte_kvargs_free(kvlist);
1220                 return 0;
1221         }
1222
1223         if (rte_kvargs_process(kvlist, key,
1224                                check_devargs_handler, NULL) < 0) {
1225                 rte_kvargs_free(kvlist);
1226                 return 0;
1227         }
1228         rte_kvargs_free(kvlist);
1229
1230         return 1;
1231 }
1232
1233 static int
1234 dpaa2_qdma_queue_setup(struct rte_rawdev *rawdev,
1235                           __rte_unused uint16_t queue_id,
1236                           rte_rawdev_obj_t queue_conf,
1237                           size_t conf_size)
1238 {
1239         char ring_name[32];
1240         char pool_name[64];
1241         int i;
1242         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1243         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1244         struct rte_qdma_queue_config *q_config =
1245                 (struct rte_qdma_queue_config *)queue_conf;
1246         uint32_t pool_size;
1247
1248         DPAA2_QDMA_FUNC_TRACE();
1249
1250         if (conf_size != sizeof(*q_config))
1251                 return -EINVAL;
1252
1253         rte_spinlock_lock(&qdma_dev->lock);
1254
1255         /* Get a free Virtual Queue */
1256         for (i = 0; i < qdma_dev->max_vqs; i++) {
1257                 if (qdma_dev->vqs[i].in_use == 0)
1258                         break;
1259         }
1260
1261         /* Return in case no VQ is free */
1262         if (i == qdma_dev->max_vqs) {
1263                 rte_spinlock_unlock(&qdma_dev->lock);
1264                 DPAA2_QDMA_ERR("Unable to get lock on QDMA device");
1265                 return -ENODEV;
1266         }
1267
1268         if (q_config->flags & RTE_QDMA_VQ_FD_SG_FORMAT) {
1269                 if (!(q_config->flags & RTE_QDMA_VQ_EXCLUSIVE_PQ)) {
1270                         DPAA2_QDMA_ERR(
1271                                 "qDMA SG format only supports physical queue!");
1272                         rte_spinlock_unlock(&qdma_dev->lock);
1273                         return -ENODEV;
1274                 }
1275                 if (!(q_config->flags & RTE_QDMA_VQ_FD_LONG_FORMAT)) {
1276                         DPAA2_QDMA_ERR(
1277                                 "qDMA SG format only supports long FD format!");
1278                         rte_spinlock_unlock(&qdma_dev->lock);
1279                         return -ENODEV;
1280                 }
1281                 pool_size = QDMA_FLE_SG_POOL_SIZE;
1282         } else {
1283                 pool_size = QDMA_FLE_SINGLE_POOL_SIZE;
1284         }
1285
1286         if (q_config->flags & RTE_QDMA_VQ_EXCLUSIVE_PQ) {
1287                 /* Allocate HW queue for a VQ */
1288                 qdma_dev->vqs[i].hw_queue = alloc_hw_queue(q_config->lcore_id);
1289                 qdma_dev->vqs[i].exclusive_hw_queue = 1;
1290         } else {
1291                 /* Allocate a Ring for Virtual Queue in VQ mode */
1292                 snprintf(ring_name, sizeof(ring_name), "status ring %d", i);
1293                 qdma_dev->vqs[i].status_ring = rte_ring_create(ring_name,
1294                         qdma_dev->fle_queue_pool_cnt, rte_socket_id(), 0);
1295                 if (!qdma_dev->vqs[i].status_ring) {
1296                         DPAA2_QDMA_ERR("Status ring creation failed for vq");
1297                         rte_spinlock_unlock(&qdma_dev->lock);
1298                         return rte_errno;
1299                 }
1300
1301                 /* Get a HW queue (shared) for a VQ */
1302                 qdma_dev->vqs[i].hw_queue = get_hw_queue(qdma_dev,
1303                                                     q_config->lcore_id);
1304                 qdma_dev->vqs[i].exclusive_hw_queue = 0;
1305         }
1306
1307         if (qdma_dev->vqs[i].hw_queue == NULL) {
1308                 DPAA2_QDMA_ERR("No H/W queue available for VQ");
1309                 if (qdma_dev->vqs[i].status_ring)
1310                         rte_ring_free(qdma_dev->vqs[i].status_ring);
1311                 qdma_dev->vqs[i].status_ring = NULL;
1312                 rte_spinlock_unlock(&qdma_dev->lock);
1313                 return -ENODEV;
1314         }
1315
1316         snprintf(pool_name, sizeof(pool_name),
1317                 "qdma_fle_pool%u_queue%d", getpid(), i);
1318         qdma_dev->vqs[i].fle_pool = rte_mempool_create(pool_name,
1319                         qdma_dev->fle_queue_pool_cnt, pool_size,
1320                         QDMA_FLE_CACHE_SIZE(qdma_dev->fle_queue_pool_cnt), 0,
1321                         NULL, NULL, NULL, NULL, SOCKET_ID_ANY, 0);
1322         if (!qdma_dev->vqs[i].fle_pool) {
1323                 DPAA2_QDMA_ERR("qdma_fle_pool create failed");
1324                 rte_spinlock_unlock(&qdma_dev->lock);
1325                 return -ENOMEM;
1326         }
1327
1328         qdma_dev->vqs[i].flags = q_config->flags;
1329         qdma_dev->vqs[i].in_use = 1;
1330         qdma_dev->vqs[i].lcore_id = q_config->lcore_id;
1331         memset(&qdma_dev->vqs[i].rbp, 0, sizeof(struct rte_qdma_rbp));
1332
1333         if (q_config->flags & RTE_QDMA_VQ_FD_LONG_FORMAT) {
1334                 if (q_config->flags & RTE_QDMA_VQ_FD_SG_FORMAT) {
1335                         qdma_dev->vqs[i].set_fd = dpdmai_dev_set_sg_fd_lf;
1336                         qdma_dev->vqs[i].get_job = dpdmai_dev_get_sg_job_lf;
1337                 } else {
1338                         if (q_config->flags & RTE_QDMA_VQ_NO_RESPONSE)
1339                                 qdma_dev->vqs[i].set_fd =
1340                                         dpdmai_dev_set_multi_fd_lf_no_rsp;
1341                         else
1342                                 qdma_dev->vqs[i].set_fd =
1343                                         dpdmai_dev_set_multi_fd_lf;
1344                         qdma_dev->vqs[i].get_job = dpdmai_dev_get_single_job_lf;
1345                 }
1346         } else {
1347                 qdma_dev->vqs[i].set_fd = dpdmai_dev_set_fd_us;
1348                 qdma_dev->vqs[i].get_job = dpdmai_dev_get_job_us;
1349         }
1350         if (dpaa2_get_devargs(rawdev->device->devargs,
1351                         DPAA2_QDMA_NO_PREFETCH) ||
1352                         (getenv("DPAA2_NO_QDMA_PREFETCH_RX"))) {
1353                 /* If no prefetch is configured. */
1354                 qdma_dev->vqs[i].dequeue_job =
1355                                 dpdmai_dev_dequeue_multijob_no_prefetch;
1356                 DPAA2_QDMA_INFO("No Prefetch RX Mode enabled");
1357         } else {
1358                 qdma_dev->vqs[i].dequeue_job =
1359                         dpdmai_dev_dequeue_multijob_prefetch;
1360         }
1361
1362         qdma_dev->vqs[i].enqueue_job = dpdmai_dev_enqueue_multi;
1363
1364         if (q_config->rbp != NULL)
1365                 memcpy(&qdma_dev->vqs[i].rbp, q_config->rbp,
1366                                 sizeof(struct rte_qdma_rbp));
1367
1368         rte_spinlock_unlock(&qdma_dev->lock);
1369
1370         return i;
1371 }
1372
1373 static int
1374 dpaa2_qdma_enqueue(struct rte_rawdev *rawdev,
1375                   __rte_unused struct rte_rawdev_buf **buffers,
1376                   unsigned int nb_jobs,
1377                   rte_rawdev_obj_t context)
1378 {
1379         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1380         struct rte_qdma_enqdeq *e_context =
1381                 (struct rte_qdma_enqdeq *)context;
1382         struct qdma_virt_queue *qdma_vq =
1383                 &dpdmai_dev->qdma_dev->vqs[e_context->vq_id];
1384         int ret;
1385
1386         /* Return error in case of wrong lcore_id */
1387         if (rte_lcore_id() != qdma_vq->lcore_id) {
1388                 DPAA2_QDMA_ERR("QDMA enqueue for vqid %d on wrong core",
1389                                 e_context->vq_id);
1390                 return -EINVAL;
1391         }
1392
1393         ret = qdma_vq->enqueue_job(qdma_vq, e_context->job, nb_jobs);
1394         if (ret < 0) {
1395                 DPAA2_QDMA_ERR("DPDMAI device enqueue failed: %d", ret);
1396                 return ret;
1397         }
1398
1399         qdma_vq->num_enqueues += ret;
1400
1401         return ret;
1402 }
1403
1404 static int
1405 dpaa2_qdma_dequeue(struct rte_rawdev *rawdev,
1406                    __rte_unused struct rte_rawdev_buf **buffers,
1407                    unsigned int nb_jobs,
1408                    rte_rawdev_obj_t cntxt)
1409 {
1410         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1411         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1412         struct rte_qdma_enqdeq *context =
1413                 (struct rte_qdma_enqdeq *)cntxt;
1414         struct qdma_virt_queue *qdma_vq = &qdma_dev->vqs[context->vq_id];
1415         struct qdma_virt_queue *temp_qdma_vq;
1416         int ret = 0, i;
1417         unsigned int ring_count;
1418
1419         if (qdma_vq->flags & RTE_QDMA_VQ_FD_SG_FORMAT) {
1420                 /** Make sure there are enough space to get jobs.*/
1421                 if (unlikely(nb_jobs < DPAA2_QDMA_MAX_SG_NB))
1422                         return -EINVAL;
1423         }
1424
1425         /* Return error in case of wrong lcore_id */
1426         if (rte_lcore_id() != (unsigned int)(qdma_vq->lcore_id)) {
1427                 DPAA2_QDMA_WARN("QDMA dequeue for vqid %d on wrong core",
1428                                 context->vq_id);
1429                 return -1;
1430         }
1431
1432         /* Only dequeue when there are pending jobs on VQ */
1433         if (qdma_vq->num_enqueues == qdma_vq->num_dequeues)
1434                 return 0;
1435
1436         if (!(qdma_vq->flags & RTE_QDMA_VQ_FD_SG_FORMAT) &&
1437                 qdma_vq->num_enqueues < (qdma_vq->num_dequeues + nb_jobs))
1438                 nb_jobs = (qdma_vq->num_enqueues - qdma_vq->num_dequeues);
1439
1440         if (qdma_vq->exclusive_hw_queue) {
1441                 /* In case of exclusive queue directly fetch from HW queue */
1442                 ret = qdma_vq->dequeue_job(qdma_vq, NULL,
1443                                         context->job, nb_jobs);
1444                 if (ret < 0) {
1445                         DPAA2_QDMA_ERR(
1446                                 "Dequeue from DPDMAI device failed: %d", ret);
1447                         return ret;
1448                 }
1449                 qdma_vq->num_dequeues += ret;
1450         } else {
1451                 uint16_t temp_vq_id[RTE_QDMA_BURST_NB_MAX];
1452                 /*
1453                  * Get the QDMA completed jobs from the software ring.
1454                  * In case they are not available on the ring poke the HW
1455                  * to fetch completed jobs from corresponding HW queues
1456                  */
1457                 ring_count = rte_ring_count(qdma_vq->status_ring);
1458                 if (ring_count < nb_jobs) {
1459                         /* TODO - How to have right budget */
1460                         ret = qdma_vq->dequeue_job(qdma_vq,
1461                                         temp_vq_id, context->job, nb_jobs);
1462                         for (i = 0; i < ret; i++) {
1463                                 temp_qdma_vq = &qdma_dev->vqs[temp_vq_id[i]];
1464                                 rte_ring_enqueue(temp_qdma_vq->status_ring,
1465                                         (void *)(context->job[i]));
1466                         }
1467                         ring_count = rte_ring_count(
1468                                         qdma_vq->status_ring);
1469                 }
1470
1471                 if (ring_count) {
1472                         /* Dequeue job from the software ring
1473                          * to provide to the user
1474                          */
1475                         ret = rte_ring_dequeue_bulk(qdma_vq->status_ring,
1476                                                     (void **)context->job,
1477                                                     ring_count, NULL);
1478                         if (ret)
1479                                 qdma_vq->num_dequeues += ret;
1480                 }
1481         }
1482
1483         return ret;
1484 }
1485
1486 void
1487 rte_qdma_vq_stats(struct rte_rawdev *rawdev,
1488                 uint16_t vq_id,
1489                 struct rte_qdma_vq_stats *vq_status)
1490 {
1491         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1492         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1493         struct qdma_virt_queue *qdma_vq = &qdma_dev->vqs[vq_id];
1494
1495         if (qdma_vq->in_use) {
1496                 vq_status->exclusive_hw_queue = qdma_vq->exclusive_hw_queue;
1497                 vq_status->lcore_id = qdma_vq->lcore_id;
1498                 vq_status->num_enqueues = qdma_vq->num_enqueues;
1499                 vq_status->num_dequeues = qdma_vq->num_dequeues;
1500                 vq_status->num_pending_jobs = vq_status->num_enqueues -
1501                                 vq_status->num_dequeues;
1502         }
1503 }
1504
1505 static int
1506 dpaa2_qdma_queue_release(struct rte_rawdev *rawdev,
1507                          uint16_t vq_id)
1508 {
1509         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1510         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1511
1512         struct qdma_virt_queue *qdma_vq = &qdma_dev->vqs[vq_id];
1513
1514         DPAA2_QDMA_FUNC_TRACE();
1515
1516         /* In case there are pending jobs on any VQ, return -EBUSY */
1517         if (qdma_vq->num_enqueues != qdma_vq->num_dequeues)
1518                 return -EBUSY;
1519
1520         rte_spinlock_lock(&qdma_dev->lock);
1521
1522         if (qdma_vq->exclusive_hw_queue)
1523                 free_hw_queue(qdma_vq->hw_queue);
1524         else {
1525                 if (qdma_vq->status_ring)
1526                         rte_ring_free(qdma_vq->status_ring);
1527
1528                 put_hw_queue(qdma_vq->hw_queue);
1529         }
1530
1531         if (qdma_vq->fle_pool)
1532                 rte_mempool_free(qdma_vq->fle_pool);
1533
1534         memset(qdma_vq, 0, sizeof(struct qdma_virt_queue));
1535
1536         rte_spinlock_unlock(&qdma_dev->lock);
1537
1538         return 0;
1539 }
1540
1541 static void
1542 dpaa2_qdma_stop(struct rte_rawdev *rawdev)
1543 {
1544         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1545         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1546
1547         DPAA2_QDMA_FUNC_TRACE();
1548
1549         qdma_dev->state = 0;
1550 }
1551
1552 static int
1553 dpaa2_qdma_close(struct rte_rawdev *rawdev)
1554 {
1555         DPAA2_QDMA_FUNC_TRACE();
1556
1557         dpaa2_qdma_reset(rawdev);
1558
1559         return 0;
1560 }
1561
1562 static struct rte_rawdev_ops dpaa2_qdma_ops = {
1563         .dev_configure            = dpaa2_qdma_configure,
1564         .dev_start                = dpaa2_qdma_start,
1565         .dev_stop                 = dpaa2_qdma_stop,
1566         .dev_reset                = dpaa2_qdma_reset,
1567         .dev_close                = dpaa2_qdma_close,
1568         .queue_setup              = dpaa2_qdma_queue_setup,
1569         .queue_release            = dpaa2_qdma_queue_release,
1570         .attr_get                 = dpaa2_qdma_attr_get,
1571         .enqueue_bufs             = dpaa2_qdma_enqueue,
1572         .dequeue_bufs             = dpaa2_qdma_dequeue,
1573 };
1574
1575 static int
1576 add_hw_queues_to_list(struct dpaa2_dpdmai_dev *dpdmai_dev)
1577 {
1578         struct qdma_hw_queue *queue;
1579         int i;
1580
1581         DPAA2_QDMA_FUNC_TRACE();
1582
1583         for (i = 0; i < dpdmai_dev->num_queues; i++) {
1584                 queue = rte_zmalloc(NULL, sizeof(struct qdma_hw_queue), 0);
1585                 if (!queue) {
1586                         DPAA2_QDMA_ERR(
1587                                 "Memory allocation failed for QDMA queue");
1588                         return -ENOMEM;
1589                 }
1590
1591                 queue->dpdmai_dev = dpdmai_dev;
1592                 queue->queue_id = i;
1593
1594                 TAILQ_INSERT_TAIL(&qdma_queue_list, queue, next);
1595                 dpdmai_dev->qdma_dev->num_hw_queues++;
1596         }
1597
1598         return 0;
1599 }
1600
1601 static void
1602 remove_hw_queues_from_list(struct dpaa2_dpdmai_dev *dpdmai_dev)
1603 {
1604         struct qdma_hw_queue *queue = NULL;
1605         struct qdma_hw_queue *tqueue = NULL;
1606
1607         DPAA2_QDMA_FUNC_TRACE();
1608
1609         RTE_TAILQ_FOREACH_SAFE(queue, &qdma_queue_list, next, tqueue) {
1610                 if (queue->dpdmai_dev == dpdmai_dev) {
1611                         TAILQ_REMOVE(&qdma_queue_list, queue, next);
1612                         rte_free(queue);
1613                         queue = NULL;
1614                 }
1615         }
1616 }
1617
1618 static int
1619 dpaa2_dpdmai_dev_uninit(struct rte_rawdev *rawdev)
1620 {
1621         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1622         int ret, i;
1623
1624         DPAA2_QDMA_FUNC_TRACE();
1625
1626         /* Remove HW queues from global list */
1627         remove_hw_queues_from_list(dpdmai_dev);
1628
1629         ret = dpdmai_disable(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1630                              dpdmai_dev->token);
1631         if (ret)
1632                 DPAA2_QDMA_ERR("dmdmai disable failed");
1633
1634         /* Set up the DQRR storage for Rx */
1635         for (i = 0; i < dpdmai_dev->num_queues; i++) {
1636                 struct dpaa2_queue *rxq = &(dpdmai_dev->rx_queue[i]);
1637
1638                 if (rxq->q_storage) {
1639                         dpaa2_free_dq_storage(rxq->q_storage);
1640                         rte_free(rxq->q_storage);
1641                 }
1642         }
1643
1644         /* Close the device at underlying layer*/
1645         ret = dpdmai_close(&dpdmai_dev->dpdmai, CMD_PRI_LOW, dpdmai_dev->token);
1646         if (ret)
1647                 DPAA2_QDMA_ERR("Failure closing dpdmai device");
1648
1649         return 0;
1650 }
1651
1652 static int
1653 dpaa2_dpdmai_dev_init(struct rte_rawdev *rawdev, int dpdmai_id)
1654 {
1655         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1656         struct dpdmai_rx_queue_cfg rx_queue_cfg;
1657         struct dpdmai_attr attr;
1658         struct dpdmai_rx_queue_attr rx_attr;
1659         struct dpdmai_tx_queue_attr tx_attr;
1660         int ret, i;
1661
1662         DPAA2_QDMA_FUNC_TRACE();
1663
1664         /* Open DPDMAI device */
1665         dpdmai_dev->dpdmai_id = dpdmai_id;
1666         dpdmai_dev->dpdmai.regs = dpaa2_get_mcp_ptr(MC_PORTAL_INDEX);
1667         dpdmai_dev->qdma_dev = &q_dev;
1668         ret = dpdmai_open(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1669                           dpdmai_dev->dpdmai_id, &dpdmai_dev->token);
1670         if (ret) {
1671                 DPAA2_QDMA_ERR("dpdmai_open() failed with err: %d", ret);
1672                 return ret;
1673         }
1674
1675         /* Get DPDMAI attributes */
1676         ret = dpdmai_get_attributes(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1677                                     dpdmai_dev->token, &attr);
1678         if (ret) {
1679                 DPAA2_QDMA_ERR("dpdmai get attributes failed with err: %d",
1680                                ret);
1681                 goto init_err;
1682         }
1683         dpdmai_dev->num_queues = attr.num_of_queues;
1684
1685         /* Set up Rx Queues */
1686         for (i = 0; i < dpdmai_dev->num_queues; i++) {
1687                 struct dpaa2_queue *rxq;
1688
1689                 memset(&rx_queue_cfg, 0, sizeof(struct dpdmai_rx_queue_cfg));
1690                 ret = dpdmai_set_rx_queue(&dpdmai_dev->dpdmai,
1691                                           CMD_PRI_LOW,
1692                                           dpdmai_dev->token,
1693                                           i, 0, &rx_queue_cfg);
1694                 if (ret) {
1695                         DPAA2_QDMA_ERR("Setting Rx queue failed with err: %d",
1696                                        ret);
1697                         goto init_err;
1698                 }
1699
1700                 /* Allocate DQ storage for the DPDMAI Rx queues */
1701                 rxq = &(dpdmai_dev->rx_queue[i]);
1702                 rxq->q_storage = rte_malloc("dq_storage",
1703                                             sizeof(struct queue_storage_info_t),
1704                                             RTE_CACHE_LINE_SIZE);
1705                 if (!rxq->q_storage) {
1706                         DPAA2_QDMA_ERR("q_storage allocation failed");
1707                         ret = -ENOMEM;
1708                         goto init_err;
1709                 }
1710
1711                 memset(rxq->q_storage, 0, sizeof(struct queue_storage_info_t));
1712                 ret = dpaa2_alloc_dq_storage(rxq->q_storage);
1713                 if (ret) {
1714                         DPAA2_QDMA_ERR("dpaa2_alloc_dq_storage failed");
1715                         goto init_err;
1716                 }
1717         }
1718
1719         /* Get Rx and Tx queues FQID's */
1720         for (i = 0; i < dpdmai_dev->num_queues; i++) {
1721                 ret = dpdmai_get_rx_queue(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1722                                           dpdmai_dev->token, i, 0, &rx_attr);
1723                 if (ret) {
1724                         DPAA2_QDMA_ERR("Reading device failed with err: %d",
1725                                        ret);
1726                         goto init_err;
1727                 }
1728                 dpdmai_dev->rx_queue[i].fqid = rx_attr.fqid;
1729
1730                 ret = dpdmai_get_tx_queue(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1731                                           dpdmai_dev->token, i, 0, &tx_attr);
1732                 if (ret) {
1733                         DPAA2_QDMA_ERR("Reading device failed with err: %d",
1734                                        ret);
1735                         goto init_err;
1736                 }
1737                 dpdmai_dev->tx_queue[i].fqid = tx_attr.fqid;
1738         }
1739
1740         /* Enable the device */
1741         ret = dpdmai_enable(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1742                             dpdmai_dev->token);
1743         if (ret) {
1744                 DPAA2_QDMA_ERR("Enabling device failed with err: %d", ret);
1745                 goto init_err;
1746         }
1747
1748         /* Add the HW queue to the global list */
1749         ret = add_hw_queues_to_list(dpdmai_dev);
1750         if (ret) {
1751                 DPAA2_QDMA_ERR("Adding H/W queue to list failed");
1752                 goto init_err;
1753         }
1754
1755         if (!dpaa2_coherent_no_alloc_cache) {
1756                 if (dpaa2_svr_family == SVR_LX2160A) {
1757                         dpaa2_coherent_no_alloc_cache =
1758                                 DPAA2_LX2_COHERENT_NO_ALLOCATE_CACHE;
1759                         dpaa2_coherent_alloc_cache =
1760                                 DPAA2_LX2_COHERENT_ALLOCATE_CACHE;
1761                 } else {
1762                         dpaa2_coherent_no_alloc_cache =
1763                                 DPAA2_COHERENT_NO_ALLOCATE_CACHE;
1764                         dpaa2_coherent_alloc_cache =
1765                                 DPAA2_COHERENT_ALLOCATE_CACHE;
1766                 }
1767         }
1768
1769         DPAA2_QDMA_DEBUG("Initialized dpdmai object successfully");
1770
1771         rte_spinlock_init(&dpdmai_dev->qdma_dev->lock);
1772
1773         return 0;
1774 init_err:
1775         dpaa2_dpdmai_dev_uninit(rawdev);
1776         return ret;
1777 }
1778
1779 static int
1780 rte_dpaa2_qdma_probe(struct rte_dpaa2_driver *dpaa2_drv,
1781                      struct rte_dpaa2_device *dpaa2_dev)
1782 {
1783         struct rte_rawdev *rawdev;
1784         int ret;
1785
1786         DPAA2_QDMA_FUNC_TRACE();
1787
1788         rawdev = rte_rawdev_pmd_allocate(dpaa2_dev->device.name,
1789                         sizeof(struct dpaa2_dpdmai_dev),
1790                         rte_socket_id());
1791         if (!rawdev) {
1792                 DPAA2_QDMA_ERR("Unable to allocate rawdevice");
1793                 return -EINVAL;
1794         }
1795
1796         dpaa2_dev->rawdev = rawdev;
1797         rawdev->dev_ops = &dpaa2_qdma_ops;
1798         rawdev->device = &dpaa2_dev->device;
1799         rawdev->driver_name = dpaa2_drv->driver.name;
1800
1801         /* Invoke PMD device initialization function */
1802         ret = dpaa2_dpdmai_dev_init(rawdev, dpaa2_dev->object_id);
1803         if (ret) {
1804                 rte_rawdev_pmd_release(rawdev);
1805                 return ret;
1806         }
1807
1808         /* Reset the QDMA device */
1809         ret = dpaa2_qdma_reset(rawdev);
1810         if (ret) {
1811                 DPAA2_QDMA_ERR("Resetting QDMA failed");
1812                 return ret;
1813         }
1814
1815         return 0;
1816 }
1817
1818 static int
1819 rte_dpaa2_qdma_remove(struct rte_dpaa2_device *dpaa2_dev)
1820 {
1821         struct rte_rawdev *rawdev = dpaa2_dev->rawdev;
1822         int ret;
1823
1824         DPAA2_QDMA_FUNC_TRACE();
1825
1826         dpaa2_dpdmai_dev_uninit(rawdev);
1827
1828         ret = rte_rawdev_pmd_release(rawdev);
1829         if (ret)
1830                 DPAA2_QDMA_ERR("Device cleanup failed");
1831
1832         return 0;
1833 }
1834
1835 static struct rte_dpaa2_driver rte_dpaa2_qdma_pmd = {
1836         .drv_flags = RTE_DPAA2_DRV_IOVA_AS_VA,
1837         .drv_type = DPAA2_QDMA,
1838         .probe = rte_dpaa2_qdma_probe,
1839         .remove = rte_dpaa2_qdma_remove,
1840 };
1841
1842 RTE_PMD_REGISTER_DPAA2(dpaa2_qdma, rte_dpaa2_qdma_pmd);
1843 RTE_PMD_REGISTER_PARAM_STRING(dpaa2_qdma,
1844         "no_prefetch=<int> ");
1845 RTE_LOG_REGISTER(dpaa2_qdma_logtype, pmd.raw.dpaa2.qdma, INFO);