drivers/raw/dpaa2_qdma/dpaa2_qdma.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018-2020 NXP
   3  */
   4
   5 #include <string.h>
   6
   7 #include <rte_eal.h>
   8 #include <rte_fslmc.h>
   9 #include <rte_atomic.h>
  10 #include <rte_lcore.h>
  11 #include <rte_rawdev.h>
  12 #include <rte_rawdev_pmd.h>
  13 #include <rte_malloc.h>
  14 #include <rte_ring.h>
  15 #include <rte_mempool.h>
  16 #include <rte_prefetch.h>
  17 #include <rte_kvargs.h>
  18
  19 #include <mc/fsl_dpdmai.h>
  20 #include <portal/dpaa2_hw_pvt.h>
  21 #include <portal/dpaa2_hw_dpio.h>
  22
  23 #include "rte_pmd_dpaa2_qdma.h"
  24 #include "dpaa2_qdma.h"
  25 #include "dpaa2_qdma_logs.h"
  26
  27 #define DPAA2_QDMA_NO_PREFETCH "no_prefetch"
  28
  29 uint32_t dpaa2_coherent_no_alloc_cache;
  30 uint32_t dpaa2_coherent_alloc_cache;
  31
  32 /* QDMA device */
  33 static struct qdma_device q_dev;
  34
  35 /* QDMA H/W queues list */
  36 TAILQ_HEAD(qdma_hw_queue_list, qdma_hw_queue);
  37 static struct qdma_hw_queue_list qdma_queue_list
  38         = TAILQ_HEAD_INITIALIZER(qdma_queue_list);
  39
  40 /* QDMA Virtual Queues */
  41 static struct qdma_virt_queue *qdma_vqs;
  42
  43 /* QDMA per core data */
  44 static struct qdma_per_core_info qdma_core_info[RTE_MAX_LCORE];
  45
  46 typedef int (dpdmai_dev_dequeue_multijob_t)(struct dpaa2_dpdmai_dev *dpdmai_dev,
  47                                             uint16_t rxq_id,
  48                                             uint16_t *vq_id,
  49                                             struct rte_qdma_job **job,
  50                                             uint16_t nb_jobs);
  51
  52 dpdmai_dev_dequeue_multijob_t *dpdmai_dev_dequeue_multijob;
  53
  54 typedef uint16_t (dpdmai_dev_get_job_t)(struct qdma_device *qdma_dev,
  55                                         const struct qbman_fd *fd,
  56                                         struct rte_qdma_job **job);
  57 typedef int (dpdmai_dev_set_fd_t)(struct qdma_device *qdma_dev,
  58                                   struct qbman_fd *fd,
  59                                   struct rte_qdma_job *job,
  60                                   struct rte_qdma_rbp *rbp,
  61                                   uint16_t vq_id);
  62 dpdmai_dev_get_job_t *dpdmai_dev_get_job;
  63 dpdmai_dev_set_fd_t *dpdmai_dev_set_fd;
  64
  65 static inline int
  66 qdma_populate_fd_pci(phys_addr_t src, phys_addr_t dest,
  67                         uint32_t len, struct qbman_fd *fd,
  68                         struct rte_qdma_rbp *rbp)
  69 {
  70         fd->simple_pci.saddr_lo = lower_32_bits((uint64_t) (src));
  71         fd->simple_pci.saddr_hi = upper_32_bits((uint64_t) (src));
  72
  73         fd->simple_pci.len_sl = len;
  74
  75         fd->simple_pci.bmt = 1;
  76         fd->simple_pci.fmt = 3;
  77         fd->simple_pci.sl = 1;
  78         fd->simple_pci.ser = 1;
  79
  80         fd->simple_pci.sportid = rbp->sportid;  /*pcie 3 */
  81         fd->simple_pci.srbp = rbp->srbp;
  82         if (rbp->srbp)
  83                 fd->simple_pci.rdttype = 0;
  84         else
  85                 fd->simple_pci.rdttype = dpaa2_coherent_alloc_cache;
  86
  87         /*dest is pcie memory */
  88         fd->simple_pci.dportid = rbp->dportid;  /*pcie 3 */
  89         fd->simple_pci.drbp = rbp->drbp;
  90         if (rbp->drbp)
  91                 fd->simple_pci.wrttype = 0;
  92         else
  93                 fd->simple_pci.wrttype = dpaa2_coherent_no_alloc_cache;
  94
  95         fd->simple_pci.daddr_lo = lower_32_bits((uint64_t) (dest));
  96         fd->simple_pci.daddr_hi = upper_32_bits((uint64_t) (dest));
  97
  98         return 0;
  99 }
 100
 101 static inline int
 102 qdma_populate_fd_ddr(phys_addr_t src, phys_addr_t dest,
 103                         uint32_t len, struct qbman_fd *fd)
 104 {
 105         fd->simple_ddr.saddr_lo = lower_32_bits((uint64_t) (src));
 106         fd->simple_ddr.saddr_hi = upper_32_bits((uint64_t) (src));
 107
 108         fd->simple_ddr.len = len;
 109
 110         fd->simple_ddr.bmt = 1;
 111         fd->simple_ddr.fmt = 3;
 112         fd->simple_ddr.sl = 1;
 113         fd->simple_ddr.ser = 1;
 114         /**
 115          * src If RBP=0 {NS,RDTTYPE[3:0]}: 0_1011
 116          * Coherent copy of cacheable memory,
 117          * lookup in downstream cache, no allocate
 118          * on miss
 119          */
 120         fd->simple_ddr.rns = 0;
 121         fd->simple_ddr.rdttype = dpaa2_coherent_alloc_cache;
 122         /**
 123          * dest If RBP=0 {NS,WRTTYPE[3:0]}: 0_0111
 124          * Coherent write of cacheable memory,
 125          * lookup in downstream cache, no allocate on miss
 126          */
 127         fd->simple_ddr.wns = 0;
 128         fd->simple_ddr.wrttype = dpaa2_coherent_no_alloc_cache;
 129
 130         fd->simple_ddr.daddr_lo = lower_32_bits((uint64_t) (dest));
 131         fd->simple_ddr.daddr_hi = upper_32_bits((uint64_t) (dest));
 132
 133         return 0;
 134 }
 135
 136 static void
 137 dpaa2_qdma_populate_fle(struct qbman_fle *fle,
 138                         struct rte_qdma_rbp *rbp,
 139                         uint64_t src, uint64_t dest,
 140                         size_t len, uint32_t flags)
 141 {
 142         struct qdma_sdd *sdd;
 143
 144         sdd = (struct qdma_sdd *)((uint8_t *)(fle) +
 145                 (DPAA2_QDMA_MAX_FLE * sizeof(struct qbman_fle)));
 146
 147         /* first frame list to source descriptor */
 148         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sdd));
 149         DPAA2_SET_FLE_LEN(fle, (2 * (sizeof(struct qdma_sdd))));
 150
 151         /* source and destination descriptor */
 152         if (rbp && rbp->enable) {
 153                 /* source */
 154                 sdd->read_cmd.portid = rbp->sportid;
 155                 sdd->rbpcmd_simple.pfid = rbp->spfid;
 156                 sdd->rbpcmd_simple.vfid = rbp->svfid;
 157
 158                 if (rbp->srbp) {
 159                         sdd->read_cmd.rbp = rbp->srbp;
 160                         sdd->read_cmd.rdtype = DPAA2_RBP_MEM_RW;
 161                 } else {
 162                         sdd->read_cmd.rdtype = dpaa2_coherent_no_alloc_cache;
 163                 }
 164                 sdd++;
 165                 /* destination */
 166                 sdd->write_cmd.portid = rbp->dportid;
 167                 sdd->rbpcmd_simple.pfid = rbp->dpfid;
 168                 sdd->rbpcmd_simple.vfid = rbp->dvfid;
 169
 170                 if (rbp->drbp) {
 171                         sdd->write_cmd.rbp = rbp->drbp;
 172                         sdd->write_cmd.wrttype = DPAA2_RBP_MEM_RW;
 173                 } else {
 174                         sdd->write_cmd.wrttype = dpaa2_coherent_alloc_cache;
 175                 }
 176
 177         } else {
 178                 sdd->read_cmd.rdtype = dpaa2_coherent_no_alloc_cache;
 179                 sdd++;
 180                 sdd->write_cmd.wrttype = dpaa2_coherent_alloc_cache;
 181         }
 182         fle++;
 183         /* source frame list to source buffer */
 184         if (flags & RTE_QDMA_JOB_SRC_PHY) {
 185                 DPAA2_SET_FLE_ADDR(fle, src);
 186                 DPAA2_SET_FLE_BMT(fle);
 187         } else {
 188                 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(src));
 189         }
 190         DPAA2_SET_FLE_LEN(fle, len);
 191
 192         fle++;
 193         /* destination frame list to destination buffer */
 194         if (flags & RTE_QDMA_JOB_DEST_PHY) {
 195                 DPAA2_SET_FLE_BMT(fle);
 196                 DPAA2_SET_FLE_ADDR(fle, dest);
 197         } else {
 198                 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(dest));
 199         }
 200         DPAA2_SET_FLE_LEN(fle, len);
 201
 202         /* Final bit: 1, for last frame list */
 203         DPAA2_SET_FLE_FIN(fle);
 204 }
 205
 206 static inline int dpdmai_dev_set_fd_us(
 207                                 struct qdma_device *qdma_dev __rte_unused,
 208                                 struct qbman_fd *fd,
 209                                 struct rte_qdma_job *job,
 210                                 struct rte_qdma_rbp *rbp,
 211                                 uint16_t vq_id)
 212 {
 213         struct rte_qdma_job **ppjob;
 214         size_t iova;
 215         int ret = 0;
 216
 217         if (job->src & QDMA_RBP_UPPER_ADDRESS_MASK)
 218                 iova = (size_t)job->dest;
 219         else
 220                 iova = (size_t)job->src;
 221
 222         /* Set the metadata */
 223         job->vq_id = vq_id;
 224         ppjob = (struct rte_qdma_job **)DPAA2_IOVA_TO_VADDR(iova) - 1;
 225         *ppjob = job;
 226
 227         if ((rbp->drbp == 1) || (rbp->srbp == 1))
 228                 ret = qdma_populate_fd_pci((phys_addr_t) job->src,
 229                                            (phys_addr_t) job->dest,
 230                                            job->len, fd, rbp);
 231         else
 232                 ret = qdma_populate_fd_ddr((phys_addr_t) job->src,
 233                                            (phys_addr_t) job->dest,
 234                                            job->len, fd);
 235         return ret;
 236 }
 237 static inline int dpdmai_dev_set_fd_lf(struct qdma_device *qdma_dev,
 238                                         struct qbman_fd *fd,
 239                                         struct rte_qdma_job *job,
 240                                         struct rte_qdma_rbp *rbp,
 241                                         uint16_t vq_id)
 242 {
 243         struct rte_qdma_job **ppjob;
 244         struct qbman_fle *fle;
 245         int ret = 0;
 246         /*
 247          * Get an FLE/SDD from FLE pool.
 248          * Note: IO metadata is before the FLE and SDD memory.
 249          */
 250         ret = rte_mempool_get(qdma_dev->fle_pool, (void **)(&ppjob));
 251         if (ret) {
 252                 DPAA2_QDMA_DP_DEBUG("Memory alloc failed for FLE");
 253                 return ret;
 254         }
 255
 256         /* Set the metadata */
 257         job->vq_id = vq_id;
 258         *ppjob = job;
 259
 260         fle = (struct qbman_fle *)(ppjob + 1);
 261
 262         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
 263         DPAA2_SET_FD_COMPOUND_FMT(fd);
 264         DPAA2_SET_FD_FRC(fd, QDMA_SER_CTX);
 265
 266         /* Populate FLE */
 267         memset(fle, 0, QDMA_FLE_POOL_SIZE);
 268         dpaa2_qdma_populate_fle(fle, rbp, job->src, job->dest,
 269                                 job->len, job->flags);
 270
 271         return 0;
 272 }
 273
 274 static inline uint16_t dpdmai_dev_get_job_us(
 275                                 struct qdma_device *qdma_dev __rte_unused,
 276                                 const struct qbman_fd *fd,
 277                                 struct rte_qdma_job **job)
 278 {
 279         uint16_t vqid;
 280         size_t iova;
 281         struct rte_qdma_job **ppjob;
 282
 283         if (fd->simple_pci.saddr_hi & (QDMA_RBP_UPPER_ADDRESS_MASK >> 32))
 284                 iova = (size_t) (((uint64_t)fd->simple_pci.daddr_hi) << 32
 285                                 | (uint64_t)fd->simple_pci.daddr_lo);
 286         else
 287                 iova = (size_t)(((uint64_t)fd->simple_pci.saddr_hi) << 32
 288                                 | (uint64_t)fd->simple_pci.saddr_lo);
 289
 290         ppjob = (struct rte_qdma_job **)DPAA2_IOVA_TO_VADDR(iova) - 1;
 291         *job = (struct rte_qdma_job *)*ppjob;
 292         (*job)->status = (fd->simple_pci.acc_err << 8) | (fd->simple_pci.error);
 293         vqid = (*job)->vq_id;
 294
 295         return vqid;
 296 }
 297
 298 static inline uint16_t dpdmai_dev_get_job_lf(struct qdma_device *qdma_dev,
 299                                              const struct qbman_fd *fd,
 300                                              struct rte_qdma_job **job)
 301 {
 302         struct rte_qdma_job **ppjob;
 303         uint16_t vqid;
 304         /*
 305          * Fetch metadata from FLE. job and vq_id were set
 306          * in metadata in the enqueue operation.
 307          */
 308         ppjob = (struct rte_qdma_job **)
 309                         DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
 310         ppjob -= 1;
 311
 312         *job = (struct rte_qdma_job *)*ppjob;
 313         (*job)->status = (DPAA2_GET_FD_ERR(fd) << 8) |
 314                          (DPAA2_GET_FD_FRC(fd) & 0xFF);
 315         vqid = (*job)->vq_id;
 316
 317         /* Free FLE to the pool */
 318         rte_mempool_put(qdma_dev->fle_pool, (void *)ppjob);
 319
 320         return vqid;
 321 }
 322
 323 static struct qdma_hw_queue *
 324 alloc_hw_queue(uint32_t lcore_id)
 325 {
 326         struct qdma_hw_queue *queue = NULL;
 327
 328         DPAA2_QDMA_FUNC_TRACE();
 329
 330         /* Get a free queue from the list */
 331         TAILQ_FOREACH(queue, &qdma_queue_list, next) {
 332                 if (queue->num_users == 0) {
 333                         queue->lcore_id = lcore_id;
 334                         queue->num_users++;
 335                         break;
 336                 }
 337         }
 338
 339         return queue;
 340 }
 341
 342 static void
 343 free_hw_queue(struct qdma_hw_queue *queue)
 344 {
 345         DPAA2_QDMA_FUNC_TRACE();
 346
 347         queue->num_users--;
 348 }
 349
 350
 351 static struct qdma_hw_queue *
 352 get_hw_queue(struct qdma_device *qdma_dev, uint32_t lcore_id)
 353 {
 354         struct qdma_per_core_info *core_info;
 355         struct qdma_hw_queue *queue, *temp;
 356         uint32_t least_num_users;
 357         int num_hw_queues, i;
 358
 359         DPAA2_QDMA_FUNC_TRACE();
 360
 361         core_info = &qdma_core_info[lcore_id];
 362         num_hw_queues = core_info->num_hw_queues;
 363
 364         /*
 365          * Allocate a HW queue if there are less queues
 366          * than maximum per core queues configured
 367          */
 368         if (num_hw_queues < qdma_dev->max_hw_queues_per_core) {
 369                 queue = alloc_hw_queue(lcore_id);
 370                 if (queue) {
 371                         core_info->hw_queues[num_hw_queues] = queue;
 372                         core_info->num_hw_queues++;
 373                         return queue;
 374                 }
 375         }
 376
 377         queue = core_info->hw_queues[0];
 378         /* In case there is no queue associated with the core return NULL */
 379         if (!queue)
 380                 return NULL;
 381
 382         /* Fetch the least loaded H/W queue */
 383         least_num_users = core_info->hw_queues[0]->num_users;
 384         for (i = 0; i < num_hw_queues; i++) {
 385                 temp = core_info->hw_queues[i];
 386                 if (temp->num_users < least_num_users)
 387                         queue = temp;
 388         }
 389
 390         if (queue)
 391                 queue->num_users++;
 392
 393         return queue;
 394 }
 395
 396 static void
 397 put_hw_queue(struct qdma_hw_queue *queue)
 398 {
 399         struct qdma_per_core_info *core_info;
 400         int lcore_id, num_hw_queues, i;
 401
 402         DPAA2_QDMA_FUNC_TRACE();
 403
 404         /*
 405          * If this is the last user of the queue free it.
 406          * Also remove it from QDMA core info.
 407          */
 408         if (queue->num_users == 1) {
 409                 free_hw_queue(queue);
 410
 411                 /* Remove the physical queue from core info */
 412                 lcore_id = queue->lcore_id;
 413                 core_info = &qdma_core_info[lcore_id];
 414                 num_hw_queues = core_info->num_hw_queues;
 415                 for (i = 0; i < num_hw_queues; i++) {
 416                         if (queue == core_info->hw_queues[i])
 417                                 break;
 418                 }
 419                 for (; i < num_hw_queues - 1; i++)
 420                         core_info->hw_queues[i] = core_info->hw_queues[i + 1];
 421                 core_info->hw_queues[i] = NULL;
 422         } else {
 423                 queue->num_users--;
 424         }
 425 }
 426
 427 static int
 428 dpaa2_qdma_attr_get(struct rte_rawdev *rawdev,
 429                     __rte_unused const char *attr_name,
 430                     uint64_t *attr_value)
 431 {
 432         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
 433         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
 434         struct rte_qdma_attr *qdma_attr = (struct rte_qdma_attr *)attr_value;
 435
 436         DPAA2_QDMA_FUNC_TRACE();
 437
 438         qdma_attr->num_hw_queues = qdma_dev->num_hw_queues;
 439
 440         return 0;
 441 }
 442
 443 static int
 444 dpaa2_qdma_reset(struct rte_rawdev *rawdev)
 445 {
 446         struct qdma_hw_queue *queue;
 447         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
 448         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
 449         int i;
 450
 451         DPAA2_QDMA_FUNC_TRACE();
 452
 453         /* In case QDMA device is not in stopped state, return -EBUSY */
 454         if (qdma_dev->state == 1) {
 455                 DPAA2_QDMA_ERR(
 456                         "Device is in running state. Stop before reset.");
 457                 return -EBUSY;
 458         }
 459
 460         /* In case there are pending jobs on any VQ, return -EBUSY */
 461         for (i = 0; i < qdma_dev->max_vqs; i++) {
 462                 if (qdma_vqs[i].in_use && (qdma_vqs[i].num_enqueues !=
 463                     qdma_vqs[i].num_dequeues)) {
 464                         DPAA2_QDMA_ERR("Jobs are still pending on VQ: %d", i);
 465                         return -EBUSY;
 466                 }
 467         }
 468
 469         /* Reset HW queues */
 470         TAILQ_FOREACH(queue, &qdma_queue_list, next)
 471                 queue->num_users = 0;
 472
 473         /* Reset and free virtual queues */
 474         for (i = 0; i < qdma_dev->max_vqs; i++) {
 475                 if (qdma_vqs[i].status_ring)
 476                         rte_ring_free(qdma_vqs[i].status_ring);
 477         }
 478         if (qdma_vqs)
 479                 rte_free(qdma_vqs);
 480         qdma_vqs = NULL;
 481
 482         /* Reset per core info */
 483         memset(&qdma_core_info, 0,
 484                 sizeof(struct qdma_per_core_info) * RTE_MAX_LCORE);
 485
 486         /* Free the FLE pool */
 487         if (qdma_dev->fle_pool)
 488                 rte_mempool_free(qdma_dev->fle_pool);
 489
 490         /* Reset QDMA device structure */
 491         qdma_dev->mode = RTE_QDMA_MODE_HW;
 492         qdma_dev->max_hw_queues_per_core = 0;
 493         qdma_dev->fle_pool = NULL;
 494         qdma_dev->fle_pool_count = 0;
 495         qdma_dev->max_vqs = 0;
 496
 497         return 0;
 498 }
 499
 500 static int
 501 dpaa2_qdma_configure(const struct rte_rawdev *rawdev,
 502                          rte_rawdev_obj_t config,
 503                          size_t config_size)
 504 {
 505         char fle_pool_name[32]; /* RTE_MEMZONE_NAMESIZE = 32 */
 506         struct rte_qdma_config *qdma_config = (struct rte_qdma_config *)config;
 507         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
 508         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
 509
 510         DPAA2_QDMA_FUNC_TRACE();
 511
 512         if (config_size != sizeof(*qdma_config))
 513                 return -EINVAL;
 514
 515         /* In case QDMA device is not in stopped state, return -EBUSY */
 516         if (qdma_dev->state == 1) {
 517                 DPAA2_QDMA_ERR(
 518                         "Device is in running state. Stop before config.");
 519                 return -1;
 520         }
 521
 522         /* Set mode */
 523         qdma_dev->mode = qdma_config->mode;
 524
 525         /* Set max HW queue per core */
 526         if (qdma_config->max_hw_queues_per_core > MAX_HW_QUEUE_PER_CORE) {
 527                 DPAA2_QDMA_ERR("H/W queues per core is more than: %d",
 528                                MAX_HW_QUEUE_PER_CORE);
 529                 return -EINVAL;
 530         }
 531         qdma_dev->max_hw_queues_per_core =
 532                 qdma_config->max_hw_queues_per_core;
 533
 534         /* Allocate Virtual Queues */
 535         qdma_vqs = rte_malloc("qdma_virtual_queues",
 536                         (sizeof(struct qdma_virt_queue) * qdma_config->max_vqs),
 537                         RTE_CACHE_LINE_SIZE);
 538         if (!qdma_vqs) {
 539                 DPAA2_QDMA_ERR("qdma_virtual_queues allocation failed");
 540                 return -ENOMEM;
 541         }
 542         qdma_dev->max_vqs = qdma_config->max_vqs;
 543
 544         /* Allocate FLE pool; just append PID so that in case of
 545          * multiprocess, the pool's don't collide.
 546          */
 547         snprintf(fle_pool_name, sizeof(fle_pool_name), "qdma_fle_pool%u",
 548                  getpid());
 549         qdma_dev->fle_pool = rte_mempool_create(fle_pool_name,
 550                         qdma_config->fle_pool_count, QDMA_FLE_POOL_SIZE,
 551                         QDMA_FLE_CACHE_SIZE(qdma_config->fle_pool_count), 0,
 552                         NULL, NULL, NULL, NULL, SOCKET_ID_ANY, 0);
 553         if (!qdma_dev->fle_pool) {
 554                 DPAA2_QDMA_ERR("qdma_fle_pool create failed");
 555                 rte_free(qdma_vqs);
 556                 qdma_vqs = NULL;
 557                 return -ENOMEM;
 558         }
 559         qdma_dev->fle_pool_count = qdma_config->fle_pool_count;
 560
 561         if (qdma_config->format == RTE_QDMA_ULTRASHORT_FORMAT) {
 562                 dpdmai_dev_get_job = dpdmai_dev_get_job_us;
 563                 dpdmai_dev_set_fd = dpdmai_dev_set_fd_us;
 564         } else {
 565                 dpdmai_dev_get_job = dpdmai_dev_get_job_lf;
 566                 dpdmai_dev_set_fd = dpdmai_dev_set_fd_lf;
 567         }
 568         return 0;
 569 }
 570
 571 static int
 572 dpaa2_qdma_start(struct rte_rawdev *rawdev)
 573 {
 574         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
 575         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
 576
 577         DPAA2_QDMA_FUNC_TRACE();
 578
 579         qdma_dev->state = 1;
 580
 581         return 0;
 582 }
 583
 584 static int
 585 dpaa2_qdma_queue_setup(struct rte_rawdev *rawdev,
 586                           __rte_unused uint16_t queue_id,
 587                           rte_rawdev_obj_t queue_conf,
 588                           size_t conf_size)
 589 {
 590         char ring_name[32];
 591         int i;
 592         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
 593         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
 594         struct rte_qdma_queue_config *q_config =
 595                 (struct rte_qdma_queue_config *)queue_conf;
 596
 597         DPAA2_QDMA_FUNC_TRACE();
 598
 599         if (conf_size != sizeof(*q_config))
 600                 return -EINVAL;
 601
 602         rte_spinlock_lock(&qdma_dev->lock);
 603
 604         /* Get a free Virtual Queue */
 605         for (i = 0; i < qdma_dev->max_vqs; i++) {
 606                 if (qdma_vqs[i].in_use == 0)
 607                         break;
 608         }
 609
 610         /* Return in case no VQ is free */
 611         if (i == qdma_dev->max_vqs) {
 612                 rte_spinlock_unlock(&qdma_dev->lock);
 613                 DPAA2_QDMA_ERR("Unable to get lock on QDMA device");
 614                 return -ENODEV;
 615         }
 616
 617         if (qdma_dev->mode == RTE_QDMA_MODE_HW ||
 618                         (q_config->flags & RTE_QDMA_VQ_EXCLUSIVE_PQ)) {
 619                 /* Allocate HW queue for a VQ */
 620                 qdma_vqs[i].hw_queue = alloc_hw_queue(q_config->lcore_id);
 621                 qdma_vqs[i].exclusive_hw_queue = 1;
 622         } else {
 623                 /* Allocate a Ring for Virutal Queue in VQ mode */
 624                 snprintf(ring_name, sizeof(ring_name), "status ring %d", i);
 625                 qdma_vqs[i].status_ring = rte_ring_create(ring_name,
 626                         qdma_dev->fle_pool_count, rte_socket_id(), 0);
 627                 if (!qdma_vqs[i].status_ring) {
 628                         DPAA2_QDMA_ERR("Status ring creation failed for vq");
 629                         rte_spinlock_unlock(&qdma_dev->lock);
 630                         return rte_errno;
 631                 }
 632
 633                 /* Get a HW queue (shared) for a VQ */
 634                 qdma_vqs[i].hw_queue = get_hw_queue(qdma_dev,
 635                                                     q_config->lcore_id);
 636                 qdma_vqs[i].exclusive_hw_queue = 0;
 637         }
 638
 639         if (qdma_vqs[i].hw_queue == NULL) {
 640                 DPAA2_QDMA_ERR("No H/W queue available for VQ");
 641                 if (qdma_vqs[i].status_ring)
 642                         rte_ring_free(qdma_vqs[i].status_ring);
 643                 qdma_vqs[i].status_ring = NULL;
 644                 rte_spinlock_unlock(&qdma_dev->lock);
 645                 return -ENODEV;
 646         }
 647
 648         qdma_vqs[i].in_use = 1;
 649         qdma_vqs[i].lcore_id = q_config->lcore_id;
 650         memset(&qdma_vqs[i].rbp, 0, sizeof(struct rte_qdma_rbp));
 651         rte_spinlock_unlock(&qdma_dev->lock);
 652
 653         if (q_config->rbp != NULL)
 654                 memcpy(&qdma_vqs[i].rbp, q_config->rbp,
 655                        sizeof(struct rte_qdma_rbp));
 656
 657         return i;
 658 }
 659
 660 static int
 661 dpdmai_dev_enqueue_multi(struct dpaa2_dpdmai_dev *dpdmai_dev,
 662                         uint16_t txq_id,
 663                         uint16_t vq_id,
 664                         struct rte_qdma_rbp *rbp,
 665                         struct rte_qdma_job **job,
 666                         uint16_t nb_jobs)
 667 {
 668         struct qbman_fd fd[RTE_QDMA_BURST_NB_MAX];
 669         struct dpaa2_queue *txq;
 670         struct qbman_eq_desc eqdesc;
 671         struct qbman_swp *swp;
 672         int ret;
 673         uint32_t num_to_send = 0;
 674         uint16_t num_tx = 0;
 675
 676         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 677                 ret = dpaa2_affine_qbman_swp();
 678                 if (ret) {
 679                         DPAA2_QDMA_ERR(
 680                                 "Failed to allocate IO portal, tid: %d\n",
 681                                 rte_gettid());
 682                         return 0;
 683                 }
 684         }
 685         swp = DPAA2_PER_LCORE_PORTAL;
 686
 687         txq = &(dpdmai_dev->tx_queue[txq_id]);
 688
 689         /* Prepare enqueue descriptor */
 690         qbman_eq_desc_clear(&eqdesc);
 691         qbman_eq_desc_set_fq(&eqdesc, txq->fqid);
 692         qbman_eq_desc_set_no_orp(&eqdesc, 0);
 693         qbman_eq_desc_set_response(&eqdesc, 0, 0);
 694
 695         memset(fd, 0, RTE_QDMA_BURST_NB_MAX * sizeof(struct qbman_fd));
 696
 697         while (nb_jobs > 0) {
 698                 uint32_t loop;
 699
 700                 num_to_send = (nb_jobs > dpaa2_eqcr_size) ?
 701                         dpaa2_eqcr_size : nb_jobs;
 702
 703                 for (loop = 0; loop < num_to_send; loop++) {
 704                         ret = dpdmai_dev_set_fd(dpdmai_dev->qdma_dev, &fd[loop],
 705                                                 job[num_tx], rbp, vq_id);
 706                         if (ret < 0) {
 707                                 /* Set nb_jobs to loop, so outer while loop
 708                                  * breaks out.
 709                                  */
 710                                 nb_jobs = loop;
 711                                 break;
 712                         }
 713
 714                         num_tx++;
 715                 }
 716
 717                 /* Enqueue the packet to the QBMAN */
 718                 uint32_t enqueue_loop = 0, retry_count = 0;
 719                 while (enqueue_loop < loop) {
 720                         ret = qbman_swp_enqueue_multiple(swp,
 721                                                 &eqdesc,
 722                                                 &fd[enqueue_loop],
 723                                                 NULL,
 724                                                 loop - enqueue_loop);
 725                         if (unlikely(ret < 0)) {
 726                                 retry_count++;
 727                                 if (retry_count > DPAA2_MAX_TX_RETRY_COUNT)
 728                                         return num_tx - (loop - enqueue_loop);
 729                         } else {
 730                                 enqueue_loop += ret;
 731                                 retry_count = 0;
 732                         }
 733                 }
 734                 nb_jobs -= loop;
 735         }
 736         return num_tx;
 737 }
 738
 739 static int
 740 dpaa2_qdma_enqueue(__rte_unused struct rte_rawdev *rawdev,
 741                   __rte_unused struct rte_rawdev_buf **buffers,
 742                   unsigned int nb_jobs,
 743                   rte_rawdev_obj_t context)
 744 {
 745         struct rte_qdma_enqdeq *e_context = (struct rte_qdma_enqdeq *)context;
 746         struct qdma_virt_queue *qdma_vq = &qdma_vqs[e_context->vq_id];
 747         struct qdma_hw_queue *qdma_pq = qdma_vq->hw_queue;
 748         struct dpaa2_dpdmai_dev *dpdmai_dev = qdma_pq->dpdmai_dev;
 749         int ret;
 750
 751         /* Return error in case of wrong lcore_id */
 752         if (rte_lcore_id() != qdma_vq->lcore_id) {
 753                 DPAA2_QDMA_ERR("QDMA enqueue for vqid %d on wrong core",
 754                                 e_context->vq_id);
 755                 return -EINVAL;
 756         }
 757
 758         ret = dpdmai_dev_enqueue_multi(dpdmai_dev,
 759                                  qdma_pq->queue_id,
 760                                  e_context->vq_id,
 761                                  &qdma_vq->rbp,
 762                                  e_context->job,
 763                                  nb_jobs);
 764         if (ret < 0) {
 765                 DPAA2_QDMA_ERR("DPDMAI device enqueue failed: %d", ret);
 766                 return ret;
 767         }
 768
 769         qdma_vq->num_enqueues += ret;
 770
 771         return ret;
 772 }
 773
 774 /* Function to receive a QDMA job for a given device and queue*/
 775 static int
 776 dpdmai_dev_dequeue_multijob_prefetch(
 777                         struct dpaa2_dpdmai_dev *dpdmai_dev,
 778                         uint16_t rxq_id,
 779                         uint16_t *vq_id,
 780                         struct rte_qdma_job **job,
 781                         uint16_t nb_jobs)
 782 {
 783         struct dpaa2_queue *rxq;
 784         struct qbman_result *dq_storage, *dq_storage1 = NULL;
 785         struct qbman_pull_desc pulldesc;
 786         struct qbman_swp *swp;
 787         struct queue_storage_info_t *q_storage;
 788         uint32_t fqid;
 789         uint8_t status, pending;
 790         uint8_t num_rx = 0;
 791         const struct qbman_fd *fd;
 792         uint16_t vqid;
 793         int ret, pull_size;
 794
 795         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 796                 ret = dpaa2_affine_qbman_swp();
 797                 if (ret) {
 798                         DPAA2_QDMA_ERR(
 799                                 "Failed to allocate IO portal, tid: %d\n",
 800                                 rte_gettid());
 801                         return 0;
 802                 }
 803         }
 804         swp = DPAA2_PER_LCORE_PORTAL;
 805
 806         pull_size = (nb_jobs > dpaa2_dqrr_size) ? dpaa2_dqrr_size : nb_jobs;
 807         rxq = &(dpdmai_dev->rx_queue[rxq_id]);
 808         fqid = rxq->fqid;
 809         q_storage = rxq->q_storage;
 810
 811         if (unlikely(!q_storage->active_dqs)) {
 812                 q_storage->toggle = 0;
 813                 dq_storage = q_storage->dq_storage[q_storage->toggle];
 814                 q_storage->last_num_pkts = pull_size;
 815                 qbman_pull_desc_clear(&pulldesc);
 816                 qbman_pull_desc_set_numframes(&pulldesc,
 817                                               q_storage->last_num_pkts);
 818                 qbman_pull_desc_set_fq(&pulldesc, fqid);
 819                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
 820                                 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
 821                 if (check_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)) {
 822                         while (!qbman_check_command_complete(
 823                                get_swp_active_dqs(
 824                                DPAA2_PER_LCORE_DPIO->index)))
 825                                 ;
 826                         clear_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index);
 827                 }
 828                 while (1) {
 829                         if (qbman_swp_pull(swp, &pulldesc)) {
 830                                 DPAA2_QDMA_DP_WARN(
 831                                         "VDQ command not issued.QBMAN busy\n");
 832                                         /* Portal was busy, try again */
 833                                 continue;
 834                         }
 835                         break;
 836                 }
 837                 q_storage->active_dqs = dq_storage;
 838                 q_storage->active_dpio_id = DPAA2_PER_LCORE_DPIO->index;
 839                 set_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index,
 840                                    dq_storage);
 841         }
 842
 843         dq_storage = q_storage->active_dqs;
 844         rte_prefetch0((void *)(size_t)(dq_storage));
 845         rte_prefetch0((void *)(size_t)(dq_storage + 1));
 846
 847         /* Prepare next pull descriptor. This will give space for the
 848          * prefething done on DQRR entries
 849          */
 850         q_storage->toggle ^= 1;
 851         dq_storage1 = q_storage->dq_storage[q_storage->toggle];
 852         qbman_pull_desc_clear(&pulldesc);
 853         qbman_pull_desc_set_numframes(&pulldesc, pull_size);
 854         qbman_pull_desc_set_fq(&pulldesc, fqid);
 855         qbman_pull_desc_set_storage(&pulldesc, dq_storage1,
 856                 (size_t)(DPAA2_VADDR_TO_IOVA(dq_storage1)), 1);
 857
 858         /* Check if the previous issued command is completed.
 859          * Also seems like the SWP is shared between the Ethernet Driver
 860          * and the SEC driver.
 861          */
 862         while (!qbman_check_command_complete(dq_storage))
 863                 ;
 864         if (dq_storage == get_swp_active_dqs(q_storage->active_dpio_id))
 865                 clear_swp_active_dqs(q_storage->active_dpio_id);
 866
 867         pending = 1;
 868
 869         do {
 870                 /* Loop until the dq_storage is updated with
 871                  * new token by QBMAN
 872                  */
 873                 while (!qbman_check_new_result(dq_storage))
 874                         ;
 875                 rte_prefetch0((void *)((size_t)(dq_storage + 2)));
 876                 /* Check whether Last Pull command is Expired and
 877                  * setting Condition for Loop termination
 878                  */
 879                 if (qbman_result_DQ_is_pull_complete(dq_storage)) {
 880                         pending = 0;
 881                         /* Check for valid frame. */
 882                         status = qbman_result_DQ_flags(dq_storage);
 883                         if (unlikely((status & QBMAN_DQ_STAT_VALIDFRAME) == 0))
 884                                 continue;
 885                 }
 886                 fd = qbman_result_DQ_fd(dq_storage);
 887
 888                 vqid = dpdmai_dev_get_job(dpdmai_dev->qdma_dev, fd,
 889                                           &job[num_rx]);
 890                 if (vq_id)
 891                         vq_id[num_rx] = vqid;
 892
 893                 dq_storage++;
 894                 num_rx++;
 895         } while (pending);
 896
 897         if (check_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)) {
 898                 while (!qbman_check_command_complete(
 899                        get_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index)))
 900                         ;
 901                 clear_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index);
 902         }
 903         /* issue a volatile dequeue command for next pull */
 904         while (1) {
 905                 if (qbman_swp_pull(swp, &pulldesc)) {
 906                         DPAA2_QDMA_DP_WARN("VDQ command is not issued."
 907                                           "QBMAN is busy (2)\n");
 908                         continue;
 909                 }
 910                 break;
 911         }
 912
 913         q_storage->active_dqs = dq_storage1;
 914         q_storage->active_dpio_id = DPAA2_PER_LCORE_DPIO->index;
 915         set_swp_active_dqs(DPAA2_PER_LCORE_DPIO->index, dq_storage1);
 916
 917         return num_rx;
 918 }
 919
 920 static int
 921 dpdmai_dev_dequeue_multijob_no_prefetch(
 922                 struct dpaa2_dpdmai_dev *dpdmai_dev,
 923                 uint16_t rxq_id,
 924                 uint16_t *vq_id,
 925                 struct rte_qdma_job **job,
 926                 uint16_t nb_jobs)
 927 {
 928         struct dpaa2_queue *rxq;
 929         struct qbman_result *dq_storage;
 930         struct qbman_pull_desc pulldesc;
 931         struct qbman_swp *swp;
 932         uint32_t fqid;
 933         uint8_t status, pending;
 934         uint8_t num_rx = 0;
 935         const struct qbman_fd *fd;
 936         uint16_t vqid;
 937         int ret, next_pull = nb_jobs, num_pulled = 0;
 938
 939         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 940                 ret = dpaa2_affine_qbman_swp();
 941                 if (ret) {
 942                         DPAA2_QDMA_ERR(
 943                                 "Failed to allocate IO portal, tid: %d\n",
 944                                 rte_gettid());
 945                         return 0;
 946                 }
 947         }
 948         swp = DPAA2_PER_LCORE_PORTAL;
 949
 950         rxq = &(dpdmai_dev->rx_queue[rxq_id]);
 951         fqid = rxq->fqid;
 952
 953         do {
 954                 dq_storage = rxq->q_storage->dq_storage[0];
 955                 /* Prepare dequeue descriptor */
 956                 qbman_pull_desc_clear(&pulldesc);
 957                 qbman_pull_desc_set_fq(&pulldesc, fqid);
 958                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
 959                         (uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
 960
 961                 if (next_pull > dpaa2_dqrr_size) {
 962                         qbman_pull_desc_set_numframes(&pulldesc,
 963                                         dpaa2_dqrr_size);
 964                         next_pull -= dpaa2_dqrr_size;
 965                 } else {
 966                         qbman_pull_desc_set_numframes(&pulldesc, next_pull);
 967                         next_pull = 0;
 968                 }
 969
 970                 while (1) {
 971                         if (qbman_swp_pull(swp, &pulldesc)) {
 972                                 DPAA2_QDMA_DP_WARN("VDQ command not issued. QBMAN busy");
 973                                 /* Portal was busy, try again */
 974                                 continue;
 975                         }
 976                         break;
 977                 }
 978
 979                 rte_prefetch0((void *)((size_t)(dq_storage + 1)));
 980                 /* Check if the previous issued command is completed. */
 981                 while (!qbman_check_command_complete(dq_storage))
 982                         ;
 983
 984                 num_pulled = 0;
 985                 pending = 1;
 986
 987                 do {
 988                         /* Loop until dq_storage is updated
 989                          * with new token by QBMAN
 990                          */
 991                         while (!qbman_check_new_result(dq_storage))
 992                                 ;
 993                         rte_prefetch0((void *)((size_t)(dq_storage + 2)));
 994
 995                         if (qbman_result_DQ_is_pull_complete(dq_storage)) {
 996                                 pending = 0;
 997                                 /* Check for valid frame. */
 998                                 status = qbman_result_DQ_flags(dq_storage);
 999                                 if (unlikely((status &
1000                                         QBMAN_DQ_STAT_VALIDFRAME) == 0))
1001                                         continue;
1002                         }
1003                         fd = qbman_result_DQ_fd(dq_storage);
1004
1005                         vqid = dpdmai_dev_get_job(dpdmai_dev->qdma_dev, fd,
1006                                                   &job[num_rx]);
1007                         if (vq_id)
1008                                 vq_id[num_rx] = vqid;
1009
1010                         dq_storage++;
1011                         num_rx++;
1012                         num_pulled++;
1013
1014                 } while (pending);
1015         /* Last VDQ provided all packets and more packets are requested */
1016         } while (next_pull && num_pulled == dpaa2_dqrr_size);
1017
1018         return num_rx;
1019 }
1020
1021 static int
1022 dpaa2_qdma_dequeue(__rte_unused struct rte_rawdev *rawdev,
1023                    __rte_unused struct rte_rawdev_buf **buffers,
1024                    unsigned int nb_jobs,
1025                    rte_rawdev_obj_t cntxt)
1026 {
1027         struct rte_qdma_enqdeq *context = (struct rte_qdma_enqdeq *)cntxt;
1028         struct qdma_virt_queue *qdma_vq = &qdma_vqs[context->vq_id];
1029         struct qdma_hw_queue *qdma_pq = qdma_vq->hw_queue;
1030         struct qdma_virt_queue *temp_qdma_vq;
1031         struct dpaa2_dpdmai_dev *dpdmai_dev = qdma_pq->dpdmai_dev;
1032         int ret = 0, i;
1033         unsigned int ring_count;
1034
1035         /* Return error in case of wrong lcore_id */
1036         if (rte_lcore_id() != (unsigned int)(qdma_vq->lcore_id)) {
1037                 DPAA2_QDMA_WARN("QDMA dequeue for vqid %d on wrong core",
1038                                 context->vq_id);
1039                 return -1;
1040         }
1041
1042         /* Only dequeue when there are pending jobs on VQ */
1043         if (qdma_vq->num_enqueues == qdma_vq->num_dequeues)
1044                 return 0;
1045
1046         if (qdma_vq->num_enqueues < (qdma_vq->num_dequeues + nb_jobs))
1047                 nb_jobs = (qdma_vq->num_enqueues -  qdma_vq->num_dequeues);
1048
1049         if (qdma_vq->exclusive_hw_queue) {
1050                 /* In case of exclusive queue directly fetch from HW queue */
1051                 ret = dpdmai_dev_dequeue_multijob(dpdmai_dev, qdma_pq->queue_id,
1052                                          NULL, context->job, nb_jobs);
1053                 if (ret < 0) {
1054                         DPAA2_QDMA_ERR(
1055                                 "Dequeue from DPDMAI device failed: %d", ret);
1056                         return ret;
1057                 }
1058                 qdma_vq->num_dequeues += ret;
1059         } else {
1060                 uint16_t temp_vq_id[RTE_QDMA_BURST_NB_MAX];
1061                 /*
1062                  * Get the QDMA completed jobs from the software ring.
1063                  * In case they are not available on the ring poke the HW
1064                  * to fetch completed jobs from corresponding HW queues
1065                  */
1066                 ring_count = rte_ring_count(qdma_vq->status_ring);
1067                 if (ring_count < nb_jobs) {
1068                         /* TODO - How to have right budget */
1069                         ret = dpdmai_dev_dequeue_multijob(dpdmai_dev,
1070                                         qdma_pq->queue_id,
1071                                         temp_vq_id, context->job, nb_jobs);
1072                         for (i = 0; i < ret; i++) {
1073                                 temp_qdma_vq = &qdma_vqs[temp_vq_id[i]];
1074                                 rte_ring_enqueue(temp_qdma_vq->status_ring,
1075                                         (void *)(context->job[i]));
1076                         }
1077                         ring_count = rte_ring_count(
1078                                         qdma_vq->status_ring);
1079                 }
1080
1081                 if (ring_count) {
1082                         /* Dequeue job from the software ring
1083                          * to provide to the user
1084                          */
1085                         ret = rte_ring_dequeue_bulk(qdma_vq->status_ring,
1086                                                     (void **)context->job,
1087                                                     ring_count, NULL);
1088                         if (ret)
1089                                 qdma_vq->num_dequeues += ret;
1090                 }
1091         }
1092
1093         return ret;
1094 }
1095
1096 void
1097 rte_qdma_vq_stats(uint16_t vq_id,
1098                   struct rte_qdma_vq_stats *vq_status)
1099 {
1100         struct qdma_virt_queue *qdma_vq = &qdma_vqs[vq_id];
1101
1102         if (qdma_vq->in_use) {
1103                 vq_status->exclusive_hw_queue = qdma_vq->exclusive_hw_queue;
1104                 vq_status->lcore_id = qdma_vq->lcore_id;
1105                 vq_status->num_enqueues = qdma_vq->num_enqueues;
1106                 vq_status->num_dequeues = qdma_vq->num_dequeues;
1107                 vq_status->num_pending_jobs = vq_status->num_enqueues -
1108                                 vq_status->num_dequeues;
1109         }
1110 }
1111
1112 static int
1113 dpaa2_qdma_queue_release(struct rte_rawdev *rawdev,
1114                          uint16_t vq_id)
1115 {
1116         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1117         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1118
1119         struct qdma_virt_queue *qdma_vq = &qdma_vqs[vq_id];
1120
1121         DPAA2_QDMA_FUNC_TRACE();
1122
1123         /* In case there are pending jobs on any VQ, return -EBUSY */
1124         if (qdma_vq->num_enqueues != qdma_vq->num_dequeues)
1125                 return -EBUSY;
1126
1127         rte_spinlock_lock(&qdma_dev->lock);
1128
1129         if (qdma_vq->exclusive_hw_queue)
1130                 free_hw_queue(qdma_vq->hw_queue);
1131         else {
1132                 if (qdma_vqs->status_ring)
1133                         rte_ring_free(qdma_vqs->status_ring);
1134
1135                 put_hw_queue(qdma_vq->hw_queue);
1136         }
1137
1138         memset(qdma_vq, 0, sizeof(struct qdma_virt_queue));
1139
1140         rte_spinlock_unlock(&qdma_dev->lock);
1141
1142         return 0;
1143 }
1144
1145 static void
1146 dpaa2_qdma_stop(struct rte_rawdev *rawdev)
1147 {
1148         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1149         struct qdma_device *qdma_dev = dpdmai_dev->qdma_dev;
1150
1151         DPAA2_QDMA_FUNC_TRACE();
1152
1153         qdma_dev->state = 0;
1154 }
1155
1156 static int
1157 dpaa2_qdma_close(struct rte_rawdev *rawdev)
1158 {
1159         DPAA2_QDMA_FUNC_TRACE();
1160
1161         dpaa2_qdma_reset(rawdev);
1162
1163         return 0;
1164 }
1165
1166 static struct rte_rawdev_ops dpaa2_qdma_ops = {
1167         .dev_configure            = dpaa2_qdma_configure,
1168         .dev_start                = dpaa2_qdma_start,
1169         .dev_stop                 = dpaa2_qdma_stop,
1170         .dev_reset                = dpaa2_qdma_reset,
1171         .dev_close                = dpaa2_qdma_close,
1172         .queue_setup              = dpaa2_qdma_queue_setup,
1173         .queue_release            = dpaa2_qdma_queue_release,
1174         .attr_get                 = dpaa2_qdma_attr_get,
1175         .enqueue_bufs             = dpaa2_qdma_enqueue,
1176         .dequeue_bufs             = dpaa2_qdma_dequeue,
1177 };
1178
1179 static int
1180 add_hw_queues_to_list(struct dpaa2_dpdmai_dev *dpdmai_dev)
1181 {
1182         struct qdma_hw_queue *queue;
1183         int i;
1184
1185         DPAA2_QDMA_FUNC_TRACE();
1186
1187         for (i = 0; i < dpdmai_dev->num_queues; i++) {
1188                 queue = rte_zmalloc(NULL, sizeof(struct qdma_hw_queue), 0);
1189                 if (!queue) {
1190                         DPAA2_QDMA_ERR(
1191                                 "Memory allocation failed for QDMA queue");
1192                         return -ENOMEM;
1193                 }
1194
1195                 queue->dpdmai_dev = dpdmai_dev;
1196                 queue->queue_id = i;
1197
1198                 TAILQ_INSERT_TAIL(&qdma_queue_list, queue, next);
1199                 dpdmai_dev->qdma_dev->num_hw_queues++;
1200         }
1201
1202         return 0;
1203 }
1204
1205 static void
1206 remove_hw_queues_from_list(struct dpaa2_dpdmai_dev *dpdmai_dev)
1207 {
1208         struct qdma_hw_queue *queue = NULL;
1209         struct qdma_hw_queue *tqueue = NULL;
1210
1211         DPAA2_QDMA_FUNC_TRACE();
1212
1213         TAILQ_FOREACH_SAFE(queue, &qdma_queue_list, next, tqueue) {
1214                 if (queue->dpdmai_dev == dpdmai_dev) {
1215                         TAILQ_REMOVE(&qdma_queue_list, queue, next);
1216                         rte_free(queue);
1217                         queue = NULL;
1218                 }
1219         }
1220 }
1221
1222 static int
1223 dpaa2_dpdmai_dev_uninit(struct rte_rawdev *rawdev)
1224 {
1225         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1226         int ret, i;
1227
1228         DPAA2_QDMA_FUNC_TRACE();
1229
1230         /* Remove HW queues from global list */
1231         remove_hw_queues_from_list(dpdmai_dev);
1232
1233         ret = dpdmai_disable(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1234                              dpdmai_dev->token);
1235         if (ret)
1236                 DPAA2_QDMA_ERR("dmdmai disable failed");
1237
1238         /* Set up the DQRR storage for Rx */
1239         for (i = 0; i < dpdmai_dev->num_queues; i++) {
1240                 struct dpaa2_queue *rxq = &(dpdmai_dev->rx_queue[i]);
1241
1242                 if (rxq->q_storage) {
1243                         dpaa2_free_dq_storage(rxq->q_storage);
1244                         rte_free(rxq->q_storage);
1245                 }
1246         }
1247
1248         /* Close the device at underlying layer*/
1249         ret = dpdmai_close(&dpdmai_dev->dpdmai, CMD_PRI_LOW, dpdmai_dev->token);
1250         if (ret)
1251                 DPAA2_QDMA_ERR("Failure closing dpdmai device");
1252
1253         return 0;
1254 }
1255
1256 static int
1257 check_devargs_handler(__rte_unused const char *key, const char *value,
1258                       __rte_unused void *opaque)
1259 {
1260         if (strcmp(value, "1"))
1261                 return -1;
1262
1263         return 0;
1264 }
1265
1266 static int
1267 dpaa2_get_devargs(struct rte_devargs *devargs, const char *key)
1268 {
1269         struct rte_kvargs *kvlist;
1270
1271         if (!devargs)
1272                 return 0;
1273
1274         kvlist = rte_kvargs_parse(devargs->args, NULL);
1275         if (!kvlist)
1276                 return 0;
1277
1278         if (!rte_kvargs_count(kvlist, key)) {
1279                 rte_kvargs_free(kvlist);
1280                 return 0;
1281         }
1282
1283         if (rte_kvargs_process(kvlist, key,
1284                                check_devargs_handler, NULL) < 0) {
1285                 rte_kvargs_free(kvlist);
1286                 return 0;
1287         }
1288         rte_kvargs_free(kvlist);
1289
1290         return 1;
1291 }
1292
1293 static int
1294 dpaa2_dpdmai_dev_init(struct rte_rawdev *rawdev, int dpdmai_id)
1295 {
1296         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
1297         struct dpdmai_rx_queue_cfg rx_queue_cfg;
1298         struct dpdmai_attr attr;
1299         struct dpdmai_rx_queue_attr rx_attr;
1300         struct dpdmai_tx_queue_attr tx_attr;
1301         int ret, i;
1302
1303         DPAA2_QDMA_FUNC_TRACE();
1304
1305         /* Open DPDMAI device */
1306         dpdmai_dev->dpdmai_id = dpdmai_id;
1307         dpdmai_dev->dpdmai.regs = dpaa2_get_mcp_ptr(MC_PORTAL_INDEX);
1308         dpdmai_dev->qdma_dev = &q_dev;
1309         ret = dpdmai_open(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1310                           dpdmai_dev->dpdmai_id, &dpdmai_dev->token);
1311         if (ret) {
1312                 DPAA2_QDMA_ERR("dpdmai_open() failed with err: %d", ret);
1313                 return ret;
1314         }
1315
1316         /* Get DPDMAI attributes */
1317         ret = dpdmai_get_attributes(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1318                                     dpdmai_dev->token, &attr);
1319         if (ret) {
1320                 DPAA2_QDMA_ERR("dpdmai get attributes failed with err: %d",
1321                                ret);
1322                 goto init_err;
1323         }
1324         dpdmai_dev->num_queues = attr.num_of_queues;
1325
1326         /* Set up Rx Queues */
1327         for (i = 0; i < dpdmai_dev->num_queues; i++) {
1328                 struct dpaa2_queue *rxq;
1329
1330                 memset(&rx_queue_cfg, 0, sizeof(struct dpdmai_rx_queue_cfg));
1331                 ret = dpdmai_set_rx_queue(&dpdmai_dev->dpdmai,
1332                                           CMD_PRI_LOW,
1333                                           dpdmai_dev->token,
1334                                           i, 0, &rx_queue_cfg);
1335                 if (ret) {
1336                         DPAA2_QDMA_ERR("Setting Rx queue failed with err: %d",
1337                                        ret);
1338                         goto init_err;
1339                 }
1340
1341                 /* Allocate DQ storage for the DPDMAI Rx queues */
1342                 rxq = &(dpdmai_dev->rx_queue[i]);
1343                 rxq->q_storage = rte_malloc("dq_storage",
1344                                             sizeof(struct queue_storage_info_t),
1345                                             RTE_CACHE_LINE_SIZE);
1346                 if (!rxq->q_storage) {
1347                         DPAA2_QDMA_ERR("q_storage allocation failed");
1348                         ret = -ENOMEM;
1349                         goto init_err;
1350                 }
1351
1352                 memset(rxq->q_storage, 0, sizeof(struct queue_storage_info_t));
1353                 ret = dpaa2_alloc_dq_storage(rxq->q_storage);
1354                 if (ret) {
1355                         DPAA2_QDMA_ERR("dpaa2_alloc_dq_storage failed");
1356                         goto init_err;
1357                 }
1358         }
1359
1360         /* Get Rx and Tx queues FQID's */
1361         for (i = 0; i < dpdmai_dev->num_queues; i++) {
1362                 ret = dpdmai_get_rx_queue(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1363                                           dpdmai_dev->token, i, 0, &rx_attr);
1364                 if (ret) {
1365                         DPAA2_QDMA_ERR("Reading device failed with err: %d",
1366                                        ret);
1367                         goto init_err;
1368                 }
1369                 dpdmai_dev->rx_queue[i].fqid = rx_attr.fqid;
1370
1371                 ret = dpdmai_get_tx_queue(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1372                                           dpdmai_dev->token, i, 0, &tx_attr);
1373                 if (ret) {
1374                         DPAA2_QDMA_ERR("Reading device failed with err: %d",
1375                                        ret);
1376                         goto init_err;
1377                 }
1378                 dpdmai_dev->tx_queue[i].fqid = tx_attr.fqid;
1379         }
1380
1381         /* Enable the device */
1382         ret = dpdmai_enable(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1383                             dpdmai_dev->token);
1384         if (ret) {
1385                 DPAA2_QDMA_ERR("Enabling device failed with err: %d", ret);
1386                 goto init_err;
1387         }
1388
1389         /* Add the HW queue to the global list */
1390         ret = add_hw_queues_to_list(dpdmai_dev);
1391         if (ret) {
1392                 DPAA2_QDMA_ERR("Adding H/W queue to list failed");
1393                 goto init_err;
1394         }
1395
1396         if (dpaa2_get_devargs(rawdev->device->devargs,
1397                 DPAA2_QDMA_NO_PREFETCH)) {
1398                 /* If no prefetch is configured. */
1399                 dpdmai_dev_dequeue_multijob =
1400                                 dpdmai_dev_dequeue_multijob_no_prefetch;
1401                 DPAA2_QDMA_INFO("No Prefetch RX Mode enabled");
1402         } else {
1403                 dpdmai_dev_dequeue_multijob =
1404                         dpdmai_dev_dequeue_multijob_prefetch;
1405         }
1406
1407         if (!dpaa2_coherent_no_alloc_cache) {
1408                 if (dpaa2_svr_family == SVR_LX2160A) {
1409                         dpaa2_coherent_no_alloc_cache =
1410                                 DPAA2_LX2_COHERENT_NO_ALLOCATE_CACHE;
1411                         dpaa2_coherent_alloc_cache =
1412                                 DPAA2_LX2_COHERENT_ALLOCATE_CACHE;
1413                 } else {
1414                         dpaa2_coherent_no_alloc_cache =
1415                                 DPAA2_COHERENT_NO_ALLOCATE_CACHE;
1416                         dpaa2_coherent_alloc_cache =
1417                                 DPAA2_COHERENT_ALLOCATE_CACHE;
1418                 }
1419         }
1420
1421         DPAA2_QDMA_DEBUG("Initialized dpdmai object successfully");
1422
1423         rte_spinlock_init(&dpdmai_dev->qdma_dev->lock);
1424
1425         return 0;
1426 init_err:
1427         dpaa2_dpdmai_dev_uninit(rawdev);
1428         return ret;
1429 }
1430
1431 static int
1432 rte_dpaa2_qdma_probe(struct rte_dpaa2_driver *dpaa2_drv,
1433                      struct rte_dpaa2_device *dpaa2_dev)
1434 {
1435         struct rte_rawdev *rawdev;
1436         int ret;
1437
1438         DPAA2_QDMA_FUNC_TRACE();
1439
1440         rawdev = rte_rawdev_pmd_allocate(dpaa2_dev->device.name,
1441                         sizeof(struct dpaa2_dpdmai_dev),
1442                         rte_socket_id());
1443         if (!rawdev) {
1444                 DPAA2_QDMA_ERR("Unable to allocate rawdevice");
1445                 return -EINVAL;
1446         }
1447
1448         dpaa2_dev->rawdev = rawdev;
1449         rawdev->dev_ops = &dpaa2_qdma_ops;
1450         rawdev->device = &dpaa2_dev->device;
1451         rawdev->driver_name = dpaa2_drv->driver.name;
1452
1453         /* Invoke PMD device initialization function */
1454         ret = dpaa2_dpdmai_dev_init(rawdev, dpaa2_dev->object_id);
1455         if (ret) {
1456                 rte_rawdev_pmd_release(rawdev);
1457                 return ret;
1458         }
1459
1460         /* Reset the QDMA device */
1461         ret = dpaa2_qdma_reset(rawdev);
1462         if (ret) {
1463                 DPAA2_QDMA_ERR("Resetting QDMA failed");
1464                 return ret;
1465         }
1466
1467         return 0;
1468 }
1469
1470 static int
1471 rte_dpaa2_qdma_remove(struct rte_dpaa2_device *dpaa2_dev)
1472 {
1473         struct rte_rawdev *rawdev = dpaa2_dev->rawdev;
1474         int ret;
1475
1476         DPAA2_QDMA_FUNC_TRACE();
1477
1478         dpaa2_dpdmai_dev_uninit(rawdev);
1479
1480         ret = rte_rawdev_pmd_release(rawdev);
1481         if (ret)
1482                 DPAA2_QDMA_ERR("Device cleanup failed");
1483
1484         return 0;
1485 }
1486
1487 static struct rte_dpaa2_driver rte_dpaa2_qdma_pmd = {
1488         .drv_flags = RTE_DPAA2_DRV_IOVA_AS_VA,
1489         .drv_type = DPAA2_QDMA,
1490         .probe = rte_dpaa2_qdma_probe,
1491         .remove = rte_dpaa2_qdma_remove,
1492 };
1493
1494 RTE_PMD_REGISTER_DPAA2(dpaa2_qdma, rte_dpaa2_qdma_pmd);
1495 RTE_PMD_REGISTER_PARAM_STRING(dpaa2_qdma,
1496         "no_prefetch=<int> ");
1497 RTE_LOG_REGISTER(dpaa2_qdma_logtype, pmd.raw.dpaa2.qdma, INFO);