drivers/raw/dpaa2_qdma/dpaa2_qdma.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2018-2019 NXP
   3  */
   4
   5 #include <string.h>
   6
   7 #include <rte_eal.h>
   8 #include <rte_fslmc.h>
   9 #include <rte_atomic.h>
  10 #include <rte_lcore.h>
  11 #include <rte_rawdev.h>
  12 #include <rte_rawdev_pmd.h>
  13 #include <rte_malloc.h>
  14 #include <rte_ring.h>
  15 #include <rte_mempool.h>
  16 #include <rte_prefetch.h>
  17
  18 #include <mc/fsl_dpdmai.h>
  19 #include <portal/dpaa2_hw_pvt.h>
  20 #include <portal/dpaa2_hw_dpio.h>
  21
  22 #include "rte_pmd_dpaa2_qdma.h"
  23 #include "dpaa2_qdma.h"
  24 #include "dpaa2_qdma_logs.h"
  25
  26 /* Dynamic log type identifier */
  27 int dpaa2_qdma_logtype;
  28
  29 uint32_t dpaa2_coherent_no_alloc_cache;
  30 uint32_t dpaa2_coherent_alloc_cache;
  31
  32 /* QDMA device */
  33 static struct qdma_device qdma_dev;
  34
  35 /* QDMA H/W queues list */
  36 TAILQ_HEAD(qdma_hw_queue_list, qdma_hw_queue);
  37 static struct qdma_hw_queue_list qdma_queue_list
  38         = TAILQ_HEAD_INITIALIZER(qdma_queue_list);
  39
  40 /* QDMA Virtual Queues */
  41 static struct qdma_virt_queue *qdma_vqs;
  42
  43 /* QDMA per core data */
  44 static struct qdma_per_core_info qdma_core_info[RTE_MAX_LCORE];
  45
  46 static struct qdma_hw_queue *
  47 alloc_hw_queue(uint32_t lcore_id)
  48 {
  49         struct qdma_hw_queue *queue = NULL;
  50
  51         DPAA2_QDMA_FUNC_TRACE();
  52
  53         /* Get a free queue from the list */
  54         TAILQ_FOREACH(queue, &qdma_queue_list, next) {
  55                 if (queue->num_users == 0) {
  56                         queue->lcore_id = lcore_id;
  57                         queue->num_users++;
  58                         break;
  59                 }
  60         }
  61
  62         return queue;
  63 }
  64
  65 static void
  66 free_hw_queue(struct qdma_hw_queue *queue)
  67 {
  68         DPAA2_QDMA_FUNC_TRACE();
  69
  70         queue->num_users--;
  71 }
  72
  73
  74 static struct qdma_hw_queue *
  75 get_hw_queue(uint32_t lcore_id)
  76 {
  77         struct qdma_per_core_info *core_info;
  78         struct qdma_hw_queue *queue, *temp;
  79         uint32_t least_num_users;
  80         int num_hw_queues, i;
  81
  82         DPAA2_QDMA_FUNC_TRACE();
  83
  84         core_info = &qdma_core_info[lcore_id];
  85         num_hw_queues = core_info->num_hw_queues;
  86
  87         /*
  88          * Allocate a HW queue if there are less queues
  89          * than maximum per core queues configured
  90          */
  91         if (num_hw_queues < qdma_dev.max_hw_queues_per_core) {
  92                 queue = alloc_hw_queue(lcore_id);
  93                 if (queue) {
  94                         core_info->hw_queues[num_hw_queues] = queue;
  95                         core_info->num_hw_queues++;
  96                         return queue;
  97                 }
  98         }
  99
 100         queue = core_info->hw_queues[0];
 101         /* In case there is no queue associated with the core return NULL */
 102         if (!queue)
 103                 return NULL;
 104
 105         /* Fetch the least loaded H/W queue */
 106         least_num_users = core_info->hw_queues[0]->num_users;
 107         for (i = 0; i < num_hw_queues; i++) {
 108                 temp = core_info->hw_queues[i];
 109                 if (temp->num_users < least_num_users)
 110                         queue = temp;
 111         }
 112
 113         if (queue)
 114                 queue->num_users++;
 115
 116         return queue;
 117 }
 118
 119 static void
 120 put_hw_queue(struct qdma_hw_queue *queue)
 121 {
 122         struct qdma_per_core_info *core_info;
 123         int lcore_id, num_hw_queues, i;
 124
 125         DPAA2_QDMA_FUNC_TRACE();
 126
 127         /*
 128          * If this is the last user of the queue free it.
 129          * Also remove it from QDMA core info.
 130          */
 131         if (queue->num_users == 1) {
 132                 free_hw_queue(queue);
 133
 134                 /* Remove the physical queue from core info */
 135                 lcore_id = queue->lcore_id;
 136                 core_info = &qdma_core_info[lcore_id];
 137                 num_hw_queues = core_info->num_hw_queues;
 138                 for (i = 0; i < num_hw_queues; i++) {
 139                         if (queue == core_info->hw_queues[i])
 140                                 break;
 141                 }
 142                 for (; i < num_hw_queues - 1; i++)
 143                         core_info->hw_queues[i] = core_info->hw_queues[i + 1];
 144                 core_info->hw_queues[i] = NULL;
 145         } else {
 146                 queue->num_users--;
 147         }
 148 }
 149
 150 int
 151 rte_qdma_init(void)
 152 {
 153         DPAA2_QDMA_FUNC_TRACE();
 154
 155         rte_spinlock_init(&qdma_dev.lock);
 156
 157         return 0;
 158 }
 159
 160 void
 161 rte_qdma_attr_get(struct rte_qdma_attr *qdma_attr)
 162 {
 163         DPAA2_QDMA_FUNC_TRACE();
 164
 165         qdma_attr->num_hw_queues = qdma_dev.num_hw_queues;
 166 }
 167
 168 int
 169 rte_qdma_reset(void)
 170 {
 171         struct qdma_hw_queue *queue;
 172         int i;
 173
 174         DPAA2_QDMA_FUNC_TRACE();
 175
 176         /* In case QDMA device is not in stopped state, return -EBUSY */
 177         if (qdma_dev.state == 1) {
 178                 DPAA2_QDMA_ERR(
 179                         "Device is in running state. Stop before reset.");
 180                 return -EBUSY;
 181         }
 182
 183         /* In case there are pending jobs on any VQ, return -EBUSY */
 184         for (i = 0; i < qdma_dev.max_vqs; i++) {
 185                 if (qdma_vqs[i].in_use && (qdma_vqs[i].num_enqueues !=
 186                     qdma_vqs[i].num_dequeues))
 187                         DPAA2_QDMA_ERR("Jobs are still pending on VQ: %d", i);
 188                         return -EBUSY;
 189         }
 190
 191         /* Reset HW queues */
 192         TAILQ_FOREACH(queue, &qdma_queue_list, next)
 193                 queue->num_users = 0;
 194
 195         /* Reset and free virtual queues */
 196         for (i = 0; i < qdma_dev.max_vqs; i++) {
 197                 if (qdma_vqs[i].status_ring)
 198                         rte_ring_free(qdma_vqs[i].status_ring);
 199         }
 200         if (qdma_vqs)
 201                 rte_free(qdma_vqs);
 202         qdma_vqs = NULL;
 203
 204         /* Reset per core info */
 205         memset(&qdma_core_info, 0,
 206                 sizeof(struct qdma_per_core_info) * RTE_MAX_LCORE);
 207
 208         /* Free the FLE pool */
 209         if (qdma_dev.fle_pool)
 210                 rte_mempool_free(qdma_dev.fle_pool);
 211
 212         /* Reset QDMA device structure */
 213         qdma_dev.mode = RTE_QDMA_MODE_HW;
 214         qdma_dev.max_hw_queues_per_core = 0;
 215         qdma_dev.fle_pool = NULL;
 216         qdma_dev.fle_pool_count = 0;
 217         qdma_dev.max_vqs = 0;
 218
 219         return 0;
 220 }
 221
 222 int
 223 rte_qdma_configure(struct rte_qdma_config *qdma_config)
 224 {
 225         int ret;
 226         char fle_pool_name[32]; /* RTE_MEMZONE_NAMESIZE = 32 */
 227
 228         DPAA2_QDMA_FUNC_TRACE();
 229
 230         /* In case QDMA device is not in stopped state, return -EBUSY */
 231         if (qdma_dev.state == 1) {
 232                 DPAA2_QDMA_ERR(
 233                         "Device is in running state. Stop before config.");
 234                 return -1;
 235         }
 236
 237         /* Reset the QDMA device */
 238         ret = rte_qdma_reset();
 239         if (ret) {
 240                 DPAA2_QDMA_ERR("Resetting QDMA failed");
 241                 return ret;
 242         }
 243
 244         /* Set mode */
 245         qdma_dev.mode = qdma_config->mode;
 246
 247         /* Set max HW queue per core */
 248         if (qdma_config->max_hw_queues_per_core > MAX_HW_QUEUE_PER_CORE) {
 249                 DPAA2_QDMA_ERR("H/W queues per core is more than: %d",
 250                                MAX_HW_QUEUE_PER_CORE);
 251                 return -EINVAL;
 252         }
 253         qdma_dev.max_hw_queues_per_core =
 254                 qdma_config->max_hw_queues_per_core;
 255
 256         /* Allocate Virtual Queues */
 257         qdma_vqs = rte_malloc("qdma_virtual_queues",
 258                         (sizeof(struct qdma_virt_queue) * qdma_config->max_vqs),
 259                         RTE_CACHE_LINE_SIZE);
 260         if (!qdma_vqs) {
 261                 DPAA2_QDMA_ERR("qdma_virtual_queues allocation failed");
 262                 return -ENOMEM;
 263         }
 264         qdma_dev.max_vqs = qdma_config->max_vqs;
 265
 266         /* Allocate FLE pool; just append PID so that in case of
 267          * multiprocess, the pool's don't collide.
 268          */
 269         snprintf(fle_pool_name, sizeof(fle_pool_name), "qdma_fle_pool%u",
 270                  getpid());
 271         qdma_dev.fle_pool = rte_mempool_create(fle_pool_name,
 272                         qdma_config->fle_pool_count, QDMA_FLE_POOL_SIZE,
 273                         QDMA_FLE_CACHE_SIZE(qdma_config->fle_pool_count), 0,
 274                         NULL, NULL, NULL, NULL, SOCKET_ID_ANY, 0);
 275         if (!qdma_dev.fle_pool) {
 276                 DPAA2_QDMA_ERR("qdma_fle_pool create failed");
 277                 rte_free(qdma_vqs);
 278                 qdma_vqs = NULL;
 279                 return -ENOMEM;
 280         }
 281         qdma_dev.fle_pool_count = qdma_config->fle_pool_count;
 282
 283         return 0;
 284 }
 285
 286 int
 287 rte_qdma_start(void)
 288 {
 289         DPAA2_QDMA_FUNC_TRACE();
 290
 291         qdma_dev.state = 1;
 292
 293         return 0;
 294 }
 295
 296 int
 297 rte_qdma_vq_create(uint32_t lcore_id, uint32_t flags)
 298 {
 299         char ring_name[32];
 300         int i;
 301
 302         DPAA2_QDMA_FUNC_TRACE();
 303
 304         rte_spinlock_lock(&qdma_dev.lock);
 305
 306         /* Get a free Virtual Queue */
 307         for (i = 0; i < qdma_dev.max_vqs; i++) {
 308                 if (qdma_vqs[i].in_use == 0)
 309                         break;
 310         }
 311
 312         /* Return in case no VQ is free */
 313         if (i == qdma_dev.max_vqs) {
 314                 rte_spinlock_unlock(&qdma_dev.lock);
 315                 DPAA2_QDMA_ERR("Unable to get lock on QDMA device");
 316                 return -ENODEV;
 317         }
 318
 319         if (qdma_dev.mode == RTE_QDMA_MODE_HW ||
 320                         (flags & RTE_QDMA_VQ_EXCLUSIVE_PQ)) {
 321                 /* Allocate HW queue for a VQ */
 322                 qdma_vqs[i].hw_queue = alloc_hw_queue(lcore_id);
 323                 qdma_vqs[i].exclusive_hw_queue = 1;
 324         } else {
 325                 /* Allocate a Ring for Virutal Queue in VQ mode */
 326                 snprintf(ring_name, sizeof(ring_name), "status ring %d", i);
 327                 qdma_vqs[i].status_ring = rte_ring_create(ring_name,
 328                         qdma_dev.fle_pool_count, rte_socket_id(), 0);
 329                 if (!qdma_vqs[i].status_ring) {
 330                         DPAA2_QDMA_ERR("Status ring creation failed for vq");
 331                         rte_spinlock_unlock(&qdma_dev.lock);
 332                         return rte_errno;
 333                 }
 334
 335                 /* Get a HW queue (shared) for a VQ */
 336                 qdma_vqs[i].hw_queue = get_hw_queue(lcore_id);
 337                 qdma_vqs[i].exclusive_hw_queue = 0;
 338         }
 339
 340         if (qdma_vqs[i].hw_queue == NULL) {
 341                 DPAA2_QDMA_ERR("No H/W queue available for VQ");
 342                 if (qdma_vqs[i].status_ring)
 343                         rte_ring_free(qdma_vqs[i].status_ring);
 344                 qdma_vqs[i].status_ring = NULL;
 345                 rte_spinlock_unlock(&qdma_dev.lock);
 346                 return -ENODEV;
 347         }
 348
 349         qdma_vqs[i].in_use = 1;
 350         qdma_vqs[i].lcore_id = lcore_id;
 351         memset(&qdma_vqs[i].rbp, 0, sizeof(struct rte_qdma_rbp));
 352         rte_spinlock_unlock(&qdma_dev.lock);
 353
 354         return i;
 355 }
 356
 357 /*create vq for route-by-port*/
 358 int
 359 rte_qdma_vq_create_rbp(uint32_t lcore_id, uint32_t flags,
 360                         struct rte_qdma_rbp *rbp)
 361 {
 362         int i;
 363
 364         i = rte_qdma_vq_create(lcore_id, flags);
 365
 366         memcpy(&qdma_vqs[i].rbp, rbp, sizeof(struct rte_qdma_rbp));
 367
 368         return i;
 369 }
 370
 371 static void
 372 dpaa2_qdma_populate_fle(struct qbman_fle *fle,
 373                         struct rte_qdma_rbp *rbp,
 374                         uint64_t src, uint64_t dest,
 375                         size_t len, uint32_t flags)
 376 {
 377         struct qdma_sdd *sdd;
 378
 379         DPAA2_QDMA_FUNC_TRACE();
 380
 381         sdd = (struct qdma_sdd *)((uint8_t *)(fle) +
 382                 (DPAA2_QDMA_MAX_FLE * sizeof(struct qbman_fle)));
 383
 384         /* first frame list to source descriptor */
 385         DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(sdd));
 386         DPAA2_SET_FLE_LEN(fle, (2 * (sizeof(struct qdma_sdd))));
 387
 388         /* source and destination descriptor */
 389         if (rbp && rbp->enable) {
 390                 /* source */
 391                 sdd->read_cmd.portid = rbp->sportid;
 392                 sdd->rbpcmd_simple.pfid = rbp->spfid;
 393                 sdd->rbpcmd_simple.vfid = rbp->svfid;
 394
 395                 if (rbp->srbp) {
 396                         sdd->read_cmd.rbp = rbp->srbp;
 397                         sdd->read_cmd.rdtype = DPAA2_RBP_MEM_RW;
 398                 } else {
 399                         sdd->read_cmd.rdtype = dpaa2_coherent_no_alloc_cache;
 400                 }
 401                 sdd++;
 402                 /* destination */
 403                 sdd->write_cmd.portid = rbp->dportid;
 404                 sdd->rbpcmd_simple.pfid = rbp->dpfid;
 405                 sdd->rbpcmd_simple.vfid = rbp->dvfid;
 406
 407                 if (rbp->drbp) {
 408                         sdd->write_cmd.rbp = rbp->drbp;
 409                         sdd->write_cmd.wrttype = DPAA2_RBP_MEM_RW;
 410                 } else {
 411                         sdd->write_cmd.wrttype = dpaa2_coherent_alloc_cache;
 412                 }
 413
 414         } else {
 415                 sdd->read_cmd.rdtype = dpaa2_coherent_no_alloc_cache;
 416                 sdd++;
 417                 sdd->write_cmd.wrttype = dpaa2_coherent_alloc_cache;
 418         }
 419         fle++;
 420         /* source frame list to source buffer */
 421         if (flags & RTE_QDMA_JOB_SRC_PHY) {
 422                 DPAA2_SET_FLE_ADDR(fle, src);
 423                 DPAA2_SET_FLE_BMT(fle);
 424         } else {
 425                 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(src));
 426         }
 427         DPAA2_SET_FLE_LEN(fle, len);
 428
 429         fle++;
 430         /* destination frame list to destination buffer */
 431         if (flags & RTE_QDMA_JOB_DEST_PHY) {
 432                 DPAA2_SET_FLE_BMT(fle);
 433                 DPAA2_SET_FLE_ADDR(fle, dest);
 434         } else {
 435                 DPAA2_SET_FLE_ADDR(fle, DPAA2_VADDR_TO_IOVA(dest));
 436         }
 437         DPAA2_SET_FLE_LEN(fle, len);
 438
 439         /* Final bit: 1, for last frame list */
 440         DPAA2_SET_FLE_FIN(fle);
 441 }
 442
 443 static inline uint16_t dpdmai_dev_set_fd(struct qbman_fd *fd,
 444                                         struct rte_qdma_job *job,
 445                                         struct rte_qdma_rbp *rbp,
 446                                         uint16_t vq_id)
 447 {
 448         struct qdma_io_meta *io_meta;
 449         struct qbman_fle *fle;
 450         int ret = 0;
 451         /*
 452          * Get an FLE/SDD from FLE pool.
 453          * Note: IO metadata is before the FLE and SDD memory.
 454          */
 455         ret = rte_mempool_get(qdma_dev.fle_pool, (void **)(&io_meta));
 456         if (ret) {
 457                 DPAA2_QDMA_DP_DEBUG("Memory alloc failed for FLE");
 458                 return ret;
 459         }
 460
 461         /* Set the metadata */
 462         io_meta->cnxt = (size_t)job;
 463         io_meta->id = vq_id;
 464
 465         fle = (struct qbman_fle *)(io_meta + 1);
 466
 467         DPAA2_SET_FD_ADDR(fd, DPAA2_VADDR_TO_IOVA(fle));
 468         DPAA2_SET_FD_COMPOUND_FMT(fd);
 469         DPAA2_SET_FD_FRC(fd, QDMA_SER_CTX);
 470
 471         /* Populate FLE */
 472         memset(fle, 0, QDMA_FLE_POOL_SIZE);
 473         dpaa2_qdma_populate_fle(fle, rbp, job->src, job->dest,
 474                                 job->len, job->flags);
 475
 476         return 0;
 477 }
 478
 479 static int
 480 dpdmai_dev_enqueue_multi(struct dpaa2_dpdmai_dev *dpdmai_dev,
 481                         uint16_t txq_id,
 482                         uint16_t vq_id,
 483                         struct rte_qdma_rbp *rbp,
 484                         struct rte_qdma_job **job,
 485                         uint16_t nb_jobs)
 486 {
 487         struct qbman_fd fd[RTE_QDMA_BURST_NB_MAX];
 488         struct dpaa2_queue *txq;
 489         struct qbman_eq_desc eqdesc;
 490         struct qbman_swp *swp;
 491         int ret;
 492         uint32_t num_to_send = 0;
 493         uint16_t num_tx = 0;
 494
 495         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 496                 ret = dpaa2_affine_qbman_swp();
 497                 if (ret) {
 498                         DPAA2_QDMA_ERR("Failure in affining portal");
 499                         return 0;
 500                 }
 501         }
 502         swp = DPAA2_PER_LCORE_PORTAL;
 503
 504         txq = &(dpdmai_dev->tx_queue[txq_id]);
 505
 506         /* Prepare enqueue descriptor */
 507         qbman_eq_desc_clear(&eqdesc);
 508         qbman_eq_desc_set_fq(&eqdesc, txq->fqid);
 509         qbman_eq_desc_set_no_orp(&eqdesc, 0);
 510         qbman_eq_desc_set_response(&eqdesc, 0, 0);
 511
 512         memset(fd, 0, RTE_QDMA_BURST_NB_MAX * sizeof(struct qbman_fd));
 513
 514         while (nb_jobs > 0) {
 515                 uint32_t loop;
 516
 517                 num_to_send = (nb_jobs > dpaa2_eqcr_size) ?
 518                         dpaa2_eqcr_size : nb_jobs;
 519
 520                 for (loop = 0; loop < num_to_send; loop++) {
 521                         ret = dpdmai_dev_set_fd(&fd[loop],
 522                                                 job[num_tx], rbp, vq_id);
 523                         if (ret < 0) {
 524                                 /* Set nb_jobs to loop, so outer while loop
 525                                  * breaks out.
 526                                  */
 527                                 nb_jobs = loop;
 528                                 break;
 529                         }
 530
 531                         num_tx++;
 532                 }
 533
 534                 /* Enqueue the packet to the QBMAN */
 535                 uint32_t enqueue_loop = 0;
 536                 while (enqueue_loop < loop) {
 537                         enqueue_loop += qbman_swp_enqueue_multiple(swp,
 538                                                 &eqdesc,
 539                                                 &fd[enqueue_loop],
 540                                                 NULL,
 541                                                 loop - enqueue_loop);
 542                 }
 543                 nb_jobs -= loop;
 544         }
 545         return num_tx;
 546 }
 547
 548 int
 549 rte_qdma_vq_enqueue_multi(uint16_t vq_id,
 550                           struct rte_qdma_job **job,
 551                           uint16_t nb_jobs)
 552 {
 553         struct qdma_virt_queue *qdma_vq = &qdma_vqs[vq_id];
 554         struct qdma_hw_queue *qdma_pq = qdma_vq->hw_queue;
 555         struct dpaa2_dpdmai_dev *dpdmai_dev = qdma_pq->dpdmai_dev;
 556         int ret;
 557
 558         DPAA2_QDMA_FUNC_TRACE();
 559
 560         /* Return error in case of wrong lcore_id */
 561         if (rte_lcore_id() != qdma_vq->lcore_id) {
 562                 DPAA2_QDMA_ERR("QDMA enqueue for vqid %d on wrong core",
 563                                 vq_id);
 564                 return -EINVAL;
 565         }
 566
 567         ret = dpdmai_dev_enqueue_multi(dpdmai_dev,
 568                                  qdma_pq->queue_id,
 569                                  vq_id,
 570                                  &qdma_vq->rbp,
 571                                  job,
 572                                  nb_jobs);
 573         if (ret < 0) {
 574                 DPAA2_QDMA_ERR("DPDMAI device enqueue failed: %d", ret);
 575                 return ret;
 576         }
 577
 578         qdma_vq->num_enqueues += ret;
 579
 580         return ret;
 581 }
 582
 583 int
 584 rte_qdma_vq_enqueue(uint16_t vq_id,
 585                     struct rte_qdma_job *job)
 586 {
 587         return rte_qdma_vq_enqueue_multi(vq_id, &job, 1);
 588 }
 589
 590 static inline uint16_t dpdmai_dev_get_job(const struct qbman_fd *fd,
 591                                         struct rte_qdma_job **job)
 592 {
 593         struct qbman_fle *fle;
 594         struct qdma_io_meta *io_meta;
 595         uint16_t vqid;
 596         /*
 597          * Fetch metadata from FLE. job and vq_id were set
 598          * in metadata in the enqueue operation.
 599          */
 600         fle = (struct qbman_fle *)DPAA2_IOVA_TO_VADDR(DPAA2_GET_FD_ADDR(fd));
 601         io_meta = (struct qdma_io_meta *)(fle) - 1;
 602
 603         *job = (struct rte_qdma_job *)(size_t)io_meta->cnxt;
 604         (*job)->status = (DPAA2_GET_FD_ERR(fd) << 8) |
 605                          (DPAA2_GET_FD_FRC(fd) & 0xFF);
 606
 607         vqid = io_meta->id;
 608
 609         /* Free FLE to the pool */
 610         rte_mempool_put(qdma_dev.fle_pool, io_meta);
 611
 612         return vqid;
 613 }
 614
 615 static int
 616 dpdmai_dev_dequeue_multijob(struct dpaa2_dpdmai_dev *dpdmai_dev,
 617                    uint16_t rxq_id,
 618                    uint16_t *vq_id,
 619                    struct rte_qdma_job **job,
 620                    uint16_t nb_jobs)
 621 {
 622         struct dpaa2_queue *rxq;
 623         struct qbman_result *dq_storage;
 624         struct qbman_pull_desc pulldesc;
 625         struct qbman_swp *swp;
 626         uint32_t fqid;
 627         uint8_t status, pending;
 628         uint8_t num_rx = 0;
 629         const struct qbman_fd *fd;
 630         uint16_t vqid;
 631         int ret, next_pull = nb_jobs, num_pulled = 0;
 632
 633         DPAA2_QDMA_FUNC_TRACE();
 634
 635         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 636                 ret = dpaa2_affine_qbman_swp();
 637                 if (ret) {
 638                         DPAA2_QDMA_ERR("Failure in affining portal");
 639                         return 0;
 640                 }
 641         }
 642         swp = DPAA2_PER_LCORE_PORTAL;
 643
 644         rxq = &(dpdmai_dev->rx_queue[rxq_id]);
 645         fqid = rxq->fqid;
 646
 647         do {
 648                 dq_storage = rxq->q_storage->dq_storage[0];
 649                 /* Prepare dequeue descriptor */
 650                 qbman_pull_desc_clear(&pulldesc);
 651                 qbman_pull_desc_set_fq(&pulldesc, fqid);
 652                 qbman_pull_desc_set_storage(&pulldesc, dq_storage,
 653                         (uint64_t)(DPAA2_VADDR_TO_IOVA(dq_storage)), 1);
 654
 655                 if (next_pull > dpaa2_dqrr_size) {
 656                         qbman_pull_desc_set_numframes(&pulldesc,
 657                                         dpaa2_dqrr_size);
 658                         next_pull -= dpaa2_dqrr_size;
 659                 } else {
 660                         qbman_pull_desc_set_numframes(&pulldesc, next_pull);
 661                         next_pull = 0;
 662                 }
 663
 664                 while (1) {
 665                         if (qbman_swp_pull(swp, &pulldesc)) {
 666                                 DPAA2_QDMA_DP_WARN("VDQ command not issued. QBMAN busy");
 667                                 /* Portal was busy, try again */
 668                                 continue;
 669                         }
 670                         break;
 671                 }
 672
 673                 rte_prefetch0((void *)((size_t)(dq_storage + 1)));
 674                 /* Check if the previous issued command is completed. */
 675                 while (!qbman_check_command_complete(dq_storage))
 676                         ;
 677
 678                 num_pulled = 0;
 679                 pending = 1;
 680
 681                 do {
 682                         /* Loop until dq_storage is updated
 683                          * with new token by QBMAN
 684                          */
 685                         while (!qbman_check_new_result(dq_storage))
 686                                 ;
 687                         rte_prefetch0((void *)((size_t)(dq_storage + 2)));
 688
 689                         if (qbman_result_DQ_is_pull_complete(dq_storage)) {
 690                                 pending = 0;
 691                                 /* Check for valid frame. */
 692                                 status = qbman_result_DQ_flags(dq_storage);
 693                                 if (unlikely((status &
 694                                         QBMAN_DQ_STAT_VALIDFRAME) == 0))
 695                                         continue;
 696                         }
 697                         fd = qbman_result_DQ_fd(dq_storage);
 698
 699                         vqid = dpdmai_dev_get_job(fd, &job[num_rx]);
 700                         if (vq_id)
 701                                 vq_id[num_rx] = vqid;
 702
 703                         dq_storage++;
 704                         num_rx++;
 705                         num_pulled++;
 706
 707                 } while (pending);
 708         /* Last VDQ provided all packets and more packets are requested */
 709         } while (next_pull && num_pulled == dpaa2_dqrr_size);
 710
 711         return num_rx;
 712 }
 713
 714 int
 715 rte_qdma_vq_dequeue_multi(uint16_t vq_id,
 716                           struct rte_qdma_job **job,
 717                           uint16_t nb_jobs)
 718 {
 719         struct qdma_virt_queue *qdma_vq = &qdma_vqs[vq_id];
 720         struct qdma_hw_queue *qdma_pq = qdma_vq->hw_queue;
 721         struct qdma_virt_queue *temp_qdma_vq;
 722         struct dpaa2_dpdmai_dev *dpdmai_dev = qdma_pq->dpdmai_dev;
 723         int ring_count, ret = 0, i;
 724
 725         /* Return error in case of wrong lcore_id */
 726         if (rte_lcore_id() != (unsigned int)(qdma_vq->lcore_id)) {
 727                 DPAA2_QDMA_WARN("QDMA dequeue for vqid %d on wrong core",
 728                                 vq_id);
 729                 return -1;
 730         }
 731
 732         /* Only dequeue when there are pending jobs on VQ */
 733         if (qdma_vq->num_enqueues == qdma_vq->num_dequeues)
 734                 return 0;
 735
 736         if (qdma_vq->num_enqueues < (qdma_vq->num_dequeues + nb_jobs))
 737                 nb_jobs = (qdma_vq->num_enqueues -  qdma_vq->num_dequeues);
 738
 739         if (qdma_vq->exclusive_hw_queue) {
 740                 /* In case of exclusive queue directly fetch from HW queue */
 741                 ret = dpdmai_dev_dequeue_multijob(dpdmai_dev, qdma_pq->queue_id,
 742                                          NULL, job, nb_jobs);
 743                 if (ret < 0) {
 744                         DPAA2_QDMA_ERR(
 745                                 "Dequeue from DPDMAI device failed: %d", ret);
 746                         return ret;
 747                 }
 748                 qdma_vq->num_dequeues += ret;
 749         } else {
 750                 uint16_t temp_vq_id[RTE_QDMA_BURST_NB_MAX];
 751                 /*
 752                  * Get the QDMA completed jobs from the software ring.
 753                  * In case they are not available on the ring poke the HW
 754                  * to fetch completed jobs from corresponding HW queues
 755                  */
 756                 ring_count = rte_ring_count(qdma_vq->status_ring);
 757                 if (ring_count < nb_jobs) {
 758                         /* TODO - How to have right budget */
 759                         ret = dpdmai_dev_dequeue_multijob(dpdmai_dev,
 760                                         qdma_pq->queue_id,
 761                                         temp_vq_id, job, nb_jobs);
 762                         for (i = 0; i < ret; i++) {
 763                                 temp_qdma_vq = &qdma_vqs[temp_vq_id[i]];
 764                                 rte_ring_enqueue(temp_qdma_vq->status_ring,
 765                                         (void *)(job[i]));
 766                         }
 767                         ring_count = rte_ring_count(
 768                                         qdma_vq->status_ring);
 769                 }
 770
 771                 if (ring_count) {
 772                         /* Dequeue job from the software ring
 773                          * to provide to the user
 774                          */
 775                         ret = rte_ring_dequeue_bulk(qdma_vq->status_ring,
 776                                         (void **)job, ring_count, NULL);
 777                         if (ret)
 778                                 qdma_vq->num_dequeues += ret;
 779                 }
 780         }
 781
 782         return ret;
 783 }
 784
 785 struct rte_qdma_job *
 786 rte_qdma_vq_dequeue(uint16_t vq_id)
 787 {
 788         int ret;
 789         struct rte_qdma_job *job = NULL;
 790
 791         ret = rte_qdma_vq_dequeue_multi(vq_id, &job, 1);
 792         if (ret < 0)
 793                 DPAA2_QDMA_DP_WARN("DPDMAI device dequeue failed: %d", ret);
 794
 795         return job;
 796 }
 797
 798 void
 799 rte_qdma_vq_stats(uint16_t vq_id,
 800                   struct rte_qdma_vq_stats *vq_status)
 801 {
 802         struct qdma_virt_queue *qdma_vq = &qdma_vqs[vq_id];
 803
 804         DPAA2_QDMA_FUNC_TRACE();
 805
 806         if (qdma_vq->in_use) {
 807                 vq_status->exclusive_hw_queue = qdma_vq->exclusive_hw_queue;
 808                 vq_status->lcore_id = qdma_vq->lcore_id;
 809                 vq_status->num_enqueues = qdma_vq->num_enqueues;
 810                 vq_status->num_dequeues = qdma_vq->num_dequeues;
 811                 vq_status->num_pending_jobs = vq_status->num_enqueues -
 812                                 vq_status->num_dequeues;
 813         }
 814 }
 815
 816 int
 817 rte_qdma_vq_destroy(uint16_t vq_id)
 818 {
 819         struct qdma_virt_queue *qdma_vq = &qdma_vqs[vq_id];
 820
 821         DPAA2_QDMA_FUNC_TRACE();
 822
 823         /* In case there are pending jobs on any VQ, return -EBUSY */
 824         if (qdma_vq->num_enqueues != qdma_vq->num_dequeues)
 825                 return -EBUSY;
 826
 827         rte_spinlock_lock(&qdma_dev.lock);
 828
 829         if (qdma_vq->exclusive_hw_queue)
 830                 free_hw_queue(qdma_vq->hw_queue);
 831         else {
 832                 if (qdma_vqs->status_ring)
 833                         rte_ring_free(qdma_vqs->status_ring);
 834
 835                 put_hw_queue(qdma_vq->hw_queue);
 836         }
 837
 838         memset(qdma_vq, 0, sizeof(struct qdma_virt_queue));
 839
 840         rte_spinlock_lock(&qdma_dev.lock);
 841
 842         return 0;
 843 }
 844
 845 int
 846 rte_qdma_vq_destroy_rbp(uint16_t vq_id)
 847 {
 848         struct qdma_virt_queue *qdma_vq = &qdma_vqs[vq_id];
 849
 850         DPAA2_QDMA_FUNC_TRACE();
 851
 852         /* In case there are pending jobs on any VQ, return -EBUSY */
 853         if (qdma_vq->num_enqueues != qdma_vq->num_dequeues)
 854                 return -EBUSY;
 855
 856         rte_spinlock_lock(&qdma_dev.lock);
 857
 858         if (qdma_vq->exclusive_hw_queue) {
 859                 free_hw_queue(qdma_vq->hw_queue);
 860         } else {
 861                 if (qdma_vqs->status_ring)
 862                         rte_ring_free(qdma_vqs->status_ring);
 863
 864                 put_hw_queue(qdma_vq->hw_queue);
 865         }
 866
 867         memset(qdma_vq, 0, sizeof(struct qdma_virt_queue));
 868
 869         rte_spinlock_lock(&qdma_dev.lock);
 870
 871         return 0;
 872 }
 873
 874 void
 875 rte_qdma_stop(void)
 876 {
 877         DPAA2_QDMA_FUNC_TRACE();
 878
 879         qdma_dev.state = 0;
 880 }
 881
 882 void
 883 rte_qdma_destroy(void)
 884 {
 885         DPAA2_QDMA_FUNC_TRACE();
 886
 887         rte_qdma_reset();
 888 }
 889
 890 static const struct rte_rawdev_ops dpaa2_qdma_ops;
 891
 892 static int
 893 add_hw_queues_to_list(struct dpaa2_dpdmai_dev *dpdmai_dev)
 894 {
 895         struct qdma_hw_queue *queue;
 896         int i;
 897
 898         DPAA2_QDMA_FUNC_TRACE();
 899
 900         for (i = 0; i < dpdmai_dev->num_queues; i++) {
 901                 queue = rte_zmalloc(NULL, sizeof(struct qdma_hw_queue), 0);
 902                 if (!queue) {
 903                         DPAA2_QDMA_ERR(
 904                                 "Memory allocation failed for QDMA queue");
 905                         return -ENOMEM;
 906                 }
 907
 908                 queue->dpdmai_dev = dpdmai_dev;
 909                 queue->queue_id = i;
 910
 911                 TAILQ_INSERT_TAIL(&qdma_queue_list, queue, next);
 912                 qdma_dev.num_hw_queues++;
 913         }
 914
 915         return 0;
 916 }
 917
 918 static void
 919 remove_hw_queues_from_list(struct dpaa2_dpdmai_dev *dpdmai_dev)
 920 {
 921         struct qdma_hw_queue *queue = NULL;
 922         struct qdma_hw_queue *tqueue = NULL;
 923
 924         DPAA2_QDMA_FUNC_TRACE();
 925
 926         TAILQ_FOREACH_SAFE(queue, &qdma_queue_list, next, tqueue) {
 927                 if (queue->dpdmai_dev == dpdmai_dev) {
 928                         TAILQ_REMOVE(&qdma_queue_list, queue, next);
 929                         rte_free(queue);
 930                         queue = NULL;
 931                 }
 932         }
 933 }
 934
 935 static int
 936 dpaa2_dpdmai_dev_uninit(struct rte_rawdev *rawdev)
 937 {
 938         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
 939         int ret, i;
 940
 941         DPAA2_QDMA_FUNC_TRACE();
 942
 943         /* Remove HW queues from global list */
 944         remove_hw_queues_from_list(dpdmai_dev);
 945
 946         ret = dpdmai_disable(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
 947                              dpdmai_dev->token);
 948         if (ret)
 949                 DPAA2_QDMA_ERR("dmdmai disable failed");
 950
 951         /* Set up the DQRR storage for Rx */
 952         for (i = 0; i < dpdmai_dev->num_queues; i++) {
 953                 struct dpaa2_queue *rxq = &(dpdmai_dev->rx_queue[i]);
 954
 955                 if (rxq->q_storage) {
 956                         dpaa2_free_dq_storage(rxq->q_storage);
 957                         rte_free(rxq->q_storage);
 958                 }
 959         }
 960
 961         /* Close the device at underlying layer*/
 962         ret = dpdmai_close(&dpdmai_dev->dpdmai, CMD_PRI_LOW, dpdmai_dev->token);
 963         if (ret)
 964                 DPAA2_QDMA_ERR("Failure closing dpdmai device");
 965
 966         return 0;
 967 }
 968
 969 static int
 970 dpaa2_dpdmai_dev_init(struct rte_rawdev *rawdev, int dpdmai_id)
 971 {
 972         struct dpaa2_dpdmai_dev *dpdmai_dev = rawdev->dev_private;
 973         struct dpdmai_rx_queue_cfg rx_queue_cfg;
 974         struct dpdmai_attr attr;
 975         struct dpdmai_rx_queue_attr rx_attr;
 976         struct dpdmai_tx_queue_attr tx_attr;
 977         int ret, i;
 978
 979         DPAA2_QDMA_FUNC_TRACE();
 980
 981         /* Open DPDMAI device */
 982         dpdmai_dev->dpdmai_id = dpdmai_id;
 983         dpdmai_dev->dpdmai.regs = rte_mcp_ptr_list[MC_PORTAL_INDEX];
 984         ret = dpdmai_open(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
 985                           dpdmai_dev->dpdmai_id, &dpdmai_dev->token);
 986         if (ret) {
 987                 DPAA2_QDMA_ERR("dpdmai_open() failed with err: %d", ret);
 988                 return ret;
 989         }
 990
 991         /* Get DPDMAI attributes */
 992         ret = dpdmai_get_attributes(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
 993                                     dpdmai_dev->token, &attr);
 994         if (ret) {
 995                 DPAA2_QDMA_ERR("dpdmai get attributes failed with err: %d",
 996                                ret);
 997                 goto init_err;
 998         }
 999         dpdmai_dev->num_queues = attr.num_of_queues;
1000
1001         /* Set up Rx Queues */
1002         for (i = 0; i < dpdmai_dev->num_queues; i++) {
1003                 struct dpaa2_queue *rxq;
1004
1005                 memset(&rx_queue_cfg, 0, sizeof(struct dpdmai_rx_queue_cfg));
1006                 ret = dpdmai_set_rx_queue(&dpdmai_dev->dpdmai,
1007                                           CMD_PRI_LOW,
1008                                           dpdmai_dev->token,
1009                                           i, 0, &rx_queue_cfg);
1010                 if (ret) {
1011                         DPAA2_QDMA_ERR("Setting Rx queue failed with err: %d",
1012                                        ret);
1013                         goto init_err;
1014                 }
1015
1016                 /* Allocate DQ storage for the DPDMAI Rx queues */
1017                 rxq = &(dpdmai_dev->rx_queue[i]);
1018                 rxq->q_storage = rte_malloc("dq_storage",
1019                                             sizeof(struct queue_storage_info_t),
1020                                             RTE_CACHE_LINE_SIZE);
1021                 if (!rxq->q_storage) {
1022                         DPAA2_QDMA_ERR("q_storage allocation failed");
1023                         ret = -ENOMEM;
1024                         goto init_err;
1025                 }
1026
1027                 memset(rxq->q_storage, 0, sizeof(struct queue_storage_info_t));
1028                 ret = dpaa2_alloc_dq_storage(rxq->q_storage);
1029                 if (ret) {
1030                         DPAA2_QDMA_ERR("dpaa2_alloc_dq_storage failed");
1031                         goto init_err;
1032                 }
1033         }
1034
1035         /* Get Rx and Tx queues FQID's */
1036         for (i = 0; i < dpdmai_dev->num_queues; i++) {
1037                 ret = dpdmai_get_rx_queue(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1038                                           dpdmai_dev->token, i, 0, &rx_attr);
1039                 if (ret) {
1040                         DPAA2_QDMA_ERR("Reading device failed with err: %d",
1041                                        ret);
1042                         goto init_err;
1043                 }
1044                 dpdmai_dev->rx_queue[i].fqid = rx_attr.fqid;
1045
1046                 ret = dpdmai_get_tx_queue(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1047                                           dpdmai_dev->token, i, 0, &tx_attr);
1048                 if (ret) {
1049                         DPAA2_QDMA_ERR("Reading device failed with err: %d",
1050                                        ret);
1051                         goto init_err;
1052                 }
1053                 dpdmai_dev->tx_queue[i].fqid = tx_attr.fqid;
1054         }
1055
1056         /* Enable the device */
1057         ret = dpdmai_enable(&dpdmai_dev->dpdmai, CMD_PRI_LOW,
1058                             dpdmai_dev->token);
1059         if (ret) {
1060                 DPAA2_QDMA_ERR("Enabling device failed with err: %d", ret);
1061                 goto init_err;
1062         }
1063
1064         /* Add the HW queue to the global list */
1065         ret = add_hw_queues_to_list(dpdmai_dev);
1066         if (ret) {
1067                 DPAA2_QDMA_ERR("Adding H/W queue to list failed");
1068                 goto init_err;
1069         }
1070
1071         if (!dpaa2_coherent_no_alloc_cache) {
1072                 if (dpaa2_svr_family == SVR_LX2160A) {
1073                         dpaa2_coherent_no_alloc_cache =
1074                                 DPAA2_LX2_COHERENT_NO_ALLOCATE_CACHE;
1075                         dpaa2_coherent_alloc_cache =
1076                                 DPAA2_LX2_COHERENT_ALLOCATE_CACHE;
1077                 } else {
1078                         dpaa2_coherent_no_alloc_cache =
1079                                 DPAA2_COHERENT_NO_ALLOCATE_CACHE;
1080                         dpaa2_coherent_alloc_cache =
1081                                 DPAA2_COHERENT_ALLOCATE_CACHE;
1082                 }
1083         }
1084
1085         DPAA2_QDMA_DEBUG("Initialized dpdmai object successfully");
1086
1087         return 0;
1088 init_err:
1089         dpaa2_dpdmai_dev_uninit(rawdev);
1090         return ret;
1091 }
1092
1093 static int
1094 rte_dpaa2_qdma_probe(struct rte_dpaa2_driver *dpaa2_drv,
1095                      struct rte_dpaa2_device *dpaa2_dev)
1096 {
1097         struct rte_rawdev *rawdev;
1098         int ret;
1099
1100         DPAA2_QDMA_FUNC_TRACE();
1101
1102         rawdev = rte_rawdev_pmd_allocate(dpaa2_dev->device.name,
1103                         sizeof(struct dpaa2_dpdmai_dev),
1104                         rte_socket_id());
1105         if (!rawdev) {
1106                 DPAA2_QDMA_ERR("Unable to allocate rawdevice");
1107                 return -EINVAL;
1108         }
1109
1110         dpaa2_dev->rawdev = rawdev;
1111         rawdev->dev_ops = &dpaa2_qdma_ops;
1112         rawdev->device = &dpaa2_dev->device;
1113         rawdev->driver_name = dpaa2_drv->driver.name;
1114
1115         /* Invoke PMD device initialization function */
1116         ret = dpaa2_dpdmai_dev_init(rawdev, dpaa2_dev->object_id);
1117         if (ret) {
1118                 rte_rawdev_pmd_release(rawdev);
1119                 return ret;
1120         }
1121
1122         return 0;
1123 }
1124
1125 static int
1126 rte_dpaa2_qdma_remove(struct rte_dpaa2_device *dpaa2_dev)
1127 {
1128         struct rte_rawdev *rawdev = dpaa2_dev->rawdev;
1129         int ret;
1130
1131         DPAA2_QDMA_FUNC_TRACE();
1132
1133         dpaa2_dpdmai_dev_uninit(rawdev);
1134
1135         ret = rte_rawdev_pmd_release(rawdev);
1136         if (ret)
1137                 DPAA2_QDMA_ERR("Device cleanup failed");
1138
1139         return 0;
1140 }
1141
1142 static struct rte_dpaa2_driver rte_dpaa2_qdma_pmd = {
1143         .drv_flags = RTE_DPAA2_DRV_IOVA_AS_VA,
1144         .drv_type = DPAA2_QDMA,
1145         .probe = rte_dpaa2_qdma_probe,
1146         .remove = rte_dpaa2_qdma_remove,
1147 };
1148
1149 RTE_PMD_REGISTER_DPAA2(dpaa2_qdma, rte_dpaa2_qdma_pmd);
1150
1151 RTE_INIT(dpaa2_qdma_init_log)
1152 {
1153         dpaa2_qdma_logtype = rte_log_register("pmd.raw.dpaa2.qdma");
1154         if (dpaa2_qdma_logtype >= 0)
1155                 rte_log_set_level(dpaa2_qdma_logtype, RTE_LOG_INFO);
1156 }