drivers/dma/dpaa/dpaa_qdma.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2021 NXP
   3  */
   4
   5 #include <rte_dpaa_bus.h>
   6 #include <rte_dmadev_pmd.h>
   7
   8 #include "dpaa_qdma.h"
   9 #include "dpaa_qdma_logs.h"
  10
  11 static inline void
  12 qdma_desc_addr_set64(struct fsl_qdma_format *ccdf, u64 addr)
  13 {
  14         ccdf->addr_hi = upper_32_bits(addr);
  15         ccdf->addr_lo = rte_cpu_to_le_32(lower_32_bits(addr));
  16 }
  17
  18 static inline u64
  19 qdma_ccdf_get_queue(const struct fsl_qdma_format *ccdf)
  20 {
  21         return ccdf->cfg8b_w1 & 0xff;
  22 }
  23
  24 static inline int
  25 qdma_ccdf_get_offset(const struct fsl_qdma_format *ccdf)
  26 {
  27         return (rte_le_to_cpu_32(ccdf->cfg) & QDMA_CCDF_MASK)
  28                 >> QDMA_CCDF_OFFSET;
  29 }
  30
  31 static inline void
  32 qdma_ccdf_set_format(struct fsl_qdma_format *ccdf, int offset)
  33 {
  34         ccdf->cfg = rte_cpu_to_le_32(QDMA_CCDF_FOTMAT | offset);
  35 }
  36
  37 static inline int
  38 qdma_ccdf_get_status(const struct fsl_qdma_format *ccdf)
  39 {
  40         return (rte_le_to_cpu_32(ccdf->status) & QDMA_CCDF_MASK)
  41                 >> QDMA_CCDF_STATUS;
  42 }
  43
  44 static inline void
  45 qdma_ccdf_set_ser(struct fsl_qdma_format *ccdf, int status)
  46 {
  47         ccdf->status = rte_cpu_to_le_32(QDMA_CCDF_SER | status);
  48 }
  49
  50 static inline void
  51 qdma_csgf_set_len(struct fsl_qdma_format *csgf, int len)
  52 {
  53         csgf->cfg = rte_cpu_to_le_32(len & QDMA_SG_LEN_MASK);
  54 }
  55
  56 static inline void
  57 qdma_csgf_set_f(struct fsl_qdma_format *csgf, int len)
  58 {
  59         csgf->cfg = rte_cpu_to_le_32(QDMA_SG_FIN | (len & QDMA_SG_LEN_MASK));
  60 }
  61
  62 static inline int
  63 ilog2(int x)
  64 {
  65         int log = 0;
  66
  67         x >>= 1;
  68
  69         while (x) {
  70                 log++;
  71                 x >>= 1;
  72         }
  73         return log;
  74 }
  75
  76 static u32
  77 qdma_readl(void *addr)
  78 {
  79         return QDMA_IN(addr);
  80 }
  81
  82 static void
  83 qdma_writel(u32 val, void *addr)
  84 {
  85         QDMA_OUT(addr, val);
  86 }
  87
  88 static u32
  89 qdma_readl_be(void *addr)
  90 {
  91         return QDMA_IN_BE(addr);
  92 }
  93
  94 static void
  95 qdma_writel_be(u32 val, void *addr)
  96 {
  97         QDMA_OUT_BE(addr, val);
  98 }
  99
 100 static void
 101 *dma_pool_alloc(int size, int aligned, dma_addr_t *phy_addr)
 102 {
 103         void *virt_addr;
 104
 105         virt_addr = rte_malloc("dma pool alloc", size, aligned);
 106         if (!virt_addr)
 107                 return NULL;
 108
 109         *phy_addr = rte_mem_virt2iova(virt_addr);
 110
 111         return virt_addr;
 112 }
 113
 114 static void
 115 dma_pool_free(void *addr)
 116 {
 117         rte_free(addr);
 118 }
 119
 120 static void
 121 fsl_qdma_free_chan_resources(struct fsl_qdma_chan *fsl_chan)
 122 {
 123         struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
 124         struct fsl_qdma_engine *fsl_qdma = fsl_chan->qdma;
 125         struct fsl_qdma_comp *comp_temp, *_comp_temp;
 126         int id;
 127
 128         if (--fsl_queue->count)
 129                 goto finally;
 130
 131         id = (fsl_qdma->block_base - fsl_queue->block_base) /
 132               fsl_qdma->block_offset;
 133
 134         while (rte_atomic32_read(&wait_task[id]) == 1)
 135                 rte_delay_us(QDMA_DELAY);
 136
 137         list_for_each_entry_safe(comp_temp, _comp_temp,
 138                                  &fsl_queue->comp_used, list) {
 139                 list_del(&comp_temp->list);
 140                 dma_pool_free(comp_temp->virt_addr);
 141                 dma_pool_free(comp_temp->desc_virt_addr);
 142                 rte_free(comp_temp);
 143         }
 144
 145         list_for_each_entry_safe(comp_temp, _comp_temp,
 146                                  &fsl_queue->comp_free, list) {
 147                 list_del(&comp_temp->list);
 148                 dma_pool_free(comp_temp->virt_addr);
 149                 dma_pool_free(comp_temp->desc_virt_addr);
 150                 rte_free(comp_temp);
 151         }
 152
 153 finally:
 154         fsl_qdma->desc_allocated--;
 155 }
 156
 157 static void
 158 fsl_qdma_comp_fill_memcpy(struct fsl_qdma_comp *fsl_comp,
 159                                       dma_addr_t dst, dma_addr_t src, u32 len)
 160 {
 161         struct fsl_qdma_format *csgf_src, *csgf_dest;
 162
 163         /* Note: command table (fsl_comp->virt_addr) is getting filled
 164          * directly in cmd descriptors of queues while enqueuing the descriptor
 165          * please refer fsl_qdma_enqueue_desc
 166          * frame list table (virt_addr) + 1) and source,
 167          * destination descriptor table
 168          * (fsl_comp->desc_virt_addr and fsl_comp->desc_virt_addr+1) move to
 169          * the control path to fsl_qdma_pre_request_enqueue_comp_sd_desc
 170          */
 171         csgf_src = (struct fsl_qdma_format *)fsl_comp->virt_addr + 2;
 172         csgf_dest = (struct fsl_qdma_format *)fsl_comp->virt_addr + 3;
 173
 174         /* Status notification is enqueued to status queue. */
 175         qdma_desc_addr_set64(csgf_src, src);
 176         qdma_csgf_set_len(csgf_src, len);
 177         qdma_desc_addr_set64(csgf_dest, dst);
 178         qdma_csgf_set_len(csgf_dest, len);
 179         /* This entry is the last entry. */
 180         qdma_csgf_set_f(csgf_dest, len);
 181 }
 182
 183 /*
 184  * Pre-request command descriptor and compound S/G for enqueue.
 185  */
 186 static int
 187 fsl_qdma_pre_request_enqueue_comp_sd_desc(
 188                                         struct fsl_qdma_queue *queue,
 189                                         int size, int aligned)
 190 {
 191         struct fsl_qdma_comp *comp_temp, *_comp_temp;
 192         struct fsl_qdma_sdf *sdf;
 193         struct fsl_qdma_ddf *ddf;
 194         struct fsl_qdma_format *csgf_desc;
 195         int i;
 196
 197         for (i = 0; i < (int)(queue->n_cq + COMMAND_QUEUE_OVERFLOW); i++) {
 198                 comp_temp = rte_zmalloc("qdma: comp temp",
 199                                         sizeof(*comp_temp), 0);
 200                 if (!comp_temp)
 201                         return -ENOMEM;
 202
 203                 comp_temp->virt_addr =
 204                 dma_pool_alloc(size, aligned, &comp_temp->bus_addr);
 205                 if (!comp_temp->virt_addr) {
 206                         rte_free(comp_temp);
 207                         goto fail;
 208                 }
 209
 210                 comp_temp->desc_virt_addr =
 211                 dma_pool_alloc(size, aligned, &comp_temp->desc_bus_addr);
 212                 if (!comp_temp->desc_virt_addr) {
 213                         rte_free(comp_temp->virt_addr);
 214                         rte_free(comp_temp);
 215                         goto fail;
 216                 }
 217
 218                 memset(comp_temp->virt_addr, 0, FSL_QDMA_COMMAND_BUFFER_SIZE);
 219                 memset(comp_temp->desc_virt_addr, 0,
 220                        FSL_QDMA_DESCRIPTOR_BUFFER_SIZE);
 221
 222                 csgf_desc = (struct fsl_qdma_format *)comp_temp->virt_addr + 1;
 223                 sdf = (struct fsl_qdma_sdf *)comp_temp->desc_virt_addr;
 224                 ddf = (struct fsl_qdma_ddf *)comp_temp->desc_virt_addr + 1;
 225                 /* Compound Command Descriptor(Frame List Table) */
 226                 qdma_desc_addr_set64(csgf_desc, comp_temp->desc_bus_addr);
 227                 /* It must be 32 as Compound S/G Descriptor */
 228                 qdma_csgf_set_len(csgf_desc, 32);
 229                 /* Descriptor Buffer */
 230                 sdf->cmd = rte_cpu_to_le_32(FSL_QDMA_CMD_RWTTYPE <<
 231                                FSL_QDMA_CMD_RWTTYPE_OFFSET);
 232                 ddf->cmd = rte_cpu_to_le_32(FSL_QDMA_CMD_RWTTYPE <<
 233                                FSL_QDMA_CMD_RWTTYPE_OFFSET);
 234                 ddf->cmd |= rte_cpu_to_le_32(FSL_QDMA_CMD_LWC <<
 235                                 FSL_QDMA_CMD_LWC_OFFSET);
 236
 237                 list_add_tail(&comp_temp->list, &queue->comp_free);
 238         }
 239
 240         return 0;
 241
 242 fail:
 243         list_for_each_entry_safe(comp_temp, _comp_temp,
 244                                  &queue->comp_free, list) {
 245                 list_del(&comp_temp->list);
 246                 rte_free(comp_temp->virt_addr);
 247                 rte_free(comp_temp->desc_virt_addr);
 248                 rte_free(comp_temp);
 249         }
 250
 251         return -ENOMEM;
 252 }
 253
 254 /*
 255  * Request a command descriptor for enqueue.
 256  */
 257 static struct fsl_qdma_comp *
 258 fsl_qdma_request_enqueue_desc(struct fsl_qdma_chan *fsl_chan)
 259 {
 260         struct fsl_qdma_queue *queue = fsl_chan->queue;
 261         struct fsl_qdma_comp *comp_temp;
 262
 263         if (!list_empty(&queue->comp_free)) {
 264                 comp_temp = list_first_entry(&queue->comp_free,
 265                                              struct fsl_qdma_comp,
 266                                              list);
 267                 list_del(&comp_temp->list);
 268                 return comp_temp;
 269         }
 270
 271         return NULL;
 272 }
 273
 274 static struct fsl_qdma_queue
 275 *fsl_qdma_alloc_queue_resources(struct fsl_qdma_engine *fsl_qdma)
 276 {
 277         struct fsl_qdma_queue *queue_head, *queue_temp;
 278         int len, i, j;
 279         int queue_num;
 280         int blocks;
 281         unsigned int queue_size[FSL_QDMA_QUEUE_MAX];
 282
 283         queue_num = fsl_qdma->n_queues;
 284         blocks = fsl_qdma->num_blocks;
 285
 286         len = sizeof(*queue_head) * queue_num * blocks;
 287         queue_head = rte_zmalloc("qdma: queue head", len, 0);
 288         if (!queue_head)
 289                 return NULL;
 290
 291         for (i = 0; i < FSL_QDMA_QUEUE_MAX; i++)
 292                 queue_size[i] = QDMA_QUEUE_SIZE;
 293
 294         for (j = 0; j < blocks; j++) {
 295                 for (i = 0; i < queue_num; i++) {
 296                         if (queue_size[i] > FSL_QDMA_CIRCULAR_DESC_SIZE_MAX ||
 297                             queue_size[i] < FSL_QDMA_CIRCULAR_DESC_SIZE_MIN) {
 298                                 DPAA_QDMA_ERR("Get wrong queue-sizes.\n");
 299                                 goto fail;
 300                         }
 301                         queue_temp = queue_head + i + (j * queue_num);
 302
 303                         queue_temp->cq =
 304                         dma_pool_alloc(sizeof(struct fsl_qdma_format) *
 305                                        queue_size[i],
 306                                        sizeof(struct fsl_qdma_format) *
 307                                        queue_size[i], &queue_temp->bus_addr);
 308
 309                         if (!queue_temp->cq)
 310                                 goto fail;
 311
 312                         memset(queue_temp->cq, 0x0, queue_size[i] *
 313                                sizeof(struct fsl_qdma_format));
 314
 315                         queue_temp->block_base = fsl_qdma->block_base +
 316                                 FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j);
 317                         queue_temp->n_cq = queue_size[i];
 318                         queue_temp->id = i;
 319                         queue_temp->count = 0;
 320                         queue_temp->pending = 0;
 321                         queue_temp->virt_head = queue_temp->cq;
 322                         queue_temp->stats = (struct rte_dma_stats){0};
 323                 }
 324         }
 325         return queue_head;
 326
 327 fail:
 328         for (j = 0; j < blocks; j++) {
 329                 for (i = 0; i < queue_num; i++) {
 330                         queue_temp = queue_head + i + (j * queue_num);
 331                         dma_pool_free(queue_temp->cq);
 332                 }
 333         }
 334         rte_free(queue_head);
 335
 336         return NULL;
 337 }
 338
 339 static struct
 340 fsl_qdma_queue *fsl_qdma_prep_status_queue(void)
 341 {
 342         struct fsl_qdma_queue *status_head;
 343         unsigned int status_size;
 344
 345         status_size = QDMA_STATUS_SIZE;
 346         if (status_size > FSL_QDMA_CIRCULAR_DESC_SIZE_MAX ||
 347             status_size < FSL_QDMA_CIRCULAR_DESC_SIZE_MIN) {
 348                 DPAA_QDMA_ERR("Get wrong status_size.\n");
 349                 return NULL;
 350         }
 351
 352         status_head = rte_zmalloc("qdma: status head", sizeof(*status_head), 0);
 353         if (!status_head)
 354                 return NULL;
 355
 356         /*
 357          * Buffer for queue command
 358          */
 359         status_head->cq = dma_pool_alloc(sizeof(struct fsl_qdma_format) *
 360                                          status_size,
 361                                          sizeof(struct fsl_qdma_format) *
 362                                          status_size,
 363                                          &status_head->bus_addr);
 364
 365         if (!status_head->cq) {
 366                 rte_free(status_head);
 367                 return NULL;
 368         }
 369
 370         memset(status_head->cq, 0x0, status_size *
 371                sizeof(struct fsl_qdma_format));
 372         status_head->n_cq = status_size;
 373         status_head->virt_head = status_head->cq;
 374
 375         return status_head;
 376 }
 377
 378 static int
 379 fsl_qdma_halt(struct fsl_qdma_engine *fsl_qdma)
 380 {
 381         void *ctrl = fsl_qdma->ctrl_base;
 382         void *block;
 383         int i, count = RETRIES;
 384         unsigned int j;
 385         u32 reg;
 386
 387         /* Disable the command queue and wait for idle state. */
 388         reg = qdma_readl(ctrl + FSL_QDMA_DMR);
 389         reg |= FSL_QDMA_DMR_DQD;
 390         qdma_writel(reg, ctrl + FSL_QDMA_DMR);
 391         for (j = 0; j < fsl_qdma->num_blocks; j++) {
 392                 block = fsl_qdma->block_base +
 393                         FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j);
 394                 for (i = 0; i < FSL_QDMA_QUEUE_NUM_MAX; i++)
 395                         qdma_writel(0, block + FSL_QDMA_BCQMR(i));
 396         }
 397         while (true) {
 398                 reg = qdma_readl(ctrl + FSL_QDMA_DSR);
 399                 if (!(reg & FSL_QDMA_DSR_DB))
 400                         break;
 401                 if (count-- < 0)
 402                         return -EBUSY;
 403                 rte_delay_us(100);
 404         }
 405
 406         for (j = 0; j < fsl_qdma->num_blocks; j++) {
 407                 block = fsl_qdma->block_base +
 408                         FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j);
 409
 410                 /* Disable status queue. */
 411                 qdma_writel(0, block + FSL_QDMA_BSQMR);
 412
 413                 /*
 414                  * clear the command queue interrupt detect register for
 415                  * all queues.
 416                  */
 417                 qdma_writel(0xffffffff, block + FSL_QDMA_BCQIDR(0));
 418         }
 419
 420         return 0;
 421 }
 422
 423 static int
 424 fsl_qdma_queue_transfer_complete(struct fsl_qdma_engine *fsl_qdma,
 425                                  void *block, int id, const uint16_t nb_cpls,
 426                                  uint16_t *last_idx,
 427                                  enum rte_dma_status_code *status)
 428 {
 429         struct fsl_qdma_queue *fsl_queue = fsl_qdma->queue;
 430         struct fsl_qdma_queue *fsl_status = fsl_qdma->status[id];
 431         struct fsl_qdma_queue *temp_queue;
 432         struct fsl_qdma_format *status_addr;
 433         struct fsl_qdma_comp *fsl_comp = NULL;
 434         u32 reg, i;
 435         int count = 0;
 436
 437         while (count < nb_cpls) {
 438                 reg = qdma_readl_be(block + FSL_QDMA_BSQSR);
 439                 if (reg & FSL_QDMA_BSQSR_QE_BE)
 440                         return count;
 441
 442                 status_addr = fsl_status->virt_head;
 443
 444                 i = qdma_ccdf_get_queue(status_addr) +
 445                         id * fsl_qdma->n_queues;
 446                 temp_queue = fsl_queue + i;
 447                 fsl_comp = list_first_entry(&temp_queue->comp_used,
 448                                             struct fsl_qdma_comp,
 449                                             list);
 450                 list_del(&fsl_comp->list);
 451
 452                 reg = qdma_readl_be(block + FSL_QDMA_BSQMR);
 453                 reg |= FSL_QDMA_BSQMR_DI_BE;
 454
 455                 qdma_desc_addr_set64(status_addr, 0x0);
 456                 fsl_status->virt_head++;
 457                 if (fsl_status->virt_head == fsl_status->cq + fsl_status->n_cq)
 458                         fsl_status->virt_head = fsl_status->cq;
 459                 qdma_writel_be(reg, block + FSL_QDMA_BSQMR);
 460                 *last_idx = fsl_comp->index;
 461                 if (status != NULL)
 462                         status[count] = RTE_DMA_STATUS_SUCCESSFUL;
 463
 464                 list_add_tail(&fsl_comp->list, &temp_queue->comp_free);
 465                 count++;
 466
 467         }
 468         return count;
 469 }
 470
 471 static int
 472 fsl_qdma_reg_init(struct fsl_qdma_engine *fsl_qdma)
 473 {
 474         struct fsl_qdma_queue *fsl_queue = fsl_qdma->queue;
 475         struct fsl_qdma_queue *temp;
 476         void *ctrl = fsl_qdma->ctrl_base;
 477         void *block;
 478         u32 i, j;
 479         u32 reg;
 480         int ret, val;
 481
 482         /* Try to halt the qDMA engine first. */
 483         ret = fsl_qdma_halt(fsl_qdma);
 484         if (ret) {
 485                 DPAA_QDMA_ERR("DMA halt failed!");
 486                 return ret;
 487         }
 488
 489         for (j = 0; j < fsl_qdma->num_blocks; j++) {
 490                 block = fsl_qdma->block_base +
 491                         FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j);
 492                 for (i = 0; i < fsl_qdma->n_queues; i++) {
 493                         temp = fsl_queue + i + (j * fsl_qdma->n_queues);
 494                         /*
 495                          * Initialize Command Queue registers to
 496                          * point to the first
 497                          * command descriptor in memory.
 498                          * Dequeue Pointer Address Registers
 499                          * Enqueue Pointer Address Registers
 500                          */
 501
 502                         qdma_writel(lower_32_bits(temp->bus_addr),
 503                                     block + FSL_QDMA_BCQDPA_SADDR(i));
 504                         qdma_writel(upper_32_bits(temp->bus_addr),
 505                                     block + FSL_QDMA_BCQEDPA_SADDR(i));
 506                         qdma_writel(lower_32_bits(temp->bus_addr),
 507                                     block + FSL_QDMA_BCQEPA_SADDR(i));
 508                         qdma_writel(upper_32_bits(temp->bus_addr),
 509                                     block + FSL_QDMA_BCQEEPA_SADDR(i));
 510
 511                         /* Initialize the queue mode. */
 512                         reg = FSL_QDMA_BCQMR_EN;
 513                         reg |= FSL_QDMA_BCQMR_CD_THLD(ilog2(temp->n_cq) - 4);
 514                         reg |= FSL_QDMA_BCQMR_CQ_SIZE(ilog2(temp->n_cq) - 6);
 515                         qdma_writel(reg, block + FSL_QDMA_BCQMR(i));
 516                 }
 517
 518                 /*
 519                  * Workaround for erratum: ERR010812.
 520                  * We must enable XOFF to avoid the enqueue rejection occurs.
 521                  * Setting SQCCMR ENTER_WM to 0x20.
 522                  */
 523
 524                 qdma_writel(FSL_QDMA_SQCCMR_ENTER_WM,
 525                             block + FSL_QDMA_SQCCMR);
 526
 527                 /*
 528                  * Initialize status queue registers to point to the first
 529                  * command descriptor in memory.
 530                  * Dequeue Pointer Address Registers
 531                  * Enqueue Pointer Address Registers
 532                  */
 533
 534                 qdma_writel(
 535                             upper_32_bits(fsl_qdma->status[j]->bus_addr),
 536                             block + FSL_QDMA_SQEEPAR);
 537                 qdma_writel(
 538                             lower_32_bits(fsl_qdma->status[j]->bus_addr),
 539                             block + FSL_QDMA_SQEPAR);
 540                 qdma_writel(
 541                             upper_32_bits(fsl_qdma->status[j]->bus_addr),
 542                             block + FSL_QDMA_SQEDPAR);
 543                 qdma_writel(
 544                             lower_32_bits(fsl_qdma->status[j]->bus_addr),
 545                             block + FSL_QDMA_SQDPAR);
 546                 /* Desiable status queue interrupt. */
 547
 548                 qdma_writel(0x0, block + FSL_QDMA_BCQIER(0));
 549                 qdma_writel(0x0, block + FSL_QDMA_BSQICR);
 550                 qdma_writel(0x0, block + FSL_QDMA_CQIER);
 551
 552                 /* Initialize the status queue mode. */
 553                 reg = FSL_QDMA_BSQMR_EN;
 554                 val = ilog2(fsl_qdma->status[j]->n_cq) - 6;
 555                 reg |= FSL_QDMA_BSQMR_CQ_SIZE(val);
 556                 qdma_writel(reg, block + FSL_QDMA_BSQMR);
 557         }
 558
 559         reg = qdma_readl(ctrl + FSL_QDMA_DMR);
 560         reg &= ~FSL_QDMA_DMR_DQD;
 561         qdma_writel(reg, ctrl + FSL_QDMA_DMR);
 562
 563         return 0;
 564 }
 565
 566 static void *
 567 fsl_qdma_prep_memcpy(void *fsl_chan, dma_addr_t dst,
 568                            dma_addr_t src, size_t len,
 569                            void *call_back,
 570                            void *param)
 571 {
 572         struct fsl_qdma_comp *fsl_comp;
 573
 574         fsl_comp =
 575         fsl_qdma_request_enqueue_desc((struct fsl_qdma_chan *)fsl_chan);
 576         if (!fsl_comp)
 577                 return NULL;
 578
 579         fsl_comp->qchan = fsl_chan;
 580         fsl_comp->call_back_func = call_back;
 581         fsl_comp->params = param;
 582
 583         fsl_qdma_comp_fill_memcpy(fsl_comp, dst, src, len);
 584         return (void *)fsl_comp;
 585 }
 586
 587 static int
 588 fsl_qdma_enqueue_desc(struct fsl_qdma_chan *fsl_chan,
 589                                   struct fsl_qdma_comp *fsl_comp,
 590                                   uint64_t flags)
 591 {
 592         struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
 593         void *block = fsl_queue->block_base;
 594         struct fsl_qdma_format *ccdf;
 595         u32 reg;
 596
 597         /* retrieve and store the register value in big endian
 598          * to avoid bits swap
 599          */
 600         reg = qdma_readl_be(block +
 601                          FSL_QDMA_BCQSR(fsl_queue->id));
 602         if (reg & (FSL_QDMA_BCQSR_QF_XOFF_BE))
 603                 return -1;
 604
 605         /* filling descriptor  command table */
 606         ccdf = (struct fsl_qdma_format *)fsl_queue->virt_head;
 607         qdma_desc_addr_set64(ccdf, fsl_comp->bus_addr + 16);
 608         qdma_ccdf_set_format(ccdf, qdma_ccdf_get_offset(fsl_comp->virt_addr));
 609         qdma_ccdf_set_ser(ccdf, qdma_ccdf_get_status(fsl_comp->virt_addr));
 610         fsl_comp->index = fsl_queue->virt_head - fsl_queue->cq;
 611         fsl_queue->virt_head++;
 612
 613         if (fsl_queue->virt_head == fsl_queue->cq + fsl_queue->n_cq)
 614                 fsl_queue->virt_head = fsl_queue->cq;
 615
 616         list_add_tail(&fsl_comp->list, &fsl_queue->comp_used);
 617
 618         if (flags == RTE_DMA_OP_FLAG_SUBMIT) {
 619                 reg = qdma_readl_be(block + FSL_QDMA_BCQMR(fsl_queue->id));
 620                 reg |= FSL_QDMA_BCQMR_EI_BE;
 621                 qdma_writel_be(reg, block + FSL_QDMA_BCQMR(fsl_queue->id));
 622                 fsl_queue->stats.submitted++;
 623         } else {
 624                 fsl_queue->pending++;
 625         }
 626         return fsl_comp->index;
 627 }
 628
 629 static int
 630 fsl_qdma_alloc_chan_resources(struct fsl_qdma_chan *fsl_chan)
 631 {
 632         struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
 633         struct fsl_qdma_engine *fsl_qdma = fsl_chan->qdma;
 634         int ret;
 635
 636         if (fsl_queue->count++)
 637                 goto finally;
 638
 639         INIT_LIST_HEAD(&fsl_queue->comp_free);
 640         INIT_LIST_HEAD(&fsl_queue->comp_used);
 641
 642         ret = fsl_qdma_pre_request_enqueue_comp_sd_desc(fsl_queue,
 643                                 FSL_QDMA_COMMAND_BUFFER_SIZE, 64);
 644         if (ret) {
 645                 DPAA_QDMA_ERR(
 646                         "failed to alloc dma buffer for comp descriptor\n");
 647                 goto exit;
 648         }
 649
 650 finally:
 651         return fsl_qdma->desc_allocated++;
 652
 653 exit:
 654         return -ENOMEM;
 655 }
 656
 657 static int
 658 dpaa_info_get(const struct rte_dma_dev *dev, struct rte_dma_info *dev_info,
 659               uint32_t info_sz)
 660 {
 661 #define DPAADMA_MAX_DESC        64
 662 #define DPAADMA_MIN_DESC        64
 663
 664         RTE_SET_USED(dev);
 665         RTE_SET_USED(info_sz);
 666
 667         dev_info->dev_capa = RTE_DMA_CAPA_MEM_TO_MEM |
 668                              RTE_DMA_CAPA_MEM_TO_DEV |
 669                              RTE_DMA_CAPA_DEV_TO_DEV |
 670                              RTE_DMA_CAPA_DEV_TO_MEM |
 671                              RTE_DMA_CAPA_SILENT |
 672                              RTE_DMA_CAPA_OPS_COPY;
 673         dev_info->max_vchans = 1;
 674         dev_info->max_desc = DPAADMA_MAX_DESC;
 675         dev_info->min_desc = DPAADMA_MIN_DESC;
 676
 677         return 0;
 678 }
 679
 680 static int
 681 dpaa_get_channel(struct fsl_qdma_engine *fsl_qdma,  uint16_t vchan)
 682 {
 683         u32 i, start, end;
 684         int ret;
 685
 686         start = fsl_qdma->free_block_id * QDMA_QUEUES;
 687         fsl_qdma->free_block_id++;
 688
 689         end = start + 1;
 690         for (i = start; i < end; i++) {
 691                 struct fsl_qdma_chan *fsl_chan = &fsl_qdma->chans[i];
 692
 693                 if (fsl_chan->free) {
 694                         fsl_chan->free = false;
 695                         ret = fsl_qdma_alloc_chan_resources(fsl_chan);
 696                         if (ret)
 697                                 return ret;
 698
 699                         fsl_qdma->vchan_map[vchan] = i;
 700                         return 0;
 701                 }
 702         }
 703
 704         return -1;
 705 }
 706
 707 static void
 708 dma_release(void *fsl_chan)
 709 {
 710         ((struct fsl_qdma_chan *)fsl_chan)->free = true;
 711         fsl_qdma_free_chan_resources((struct fsl_qdma_chan *)fsl_chan);
 712 }
 713
 714 static int
 715 dpaa_qdma_configure(__rte_unused struct rte_dma_dev *dmadev,
 716                     __rte_unused const struct rte_dma_conf *dev_conf,
 717                     __rte_unused uint32_t conf_sz)
 718 {
 719         return 0;
 720 }
 721
 722 static int
 723 dpaa_qdma_start(__rte_unused struct rte_dma_dev *dev)
 724 {
 725         return 0;
 726 }
 727
 728 static int
 729 dpaa_qdma_close(__rte_unused struct rte_dma_dev *dev)
 730 {
 731         return 0;
 732 }
 733
 734 static int
 735 dpaa_qdma_queue_setup(struct rte_dma_dev *dmadev,
 736                       uint16_t vchan,
 737                       __rte_unused const struct rte_dma_vchan_conf *conf,
 738                       __rte_unused uint32_t conf_sz)
 739 {
 740         struct fsl_qdma_engine *fsl_qdma = dmadev->data->dev_private;
 741
 742         return dpaa_get_channel(fsl_qdma, vchan);
 743 }
 744
 745 static int
 746 dpaa_qdma_submit(void *dev_private, uint16_t vchan)
 747 {
 748         struct fsl_qdma_engine *fsl_qdma = (struct fsl_qdma_engine *)dev_private;
 749         struct fsl_qdma_chan *fsl_chan =
 750                 &fsl_qdma->chans[fsl_qdma->vchan_map[vchan]];
 751         struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
 752         void *block = fsl_queue->block_base;
 753         u32 reg;
 754
 755         while (fsl_queue->pending) {
 756                 reg = qdma_readl_be(block + FSL_QDMA_BCQMR(fsl_queue->id));
 757                 reg |= FSL_QDMA_BCQMR_EI_BE;
 758                 qdma_writel_be(reg, block + FSL_QDMA_BCQMR(fsl_queue->id));
 759                 fsl_queue->pending--;
 760                 fsl_queue->stats.submitted++;
 761         }
 762
 763         return 0;
 764 }
 765
 766 static int
 767 dpaa_qdma_enqueue(void *dev_private, uint16_t vchan,
 768                   rte_iova_t src, rte_iova_t dst,
 769                   uint32_t length, uint64_t flags)
 770 {
 771         struct fsl_qdma_engine *fsl_qdma = (struct fsl_qdma_engine *)dev_private;
 772         struct fsl_qdma_chan *fsl_chan =
 773                 &fsl_qdma->chans[fsl_qdma->vchan_map[vchan]];
 774         int ret;
 775
 776         void *fsl_comp = NULL;
 777
 778         fsl_comp = fsl_qdma_prep_memcpy(fsl_chan,
 779                         (dma_addr_t)dst, (dma_addr_t)src,
 780                         length, NULL, NULL);
 781         if (!fsl_comp) {
 782                 DPAA_QDMA_DP_DEBUG("fsl_comp is NULL\n");
 783                 return -1;
 784         }
 785         ret = fsl_qdma_enqueue_desc(fsl_chan, fsl_comp, flags);
 786
 787         return ret;
 788 }
 789
 790 static uint16_t
 791 dpaa_qdma_dequeue_status(void *dev_private, uint16_t vchan,
 792                          const uint16_t nb_cpls, uint16_t *last_idx,
 793                          enum rte_dma_status_code *st)
 794 {
 795         struct fsl_qdma_engine *fsl_qdma = (struct fsl_qdma_engine *)dev_private;
 796         int id = (int)((fsl_qdma->vchan_map[vchan]) / QDMA_QUEUES);
 797         void *block;
 798         int intr;
 799         void *status = fsl_qdma->status_base;
 800         struct fsl_qdma_chan *fsl_chan =
 801                 &fsl_qdma->chans[fsl_qdma->vchan_map[vchan]];
 802         struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
 803
 804         intr = qdma_readl_be(status + FSL_QDMA_DEDR);
 805         if (intr) {
 806                 DPAA_QDMA_ERR("DMA transaction error! %x\n", intr);
 807                 intr = qdma_readl(status + FSL_QDMA_DECFDW0R);
 808                 DPAA_QDMA_INFO("reg FSL_QDMA_DECFDW0R %x\n", intr);
 809                 intr = qdma_readl(status + FSL_QDMA_DECFDW1R);
 810                 DPAA_QDMA_INFO("reg FSL_QDMA_DECFDW1R %x\n", intr);
 811                 intr = qdma_readl(status + FSL_QDMA_DECFDW2R);
 812                 DPAA_QDMA_INFO("reg FSL_QDMA_DECFDW2R %x\n", intr);
 813                 intr = qdma_readl(status + FSL_QDMA_DECFDW3R);
 814                 DPAA_QDMA_INFO("reg FSL_QDMA_DECFDW3R %x\n", intr);
 815                 intr = qdma_readl(status + FSL_QDMA_DECFQIDR);
 816                 DPAA_QDMA_INFO("reg FSL_QDMA_DECFQIDR %x\n", intr);
 817                 intr = qdma_readl(status + FSL_QDMA_DECBR);
 818                 DPAA_QDMA_INFO("reg FSL_QDMA_DECBR %x\n", intr);
 819                 qdma_writel(0xffffffff,
 820                             status + FSL_QDMA_DEDR);
 821                 intr = qdma_readl(status + FSL_QDMA_DEDR);
 822                 fsl_queue->stats.errors++;
 823         }
 824
 825         block = fsl_qdma->block_base +
 826                 FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, id);
 827
 828         intr = fsl_qdma_queue_transfer_complete(fsl_qdma, block, id, nb_cpls,
 829                                                 last_idx, st);
 830         fsl_queue->stats.completed += intr;
 831
 832         return intr;
 833 }
 834
 835
 836 static uint16_t
 837 dpaa_qdma_dequeue(void *dev_private,
 838                   uint16_t vchan, const uint16_t nb_cpls,
 839                   uint16_t *last_idx, bool *has_error)
 840 {
 841         struct fsl_qdma_engine *fsl_qdma = (struct fsl_qdma_engine *)dev_private;
 842         int id = (int)((fsl_qdma->vchan_map[vchan]) / QDMA_QUEUES);
 843         void *block;
 844         int intr;
 845         void *status = fsl_qdma->status_base;
 846         struct fsl_qdma_chan *fsl_chan =
 847                 &fsl_qdma->chans[fsl_qdma->vchan_map[vchan]];
 848         struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
 849
 850         intr = qdma_readl_be(status + FSL_QDMA_DEDR);
 851         if (intr) {
 852                 DPAA_QDMA_ERR("DMA transaction error! %x\n", intr);
 853                 intr = qdma_readl(status + FSL_QDMA_DECFDW0R);
 854                 DPAA_QDMA_INFO("reg FSL_QDMA_DECFDW0R %x\n", intr);
 855                 intr = qdma_readl(status + FSL_QDMA_DECFDW1R);
 856                 DPAA_QDMA_INFO("reg FSL_QDMA_DECFDW1R %x\n", intr);
 857                 intr = qdma_readl(status + FSL_QDMA_DECFDW2R);
 858                 DPAA_QDMA_INFO("reg FSL_QDMA_DECFDW2R %x\n", intr);
 859                 intr = qdma_readl(status + FSL_QDMA_DECFDW3R);
 860                 DPAA_QDMA_INFO("reg FSL_QDMA_DECFDW3R %x\n", intr);
 861                 intr = qdma_readl(status + FSL_QDMA_DECFQIDR);
 862                 DPAA_QDMA_INFO("reg FSL_QDMA_DECFQIDR %x\n", intr);
 863                 intr = qdma_readl(status + FSL_QDMA_DECBR);
 864                 DPAA_QDMA_INFO("reg FSL_QDMA_DECBR %x\n", intr);
 865                 qdma_writel(0xffffffff,
 866                             status + FSL_QDMA_DEDR);
 867                 intr = qdma_readl(status + FSL_QDMA_DEDR);
 868                 *has_error = true;
 869                 fsl_queue->stats.errors++;
 870         }
 871
 872         block = fsl_qdma->block_base +
 873                 FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, id);
 874
 875         intr = fsl_qdma_queue_transfer_complete(fsl_qdma, block, id, nb_cpls,
 876                                                 last_idx, NULL);
 877         fsl_queue->stats.completed += intr;
 878
 879         return intr;
 880 }
 881
 882 static int
 883 dpaa_qdma_stats_get(const struct rte_dma_dev *dmadev, uint16_t vchan,
 884                     struct rte_dma_stats *rte_stats, uint32_t size)
 885 {
 886         struct fsl_qdma_engine *fsl_qdma = dmadev->data->dev_private;
 887         struct fsl_qdma_chan *fsl_chan =
 888                 &fsl_qdma->chans[fsl_qdma->vchan_map[vchan]];
 889         struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
 890         struct rte_dma_stats *stats = &fsl_queue->stats;
 891
 892         if (size < sizeof(rte_stats))
 893                 return -EINVAL;
 894         if (rte_stats == NULL)
 895                 return -EINVAL;
 896
 897         *rte_stats = *stats;
 898
 899         return 0;
 900 }
 901
 902 static int
 903 dpaa_qdma_stats_reset(struct rte_dma_dev *dmadev, uint16_t vchan)
 904 {
 905         struct fsl_qdma_engine *fsl_qdma = dmadev->data->dev_private;
 906         struct fsl_qdma_chan *fsl_chan =
 907                 &fsl_qdma->chans[fsl_qdma->vchan_map[vchan]];
 908         struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
 909
 910         fsl_queue->stats = (struct rte_dma_stats){0};
 911
 912         return 0;
 913 }
 914
 915 static struct rte_dma_dev_ops dpaa_qdma_ops = {
 916         .dev_info_get             = dpaa_info_get,
 917         .dev_configure            = dpaa_qdma_configure,
 918         .dev_start                = dpaa_qdma_start,
 919         .dev_close                = dpaa_qdma_close,
 920         .vchan_setup              = dpaa_qdma_queue_setup,
 921         .stats_get                = dpaa_qdma_stats_get,
 922         .stats_reset              = dpaa_qdma_stats_reset,
 923 };
 924
 925 static int
 926 dpaa_qdma_init(struct rte_dma_dev *dmadev)
 927 {
 928         struct fsl_qdma_engine *fsl_qdma = dmadev->data->dev_private;
 929         struct fsl_qdma_chan *fsl_chan;
 930         uint64_t phys_addr;
 931         unsigned int len;
 932         int ccsr_qdma_fd;
 933         int regs_size;
 934         int ret;
 935         u32 i;
 936
 937         fsl_qdma->desc_allocated = 0;
 938         fsl_qdma->n_chans = VIRT_CHANNELS;
 939         fsl_qdma->n_queues = QDMA_QUEUES;
 940         fsl_qdma->num_blocks = QDMA_BLOCKS;
 941         fsl_qdma->block_offset = QDMA_BLOCK_OFFSET;
 942
 943         len = sizeof(*fsl_chan) * fsl_qdma->n_chans;
 944         fsl_qdma->chans = rte_zmalloc("qdma: fsl chans", len, 0);
 945         if (!fsl_qdma->chans)
 946                 return -1;
 947
 948         len = sizeof(struct fsl_qdma_queue *) * fsl_qdma->num_blocks;
 949         fsl_qdma->status = rte_zmalloc("qdma: fsl status", len, 0);
 950         if (!fsl_qdma->status) {
 951                 rte_free(fsl_qdma->chans);
 952                 return -1;
 953         }
 954
 955         for (i = 0; i < fsl_qdma->num_blocks; i++) {
 956                 rte_atomic32_init(&wait_task[i]);
 957                 fsl_qdma->status[i] = fsl_qdma_prep_status_queue();
 958                 if (!fsl_qdma->status[i])
 959                         goto err;
 960         }
 961
 962         ccsr_qdma_fd = open("/dev/mem", O_RDWR);
 963         if (unlikely(ccsr_qdma_fd < 0)) {
 964                 DPAA_QDMA_ERR("Can not open /dev/mem for qdma CCSR map");
 965                 goto err;
 966         }
 967
 968         regs_size = fsl_qdma->block_offset * (fsl_qdma->num_blocks + 2);
 969         phys_addr = QDMA_CCSR_BASE;
 970         fsl_qdma->ctrl_base = mmap(NULL, regs_size, PROT_READ |
 971                                          PROT_WRITE, MAP_SHARED,
 972                                          ccsr_qdma_fd, phys_addr);
 973
 974         close(ccsr_qdma_fd);
 975         if (fsl_qdma->ctrl_base == MAP_FAILED) {
 976                 DPAA_QDMA_ERR("Can not map CCSR base qdma: Phys: %08" PRIx64
 977                        "size %d\n", phys_addr, regs_size);
 978                 goto err;
 979         }
 980
 981         fsl_qdma->status_base = fsl_qdma->ctrl_base + QDMA_BLOCK_OFFSET;
 982         fsl_qdma->block_base = fsl_qdma->status_base + QDMA_BLOCK_OFFSET;
 983
 984         fsl_qdma->queue = fsl_qdma_alloc_queue_resources(fsl_qdma);
 985         if (!fsl_qdma->queue) {
 986                 munmap(fsl_qdma->ctrl_base, regs_size);
 987                 goto err;
 988         }
 989
 990         for (i = 0; i < fsl_qdma->n_chans; i++) {
 991                 struct fsl_qdma_chan *fsl_chan = &fsl_qdma->chans[i];
 992
 993                 fsl_chan->qdma = fsl_qdma;
 994                 fsl_chan->queue = fsl_qdma->queue + i % (fsl_qdma->n_queues *
 995                                                         fsl_qdma->num_blocks);
 996                 fsl_chan->free = true;
 997         }
 998
 999         ret = fsl_qdma_reg_init(fsl_qdma);
1000         if (ret) {
1001                 DPAA_QDMA_ERR("Can't Initialize the qDMA engine.\n");
1002                 munmap(fsl_qdma->ctrl_base, regs_size);
1003                 goto err;
1004         }
1005
1006         return 0;
1007
1008 err:
1009         rte_free(fsl_qdma->chans);
1010         rte_free(fsl_qdma->status);
1011
1012         return -1;
1013 }
1014
1015 static int
1016 dpaa_qdma_probe(__rte_unused struct rte_dpaa_driver *dpaa_drv,
1017                 struct rte_dpaa_device *dpaa_dev)
1018 {
1019         struct rte_dma_dev *dmadev;
1020         int ret;
1021
1022         dmadev = rte_dma_pmd_allocate(dpaa_dev->device.name,
1023                                       rte_socket_id(),
1024                                       sizeof(struct fsl_qdma_engine));
1025         if (!dmadev) {
1026                 DPAA_QDMA_ERR("Unable to allocate dmadevice");
1027                 return -EINVAL;
1028         }
1029
1030         dpaa_dev->dmadev = dmadev;
1031         dmadev->dev_ops = &dpaa_qdma_ops;
1032         dmadev->device = &dpaa_dev->device;
1033         dmadev->fp_obj->dev_private = dmadev->data->dev_private;
1034         dmadev->fp_obj->copy = dpaa_qdma_enqueue;
1035         dmadev->fp_obj->submit = dpaa_qdma_submit;
1036         dmadev->fp_obj->completed = dpaa_qdma_dequeue;
1037         dmadev->fp_obj->completed_status = dpaa_qdma_dequeue_status;
1038
1039         /* Invoke PMD device initialization function */
1040         ret = dpaa_qdma_init(dmadev);
1041         if (ret) {
1042                 (void)rte_dma_pmd_release(dpaa_dev->device.name);
1043                 return ret;
1044         }
1045
1046         dmadev->state = RTE_DMA_DEV_READY;
1047         return 0;
1048 }
1049
1050 static int
1051 dpaa_qdma_remove(struct rte_dpaa_device *dpaa_dev)
1052 {
1053         struct rte_dma_dev *dmadev = dpaa_dev->dmadev;
1054         struct fsl_qdma_engine *fsl_qdma = dmadev->data->dev_private;
1055         int i = 0, max = QDMA_QUEUES * QDMA_BLOCKS;
1056
1057         for (i = 0; i < max; i++) {
1058                 struct fsl_qdma_chan *fsl_chan = &fsl_qdma->chans[i];
1059
1060                 if (fsl_chan->free == false)
1061                         dma_release(fsl_chan);
1062         }
1063
1064         rte_free(fsl_qdma->status);
1065         rte_free(fsl_qdma->chans);
1066
1067         (void)rte_dma_pmd_release(dpaa_dev->device.name);
1068
1069         return 0;
1070 }
1071
1072 static struct rte_dpaa_driver rte_dpaa_qdma_pmd;
1073
1074 static struct rte_dpaa_driver rte_dpaa_qdma_pmd = {
1075         .drv_type = FSL_DPAA_QDMA,
1076         .probe = dpaa_qdma_probe,
1077         .remove = dpaa_qdma_remove,
1078 };
1079
1080 RTE_PMD_REGISTER_DPAA(dpaa_qdma, rte_dpaa_qdma_pmd);
1081 RTE_LOG_REGISTER_DEFAULT(dpaa_qdma_logtype, INFO);