drivers/dma/dpaa/dpaa_qdma.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2021 NXP
   3  */
   4
   5 #include <rte_dpaa_bus.h>
   6 #include <rte_dmadev_pmd.h>
   7
   8 #include "dpaa_qdma.h"
   9 #include "dpaa_qdma_logs.h"
  10
  11 static inline int
  12 ilog2(int x)
  13 {
  14         int log = 0;
  15
  16         x >>= 1;
  17
  18         while (x) {
  19                 log++;
  20                 x >>= 1;
  21         }
  22         return log;
  23 }
  24
  25 static u32
  26 qdma_readl(void *addr)
  27 {
  28         return QDMA_IN(addr);
  29 }
  30
  31 static void
  32 qdma_writel(u32 val, void *addr)
  33 {
  34         QDMA_OUT(addr, val);
  35 }
  36
  37 static void
  38 *dma_pool_alloc(int size, int aligned, dma_addr_t *phy_addr)
  39 {
  40         void *virt_addr;
  41
  42         virt_addr = rte_malloc("dma pool alloc", size, aligned);
  43         if (!virt_addr)
  44                 return NULL;
  45
  46         *phy_addr = rte_mem_virt2iova(virt_addr);
  47
  48         return virt_addr;
  49 }
  50
  51 static void
  52 dma_pool_free(void *addr)
  53 {
  54         rte_free(addr);
  55 }
  56
  57 static void
  58 fsl_qdma_free_chan_resources(struct fsl_qdma_chan *fsl_chan)
  59 {
  60         struct fsl_qdma_queue *fsl_queue = fsl_chan->queue;
  61         struct fsl_qdma_engine *fsl_qdma = fsl_chan->qdma;
  62         struct fsl_qdma_comp *comp_temp, *_comp_temp;
  63         int id;
  64
  65         if (--fsl_queue->count)
  66                 goto finally;
  67
  68         id = (fsl_qdma->block_base - fsl_queue->block_base) /
  69               fsl_qdma->block_offset;
  70
  71         while (rte_atomic32_read(&wait_task[id]) == 1)
  72                 rte_delay_us(QDMA_DELAY);
  73
  74         list_for_each_entry_safe(comp_temp, _comp_temp,
  75                                  &fsl_queue->comp_used, list) {
  76                 list_del(&comp_temp->list);
  77                 dma_pool_free(comp_temp->virt_addr);
  78                 dma_pool_free(comp_temp->desc_virt_addr);
  79                 rte_free(comp_temp);
  80         }
  81
  82         list_for_each_entry_safe(comp_temp, _comp_temp,
  83                                  &fsl_queue->comp_free, list) {
  84                 list_del(&comp_temp->list);
  85                 dma_pool_free(comp_temp->virt_addr);
  86                 dma_pool_free(comp_temp->desc_virt_addr);
  87                 rte_free(comp_temp);
  88         }
  89
  90 finally:
  91         fsl_qdma->desc_allocated--;
  92 }
  93
  94 static struct fsl_qdma_queue
  95 *fsl_qdma_alloc_queue_resources(struct fsl_qdma_engine *fsl_qdma)
  96 {
  97         struct fsl_qdma_queue *queue_head, *queue_temp;
  98         int len, i, j;
  99         int queue_num;
 100         int blocks;
 101         unsigned int queue_size[FSL_QDMA_QUEUE_MAX];
 102
 103         queue_num = fsl_qdma->n_queues;
 104         blocks = fsl_qdma->num_blocks;
 105
 106         len = sizeof(*queue_head) * queue_num * blocks;
 107         queue_head = rte_zmalloc("qdma: queue head", len, 0);
 108         if (!queue_head)
 109                 return NULL;
 110
 111         for (i = 0; i < FSL_QDMA_QUEUE_MAX; i++)
 112                 queue_size[i] = QDMA_QUEUE_SIZE;
 113
 114         for (j = 0; j < blocks; j++) {
 115                 for (i = 0; i < queue_num; i++) {
 116                         if (queue_size[i] > FSL_QDMA_CIRCULAR_DESC_SIZE_MAX ||
 117                             queue_size[i] < FSL_QDMA_CIRCULAR_DESC_SIZE_MIN) {
 118                                 DPAA_QDMA_ERR("Get wrong queue-sizes.\n");
 119                                 goto fail;
 120                         }
 121                         queue_temp = queue_head + i + (j * queue_num);
 122
 123                         queue_temp->cq =
 124                         dma_pool_alloc(sizeof(struct fsl_qdma_format) *
 125                                        queue_size[i],
 126                                        sizeof(struct fsl_qdma_format) *
 127                                        queue_size[i], &queue_temp->bus_addr);
 128
 129                         if (!queue_temp->cq)
 130                                 goto fail;
 131
 132                         memset(queue_temp->cq, 0x0, queue_size[i] *
 133                                sizeof(struct fsl_qdma_format));
 134
 135                         queue_temp->block_base = fsl_qdma->block_base +
 136                                 FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j);
 137                         queue_temp->n_cq = queue_size[i];
 138                         queue_temp->id = i;
 139                         queue_temp->count = 0;
 140                         queue_temp->pending = 0;
 141                         queue_temp->virt_head = queue_temp->cq;
 142
 143                 }
 144         }
 145         return queue_head;
 146
 147 fail:
 148         for (j = 0; j < blocks; j++) {
 149                 for (i = 0; i < queue_num; i++) {
 150                         queue_temp = queue_head + i + (j * queue_num);
 151                         dma_pool_free(queue_temp->cq);
 152                 }
 153         }
 154         rte_free(queue_head);
 155
 156         return NULL;
 157 }
 158
 159 static struct
 160 fsl_qdma_queue *fsl_qdma_prep_status_queue(void)
 161 {
 162         struct fsl_qdma_queue *status_head;
 163         unsigned int status_size;
 164
 165         status_size = QDMA_STATUS_SIZE;
 166         if (status_size > FSL_QDMA_CIRCULAR_DESC_SIZE_MAX ||
 167             status_size < FSL_QDMA_CIRCULAR_DESC_SIZE_MIN) {
 168                 DPAA_QDMA_ERR("Get wrong status_size.\n");
 169                 return NULL;
 170         }
 171
 172         status_head = rte_zmalloc("qdma: status head", sizeof(*status_head), 0);
 173         if (!status_head)
 174                 return NULL;
 175
 176         /*
 177          * Buffer for queue command
 178          */
 179         status_head->cq = dma_pool_alloc(sizeof(struct fsl_qdma_format) *
 180                                          status_size,
 181                                          sizeof(struct fsl_qdma_format) *
 182                                          status_size,
 183                                          &status_head->bus_addr);
 184
 185         if (!status_head->cq) {
 186                 rte_free(status_head);
 187                 return NULL;
 188         }
 189
 190         memset(status_head->cq, 0x0, status_size *
 191                sizeof(struct fsl_qdma_format));
 192         status_head->n_cq = status_size;
 193         status_head->virt_head = status_head->cq;
 194
 195         return status_head;
 196 }
 197
 198 static int
 199 fsl_qdma_halt(struct fsl_qdma_engine *fsl_qdma)
 200 {
 201         void *ctrl = fsl_qdma->ctrl_base;
 202         void *block;
 203         int i, count = RETRIES;
 204         unsigned int j;
 205         u32 reg;
 206
 207         /* Disable the command queue and wait for idle state. */
 208         reg = qdma_readl(ctrl + FSL_QDMA_DMR);
 209         reg |= FSL_QDMA_DMR_DQD;
 210         qdma_writel(reg, ctrl + FSL_QDMA_DMR);
 211         for (j = 0; j < fsl_qdma->num_blocks; j++) {
 212                 block = fsl_qdma->block_base +
 213                         FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j);
 214                 for (i = 0; i < FSL_QDMA_QUEUE_NUM_MAX; i++)
 215                         qdma_writel(0, block + FSL_QDMA_BCQMR(i));
 216         }
 217         while (true) {
 218                 reg = qdma_readl(ctrl + FSL_QDMA_DSR);
 219                 if (!(reg & FSL_QDMA_DSR_DB))
 220                         break;
 221                 if (count-- < 0)
 222                         return -EBUSY;
 223                 rte_delay_us(100);
 224         }
 225
 226         for (j = 0; j < fsl_qdma->num_blocks; j++) {
 227                 block = fsl_qdma->block_base +
 228                         FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j);
 229
 230                 /* Disable status queue. */
 231                 qdma_writel(0, block + FSL_QDMA_BSQMR);
 232
 233                 /*
 234                  * clear the command queue interrupt detect register for
 235                  * all queues.
 236                  */
 237                 qdma_writel(0xffffffff, block + FSL_QDMA_BCQIDR(0));
 238         }
 239
 240         return 0;
 241 }
 242
 243 static int
 244 fsl_qdma_reg_init(struct fsl_qdma_engine *fsl_qdma)
 245 {
 246         struct fsl_qdma_queue *fsl_queue = fsl_qdma->queue;
 247         struct fsl_qdma_queue *temp;
 248         void *ctrl = fsl_qdma->ctrl_base;
 249         void *block;
 250         u32 i, j;
 251         u32 reg;
 252         int ret, val;
 253
 254         /* Try to halt the qDMA engine first. */
 255         ret = fsl_qdma_halt(fsl_qdma);
 256         if (ret) {
 257                 DPAA_QDMA_ERR("DMA halt failed!");
 258                 return ret;
 259         }
 260
 261         for (j = 0; j < fsl_qdma->num_blocks; j++) {
 262                 block = fsl_qdma->block_base +
 263                         FSL_QDMA_BLOCK_BASE_OFFSET(fsl_qdma, j);
 264                 for (i = 0; i < fsl_qdma->n_queues; i++) {
 265                         temp = fsl_queue + i + (j * fsl_qdma->n_queues);
 266                         /*
 267                          * Initialize Command Queue registers to
 268                          * point to the first
 269                          * command descriptor in memory.
 270                          * Dequeue Pointer Address Registers
 271                          * Enqueue Pointer Address Registers
 272                          */
 273
 274                         qdma_writel(lower_32_bits(temp->bus_addr),
 275                                     block + FSL_QDMA_BCQDPA_SADDR(i));
 276                         qdma_writel(upper_32_bits(temp->bus_addr),
 277                                     block + FSL_QDMA_BCQEDPA_SADDR(i));
 278                         qdma_writel(lower_32_bits(temp->bus_addr),
 279                                     block + FSL_QDMA_BCQEPA_SADDR(i));
 280                         qdma_writel(upper_32_bits(temp->bus_addr),
 281                                     block + FSL_QDMA_BCQEEPA_SADDR(i));
 282
 283                         /* Initialize the queue mode. */
 284                         reg = FSL_QDMA_BCQMR_EN;
 285                         reg |= FSL_QDMA_BCQMR_CD_THLD(ilog2(temp->n_cq) - 4);
 286                         reg |= FSL_QDMA_BCQMR_CQ_SIZE(ilog2(temp->n_cq) - 6);
 287                         qdma_writel(reg, block + FSL_QDMA_BCQMR(i));
 288                 }
 289
 290                 /*
 291                  * Workaround for erratum: ERR010812.
 292                  * We must enable XOFF to avoid the enqueue rejection occurs.
 293                  * Setting SQCCMR ENTER_WM to 0x20.
 294                  */
 295
 296                 qdma_writel(FSL_QDMA_SQCCMR_ENTER_WM,
 297                             block + FSL_QDMA_SQCCMR);
 298
 299                 /*
 300                  * Initialize status queue registers to point to the first
 301                  * command descriptor in memory.
 302                  * Dequeue Pointer Address Registers
 303                  * Enqueue Pointer Address Registers
 304                  */
 305
 306                 qdma_writel(
 307                             upper_32_bits(fsl_qdma->status[j]->bus_addr),
 308                             block + FSL_QDMA_SQEEPAR);
 309                 qdma_writel(
 310                             lower_32_bits(fsl_qdma->status[j]->bus_addr),
 311                             block + FSL_QDMA_SQEPAR);
 312                 qdma_writel(
 313                             upper_32_bits(fsl_qdma->status[j]->bus_addr),
 314                             block + FSL_QDMA_SQEDPAR);
 315                 qdma_writel(
 316                             lower_32_bits(fsl_qdma->status[j]->bus_addr),
 317                             block + FSL_QDMA_SQDPAR);
 318                 /* Desiable status queue interrupt. */
 319
 320                 qdma_writel(0x0, block + FSL_QDMA_BCQIER(0));
 321                 qdma_writel(0x0, block + FSL_QDMA_BSQICR);
 322                 qdma_writel(0x0, block + FSL_QDMA_CQIER);
 323
 324                 /* Initialize the status queue mode. */
 325                 reg = FSL_QDMA_BSQMR_EN;
 326                 val = ilog2(fsl_qdma->status[j]->n_cq) - 6;
 327                 reg |= FSL_QDMA_BSQMR_CQ_SIZE(val);
 328                 qdma_writel(reg, block + FSL_QDMA_BSQMR);
 329         }
 330
 331         reg = qdma_readl(ctrl + FSL_QDMA_DMR);
 332         reg &= ~FSL_QDMA_DMR_DQD;
 333         qdma_writel(reg, ctrl + FSL_QDMA_DMR);
 334
 335         return 0;
 336 }
 337
 338 static void
 339 dma_release(void *fsl_chan)
 340 {
 341         ((struct fsl_qdma_chan *)fsl_chan)->free = true;
 342         fsl_qdma_free_chan_resources((struct fsl_qdma_chan *)fsl_chan);
 343 }
 344
 345 static int
 346 dpaa_qdma_init(struct rte_dma_dev *dmadev)
 347 {
 348         struct fsl_qdma_engine *fsl_qdma = dmadev->data->dev_private;
 349         struct fsl_qdma_chan *fsl_chan;
 350         uint64_t phys_addr;
 351         unsigned int len;
 352         int ccsr_qdma_fd;
 353         int regs_size;
 354         int ret;
 355         u32 i;
 356
 357         fsl_qdma->desc_allocated = 0;
 358         fsl_qdma->n_chans = VIRT_CHANNELS;
 359         fsl_qdma->n_queues = QDMA_QUEUES;
 360         fsl_qdma->num_blocks = QDMA_BLOCKS;
 361         fsl_qdma->block_offset = QDMA_BLOCK_OFFSET;
 362
 363         len = sizeof(*fsl_chan) * fsl_qdma->n_chans;
 364         fsl_qdma->chans = rte_zmalloc("qdma: fsl chans", len, 0);
 365         if (!fsl_qdma->chans)
 366                 return -1;
 367
 368         len = sizeof(struct fsl_qdma_queue *) * fsl_qdma->num_blocks;
 369         fsl_qdma->status = rte_zmalloc("qdma: fsl status", len, 0);
 370         if (!fsl_qdma->status) {
 371                 rte_free(fsl_qdma->chans);
 372                 return -1;
 373         }
 374
 375         for (i = 0; i < fsl_qdma->num_blocks; i++) {
 376                 rte_atomic32_init(&wait_task[i]);
 377                 fsl_qdma->status[i] = fsl_qdma_prep_status_queue();
 378                 if (!fsl_qdma->status[i])
 379                         goto err;
 380         }
 381
 382         ccsr_qdma_fd = open("/dev/mem", O_RDWR);
 383         if (unlikely(ccsr_qdma_fd < 0)) {
 384                 DPAA_QDMA_ERR("Can not open /dev/mem for qdma CCSR map");
 385                 goto err;
 386         }
 387
 388         regs_size = fsl_qdma->block_offset * (fsl_qdma->num_blocks + 2);
 389         phys_addr = QDMA_CCSR_BASE;
 390         fsl_qdma->ctrl_base = mmap(NULL, regs_size, PROT_READ |
 391                                          PROT_WRITE, MAP_SHARED,
 392                                          ccsr_qdma_fd, phys_addr);
 393
 394         close(ccsr_qdma_fd);
 395         if (fsl_qdma->ctrl_base == MAP_FAILED) {
 396                 DPAA_QDMA_ERR("Can not map CCSR base qdma: Phys: %08" PRIx64
 397                        "size %d\n", phys_addr, regs_size);
 398                 goto err;
 399         }
 400
 401         fsl_qdma->status_base = fsl_qdma->ctrl_base + QDMA_BLOCK_OFFSET;
 402         fsl_qdma->block_base = fsl_qdma->status_base + QDMA_BLOCK_OFFSET;
 403
 404         fsl_qdma->queue = fsl_qdma_alloc_queue_resources(fsl_qdma);
 405         if (!fsl_qdma->queue) {
 406                 munmap(fsl_qdma->ctrl_base, regs_size);
 407                 goto err;
 408         }
 409
 410         for (i = 0; i < fsl_qdma->n_chans; i++) {
 411                 struct fsl_qdma_chan *fsl_chan = &fsl_qdma->chans[i];
 412
 413                 fsl_chan->qdma = fsl_qdma;
 414                 fsl_chan->queue = fsl_qdma->queue + i % (fsl_qdma->n_queues *
 415                                                         fsl_qdma->num_blocks);
 416                 fsl_chan->free = true;
 417         }
 418
 419         ret = fsl_qdma_reg_init(fsl_qdma);
 420         if (ret) {
 421                 DPAA_QDMA_ERR("Can't Initialize the qDMA engine.\n");
 422                 munmap(fsl_qdma->ctrl_base, regs_size);
 423                 goto err;
 424         }
 425
 426         return 0;
 427
 428 err:
 429         rte_free(fsl_qdma->chans);
 430         rte_free(fsl_qdma->status);
 431
 432         return -1;
 433 }
 434
 435 static int
 436 dpaa_qdma_probe(__rte_unused struct rte_dpaa_driver *dpaa_drv,
 437                 struct rte_dpaa_device *dpaa_dev)
 438 {
 439         struct rte_dma_dev *dmadev;
 440         int ret;
 441
 442         dmadev = rte_dma_pmd_allocate(dpaa_dev->device.name,
 443                                       rte_socket_id(),
 444                                       sizeof(struct fsl_qdma_engine));
 445         if (!dmadev) {
 446                 DPAA_QDMA_ERR("Unable to allocate dmadevice");
 447                 return -EINVAL;
 448         }
 449
 450         dpaa_dev->dmadev = dmadev;
 451
 452         /* Invoke PMD device initialization function */
 453         ret = dpaa_qdma_init(dmadev);
 454         if (ret) {
 455                 (void)rte_dma_pmd_release(dpaa_dev->device.name);
 456                 return ret;
 457         }
 458
 459         dmadev->state = RTE_DMA_DEV_READY;
 460         return 0;
 461 }
 462
 463 static int
 464 dpaa_qdma_remove(struct rte_dpaa_device *dpaa_dev)
 465 {
 466         struct rte_dma_dev *dmadev = dpaa_dev->dmadev;
 467         struct fsl_qdma_engine *fsl_qdma = dmadev->data->dev_private;
 468         int i = 0, max = QDMA_QUEUES * QDMA_BLOCKS;
 469
 470         for (i = 0; i < max; i++) {
 471                 struct fsl_qdma_chan *fsl_chan = &fsl_qdma->chans[i];
 472
 473                 if (fsl_chan->free == false)
 474                         dma_release(fsl_chan);
 475         }
 476
 477         rte_free(fsl_qdma->status);
 478         rte_free(fsl_qdma->chans);
 479
 480         (void)rte_dma_pmd_release(dpaa_dev->device.name);
 481
 482         return 0;
 483 }
 484
 485 static struct rte_dpaa_driver rte_dpaa_qdma_pmd;
 486
 487 static struct rte_dpaa_driver rte_dpaa_qdma_pmd = {
 488         .drv_type = FSL_DPAA_QDMA,
 489         .probe = dpaa_qdma_probe,
 490         .remove = dpaa_qdma_remove,
 491 };
 492
 493 RTE_PMD_REGISTER_DPAA(dpaa_qdma, rte_dpaa_qdma_pmd);
 494 RTE_LOG_REGISTER_DEFAULT(dpaa_qdma_logtype, INFO);