drivers/dma/cnxk/cnxk_dmadev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (C) 2021 Marvell International Ltd.
   3  */
   4
   5 #include <string.h>
   6 #include <unistd.h>
   7
   8 #include <rte_bus.h>
   9 #include <rte_bus_pci.h>
  10 #include <rte_common.h>
  11 #include <rte_eal.h>
  12 #include <rte_lcore.h>
  13 #include <rte_mempool.h>
  14 #include <rte_pci.h>
  15 #include <rte_dmadev.h>
  16 #include <rte_dmadev_pmd.h>
  17
  18 #include <roc_api.h>
  19 #include <cnxk_dmadev.h>
  20
  21 static int
  22 cnxk_dmadev_info_get(const struct rte_dma_dev *dev,
  23                      struct rte_dma_info *dev_info, uint32_t size)
  24 {
  25         RTE_SET_USED(dev);
  26         RTE_SET_USED(size);
  27
  28         dev_info->max_vchans = 1;
  29         dev_info->nb_vchans = 1;
  30         dev_info->dev_capa = RTE_DMA_CAPA_MEM_TO_MEM |
  31                 RTE_DMA_CAPA_MEM_TO_DEV | RTE_DMA_CAPA_DEV_TO_MEM |
  32                 RTE_DMA_CAPA_DEV_TO_DEV | RTE_DMA_CAPA_OPS_COPY |
  33                 RTE_DMA_CAPA_OPS_COPY_SG;
  34         dev_info->max_desc = DPI_MAX_DESC;
  35         dev_info->min_desc = 1;
  36         dev_info->max_sges = DPI_MAX_POINTER;
  37
  38         return 0;
  39 }
  40
  41 static int
  42 cnxk_dmadev_configure(struct rte_dma_dev *dev,
  43                       const struct rte_dma_conf *conf, uint32_t conf_sz)
  44 {
  45         struct cnxk_dpi_vf_s *dpivf = NULL;
  46         int rc = 0;
  47
  48         RTE_SET_USED(conf);
  49         RTE_SET_USED(conf);
  50         RTE_SET_USED(conf_sz);
  51         RTE_SET_USED(conf_sz);
  52         dpivf = dev->fp_obj->dev_private;
  53         rc = roc_dpi_configure(&dpivf->rdpi);
  54         if (rc < 0)
  55                 plt_err("DMA configure failed err = %d", rc);
  56
  57         return rc;
  58 }
  59
  60 static int
  61 cnxk_dmadev_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan,
  62                         const struct rte_dma_vchan_conf *conf,
  63                         uint32_t conf_sz)
  64 {
  65         struct cnxk_dpi_vf_s *dpivf = dev->fp_obj->dev_private;
  66         struct cnxk_dpi_compl_s *comp_data;
  67         union dpi_instr_hdr_s *header = &dpivf->conf.hdr;
  68         int i;
  69
  70         RTE_SET_USED(vchan);
  71         RTE_SET_USED(conf_sz);
  72
  73         header->s.pt = DPI_HDR_PT_ZBW_CA;
  74
  75         switch (conf->direction) {
  76         case RTE_DMA_DIR_DEV_TO_MEM:
  77                 header->s.xtype = DPI_XTYPE_INBOUND;
  78                 header->s.lport = conf->src_port.pcie.coreid;
  79                 header->s.fport = 0;
  80                 header->s.pvfe = 1;
  81                 break;
  82         case RTE_DMA_DIR_MEM_TO_DEV:
  83                 header->s.xtype = DPI_XTYPE_OUTBOUND;
  84                 header->s.lport = 0;
  85                 header->s.fport = conf->dst_port.pcie.coreid;
  86                 header->s.pvfe = 1;
  87                 break;
  88         case RTE_DMA_DIR_MEM_TO_MEM:
  89                 header->s.xtype = DPI_XTYPE_INTERNAL_ONLY;
  90                 header->s.lport = 0;
  91                 header->s.fport = 0;
  92                 header->s.pvfe = 0;
  93                 break;
  94         case RTE_DMA_DIR_DEV_TO_DEV:
  95                 header->s.xtype = DPI_XTYPE_EXTERNAL_ONLY;
  96                 header->s.lport = conf->src_port.pcie.coreid;
  97                 header->s.fport = conf->dst_port.pcie.coreid;
  98         };
  99
 100         for (i = 0; i < conf->nb_desc; i++) {
 101                 comp_data = rte_zmalloc(NULL, sizeof(*comp_data), 0);
 102                 if (comp_data == NULL) {
 103                         plt_err("Failed to allocate for comp_data");
 104                         return -ENOMEM;
 105                 }
 106                 dpivf->conf.c_desc.compl_ptr[i] = comp_data;
 107         };
 108         dpivf->conf.c_desc.max_cnt = DPI_MAX_DESC;
 109         dpivf->conf.c_desc.head = 0;
 110         dpivf->conf.c_desc.tail = 0;
 111
 112         return 0;
 113 }
 114
 115 static int
 116 cnxk_dmadev_start(struct rte_dma_dev *dev)
 117 {
 118         struct cnxk_dpi_vf_s *dpivf = dev->fp_obj->dev_private;
 119
 120         dpivf->desc_idx = 0;
 121         dpivf->num_words = 0;
 122         roc_dpi_enable(&dpivf->rdpi);
 123
 124         return 0;
 125 }
 126
 127 static int
 128 cnxk_dmadev_stop(struct rte_dma_dev *dev)
 129 {
 130         struct cnxk_dpi_vf_s *dpivf = dev->fp_obj->dev_private;
 131
 132         roc_dpi_disable(&dpivf->rdpi);
 133
 134         return 0;
 135 }
 136
 137 static int
 138 cnxk_dmadev_close(struct rte_dma_dev *dev)
 139 {
 140         struct cnxk_dpi_vf_s *dpivf = dev->fp_obj->dev_private;
 141
 142         roc_dpi_disable(&dpivf->rdpi);
 143         roc_dpi_dev_fini(&dpivf->rdpi);
 144
 145         return 0;
 146 }
 147
 148 static inline int
 149 __dpi_queue_write(struct roc_dpi *dpi, uint64_t *cmds, int cmd_count)
 150 {
 151         uint64_t *ptr = dpi->chunk_base;
 152
 153         if ((cmd_count < DPI_MIN_CMD_SIZE) || (cmd_count > DPI_MAX_CMD_SIZE) ||
 154             cmds == NULL)
 155                 return -EINVAL;
 156
 157         /*
 158          * Normally there is plenty of room in the current buffer for the
 159          * command
 160          */
 161         if (dpi->chunk_head + cmd_count < dpi->pool_size_m1) {
 162                 ptr += dpi->chunk_head;
 163                 dpi->chunk_head += cmd_count;
 164                 while (cmd_count--)
 165                         *ptr++ = *cmds++;
 166         } else {
 167                 int count;
 168                 uint64_t *new_buff = dpi->chunk_next;
 169
 170                 dpi->chunk_next =
 171                         (void *)roc_npa_aura_op_alloc(dpi->aura_handle, 0);
 172                 if (!dpi->chunk_next) {
 173                         plt_err("Failed to alloc next buffer from NPA");
 174                         return -ENOMEM;
 175                 }
 176
 177                 /*
 178                  * Figure out how many cmd words will fit in this buffer.
 179                  * One location will be needed for the next buffer pointer.
 180                  */
 181                 count = dpi->pool_size_m1 - dpi->chunk_head;
 182                 ptr += dpi->chunk_head;
 183                 cmd_count -= count;
 184                 while (count--)
 185                         *ptr++ = *cmds++;
 186
 187                 /*
 188                  * chunk next ptr is 2 DWORDS
 189                  * second DWORD is reserved.
 190                  */
 191                 *ptr++ = (uint64_t)new_buff;
 192                 *ptr = 0;
 193
 194                 /*
 195                  * The current buffer is full and has a link to the next
 196                  * buffers. Time to write the rest of the commands into the new
 197                  * buffer.
 198                  */
 199                 dpi->chunk_base = new_buff;
 200                 dpi->chunk_head = cmd_count;
 201                 ptr = new_buff;
 202                 while (cmd_count--)
 203                         *ptr++ = *cmds++;
 204
 205                 /* queue index may be greater than pool size */
 206                 if (dpi->chunk_head >= dpi->pool_size_m1) {
 207                         new_buff = dpi->chunk_next;
 208                         dpi->chunk_next =
 209                                 (void *)roc_npa_aura_op_alloc(dpi->aura_handle,
 210                                                               0);
 211                         if (!dpi->chunk_next) {
 212                                 plt_err("Failed to alloc next buffer from NPA");
 213                                 return -ENOMEM;
 214                         }
 215                         /* Write next buffer address */
 216                         *ptr = (uint64_t)new_buff;
 217                         dpi->chunk_base = new_buff;
 218                         dpi->chunk_head = 0;
 219                 }
 220         }
 221
 222         return 0;
 223 }
 224
 225 static int
 226 cnxk_dmadev_copy(void *dev_private, uint16_t vchan, rte_iova_t src,
 227                  rte_iova_t dst, uint32_t length, uint64_t flags)
 228 {
 229         struct cnxk_dpi_vf_s *dpivf = dev_private;
 230         union dpi_instr_hdr_s *header = &dpivf->conf.hdr;
 231         struct cnxk_dpi_compl_s *comp_ptr;
 232         rte_iova_t fptr, lptr;
 233         int num_words = 0;
 234         int rc;
 235
 236         RTE_SET_USED(vchan);
 237
 238         comp_ptr = dpivf->conf.c_desc.compl_ptr[dpivf->conf.c_desc.tail];
 239         comp_ptr->cdata = DPI_REQ_CDATA;
 240         header->s.ptr = (uint64_t)comp_ptr;
 241         STRM_INC(dpivf->conf.c_desc);
 242
 243         header->s.nfst = 1;
 244         header->s.nlst = 1;
 245
 246         /*
 247          * For inbound case, src pointers are last pointers.
 248          * For all other cases, src pointers are first pointers.
 249          */
 250         if (header->s.xtype == DPI_XTYPE_INBOUND) {
 251                 fptr = dst;
 252                 lptr = src;
 253         } else {
 254                 fptr = src;
 255                 lptr = dst;
 256         }
 257
 258         dpivf->cmd[0] = header->u[0];
 259         dpivf->cmd[1] = header->u[1];
 260         dpivf->cmd[2] = header->u[2];
 261         /* word3 is always 0 */
 262         num_words += 4;
 263         dpivf->cmd[num_words++] = length;
 264         dpivf->cmd[num_words++] = fptr;
 265         dpivf->cmd[num_words++] = length;
 266         dpivf->cmd[num_words++] = lptr;
 267
 268         rc = __dpi_queue_write(&dpivf->rdpi, dpivf->cmd, num_words);
 269         if (!rc) {
 270                 if (flags & RTE_DMA_OP_FLAG_SUBMIT) {
 271                         rte_wmb();
 272                         plt_write64(num_words,
 273                                     dpivf->rdpi.rbase + DPI_VDMA_DBELL);
 274                         dpivf->stats.submitted++;
 275                 }
 276                 dpivf->num_words += num_words;
 277         }
 278
 279         return dpivf->desc_idx++;
 280 }
 281
 282 static int
 283 cnxk_dmadev_copy_sg(void *dev_private, uint16_t vchan,
 284                     const struct rte_dma_sge *src,
 285                     const struct rte_dma_sge *dst,
 286                     uint16_t nb_src, uint16_t nb_dst, uint64_t flags)
 287 {
 288         struct cnxk_dpi_vf_s *dpivf = dev_private;
 289         union dpi_instr_hdr_s *header = &dpivf->conf.hdr;
 290         const struct rte_dma_sge *fptr, *lptr;
 291         struct cnxk_dpi_compl_s *comp_ptr;
 292         int num_words = 0;
 293         int i, rc;
 294
 295         RTE_SET_USED(vchan);
 296
 297         comp_ptr = dpivf->conf.c_desc.compl_ptr[dpivf->conf.c_desc.tail];
 298         comp_ptr->cdata = DPI_REQ_CDATA;
 299         header->s.ptr = (uint64_t)comp_ptr;
 300         STRM_INC(dpivf->conf.c_desc);
 301
 302         /*
 303          * For inbound case, src pointers are last pointers.
 304          * For all other cases, src pointers are first pointers.
 305          */
 306         if (header->s.xtype == DPI_XTYPE_INBOUND) {
 307                 header->s.nfst = nb_dst & 0xf;
 308                 header->s.nlst = nb_src & 0xf;
 309                 fptr = &dst[0];
 310                 lptr = &src[0];
 311         } else {
 312                 header->s.nfst = nb_src & 0xf;
 313                 header->s.nlst = nb_dst & 0xf;
 314                 fptr = &src[0];
 315                 lptr = &dst[0];
 316         }
 317
 318         dpivf->cmd[0] = header->u[0];
 319         dpivf->cmd[1] = header->u[1];
 320         dpivf->cmd[2] = header->u[2];
 321         num_words += 4;
 322         for (i = 0; i < header->s.nfst; i++) {
 323                 dpivf->cmd[num_words++] = (uint64_t)fptr->length;
 324                 dpivf->cmd[num_words++] = fptr->addr;
 325                 fptr++;
 326         }
 327
 328         for (i = 0; i < header->s.nlst; i++) {
 329                 dpivf->cmd[num_words++] = (uint64_t)lptr->length;
 330                 dpivf->cmd[num_words++] = lptr->addr;
 331                 lptr++;
 332         }
 333
 334         rc = __dpi_queue_write(&dpivf->rdpi, dpivf->cmd, num_words);
 335         if (!rc) {
 336                 if (flags & RTE_DMA_OP_FLAG_SUBMIT) {
 337                         rte_wmb();
 338                         plt_write64(num_words,
 339                                     dpivf->rdpi.rbase + DPI_VDMA_DBELL);
 340                         dpivf->stats.submitted += nb_src;
 341                 }
 342                 dpivf->num_words += num_words;
 343         }
 344
 345         return dpivf->desc_idx++;
 346 }
 347
 348 static uint16_t
 349 cnxk_dmadev_completed(void *dev_private, uint16_t vchan, const uint16_t nb_cpls,
 350                       uint16_t *last_idx, bool *has_error)
 351 {
 352         struct cnxk_dpi_vf_s *dpivf = dev_private;
 353         int cnt;
 354
 355         RTE_SET_USED(vchan);
 356         for (cnt = 0; cnt < nb_cpls; cnt++) {
 357                 struct cnxk_dpi_compl_s *comp_ptr =
 358                         dpivf->conf.c_desc.compl_ptr[cnt];
 359
 360                 if (comp_ptr->cdata) {
 361                         *has_error = 1;
 362                         dpivf->stats.errors++;
 363                         break;
 364                 }
 365         }
 366
 367         *last_idx = cnt - 1;
 368         dpivf->conf.c_desc.tail = cnt;
 369         dpivf->stats.completed += cnt;
 370
 371         return cnt;
 372 }
 373
 374 static uint16_t
 375 cnxk_dmadev_completed_status(void *dev_private, uint16_t vchan,
 376                              const uint16_t nb_cpls, uint16_t *last_idx,
 377                              enum rte_dma_status_code *status)
 378 {
 379         struct cnxk_dpi_vf_s *dpivf = dev_private;
 380         int cnt;
 381
 382         RTE_SET_USED(vchan);
 383         RTE_SET_USED(last_idx);
 384         for (cnt = 0; cnt < nb_cpls; cnt++) {
 385                 struct cnxk_dpi_compl_s *comp_ptr =
 386                         dpivf->conf.c_desc.compl_ptr[cnt];
 387                 status[cnt] = comp_ptr->cdata;
 388                 if (comp_ptr->cdata)
 389                         dpivf->stats.errors++;
 390         }
 391
 392         *last_idx = cnt - 1;
 393         dpivf->conf.c_desc.tail = 0;
 394         dpivf->stats.completed += cnt;
 395
 396         return cnt;
 397 }
 398
 399 static int
 400 cnxk_dmadev_submit(void *dev_private, uint16_t vchan __rte_unused)
 401 {
 402         struct cnxk_dpi_vf_s *dpivf = dev_private;
 403
 404         rte_wmb();
 405         plt_write64(dpivf->num_words, dpivf->rdpi.rbase + DPI_VDMA_DBELL);
 406         dpivf->stats.submitted++;
 407
 408         return 0;
 409 }
 410
 411 static int
 412 cnxk_stats_get(const struct rte_dma_dev *dev, uint16_t vchan,
 413                struct rte_dma_stats *rte_stats, uint32_t size)
 414 {
 415         struct cnxk_dpi_vf_s *dpivf = dev->fp_obj->dev_private;
 416         struct rte_dma_stats *stats = &dpivf->stats;
 417
 418         RTE_SET_USED(vchan);
 419
 420         if (size < sizeof(rte_stats))
 421                 return -EINVAL;
 422         if (rte_stats == NULL)
 423                 return -EINVAL;
 424
 425         *rte_stats = *stats;
 426         return 0;
 427 }
 428
 429 static int
 430 cnxk_stats_reset(struct rte_dma_dev *dev, uint16_t vchan __rte_unused)
 431 {
 432         struct cnxk_dpi_vf_s *dpivf = dev->fp_obj->dev_private;
 433
 434         dpivf->stats = (struct rte_dma_stats){0};
 435         return 0;
 436 }
 437
 438 static const struct rte_dma_dev_ops cnxk_dmadev_ops = {
 439         .dev_close = cnxk_dmadev_close,
 440         .dev_configure = cnxk_dmadev_configure,
 441         .dev_info_get = cnxk_dmadev_info_get,
 442         .dev_start = cnxk_dmadev_start,
 443         .dev_stop = cnxk_dmadev_stop,
 444         .stats_get = cnxk_stats_get,
 445         .stats_reset = cnxk_stats_reset,
 446         .vchan_setup = cnxk_dmadev_vchan_setup,
 447 };
 448
 449 static int
 450 cnxk_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused,
 451                   struct rte_pci_device *pci_dev)
 452 {
 453         struct cnxk_dpi_vf_s *dpivf = NULL;
 454         char name[RTE_DEV_NAME_MAX_LEN];
 455         struct rte_dma_dev *dmadev;
 456         struct roc_dpi *rdpi = NULL;
 457         int rc;
 458
 459         if (!pci_dev->mem_resource[0].addr)
 460                 return -ENODEV;
 461
 462         rc = roc_plt_init();
 463         if (rc) {
 464                 plt_err("Failed to initialize platform model, rc=%d", rc);
 465                 return rc;
 466         }
 467         memset(name, 0, sizeof(name));
 468         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 469
 470         dmadev = rte_dma_pmd_allocate(name, pci_dev->device.numa_node,
 471                                       sizeof(*dpivf));
 472         if (dmadev == NULL) {
 473                 plt_err("dma device allocation failed for %s", name);
 474                 return -ENOMEM;
 475         }
 476
 477         dpivf = dmadev->data->dev_private;
 478
 479         dmadev->device = &pci_dev->device;
 480         dmadev->fp_obj->dev_private = dpivf;
 481         dmadev->dev_ops = &cnxk_dmadev_ops;
 482
 483         dmadev->fp_obj->copy = cnxk_dmadev_copy;
 484         dmadev->fp_obj->copy_sg = cnxk_dmadev_copy_sg;
 485         dmadev->fp_obj->submit = cnxk_dmadev_submit;
 486         dmadev->fp_obj->completed = cnxk_dmadev_completed;
 487         dmadev->fp_obj->completed_status = cnxk_dmadev_completed_status;
 488
 489         rdpi = &dpivf->rdpi;
 490
 491         rdpi->pci_dev = pci_dev;
 492         rc = roc_dpi_dev_init(rdpi);
 493         if (rc < 0)
 494                 goto err_out_free;
 495
 496         return 0;
 497
 498 err_out_free:
 499         if (dmadev)
 500                 rte_dma_pmd_release(name);
 501
 502         return rc;
 503 }
 504
 505 static int
 506 cnxk_dmadev_remove(struct rte_pci_device *pci_dev)
 507 {
 508         char name[RTE_DEV_NAME_MAX_LEN];
 509
 510         memset(name, 0, sizeof(name));
 511         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 512
 513         return rte_dma_pmd_release(name);
 514 }
 515
 516 static const struct rte_pci_id cnxk_dma_pci_map[] = {
 517         {
 518                 RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM,
 519                                PCI_DEVID_CNXK_DPI_VF)
 520         },
 521         {
 522                 .vendor_id = 0,
 523         },
 524 };
 525
 526 static struct rte_pci_driver cnxk_dmadev = {
 527         .id_table  = cnxk_dma_pci_map,
 528         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_NEED_IOVA_AS_VA,
 529         .probe     = cnxk_dmadev_probe,
 530         .remove    = cnxk_dmadev_remove,
 531 };
 532
 533 RTE_PMD_REGISTER_PCI(cnxk_dmadev_pci_driver, cnxk_dmadev);
 534 RTE_PMD_REGISTER_PCI_TABLE(cnxk_dmadev_pci_driver, cnxk_dma_pci_map);
 535 RTE_PMD_REGISTER_KMOD_DEP(cnxk_dmadev_pci_driver, "vfio-pci");