drivers/dma/cnxk/cnxk_dmadev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (C) 2021 Marvell International Ltd.
   3  */
   4
   5 #include <string.h>
   6 #include <unistd.h>
   7
   8 #include <rte_bus.h>
   9 #include <rte_bus_pci.h>
  10 #include <rte_common.h>
  11 #include <rte_eal.h>
  12 #include <rte_lcore.h>
  13 #include <rte_mempool.h>
  14 #include <rte_pci.h>
  15 #include <rte_dmadev.h>
  16 #include <rte_dmadev_pmd.h>
  17
  18 #include <roc_api.h>
  19 #include <cnxk_dmadev.h>
  20
  21 static int
  22 cnxk_dmadev_info_get(const struct rte_dma_dev *dev,
  23                      struct rte_dma_info *dev_info, uint32_t size)
  24 {
  25         RTE_SET_USED(dev);
  26         RTE_SET_USED(size);
  27
  28         dev_info->max_vchans = 1;
  29         dev_info->nb_vchans = 1;
  30         dev_info->dev_capa = RTE_DMA_CAPA_MEM_TO_MEM |
  31                 RTE_DMA_CAPA_MEM_TO_DEV | RTE_DMA_CAPA_DEV_TO_MEM |
  32                 RTE_DMA_CAPA_DEV_TO_DEV | RTE_DMA_CAPA_OPS_COPY;
  33         dev_info->max_desc = DPI_MAX_DESC;
  34         dev_info->min_desc = 1;
  35         dev_info->max_sges = DPI_MAX_POINTER;
  36
  37         return 0;
  38 }
  39
  40 static int
  41 cnxk_dmadev_configure(struct rte_dma_dev *dev,
  42                       const struct rte_dma_conf *conf, uint32_t conf_sz)
  43 {
  44         struct cnxk_dpi_vf_s *dpivf = NULL;
  45         int rc = 0;
  46
  47         RTE_SET_USED(conf);
  48         RTE_SET_USED(conf);
  49         RTE_SET_USED(conf_sz);
  50         RTE_SET_USED(conf_sz);
  51         dpivf = dev->fp_obj->dev_private;
  52         rc = roc_dpi_configure(&dpivf->rdpi);
  53         if (rc < 0)
  54                 plt_err("DMA configure failed err = %d", rc);
  55
  56         return rc;
  57 }
  58
  59 static int
  60 cnxk_dmadev_vchan_setup(struct rte_dma_dev *dev, uint16_t vchan,
  61                         const struct rte_dma_vchan_conf *conf,
  62                         uint32_t conf_sz)
  63 {
  64         struct cnxk_dpi_vf_s *dpivf = dev->fp_obj->dev_private;
  65         struct cnxk_dpi_compl_s *comp_data;
  66         union dpi_instr_hdr_s *header = &dpivf->conf.hdr;
  67         int i;
  68
  69         RTE_SET_USED(vchan);
  70         RTE_SET_USED(conf_sz);
  71
  72         header->s.pt = DPI_HDR_PT_ZBW_CA;
  73
  74         switch (conf->direction) {
  75         case RTE_DMA_DIR_DEV_TO_MEM:
  76                 header->s.xtype = DPI_XTYPE_INBOUND;
  77                 header->s.lport = conf->src_port.pcie.coreid;
  78                 header->s.fport = 0;
  79                 header->s.pvfe = 1;
  80                 break;
  81         case RTE_DMA_DIR_MEM_TO_DEV:
  82                 header->s.xtype = DPI_XTYPE_OUTBOUND;
  83                 header->s.lport = 0;
  84                 header->s.fport = conf->dst_port.pcie.coreid;
  85                 header->s.pvfe = 1;
  86                 break;
  87         case RTE_DMA_DIR_MEM_TO_MEM:
  88                 header->s.xtype = DPI_XTYPE_INTERNAL_ONLY;
  89                 header->s.lport = 0;
  90                 header->s.fport = 0;
  91                 header->s.pvfe = 0;
  92                 break;
  93         case RTE_DMA_DIR_DEV_TO_DEV:
  94                 header->s.xtype = DPI_XTYPE_EXTERNAL_ONLY;
  95                 header->s.lport = conf->src_port.pcie.coreid;
  96                 header->s.fport = conf->dst_port.pcie.coreid;
  97         };
  98
  99         for (i = 0; i < conf->nb_desc; i++) {
 100                 comp_data = rte_zmalloc(NULL, sizeof(*comp_data), 0);
 101                 if (comp_data == NULL) {
 102                         plt_err("Failed to allocate for comp_data");
 103                         return -ENOMEM;
 104                 }
 105                 dpivf->conf.c_desc.compl_ptr[i] = comp_data;
 106         };
 107         dpivf->conf.c_desc.max_cnt = DPI_MAX_DESC;
 108         dpivf->conf.c_desc.head = 0;
 109         dpivf->conf.c_desc.tail = 0;
 110
 111         return 0;
 112 }
 113
 114 static int
 115 cnxk_dmadev_start(struct rte_dma_dev *dev)
 116 {
 117         struct cnxk_dpi_vf_s *dpivf = dev->fp_obj->dev_private;
 118
 119         dpivf->desc_idx = 0;
 120         dpivf->num_words = 0;
 121         roc_dpi_enable(&dpivf->rdpi);
 122
 123         return 0;
 124 }
 125
 126 static int
 127 cnxk_dmadev_stop(struct rte_dma_dev *dev)
 128 {
 129         struct cnxk_dpi_vf_s *dpivf = dev->fp_obj->dev_private;
 130
 131         roc_dpi_disable(&dpivf->rdpi);
 132
 133         return 0;
 134 }
 135
 136 static int
 137 cnxk_dmadev_close(struct rte_dma_dev *dev)
 138 {
 139         struct cnxk_dpi_vf_s *dpivf = dev->fp_obj->dev_private;
 140
 141         roc_dpi_disable(&dpivf->rdpi);
 142         roc_dpi_dev_fini(&dpivf->rdpi);
 143
 144         return 0;
 145 }
 146
 147 static inline int
 148 __dpi_queue_write(struct roc_dpi *dpi, uint64_t *cmds, int cmd_count)
 149 {
 150         uint64_t *ptr = dpi->chunk_base;
 151
 152         if ((cmd_count < DPI_MIN_CMD_SIZE) || (cmd_count > DPI_MAX_CMD_SIZE) ||
 153             cmds == NULL)
 154                 return -EINVAL;
 155
 156         /*
 157          * Normally there is plenty of room in the current buffer for the
 158          * command
 159          */
 160         if (dpi->chunk_head + cmd_count < dpi->pool_size_m1) {
 161                 ptr += dpi->chunk_head;
 162                 dpi->chunk_head += cmd_count;
 163                 while (cmd_count--)
 164                         *ptr++ = *cmds++;
 165         } else {
 166                 int count;
 167                 uint64_t *new_buff = dpi->chunk_next;
 168
 169                 dpi->chunk_next =
 170                         (void *)roc_npa_aura_op_alloc(dpi->aura_handle, 0);
 171                 if (!dpi->chunk_next) {
 172                         plt_err("Failed to alloc next buffer from NPA");
 173                         return -ENOMEM;
 174                 }
 175
 176                 /*
 177                  * Figure out how many cmd words will fit in this buffer.
 178                  * One location will be needed for the next buffer pointer.
 179                  */
 180                 count = dpi->pool_size_m1 - dpi->chunk_head;
 181                 ptr += dpi->chunk_head;
 182                 cmd_count -= count;
 183                 while (count--)
 184                         *ptr++ = *cmds++;
 185
 186                 /*
 187                  * chunk next ptr is 2 DWORDS
 188                  * second DWORD is reserved.
 189                  */
 190                 *ptr++ = (uint64_t)new_buff;
 191                 *ptr = 0;
 192
 193                 /*
 194                  * The current buffer is full and has a link to the next
 195                  * buffers. Time to write the rest of the commands into the new
 196                  * buffer.
 197                  */
 198                 dpi->chunk_base = new_buff;
 199                 dpi->chunk_head = cmd_count;
 200                 ptr = new_buff;
 201                 while (cmd_count--)
 202                         *ptr++ = *cmds++;
 203
 204                 /* queue index may be greater than pool size */
 205                 if (dpi->chunk_head >= dpi->pool_size_m1) {
 206                         new_buff = dpi->chunk_next;
 207                         dpi->chunk_next =
 208                                 (void *)roc_npa_aura_op_alloc(dpi->aura_handle,
 209                                                               0);
 210                         if (!dpi->chunk_next) {
 211                                 plt_err("Failed to alloc next buffer from NPA");
 212                                 return -ENOMEM;
 213                         }
 214                         /* Write next buffer address */
 215                         *ptr = (uint64_t)new_buff;
 216                         dpi->chunk_base = new_buff;
 217                         dpi->chunk_head = 0;
 218                 }
 219         }
 220
 221         return 0;
 222 }
 223
 224 static int
 225 cnxk_dmadev_copy(void *dev_private, uint16_t vchan, rte_iova_t src,
 226                  rte_iova_t dst, uint32_t length, uint64_t flags)
 227 {
 228         struct cnxk_dpi_vf_s *dpivf = dev_private;
 229         union dpi_instr_hdr_s *header = &dpivf->conf.hdr;
 230         struct cnxk_dpi_compl_s *comp_ptr;
 231         rte_iova_t fptr, lptr;
 232         int num_words = 0;
 233         int rc;
 234
 235         RTE_SET_USED(vchan);
 236
 237         comp_ptr = dpivf->conf.c_desc.compl_ptr[dpivf->conf.c_desc.tail];
 238         comp_ptr->cdata = DPI_REQ_CDATA;
 239         header->s.ptr = (uint64_t)comp_ptr;
 240         STRM_INC(dpivf->conf.c_desc);
 241
 242         header->s.nfst = 1;
 243         header->s.nlst = 1;
 244
 245         /*
 246          * For inbound case, src pointers are last pointers.
 247          * For all other cases, src pointers are first pointers.
 248          */
 249         if (header->s.xtype == DPI_XTYPE_INBOUND) {
 250                 fptr = dst;
 251                 lptr = src;
 252         } else {
 253                 fptr = src;
 254                 lptr = dst;
 255         }
 256
 257         dpivf->cmd[0] = header->u[0];
 258         dpivf->cmd[1] = header->u[1];
 259         dpivf->cmd[2] = header->u[2];
 260         /* word3 is always 0 */
 261         num_words += 4;
 262         dpivf->cmd[num_words++] = length;
 263         dpivf->cmd[num_words++] = fptr;
 264         dpivf->cmd[num_words++] = length;
 265         dpivf->cmd[num_words++] = lptr;
 266
 267         rc = __dpi_queue_write(&dpivf->rdpi, dpivf->cmd, num_words);
 268         if (!rc) {
 269                 if (flags & RTE_DMA_OP_FLAG_SUBMIT) {
 270                         rte_wmb();
 271                         plt_write64(num_words,
 272                                     dpivf->rdpi.rbase + DPI_VDMA_DBELL);
 273                 }
 274                 dpivf->num_words += num_words;
 275         }
 276
 277         return dpivf->desc_idx++;
 278 }
 279
 280 static uint16_t
 281 cnxk_dmadev_completed(void *dev_private, uint16_t vchan, const uint16_t nb_cpls,
 282                       uint16_t *last_idx, bool *has_error)
 283 {
 284         struct cnxk_dpi_vf_s *dpivf = dev_private;
 285         int cnt;
 286
 287         RTE_SET_USED(vchan);
 288         for (cnt = 0; cnt < nb_cpls; cnt++) {
 289                 struct cnxk_dpi_compl_s *comp_ptr =
 290                         dpivf->conf.c_desc.compl_ptr[cnt];
 291
 292                 if (comp_ptr->cdata) {
 293                         *has_error = 1;
 294                         break;
 295                 }
 296         }
 297
 298         *last_idx = cnt - 1;
 299         dpivf->conf.c_desc.tail = cnt;
 300
 301         return cnt;
 302 }
 303
 304 static uint16_t
 305 cnxk_dmadev_completed_status(void *dev_private, uint16_t vchan,
 306                              const uint16_t nb_cpls, uint16_t *last_idx,
 307                              enum rte_dma_status_code *status)
 308 {
 309         struct cnxk_dpi_vf_s *dpivf = dev_private;
 310         int cnt;
 311
 312         RTE_SET_USED(vchan);
 313         RTE_SET_USED(last_idx);
 314         for (cnt = 0; cnt < nb_cpls; cnt++) {
 315                 struct cnxk_dpi_compl_s *comp_ptr =
 316                         dpivf->conf.c_desc.compl_ptr[cnt];
 317                 status[cnt] = comp_ptr->cdata;
 318         }
 319
 320         *last_idx = cnt - 1;
 321         dpivf->conf.c_desc.tail = 0;
 322
 323         return cnt;
 324 }
 325
 326 static int
 327 cnxk_dmadev_submit(void *dev_private, uint16_t vchan __rte_unused)
 328 {
 329         struct cnxk_dpi_vf_s *dpivf = dev_private;
 330
 331         rte_wmb();
 332         plt_write64(dpivf->num_words, dpivf->rdpi.rbase + DPI_VDMA_DBELL);
 333
 334         return 0;
 335 }
 336
 337 static const struct rte_dma_dev_ops cnxk_dmadev_ops = {
 338         .dev_close = cnxk_dmadev_close,
 339         .dev_configure = cnxk_dmadev_configure,
 340         .dev_info_get = cnxk_dmadev_info_get,
 341         .dev_start = cnxk_dmadev_start,
 342         .dev_stop = cnxk_dmadev_stop,
 343         .vchan_setup = cnxk_dmadev_vchan_setup,
 344 };
 345
 346 static int
 347 cnxk_dmadev_probe(struct rte_pci_driver *pci_drv __rte_unused,
 348                   struct rte_pci_device *pci_dev)
 349 {
 350         struct cnxk_dpi_vf_s *dpivf = NULL;
 351         char name[RTE_DEV_NAME_MAX_LEN];
 352         struct rte_dma_dev *dmadev;
 353         struct roc_dpi *rdpi = NULL;
 354         int rc;
 355
 356         if (!pci_dev->mem_resource[0].addr)
 357                 return -ENODEV;
 358
 359         rc = roc_plt_init();
 360         if (rc) {
 361                 plt_err("Failed to initialize platform model, rc=%d", rc);
 362                 return rc;
 363         }
 364         memset(name, 0, sizeof(name));
 365         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 366
 367         dmadev = rte_dma_pmd_allocate(name, pci_dev->device.numa_node,
 368                                       sizeof(*dpivf));
 369         if (dmadev == NULL) {
 370                 plt_err("dma device allocation failed for %s", name);
 371                 return -ENOMEM;
 372         }
 373
 374         dpivf = dmadev->data->dev_private;
 375
 376         dmadev->device = &pci_dev->device;
 377         dmadev->fp_obj->dev_private = dpivf;
 378         dmadev->dev_ops = &cnxk_dmadev_ops;
 379
 380         dmadev->fp_obj->copy = cnxk_dmadev_copy;
 381         dmadev->fp_obj->submit = cnxk_dmadev_submit;
 382         dmadev->fp_obj->completed = cnxk_dmadev_completed;
 383         dmadev->fp_obj->completed_status = cnxk_dmadev_completed_status;
 384
 385         rdpi = &dpivf->rdpi;
 386
 387         rdpi->pci_dev = pci_dev;
 388         rc = roc_dpi_dev_init(rdpi);
 389         if (rc < 0)
 390                 goto err_out_free;
 391
 392         return 0;
 393
 394 err_out_free:
 395         if (dmadev)
 396                 rte_dma_pmd_release(name);
 397
 398         return rc;
 399 }
 400
 401 static int
 402 cnxk_dmadev_remove(struct rte_pci_device *pci_dev)
 403 {
 404         char name[RTE_DEV_NAME_MAX_LEN];
 405
 406         memset(name, 0, sizeof(name));
 407         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 408
 409         return rte_dma_pmd_release(name);
 410 }
 411
 412 static const struct rte_pci_id cnxk_dma_pci_map[] = {
 413         {
 414                 RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM,
 415                                PCI_DEVID_CNXK_DPI_VF)
 416         },
 417         {
 418                 .vendor_id = 0,
 419         },
 420 };
 421
 422 static struct rte_pci_driver cnxk_dmadev = {
 423         .id_table  = cnxk_dma_pci_map,
 424         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_NEED_IOVA_AS_VA,
 425         .probe     = cnxk_dmadev_probe,
 426         .remove    = cnxk_dmadev_remove,
 427 };
 428
 429 RTE_PMD_REGISTER_PCI(cnxk_dmadev_pci_driver, cnxk_dmadev);
 430 RTE_PMD_REGISTER_PCI_TABLE(cnxk_dmadev_pci_driver, cnxk_dma_pci_map);
 431 RTE_PMD_REGISTER_KMOD_DEP(cnxk_dmadev_pci_driver, "vfio-pci");