drivers/raw/octeontx2_dma/otx2_dpi_rawdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #include <string.h>
   6 #include <unistd.h>
   7
   8 #include <rte_bus.h>
   9 #include <rte_bus_pci.h>
  10 #include <rte_common.h>
  11 #include <rte_eal.h>
  12 #include <rte_lcore.h>
  13 #include <rte_mempool.h>
  14 #include <rte_pci.h>
  15 #include <rte_rawdev.h>
  16 #include <rte_rawdev_pmd.h>
  17
  18 #include <otx2_common.h>
  19
  20 #include "otx2_dpi_rawdev.h"
  21
  22 static const struct rte_pci_id pci_dma_map[] = {
  23         {
  24                 RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM,
  25                                PCI_DEVID_OCTEONTX2_DPI_VF)
  26         },
  27         {
  28                 .vendor_id = 0,
  29         },
  30 };
  31
  32 /* Enable/Disable DMA queue */
  33 static inline int
  34 dma_engine_enb_dis(struct dpi_vf_s *dpivf, const bool enb)
  35 {
  36         if (enb)
  37                 otx2_write64(0x1, dpivf->vf_bar0 + DPI_VDMA_EN);
  38         else
  39                 otx2_write64(0x0, dpivf->vf_bar0 + DPI_VDMA_EN);
  40
  41         return DPI_DMA_QUEUE_SUCCESS;
  42 }
  43
  44 /* Free DMA Queue instruction buffers, and send close notification to PF */
  45 static inline int
  46 dma_queue_finish(struct dpi_vf_s *dpivf)
  47 {
  48         uint32_t timeout = 0, sleep = 1;
  49         uint64_t reg = 0ULL;
  50
  51         /* Wait for SADDR to become idle */
  52         reg = otx2_read64(dpivf->vf_bar0 + DPI_VDMA_SADDR);
  53         while (!(reg & BIT_ULL(DPI_VDMA_SADDR_REQ_IDLE))) {
  54                 rte_delay_ms(sleep);
  55                 timeout++;
  56                 if (timeout >= DPI_QFINISH_TIMEOUT) {
  57                         otx2_dpi_dbg("Timeout!!! Closing Forcibly");
  58                         break;
  59                 }
  60                 reg = otx2_read64(dpivf->vf_bar0 + DPI_VDMA_SADDR);
  61         }
  62
  63         if (otx2_dpi_queue_close(dpivf->vf_id) < 0)
  64                 return -EACCES;
  65
  66         rte_mempool_put(dpivf->chunk_pool, dpivf->base_ptr);
  67         dpivf->vf_bar0 = (uintptr_t)NULL;
  68
  69         return DPI_DMA_QUEUE_SUCCESS;
  70 }
  71
  72 /* Write an arbitrary number of command words to a command queue */
  73 static __rte_always_inline enum dpi_dma_queue_result_e
  74 dma_queue_write(struct dpi_vf_s *dpi, uint16_t cmd_count, uint64_t *cmds)
  75 {
  76         if ((cmd_count < 1) || (cmd_count > 64))
  77                 return DPI_DMA_QUEUE_INVALID_PARAM;
  78
  79         if (cmds == NULL)
  80                 return DPI_DMA_QUEUE_INVALID_PARAM;
  81
  82         /* Room available in the current buffer for the command */
  83         if (dpi->index + cmd_count < dpi->pool_size_m1) {
  84                 uint64_t *ptr = dpi->base_ptr;
  85
  86                 ptr += dpi->index;
  87                 dpi->index += cmd_count;
  88                 while (cmd_count--)
  89                         *ptr++ = *cmds++;
  90         } else {
  91                 void *new_buffer;
  92                 uint64_t *ptr;
  93                 int count;
  94
  95                 /* Allocate new command buffer, return if failed */
  96                 if (rte_mempool_get(dpi->chunk_pool, &new_buffer) ||
  97                     new_buffer == NULL) {
  98                         return DPI_DMA_QUEUE_NO_MEMORY;
  99                 }
 100                 ptr = dpi->base_ptr;
 101                 /* Figure out how many command words will fit in this buffer.
 102                  * One location will be needed for the next buffer pointer.
 103                  **/
 104                 count = dpi->pool_size_m1 - dpi->index;
 105                 ptr += dpi->index;
 106                 cmd_count -= count;
 107                 while (count--)
 108                         *ptr++ = *cmds++;
 109                 /* Chunk next ptr is 2DWORDs, second DWORD is reserved. */
 110                 *ptr++ = (uint64_t)new_buffer;
 111                 *ptr   = 0;
 112                 /* The current buffer is full and has a link to the next buffer.
 113                  * Time to write the rest of the commands into the new buffer.
 114                  **/
 115                 dpi->base_ptr = new_buffer;
 116                 dpi->index = cmd_count;
 117                 ptr = new_buffer;
 118                 while (cmd_count--)
 119                         *ptr++ = *cmds++;
 120                 /* queue index may greater than pool size */
 121                 if (dpi->index >= dpi->pool_size_m1) {
 122                         if (rte_mempool_get(dpi->chunk_pool, &new_buffer) ||
 123                             new_buffer == NULL) {
 124                                 return DPI_DMA_QUEUE_NO_MEMORY;
 125                         }
 126                         /* Write next buffer address */
 127                         *ptr = (uint64_t)new_buffer;
 128                         dpi->base_ptr = new_buffer;
 129                         dpi->index = 0;
 130                 }
 131         }
 132         return DPI_DMA_QUEUE_SUCCESS;
 133 }
 134
 135 /* Submit a DMA command to the DMA queues. */
 136 static __rte_always_inline int
 137 dma_queue_submit(struct rte_rawdev *dev, uint16_t cmd_count, uint64_t *cmds)
 138 {
 139         struct dpi_vf_s *dpivf = dev->dev_private;
 140         enum dpi_dma_queue_result_e result;
 141
 142         result = dma_queue_write(dpivf, cmd_count, cmds);
 143         rte_wmb();
 144         if (likely(result == DPI_DMA_QUEUE_SUCCESS))
 145                 otx2_write64((uint64_t)cmd_count,
 146                              dpivf->vf_bar0 + DPI_VDMA_DBELL);
 147
 148         return result;
 149 }
 150
 151 /* Enqueue buffers to DMA queue
 152  * returns number of buffers enqueued successfully
 153  */
 154 static int
 155 otx2_dpi_rawdev_enqueue_bufs(struct rte_rawdev *dev,
 156                              struct rte_rawdev_buf **buffers,
 157                              unsigned int count, rte_rawdev_obj_t context)
 158 {
 159         struct dpi_dma_queue_ctx_s *ctx = (struct dpi_dma_queue_ctx_s *)context;
 160         struct dpi_dma_buf_ptr_s *cmd;
 161         uint32_t c = 0;
 162
 163         for (c = 0; c < count; c++) {
 164                 uint64_t dpi_cmd[DPI_DMA_CMD_SIZE] = {0};
 165                 union dpi_dma_instr_hdr_u *hdr;
 166                 uint16_t index = 0, i;
 167
 168                 hdr = (union dpi_dma_instr_hdr_u *)&dpi_cmd[0];
 169                 cmd = (struct dpi_dma_buf_ptr_s *)buffers[c]->buf_addr;
 170
 171                 hdr->s.xtype = ctx->xtype & DPI_XTYPE_MASK;
 172                 hdr->s.pt = ctx->pt & DPI_HDR_PT_MASK;
 173                 /* Request initiated with byte write completion, but completion
 174                  * pointer not provided
 175                  */
 176                 if ((hdr->s.pt == DPI_HDR_PT_ZBW_CA ||
 177                      hdr->s.pt == DPI_HDR_PT_ZBW_NC) && cmd->comp_ptr == NULL)
 178                         return c;
 179
 180                 cmd->comp_ptr->cdata = DPI_REQ_CDATA;
 181                 hdr->s.ptr = (uint64_t)cmd->comp_ptr;
 182                 hdr->s.deallocv = ctx->deallocv;
 183                 hdr->s.tt = ctx->tt & DPI_W0_TT_MASK;
 184                 hdr->s.grp = ctx->grp & DPI_W0_GRP_MASK;
 185
 186                 /* If caller provides completion ring details, then only queue
 187                  * completion address for later polling.
 188                  */
 189                 if (ctx->c_ring) {
 190                         ctx->c_ring->compl_data[ctx->c_ring->tail] =
 191                                                                  cmd->comp_ptr;
 192                         STRM_INC(ctx->c_ring);
 193                 }
 194
 195                 if (hdr->s.deallocv)
 196                         hdr->s.pvfe = 1;
 197
 198                 if (hdr->s.pt == DPI_HDR_PT_WQP)
 199                         hdr->s.ptr = hdr->s.ptr | DPI_HDR_PT_WQP_STATUSNC;
 200
 201                 index += 4;
 202                 hdr->s.fport = 0;
 203                 hdr->s.lport = 0;
 204
 205                 /* For inbound case, src pointers are last pointers.
 206                  * For all other cases, src pointers are first pointers.
 207                  */
 208                 if (ctx->xtype ==  DPI_XTYPE_INBOUND) {
 209                         hdr->s.nfst = cmd->wptr_cnt & DPI_MAX_POINTER;
 210                         hdr->s.nlst = cmd->rptr_cnt & DPI_MAX_POINTER;
 211                         for (i = 0; i < hdr->s.nfst; i++) {
 212                                 dpi_cmd[index++] = cmd->wptr[i]->u[0];
 213                                 dpi_cmd[index++] = cmd->wptr[i]->u[1];
 214                         }
 215                         for (i = 0; i < hdr->s.nlst; i++) {
 216                                 dpi_cmd[index++] = cmd->rptr[i]->u[0];
 217                                 dpi_cmd[index++] = cmd->rptr[i]->u[1];
 218                         }
 219                 } else {
 220                         hdr->s.nfst = cmd->rptr_cnt & DPI_MAX_POINTER;
 221                         hdr->s.nlst = cmd->wptr_cnt & DPI_MAX_POINTER;
 222                         for (i = 0; i < hdr->s.nfst; i++) {
 223                                 dpi_cmd[index++] = cmd->rptr[i]->u[0];
 224                                 dpi_cmd[index++] = cmd->rptr[i]->u[1];
 225                         }
 226                         for (i = 0; i < hdr->s.nlst; i++) {
 227                                 dpi_cmd[index++] = cmd->wptr[i]->u[0];
 228                                 dpi_cmd[index++] = cmd->wptr[i]->u[1];
 229                         }
 230                 }
 231                 if (dma_queue_submit(dev, index, dpi_cmd))
 232                         return c;
 233         }
 234         return c;
 235 }
 236
 237 static int
 238 otx2_dpi_rawdev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config)
 239 {
 240         struct dpi_rawdev_conf_s *conf = config;
 241         struct dpi_vf_s *dpivf = NULL;
 242         void *buf = NULL;
 243         uintptr_t pool;
 244         uint32_t gaura;
 245
 246         if (conf == NULL) {
 247                 otx2_dpi_dbg("NULL configuration");
 248                 return -EINVAL;
 249         }
 250         dpivf = (struct dpi_vf_s *)dev->dev_private;
 251         dpivf->chunk_pool = conf->chunk_pool;
 252         if (rte_mempool_get(conf->chunk_pool, &buf) || (buf == NULL)) {
 253                 otx2_err("Unable allocate buffer");
 254                 return -ENODEV;
 255         }
 256         dpivf->base_ptr = buf;
 257         otx2_write64(0x0, dpivf->vf_bar0 + DPI_VDMA_EN);
 258         dpivf->pool_size_m1 = (DPI_CHUNK_SIZE >> 3) - 2;
 259         pool = (uintptr_t)((struct rte_mempool *)conf->chunk_pool)->pool_id;
 260         gaura = npa_lf_aura_handle_to_aura(pool);
 261         otx2_write64(0, dpivf->vf_bar0 + DPI_VDMA_REQQ_CTL);
 262         otx2_write64(((uint64_t)buf >> 7) << 7,
 263                      dpivf->vf_bar0 + DPI_VDMA_SADDR);
 264         if (otx2_dpi_queue_open(dpivf->vf_id, DPI_CHUNK_SIZE, gaura) < 0) {
 265                 otx2_err("Unable to open DPI VF %d", dpivf->vf_id);
 266                 rte_mempool_put(conf->chunk_pool, buf);
 267                 return -EACCES;
 268         }
 269         dma_engine_enb_dis(dpivf, true);
 270
 271         return DPI_DMA_QUEUE_SUCCESS;
 272 }
 273
 274 static const struct rte_rawdev_ops dpi_rawdev_ops = {
 275         .dev_configure = otx2_dpi_rawdev_configure,
 276         .enqueue_bufs = otx2_dpi_rawdev_enqueue_bufs,
 277 };
 278
 279 static int
 280 otx2_dpi_rawdev_probe(struct rte_pci_driver *pci_drv __rte_unused,
 281                       struct rte_pci_device *pci_dev)
 282 {
 283         char name[RTE_RAWDEV_NAME_MAX_LEN];
 284         struct dpi_vf_s *dpivf = NULL;
 285         struct rte_rawdev *rawdev;
 286         uint16_t vf_id;
 287
 288         /* For secondary processes, the primary has done all the work */
 289         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 290                 return DPI_DMA_QUEUE_SUCCESS;
 291
 292         if (pci_dev->mem_resource[0].addr == NULL) {
 293                 otx2_dpi_dbg("Empty bars %p %p", pci_dev->mem_resource[0].addr,
 294                              pci_dev->mem_resource[2].addr);
 295                 return -ENODEV;
 296         }
 297
 298         memset(name, 0, sizeof(name));
 299         snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "DPI:%x:%02x.%x",
 300                  pci_dev->addr.bus, pci_dev->addr.devid,
 301                  pci_dev->addr.function);
 302
 303         /* Allocate device structure */
 304         rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct dpi_vf_s),
 305                                          rte_socket_id());
 306         if (rawdev == NULL) {
 307                 otx2_err("Rawdev allocation failed");
 308                 return -EINVAL;
 309         }
 310
 311         rawdev->dev_ops = &dpi_rawdev_ops;
 312         rawdev->device = &pci_dev->device;
 313         rawdev->driver_name = pci_dev->driver->driver.name;
 314
 315         dpivf = rawdev->dev_private;
 316         if (dpivf->state != DPI_QUEUE_STOP) {
 317                 otx2_dpi_dbg("Device already started!!!");
 318                 return -ENODEV;
 319         }
 320
 321         vf_id = ((pci_dev->addr.devid & 0x1F) << 3) |
 322                  (pci_dev->addr.function & 0x7);
 323         vf_id -= 1;
 324         dpivf->state = DPI_QUEUE_START;
 325         dpivf->vf_id = vf_id;
 326         dpivf->vf_bar0 = (uintptr_t)pci_dev->mem_resource[0].addr;
 327         dpivf->vf_bar2 = (uintptr_t)pci_dev->mem_resource[2].addr;
 328
 329         return DPI_DMA_QUEUE_SUCCESS;
 330 }
 331
 332 static int
 333 otx2_dpi_rawdev_remove(struct rte_pci_device *pci_dev)
 334 {
 335         char name[RTE_RAWDEV_NAME_MAX_LEN];
 336         struct rte_rawdev *rawdev;
 337         struct dpi_vf_s *dpivf;
 338
 339         if (pci_dev == NULL) {
 340                 otx2_dpi_dbg("Invalid pci_dev of the device!");
 341                 return -EINVAL;
 342         }
 343
 344         memset(name, 0, sizeof(name));
 345         snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "DPI:%x:%02x.%x",
 346                  pci_dev->addr.bus, pci_dev->addr.devid,
 347                  pci_dev->addr.function);
 348
 349         rawdev = rte_rawdev_pmd_get_named_dev(name);
 350         if (rawdev == NULL) {
 351                 otx2_dpi_dbg("Invalid device name (%s)", name);
 352                 return -EINVAL;
 353         }
 354
 355         dpivf = (struct dpi_vf_s *)rawdev->dev_private;
 356         dma_engine_enb_dis(dpivf, false);
 357         dma_queue_finish(dpivf);
 358
 359         /* rte_rawdev_close is called by pmd_release */
 360         return rte_rawdev_pmd_release(rawdev);
 361 }
 362
 363 static struct rte_pci_driver rte_dpi_rawdev_pmd = {
 364         .id_table  = pci_dma_map,
 365         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_IOVA_AS_VA,
 366         .probe     = otx2_dpi_rawdev_probe,
 367         .remove    = otx2_dpi_rawdev_remove,
 368 };
 369
 370 RTE_PMD_REGISTER_PCI(dpi_rawdev_pci_driver, rte_dpi_rawdev_pmd);
 371 RTE_PMD_REGISTER_PCI_TABLE(dpi_rawdev_pci_driver, pci_dma_map);
 372 RTE_PMD_REGISTER_KMOD_DEP(dpi_rawdev_pci_driver, "vfio-pci");