drivers/raw/octeontx2_dma/otx2_dpi_rawdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #include <string.h>
   6 #include <unistd.h>
   7
   8 #include <rte_bus.h>
   9 #include <rte_bus_pci.h>
  10 #include <rte_common.h>
  11 #include <rte_eal.h>
  12 #include <rte_lcore.h>
  13 #include <rte_mempool.h>
  14 #include <rte_pci.h>
  15 #include <rte_rawdev.h>
  16 #include <rte_rawdev_pmd.h>
  17
  18 #include <otx2_common.h>
  19
  20 #include "otx2_dpi_rawdev.h"
  21
  22 static const struct rte_pci_id pci_dma_map[] = {
  23         {
  24                 RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM,
  25                                PCI_DEVID_OCTEONTX2_DPI_VF)
  26         },
  27         {
  28                 .vendor_id = 0,
  29         },
  30 };
  31
  32 /* Enable/Disable DMA queue */
  33 static inline int
  34 dma_engine_enb_dis(struct dpi_vf_s *dpivf, const bool enb)
  35 {
  36         if (enb)
  37                 otx2_write64(0x1, dpivf->vf_bar0 + DPI_VDMA_EN);
  38         else
  39                 otx2_write64(0x0, dpivf->vf_bar0 + DPI_VDMA_EN);
  40
  41         return DPI_DMA_QUEUE_SUCCESS;
  42 }
  43
  44 /* Free DMA Queue instruction buffers, and send close notification to PF */
  45 static inline int
  46 dma_queue_finish(struct dpi_vf_s *dpivf)
  47 {
  48         uint32_t timeout = 0, sleep = 1;
  49         uint64_t reg = 0ULL;
  50
  51         /* Wait for SADDR to become idle */
  52         reg = otx2_read64(dpivf->vf_bar0 + DPI_VDMA_SADDR);
  53         while (!(reg & BIT_ULL(DPI_VDMA_SADDR_REQ_IDLE))) {
  54                 rte_delay_ms(sleep);
  55                 timeout++;
  56                 if (timeout >= DPI_QFINISH_TIMEOUT) {
  57                         otx2_dpi_dbg("Timeout!!! Closing Forcibly");
  58                         break;
  59                 }
  60                 reg = otx2_read64(dpivf->vf_bar0 + DPI_VDMA_SADDR);
  61         }
  62
  63         if (otx2_dpi_queue_close(dpivf->vf_id) < 0)
  64                 return -EACCES;
  65
  66         rte_mempool_put(dpivf->chunk_pool, dpivf->base_ptr);
  67         dpivf->vf_bar0 = (uintptr_t)NULL;
  68
  69         return DPI_DMA_QUEUE_SUCCESS;
  70 }
  71
  72 /* Write an arbitrary number of command words to a command queue */
  73 static __rte_always_inline enum dpi_dma_queue_result_e
  74 dma_queue_write(struct dpi_vf_s *dpi, uint16_t cmd_count, uint64_t *cmds)
  75 {
  76         if ((cmd_count < 1) || (cmd_count > 64))
  77                 return DPI_DMA_QUEUE_INVALID_PARAM;
  78
  79         if (cmds == NULL)
  80                 return DPI_DMA_QUEUE_INVALID_PARAM;
  81
  82         /* Room available in the current buffer for the command */
  83         if (dpi->index + cmd_count < dpi->pool_size_m1) {
  84                 uint64_t *ptr = dpi->base_ptr;
  85
  86                 ptr += dpi->index;
  87                 dpi->index += cmd_count;
  88                 while (cmd_count--)
  89                         *ptr++ = *cmds++;
  90         } else {
  91                 void *new_buffer;
  92                 uint64_t *ptr;
  93                 int count;
  94
  95                 /* Allocate new command buffer, return if failed */
  96                 if (rte_mempool_get(dpi->chunk_pool, &new_buffer) ||
  97                     new_buffer == NULL) {
  98                         return DPI_DMA_QUEUE_NO_MEMORY;
  99                 }
 100                 ptr = dpi->base_ptr;
 101                 /* Figure out how many command words will fit in this buffer.
 102                  * One location will be needed for the next buffer pointer.
 103                  **/
 104                 count = dpi->pool_size_m1 - dpi->index;
 105                 ptr += dpi->index;
 106                 cmd_count -= count;
 107                 while (count--)
 108                         *ptr++ = *cmds++;
 109                 /* Chunk next ptr is 2DWORDs, second DWORD is reserved. */
 110                 *ptr++ = (uint64_t)new_buffer;
 111                 *ptr   = 0;
 112                 /* The current buffer is full and has a link to the next buffer.
 113                  * Time to write the rest of the commands into the new buffer.
 114                  **/
 115                 dpi->base_ptr = new_buffer;
 116                 dpi->index = cmd_count;
 117                 ptr = new_buffer;
 118                 while (cmd_count--)
 119                         *ptr++ = *cmds++;
 120                 /* queue index may greater than pool size */
 121                 if (dpi->index >= dpi->pool_size_m1) {
 122                         if (rte_mempool_get(dpi->chunk_pool, &new_buffer) ||
 123                             new_buffer == NULL) {
 124                                 return DPI_DMA_QUEUE_NO_MEMORY;
 125                         }
 126                         /* Write next buffer address */
 127                         *ptr = (uint64_t)new_buffer;
 128                         dpi->base_ptr = new_buffer;
 129                         dpi->index = 0;
 130                 }
 131         }
 132         return DPI_DMA_QUEUE_SUCCESS;
 133 }
 134
 135 /* Submit a DMA command to the DMA queues. */
 136 static __rte_always_inline int
 137 dma_queue_submit(struct rte_rawdev *dev, uint16_t cmd_count, uint64_t *cmds)
 138 {
 139         struct dpi_vf_s *dpivf = dev->dev_private;
 140         enum dpi_dma_queue_result_e result;
 141
 142         result = dma_queue_write(dpivf, cmd_count, cmds);
 143         rte_wmb();
 144         if (likely(result == DPI_DMA_QUEUE_SUCCESS))
 145                 otx2_write64((uint64_t)cmd_count,
 146                              dpivf->vf_bar0 + DPI_VDMA_DBELL);
 147
 148         return result;
 149 }
 150
 151 /* Enqueue buffers to DMA queue
 152  * returns number of buffers enqueued successfully
 153  */
 154 static int
 155 otx2_dpi_rawdev_enqueue_bufs(struct rte_rawdev *dev,
 156                              struct rte_rawdev_buf **buffers,
 157                              unsigned int count, rte_rawdev_obj_t context)
 158 {
 159         struct dpi_dma_queue_ctx_s *ctx = (struct dpi_dma_queue_ctx_s *)context;
 160         struct dpi_dma_buf_ptr_s *cmd;
 161         uint32_t c = 0;
 162
 163         for (c = 0; c < count; c++) {
 164                 uint64_t dpi_cmd[DPI_DMA_CMD_SIZE] = {0};
 165                 union dpi_dma_instr_hdr_u *hdr;
 166                 uint16_t index = 0, i;
 167
 168                 hdr = (union dpi_dma_instr_hdr_u *)&dpi_cmd[0];
 169                 cmd = (struct dpi_dma_buf_ptr_s *)buffers[c]->buf_addr;
 170
 171                 hdr->s.xtype = ctx->xtype & DPI_XTYPE_MASK;
 172                 hdr->s.pt = ctx->pt & DPI_HDR_PT_MASK;
 173                 /* Request initiated with byte write completion, but completion
 174                  * pointer not provided
 175                  */
 176                 if ((hdr->s.pt == DPI_HDR_PT_ZBW_CA ||
 177                      hdr->s.pt == DPI_HDR_PT_ZBW_NC) && cmd->comp_ptr == NULL)
 178                         return c;
 179
 180                 cmd->comp_ptr->cdata = DPI_REQ_CDATA;
 181                 hdr->s.ptr = (uint64_t)cmd->comp_ptr;
 182                 hdr->s.deallocv = ctx->deallocv;
 183                 hdr->s.tt = ctx->tt & DPI_W0_TT_MASK;
 184                 hdr->s.grp = ctx->grp & DPI_W0_GRP_MASK;
 185
 186                 /* If caller provides completion ring details, then only queue
 187                  * completion address for later polling.
 188                  */
 189                 if (ctx->c_ring) {
 190                         ctx->c_ring->compl_data[ctx->c_ring->tail] =
 191                                                                  cmd->comp_ptr;
 192                         STRM_INC(ctx->c_ring);
 193                 }
 194
 195                 if (hdr->s.deallocv)
 196                         hdr->s.pvfe = 1;
 197
 198                 if (hdr->s.pt == DPI_HDR_PT_WQP)
 199                         hdr->s.ptr = hdr->s.ptr | DPI_HDR_PT_WQP_STATUSNC;
 200
 201                 index += 4;
 202                 hdr->s.fport = 0;
 203                 hdr->s.lport = 0;
 204                 if (ctx->xtype !=  DPI_XTYPE_INTERNAL_ONLY)
 205                         hdr->s.lport = ctx->pem_id;
 206
 207                 /* For inbound case, src pointers are last pointers.
 208                  * For all other cases, src pointers are first pointers.
 209                  */
 210                 if (ctx->xtype ==  DPI_XTYPE_INBOUND) {
 211                         hdr->s.nfst = cmd->wptr_cnt & DPI_MAX_POINTER;
 212                         hdr->s.nlst = cmd->rptr_cnt & DPI_MAX_POINTER;
 213                         for (i = 0; i < hdr->s.nfst; i++) {
 214                                 dpi_cmd[index++] = cmd->wptr[i]->u[0];
 215                                 dpi_cmd[index++] = cmd->wptr[i]->u[1];
 216                         }
 217                         for (i = 0; i < hdr->s.nlst; i++) {
 218                                 dpi_cmd[index++] = cmd->rptr[i]->u[0];
 219                                 dpi_cmd[index++] = cmd->rptr[i]->u[1];
 220                         }
 221                 } else {
 222                         hdr->s.nfst = cmd->rptr_cnt & DPI_MAX_POINTER;
 223                         hdr->s.nlst = cmd->wptr_cnt & DPI_MAX_POINTER;
 224                         for (i = 0; i < hdr->s.nfst; i++) {
 225                                 dpi_cmd[index++] = cmd->rptr[i]->u[0];
 226                                 dpi_cmd[index++] = cmd->rptr[i]->u[1];
 227                         }
 228                         for (i = 0; i < hdr->s.nlst; i++) {
 229                                 dpi_cmd[index++] = cmd->wptr[i]->u[0];
 230                                 dpi_cmd[index++] = cmd->wptr[i]->u[1];
 231                         }
 232                 }
 233                 if (dma_queue_submit(dev, index, dpi_cmd))
 234                         return c;
 235         }
 236         return c;
 237 }
 238
 239 /* Check for command completion, returns number of commands completed */
 240 static int
 241 otx2_dpi_rawdev_dequeue_bufs(struct rte_rawdev *dev __rte_unused,
 242                              struct rte_rawdev_buf **buffers,
 243                              unsigned int count, rte_rawdev_obj_t context)
 244 {
 245         struct dpi_dma_queue_ctx_s *ctx = (struct dpi_dma_queue_ctx_s *)context;
 246         unsigned int i = 0, headp;
 247
 248         /* No completion ring to poll */
 249         if (ctx->c_ring == NULL)
 250                 return 0;
 251
 252         headp = ctx->c_ring->head;
 253         for (i = 0; i < count && (headp != ctx->c_ring->tail); i++) {
 254                 struct dpi_dma_req_compl_s *comp_ptr =
 255                                          ctx->c_ring->compl_data[headp];
 256
 257                 if (comp_ptr->cdata)
 258                         break;
 259
 260                 /* Request Completed */
 261                 buffers[i] = (void *)comp_ptr;
 262                 headp = (headp + 1) % ctx->c_ring->max_cnt;
 263         }
 264         ctx->c_ring->head = headp;
 265
 266         return i;
 267 }
 268
 269 static int
 270 otx2_dpi_rawdev_start(struct rte_rawdev *dev)
 271 {
 272         dev->started = DPI_QUEUE_START;
 273
 274         return DPI_DMA_QUEUE_SUCCESS;
 275 }
 276
 277 static void
 278 otx2_dpi_rawdev_stop(struct rte_rawdev *dev)
 279 {
 280         dev->started = DPI_QUEUE_STOP;
 281 }
 282
 283 static int
 284 otx2_dpi_rawdev_close(struct rte_rawdev *dev)
 285 {
 286         dma_engine_enb_dis(dev->dev_private, false);
 287         dma_queue_finish(dev->dev_private);
 288
 289         return DPI_DMA_QUEUE_SUCCESS;
 290 }
 291
 292 static int
 293 otx2_dpi_rawdev_reset(struct rte_rawdev *dev)
 294 {
 295         return dev ? DPI_QUEUE_STOP : DPI_QUEUE_START;
 296 }
 297
 298 static int
 299 otx2_dpi_rawdev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config,
 300                 size_t config_size)
 301 {
 302         struct dpi_rawdev_conf_s *conf = config;
 303         struct dpi_vf_s *dpivf = NULL;
 304         void *buf = NULL;
 305         uintptr_t pool;
 306         uint32_t gaura;
 307
 308         if (conf == NULL || config_size != sizeof(*conf)) {
 309                 otx2_dpi_dbg("NULL or invalid configuration");
 310                 return -EINVAL;
 311         }
 312         dpivf = (struct dpi_vf_s *)dev->dev_private;
 313         dpivf->chunk_pool = conf->chunk_pool;
 314         if (rte_mempool_get(conf->chunk_pool, &buf) || (buf == NULL)) {
 315                 otx2_err("Unable allocate buffer");
 316                 return -ENODEV;
 317         }
 318         dpivf->base_ptr = buf;
 319         otx2_write64(0x0, dpivf->vf_bar0 + DPI_VDMA_EN);
 320         dpivf->pool_size_m1 = (DPI_CHUNK_SIZE >> 3) - 2;
 321         pool = (uintptr_t)((struct rte_mempool *)conf->chunk_pool)->pool_id;
 322         gaura = npa_lf_aura_handle_to_aura(pool);
 323         otx2_write64(0, dpivf->vf_bar0 + DPI_VDMA_REQQ_CTL);
 324         otx2_write64(((uint64_t)buf >> 7) << 7,
 325                      dpivf->vf_bar0 + DPI_VDMA_SADDR);
 326         if (otx2_dpi_queue_open(dpivf->vf_id, DPI_CHUNK_SIZE, gaura) < 0) {
 327                 otx2_err("Unable to open DPI VF %d", dpivf->vf_id);
 328                 rte_mempool_put(conf->chunk_pool, buf);
 329                 return -EACCES;
 330         }
 331         dma_engine_enb_dis(dpivf, true);
 332
 333         return DPI_DMA_QUEUE_SUCCESS;
 334 }
 335
 336 static const struct rte_rawdev_ops dpi_rawdev_ops = {
 337         .dev_configure = otx2_dpi_rawdev_configure,
 338         .dev_start = otx2_dpi_rawdev_start,
 339         .dev_stop = otx2_dpi_rawdev_stop,
 340         .dev_close = otx2_dpi_rawdev_close,
 341         .dev_reset = otx2_dpi_rawdev_reset,
 342         .enqueue_bufs = otx2_dpi_rawdev_enqueue_bufs,
 343         .dequeue_bufs = otx2_dpi_rawdev_dequeue_bufs,
 344         .dev_selftest = test_otx2_dma_rawdev,
 345 };
 346
 347 static int
 348 otx2_dpi_rawdev_probe(struct rte_pci_driver *pci_drv __rte_unused,
 349                       struct rte_pci_device *pci_dev)
 350 {
 351         char name[RTE_RAWDEV_NAME_MAX_LEN];
 352         struct dpi_vf_s *dpivf = NULL;
 353         struct rte_rawdev *rawdev;
 354         uint16_t vf_id;
 355
 356         /* For secondary processes, the primary has done all the work */
 357         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 358                 return DPI_DMA_QUEUE_SUCCESS;
 359
 360         if (pci_dev->mem_resource[0].addr == NULL) {
 361                 otx2_dpi_dbg("Empty bars %p %p", pci_dev->mem_resource[0].addr,
 362                              pci_dev->mem_resource[2].addr);
 363                 return -ENODEV;
 364         }
 365
 366         memset(name, 0, sizeof(name));
 367         snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "DPI:%x:%02x.%x",
 368                  pci_dev->addr.bus, pci_dev->addr.devid,
 369                  pci_dev->addr.function);
 370
 371         /* Allocate device structure */
 372         rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct dpi_vf_s),
 373                                          rte_socket_id());
 374         if (rawdev == NULL) {
 375                 otx2_err("Rawdev allocation failed");
 376                 return -EINVAL;
 377         }
 378
 379         rawdev->dev_ops = &dpi_rawdev_ops;
 380         rawdev->device = &pci_dev->device;
 381         rawdev->driver_name = pci_dev->driver->driver.name;
 382
 383         dpivf = rawdev->dev_private;
 384         if (dpivf->state != DPI_QUEUE_STOP) {
 385                 otx2_dpi_dbg("Device already started!!!");
 386                 return -ENODEV;
 387         }
 388
 389         vf_id = ((pci_dev->addr.devid & 0x1F) << 3) |
 390                  (pci_dev->addr.function & 0x7);
 391         vf_id -= 1;
 392         dpivf->state = DPI_QUEUE_START;
 393         dpivf->vf_id = vf_id;
 394         dpivf->vf_bar0 = (uintptr_t)pci_dev->mem_resource[0].addr;
 395         dpivf->vf_bar2 = (uintptr_t)pci_dev->mem_resource[2].addr;
 396
 397         return DPI_DMA_QUEUE_SUCCESS;
 398 }
 399
 400 static int
 401 otx2_dpi_rawdev_remove(struct rte_pci_device *pci_dev)
 402 {
 403         char name[RTE_RAWDEV_NAME_MAX_LEN];
 404         struct rte_rawdev *rawdev;
 405         struct dpi_vf_s *dpivf;
 406
 407         if (pci_dev == NULL) {
 408                 otx2_dpi_dbg("Invalid pci_dev of the device!");
 409                 return -EINVAL;
 410         }
 411
 412         memset(name, 0, sizeof(name));
 413         snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "DPI:%x:%02x.%x",
 414                  pci_dev->addr.bus, pci_dev->addr.devid,
 415                  pci_dev->addr.function);
 416
 417         rawdev = rte_rawdev_pmd_get_named_dev(name);
 418         if (rawdev == NULL) {
 419                 otx2_dpi_dbg("Invalid device name (%s)", name);
 420                 return -EINVAL;
 421         }
 422
 423         dpivf = (struct dpi_vf_s *)rawdev->dev_private;
 424         dma_engine_enb_dis(dpivf, false);
 425         dma_queue_finish(dpivf);
 426
 427         /* rte_rawdev_close is called by pmd_release */
 428         return rte_rawdev_pmd_release(rawdev);
 429 }
 430
 431 static struct rte_pci_driver rte_dpi_rawdev_pmd = {
 432         .id_table  = pci_dma_map,
 433         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_NEED_IOVA_AS_VA,
 434         .probe     = otx2_dpi_rawdev_probe,
 435         .remove    = otx2_dpi_rawdev_remove,
 436 };
 437
 438 RTE_PMD_REGISTER_PCI(dpi_rawdev_pci_driver, rte_dpi_rawdev_pmd);
 439 RTE_PMD_REGISTER_PCI_TABLE(dpi_rawdev_pci_driver, pci_dma_map);
 440 RTE_PMD_REGISTER_KMOD_DEP(dpi_rawdev_pci_driver, "vfio-pci");