drivers/raw/octeontx2_dma/otx2_dpi_rawdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #include <string.h>
   6 #include <unistd.h>
   7
   8 #include <rte_bus.h>
   9 #include <rte_bus_pci.h>
  10 #include <rte_common.h>
  11 #include <rte_eal.h>
  12 #include <rte_lcore.h>
  13 #include <rte_mempool.h>
  14 #include <rte_pci.h>
  15 #include <rte_rawdev.h>
  16 #include <rte_rawdev_pmd.h>
  17
  18 #include <otx2_common.h>
  19
  20 #include "otx2_dpi_rawdev.h"
  21
  22 static const struct rte_pci_id pci_dma_map[] = {
  23         {
  24                 RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM,
  25                                PCI_DEVID_OCTEONTX2_DPI_VF)
  26         },
  27         {
  28                 .vendor_id = 0,
  29         },
  30 };
  31
  32 /* Enable/Disable DMA queue */
  33 static inline int
  34 dma_engine_enb_dis(struct dpi_vf_s *dpivf, const bool enb)
  35 {
  36         if (enb)
  37                 otx2_write64(0x1, dpivf->vf_bar0 + DPI_VDMA_EN);
  38         else
  39                 otx2_write64(0x0, dpivf->vf_bar0 + DPI_VDMA_EN);
  40
  41         return DPI_DMA_QUEUE_SUCCESS;
  42 }
  43
  44 /* Free DMA Queue instruction buffers, and send close notification to PF */
  45 static inline int
  46 dma_queue_finish(struct dpi_vf_s *dpivf)
  47 {
  48         uint32_t timeout = 0, sleep = 1;
  49         uint64_t reg = 0ULL;
  50
  51         /* Wait for SADDR to become idle */
  52         reg = otx2_read64(dpivf->vf_bar0 + DPI_VDMA_SADDR);
  53         while (!(reg & BIT_ULL(DPI_VDMA_SADDR_REQ_IDLE))) {
  54                 rte_delay_ms(sleep);
  55                 timeout++;
  56                 if (timeout >= DPI_QFINISH_TIMEOUT) {
  57                         otx2_dpi_dbg("Timeout!!! Closing Forcibly");
  58                         break;
  59                 }
  60                 reg = otx2_read64(dpivf->vf_bar0 + DPI_VDMA_SADDR);
  61         }
  62
  63         if (otx2_dpi_queue_close(dpivf->vf_id) < 0)
  64                 return -EACCES;
  65
  66         rte_mempool_put(dpivf->chunk_pool, dpivf->base_ptr);
  67         dpivf->vf_bar0 = (uintptr_t)NULL;
  68
  69         return DPI_DMA_QUEUE_SUCCESS;
  70 }
  71
  72 /* Write an arbitrary number of command words to a command queue */
  73 static __rte_always_inline enum dpi_dma_queue_result_e
  74 dma_queue_write(struct dpi_vf_s *dpi, uint16_t cmd_count, uint64_t *cmds)
  75 {
  76         if ((cmd_count < 1) || (cmd_count > 64))
  77                 return DPI_DMA_QUEUE_INVALID_PARAM;
  78
  79         if (cmds == NULL)
  80                 return DPI_DMA_QUEUE_INVALID_PARAM;
  81
  82         /* Room available in the current buffer for the command */
  83         if (dpi->index + cmd_count < dpi->pool_size_m1) {
  84                 uint64_t *ptr = dpi->base_ptr;
  85
  86                 ptr += dpi->index;
  87                 dpi->index += cmd_count;
  88                 while (cmd_count--)
  89                         *ptr++ = *cmds++;
  90         } else {
  91                 void *new_buffer;
  92                 uint64_t *ptr;
  93                 int count;
  94
  95                 /* Allocate new command buffer, return if failed */
  96                 if (rte_mempool_get(dpi->chunk_pool, &new_buffer) ||
  97                     new_buffer == NULL) {
  98                         return DPI_DMA_QUEUE_NO_MEMORY;
  99                 }
 100                 ptr = dpi->base_ptr;
 101                 /* Figure out how many command words will fit in this buffer.
 102                  * One location will be needed for the next buffer pointer.
 103                  **/
 104                 count = dpi->pool_size_m1 - dpi->index;
 105                 ptr += dpi->index;
 106                 cmd_count -= count;
 107                 while (count--)
 108                         *ptr++ = *cmds++;
 109                 /* Chunk next ptr is 2DWORDs, second DWORD is reserved. */
 110                 *ptr++ = (uint64_t)new_buffer;
 111                 *ptr   = 0;
 112                 /* The current buffer is full and has a link to the next buffer.
 113                  * Time to write the rest of the commands into the new buffer.
 114                  **/
 115                 dpi->base_ptr = new_buffer;
 116                 dpi->index = cmd_count;
 117                 ptr = new_buffer;
 118                 while (cmd_count--)
 119                         *ptr++ = *cmds++;
 120                 /* queue index may greater than pool size */
 121                 if (dpi->index >= dpi->pool_size_m1) {
 122                         if (rte_mempool_get(dpi->chunk_pool, &new_buffer) ||
 123                             new_buffer == NULL) {
 124                                 return DPI_DMA_QUEUE_NO_MEMORY;
 125                         }
 126                         /* Write next buffer address */
 127                         *ptr = (uint64_t)new_buffer;
 128                         dpi->base_ptr = new_buffer;
 129                         dpi->index = 0;
 130                 }
 131         }
 132         return DPI_DMA_QUEUE_SUCCESS;
 133 }
 134
 135 /* Submit a DMA command to the DMA queues. */
 136 static __rte_always_inline int
 137 dma_queue_submit(struct rte_rawdev *dev, uint16_t cmd_count, uint64_t *cmds)
 138 {
 139         struct dpi_vf_s *dpivf = dev->dev_private;
 140         enum dpi_dma_queue_result_e result;
 141
 142         result = dma_queue_write(dpivf, cmd_count, cmds);
 143         rte_wmb();
 144         if (likely(result == DPI_DMA_QUEUE_SUCCESS))
 145                 otx2_write64((uint64_t)cmd_count,
 146                              dpivf->vf_bar0 + DPI_VDMA_DBELL);
 147
 148         return result;
 149 }
 150
 151 /* Enqueue buffers to DMA queue
 152  * returns number of buffers enqueued successfully
 153  */
 154 static int
 155 otx2_dpi_rawdev_enqueue_bufs(struct rte_rawdev *dev,
 156                              struct rte_rawdev_buf **buffers,
 157                              unsigned int count, rte_rawdev_obj_t context)
 158 {
 159         struct dpi_dma_queue_ctx_s *ctx = (struct dpi_dma_queue_ctx_s *)context;
 160         struct dpi_dma_buf_ptr_s *cmd;
 161         uint32_t c = 0;
 162
 163         for (c = 0; c < count; c++) {
 164                 uint64_t dpi_cmd[DPI_DMA_CMD_SIZE] = {0};
 165                 union dpi_dma_instr_hdr_u *hdr;
 166                 uint16_t index = 0, i;
 167
 168                 hdr = (union dpi_dma_instr_hdr_u *)&dpi_cmd[0];
 169                 cmd = (struct dpi_dma_buf_ptr_s *)buffers[c]->buf_addr;
 170
 171                 hdr->s.xtype = ctx->xtype & DPI_XTYPE_MASK;
 172                 hdr->s.pt = ctx->pt & DPI_HDR_PT_MASK;
 173                 /* Request initiated with byte write completion, but completion
 174                  * pointer not provided
 175                  */
 176                 if ((hdr->s.pt == DPI_HDR_PT_ZBW_CA ||
 177                      hdr->s.pt == DPI_HDR_PT_ZBW_NC) && cmd->comp_ptr == NULL)
 178                         return c;
 179
 180                 cmd->comp_ptr->cdata = DPI_REQ_CDATA;
 181                 hdr->s.ptr = (uint64_t)cmd->comp_ptr;
 182                 hdr->s.deallocv = ctx->deallocv;
 183                 hdr->s.tt = ctx->tt & DPI_W0_TT_MASK;
 184                 hdr->s.grp = ctx->grp & DPI_W0_GRP_MASK;
 185
 186                 /* If caller provides completion ring details, then only queue
 187                  * completion address for later polling.
 188                  */
 189                 if (ctx->c_ring) {
 190                         ctx->c_ring->compl_data[ctx->c_ring->tail] =
 191                                                                  cmd->comp_ptr;
 192                         STRM_INC(ctx->c_ring);
 193                 }
 194
 195                 if (hdr->s.deallocv)
 196                         hdr->s.pvfe = 1;
 197
 198                 if (hdr->s.pt == DPI_HDR_PT_WQP)
 199                         hdr->s.ptr = hdr->s.ptr | DPI_HDR_PT_WQP_STATUSNC;
 200
 201                 index += 4;
 202                 hdr->s.fport = 0;
 203                 hdr->s.lport = 0;
 204
 205                 /* For inbound case, src pointers are last pointers.
 206                  * For all other cases, src pointers are first pointers.
 207                  */
 208                 if (ctx->xtype ==  DPI_XTYPE_INBOUND) {
 209                         hdr->s.nfst = cmd->wptr_cnt & DPI_MAX_POINTER;
 210                         hdr->s.nlst = cmd->rptr_cnt & DPI_MAX_POINTER;
 211                         for (i = 0; i < hdr->s.nfst; i++) {
 212                                 dpi_cmd[index++] = cmd->wptr[i]->u[0];
 213                                 dpi_cmd[index++] = cmd->wptr[i]->u[1];
 214                         }
 215                         for (i = 0; i < hdr->s.nlst; i++) {
 216                                 dpi_cmd[index++] = cmd->rptr[i]->u[0];
 217                                 dpi_cmd[index++] = cmd->rptr[i]->u[1];
 218                         }
 219                 } else {
 220                         hdr->s.nfst = cmd->rptr_cnt & DPI_MAX_POINTER;
 221                         hdr->s.nlst = cmd->wptr_cnt & DPI_MAX_POINTER;
 222                         for (i = 0; i < hdr->s.nfst; i++) {
 223                                 dpi_cmd[index++] = cmd->rptr[i]->u[0];
 224                                 dpi_cmd[index++] = cmd->rptr[i]->u[1];
 225                         }
 226                         for (i = 0; i < hdr->s.nlst; i++) {
 227                                 dpi_cmd[index++] = cmd->wptr[i]->u[0];
 228                                 dpi_cmd[index++] = cmd->wptr[i]->u[1];
 229                         }
 230                 }
 231                 if (dma_queue_submit(dev, index, dpi_cmd))
 232                         return c;
 233         }
 234         return c;
 235 }
 236
 237 /* Check for command completion, returns number of commands completed */
 238 static int
 239 otx2_dpi_rawdev_dequeue_bufs(struct rte_rawdev *dev __rte_unused,
 240                              struct rte_rawdev_buf **buffers,
 241                              unsigned int count, rte_rawdev_obj_t context)
 242 {
 243         struct dpi_dma_queue_ctx_s *ctx = (struct dpi_dma_queue_ctx_s *)context;
 244         unsigned int i = 0, headp;
 245
 246         /* No completion ring to poll */
 247         if (ctx->c_ring == NULL)
 248                 return 0;
 249
 250         headp = ctx->c_ring->head;
 251         for (i = 0; i < count && (headp != ctx->c_ring->tail); i++) {
 252                 struct dpi_dma_req_compl_s *comp_ptr =
 253                                          ctx->c_ring->compl_data[headp];
 254
 255                 if (comp_ptr->cdata)
 256                         break;
 257
 258                 /* Request Completed */
 259                 buffers[i] = (void *)comp_ptr;
 260                 headp = (headp + 1) % ctx->c_ring->max_cnt;
 261         }
 262         ctx->c_ring->head = headp;
 263
 264         return i;
 265 }
 266
 267 static int
 268 otx2_dpi_rawdev_start(struct rte_rawdev *dev)
 269 {
 270         dev->started = DPI_QUEUE_START;
 271
 272         return DPI_DMA_QUEUE_SUCCESS;
 273 }
 274
 275 static void
 276 otx2_dpi_rawdev_stop(struct rte_rawdev *dev)
 277 {
 278         dev->started = DPI_QUEUE_STOP;
 279 }
 280
 281 static int
 282 otx2_dpi_rawdev_close(struct rte_rawdev *dev)
 283 {
 284         dma_engine_enb_dis(dev->dev_private, false);
 285         dma_queue_finish(dev->dev_private);
 286
 287         return DPI_DMA_QUEUE_SUCCESS;
 288 }
 289
 290 static int
 291 otx2_dpi_rawdev_reset(struct rte_rawdev *dev)
 292 {
 293         return dev ? DPI_QUEUE_STOP : DPI_QUEUE_START;
 294 }
 295
 296 static int
 297 otx2_dpi_rawdev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config)
 298 {
 299         struct dpi_rawdev_conf_s *conf = config;
 300         struct dpi_vf_s *dpivf = NULL;
 301         void *buf = NULL;
 302         uintptr_t pool;
 303         uint32_t gaura;
 304
 305         if (conf == NULL) {
 306                 otx2_dpi_dbg("NULL configuration");
 307                 return -EINVAL;
 308         }
 309         dpivf = (struct dpi_vf_s *)dev->dev_private;
 310         dpivf->chunk_pool = conf->chunk_pool;
 311         if (rte_mempool_get(conf->chunk_pool, &buf) || (buf == NULL)) {
 312                 otx2_err("Unable allocate buffer");
 313                 return -ENODEV;
 314         }
 315         dpivf->base_ptr = buf;
 316         otx2_write64(0x0, dpivf->vf_bar0 + DPI_VDMA_EN);
 317         dpivf->pool_size_m1 = (DPI_CHUNK_SIZE >> 3) - 2;
 318         pool = (uintptr_t)((struct rte_mempool *)conf->chunk_pool)->pool_id;
 319         gaura = npa_lf_aura_handle_to_aura(pool);
 320         otx2_write64(0, dpivf->vf_bar0 + DPI_VDMA_REQQ_CTL);
 321         otx2_write64(((uint64_t)buf >> 7) << 7,
 322                      dpivf->vf_bar0 + DPI_VDMA_SADDR);
 323         if (otx2_dpi_queue_open(dpivf->vf_id, DPI_CHUNK_SIZE, gaura) < 0) {
 324                 otx2_err("Unable to open DPI VF %d", dpivf->vf_id);
 325                 rte_mempool_put(conf->chunk_pool, buf);
 326                 return -EACCES;
 327         }
 328         dma_engine_enb_dis(dpivf, true);
 329
 330         return DPI_DMA_QUEUE_SUCCESS;
 331 }
 332
 333 static const struct rte_rawdev_ops dpi_rawdev_ops = {
 334         .dev_configure = otx2_dpi_rawdev_configure,
 335         .dev_start = otx2_dpi_rawdev_start,
 336         .dev_stop = otx2_dpi_rawdev_stop,
 337         .dev_close = otx2_dpi_rawdev_close,
 338         .dev_reset = otx2_dpi_rawdev_reset,
 339         .enqueue_bufs = otx2_dpi_rawdev_enqueue_bufs,
 340         .dequeue_bufs = otx2_dpi_rawdev_dequeue_bufs,
 341         .dev_selftest = test_otx2_dma_rawdev,
 342 };
 343
 344 static int
 345 otx2_dpi_rawdev_probe(struct rte_pci_driver *pci_drv __rte_unused,
 346                       struct rte_pci_device *pci_dev)
 347 {
 348         char name[RTE_RAWDEV_NAME_MAX_LEN];
 349         struct dpi_vf_s *dpivf = NULL;
 350         struct rte_rawdev *rawdev;
 351         uint16_t vf_id;
 352
 353         /* For secondary processes, the primary has done all the work */
 354         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 355                 return DPI_DMA_QUEUE_SUCCESS;
 356
 357         if (pci_dev->mem_resource[0].addr == NULL) {
 358                 otx2_dpi_dbg("Empty bars %p %p", pci_dev->mem_resource[0].addr,
 359                              pci_dev->mem_resource[2].addr);
 360                 return -ENODEV;
 361         }
 362
 363         memset(name, 0, sizeof(name));
 364         snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "DPI:%x:%02x.%x",
 365                  pci_dev->addr.bus, pci_dev->addr.devid,
 366                  pci_dev->addr.function);
 367
 368         /* Allocate device structure */
 369         rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct dpi_vf_s),
 370                                          rte_socket_id());
 371         if (rawdev == NULL) {
 372                 otx2_err("Rawdev allocation failed");
 373                 return -EINVAL;
 374         }
 375
 376         rawdev->dev_ops = &dpi_rawdev_ops;
 377         rawdev->device = &pci_dev->device;
 378         rawdev->driver_name = pci_dev->driver->driver.name;
 379
 380         dpivf = rawdev->dev_private;
 381         if (dpivf->state != DPI_QUEUE_STOP) {
 382                 otx2_dpi_dbg("Device already started!!!");
 383                 return -ENODEV;
 384         }
 385
 386         vf_id = ((pci_dev->addr.devid & 0x1F) << 3) |
 387                  (pci_dev->addr.function & 0x7);
 388         vf_id -= 1;
 389         dpivf->state = DPI_QUEUE_START;
 390         dpivf->vf_id = vf_id;
 391         dpivf->vf_bar0 = (uintptr_t)pci_dev->mem_resource[0].addr;
 392         dpivf->vf_bar2 = (uintptr_t)pci_dev->mem_resource[2].addr;
 393
 394         return DPI_DMA_QUEUE_SUCCESS;
 395 }
 396
 397 static int
 398 otx2_dpi_rawdev_remove(struct rte_pci_device *pci_dev)
 399 {
 400         char name[RTE_RAWDEV_NAME_MAX_LEN];
 401         struct rte_rawdev *rawdev;
 402         struct dpi_vf_s *dpivf;
 403
 404         if (pci_dev == NULL) {
 405                 otx2_dpi_dbg("Invalid pci_dev of the device!");
 406                 return -EINVAL;
 407         }
 408
 409         memset(name, 0, sizeof(name));
 410         snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "DPI:%x:%02x.%x",
 411                  pci_dev->addr.bus, pci_dev->addr.devid,
 412                  pci_dev->addr.function);
 413
 414         rawdev = rte_rawdev_pmd_get_named_dev(name);
 415         if (rawdev == NULL) {
 416                 otx2_dpi_dbg("Invalid device name (%s)", name);
 417                 return -EINVAL;
 418         }
 419
 420         dpivf = (struct dpi_vf_s *)rawdev->dev_private;
 421         dma_engine_enb_dis(dpivf, false);
 422         dma_queue_finish(dpivf);
 423
 424         /* rte_rawdev_close is called by pmd_release */
 425         return rte_rawdev_pmd_release(rawdev);
 426 }
 427
 428 static struct rte_pci_driver rte_dpi_rawdev_pmd = {
 429         .id_table  = pci_dma_map,
 430         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_NEED_IOVA_AS_VA,
 431         .probe     = otx2_dpi_rawdev_probe,
 432         .remove    = otx2_dpi_rawdev_remove,
 433 };
 434
 435 RTE_PMD_REGISTER_PCI(dpi_rawdev_pci_driver, rte_dpi_rawdev_pmd);
 436 RTE_PMD_REGISTER_PCI_TABLE(dpi_rawdev_pci_driver, pci_dma_map);
 437 RTE_PMD_REGISTER_KMOD_DEP(dpi_rawdev_pci_driver, "vfio-pci");