1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2019 Marvell International Ltd.
9 #include <rte_bus_pci.h>
10 #include <rte_common.h>
12 #include <rte_lcore.h>
13 #include <rte_mempool.h>
15 #include <rte_rawdev.h>
16 #include <rte_rawdev_pmd.h>
18 #include <otx2_common.h>
20 #include "otx2_dpi_rawdev.h"
22 static const struct rte_pci_id pci_dma_map[] = {
24 RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM,
25 PCI_DEVID_OCTEONTX2_DPI_VF)
32 /* Enable/Disable DMA queue */
34 dma_engine_enb_dis(struct dpi_vf_s *dpivf, const bool enb)
37 otx2_write64(0x1, dpivf->vf_bar0 + DPI_VDMA_EN);
39 otx2_write64(0x0, dpivf->vf_bar0 + DPI_VDMA_EN);
41 return DPI_DMA_QUEUE_SUCCESS;
44 /* Free DMA Queue instruction buffers, and send close notification to PF */
46 dma_queue_finish(struct dpi_vf_s *dpivf)
48 uint32_t timeout = 0, sleep = 1;
51 /* Wait for SADDR to become idle */
52 reg = otx2_read64(dpivf->vf_bar0 + DPI_VDMA_SADDR);
53 while (!(reg & BIT_ULL(DPI_VDMA_SADDR_REQ_IDLE))) {
56 if (timeout >= DPI_QFINISH_TIMEOUT) {
57 otx2_dpi_dbg("Timeout!!! Closing Forcibly");
60 reg = otx2_read64(dpivf->vf_bar0 + DPI_VDMA_SADDR);
63 if (otx2_dpi_queue_close(dpivf->vf_id) < 0)
66 rte_mempool_put(dpivf->chunk_pool, dpivf->base_ptr);
67 dpivf->vf_bar0 = (uintptr_t)NULL;
69 return DPI_DMA_QUEUE_SUCCESS;
72 /* Write an arbitrary number of command words to a command queue */
73 static __rte_always_inline enum dpi_dma_queue_result_e
74 dma_queue_write(struct dpi_vf_s *dpi, uint16_t cmd_count, uint64_t *cmds)
76 if ((cmd_count < 1) || (cmd_count > 64))
77 return DPI_DMA_QUEUE_INVALID_PARAM;
80 return DPI_DMA_QUEUE_INVALID_PARAM;
82 /* Room available in the current buffer for the command */
83 if (dpi->index + cmd_count < dpi->pool_size_m1) {
84 uint64_t *ptr = dpi->base_ptr;
87 dpi->index += cmd_count;
95 /* Allocate new command buffer, return if failed */
96 if (rte_mempool_get(dpi->chunk_pool, &new_buffer) ||
98 return DPI_DMA_QUEUE_NO_MEMORY;
101 /* Figure out how many command words will fit in this buffer.
102 * One location will be needed for the next buffer pointer.
104 count = dpi->pool_size_m1 - dpi->index;
109 /* Chunk next ptr is 2DWORDs, second DWORD is reserved. */
110 *ptr++ = (uint64_t)new_buffer;
112 /* The current buffer is full and has a link to the next buffer.
113 * Time to write the rest of the commands into the new buffer.
115 dpi->base_ptr = new_buffer;
116 dpi->index = cmd_count;
120 /* queue index may greater than pool size */
121 if (dpi->index >= dpi->pool_size_m1) {
122 if (rte_mempool_get(dpi->chunk_pool, &new_buffer) ||
123 new_buffer == NULL) {
124 return DPI_DMA_QUEUE_NO_MEMORY;
126 /* Write next buffer address */
127 *ptr = (uint64_t)new_buffer;
128 dpi->base_ptr = new_buffer;
132 return DPI_DMA_QUEUE_SUCCESS;
135 /* Submit a DMA command to the DMA queues. */
136 static __rte_always_inline int
137 dma_queue_submit(struct rte_rawdev *dev, uint16_t cmd_count, uint64_t *cmds)
139 struct dpi_vf_s *dpivf = dev->dev_private;
140 enum dpi_dma_queue_result_e result;
142 result = dma_queue_write(dpivf, cmd_count, cmds);
144 if (likely(result == DPI_DMA_QUEUE_SUCCESS))
145 otx2_write64((uint64_t)cmd_count,
146 dpivf->vf_bar0 + DPI_VDMA_DBELL);
151 /* Enqueue buffers to DMA queue
152 * returns number of buffers enqueued successfully
155 otx2_dpi_rawdev_enqueue_bufs(struct rte_rawdev *dev,
156 struct rte_rawdev_buf **buffers,
157 unsigned int count, rte_rawdev_obj_t context)
159 struct dpi_dma_queue_ctx_s *ctx = (struct dpi_dma_queue_ctx_s *)context;
160 struct dpi_dma_buf_ptr_s *cmd;
163 for (c = 0; c < count; c++) {
164 uint64_t dpi_cmd[DPI_DMA_CMD_SIZE] = {0};
165 union dpi_dma_instr_hdr_u *hdr;
166 uint16_t index = 0, i;
168 hdr = (union dpi_dma_instr_hdr_u *)&dpi_cmd[0];
169 cmd = (struct dpi_dma_buf_ptr_s *)buffers[c]->buf_addr;
171 hdr->s.xtype = ctx->xtype & DPI_XTYPE_MASK;
172 hdr->s.pt = ctx->pt & DPI_HDR_PT_MASK;
173 /* Request initiated with byte write completion, but completion
174 * pointer not provided
176 if ((hdr->s.pt == DPI_HDR_PT_ZBW_CA ||
177 hdr->s.pt == DPI_HDR_PT_ZBW_NC) && cmd->comp_ptr == NULL)
180 cmd->comp_ptr->cdata = DPI_REQ_CDATA;
181 hdr->s.ptr = (uint64_t)cmd->comp_ptr;
182 hdr->s.deallocv = ctx->deallocv;
183 hdr->s.tt = ctx->tt & DPI_W0_TT_MASK;
184 hdr->s.grp = ctx->grp & DPI_W0_GRP_MASK;
186 /* If caller provides completion ring details, then only queue
187 * completion address for later polling.
190 ctx->c_ring->compl_data[ctx->c_ring->tail] =
192 STRM_INC(ctx->c_ring);
198 if (hdr->s.pt == DPI_HDR_PT_WQP)
199 hdr->s.ptr = hdr->s.ptr | DPI_HDR_PT_WQP_STATUSNC;
205 /* For inbound case, src pointers are last pointers.
206 * For all other cases, src pointers are first pointers.
208 if (ctx->xtype == DPI_XTYPE_INBOUND) {
209 hdr->s.nfst = cmd->wptr_cnt & DPI_MAX_POINTER;
210 hdr->s.nlst = cmd->rptr_cnt & DPI_MAX_POINTER;
211 for (i = 0; i < hdr->s.nfst; i++) {
212 dpi_cmd[index++] = cmd->wptr[i]->u[0];
213 dpi_cmd[index++] = cmd->wptr[i]->u[1];
215 for (i = 0; i < hdr->s.nlst; i++) {
216 dpi_cmd[index++] = cmd->rptr[i]->u[0];
217 dpi_cmd[index++] = cmd->rptr[i]->u[1];
220 hdr->s.nfst = cmd->rptr_cnt & DPI_MAX_POINTER;
221 hdr->s.nlst = cmd->wptr_cnt & DPI_MAX_POINTER;
222 for (i = 0; i < hdr->s.nfst; i++) {
223 dpi_cmd[index++] = cmd->rptr[i]->u[0];
224 dpi_cmd[index++] = cmd->rptr[i]->u[1];
226 for (i = 0; i < hdr->s.nlst; i++) {
227 dpi_cmd[index++] = cmd->wptr[i]->u[0];
228 dpi_cmd[index++] = cmd->wptr[i]->u[1];
231 if (dma_queue_submit(dev, index, dpi_cmd))
237 /* Check for command completion, returns number of commands completed */
239 otx2_dpi_rawdev_dequeue_bufs(struct rte_rawdev *dev __rte_unused,
240 struct rte_rawdev_buf **buffers,
241 unsigned int count, rte_rawdev_obj_t context)
243 struct dpi_dma_queue_ctx_s *ctx = (struct dpi_dma_queue_ctx_s *)context;
244 unsigned int i = 0, headp;
246 /* No completion ring to poll */
247 if (ctx->c_ring == NULL)
250 headp = ctx->c_ring->head;
251 for (i = 0; i < count && (headp != ctx->c_ring->tail); i++) {
252 struct dpi_dma_req_compl_s *comp_ptr =
253 ctx->c_ring->compl_data[headp];
258 /* Request Completed */
259 buffers[i] = (void *)comp_ptr;
260 headp = (headp + 1) % ctx->c_ring->max_cnt;
262 ctx->c_ring->head = headp;
268 otx2_dpi_rawdev_start(struct rte_rawdev *dev)
270 dev->started = DPI_QUEUE_START;
272 return DPI_DMA_QUEUE_SUCCESS;
276 otx2_dpi_rawdev_stop(struct rte_rawdev *dev)
278 dev->started = DPI_QUEUE_STOP;
282 otx2_dpi_rawdev_close(struct rte_rawdev *dev)
284 dma_engine_enb_dis(dev->dev_private, false);
285 dma_queue_finish(dev->dev_private);
287 return DPI_DMA_QUEUE_SUCCESS;
291 otx2_dpi_rawdev_reset(struct rte_rawdev *dev)
293 return dev ? DPI_QUEUE_STOP : DPI_QUEUE_START;
297 otx2_dpi_rawdev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config,
300 struct dpi_rawdev_conf_s *conf = config;
301 struct dpi_vf_s *dpivf = NULL;
306 if (conf == NULL || config_size != sizeof(*conf)) {
307 otx2_dpi_dbg("NULL or invalid configuration");
310 dpivf = (struct dpi_vf_s *)dev->dev_private;
311 dpivf->chunk_pool = conf->chunk_pool;
312 if (rte_mempool_get(conf->chunk_pool, &buf) || (buf == NULL)) {
313 otx2_err("Unable allocate buffer");
316 dpivf->base_ptr = buf;
317 otx2_write64(0x0, dpivf->vf_bar0 + DPI_VDMA_EN);
318 dpivf->pool_size_m1 = (DPI_CHUNK_SIZE >> 3) - 2;
319 pool = (uintptr_t)((struct rte_mempool *)conf->chunk_pool)->pool_id;
320 gaura = npa_lf_aura_handle_to_aura(pool);
321 otx2_write64(0, dpivf->vf_bar0 + DPI_VDMA_REQQ_CTL);
322 otx2_write64(((uint64_t)buf >> 7) << 7,
323 dpivf->vf_bar0 + DPI_VDMA_SADDR);
324 if (otx2_dpi_queue_open(dpivf->vf_id, DPI_CHUNK_SIZE, gaura) < 0) {
325 otx2_err("Unable to open DPI VF %d", dpivf->vf_id);
326 rte_mempool_put(conf->chunk_pool, buf);
329 dma_engine_enb_dis(dpivf, true);
331 return DPI_DMA_QUEUE_SUCCESS;
334 static const struct rte_rawdev_ops dpi_rawdev_ops = {
335 .dev_configure = otx2_dpi_rawdev_configure,
336 .dev_start = otx2_dpi_rawdev_start,
337 .dev_stop = otx2_dpi_rawdev_stop,
338 .dev_close = otx2_dpi_rawdev_close,
339 .dev_reset = otx2_dpi_rawdev_reset,
340 .enqueue_bufs = otx2_dpi_rawdev_enqueue_bufs,
341 .dequeue_bufs = otx2_dpi_rawdev_dequeue_bufs,
342 .dev_selftest = test_otx2_dma_rawdev,
346 otx2_dpi_rawdev_probe(struct rte_pci_driver *pci_drv __rte_unused,
347 struct rte_pci_device *pci_dev)
349 char name[RTE_RAWDEV_NAME_MAX_LEN];
350 struct dpi_vf_s *dpivf = NULL;
351 struct rte_rawdev *rawdev;
354 /* For secondary processes, the primary has done all the work */
355 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
356 return DPI_DMA_QUEUE_SUCCESS;
358 if (pci_dev->mem_resource[0].addr == NULL) {
359 otx2_dpi_dbg("Empty bars %p %p", pci_dev->mem_resource[0].addr,
360 pci_dev->mem_resource[2].addr);
364 memset(name, 0, sizeof(name));
365 snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "DPI:%x:%02x.%x",
366 pci_dev->addr.bus, pci_dev->addr.devid,
367 pci_dev->addr.function);
369 /* Allocate device structure */
370 rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct dpi_vf_s),
372 if (rawdev == NULL) {
373 otx2_err("Rawdev allocation failed");
377 rawdev->dev_ops = &dpi_rawdev_ops;
378 rawdev->device = &pci_dev->device;
379 rawdev->driver_name = pci_dev->driver->driver.name;
381 dpivf = rawdev->dev_private;
382 if (dpivf->state != DPI_QUEUE_STOP) {
383 otx2_dpi_dbg("Device already started!!!");
387 vf_id = ((pci_dev->addr.devid & 0x1F) << 3) |
388 (pci_dev->addr.function & 0x7);
390 dpivf->state = DPI_QUEUE_START;
391 dpivf->vf_id = vf_id;
392 dpivf->vf_bar0 = (uintptr_t)pci_dev->mem_resource[0].addr;
393 dpivf->vf_bar2 = (uintptr_t)pci_dev->mem_resource[2].addr;
395 return DPI_DMA_QUEUE_SUCCESS;
399 otx2_dpi_rawdev_remove(struct rte_pci_device *pci_dev)
401 char name[RTE_RAWDEV_NAME_MAX_LEN];
402 struct rte_rawdev *rawdev;
403 struct dpi_vf_s *dpivf;
405 if (pci_dev == NULL) {
406 otx2_dpi_dbg("Invalid pci_dev of the device!");
410 memset(name, 0, sizeof(name));
411 snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "DPI:%x:%02x.%x",
412 pci_dev->addr.bus, pci_dev->addr.devid,
413 pci_dev->addr.function);
415 rawdev = rte_rawdev_pmd_get_named_dev(name);
416 if (rawdev == NULL) {
417 otx2_dpi_dbg("Invalid device name (%s)", name);
421 dpivf = (struct dpi_vf_s *)rawdev->dev_private;
422 dma_engine_enb_dis(dpivf, false);
423 dma_queue_finish(dpivf);
425 /* rte_rawdev_close is called by pmd_release */
426 return rte_rawdev_pmd_release(rawdev);
429 static struct rte_pci_driver rte_dpi_rawdev_pmd = {
430 .id_table = pci_dma_map,
431 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_NEED_IOVA_AS_VA,
432 .probe = otx2_dpi_rawdev_probe,
433 .remove = otx2_dpi_rawdev_remove,
436 RTE_PMD_REGISTER_PCI(dpi_rawdev_pci_driver, rte_dpi_rawdev_pmd);
437 RTE_PMD_REGISTER_PCI_TABLE(dpi_rawdev_pci_driver, pci_dma_map);
438 RTE_PMD_REGISTER_KMOD_DEP(dpi_rawdev_pci_driver, "vfio-pci");