1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017 Cavium, Inc
10 #include <rte_cycles.h>
11 #include <rte_malloc.h>
12 #include <rte_memory.h>
13 #include <rte_bus_pci.h>
14 #include <rte_spinlock.h>
16 #include "../octeontx_logs.h"
17 #include "octeontx_io.h"
18 #include "octeontx_pkovf.h"
20 struct octeontx_pko_iomem {
26 #define PKO_IOMEM_NULL (struct octeontx_pko_iomem){0, 0, 0}
30 struct octeontx_pko_fc_ctl_s {
32 int64_t padding[(PKO_DQ_FC_STRIDE / 8) - 1];
35 struct octeontx_pkovf {
43 struct octeontx_pko_vf_ctl_s {
45 uint16_t global_domain;
46 struct octeontx_pko_iomem fc_iomem;
47 struct octeontx_pko_fc_ctl_s *fc_ctl;
48 struct octeontx_pkovf pko[PKO_VF_MAX];
51 } dq_map[PKO_VF_MAX * PKO_VF_NUM_DQ];
54 static struct octeontx_pko_vf_ctl_s pko_vf_ctl;
57 octeontx_pko_dq_vf_bar0(uint16_t txq)
61 vf_ix = txq / PKO_VF_NUM_DQ;
62 return pko_vf_ctl.pko[vf_ix].bar0;
66 octeontx_pko_dq_gdq(uint16_t txq)
68 return txq % PKO_VF_NUM_DQ;
75 int octeontx_pko_dq_open(uint16_t txq)
82 vf_bar0 = octeontx_pko_dq_vf_bar0(txq);
83 gdq = octeontx_pko_dq_gdq(txq);
85 if (unlikely(gdq < 0 || vf_bar0 == NULL))
87 *(volatile int64_t*)(pko_vf_ctl.fc_ctl + txq) =
88 PKO_DQ_FC_DEPTH_PAGES - PKO_DQ_FC_SKID;
92 octeontx_write64(PKO_DQ_FC_DEPTH_PAGES,
93 vf_bar0 + PKO_VF_DQ_FC_STATUS(gdq));
95 /* Set the register to return descriptor (packet) count as DEPTH */
96 /* KIND=1, NCB_QUERY_RSP=0 */
97 octeontx_write64(1ull << PKO_DQ_KIND_BIT,
98 vf_bar0 + PKO_VF_DQ_WM_CTL(gdq));
99 reg_off = PKO_VF_DQ_OP_OPEN(gdq);
101 rtn = octeontx_reg_ldadd_u64(vf_bar0 + reg_off, 0);
103 /* PKO_DQOP_E::OPEN */
104 if (((rtn >> PKO_DQ_OP_BIT) & 0x3) != 0x1)
107 switch (rtn >> PKO_DQ_STATUS_BIT) {
108 case 0xC: /* DQALREADYCREATED */
115 /* DRAIN=0, DRAIN_NULL_LINK=0, SW_XOFF=0 */
116 octeontx_write64(0, vf_bar0 + PKO_VF_DQ_SW_XOFF(gdq));
118 return rtn & ((1ull << PKO_DQ_OP_BIT) - 1);
123 * Flush all packets pending.
126 int octeontx_pko_dq_close(uint16_t txq)
128 unsigned int reg_off;
133 vf_bar0 = octeontx_pko_dq_vf_bar0(txq);
134 res = octeontx_pko_dq_gdq(txq);
136 if (unlikely(res < 0 || vf_bar0 == NULL))
139 reg_off = PKO_VF_DQ_OP_CLOSE(res);
141 rtn = octeontx_reg_ldadd_u64(vf_bar0 + reg_off, 0);
143 /* PKO_DQOP_E::CLOSE */
144 if (((rtn >> PKO_DQ_OP_BIT) & 0x3) != 0x2)
147 switch (rtn >> PKO_DQ_STATUS_BIT) {
148 case 0xD: /* DQNOTCREATED */
155 res = rtn & ((1ull << PKO_DQ_OP_BIT) - 1); /* DEPTH */
159 /* Flush all packets pending on a DQ */
161 int octeontx_pko_dq_drain(uint16_t txq)
166 int res, timo = PKO_DQ_DRAIN_TO;
168 vf_bar0 = octeontx_pko_dq_vf_bar0(txq);
169 res = octeontx_pko_dq_gdq(txq);
172 /* DRAIN=1, DRAIN_NULL_LINK=0, SW_XOFF=1 */
173 octeontx_write64(0x3, vf_bar0 + PKO_VF_DQ_SW_XOFF(gdq));
174 /* Wait until buffers leave DQs */
175 reg = octeontx_read64(vf_bar0 + PKO_VF_DQ_WM_CNT(gdq));
176 while (reg && timo > 0) {
179 reg = octeontx_read64(vf_bar0 + PKO_VF_DQ_WM_CNT(gdq));
181 /* DRAIN=0, DRAIN_NULL_LINK=0, SW_XOFF=0 */
182 octeontx_write64(0, vf_bar0 + PKO_VF_DQ_SW_XOFF(gdq));
188 octeontx_pko_dq_range_lookup(struct octeontx_pko_vf_ctl_s *ctl, uint64_t chanid,
189 unsigned int dq_num, unsigned int dq_from)
191 unsigned int dq, dq_cnt;
192 unsigned int dq_base;
196 while (dq < RTE_DIM(ctl->dq_map)) {
199 while (ctl->dq_map[dq].chanid == ~chanid &&
200 dq < RTE_DIM(ctl->dq_map)) {
202 if (dq_cnt == dq_num)
212 octeontx_pko_dq_range_assign(struct octeontx_pko_vf_ctl_s *ctl, uint64_t chanid,
213 unsigned int dq_base, unsigned int dq_num)
215 unsigned int dq, dq_cnt;
218 while (dq_cnt < dq_num) {
219 dq = dq_base + dq_cnt;
221 octeontx_log_dbg("DQ# %u assigned to CHAN# %" PRIx64 "", dq,
224 ctl->dq_map[dq].chanid = ~chanid;
230 octeontx_pko_dq_claim(struct octeontx_pko_vf_ctl_s *ctl, unsigned int dq_base,
231 unsigned int dq_num, uint64_t chanid)
233 const uint64_t null_chanid = ~0ull;
236 rte_spinlock_lock(&ctl->lock);
238 dq = octeontx_pko_dq_range_lookup(ctl, null_chanid, dq_num, dq_base);
239 if (dq < 0 || (unsigned int)dq != dq_base) {
240 rte_spinlock_unlock(&ctl->lock);
243 octeontx_pko_dq_range_assign(ctl, chanid, dq_base, dq_num);
245 rte_spinlock_unlock(&ctl->lock);
251 octeontx_pko_dq_free(struct octeontx_pko_vf_ctl_s *ctl, uint64_t chanid)
253 const uint64_t null_chanid = ~0ull;
254 unsigned int dq = 0, dq_cnt = 0;
256 rte_spinlock_lock(&ctl->lock);
257 while (dq < RTE_DIM(ctl->dq_map)) {
258 if (ctl->dq_map[dq].chanid == ~chanid) {
259 ctl->dq_map[dq].chanid = ~null_chanid;
264 rte_spinlock_unlock(&ctl->lock);
266 return dq_cnt > 0 ? 0 : -EINVAL;
270 octeontx_pko_channel_open(int dq_base, int dq_num, int chanid)
272 struct octeontx_pko_vf_ctl_s *ctl = &pko_vf_ctl;
275 res = octeontx_pko_dq_claim(ctl, dq_base, dq_num, chanid);
283 octeontx_pko_channel_close(int chanid)
285 struct octeontx_pko_vf_ctl_s *ctl = &pko_vf_ctl;
288 res = octeontx_pko_dq_free(ctl, chanid);
296 octeontx_pko_chan_start(struct octeontx_pko_vf_ctl_s *ctl, uint64_t chanid)
299 unsigned int dq, dq_cnt;
303 while (dq < RTE_DIM(ctl->dq_map)) {
304 dq_vf = dq / PKO_VF_NUM_DQ;
306 if (!ctl->pko[dq_vf].bar0) {
311 if (ctl->dq_map[dq].chanid != ~chanid) {
316 if (octeontx_pko_dq_open(dq) < 0)
327 octeontx_pko_channel_start(int chanid)
329 struct octeontx_pko_vf_ctl_s *ctl = &pko_vf_ctl;
332 dq_cnt = octeontx_pko_chan_start(ctl, chanid);
340 octeontx_pko_chan_stop(struct octeontx_pko_vf_ctl_s *ctl, uint64_t chanid)
342 unsigned int dq, dq_cnt, dq_vf;
347 while (dq < RTE_DIM(ctl->dq_map)) {
348 dq_vf = dq / PKO_VF_NUM_DQ;
350 if (!ctl->pko[dq_vf].bar0) {
355 if (ctl->dq_map[dq].chanid != ~chanid) {
360 res = octeontx_pko_dq_drain(dq);
362 octeontx_log_err("draining DQ%d, buffers left: %x",
365 res = octeontx_pko_dq_close(dq);
367 octeontx_log_err("closing DQ%d failed\n", dq);
376 octeontx_pko_channel_stop(int chanid)
378 struct octeontx_pko_vf_ctl_s *ctl = &pko_vf_ctl;
380 octeontx_pko_chan_stop(ctl, chanid);
385 octeontx_pko_channel_query(struct octeontx_pko_vf_ctl_s *ctl, uint64_t chanid,
386 void *out, size_t out_elem_size,
387 size_t dq_num, octeontx_pko_dq_getter_t getter)
393 RTE_SET_USED(out_elem_size);
394 memset(&curr, 0, sizeof(octeontx_dq_t));
396 dq_vf = dq_num / PKO_VF_NUM_DQ;
397 dq = dq_num % PKO_VF_NUM_DQ;
399 if (!ctl->pko[dq_vf].bar0)
402 if (ctl->dq_map[dq_num].chanid != ~chanid)
405 uint8_t *iter = (uint8_t *)out;
406 curr.lmtline_va = ctl->pko[dq_vf].bar2;
407 curr.ioreg_va = (void *)((uintptr_t)ctl->pko[dq_vf].bar0
408 + PKO_VF_DQ_OP_SEND((dq), 0));
409 curr.fc_status_va = ctl->fc_ctl + dq_num;
411 octeontx_log_dbg("lmtline=%p ioreg_va=%p fc_status_va=%p",
412 curr.lmtline_va, curr.ioreg_va,
415 getter(&curr, (void *)iter);
420 octeontx_pko_channel_query_dqs(int chanid, void *out, size_t out_elem_size,
421 size_t dq_num, octeontx_pko_dq_getter_t getter)
423 struct octeontx_pko_vf_ctl_s *ctl = &pko_vf_ctl;
426 dq_cnt = octeontx_pko_channel_query(ctl, chanid, out, out_elem_size,
435 octeontx_pko_vf_count(void)
437 uint16_t global_domain = octeontx_get_global_domain();
440 pko_vf_ctl.global_domain = global_domain;
442 while (pko_vf_ctl.pko[vf_cnt].bar0)
449 octeontx_pko_get_vfid(void)
451 size_t vf_cnt = octeontx_pko_vf_count();
455 for (vf_idx = 0; vf_idx < vf_cnt; vf_idx++) {
456 if (!(pko_vf_ctl.pko[vf_idx].status & PKO_VALID))
458 if (pko_vf_ctl.pko[vf_idx].status & PKO_INUSE)
461 pko_vf_ctl.pko[vf_idx].status |= PKO_INUSE;
462 return pko_vf_ctl.pko[vf_idx].vfid;
469 octeontx_pko_init_fc(const size_t pko_vf_count)
477 fc_mem_size = sizeof(struct octeontx_pko_fc_ctl_s) *
478 pko_vf_count * PKO_VF_NUM_DQ;
480 pko_vf_ctl.fc_iomem.va = rte_malloc(NULL, fc_mem_size, 128);
481 if (unlikely(!pko_vf_ctl.fc_iomem.va)) {
482 octeontx_log_err("fc_iomem: not enough memory");
486 pko_vf_ctl.fc_iomem.iova = rte_malloc_virt2iova((void *)
487 pko_vf_ctl.fc_iomem.va);
488 pko_vf_ctl.fc_iomem.size = fc_mem_size;
491 (struct octeontx_pko_fc_ctl_s *)pko_vf_ctl.fc_iomem.va;
493 /* Configure Flow-Control feature for all DQs of open VFs */
494 for (vf_idx = 0; vf_idx < pko_vf_count; vf_idx++) {
495 if (pko_vf_ctl.pko[vf_idx].domain != pko_vf_ctl.global_domain)
498 dq_ix = pko_vf_ctl.pko[vf_idx].vfid * PKO_VF_NUM_DQ;
499 vf_bar0 = pko_vf_ctl.pko[vf_idx].bar0;
501 reg = (pko_vf_ctl.fc_iomem.iova +
502 (sizeof(struct octeontx_pko_fc_ctl_s) * dq_ix)) & ~0x7F;
504 (0x2 << 3) | /* HYST_BITS */
505 (((PKO_DQ_FC_STRIDE == PKO_DQ_FC_STRIDE_16) ? 1 : 0) << 2) |
506 (0x1 << 0); /* ENABLE */
508 octeontx_write64(reg, vf_bar0 + PKO_VF_DQ_FC_CONFIG);
509 pko_vf_ctl.pko[vf_idx].status = PKO_VALID;
511 octeontx_log_dbg("PKO: bar0 %p VF_idx %d DQ_FC_CFG=%" PRIx64 "",
512 vf_bar0, (int)vf_idx, reg);
518 octeontx_pko_fc_free(void)
520 rte_free(pko_vf_ctl.fc_iomem.va);
524 octeontx_pkovf_setup(void)
526 static bool init_once;
531 rte_spinlock_init(&pko_vf_ctl.lock);
533 pko_vf_ctl.fc_iomem = PKO_IOMEM_NULL;
534 pko_vf_ctl.fc_ctl = NULL;
536 for (i = 0; i < PKO_VF_MAX; i++) {
537 pko_vf_ctl.pko[i].bar0 = NULL;
538 pko_vf_ctl.pko[i].bar2 = NULL;
539 pko_vf_ctl.pko[i].domain = ~(uint16_t)0;
540 pko_vf_ctl.pko[i].vfid = ~(uint16_t)0;
543 for (i = 0; i < (PKO_VF_MAX * PKO_VF_NUM_DQ); i++)
544 pko_vf_ctl.dq_map[i].chanid = 0;
550 /* PKOVF pcie device*/
552 pkovf_probe(struct rte_pci_driver *pci_drv, struct rte_pci_device *pci_dev)
559 static uint8_t vf_cnt;
560 struct octeontx_pkovf *res;
562 RTE_SET_USED(pci_drv);
564 /* For secondary processes, the primary has done all the work */
565 if (rte_eal_process_type() != RTE_PROC_PRIMARY)
568 if (pci_dev->mem_resource[0].addr == NULL ||
569 pci_dev->mem_resource[2].addr == NULL) {
570 octeontx_log_err("Empty bars %p %p",
571 pci_dev->mem_resource[0].addr,
572 pci_dev->mem_resource[2].addr);
575 bar0 = pci_dev->mem_resource[0].addr;
576 bar2 = pci_dev->mem_resource[2].addr;
578 octeontx_pkovf_setup();
580 /* get vfid and domain */
581 val = octeontx_read64(bar0 + PKO_VF_DQ_FC_CONFIG);
582 domain = (val >> 7) & 0xffff;
583 vfid = (val >> 23) & 0xffff;
585 if (unlikely(vfid >= PKO_VF_MAX)) {
586 octeontx_log_err("pko: Invalid vfid %d", vfid);
590 res = &pko_vf_ctl.pko[vf_cnt++];
592 res->domain = domain;
596 octeontx_log_dbg("Domain=%d group=%d", res->domain, res->vfid);
600 #define PCI_VENDOR_ID_CAVIUM 0x177D
601 #define PCI_DEVICE_ID_OCTEONTX_PKO_VF 0xA049
603 static const struct rte_pci_id pci_pkovf_map[] = {
605 RTE_PCI_DEVICE(PCI_VENDOR_ID_CAVIUM,
606 PCI_DEVICE_ID_OCTEONTX_PKO_VF)
613 static struct rte_pci_driver pci_pkovf = {
614 .id_table = pci_pkovf_map,
615 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
616 .probe = pkovf_probe,
619 RTE_PMD_REGISTER_PCI(octeontx_pkovf, pci_pkovf);