1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2019 Marvell International Ltd.
10 #include <rte_alarm.h>
11 #include <rte_common.h>
13 #include <rte_memcpy.h>
16 #include "otx2_mbox.h"
18 #define RVU_MAX_VF 64 /* RVU_PF_VFPF_MBOX_INT(0..1) */
19 #define RVU_MAX_INT_RETRY 3
21 /* PF/VF message handling timer */
22 #define VF_PF_MBOX_TIMER_MS (20 * 1000)
25 mbox_mem_map(off_t off, size_t size)
27 void *va = MAP_FAILED;
33 mem_fd = open("/dev/mem", O_RDWR);
37 va = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, mem_fd, off);
41 otx2_err("Failed to mmap sz=0x%zx, fd=%d, off=%jd",
42 size, mem_fd, (intmax_t)off);
48 mbox_mem_unmap(void *va, size_t size)
55 af_pf_wait_msg(struct otx2_dev *dev, uint16_t vf, int num_msg)
57 uint32_t timeout = 0, sleep = 1; struct otx2_mbox *mbox = dev->mbox;
58 struct otx2_mbox_dev *mdev = &mbox->dev[0];
59 volatile uint64_t int_status;
60 struct mbox_hdr *req_hdr;
61 struct mbox_msghdr *msg;
62 struct mbox_msghdr *rsp;
67 /* We need to disable PF interrupts. We are in timer interrupt */
68 otx2_write64(~0ull, dev->bar2 + RVU_PF_INT_ENA_W1C);
71 otx2_mbox_msg_send(mbox, 0);
76 if (timeout >= MBOX_RSP_TIMEOUT) {
77 otx2_err("Routed messages %d timeout: %dms",
78 num_msg, MBOX_RSP_TIMEOUT);
81 int_status = otx2_read64(dev->bar2 + RVU_PF_INT);
82 } while ((int_status & 0x1) != 0x1);
85 otx2_write64(~0ull, dev->bar2 + RVU_PF_INT);
87 /* Enable interrupts */
88 otx2_write64(~0ull, dev->bar2 + RVU_PF_INT_ENA_W1S);
90 rte_spinlock_lock(&mdev->mbox_lock);
92 req_hdr = (struct mbox_hdr *)((uintptr_t)mdev->mbase + mbox->rx_start);
93 if (req_hdr->num_msgs != num_msg)
94 otx2_err("Routed messages: %d received: %d", num_msg,
97 /* Get messages from mbox */
98 offset = mbox->rx_start +
99 RTE_ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN);
100 for (i = 0; i < req_hdr->num_msgs; i++) {
101 msg = (struct mbox_msghdr *)((uintptr_t)mdev->mbase + offset);
102 size = mbox->rx_start + msg->next_msgoff - offset;
104 /* Reserve PF/VF mbox message */
105 size = RTE_ALIGN(size, MBOX_MSG_ALIGN);
106 rsp = otx2_mbox_alloc_msg(&dev->mbox_vfpf, vf, size);
107 otx2_mbox_rsp_init(msg->id, rsp);
109 /* Copy message from AF<->PF mbox to PF<->VF mbox */
110 otx2_mbox_memcpy((uint8_t *)rsp + sizeof(struct mbox_msghdr),
111 (uint8_t *)msg + sizeof(struct mbox_msghdr),
112 size - sizeof(struct mbox_msghdr));
114 /* Set status and sender pf_func data */
116 rsp->pcifunc = msg->pcifunc;
118 offset = mbox->rx_start + msg->next_msgoff;
120 rte_spinlock_unlock(&mdev->mbox_lock);
122 return req_hdr->num_msgs;
126 vf_pf_process_msgs(struct otx2_dev *dev, uint16_t vf)
128 int offset, routed = 0; struct otx2_mbox *mbox = &dev->mbox_vfpf;
129 struct otx2_mbox_dev *mdev = &mbox->dev[vf];
130 struct mbox_hdr *req_hdr;
131 struct mbox_msghdr *msg;
135 req_hdr = (struct mbox_hdr *)((uintptr_t)mdev->mbase + mbox->rx_start);
136 if (!req_hdr->num_msgs)
139 offset = mbox->rx_start + RTE_ALIGN(sizeof(*req_hdr), MBOX_MSG_ALIGN);
141 for (i = 0; i < req_hdr->num_msgs; i++) {
143 msg = (struct mbox_msghdr *)((uintptr_t)mdev->mbase + offset);
144 size = mbox->rx_start + msg->next_msgoff - offset;
147 msg->pcifunc = otx2_pfvf_func(dev->pf, vf);
149 if (msg->id == MBOX_MSG_READY) {
150 struct ready_msg_rsp *rsp;
151 uint16_t max_bits = sizeof(dev->active_vfs[0]) * 8;
153 /* Handle READY message in PF */
154 dev->active_vfs[vf / max_bits] |=
155 BIT_ULL(vf % max_bits);
156 rsp = (struct ready_msg_rsp *)
157 otx2_mbox_alloc_msg(mbox, vf, sizeof(*rsp));
158 otx2_mbox_rsp_init(msg->id, rsp);
160 /* PF/VF function ID */
161 rsp->hdr.pcifunc = msg->pcifunc;
164 struct mbox_msghdr *af_req;
165 /* Reserve AF/PF mbox message */
166 size = RTE_ALIGN(size, MBOX_MSG_ALIGN);
167 af_req = otx2_mbox_alloc_msg(dev->mbox, 0, size);
168 otx2_mbox_req_init(msg->id, af_req);
170 /* Copy message from VF<->PF mbox to PF<->AF mbox */
171 otx2_mbox_memcpy((uint8_t *)af_req +
172 sizeof(struct mbox_msghdr),
173 (uint8_t *)msg + sizeof(struct mbox_msghdr),
174 size - sizeof(struct mbox_msghdr));
175 af_req->pcifunc = msg->pcifunc;
178 offset = mbox->rx_start + msg->next_msgoff;
182 otx2_base_dbg("pf:%d routed %d messages from vf:%d to AF",
183 dev->pf, routed, vf);
184 af_pf_wait_msg(dev, vf, routed);
185 otx2_mbox_reset(dev->mbox, 0);
188 /* Send mbox responses to VF */
189 if (mdev->num_msgs) {
190 otx2_base_dbg("pf:%d reply %d messages to vf:%d",
191 dev->pf, mdev->num_msgs, vf);
192 otx2_mbox_msg_send(mbox, vf);
199 otx2_vf_pf_mbox_handle_msg(void *param)
201 uint16_t vf, max_vf, max_bits;
202 struct otx2_dev *dev = param;
204 max_bits = sizeof(dev->intr.bits[0]) * sizeof(uint64_t);
205 max_vf = max_bits * MAX_VFPF_DWORD_BITS;
207 for (vf = 0; vf < max_vf; vf++) {
208 if (dev->intr.bits[vf/max_bits] & BIT_ULL(vf%max_bits)) {
209 otx2_base_dbg("Process vf:%d request (pf:%d, vf:%d)",
210 vf, dev->pf, dev->vf);
211 vf_pf_process_msgs(dev, vf);
212 dev->intr.bits[vf/max_bits] &= ~(BIT_ULL(vf%max_bits));
219 otx2_vf_pf_mbox_irq(void *param)
221 struct otx2_dev *dev = param;
222 bool alarm_set = false;
226 for (vfpf = 0; vfpf < MAX_VFPF_DWORD_BITS; ++vfpf) {
227 intr = otx2_read64(dev->bar2 + RVU_PF_VFPF_MBOX_INTX(vfpf));
231 otx2_base_dbg("vfpf: %d intr: 0x%" PRIx64 " (pf:%d, vf:%d)",
232 vfpf, intr, dev->pf, dev->vf);
234 /* Save and clear intr bits */
235 dev->intr.bits[vfpf] |= intr;
236 otx2_write64(intr, dev->bar2 + RVU_PF_VFPF_MBOX_INTX(vfpf));
240 if (!dev->timer_set && alarm_set) {
242 /* Start timer to handle messages */
243 rte_eal_alarm_set(VF_PF_MBOX_TIMER_MS,
244 otx2_vf_pf_mbox_handle_msg, dev);
249 otx2_process_msgs(struct otx2_dev *dev, struct otx2_mbox *mbox)
251 struct otx2_mbox_dev *mdev = &mbox->dev[0];
252 struct mbox_hdr *req_hdr;
253 struct mbox_msghdr *msg;
258 req_hdr = (struct mbox_hdr *)((uintptr_t)mdev->mbase + mbox->rx_start);
259 if (req_hdr->num_msgs == 0)
262 offset = mbox->rx_start + RTE_ALIGN(sizeof(*req_hdr), MBOX_MSG_ALIGN);
263 for (i = 0; i < req_hdr->num_msgs; i++) {
264 msg = (struct mbox_msghdr *)((uintptr_t)mdev->mbase + offset);
267 otx2_base_dbg("Message 0x%x (%s) pf:%d/vf:%d",
268 msg->id, otx2_mbox_id2name(msg->id),
269 otx2_get_pf(msg->pcifunc),
270 otx2_get_vf(msg->pcifunc));
273 /* Add message id's that are handled here */
275 /* Get our identity */
276 dev->pf_func = msg->pcifunc;
281 otx2_err("Message (%s) response has err=%d",
282 otx2_mbox_id2name(msg->id), msg->rc);
285 offset = mbox->rx_start + msg->next_msgoff;
288 otx2_mbox_reset(mbox, 0);
289 /* Update acked if someone is waiting a message */
290 mdev->msgs_acked = msgs_acked;
295 otx2_pf_vf_mbox_irq(void *param)
297 struct otx2_dev *dev = param;
300 intr = otx2_read64(dev->bar2 + RVU_VF_INT);
304 otx2_write64(intr, dev->bar2 + RVU_VF_INT);
305 otx2_base_dbg("Irq 0x%" PRIx64 "(pf:%d,vf:%d)", intr, dev->pf, dev->vf);
307 /* First process all configuration messages */
308 otx2_process_msgs(dev, dev->mbox);
313 otx2_af_pf_mbox_irq(void *param)
315 struct otx2_dev *dev = param;
318 intr = otx2_read64(dev->bar2 + RVU_PF_INT);
322 otx2_write64(intr, dev->bar2 + RVU_PF_INT);
324 otx2_base_dbg("Irq 0x%" PRIx64 "(pf:%d,vf:%d)", intr, dev->pf, dev->vf);
326 /* First process all configuration messages */
327 otx2_process_msgs(dev, dev->mbox);
331 mbox_register_pf_irq(struct rte_pci_device *pci_dev, struct otx2_dev *dev)
333 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
337 for (i = 0; i < MAX_VFPF_DWORD_BITS; ++i)
338 otx2_write64(~0ull, dev->bar2 +
339 RVU_PF_VFPF_MBOX_INT_ENA_W1CX(i));
341 otx2_write64(~0ull, dev->bar2 + RVU_PF_INT_ENA_W1C);
345 /* MBOX interrupt for VF(0...63) <-> PF */
346 rc = otx2_register_irq(intr_handle, otx2_vf_pf_mbox_irq, dev,
347 RVU_PF_INT_VEC_VFPF_MBOX0);
350 otx2_err("Fail to register PF(VF0-63) mbox irq");
353 /* MBOX interrupt for VF(64...128) <-> PF */
354 rc = otx2_register_irq(intr_handle, otx2_vf_pf_mbox_irq, dev,
355 RVU_PF_INT_VEC_VFPF_MBOX1);
358 otx2_err("Fail to register PF(VF64-128) mbox irq");
361 /* MBOX interrupt AF <-> PF */
362 rc = otx2_register_irq(intr_handle, otx2_af_pf_mbox_irq,
363 dev, RVU_PF_INT_VEC_AFPF_MBOX);
365 otx2_err("Fail to register AF<->PF mbox irq");
370 for (i = 0; i < MAX_VFPF_DWORD_BITS; ++i)
371 otx2_write64(~0ull, dev->bar2 +
372 RVU_PF_VFPF_MBOX_INT_ENA_W1SX(i));
374 otx2_write64(~0ull, dev->bar2 + RVU_PF_INT);
375 otx2_write64(~0ull, dev->bar2 + RVU_PF_INT_ENA_W1S);
381 mbox_register_vf_irq(struct rte_pci_device *pci_dev, struct otx2_dev *dev)
383 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
387 otx2_write64(~0ull, dev->bar2 + RVU_VF_INT_ENA_W1C);
389 /* MBOX interrupt PF <-> VF */
390 rc = otx2_register_irq(intr_handle, otx2_pf_vf_mbox_irq,
391 dev, RVU_VF_INT_VEC_MBOX);
393 otx2_err("Fail to register PF<->VF mbox irq");
398 otx2_write64(~0ull, dev->bar2 + RVU_VF_INT);
399 otx2_write64(~0ull, dev->bar2 + RVU_VF_INT_ENA_W1S);
405 mbox_register_irq(struct rte_pci_device *pci_dev, struct otx2_dev *dev)
407 if (otx2_dev_is_vf(dev))
408 return mbox_register_vf_irq(pci_dev, dev);
410 return mbox_register_pf_irq(pci_dev, dev);
414 mbox_unregister_pf_irq(struct rte_pci_device *pci_dev, struct otx2_dev *dev)
416 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
420 for (i = 0; i < MAX_VFPF_DWORD_BITS; ++i)
421 otx2_write64(~0ull, dev->bar2 +
422 RVU_PF_VFPF_MBOX_INT_ENA_W1CX(i));
424 otx2_write64(~0ull, dev->bar2 + RVU_PF_INT_ENA_W1C);
428 rte_eal_alarm_cancel(otx2_vf_pf_mbox_handle_msg, dev);
430 /* Unregister the interrupt handler for each vectors */
431 /* MBOX interrupt for VF(0...63) <-> PF */
432 otx2_unregister_irq(intr_handle, otx2_vf_pf_mbox_irq, dev,
433 RVU_PF_INT_VEC_VFPF_MBOX0);
435 /* MBOX interrupt for VF(64...128) <-> PF */
436 otx2_unregister_irq(intr_handle, otx2_vf_pf_mbox_irq, dev,
437 RVU_PF_INT_VEC_VFPF_MBOX1);
439 /* MBOX interrupt AF <-> PF */
440 otx2_unregister_irq(intr_handle, otx2_af_pf_mbox_irq, dev,
441 RVU_PF_INT_VEC_AFPF_MBOX);
446 mbox_unregister_vf_irq(struct rte_pci_device *pci_dev, struct otx2_dev *dev)
448 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
451 otx2_write64(~0ull, dev->bar2 + RVU_VF_INT_ENA_W1C);
453 /* Unregister the interrupt handler */
454 otx2_unregister_irq(intr_handle, otx2_pf_vf_mbox_irq, dev,
455 RVU_VF_INT_VEC_MBOX);
459 mbox_unregister_irq(struct rte_pci_device *pci_dev, struct otx2_dev *dev)
461 if (otx2_dev_is_vf(dev))
462 return mbox_unregister_vf_irq(pci_dev, dev);
464 return mbox_unregister_pf_irq(pci_dev, dev);
468 otx2_update_pass_hwcap(struct rte_pci_device *pci_dev, struct otx2_dev *dev)
470 RTE_SET_USED(pci_dev);
472 /* Update this logic when we have A1 */
473 dev->hwcap |= OTX2_HWCAP_F_A0;
477 otx2_update_vf_hwcap(struct rte_pci_device *pci_dev, struct otx2_dev *dev)
481 switch (pci_dev->id.device_id) {
482 case PCI_DEVID_OCTEONTX2_RVU_PF:
484 case PCI_DEVID_OCTEONTX2_RVU_SSO_TIM_VF:
485 case PCI_DEVID_OCTEONTX2_RVU_NPA_VF:
486 case PCI_DEVID_OCTEONTX2_RVU_CPT_VF:
487 case PCI_DEVID_OCTEONTX2_RVU_AF_VF:
488 case PCI_DEVID_OCTEONTX2_RVU_VF:
489 dev->hwcap |= OTX2_HWCAP_F_VF;
496 * Initialize the otx2 device
499 otx2_dev_init(struct rte_pci_device *pci_dev, void *otx2_dev)
501 int up_direction = MBOX_DIR_PFAF_UP;
502 int rc, direction = MBOX_DIR_PFAF;
503 struct otx2_dev *dev = otx2_dev;
504 uintptr_t bar2, bar4;
508 bar2 = (uintptr_t)pci_dev->mem_resource[2].addr;
509 bar4 = (uintptr_t)pci_dev->mem_resource[4].addr;
511 if (bar2 == 0 || bar4 == 0) {
512 otx2_err("Failed to get pci bars");
517 dev->node = pci_dev->device.numa_node;
518 dev->maxvf = pci_dev->max_vfs;
522 otx2_update_vf_hwcap(pci_dev, dev);
523 otx2_update_pass_hwcap(pci_dev, dev);
525 if (otx2_dev_is_vf(dev)) {
526 direction = MBOX_DIR_VFPF;
527 up_direction = MBOX_DIR_VFPF_UP;
530 /* Initialize the local mbox */
531 rc = otx2_mbox_init(&dev->mbox_local, bar4, bar2, direction, 1);
534 dev->mbox = &dev->mbox_local;
536 rc = otx2_mbox_init(&dev->mbox_up, bar4, bar2, up_direction, 1);
540 /* Register mbox interrupts */
541 rc = mbox_register_irq(pci_dev, dev);
545 /* Check the readiness of PF/VF */
546 rc = otx2_send_ready_msg(dev->mbox, &dev->pf_func);
548 goto mbox_unregister;
550 dev->pf = otx2_get_pf(dev->pf_func);
551 dev->vf = otx2_get_vf(dev->pf_func);
552 memset(&dev->active_vfs, 0, sizeof(dev->active_vfs));
554 /* Found VF devices in a PF device */
555 if (pci_dev->max_vfs > 0) {
557 /* Remap mbox area for all vf's */
558 bar4_addr = otx2_read64(bar2 + RVU_PF_VF_BAR4_ADDR);
559 if (bar4_addr == 0) {
564 hwbase = mbox_mem_map(bar4_addr, MBOX_SIZE * pci_dev->max_vfs);
565 if (hwbase == MAP_FAILED) {
569 /* Init mbox object */
570 rc = otx2_mbox_init(&dev->mbox_vfpf, (uintptr_t)hwbase,
571 bar2, MBOX_DIR_PFVF, pci_dev->max_vfs);
575 /* PF -> VF UP messages */
576 rc = otx2_mbox_init(&dev->mbox_vfpf_up, (uintptr_t)hwbase,
577 bar2, MBOX_DIR_PFVF_UP, pci_dev->max_vfs);
582 dev->mbox_active = 1;
586 mbox_mem_unmap(hwbase, MBOX_SIZE * pci_dev->max_vfs);
588 mbox_unregister_irq(pci_dev, dev);
590 otx2_mbox_fini(dev->mbox);
591 otx2_mbox_fini(&dev->mbox_up);
598 * Finalize the otx2 device
601 otx2_dev_fini(struct rte_pci_device *pci_dev, void *otx2_dev)
603 struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
604 struct otx2_dev *dev = otx2_dev;
605 struct otx2_idev_cfg *idev;
606 struct otx2_mbox *mbox;
608 /* Clear references to this pci dev */
609 idev = otx2_intra_dev_get_cfg();
610 if (idev->npa_lf && idev->npa_lf->pci_dev == pci_dev)
613 mbox_unregister_irq(pci_dev, dev);
615 /* Release PF - VF */
616 mbox = &dev->mbox_vfpf;
617 if (mbox->hwbase && mbox->dev)
618 mbox_mem_unmap((void *)mbox->hwbase,
619 MBOX_SIZE * pci_dev->max_vfs);
620 otx2_mbox_fini(mbox);
621 mbox = &dev->mbox_vfpf_up;
622 otx2_mbox_fini(mbox);
624 /* Release PF - AF */
626 otx2_mbox_fini(mbox);
627 mbox = &dev->mbox_up;
628 otx2_mbox_fini(mbox);
629 dev->mbox_active = 0;
631 /* Disable MSIX vectors */
632 otx2_disable_irqs(intr_handle);