1 /* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0)
2 * Copyright(c) 2018-2019 Pensando Systems, Inc. All rights reserved.
5 #include <rte_malloc.h>
6 #include <rte_ethdev_driver.h>
9 #include "ionic_logs.h"
10 #include "ionic_lif.h"
11 #include "ionic_ethdev.h"
14 ionic_qcq_enable(struct ionic_qcq *qcq)
16 struct ionic_queue *q = &qcq->q;
17 struct ionic_lif *lif = q->lif;
18 struct ionic_dev *idev = &lif->adapter->idev;
19 struct ionic_admin_ctx ctx = {
22 .opcode = IONIC_CMD_Q_CONTROL,
23 .lif_index = lif->index,
26 .oper = IONIC_Q_ENABLE,
30 if (qcq->flags & IONIC_QCQ_F_INTR) {
31 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
32 IONIC_INTR_MASK_CLEAR);
35 return ionic_adminq_post_wait(lif, &ctx);
39 ionic_qcq_disable(struct ionic_qcq *qcq)
41 struct ionic_queue *q = &qcq->q;
42 struct ionic_lif *lif = q->lif;
43 struct ionic_dev *idev = &lif->adapter->idev;
44 struct ionic_admin_ctx ctx = {
47 .opcode = IONIC_CMD_Q_CONTROL,
48 .lif_index = lif->index,
51 .oper = IONIC_Q_DISABLE,
55 if (qcq->flags & IONIC_QCQ_F_INTR) {
56 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
60 return ionic_adminq_post_wait(lif, &ctx);
64 ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
66 struct ionic_adapter *adapter = lif->adapter;
67 struct ionic_dev *idev = &adapter->idev;
71 * Note: interrupt handler is called for index = 0 only
72 * (we use interrupts for the notifyq only anyway,
73 * which hash index = 0)
76 for (index = 0; index < adapter->nintrs; index++)
77 if (!adapter->intrs[index])
80 if (index == adapter->nintrs)
83 adapter->intrs[index] = true;
85 ionic_intr_init(idev, intr, index);
91 ionic_intr_free(struct ionic_lif *lif, struct ionic_intr_info *intr)
93 if (intr->index != IONIC_INTR_INDEX_NOT_ASSIGNED)
94 lif->adapter->intrs[intr->index] = false;
98 ionic_qcq_alloc(struct ionic_lif *lif, uint8_t type,
100 const char *base, uint32_t flags,
103 uint32_t cq_desc_size,
104 uint32_t sg_desc_size,
105 uint32_t pid, struct ionic_qcq **qcq)
107 struct ionic_dev *idev = &lif->adapter->idev;
108 struct ionic_qcq *new;
109 uint32_t q_size, cq_size, sg_size, total_size;
110 void *q_base, *cq_base, *sg_base;
111 rte_iova_t q_base_pa = 0;
112 rte_iova_t cq_base_pa = 0;
113 rte_iova_t sg_base_pa = 0;
114 uint32_t socket_id = rte_socket_id();
119 q_size = num_descs * desc_size;
120 cq_size = num_descs * cq_desc_size;
121 sg_size = num_descs * sg_desc_size;
123 total_size = RTE_ALIGN(q_size, PAGE_SIZE) +
124 RTE_ALIGN(cq_size, PAGE_SIZE);
126 * Note: aligning q_size/cq_size is not enough due to cq_base address
127 * aligning as q_base could be not aligned to the page.
130 total_size += PAGE_SIZE;
132 if (flags & IONIC_QCQ_F_SG) {
133 total_size += RTE_ALIGN(sg_size, PAGE_SIZE);
134 total_size += PAGE_SIZE;
137 new = rte_zmalloc("ionic", sizeof(*new), 0);
139 IONIC_PRINT(ERR, "Cannot allocate queue structure");
146 new->q.info = rte_zmalloc("ionic", sizeof(*new->q.info) * num_descs, 0);
148 IONIC_PRINT(ERR, "Cannot allocate queue info");
154 err = ionic_q_init(lif, idev, &new->q, index, num_descs,
155 desc_size, sg_desc_size, pid);
157 IONIC_PRINT(ERR, "Queue initialization failed");
161 if (flags & IONIC_QCQ_F_INTR) {
162 err = ionic_intr_alloc(lif, &new->intr);
166 ionic_intr_mask_assert(idev->intr_ctrl, new->intr.index,
167 IONIC_INTR_MASK_SET);
169 new->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
172 err = ionic_cq_init(lif, &new->cq, &new->intr,
173 num_descs, cq_desc_size);
175 IONIC_PRINT(ERR, "Completion queue initialization failed");
176 goto err_out_free_intr;
179 new->base_z = rte_eth_dma_zone_reserve(lif->eth_dev,
180 base /* name */, index /* queue_idx */,
181 total_size, IONIC_ALIGN, socket_id);
184 IONIC_PRINT(ERR, "Cannot reserve queue DMA memory");
186 goto err_out_free_intr;
189 new->base = new->base_z->addr;
190 new->base_pa = new->base_z->iova;
191 new->total_size = total_size;
194 q_base_pa = new->base_pa;
196 cq_base = (void *)RTE_ALIGN((uintptr_t)q_base + q_size, PAGE_SIZE);
197 cq_base_pa = RTE_ALIGN(q_base_pa + q_size, PAGE_SIZE);
199 if (flags & IONIC_QCQ_F_SG) {
200 sg_base = (void *)RTE_ALIGN((uintptr_t)cq_base + cq_size,
202 sg_base_pa = RTE_ALIGN(cq_base_pa + cq_size, PAGE_SIZE);
203 ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
206 IONIC_PRINT(DEBUG, "Q-Base-PA = %ju CQ-Base-PA = %ju "
208 q_base_pa, cq_base_pa, sg_base_pa);
210 ionic_q_map(&new->q, q_base, q_base_pa);
211 ionic_cq_map(&new->cq, cq_base, cq_base_pa);
212 ionic_cq_bind(&new->cq, &new->q);
219 if (flags & IONIC_QCQ_F_INTR)
220 ionic_intr_free(lif, &new->intr);
226 ionic_qcq_free(struct ionic_qcq *qcq)
231 rte_memzone_free(qcq->base_z);
236 rte_free(qcq->q.info);
244 ionic_admin_qcq_alloc(struct ionic_lif *lif)
250 err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
252 sizeof(struct ionic_admin_cmd),
253 sizeof(struct ionic_admin_comp),
255 lif->kern_pid, &lif->adminqcq);
263 ionic_notify_qcq_alloc(struct ionic_lif *lif)
268 flags = IONIC_QCQ_F_NOTIFYQ | IONIC_QCQ_F_INTR;
270 err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notify",
272 IONIC_NOTIFYQ_LENGTH,
273 sizeof(struct ionic_notifyq_cmd),
274 sizeof(union ionic_notifyq_comp),
276 lif->kern_pid, &lif->notifyqcq);
284 ionic_bus_map_dbpage(struct ionic_adapter *adapter, int page_num)
286 char *vaddr = adapter->bars[IONIC_PCI_BAR_DBELL].vaddr;
288 if (adapter->num_bars <= IONIC_PCI_BAR_DBELL)
291 return (void *)&vaddr[page_num << PAGE_SHIFT];
295 ionic_lif_alloc(struct ionic_lif *lif)
297 struct ionic_adapter *adapter = lif->adapter;
298 uint32_t socket_id = rte_socket_id();
302 snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);
304 IONIC_PRINT(DEBUG, "Allocating Lif Info");
306 rte_spinlock_init(&lif->adminq_lock);
307 rte_spinlock_init(&lif->adminq_service_lock);
311 dbpage_num = ionic_db_page_num(lif, 0);
313 lif->kern_dbpage = ionic_bus_map_dbpage(adapter, dbpage_num);
314 if (!lif->kern_dbpage) {
315 IONIC_PRINT(ERR, "Cannot map dbpage, aborting");
319 IONIC_PRINT(DEBUG, "Allocating Notify Queue");
321 err = ionic_notify_qcq_alloc(lif);
324 IONIC_PRINT(ERR, "Cannot allocate notify queue");
328 IONIC_PRINT(DEBUG, "Allocating Admin Queue");
330 IONIC_PRINT(DEBUG, "Allocating Admin Queue");
332 err = ionic_admin_qcq_alloc(lif);
334 IONIC_PRINT(ERR, "Cannot allocate admin queue");
338 IONIC_PRINT(DEBUG, "Allocating Lif Info");
340 lif->info_sz = RTE_ALIGN(sizeof(*lif->info), PAGE_SIZE);
342 lif->info_z = rte_eth_dma_zone_reserve(lif->eth_dev,
343 "lif_info", 0 /* queue_idx*/,
344 lif->info_sz, IONIC_ALIGN, socket_id);
346 IONIC_PRINT(ERR, "Cannot allocate lif info memory");
350 lif->info = lif->info_z->addr;
351 lif->info_pa = lif->info_z->iova;
357 ionic_lif_free(struct ionic_lif *lif)
359 if (lif->notifyqcq) {
360 ionic_qcq_free(lif->notifyqcq);
361 lif->notifyqcq = NULL;
365 ionic_qcq_free(lif->adminqcq);
366 lif->adminqcq = NULL;
370 rte_memzone_free(lif->info_z);
376 ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
378 struct ionic_dev *idev = &lif->adapter->idev;
380 if (!(qcq->flags & IONIC_QCQ_F_INITED))
383 if (qcq->flags & IONIC_QCQ_F_INTR)
384 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
385 IONIC_INTR_MASK_SET);
387 qcq->flags &= ~IONIC_QCQ_F_INITED;
391 ionic_adminq_service(struct ionic_cq *cq, uint32_t cq_desc_index,
392 void *cb_arg __rte_unused)
394 struct ionic_admin_comp *cq_desc_base = cq->base;
395 struct ionic_admin_comp *cq_desc = &cq_desc_base[cq_desc_index];
397 if (!color_match(cq_desc->color, cq->done_color))
400 ionic_q_service(cq->bound_q, cq_desc_index, cq_desc->comp_index, NULL);
405 /* This acts like ionic_napi */
407 ionic_qcq_service(struct ionic_qcq *qcq, int budget, ionic_cq_cb cb,
410 struct ionic_cq *cq = &qcq->cq;
413 work_done = ionic_cq_service(cq, budget, cb, cb_arg);
419 ionic_link_status_check(struct ionic_lif *lif)
421 struct ionic_adapter *adapter = lif->adapter;
424 lif->state &= ~IONIC_LIF_F_LINK_CHECK_NEEDED;
429 link_up = (lif->info->status.link_status == IONIC_PORT_OPER_STATUS_UP);
431 if ((link_up && adapter->link_up) ||
432 (!link_up && !adapter->link_up))
436 IONIC_PRINT(DEBUG, "Link up - %d Gbps",
437 lif->info->status.link_speed);
438 adapter->link_speed = lif->info->status.link_speed;
440 IONIC_PRINT(DEBUG, "Link down");
443 adapter->link_up = link_up;
447 ionic_notifyq_cb(struct ionic_cq *cq, uint32_t cq_desc_index, void *cb_arg)
449 union ionic_notifyq_comp *cq_desc_base = cq->base;
450 union ionic_notifyq_comp *cq_desc = &cq_desc_base[cq_desc_index];
451 struct ionic_lif *lif = cb_arg;
453 IONIC_PRINT(DEBUG, "Notifyq callback eid = %jd ecode = %d",
454 cq_desc->event.eid, cq_desc->event.ecode);
456 /* Have we run out of new completions to process? */
457 if (!(cq_desc->event.eid > lif->last_eid))
460 lif->last_eid = cq_desc->event.eid;
462 switch (cq_desc->event.ecode) {
463 case IONIC_EVENT_LINK_CHANGE:
465 "Notifyq IONIC_EVENT_LINK_CHANGE eid=%jd link_status=%d link_speed=%d",
467 cq_desc->link_change.link_status,
468 cq_desc->link_change.link_speed);
470 lif->state |= IONIC_LIF_F_LINK_CHECK_NEEDED;
474 IONIC_PRINT(WARNING, "Notifyq bad event ecode=%d eid=%jd",
475 cq_desc->event.ecode, cq_desc->event.eid);
483 ionic_notifyq_handler(struct ionic_lif *lif, int budget)
485 struct ionic_dev *idev = &lif->adapter->idev;
486 struct ionic_qcq *qcq = lif->notifyqcq;
489 if (!(qcq->flags & IONIC_QCQ_F_INITED)) {
490 IONIC_PRINT(DEBUG, "Notifyq not yet initialized");
494 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
495 IONIC_INTR_MASK_SET);
497 work_done = ionic_qcq_service(qcq, budget, ionic_notifyq_cb, lif);
499 if (lif->state & IONIC_LIF_F_LINK_CHECK_NEEDED)
500 ionic_link_status_check(lif);
502 ionic_intr_credits(idev->intr_ctrl, qcq->intr.index,
503 work_done, IONIC_INTR_CRED_RESET_COALESCE);
505 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
506 IONIC_INTR_MASK_CLEAR);
512 ionic_lif_adminq_init(struct ionic_lif *lif)
514 struct ionic_dev *idev = &lif->adapter->idev;
515 struct ionic_qcq *qcq = lif->adminqcq;
516 struct ionic_queue *q = &qcq->q;
517 struct ionic_q_init_comp comp;
520 ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
521 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
525 ionic_dev_cmd_comp(idev, &comp);
527 q->hw_type = comp.hw_type;
528 q->hw_index = comp.hw_index;
529 q->db = ionic_db_map(lif, q);
531 IONIC_PRINT(DEBUG, "adminq->hw_type %d", q->hw_type);
532 IONIC_PRINT(DEBUG, "adminq->hw_index %d", q->hw_index);
533 IONIC_PRINT(DEBUG, "adminq->db %p", q->db);
535 if (qcq->flags & IONIC_QCQ_F_INTR)
536 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
537 IONIC_INTR_MASK_CLEAR);
539 qcq->flags |= IONIC_QCQ_F_INITED;
545 ionic_lif_notifyq_init(struct ionic_lif *lif)
547 struct ionic_dev *idev = &lif->adapter->idev;
548 struct ionic_qcq *qcq = lif->notifyqcq;
549 struct ionic_queue *q = &qcq->q;
552 struct ionic_admin_ctx ctx = {
553 .pending_work = true,
555 .opcode = IONIC_CMD_Q_INIT,
556 .lif_index = lif->index,
559 .flags = (IONIC_QINIT_F_IRQ | IONIC_QINIT_F_ENA),
560 .intr_index = qcq->intr.index,
562 .ring_size = rte_log2_u32(q->num_descs),
563 .ring_base = q->base_pa,
567 IONIC_PRINT(DEBUG, "notifyq_init.pid %d", ctx.cmd.q_init.pid);
568 IONIC_PRINT(DEBUG, "notifyq_init.index %d",
569 ctx.cmd.q_init.index);
570 IONIC_PRINT(DEBUG, "notifyq_init.ring_base 0x%" PRIx64 "",
571 ctx.cmd.q_init.ring_base);
572 IONIC_PRINT(DEBUG, "notifyq_init.ring_size %d",
573 ctx.cmd.q_init.ring_size);
575 err = ionic_adminq_post_wait(lif, &ctx);
579 q->hw_type = ctx.comp.q_init.hw_type;
580 q->hw_index = ctx.comp.q_init.hw_index;
583 IONIC_PRINT(DEBUG, "notifyq->hw_type %d", q->hw_type);
584 IONIC_PRINT(DEBUG, "notifyq->hw_index %d", q->hw_index);
585 IONIC_PRINT(DEBUG, "notifyq->db %p", q->db);
587 if (qcq->flags & IONIC_QCQ_F_INTR)
588 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
589 IONIC_INTR_MASK_CLEAR);
591 qcq->flags |= IONIC_QCQ_F_INITED;
597 ionic_lif_init(struct ionic_lif *lif)
599 struct ionic_dev *idev = &lif->adapter->idev;
600 struct ionic_q_init_comp comp;
603 ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
604 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
605 ionic_dev_cmd_comp(idev, &comp);
609 lif->hw_index = comp.hw_index;
611 err = ionic_lif_adminq_init(lif);
615 err = ionic_lif_notifyq_init(lif);
617 goto err_out_adminq_deinit;
619 lif->state |= IONIC_LIF_F_INITED;
623 err_out_adminq_deinit:
624 ionic_lif_qcq_deinit(lif, lif->adminqcq);
630 ionic_lif_deinit(struct ionic_lif *lif)
632 if (!(lif->state & IONIC_LIF_F_INITED))
635 ionic_lif_qcq_deinit(lif, lif->notifyqcq);
636 ionic_lif_qcq_deinit(lif, lif->adminqcq);
638 lif->state &= ~IONIC_LIF_F_INITED;
642 ionic_lif_identify(struct ionic_adapter *adapter)
644 struct ionic_dev *idev = &adapter->idev;
645 struct ionic_identity *ident = &adapter->ident;
648 unsigned int lif_words = sizeof(ident->lif.words) /
649 sizeof(ident->lif.words[0]);
650 unsigned int cmd_words = sizeof(idev->dev_cmd->data) /
651 sizeof(idev->dev_cmd->data[0]);
654 ionic_dev_cmd_lif_identify(idev, IONIC_LIF_TYPE_CLASSIC,
655 IONIC_IDENTITY_VERSION_1);
656 err = ionic_dev_cmd_wait_check(idev, IONIC_DEVCMD_TIMEOUT);
660 nwords = RTE_MIN(lif_words, cmd_words);
661 for (i = 0; i < nwords; i++)
662 ident->lif.words[i] = ioread32(&idev->dev_cmd->data[i]);
664 IONIC_PRINT(INFO, "capabilities 0x%" PRIx64 " ",
665 ident->lif.capabilities);
667 IONIC_PRINT(INFO, "eth.max_ucast_filters 0x%" PRIx32 " ",
668 ident->lif.eth.max_ucast_filters);
669 IONIC_PRINT(INFO, "eth.max_mcast_filters 0x%" PRIx32 " ",
670 ident->lif.eth.max_mcast_filters);
672 IONIC_PRINT(INFO, "eth.features 0x%" PRIx64 " ",
673 ident->lif.eth.config.features);
674 IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_ADMINQ] 0x%" PRIx32 " ",
675 ident->lif.eth.config.queue_count[IONIC_QTYPE_ADMINQ]);
676 IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] 0x%" PRIx32 " ",
677 ident->lif.eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]);
678 IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_RXQ] 0x%" PRIx32 " ",
679 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ]);
680 IONIC_PRINT(INFO, "eth.queue_count[IONIC_QTYPE_TXQ] 0x%" PRIx32 " ",
681 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ]);
687 ionic_lifs_size(struct ionic_adapter *adapter)
689 struct ionic_identity *ident = &adapter->ident;
690 uint32_t nlifs = ident->dev.nlifs;
691 uint32_t nintrs, dev_nintrs = ident->dev.nintrs;
693 adapter->max_ntxqs_per_lif =
694 ident->lif.eth.config.queue_count[IONIC_QTYPE_TXQ];
695 adapter->max_nrxqs_per_lif =
696 ident->lif.eth.config.queue_count[IONIC_QTYPE_RXQ];
698 nintrs = nlifs * 1 /* notifyq */;
700 if (nintrs > dev_nintrs) {
701 IONIC_PRINT(ERR, "At most %d intr queues supported, minimum required is %u",
706 adapter->nintrs = nintrs;