X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fbaseband%2Ffpga_5gnr_fec%2Frte_fpga_5gnr_fec.c;h=bd99fb4881745023a5148bab0a2124d136b0e992;hb=8b3799531b9bc46fb518c4ec84f4960ecc78a0d0;hp=d9c1883d26bca24f60857be9dcf3bac015439a00;hpb=2d4306438c926208812f6170844c9e49eb577ead;p=dpdk.git diff --git a/drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c b/drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c index d9c1883d26..bd99fb4881 100644 --- a/drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c +++ b/drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c @@ -23,8 +23,11 @@ #include "fpga_5gnr_fec.h" #include "rte_pmd_fpga_5gnr_fec.h" -/* 5GNR SW PMD logging ID */ -static int fpga_5gnr_fec_logtype; +#ifdef RTE_LIBRTE_BBDEV_DEBUG +RTE_LOG_REGISTER(fpga_5gnr_fec_logtype, pmd.bb.fpga_5gnr_fec, DEBUG); +#else +RTE_LOG_REGISTER(fpga_5gnr_fec_logtype, pmd.bb.fpga_5gnr_fec, NOTICE); +#endif #ifdef RTE_LIBRTE_BBDEV_DEBUG @@ -324,6 +327,8 @@ fpga_dev_info_get(struct rte_bbdev *dev, RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE | RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE | RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE | + RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK | + RTE_BBDEV_LDPC_DEC_INTERRUPTS | RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS, .llr_size = 6, .llr_decimals = 2, @@ -635,14 +640,152 @@ fpga_queue_stop(struct rte_bbdev *dev, uint16_t queue_id) return 0; } +static inline uint16_t +get_queue_id(struct rte_bbdev_data *data, uint8_t q_idx) +{ + uint16_t queue_id; + + for (queue_id = 0; queue_id < data->num_queues; ++queue_id) { + struct fpga_queue *q = data->queues[queue_id].queue_private; + if (q != NULL && q->q_idx == q_idx) + return queue_id; + } + + return -1; +} + +/* Interrupt handler triggered by FPGA dev for handling specific interrupt */ +static void +fpga_dev_interrupt_handler(void *cb_arg) +{ + struct rte_bbdev *dev = cb_arg; + struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private; + struct fpga_queue *q; + uint64_t ring_head; + uint64_t q_idx; + uint16_t queue_id; + uint8_t i; + + /* Scan queue assigned to this device */ + for (i = 0; i < FPGA_TOTAL_NUM_QUEUES; ++i) { + q_idx = 1ULL << i; + if (fpga_dev->q_bound_bit_map & q_idx) { + queue_id = get_queue_id(dev->data, i); + if (queue_id == (uint16_t) -1) + continue; + + /* Check if completion head was changed */ + q = dev->data->queues[queue_id].queue_private; + ring_head = *q->ring_head_addr; + if (q->shadow_completion_head != ring_head && + q->irq_enable == 1) { + q->shadow_completion_head = ring_head; + rte_bbdev_pmd_callback_process( + dev, + RTE_BBDEV_EVENT_DEQUEUE, + &queue_id); + } + } + } +} + +static int +fpga_queue_intr_enable(struct rte_bbdev *dev, uint16_t queue_id) +{ + struct fpga_queue *q = dev->data->queues[queue_id].queue_private; + + if (!rte_intr_cap_multiple(dev->intr_handle)) + return -ENOTSUP; + + q->irq_enable = 1; + + return 0; +} + +static int +fpga_queue_intr_disable(struct rte_bbdev *dev, uint16_t queue_id) +{ + struct fpga_queue *q = dev->data->queues[queue_id].queue_private; + q->irq_enable = 0; + + return 0; +} + +static int +fpga_intr_enable(struct rte_bbdev *dev) +{ + int ret; + uint8_t i; + + if (!rte_intr_cap_multiple(dev->intr_handle)) { + rte_bbdev_log(ERR, "Multiple intr vector is not supported by FPGA (%s)", + dev->data->name); + return -ENOTSUP; + } + + /* Create event file descriptors for each of 64 queue. Event fds will be + * mapped to FPGA IRQs in rte_intr_enable(). This is a 1:1 mapping where + * the IRQ number is a direct translation to the queue number. + * + * 63 (FPGA_NUM_INTR_VEC) event fds are created as rte_intr_enable() + * mapped the first IRQ to already created interrupt event file + * descriptor (intr_handle->fd). + */ + if (rte_intr_efd_enable(dev->intr_handle, FPGA_NUM_INTR_VEC)) { + rte_bbdev_log(ERR, "Failed to create fds for %u queues", + dev->data->num_queues); + return -1; + } + + /* TODO Each event file descriptor is overwritten by interrupt event + * file descriptor. That descriptor is added to epoll observed list. + * It ensures that callback function assigned to that descriptor will + * invoked when any FPGA queue issues interrupt. + */ + for (i = 0; i < FPGA_NUM_INTR_VEC; ++i) + dev->intr_handle->efds[i] = dev->intr_handle->fd; + + if (!dev->intr_handle->intr_vec) { + dev->intr_handle->intr_vec = rte_zmalloc("intr_vec", + dev->data->num_queues * sizeof(int), 0); + if (!dev->intr_handle->intr_vec) { + rte_bbdev_log(ERR, "Failed to allocate %u vectors", + dev->data->num_queues); + return -ENOMEM; + } + } + + ret = rte_intr_enable(dev->intr_handle); + if (ret < 0) { + rte_bbdev_log(ERR, + "Couldn't enable interrupts for device: %s", + dev->data->name); + return ret; + } + + ret = rte_intr_callback_register(dev->intr_handle, + fpga_dev_interrupt_handler, dev); + if (ret < 0) { + rte_bbdev_log(ERR, + "Couldn't register interrupt callback for device: %s", + dev->data->name); + return ret; + } + + return 0; +} + static const struct rte_bbdev_ops fpga_ops = { .setup_queues = fpga_setup_queues, + .intr_enable = fpga_intr_enable, .close = fpga_dev_close, .info_get = fpga_dev_info_get, .queue_setup = fpga_queue_setup, .queue_stop = fpga_queue_stop, .queue_start = fpga_queue_start, .queue_release = fpga_queue_release, + .queue_intr_enable = fpga_queue_intr_enable, + .queue_intr_disable = fpga_queue_intr_disable }; static inline void @@ -807,14 +950,14 @@ fpga_dma_desc_te_fill(struct rte_bbdev_enc_op *op, desc->num_null = op->ldpc_enc.n_filler; /* Set inbound data buffer address */ desc->in_addr_hi = (uint32_t)( - rte_pktmbuf_mtophys_offset(input, in_offset) >> 32); + rte_pktmbuf_iova_offset(input, in_offset) >> 32); desc->in_addr_lw = (uint32_t)( - rte_pktmbuf_mtophys_offset(input, in_offset)); + rte_pktmbuf_iova_offset(input, in_offset)); desc->out_addr_hi = (uint32_t)( - rte_pktmbuf_mtophys_offset(output, out_offset) >> 32); + rte_pktmbuf_iova_offset(output, out_offset) >> 32); desc->out_addr_lw = (uint32_t)( - rte_pktmbuf_mtophys_offset(output, out_offset)); + rte_pktmbuf_iova_offset(output, out_offset)); /* Save software context needed for dequeue */ desc->op_addr = op; /* Set total number of CBs in an op */ @@ -855,9 +998,9 @@ fpga_dma_desc_ld_fill(struct rte_bbdev_dec_op *op, desc->error = 0; /* Set inbound data buffer address */ desc->in_addr_hi = (uint32_t)( - rte_pktmbuf_mtophys_offset(input, in_offset) >> 32); + rte_pktmbuf_iova_offset(input, in_offset) >> 32); desc->in_addr_lw = (uint32_t)( - rte_pktmbuf_mtophys_offset(input, in_offset)); + rte_pktmbuf_iova_offset(input, in_offset)); desc->rm_e = op->ldpc_dec.cb_params.e; desc->harq_input_length = harq_in_length; desc->et_dis = !check_bit(op->ldpc_dec.op_flags, @@ -878,9 +1021,9 @@ fpga_dma_desc_ld_fill(struct rte_bbdev_dec_op *op, desc->max_iter = op->ldpc_dec.iter_max; desc->qm_idx = op->ldpc_dec.q_m / 2; desc->out_addr_hi = (uint32_t)( - rte_pktmbuf_mtophys_offset(output, out_offset) >> 32); + rte_pktmbuf_iova_offset(output, out_offset) >> 32); desc->out_addr_lw = (uint32_t)( - rte_pktmbuf_mtophys_offset(output, out_offset)); + rte_pktmbuf_iova_offset(output, out_offset)); /* Save software context needed for dequeue */ desc->op_addr = op; /* Set total number of CBs in an op */ @@ -1054,6 +1197,140 @@ validate_dec_op(struct rte_bbdev_dec_op *op __rte_unused) } #endif +static inline int +fpga_harq_write_loopback(struct fpga_5gnr_fec_device *fpga_dev, + struct rte_mbuf *harq_input, uint16_t harq_in_length, + uint32_t harq_in_offset, uint32_t harq_out_offset) +{ + uint32_t out_offset = harq_out_offset; + uint32_t in_offset = harq_in_offset; + uint32_t left_length = harq_in_length; + uint32_t reg_32, increment = 0; + uint64_t *input = NULL; + uint32_t last_transaction = left_length + % FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES; + uint64_t last_word; + + if (last_transaction > 0) + left_length -= last_transaction; + + /* + * Get HARQ buffer size for each VF/PF: When 0x00, there is no + * available DDR space for the corresponding VF/PF. + */ + reg_32 = fpga_reg_read_32(fpga_dev->mmio_base, + FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS); + if (reg_32 < harq_in_length) { + left_length = reg_32; + rte_bbdev_log(ERR, "HARQ in length > HARQ buffer size\n"); + } + + input = (uint64_t *)rte_pktmbuf_mtod_offset(harq_input, + uint8_t *, in_offset); + + while (left_length > 0) { + if (fpga_reg_read_8(fpga_dev->mmio_base, + FPGA_5GNR_FEC_DDR4_ADDR_RDY_REGS) == 1) { + fpga_reg_write_32(fpga_dev->mmio_base, + FPGA_5GNR_FEC_DDR4_WR_ADDR_REGS, + out_offset); + fpga_reg_write_64(fpga_dev->mmio_base, + FPGA_5GNR_FEC_DDR4_WR_DATA_REGS, + input[increment]); + left_length -= FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES; + out_offset += FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES; + increment++; + fpga_reg_write_8(fpga_dev->mmio_base, + FPGA_5GNR_FEC_DDR4_WR_DONE_REGS, 1); + } + } + while (last_transaction > 0) { + if (fpga_reg_read_8(fpga_dev->mmio_base, + FPGA_5GNR_FEC_DDR4_ADDR_RDY_REGS) == 1) { + fpga_reg_write_32(fpga_dev->mmio_base, + FPGA_5GNR_FEC_DDR4_WR_ADDR_REGS, + out_offset); + last_word = input[increment]; + last_word &= (uint64_t)(1 << (last_transaction * 4)) + - 1; + fpga_reg_write_64(fpga_dev->mmio_base, + FPGA_5GNR_FEC_DDR4_WR_DATA_REGS, + last_word); + fpga_reg_write_8(fpga_dev->mmio_base, + FPGA_5GNR_FEC_DDR4_WR_DONE_REGS, 1); + last_transaction = 0; + } + } + return 1; +} + +static inline int +fpga_harq_read_loopback(struct fpga_5gnr_fec_device *fpga_dev, + struct rte_mbuf *harq_output, uint16_t harq_in_length, + uint32_t harq_in_offset, uint32_t harq_out_offset) +{ + uint32_t left_length, in_offset = harq_in_offset; + uint64_t reg; + uint32_t increment = 0; + uint64_t *input = NULL; + uint32_t last_transaction = harq_in_length + % FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES; + + if (last_transaction > 0) + harq_in_length += (8 - last_transaction); + + reg = fpga_reg_read_32(fpga_dev->mmio_base, + FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS); + if (reg < harq_in_length) { + harq_in_length = reg; + rte_bbdev_log(ERR, "HARQ in length > HARQ buffer size\n"); + } + + if (!mbuf_append(harq_output, harq_output, harq_in_length)) { + rte_bbdev_log(ERR, "HARQ output buffer warning %d %d\n", + harq_output->buf_len - + rte_pktmbuf_headroom(harq_output), + harq_in_length); + harq_in_length = harq_output->buf_len - + rte_pktmbuf_headroom(harq_output); + if (!mbuf_append(harq_output, harq_output, harq_in_length)) { + rte_bbdev_log(ERR, "HARQ output buffer issue %d %d\n", + harq_output->buf_len, harq_in_length); + return -1; + } + } + left_length = harq_in_length; + + input = (uint64_t *)rte_pktmbuf_mtod_offset(harq_output, + uint8_t *, harq_out_offset); + + while (left_length > 0) { + fpga_reg_write_32(fpga_dev->mmio_base, + FPGA_5GNR_FEC_DDR4_RD_ADDR_REGS, in_offset); + fpga_reg_write_8(fpga_dev->mmio_base, + FPGA_5GNR_FEC_DDR4_RD_DONE_REGS, 1); + reg = fpga_reg_read_8(fpga_dev->mmio_base, + FPGA_5GNR_FEC_DDR4_RD_RDY_REGS); + while (reg != 1) { + reg = fpga_reg_read_8(fpga_dev->mmio_base, + FPGA_5GNR_FEC_DDR4_RD_RDY_REGS); + if (reg == FPGA_DDR_OVERFLOW) { + rte_bbdev_log(ERR, + "Read address is overflow!\n"); + return -1; + } + } + input[increment] = fpga_reg_read_64(fpga_dev->mmio_base, + FPGA_5GNR_FEC_DDR4_RD_DATA_REGS); + left_length -= FPGA_5GNR_FEC_DDR_RD_DATA_LEN_IN_BYTES; + in_offset += FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES; + increment++; + fpga_reg_write_8(fpga_dev->mmio_base, + FPGA_5GNR_FEC_DDR4_RD_DONE_REGS, 0); + } + return 1; +} + static inline int enqueue_ldpc_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op *op, uint16_t desc_offset) @@ -1182,6 +1459,42 @@ enqueue_ldpc_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op *op, ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); desc = q->ring_addr + ring_offset; + if (check_bit(dec->op_flags, + RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { + struct rte_mbuf *harq_in = dec->harq_combined_input.data; + struct rte_mbuf *harq_out = dec->harq_combined_output.data; + harq_in_length = dec->harq_combined_input.length; + uint32_t harq_in_offset = dec->harq_combined_input.offset; + uint32_t harq_out_offset = dec->harq_combined_output.offset; + + if (check_bit(dec->op_flags, + RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE + )) { + ret = fpga_harq_write_loopback(q->d, harq_in, + harq_in_length, harq_in_offset, + harq_out_offset); + } else if (check_bit(dec->op_flags, + RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE + )) { + ret = fpga_harq_read_loopback(q->d, harq_out, + harq_in_length, harq_in_offset, + harq_out_offset); + dec->harq_combined_output.length = harq_in_length; + } else { + rte_bbdev_log(ERR, "OP flag Err!"); + ret = -1; + } + /* Set descriptor for dequeue */ + desc->dec_req.done = 1; + desc->dec_req.error = 0; + desc->dec_req.op_addr = op; + desc->dec_req.cbs_in_op = 1; + /* Mark this dummy descriptor to be dropped by HW */ + desc->dec_req.desc_idx = (ring_offset + 1) + & q->sw_ring_wrap_mask; + return ret; /* Error or number of CB */ + } + if (m_in == NULL || m_out == NULL) { rte_bbdev_log(ERR, "Invalid mbuf pointer"); op->status = 1 << RTE_BBDEV_DATA_ERROR; @@ -1526,7 +1839,7 @@ fpga_5gnr_fec_init(struct rte_bbdev *dev, struct rte_pci_driver *drv) rte_bbdev_log_debug( "Init device %s [%s] @ virtaddr %p phyaddr %#"PRIx64, - dev->device->driver->name, dev->data->name, + drv->driver.name, dev->data->name, (void *)pci_dev->mem_resource[0].addr, pci_dev->mem_resource[0].phys_addr); } @@ -1582,7 +1895,7 @@ fpga_5gnr_fec_probe(struct rte_pci_driver *pci_drv, ((uint16_t)(version_id >> 16)), ((uint16_t)version_id)); #ifdef RTE_LIBRTE_BBDEV_DEBUG - if (!strcmp(bbdev->device->driver->name, + if (!strcmp(pci_drv->driver.name, RTE_STR(FPGA_5GNR_FEC_PF_DRIVER_NAME))) print_static_reg_debug_info(d->mmio_base); #endif @@ -1631,10 +1944,10 @@ fpga_5gnr_fec_remove(struct rte_pci_device *pci_dev) } static inline void -set_default_fpga_conf(struct fpga_5gnr_fec_conf *def_conf) +set_default_fpga_conf(struct rte_fpga_5gnr_fec_conf *def_conf) { /* clear default configuration before initialization */ - memset(def_conf, 0, sizeof(struct fpga_5gnr_fec_conf)); + memset(def_conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf)); /* Set pf mode to true */ def_conf->pf_mode_en = true; @@ -1649,15 +1962,15 @@ set_default_fpga_conf(struct fpga_5gnr_fec_conf *def_conf) /* Initial configuration of FPGA 5GNR FEC device */ int -fpga_5gnr_fec_configure(const char *dev_name, - const struct fpga_5gnr_fec_conf *conf) +rte_fpga_5gnr_fec_configure(const char *dev_name, + const struct rte_fpga_5gnr_fec_conf *conf) { uint32_t payload_32, address; uint16_t payload_16; uint8_t payload_8; uint16_t q_id, vf_id, total_q_id, total_ul_q_id, total_dl_q_id; struct rte_bbdev *bbdev = rte_bbdev_get_named_dev(dev_name); - struct fpga_5gnr_fec_conf def_conf; + struct rte_fpga_5gnr_fec_conf def_conf; if (bbdev == NULL) { rte_bbdev_log(ERR, @@ -1864,14 +2177,3 @@ RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_PF_DRIVER_NAME, RTE_PMD_REGISTER_PCI(FPGA_5GNR_FEC_VF_DRIVER_NAME, fpga_5gnr_fec_pci_vf_driver); RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_VF_DRIVER_NAME, pci_id_fpga_5gnr_fec_vf_map); - -RTE_INIT(fpga_5gnr_fec_init_log) -{ - fpga_5gnr_fec_logtype = rte_log_register("pmd.bb.fpga_5gnr_fec"); - if (fpga_5gnr_fec_logtype >= 0) -#ifdef RTE_LIBRTE_BBDEV_DEBUG - rte_log_set_level(fpga_5gnr_fec_logtype, RTE_LOG_DEBUG); -#else - rte_log_set_level(fpga_5gnr_fec_logtype, RTE_LOG_NOTICE); -#endif -}