#include "fpga_5gnr_fec.h"
#include "rte_pmd_fpga_5gnr_fec.h"
-/* 5GNR SW PMD logging ID */
-static int fpga_5gnr_fec_logtype;
+#ifdef RTE_LIBRTE_BBDEV_DEBUG
+RTE_LOG_REGISTER_DEFAULT(fpga_5gnr_fec_logtype, DEBUG);
+#else
+RTE_LOG_REGISTER_DEFAULT(fpga_5gnr_fec_logtype, NOTICE);
+#endif
#ifdef RTE_LIBRTE_BBDEV_DEBUG
RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE |
RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE |
RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE |
+ RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK |
+ RTE_BBDEV_LDPC_DEC_INTERRUPTS |
RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS,
.llr_size = 6,
.llr_decimals = 2,
dev_info->default_queue_conf = default_queue_conf;
dev_info->capabilities = bbdev_capabilities;
dev_info->cpu_flag_reqs = NULL;
+ dev_info->data_endianness = RTE_LITTLE_ENDIAN;
/* Calculates number of queues assigned to device */
dev_info->max_num_queues = 0;
return 0;
}
+static inline uint16_t
+get_queue_id(struct rte_bbdev_data *data, uint8_t q_idx)
+{
+ uint16_t queue_id;
+
+ for (queue_id = 0; queue_id < data->num_queues; ++queue_id) {
+ struct fpga_queue *q = data->queues[queue_id].queue_private;
+ if (q != NULL && q->q_idx == q_idx)
+ return queue_id;
+ }
+
+ return -1;
+}
+
+/* Interrupt handler triggered by FPGA dev for handling specific interrupt */
+static void
+fpga_dev_interrupt_handler(void *cb_arg)
+{
+ struct rte_bbdev *dev = cb_arg;
+ struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private;
+ struct fpga_queue *q;
+ uint64_t ring_head;
+ uint64_t q_idx;
+ uint16_t queue_id;
+ uint8_t i;
+
+ /* Scan queue assigned to this device */
+ for (i = 0; i < FPGA_TOTAL_NUM_QUEUES; ++i) {
+ q_idx = 1ULL << i;
+ if (fpga_dev->q_bound_bit_map & q_idx) {
+ queue_id = get_queue_id(dev->data, i);
+ if (queue_id == (uint16_t) -1)
+ continue;
+
+ /* Check if completion head was changed */
+ q = dev->data->queues[queue_id].queue_private;
+ ring_head = *q->ring_head_addr;
+ if (q->shadow_completion_head != ring_head &&
+ q->irq_enable == 1) {
+ q->shadow_completion_head = ring_head;
+ rte_bbdev_pmd_callback_process(
+ dev,
+ RTE_BBDEV_EVENT_DEQUEUE,
+ &queue_id);
+ }
+ }
+ }
+}
+
+static int
+fpga_queue_intr_enable(struct rte_bbdev *dev, uint16_t queue_id)
+{
+ struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
+
+ if (!rte_intr_cap_multiple(dev->intr_handle))
+ return -ENOTSUP;
+
+ q->irq_enable = 1;
+
+ return 0;
+}
+
+static int
+fpga_queue_intr_disable(struct rte_bbdev *dev, uint16_t queue_id)
+{
+ struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
+ q->irq_enable = 0;
+
+ return 0;
+}
+
+static int
+fpga_intr_enable(struct rte_bbdev *dev)
+{
+ int ret;
+ uint8_t i;
+
+ if (!rte_intr_cap_multiple(dev->intr_handle)) {
+ rte_bbdev_log(ERR, "Multiple intr vector is not supported by FPGA (%s)",
+ dev->data->name);
+ return -ENOTSUP;
+ }
+
+ /* Create event file descriptors for each of 64 queue. Event fds will be
+ * mapped to FPGA IRQs in rte_intr_enable(). This is a 1:1 mapping where
+ * the IRQ number is a direct translation to the queue number.
+ *
+ * 63 (FPGA_NUM_INTR_VEC) event fds are created as rte_intr_enable()
+ * mapped the first IRQ to already created interrupt event file
+ * descriptor (intr_handle->fd).
+ */
+ if (rte_intr_efd_enable(dev->intr_handle, FPGA_NUM_INTR_VEC)) {
+ rte_bbdev_log(ERR, "Failed to create fds for %u queues",
+ dev->data->num_queues);
+ return -1;
+ }
+
+ /* TODO Each event file descriptor is overwritten by interrupt event
+ * file descriptor. That descriptor is added to epoll observed list.
+ * It ensures that callback function assigned to that descriptor will
+ * invoked when any FPGA queue issues interrupt.
+ */
+ for (i = 0; i < FPGA_NUM_INTR_VEC; ++i) {
+ if (rte_intr_efds_index_set(dev->intr_handle, i,
+ rte_intr_fd_get(dev->intr_handle)))
+ return -rte_errno;
+ }
+
+ if (rte_intr_vec_list_alloc(dev->intr_handle, "intr_vec",
+ dev->data->num_queues)) {
+ rte_bbdev_log(ERR, "Failed to allocate %u vectors",
+ dev->data->num_queues);
+ return -ENOMEM;
+ }
+
+ ret = rte_intr_enable(dev->intr_handle);
+ if (ret < 0) {
+ rte_bbdev_log(ERR,
+ "Couldn't enable interrupts for device: %s",
+ dev->data->name);
+ return ret;
+ }
+
+ ret = rte_intr_callback_register(dev->intr_handle,
+ fpga_dev_interrupt_handler, dev);
+ if (ret < 0) {
+ rte_bbdev_log(ERR,
+ "Couldn't register interrupt callback for device: %s",
+ dev->data->name);
+ return ret;
+ }
+
+ return 0;
+}
+
static const struct rte_bbdev_ops fpga_ops = {
.setup_queues = fpga_setup_queues,
+ .intr_enable = fpga_intr_enable,
.close = fpga_dev_close,
.info_get = fpga_dev_info_get,
.queue_setup = fpga_queue_setup,
.queue_stop = fpga_queue_stop,
.queue_start = fpga_queue_start,
.queue_release = fpga_queue_release,
+ .queue_intr_enable = fpga_queue_intr_enable,
+ .queue_intr_disable = fpga_queue_intr_disable
};
static inline void
desc->num_null = op->ldpc_enc.n_filler;
/* Set inbound data buffer address */
desc->in_addr_hi = (uint32_t)(
- rte_pktmbuf_mtophys_offset(input, in_offset) >> 32);
+ rte_pktmbuf_iova_offset(input, in_offset) >> 32);
desc->in_addr_lw = (uint32_t)(
- rte_pktmbuf_mtophys_offset(input, in_offset));
+ rte_pktmbuf_iova_offset(input, in_offset));
desc->out_addr_hi = (uint32_t)(
- rte_pktmbuf_mtophys_offset(output, out_offset) >> 32);
+ rte_pktmbuf_iova_offset(output, out_offset) >> 32);
desc->out_addr_lw = (uint32_t)(
- rte_pktmbuf_mtophys_offset(output, out_offset));
+ rte_pktmbuf_iova_offset(output, out_offset));
/* Save software context needed for dequeue */
desc->op_addr = op;
/* Set total number of CBs in an op */
desc->error = 0;
/* Set inbound data buffer address */
desc->in_addr_hi = (uint32_t)(
- rte_pktmbuf_mtophys_offset(input, in_offset) >> 32);
+ rte_pktmbuf_iova_offset(input, in_offset) >> 32);
desc->in_addr_lw = (uint32_t)(
- rte_pktmbuf_mtophys_offset(input, in_offset));
+ rte_pktmbuf_iova_offset(input, in_offset));
desc->rm_e = op->ldpc_dec.cb_params.e;
desc->harq_input_length = harq_in_length;
desc->et_dis = !check_bit(op->ldpc_dec.op_flags,
desc->max_iter = op->ldpc_dec.iter_max;
desc->qm_idx = op->ldpc_dec.q_m / 2;
desc->out_addr_hi = (uint32_t)(
- rte_pktmbuf_mtophys_offset(output, out_offset) >> 32);
+ rte_pktmbuf_iova_offset(output, out_offset) >> 32);
desc->out_addr_lw = (uint32_t)(
- rte_pktmbuf_mtophys_offset(output, out_offset));
+ rte_pktmbuf_iova_offset(output, out_offset));
/* Save software context needed for dequeue */
desc->op_addr = op;
/* Set total number of CBs in an op */
ldpc_enc->basegraph);
return -1;
}
- if (ldpc_enc->code_block_mode > 1) {
+ if (ldpc_enc->code_block_mode > RTE_BBDEV_CODE_BLOCK) {
rte_bbdev_log(ERR,
"code_block_mode (%u) is out of range 0:Tb 1:CB",
ldpc_enc->code_block_mode);
return -1;
}
- if (ldpc_enc->code_block_mode == 0) {
+ if (ldpc_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
tb = &ldpc_enc->tb_params;
if (tb->c == 0) {
rte_bbdev_log(ERR,
return -1;
}
- if (ldpc_dec->code_block_mode > 1) {
+ if (ldpc_dec->code_block_mode > RTE_BBDEV_CODE_BLOCK) {
rte_bbdev_log(ERR,
"code_block_mode (%u) is out of range 0 <= value <= 1",
ldpc_dec->code_block_mode);
return -1;
}
- if (ldpc_dec->code_block_mode == 0) {
+ if (ldpc_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
tb = &ldpc_dec->tb_params;
if (tb->c < 1) {
rte_bbdev_log(ERR,
}
#endif
+static inline int
+fpga_harq_write_loopback(struct fpga_5gnr_fec_device *fpga_dev,
+ struct rte_mbuf *harq_input, uint16_t harq_in_length,
+ uint32_t harq_in_offset, uint32_t harq_out_offset)
+{
+ uint32_t out_offset = harq_out_offset;
+ uint32_t in_offset = harq_in_offset;
+ uint32_t left_length = harq_in_length;
+ uint32_t reg_32, increment = 0;
+ uint64_t *input = NULL;
+ uint32_t last_transaction = left_length
+ % FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
+ uint64_t last_word;
+
+ if (last_transaction > 0)
+ left_length -= last_transaction;
+
+ /*
+ * Get HARQ buffer size for each VF/PF: When 0x00, there is no
+ * available DDR space for the corresponding VF/PF.
+ */
+ reg_32 = fpga_reg_read_32(fpga_dev->mmio_base,
+ FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS);
+ if (reg_32 < harq_in_length) {
+ left_length = reg_32;
+ rte_bbdev_log(ERR, "HARQ in length > HARQ buffer size\n");
+ }
+
+ input = (uint64_t *)rte_pktmbuf_mtod_offset(harq_input,
+ uint8_t *, in_offset);
+
+ while (left_length > 0) {
+ if (fpga_reg_read_8(fpga_dev->mmio_base,
+ FPGA_5GNR_FEC_DDR4_ADDR_RDY_REGS) == 1) {
+ fpga_reg_write_32(fpga_dev->mmio_base,
+ FPGA_5GNR_FEC_DDR4_WR_ADDR_REGS,
+ out_offset);
+ fpga_reg_write_64(fpga_dev->mmio_base,
+ FPGA_5GNR_FEC_DDR4_WR_DATA_REGS,
+ input[increment]);
+ left_length -= FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
+ out_offset += FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
+ increment++;
+ fpga_reg_write_8(fpga_dev->mmio_base,
+ FPGA_5GNR_FEC_DDR4_WR_DONE_REGS, 1);
+ }
+ }
+ while (last_transaction > 0) {
+ if (fpga_reg_read_8(fpga_dev->mmio_base,
+ FPGA_5GNR_FEC_DDR4_ADDR_RDY_REGS) == 1) {
+ fpga_reg_write_32(fpga_dev->mmio_base,
+ FPGA_5GNR_FEC_DDR4_WR_ADDR_REGS,
+ out_offset);
+ last_word = input[increment];
+ last_word &= (uint64_t)(1 << (last_transaction * 4))
+ - 1;
+ fpga_reg_write_64(fpga_dev->mmio_base,
+ FPGA_5GNR_FEC_DDR4_WR_DATA_REGS,
+ last_word);
+ fpga_reg_write_8(fpga_dev->mmio_base,
+ FPGA_5GNR_FEC_DDR4_WR_DONE_REGS, 1);
+ last_transaction = 0;
+ }
+ }
+ return 1;
+}
+
+static inline int
+fpga_harq_read_loopback(struct fpga_5gnr_fec_device *fpga_dev,
+ struct rte_mbuf *harq_output, uint16_t harq_in_length,
+ uint32_t harq_in_offset, uint32_t harq_out_offset)
+{
+ uint32_t left_length, in_offset = harq_in_offset;
+ uint64_t reg;
+ uint32_t increment = 0;
+ uint64_t *input = NULL;
+ uint32_t last_transaction = harq_in_length
+ % FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
+
+ if (last_transaction > 0)
+ harq_in_length += (8 - last_transaction);
+
+ reg = fpga_reg_read_32(fpga_dev->mmio_base,
+ FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS);
+ if (reg < harq_in_length) {
+ harq_in_length = reg;
+ rte_bbdev_log(ERR, "HARQ in length > HARQ buffer size\n");
+ }
+
+ if (!mbuf_append(harq_output, harq_output, harq_in_length)) {
+ rte_bbdev_log(ERR, "HARQ output buffer warning %d %d\n",
+ harq_output->buf_len -
+ rte_pktmbuf_headroom(harq_output),
+ harq_in_length);
+ harq_in_length = harq_output->buf_len -
+ rte_pktmbuf_headroom(harq_output);
+ if (!mbuf_append(harq_output, harq_output, harq_in_length)) {
+ rte_bbdev_log(ERR, "HARQ output buffer issue %d %d\n",
+ harq_output->buf_len, harq_in_length);
+ return -1;
+ }
+ }
+ left_length = harq_in_length;
+
+ input = (uint64_t *)rte_pktmbuf_mtod_offset(harq_output,
+ uint8_t *, harq_out_offset);
+
+ while (left_length > 0) {
+ fpga_reg_write_32(fpga_dev->mmio_base,
+ FPGA_5GNR_FEC_DDR4_RD_ADDR_REGS, in_offset);
+ fpga_reg_write_8(fpga_dev->mmio_base,
+ FPGA_5GNR_FEC_DDR4_RD_DONE_REGS, 1);
+ reg = fpga_reg_read_8(fpga_dev->mmio_base,
+ FPGA_5GNR_FEC_DDR4_RD_RDY_REGS);
+ while (reg != 1) {
+ reg = fpga_reg_read_8(fpga_dev->mmio_base,
+ FPGA_5GNR_FEC_DDR4_RD_RDY_REGS);
+ if (reg == FPGA_DDR_OVERFLOW) {
+ rte_bbdev_log(ERR,
+ "Read address is overflow!\n");
+ return -1;
+ }
+ }
+ input[increment] = fpga_reg_read_64(fpga_dev->mmio_base,
+ FPGA_5GNR_FEC_DDR4_RD_DATA_REGS);
+ left_length -= FPGA_5GNR_FEC_DDR_RD_DATA_LEN_IN_BYTES;
+ in_offset += FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
+ increment++;
+ fpga_reg_write_8(fpga_dev->mmio_base,
+ FPGA_5GNR_FEC_DDR4_RD_DONE_REGS, 0);
+ }
+ return 1;
+}
+
static inline int
enqueue_ldpc_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op *op,
uint16_t desc_offset)
if (enc->op_flags & RTE_BBDEV_LDPC_CRC_24B_ATTACH)
crc24_bits = 24;
- if (enc->code_block_mode == 0) {
+ if (enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
/* For Transport Block mode */
/* FIXME */
c = enc->tb_params.c;
ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask);
desc = q->ring_addr + ring_offset;
+ if (check_bit(dec->op_flags,
+ RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
+ struct rte_mbuf *harq_in = dec->harq_combined_input.data;
+ struct rte_mbuf *harq_out = dec->harq_combined_output.data;
+ harq_in_length = dec->harq_combined_input.length;
+ uint32_t harq_in_offset = dec->harq_combined_input.offset;
+ uint32_t harq_out_offset = dec->harq_combined_output.offset;
+
+ if (check_bit(dec->op_flags,
+ RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE
+ )) {
+ ret = fpga_harq_write_loopback(q->d, harq_in,
+ harq_in_length, harq_in_offset,
+ harq_out_offset);
+ } else if (check_bit(dec->op_flags,
+ RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE
+ )) {
+ ret = fpga_harq_read_loopback(q->d, harq_out,
+ harq_in_length, harq_in_offset,
+ harq_out_offset);
+ dec->harq_combined_output.length = harq_in_length;
+ } else {
+ rte_bbdev_log(ERR, "OP flag Err!");
+ ret = -1;
+ }
+ /* Set descriptor for dequeue */
+ desc->dec_req.done = 1;
+ desc->dec_req.error = 0;
+ desc->dec_req.op_addr = op;
+ desc->dec_req.cbs_in_op = 1;
+ /* Mark this dummy descriptor to be dropped by HW */
+ desc->dec_req.desc_idx = (ring_offset + 1)
+ & q->sw_ring_wrap_mask;
+ return ret; /* Error or number of CB */
+ }
+
if (m_in == NULL || m_out == NULL) {
rte_bbdev_log(ERR, "Invalid mbuf pointer");
op->status = 1 << RTE_BBDEV_DATA_ERROR;
rte_bbdev_log_debug(
"Init device %s [%s] @ virtaddr %p phyaddr %#"PRIx64,
- dev->device->driver->name, dev->data->name,
+ drv->driver.name, dev->data->name,
(void *)pci_dev->mem_resource[0].addr,
pci_dev->mem_resource[0].phys_addr);
}
/* Fill HW specific part of device structure */
bbdev->device = &pci_dev->device;
- bbdev->intr_handle = &pci_dev->intr_handle;
+ bbdev->intr_handle = pci_dev->intr_handle;
bbdev->data->socket_id = pci_dev->device.numa_node;
/* Invoke FEC FPGA device initialization function */
((uint16_t)(version_id >> 16)), ((uint16_t)version_id));
#ifdef RTE_LIBRTE_BBDEV_DEBUG
- if (!strcmp(bbdev->device->driver->name,
+ if (!strcmp(pci_drv->driver.name,
RTE_STR(FPGA_5GNR_FEC_PF_DRIVER_NAME)))
print_static_reg_debug_info(d->mmio_base);
#endif
}
static inline void
-set_default_fpga_conf(struct fpga_5gnr_fec_conf *def_conf)
+set_default_fpga_conf(struct rte_fpga_5gnr_fec_conf *def_conf)
{
/* clear default configuration before initialization */
- memset(def_conf, 0, sizeof(struct fpga_5gnr_fec_conf));
+ memset(def_conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf));
/* Set pf mode to true */
def_conf->pf_mode_en = true;
/* Initial configuration of FPGA 5GNR FEC device */
int
-fpga_5gnr_fec_configure(const char *dev_name,
- const struct fpga_5gnr_fec_conf *conf)
+rte_fpga_5gnr_fec_configure(const char *dev_name,
+ const struct rte_fpga_5gnr_fec_conf *conf)
{
uint32_t payload_32, address;
uint16_t payload_16;
uint8_t payload_8;
uint16_t q_id, vf_id, total_q_id, total_ul_q_id, total_dl_q_id;
struct rte_bbdev *bbdev = rte_bbdev_get_named_dev(dev_name);
- struct fpga_5gnr_fec_conf def_conf;
+ struct rte_fpga_5gnr_fec_conf def_conf;
if (bbdev == NULL) {
rte_bbdev_log(ERR,
RTE_PMD_REGISTER_PCI(FPGA_5GNR_FEC_VF_DRIVER_NAME, fpga_5gnr_fec_pci_vf_driver);
RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_VF_DRIVER_NAME,
pci_id_fpga_5gnr_fec_vf_map);
-
-RTE_INIT(fpga_5gnr_fec_init_log)
-{
- fpga_5gnr_fec_logtype = rte_log_register("pmd.bb.fpga_5gnr_fec");
- if (fpga_5gnr_fec_logtype >= 0)
-#ifdef RTE_LIBRTE_BBDEV_DEBUG
- rte_log_set_level(fpga_5gnr_fec_logtype, RTE_LOG_DEBUG);
-#else
- rte_log_set_level(fpga_5gnr_fec_logtype, RTE_LOG_NOTICE);
-#endif
-}
git.droids-corp.org / dpdk.git / blobdiff