1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
7 #include <rte_common.h>
10 #include <rte_malloc.h>
11 #include <rte_mempool.h>
12 #include <rte_errno.h>
14 #include <rte_bus_pci.h>
15 #include <rte_byteorder.h>
16 #ifdef RTE_BBDEV_OFFLOAD_COST
17 #include <rte_cycles.h>
20 #include <rte_bbdev.h>
21 #include <rte_bbdev_pmd.h>
23 #include "fpga_5gnr_fec.h"
25 /* 5GNR SW PMD logging ID */
26 static int fpga_5gnr_fec_logtype;
29 fpga_setup_queues(struct rte_bbdev *dev, uint16_t num_queues, int socket_id)
31 /* Number of queues bound to a PF/VF */
32 uint32_t hw_q_num = 0;
33 uint32_t ring_size, payload, address, q_id, offset;
35 struct fpga_ring_ctrl_reg ring_reg;
36 struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private;
38 address = FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE;
39 if (!(fpga_reg_read_32(fpga_dev->mmio_base, address) & 0x1)) {
41 "Queue-PF/VF mapping is not set! Was PF configured for device (%s) ?",
46 /* Clear queue registers structure */
47 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg));
50 * If a queue is valid and mapped to a calling PF/VF the read value is
51 * replaced with a queue ID and if it's not then
52 * FPGA_INVALID_HW_QUEUE_ID is returned.
54 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) {
55 uint32_t hw_q_id = fpga_reg_read_32(fpga_dev->mmio_base,
56 FPGA_5GNR_FEC_QUEUE_MAP + (q_id << 2));
58 rte_bbdev_log_debug("%s: queue ID: %u, registry queue ID: %u",
59 dev->device->name, q_id, hw_q_id);
61 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID) {
62 fpga_dev->q_bound_bit_map |= (1ULL << q_id);
63 /* Clear queue register of found queue */
64 offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
65 (sizeof(struct fpga_ring_ctrl_reg) * q_id);
66 fpga_ring_reg_write(fpga_dev->mmio_base,
73 "No HW queues assigned to this device. Probably this is a VF configured for PF mode. Check device configuration!");
77 if (num_queues > hw_q_num) {
79 "Not enough queues for device %s! Requested: %u, available: %u",
80 dev->device->name, num_queues, hw_q_num);
84 ring_size = FPGA_RING_MAX_SIZE * sizeof(struct fpga_dma_dec_desc);
86 /* Enforce 32 byte alignment */
87 RTE_BUILD_BUG_ON((RTE_CACHE_LINE_SIZE % 32) != 0);
89 /* Allocate memory for SW descriptor rings */
90 fpga_dev->sw_rings = rte_zmalloc_socket(dev->device->driver->name,
91 num_queues * ring_size, RTE_CACHE_LINE_SIZE,
93 if (fpga_dev->sw_rings == NULL) {
95 "Failed to allocate memory for %s:%u sw_rings",
96 dev->device->driver->name, dev->data->dev_id);
100 fpga_dev->sw_rings_phys = rte_malloc_virt2iova(fpga_dev->sw_rings);
101 fpga_dev->sw_ring_size = ring_size;
102 fpga_dev->sw_ring_max_depth = FPGA_RING_MAX_SIZE;
104 /* Allocate memory for ring flush status */
105 fpga_dev->flush_queue_status = rte_zmalloc_socket(NULL,
106 sizeof(uint64_t), RTE_CACHE_LINE_SIZE, socket_id);
107 if (fpga_dev->flush_queue_status == NULL) {
109 "Failed to allocate memory for %s:%u flush_queue_status",
110 dev->device->driver->name, dev->data->dev_id);
114 /* Set the flush status address registers */
115 phys_addr = rte_malloc_virt2iova(fpga_dev->flush_queue_status);
117 address = FPGA_5GNR_FEC_VFQ_FLUSH_STATUS_LW;
118 payload = (uint32_t)(phys_addr);
119 fpga_reg_write_32(fpga_dev->mmio_base, address, payload);
121 address = FPGA_5GNR_FEC_VFQ_FLUSH_STATUS_HI;
122 payload = (uint32_t)(phys_addr >> 32);
123 fpga_reg_write_32(fpga_dev->mmio_base, address, payload);
129 fpga_dev_close(struct rte_bbdev *dev)
131 struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private;
133 rte_free(fpga_dev->sw_rings);
134 rte_free(fpga_dev->flush_queue_status);
140 fpga_dev_info_get(struct rte_bbdev *dev,
141 struct rte_bbdev_driver_info *dev_info)
143 struct fpga_5gnr_fec_device *d = dev->data->dev_private;
146 static const struct rte_bbdev_op_cap bbdev_capabilities[] = {
148 .type = RTE_BBDEV_OP_LDPC_ENC,
151 RTE_BBDEV_LDPC_RATE_MATCH |
152 RTE_BBDEV_LDPC_ENC_INTERRUPTS |
153 RTE_BBDEV_LDPC_CRC_24B_ATTACH,
155 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
157 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
161 .type = RTE_BBDEV_OP_LDPC_DEC,
164 RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK |
165 RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP |
166 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE |
167 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE |
168 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE |
169 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE |
170 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE |
171 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS,
175 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
176 .num_buffers_hard_out =
177 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
178 .num_buffers_soft_out = 0,
181 RTE_BBDEV_END_OF_CAPABILITIES_LIST()
184 /* Check the HARQ DDR size available */
185 uint8_t timeout_counter = 0;
186 uint32_t harq_buf_ready = fpga_reg_read_32(d->mmio_base,
187 FPGA_5GNR_FEC_HARQ_BUF_SIZE_RDY_REGS);
188 while (harq_buf_ready != 1) {
189 usleep(FPGA_TIMEOUT_CHECK_INTERVAL);
191 harq_buf_ready = fpga_reg_read_32(d->mmio_base,
192 FPGA_5GNR_FEC_HARQ_BUF_SIZE_RDY_REGS);
193 if (timeout_counter > FPGA_HARQ_RDY_TIMEOUT) {
194 rte_bbdev_log(ERR, "HARQ Buffer not ready %d",
199 uint32_t harq_buf_size = fpga_reg_read_32(d->mmio_base,
200 FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS);
202 static struct rte_bbdev_queue_conf default_queue_conf;
203 default_queue_conf.socket = dev->data->socket_id;
204 default_queue_conf.queue_size = FPGA_RING_MAX_SIZE;
206 dev_info->driver_name = dev->device->driver->name;
207 dev_info->queue_size_lim = FPGA_RING_MAX_SIZE;
208 dev_info->hardware_accelerated = true;
209 dev_info->min_alignment = 64;
210 dev_info->harq_buffer_size = (harq_buf_size >> 10) + 1;
211 dev_info->default_queue_conf = default_queue_conf;
212 dev_info->capabilities = bbdev_capabilities;
213 dev_info->cpu_flag_reqs = NULL;
215 /* Calculates number of queues assigned to device */
216 dev_info->max_num_queues = 0;
217 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) {
218 uint32_t hw_q_id = fpga_reg_read_32(d->mmio_base,
219 FPGA_5GNR_FEC_QUEUE_MAP + (q_id << 2));
220 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID)
221 dev_info->max_num_queues++;
226 * Find index of queue bound to current PF/VF which is unassigned. Return -1
227 * when there is no available queue
230 fpga_find_free_queue_idx(struct rte_bbdev *dev,
231 const struct rte_bbdev_queue_conf *conf)
233 struct fpga_5gnr_fec_device *d = dev->data->dev_private;
236 uint8_t range = FPGA_TOTAL_NUM_QUEUES >> 1;
238 if (conf->op_type == RTE_BBDEV_OP_LDPC_ENC) {
239 i = FPGA_NUM_DL_QUEUES;
240 range = FPGA_TOTAL_NUM_QUEUES;
243 for (; i < range; ++i) {
245 /* Check if index of queue is bound to current PF/VF */
246 if (d->q_bound_bit_map & q_idx)
247 /* Check if found queue was not already assigned */
248 if (!(d->q_assigned_bit_map & q_idx)) {
249 d->q_assigned_bit_map |= q_idx;
254 rte_bbdev_log(INFO, "Failed to find free queue on %s", dev->data->name);
260 fpga_queue_setup(struct rte_bbdev *dev, uint16_t queue_id,
261 const struct rte_bbdev_queue_conf *conf)
263 uint32_t address, ring_offset;
264 struct fpga_5gnr_fec_device *d = dev->data->dev_private;
265 struct fpga_queue *q;
268 /* Check if there is a free queue to assign */
269 q_idx = fpga_find_free_queue_idx(dev, conf);
273 /* Allocate the queue data structure. */
274 q = rte_zmalloc_socket(dev->device->driver->name, sizeof(*q),
275 RTE_CACHE_LINE_SIZE, conf->socket);
277 /* Mark queue as un-assigned */
278 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx));
279 rte_bbdev_log(ERR, "Failed to allocate queue memory");
286 /* Set ring_base_addr */
287 q->ring_addr = RTE_PTR_ADD(d->sw_rings, (d->sw_ring_size * queue_id));
288 q->ring_ctrl_reg.ring_base_addr = d->sw_rings_phys +
289 (d->sw_ring_size * queue_id);
291 /* Allocate memory for Completion Head variable*/
292 q->ring_head_addr = rte_zmalloc_socket(dev->device->driver->name,
293 sizeof(uint64_t), RTE_CACHE_LINE_SIZE, conf->socket);
294 if (q->ring_head_addr == NULL) {
295 /* Mark queue as un-assigned */
296 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx));
299 "Failed to allocate memory for %s:%u completion_head",
300 dev->device->driver->name, dev->data->dev_id);
303 /* Set ring_head_addr */
304 q->ring_ctrl_reg.ring_head_addr =
305 rte_malloc_virt2iova(q->ring_head_addr);
307 /* Clear shadow_completion_head */
308 q->shadow_completion_head = 0;
311 if (conf->queue_size > FPGA_RING_MAX_SIZE) {
312 /* Mark queue as un-assigned */
313 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx));
314 rte_free(q->ring_head_addr);
317 "Size of queue is too big %d (MAX: %d ) for %s:%u",
318 conf->queue_size, FPGA_RING_MAX_SIZE,
319 dev->device->driver->name, dev->data->dev_id);
322 q->ring_ctrl_reg.ring_size = conf->queue_size;
324 /* Set Miscellaneous FPGA register*/
325 /* Max iteration number for TTI mitigation - todo */
326 q->ring_ctrl_reg.max_ul_dec = 0;
327 /* Enable max iteration number for TTI - todo */
328 q->ring_ctrl_reg.max_ul_dec_en = 0;
330 /* Enable the ring */
331 q->ring_ctrl_reg.enable = 1;
333 /* Set FPGA head_point and tail registers */
334 q->ring_ctrl_reg.head_point = q->tail = 0;
336 /* Set FPGA shadow_tail register */
337 q->ring_ctrl_reg.shadow_tail = q->tail;
339 /* Calculates the ring offset for found queue */
340 ring_offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
341 (sizeof(struct fpga_ring_ctrl_reg) * q_idx);
343 /* Set FPGA Ring Control Registers */
344 fpga_ring_reg_write(d->mmio_base, ring_offset, q->ring_ctrl_reg);
346 /* Store MMIO register of shadow_tail */
347 address = ring_offset + FPGA_5GNR_FEC_RING_SHADOW_TAIL;
348 q->shadow_tail_addr = RTE_PTR_ADD(d->mmio_base, address);
350 q->head_free_desc = q->tail;
353 q->sw_ring_wrap_mask = conf->queue_size - 1;
355 rte_bbdev_log_debug("Setup dev%u q%u: queue_idx=%u",
356 dev->data->dev_id, queue_id, q->q_idx);
358 dev->data->queues[queue_id].queue_private = q;
360 rte_bbdev_log_debug("BBDEV queue[%d] set up for FPGA queue[%d]",
367 fpga_queue_release(struct rte_bbdev *dev, uint16_t queue_id)
369 struct fpga_5gnr_fec_device *d = dev->data->dev_private;
370 struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
371 struct fpga_ring_ctrl_reg ring_reg;
374 rte_bbdev_log_debug("FPGA Queue[%d] released", queue_id);
377 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg));
378 offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
379 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx);
381 fpga_reg_write_8(d->mmio_base,
382 offset + FPGA_5GNR_FEC_RING_ENABLE, 0x00);
383 /* Clear queue registers */
384 fpga_ring_reg_write(d->mmio_base, offset, ring_reg);
386 /* Mark the Queue as un-assigned */
387 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q->q_idx));
388 rte_free(q->ring_head_addr);
390 dev->data->queues[queue_id].queue_private = NULL;
396 /* Function starts a device queue. */
398 fpga_queue_start(struct rte_bbdev *dev, uint16_t queue_id)
400 struct fpga_5gnr_fec_device *d = dev->data->dev_private;
401 #ifdef RTE_LIBRTE_BBDEV_DEBUG
403 rte_bbdev_log(ERR, "Invalid device pointer");
407 struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
408 uint32_t offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
409 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx);
410 uint8_t enable = 0x01;
411 uint16_t zero = 0x0000;
413 /* Clear queue head and tail variables */
414 q->tail = q->head_free_desc = 0;
416 /* Clear FPGA head_point and tail registers */
417 fpga_reg_write_16(d->mmio_base, offset + FPGA_5GNR_FEC_RING_HEAD_POINT,
419 fpga_reg_write_16(d->mmio_base, offset + FPGA_5GNR_FEC_RING_SHADOW_TAIL,
423 fpga_reg_write_8(d->mmio_base, offset + FPGA_5GNR_FEC_RING_ENABLE,
426 rte_bbdev_log_debug("FPGA Queue[%d] started", queue_id);
430 /* Function stops a device queue. */
432 fpga_queue_stop(struct rte_bbdev *dev, uint16_t queue_id)
434 struct fpga_5gnr_fec_device *d = dev->data->dev_private;
435 #ifdef RTE_LIBRTE_BBDEV_DEBUG
437 rte_bbdev_log(ERR, "Invalid device pointer");
441 struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
442 uint32_t offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
443 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx);
444 uint8_t payload = 0x01;
446 uint8_t timeout = FPGA_QUEUE_FLUSH_TIMEOUT_US /
447 FPGA_TIMEOUT_CHECK_INTERVAL;
449 /* Set flush_queue_en bit to trigger queue flushing */
450 fpga_reg_write_8(d->mmio_base,
451 offset + FPGA_5GNR_FEC_RING_FLUSH_QUEUE_EN, payload);
453 /** Check if queue flush is completed.
454 * FPGA will update the completion flag after queue flushing is
455 * completed. If completion flag is not updated within 1ms it is
456 * considered as a failure.
458 while (!(*((volatile uint8_t *)d->flush_queue_status + q->q_idx)
460 if (counter > timeout) {
461 rte_bbdev_log(ERR, "FPGA Queue Flush failed for queue %d",
465 usleep(FPGA_TIMEOUT_CHECK_INTERVAL);
471 fpga_reg_write_8(d->mmio_base, offset + FPGA_5GNR_FEC_RING_ENABLE,
474 rte_bbdev_log_debug("FPGA Queue[%d] stopped", queue_id);
478 static const struct rte_bbdev_ops fpga_ops = {
479 .setup_queues = fpga_setup_queues,
480 .close = fpga_dev_close,
481 .info_get = fpga_dev_info_get,
482 .queue_setup = fpga_queue_setup,
483 .queue_stop = fpga_queue_stop,
484 .queue_start = fpga_queue_start,
485 .queue_release = fpga_queue_release,
488 fpga_dma_enqueue(struct fpga_queue *q, uint16_t num_desc,
489 struct rte_bbdev_stats *queue_stats)
491 #ifdef RTE_BBDEV_OFFLOAD_COST
492 uint64_t start_time = 0;
493 queue_stats->acc_offload_cycles = 0;
495 RTE_SET_USED(queue_stats);
498 /* Update tail and shadow_tail register */
499 q->tail = (q->tail + num_desc) & q->sw_ring_wrap_mask;
503 #ifdef RTE_BBDEV_OFFLOAD_COST
504 /* Start time measurement for enqueue function offload. */
505 start_time = rte_rdtsc_precise();
507 mmio_write_16(q->shadow_tail_addr, q->tail);
509 #ifdef RTE_BBDEV_OFFLOAD_COST
511 queue_stats->acc_offload_cycles += rte_rdtsc_precise() - start_time;
515 /* Read flag value 0/1/ from bitmap */
517 check_bit(uint32_t bitmap, uint32_t bitmask)
519 return bitmap & bitmask;
522 /* Print an error if a descriptor error has occurred.
523 * Return 0 on success, 1 on failure
526 check_desc_error(uint32_t error_code) {
527 switch (error_code) {
528 case DESC_ERR_NO_ERR:
530 case DESC_ERR_K_P_OUT_OF_RANGE:
531 rte_bbdev_log(ERR, "Encode block size K' is out of range");
533 case DESC_ERR_Z_C_NOT_LEGAL:
534 rte_bbdev_log(ERR, "Zc is illegal");
536 case DESC_ERR_DESC_OFFSET_ERR:
538 "Queue offset does not meet the expectation in the FPGA"
541 case DESC_ERR_DESC_READ_FAIL:
542 rte_bbdev_log(ERR, "Unsuccessful completion for descriptor read");
544 case DESC_ERR_DESC_READ_TIMEOUT:
545 rte_bbdev_log(ERR, "Descriptor read time-out");
547 case DESC_ERR_DESC_READ_TLP_POISONED:
548 rte_bbdev_log(ERR, "Descriptor read TLP poisoned");
550 case DESC_ERR_CB_READ_FAIL:
551 rte_bbdev_log(ERR, "Unsuccessful completion for code block");
553 case DESC_ERR_CB_READ_TIMEOUT:
554 rte_bbdev_log(ERR, "Code block read time-out");
556 case DESC_ERR_CB_READ_TLP_POISONED:
557 rte_bbdev_log(ERR, "Code block read TLP poisoned");
559 case DESC_ERR_HBSTORE_ERR:
560 rte_bbdev_log(ERR, "Hbstroe exceeds HARQ buffer size.");
563 rte_bbdev_log(ERR, "Descriptor error unknown error code %u",
570 /* Compute value of k0.
571 * Based on 3GPP 38.212 Table 5.4.2.1-2
572 * Starting position of different redundancy versions, k0
574 static inline uint16_t
575 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index)
579 uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c;
582 return (bg == 1 ? K0_1_1 : K0_1_2) * z_c;
583 else if (rv_index == 2)
584 return (bg == 1 ? K0_2_1 : K0_2_2) * z_c;
586 return (bg == 1 ? K0_3_1 : K0_3_2) * z_c;
588 /* LBRM case - includes a division by N */
590 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb)
592 else if (rv_index == 2)
593 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb)
596 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb)
601 * Set DMA descriptor for encode operation (1 Code Block)
604 * Pointer to a single encode operation.
606 * Pointer to DMA descriptor.
608 * Pointer to pointer to input data which will be decoded.
610 * E value (length of output in bits).
612 * Ncb value (size of the soft buffer).
614 * Length of output buffer
616 * Input offset in rte_mbuf structure. It is used for calculating the point
617 * where data is starting.
619 * Output offset in rte_mbuf structure. It is used for calculating the point
620 * where hard output data will be stored.
622 * Number of CBs contained in one operation.
625 fpga_dma_desc_te_fill(struct rte_bbdev_enc_op *op,
626 struct fpga_dma_enc_desc *desc, struct rte_mbuf *input,
627 struct rte_mbuf *output, uint16_t k_, uint16_t e,
628 uint32_t in_offset, uint32_t out_offset, uint16_t desc_offset,
636 desc->desc_idx = desc_offset;
637 desc->zc = op->ldpc_enc.z_c;
638 desc->bg_idx = op->ldpc_enc.basegraph - 1;
639 desc->qm_idx = op->ldpc_enc.q_m / 2;
640 desc->crc_en = check_bit(op->ldpc_enc.op_flags,
641 RTE_BBDEV_LDPC_CRC_24B_ATTACH);
643 desc->k0 = get_k0(op->ldpc_enc.n_cb, op->ldpc_enc.z_c,
644 op->ldpc_enc.basegraph, op->ldpc_enc.rv_index);
645 desc->ncb = op->ldpc_enc.n_cb;
646 desc->num_null = op->ldpc_enc.n_filler;
647 /* Set inbound data buffer address */
648 desc->in_addr_hi = (uint32_t)(
649 rte_pktmbuf_mtophys_offset(input, in_offset) >> 32);
650 desc->in_addr_lw = (uint32_t)(
651 rte_pktmbuf_mtophys_offset(input, in_offset));
653 desc->out_addr_hi = (uint32_t)(
654 rte_pktmbuf_mtophys_offset(output, out_offset) >> 32);
655 desc->out_addr_lw = (uint32_t)(
656 rte_pktmbuf_mtophys_offset(output, out_offset));
657 /* Save software context needed for dequeue */
659 /* Set total number of CBs in an op */
660 desc->cbs_in_op = cbs_in_op;
665 * Set DMA descriptor for decode operation (1 Code Block)
668 * Pointer to a single encode operation.
670 * Pointer to DMA descriptor.
672 * Pointer to pointer to input data which will be decoded.
674 * Input offset in rte_mbuf structure. It is used for calculating the point
675 * where data is starting.
677 * Output offset in rte_mbuf structure. It is used for calculating the point
678 * where hard output data will be stored.
680 * Number of CBs contained in one operation.
683 fpga_dma_desc_ld_fill(struct rte_bbdev_dec_op *op,
684 struct fpga_dma_dec_desc *desc,
685 struct rte_mbuf *input, struct rte_mbuf *output,
686 uint16_t harq_in_length,
687 uint32_t in_offset, uint32_t out_offset,
688 uint32_t harq_offset,
689 uint16_t desc_offset,
695 /* Set inbound data buffer address */
696 desc->in_addr_hi = (uint32_t)(
697 rte_pktmbuf_mtophys_offset(input, in_offset) >> 32);
698 desc->in_addr_lw = (uint32_t)(
699 rte_pktmbuf_mtophys_offset(input, in_offset));
700 desc->rm_e = op->ldpc_dec.cb_params.e;
701 desc->harq_input_length = harq_in_length;
702 desc->et_dis = !check_bit(op->ldpc_dec.op_flags,
703 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE);
704 desc->rv = op->ldpc_dec.rv_index;
705 desc->crc24b_ind = check_bit(op->ldpc_dec.op_flags,
706 RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK);
707 desc->drop_crc24b = check_bit(op->ldpc_dec.op_flags,
708 RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP);
709 desc->desc_idx = desc_offset;
710 desc->ncb = op->ldpc_dec.n_cb;
711 desc->num_null = op->ldpc_dec.n_filler;
712 desc->hbstroe_offset = harq_offset >> 10;
713 desc->zc = op->ldpc_dec.z_c;
714 desc->harqin_en = check_bit(op->ldpc_dec.op_flags,
715 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE);
716 desc->bg_idx = op->ldpc_dec.basegraph - 1;
717 desc->max_iter = op->ldpc_dec.iter_max;
718 desc->qm_idx = op->ldpc_dec.q_m / 2;
719 desc->out_addr_hi = (uint32_t)(
720 rte_pktmbuf_mtophys_offset(output, out_offset) >> 32);
721 desc->out_addr_lw = (uint32_t)(
722 rte_pktmbuf_mtophys_offset(output, out_offset));
723 /* Save software context needed for dequeue */
725 /* Set total number of CBs in an op */
726 desc->cbs_in_op = cbs_in_op;
732 mbuf_append(struct rte_mbuf *m_head, struct rte_mbuf *m, uint16_t len)
734 if (unlikely(len > rte_pktmbuf_tailroom(m)))
737 char *tail = (char *)m->buf_addr + m->data_off + m->data_len;
738 m->data_len = (uint16_t)(m->data_len + len);
739 m_head->pkt_len = (m_head->pkt_len + len);
744 enqueue_ldpc_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op *op,
745 uint16_t desc_offset)
747 union fpga_dma_desc *desc;
749 uint8_t c, crc24_bits = 0;
750 struct rte_bbdev_op_ldpc_enc *enc = &op->ldpc_enc;
751 uint16_t in_offset = enc->input.offset;
752 uint16_t out_offset = enc->output.offset;
753 struct rte_mbuf *m_in = enc->input.data;
754 struct rte_mbuf *m_out = enc->output.data;
755 struct rte_mbuf *m_out_head = enc->output.data;
756 uint32_t in_length, out_length, e;
757 uint16_t total_left = enc->input.length;
758 uint16_t ring_offset;
761 /* Clear op status */
764 if (m_in == NULL || m_out == NULL) {
765 rte_bbdev_log(ERR, "Invalid mbuf pointer");
766 op->status = 1 << RTE_BBDEV_DATA_ERROR;
770 if (enc->op_flags & RTE_BBDEV_LDPC_CRC_24B_ATTACH)
773 if (enc->code_block_mode == 0) {
774 /* For Transport Block mode */
776 c = enc->tb_params.c;
777 e = enc->tb_params.ea;
778 } else { /* For Code Block mode */
780 e = enc->cb_params.e;
783 /* Update total_left */
784 K = (enc->basegraph == 1 ? 22 : 10) * enc->z_c;
785 k_ = K - enc->n_filler;
786 in_length = (k_ - crc24_bits) >> 3;
787 out_length = (e + 7) >> 3;
789 total_left = rte_pktmbuf_data_len(m_in) - in_offset;
792 if (total_left != in_length) {
793 op->status |= 1 << RTE_BBDEV_DATA_ERROR;
795 "Mismatch between mbuf length and included CBs sizes %d",
799 mbuf_append(m_out_head, m_out, out_length);
801 /* Offset into the ring */
802 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask);
803 /* Setup DMA Descriptor */
804 desc = q->ring_addr + ring_offset;
806 ret = fpga_dma_desc_te_fill(op, &desc->enc_req, m_in, m_out,
807 k_, e, in_offset, out_offset, ring_offset, c);
808 if (unlikely(ret < 0))
812 total_left -= in_length;
813 op->ldpc_enc.output.length += out_length;
815 if (total_left > 0) {
817 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u",
818 total_left, in_length);
826 enqueue_ldpc_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op *op,
827 uint16_t desc_offset)
829 union fpga_dma_desc *desc;
831 uint16_t ring_offset;
833 uint16_t e, in_length, out_length, k0, l, seg_total_left, sys_cols;
834 uint16_t K, parity_offset, harq_in_length = 0, harq_out_length = 0;
835 uint16_t crc24_overlap = 0;
836 struct rte_bbdev_op_ldpc_dec *dec = &op->ldpc_dec;
837 struct rte_mbuf *m_in = dec->input.data;
838 struct rte_mbuf *m_out = dec->hard_output.data;
839 struct rte_mbuf *m_out_head = dec->hard_output.data;
840 uint16_t in_offset = dec->input.offset;
841 uint16_t out_offset = dec->hard_output.offset;
842 uint32_t harq_offset = 0;
844 /* Clear op status */
847 /* Setup DMA Descriptor */
848 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask);
849 desc = q->ring_addr + ring_offset;
851 if (m_in == NULL || m_out == NULL) {
852 rte_bbdev_log(ERR, "Invalid mbuf pointer");
853 op->status = 1 << RTE_BBDEV_DATA_ERROR;
858 e = dec->cb_params.e;
860 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP))
863 sys_cols = (dec->basegraph == 1) ? 22 : 10;
864 K = sys_cols * dec->z_c;
865 parity_offset = K - 2 * dec->z_c;
867 out_length = ((K - crc24_overlap - dec->n_filler) >> 3);
869 seg_total_left = dec->input.length;
871 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) {
872 harq_in_length = RTE_MIN(dec->harq_combined_input.length,
873 (uint32_t)dec->n_cb);
876 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE)) {
877 k0 = get_k0(dec->n_cb, dec->z_c,
878 dec->basegraph, dec->rv_index);
879 if (k0 > parity_offset)
882 l = k0 + e + dec->n_filler;
883 harq_out_length = RTE_MIN(RTE_MAX(harq_in_length, l),
884 dec->n_cb - dec->n_filler);
885 dec->harq_combined_output.length = harq_out_length;
888 mbuf_append(m_out_head, m_out, out_length);
889 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE))
890 harq_offset = dec->harq_combined_input.offset;
891 else if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE))
892 harq_offset = dec->harq_combined_output.offset;
894 if ((harq_offset & 0x3FF) > 0) {
895 rte_bbdev_log(ERR, "Invalid HARQ offset %d", harq_offset);
896 op->status = 1 << RTE_BBDEV_DATA_ERROR;
900 ret = fpga_dma_desc_ld_fill(op, &desc->dec_req, m_in, m_out,
901 harq_in_length, in_offset, out_offset, harq_offset,
903 if (unlikely(ret < 0))
906 seg_total_left -= in_length;
907 op->ldpc_dec.hard_output.length += out_length;
908 if (seg_total_left > 0) {
910 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u",
911 seg_total_left, in_length);
919 fpga_enqueue_ldpc_enc(struct rte_bbdev_queue_data *q_data,
920 struct rte_bbdev_enc_op **ops, uint16_t num)
922 uint16_t i, total_enqueued_cbs = 0;
925 struct fpga_queue *q = q_data->queue_private;
926 union fpga_dma_desc *desc;
928 /* Check if queue is not full */
929 if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) ==
933 /* Calculates available space */
934 avail = (q->head_free_desc > q->tail) ?
935 q->head_free_desc - q->tail - 1 :
936 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1;
938 for (i = 0; i < num; ++i) {
940 /* Check if there is available space for further
943 if (unlikely(avail - 1 < 0))
946 enqueued_cbs = enqueue_ldpc_enc_one_op_cb(q, ops[i],
949 if (enqueued_cbs < 0)
952 total_enqueued_cbs += enqueued_cbs;
954 rte_bbdev_log_debug("enqueuing enc ops [%d/%d] | head %d | tail %d",
955 total_enqueued_cbs, num,
956 q->head_free_desc, q->tail);
959 /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt
960 * only when all previous CBs were already processed.
962 desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1)
963 & q->sw_ring_wrap_mask);
964 desc->enc_req.irq_en = q->irq_enable;
966 fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats);
969 q_data->queue_stats.enqueued_count += i;
970 q_data->queue_stats.enqueue_err_count += num - i;
976 fpga_enqueue_ldpc_dec(struct rte_bbdev_queue_data *q_data,
977 struct rte_bbdev_dec_op **ops, uint16_t num)
979 uint16_t i, total_enqueued_cbs = 0;
982 struct fpga_queue *q = q_data->queue_private;
983 union fpga_dma_desc *desc;
985 /* Check if queue is not full */
986 if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) ==
990 /* Calculates available space */
991 avail = (q->head_free_desc > q->tail) ?
992 q->head_free_desc - q->tail - 1 :
993 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1;
995 for (i = 0; i < num; ++i) {
997 /* Check if there is available space for further
1000 if (unlikely(avail - 1 < 0))
1003 enqueued_cbs = enqueue_ldpc_dec_one_op_cb(q, ops[i],
1004 total_enqueued_cbs);
1006 if (enqueued_cbs < 0)
1009 total_enqueued_cbs += enqueued_cbs;
1011 rte_bbdev_log_debug("enqueuing dec ops [%d/%d] | head %d | tail %d",
1012 total_enqueued_cbs, num,
1013 q->head_free_desc, q->tail);
1017 q_data->queue_stats.enqueued_count += i;
1018 q_data->queue_stats.enqueue_err_count += num - i;
1020 /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt
1021 * only when all previous CBs were already processed.
1023 desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1)
1024 & q->sw_ring_wrap_mask);
1025 desc->enc_req.irq_en = q->irq_enable;
1026 fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats);
1032 dequeue_ldpc_enc_one_op_cb(struct fpga_queue *q,
1033 struct rte_bbdev_enc_op **op,
1034 uint16_t desc_offset)
1036 union fpga_dma_desc *desc;
1038 /* Set current desc */
1039 desc = q->ring_addr + ((q->head_free_desc + desc_offset)
1040 & q->sw_ring_wrap_mask);
1043 if (desc->enc_req.done == 0)
1046 /* make sure the response is read atomically */
1049 rte_bbdev_log_debug("DMA response desc %p", desc);
1051 *op = desc->enc_req.op_addr;
1052 /* Check the descriptor error field, return 1 on error */
1053 desc_error = check_desc_error(desc->enc_req.error);
1054 (*op)->status = desc_error << RTE_BBDEV_DATA_ERROR;
1061 dequeue_ldpc_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op **op,
1062 uint16_t desc_offset)
1064 union fpga_dma_desc *desc;
1066 /* Set descriptor */
1067 desc = q->ring_addr + ((q->head_free_desc + desc_offset)
1068 & q->sw_ring_wrap_mask);
1070 /* Verify done bit is set */
1071 if (desc->dec_req.done == 0)
1074 /* make sure the response is read atomically */
1077 *op = desc->dec_req.op_addr;
1079 if (check_bit((*op)->ldpc_dec.op_flags,
1080 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1085 /* FPGA reports iterations based on round-up minus 1 */
1086 (*op)->ldpc_dec.iter_count = desc->dec_req.iter + 1;
1087 /* CRC Check criteria */
1088 if (desc->dec_req.crc24b_ind && !(desc->dec_req.crcb_pass))
1089 (*op)->status = 1 << RTE_BBDEV_CRC_ERROR;
1090 /* et_pass = 0 when decoder fails */
1091 (*op)->status |= !(desc->dec_req.et_pass) << RTE_BBDEV_SYNDROME_ERROR;
1092 /* Check the descriptor error field, return 1 on error */
1093 desc_error = check_desc_error(desc->dec_req.error);
1094 (*op)->status |= desc_error << RTE_BBDEV_DATA_ERROR;
1099 fpga_dequeue_ldpc_enc(struct rte_bbdev_queue_data *q_data,
1100 struct rte_bbdev_enc_op **ops, uint16_t num)
1102 struct fpga_queue *q = q_data->queue_private;
1103 uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask;
1105 uint16_t dequeued_cbs = 0;
1108 for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) {
1109 ret = dequeue_ldpc_enc_one_op_cb(q, &ops[i], dequeued_cbs);
1114 dequeued_cbs += ret;
1116 rte_bbdev_log_debug("dequeuing enc ops [%d/%d] | head %d | tail %d",
1117 dequeued_cbs, num, q->head_free_desc, q->tail);
1121 q->head_free_desc = (q->head_free_desc + dequeued_cbs) &
1122 q->sw_ring_wrap_mask;
1125 q_data->queue_stats.dequeued_count += i;
1131 fpga_dequeue_ldpc_dec(struct rte_bbdev_queue_data *q_data,
1132 struct rte_bbdev_dec_op **ops, uint16_t num)
1134 struct fpga_queue *q = q_data->queue_private;
1135 uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask;
1137 uint16_t dequeued_cbs = 0;
1140 for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) {
1141 ret = dequeue_ldpc_dec_one_op_cb(q, &ops[i], dequeued_cbs);
1146 dequeued_cbs += ret;
1148 rte_bbdev_log_debug("dequeuing dec ops [%d/%d] | head %d | tail %d",
1149 dequeued_cbs, num, q->head_free_desc, q->tail);
1153 q->head_free_desc = (q->head_free_desc + dequeued_cbs) &
1154 q->sw_ring_wrap_mask;
1157 q_data->queue_stats.dequeued_count += i;
1163 /* Initialization Function */
1165 fpga_5gnr_fec_init(struct rte_bbdev *dev, struct rte_pci_driver *drv)
1167 struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device);
1169 dev->dev_ops = &fpga_ops;
1170 dev->enqueue_ldpc_enc_ops = fpga_enqueue_ldpc_enc;
1171 dev->enqueue_ldpc_dec_ops = fpga_enqueue_ldpc_dec;
1172 dev->dequeue_ldpc_enc_ops = fpga_dequeue_ldpc_enc;
1173 dev->dequeue_ldpc_dec_ops = fpga_dequeue_ldpc_dec;
1175 ((struct fpga_5gnr_fec_device *) dev->data->dev_private)->pf_device =
1176 !strcmp(drv->driver.name,
1177 RTE_STR(FPGA_5GNR_FEC_PF_DRIVER_NAME));
1178 ((struct fpga_5gnr_fec_device *) dev->data->dev_private)->mmio_base =
1179 pci_dev->mem_resource[0].addr;
1181 rte_bbdev_log_debug(
1182 "Init device %s [%s] @ virtaddr %p phyaddr %#"PRIx64,
1183 dev->device->driver->name, dev->data->name,
1184 (void *)pci_dev->mem_resource[0].addr,
1185 pci_dev->mem_resource[0].phys_addr);
1189 fpga_5gnr_fec_probe(struct rte_pci_driver *pci_drv,
1190 struct rte_pci_device *pci_dev)
1192 struct rte_bbdev *bbdev = NULL;
1193 char dev_name[RTE_BBDEV_NAME_MAX_LEN];
1195 if (pci_dev == NULL) {
1196 rte_bbdev_log(ERR, "NULL PCI device");
1200 rte_pci_device_name(&pci_dev->addr, dev_name, sizeof(dev_name));
1202 /* Allocate memory to be used privately by drivers */
1203 bbdev = rte_bbdev_allocate(pci_dev->device.name);
1207 /* allocate device private memory */
1208 bbdev->data->dev_private = rte_zmalloc_socket(dev_name,
1209 sizeof(struct fpga_5gnr_fec_device),
1210 RTE_CACHE_LINE_SIZE,
1211 pci_dev->device.numa_node);
1213 if (bbdev->data->dev_private == NULL) {
1215 "Allocate of %zu bytes for device \"%s\" failed",
1216 sizeof(struct fpga_5gnr_fec_device), dev_name);
1217 rte_bbdev_release(bbdev);
1221 /* Fill HW specific part of device structure */
1222 bbdev->device = &pci_dev->device;
1223 bbdev->intr_handle = &pci_dev->intr_handle;
1224 bbdev->data->socket_id = pci_dev->device.numa_node;
1226 /* Invoke FEC FPGA device initialization function */
1227 fpga_5gnr_fec_init(bbdev, pci_drv);
1229 rte_bbdev_log_debug("bbdev id = %u [%s]",
1230 bbdev->data->dev_id, dev_name);
1236 fpga_5gnr_fec_remove(struct rte_pci_device *pci_dev)
1238 struct rte_bbdev *bbdev;
1242 if (pci_dev == NULL)
1246 bbdev = rte_bbdev_get_named_dev(pci_dev->device.name);
1247 if (bbdev == NULL) {
1249 "Couldn't find HW dev \"%s\" to uninitialise it",
1250 pci_dev->device.name);
1253 dev_id = bbdev->data->dev_id;
1255 /* free device private memory before close */
1256 rte_free(bbdev->data->dev_private);
1259 ret = rte_bbdev_close(dev_id);
1262 "Device %i failed to close during uninit: %i",
1265 /* release bbdev from library */
1266 ret = rte_bbdev_release(bbdev);
1268 rte_bbdev_log(ERR, "Device %i failed to uninit: %i", dev_id,
1271 rte_bbdev_log_debug("Destroyed bbdev = %u", dev_id);
1276 /* FPGA 5GNR FEC PCI PF address map */
1277 static struct rte_pci_id pci_id_fpga_5gnr_fec_pf_map[] = {
1279 RTE_PCI_DEVICE(FPGA_5GNR_FEC_VENDOR_ID,
1280 FPGA_5GNR_FEC_PF_DEVICE_ID)
1285 static struct rte_pci_driver fpga_5gnr_fec_pci_pf_driver = {
1286 .probe = fpga_5gnr_fec_probe,
1287 .remove = fpga_5gnr_fec_remove,
1288 .id_table = pci_id_fpga_5gnr_fec_pf_map,
1289 .drv_flags = RTE_PCI_DRV_NEED_MAPPING
1292 /* FPGA 5GNR FEC PCI VF address map */
1293 static struct rte_pci_id pci_id_fpga_5gnr_fec_vf_map[] = {
1295 RTE_PCI_DEVICE(FPGA_5GNR_FEC_VENDOR_ID,
1296 FPGA_5GNR_FEC_VF_DEVICE_ID)
1301 static struct rte_pci_driver fpga_5gnr_fec_pci_vf_driver = {
1302 .probe = fpga_5gnr_fec_probe,
1303 .remove = fpga_5gnr_fec_remove,
1304 .id_table = pci_id_fpga_5gnr_fec_vf_map,
1305 .drv_flags = RTE_PCI_DRV_NEED_MAPPING
1309 RTE_PMD_REGISTER_PCI(FPGA_5GNR_FEC_PF_DRIVER_NAME, fpga_5gnr_fec_pci_pf_driver);
1310 RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_PF_DRIVER_NAME,
1311 pci_id_fpga_5gnr_fec_pf_map);
1312 RTE_PMD_REGISTER_PCI(FPGA_5GNR_FEC_VF_DRIVER_NAME, fpga_5gnr_fec_pci_vf_driver);
1313 RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_VF_DRIVER_NAME,
1314 pci_id_fpga_5gnr_fec_vf_map);
1316 RTE_INIT(fpga_5gnr_fec_init_log)
1318 fpga_5gnr_fec_logtype = rte_log_register("pmd.bb.fpga_5gnr_fec");
1319 if (fpga_5gnr_fec_logtype >= 0)
1320 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1321 rte_log_set_level(fpga_5gnr_fec_logtype, RTE_LOG_DEBUG);
1323 rte_log_set_level(fpga_5gnr_fec_logtype, RTE_LOG_NOTICE);