1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
7 #include <rte_common.h>
10 #include <rte_malloc.h>
11 #include <rte_mempool.h>
12 #include <rte_errno.h>
14 #include <rte_bus_pci.h>
15 #include <rte_byteorder.h>
16 #ifdef RTE_BBDEV_OFFLOAD_COST
17 #include <rte_cycles.h>
20 #include <rte_bbdev.h>
21 #include <rte_bbdev_pmd.h>
23 #include "fpga_5gnr_fec.h"
24 #include "rte_pmd_fpga_5gnr_fec.h"
26 #ifdef RTE_LIBRTE_BBDEV_DEBUG
27 RTE_LOG_REGISTER_DEFAULT(fpga_5gnr_fec_logtype, DEBUG);
29 RTE_LOG_REGISTER_DEFAULT(fpga_5gnr_fec_logtype, NOTICE);
32 #ifdef RTE_LIBRTE_BBDEV_DEBUG
34 /* Read Ring Control Register of FPGA 5GNR FEC device */
36 print_ring_reg_debug_info(void *mmio_base, uint32_t offset)
39 "FPGA MMIO base address @ %p | Ring Control Register @ offset = 0x%08"
40 PRIx32, mmio_base, offset);
42 "RING_BASE_ADDR = 0x%016"PRIx64,
43 fpga_reg_read_64(mmio_base, offset));
45 "RING_HEAD_ADDR = 0x%016"PRIx64,
46 fpga_reg_read_64(mmio_base, offset +
47 FPGA_5GNR_FEC_RING_HEAD_ADDR));
49 "RING_SIZE = 0x%04"PRIx16,
50 fpga_reg_read_16(mmio_base, offset +
51 FPGA_5GNR_FEC_RING_SIZE));
53 "RING_MISC = 0x%02"PRIx8,
54 fpga_reg_read_8(mmio_base, offset +
55 FPGA_5GNR_FEC_RING_MISC));
57 "RING_ENABLE = 0x%02"PRIx8,
58 fpga_reg_read_8(mmio_base, offset +
59 FPGA_5GNR_FEC_RING_ENABLE));
61 "RING_FLUSH_QUEUE_EN = 0x%02"PRIx8,
62 fpga_reg_read_8(mmio_base, offset +
63 FPGA_5GNR_FEC_RING_FLUSH_QUEUE_EN));
65 "RING_SHADOW_TAIL = 0x%04"PRIx16,
66 fpga_reg_read_16(mmio_base, offset +
67 FPGA_5GNR_FEC_RING_SHADOW_TAIL));
69 "RING_HEAD_POINT = 0x%04"PRIx16,
70 fpga_reg_read_16(mmio_base, offset +
71 FPGA_5GNR_FEC_RING_HEAD_POINT));
74 /* Read Static Register of FPGA 5GNR FEC device */
76 print_static_reg_debug_info(void *mmio_base)
78 uint16_t config = fpga_reg_read_16(mmio_base,
79 FPGA_5GNR_FEC_CONFIGURATION);
80 uint8_t qmap_done = fpga_reg_read_8(mmio_base,
81 FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE);
82 uint16_t lb_factor = fpga_reg_read_16(mmio_base,
83 FPGA_5GNR_FEC_LOAD_BALANCE_FACTOR);
84 uint16_t ring_desc_len = fpga_reg_read_16(mmio_base,
85 FPGA_5GNR_FEC_RING_DESC_LEN);
86 uint16_t flr_time_out = fpga_reg_read_16(mmio_base,
87 FPGA_5GNR_FEC_FLR_TIME_OUT);
89 rte_bbdev_log_debug("UL.DL Weights = %u.%u",
90 ((uint8_t)config), ((uint8_t)(config >> 8)));
91 rte_bbdev_log_debug("UL.DL Load Balance = %u.%u",
92 ((uint8_t)lb_factor), ((uint8_t)(lb_factor >> 8)));
93 rte_bbdev_log_debug("Queue-PF/VF Mapping Table = %s",
94 (qmap_done > 0) ? "READY" : "NOT-READY");
95 rte_bbdev_log_debug("Ring Descriptor Size = %u bytes",
96 ring_desc_len*FPGA_RING_DESC_LEN_UNIT_BYTES);
97 rte_bbdev_log_debug("FLR Timeout = %f usec",
98 (float)flr_time_out*FPGA_FLR_TIMEOUT_UNIT);
101 /* Print decode DMA Descriptor of FPGA 5GNR Decoder device */
103 print_dma_dec_desc_debug_info(union fpga_dma_desc *desc)
105 rte_bbdev_log_debug("DMA response desc %p\n"
106 "\t-- done(%"PRIu32") | iter(%"PRIu32") | et_pass(%"PRIu32")"
107 " | crcb_pass (%"PRIu32") | error(%"PRIu32")\n"
108 "\t-- qm_idx(%"PRIu32") | max_iter(%"PRIu32") | "
109 "bg_idx (%"PRIu32") | harqin_en(%"PRIu32") | zc(%"PRIu32")\n"
110 "\t-- hbstroe_offset(%"PRIu32") | num_null (%"PRIu32") "
111 "| irq_en(%"PRIu32")\n"
112 "\t-- ncb(%"PRIu32") | desc_idx (%"PRIu32") | "
113 "drop_crc24b(%"PRIu32") | RV (%"PRIu32")\n"
114 "\t-- crc24b_ind(%"PRIu32") | et_dis (%"PRIu32")\n"
115 "\t-- harq_input_length(%"PRIu32") | rm_e(%"PRIu32")\n"
116 "\t-- cbs_in_op(%"PRIu32") | in_add (0x%08"PRIx32"%08"PRIx32")"
117 "| out_add (0x%08"PRIx32"%08"PRIx32")",
119 (uint32_t)desc->dec_req.done,
120 (uint32_t)desc->dec_req.iter,
121 (uint32_t)desc->dec_req.et_pass,
122 (uint32_t)desc->dec_req.crcb_pass,
123 (uint32_t)desc->dec_req.error,
124 (uint32_t)desc->dec_req.qm_idx,
125 (uint32_t)desc->dec_req.max_iter,
126 (uint32_t)desc->dec_req.bg_idx,
127 (uint32_t)desc->dec_req.harqin_en,
128 (uint32_t)desc->dec_req.zc,
129 (uint32_t)desc->dec_req.hbstroe_offset,
130 (uint32_t)desc->dec_req.num_null,
131 (uint32_t)desc->dec_req.irq_en,
132 (uint32_t)desc->dec_req.ncb,
133 (uint32_t)desc->dec_req.desc_idx,
134 (uint32_t)desc->dec_req.drop_crc24b,
135 (uint32_t)desc->dec_req.rv,
136 (uint32_t)desc->dec_req.crc24b_ind,
137 (uint32_t)desc->dec_req.et_dis,
138 (uint32_t)desc->dec_req.harq_input_length,
139 (uint32_t)desc->dec_req.rm_e,
140 (uint32_t)desc->dec_req.cbs_in_op,
141 (uint32_t)desc->dec_req.in_addr_hi,
142 (uint32_t)desc->dec_req.in_addr_lw,
143 (uint32_t)desc->dec_req.out_addr_hi,
144 (uint32_t)desc->dec_req.out_addr_lw);
145 uint32_t *word = (uint32_t *) desc;
146 rte_bbdev_log_debug("%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n"
147 "%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n",
148 word[0], word[1], word[2], word[3],
149 word[4], word[5], word[6], word[7]);
152 /* Print decode DMA Descriptor of FPGA 5GNR encoder device */
154 print_dma_enc_desc_debug_info(union fpga_dma_desc *desc)
156 rte_bbdev_log_debug("DMA response desc %p\n"
157 "%"PRIu32" %"PRIu32"\n"
158 "K' %"PRIu32" E %"PRIu32" desc %"PRIu32" Z %"PRIu32"\n"
159 "BG %"PRIu32" Qm %"PRIu32" CRC %"PRIu32" IRQ %"PRIu32"\n"
160 "k0 %"PRIu32" Ncb %"PRIu32" F %"PRIu32"\n",
162 (uint32_t)desc->enc_req.done,
163 (uint32_t)desc->enc_req.error,
165 (uint32_t)desc->enc_req.k_,
166 (uint32_t)desc->enc_req.rm_e,
167 (uint32_t)desc->enc_req.desc_idx,
168 (uint32_t)desc->enc_req.zc,
170 (uint32_t)desc->enc_req.bg_idx,
171 (uint32_t)desc->enc_req.qm_idx,
172 (uint32_t)desc->enc_req.crc_en,
173 (uint32_t)desc->enc_req.irq_en,
175 (uint32_t)desc->enc_req.k0,
176 (uint32_t)desc->enc_req.ncb,
177 (uint32_t)desc->enc_req.num_null);
178 uint32_t *word = (uint32_t *) desc;
179 rte_bbdev_log_debug("%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n"
180 "%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n",
181 word[0], word[1], word[2], word[3],
182 word[4], word[5], word[6], word[7]);
188 fpga_setup_queues(struct rte_bbdev *dev, uint16_t num_queues, int socket_id)
190 /* Number of queues bound to a PF/VF */
191 uint32_t hw_q_num = 0;
192 uint32_t ring_size, payload, address, q_id, offset;
193 rte_iova_t phys_addr;
194 struct fpga_ring_ctrl_reg ring_reg;
195 struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private;
197 address = FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE;
198 if (!(fpga_reg_read_32(fpga_dev->mmio_base, address) & 0x1)) {
200 "Queue-PF/VF mapping is not set! Was PF configured for device (%s) ?",
205 /* Clear queue registers structure */
206 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg));
209 * If a queue is valid and mapped to a calling PF/VF the read value is
210 * replaced with a queue ID and if it's not then
211 * FPGA_INVALID_HW_QUEUE_ID is returned.
213 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) {
214 uint32_t hw_q_id = fpga_reg_read_32(fpga_dev->mmio_base,
215 FPGA_5GNR_FEC_QUEUE_MAP + (q_id << 2));
217 rte_bbdev_log_debug("%s: queue ID: %u, registry queue ID: %u",
218 dev->device->name, q_id, hw_q_id);
220 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID) {
221 fpga_dev->q_bound_bit_map |= (1ULL << q_id);
222 /* Clear queue register of found queue */
223 offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
224 (sizeof(struct fpga_ring_ctrl_reg) * q_id);
225 fpga_ring_reg_write(fpga_dev->mmio_base,
232 "No HW queues assigned to this device. Probably this is a VF configured for PF mode. Check device configuration!");
236 if (num_queues > hw_q_num) {
238 "Not enough queues for device %s! Requested: %u, available: %u",
239 dev->device->name, num_queues, hw_q_num);
243 ring_size = FPGA_RING_MAX_SIZE * sizeof(struct fpga_dma_dec_desc);
245 /* Enforce 32 byte alignment */
246 RTE_BUILD_BUG_ON((RTE_CACHE_LINE_SIZE % 32) != 0);
248 /* Allocate memory for SW descriptor rings */
249 fpga_dev->sw_rings = rte_zmalloc_socket(dev->device->driver->name,
250 num_queues * ring_size, RTE_CACHE_LINE_SIZE,
252 if (fpga_dev->sw_rings == NULL) {
254 "Failed to allocate memory for %s:%u sw_rings",
255 dev->device->driver->name, dev->data->dev_id);
259 fpga_dev->sw_rings_phys = rte_malloc_virt2iova(fpga_dev->sw_rings);
260 fpga_dev->sw_ring_size = ring_size;
261 fpga_dev->sw_ring_max_depth = FPGA_RING_MAX_SIZE;
263 /* Allocate memory for ring flush status */
264 fpga_dev->flush_queue_status = rte_zmalloc_socket(NULL,
265 sizeof(uint64_t), RTE_CACHE_LINE_SIZE, socket_id);
266 if (fpga_dev->flush_queue_status == NULL) {
268 "Failed to allocate memory for %s:%u flush_queue_status",
269 dev->device->driver->name, dev->data->dev_id);
273 /* Set the flush status address registers */
274 phys_addr = rte_malloc_virt2iova(fpga_dev->flush_queue_status);
276 address = FPGA_5GNR_FEC_VFQ_FLUSH_STATUS_LW;
277 payload = (uint32_t)(phys_addr);
278 fpga_reg_write_32(fpga_dev->mmio_base, address, payload);
280 address = FPGA_5GNR_FEC_VFQ_FLUSH_STATUS_HI;
281 payload = (uint32_t)(phys_addr >> 32);
282 fpga_reg_write_32(fpga_dev->mmio_base, address, payload);
288 fpga_dev_close(struct rte_bbdev *dev)
290 struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private;
292 rte_free(fpga_dev->sw_rings);
293 rte_free(fpga_dev->flush_queue_status);
299 fpga_dev_info_get(struct rte_bbdev *dev,
300 struct rte_bbdev_driver_info *dev_info)
302 struct fpga_5gnr_fec_device *d = dev->data->dev_private;
305 static const struct rte_bbdev_op_cap bbdev_capabilities[] = {
307 .type = RTE_BBDEV_OP_LDPC_ENC,
310 RTE_BBDEV_LDPC_RATE_MATCH |
311 RTE_BBDEV_LDPC_ENC_INTERRUPTS |
312 RTE_BBDEV_LDPC_CRC_24B_ATTACH,
314 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
316 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
320 .type = RTE_BBDEV_OP_LDPC_DEC,
323 RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK |
324 RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP |
325 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE |
326 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE |
327 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE |
328 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE |
329 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE |
330 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK |
331 RTE_BBDEV_LDPC_DEC_INTERRUPTS |
332 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS,
336 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
337 .num_buffers_hard_out =
338 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
339 .num_buffers_soft_out = 0,
342 RTE_BBDEV_END_OF_CAPABILITIES_LIST()
345 /* Check the HARQ DDR size available */
346 uint8_t timeout_counter = 0;
347 uint32_t harq_buf_ready = fpga_reg_read_32(d->mmio_base,
348 FPGA_5GNR_FEC_HARQ_BUF_SIZE_RDY_REGS);
349 while (harq_buf_ready != 1) {
350 usleep(FPGA_TIMEOUT_CHECK_INTERVAL);
352 harq_buf_ready = fpga_reg_read_32(d->mmio_base,
353 FPGA_5GNR_FEC_HARQ_BUF_SIZE_RDY_REGS);
354 if (timeout_counter > FPGA_HARQ_RDY_TIMEOUT) {
355 rte_bbdev_log(ERR, "HARQ Buffer not ready %d",
360 uint32_t harq_buf_size = fpga_reg_read_32(d->mmio_base,
361 FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS);
363 static struct rte_bbdev_queue_conf default_queue_conf;
364 default_queue_conf.socket = dev->data->socket_id;
365 default_queue_conf.queue_size = FPGA_RING_MAX_SIZE;
367 dev_info->driver_name = dev->device->driver->name;
368 dev_info->queue_size_lim = FPGA_RING_MAX_SIZE;
369 dev_info->hardware_accelerated = true;
370 dev_info->min_alignment = 64;
371 dev_info->harq_buffer_size = (harq_buf_size >> 10) + 1;
372 dev_info->default_queue_conf = default_queue_conf;
373 dev_info->capabilities = bbdev_capabilities;
374 dev_info->cpu_flag_reqs = NULL;
375 dev_info->data_endianness = RTE_LITTLE_ENDIAN;
377 /* Calculates number of queues assigned to device */
378 dev_info->max_num_queues = 0;
379 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) {
380 uint32_t hw_q_id = fpga_reg_read_32(d->mmio_base,
381 FPGA_5GNR_FEC_QUEUE_MAP + (q_id << 2));
382 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID)
383 dev_info->max_num_queues++;
388 * Find index of queue bound to current PF/VF which is unassigned. Return -1
389 * when there is no available queue
392 fpga_find_free_queue_idx(struct rte_bbdev *dev,
393 const struct rte_bbdev_queue_conf *conf)
395 struct fpga_5gnr_fec_device *d = dev->data->dev_private;
398 uint8_t range = FPGA_TOTAL_NUM_QUEUES >> 1;
400 if (conf->op_type == RTE_BBDEV_OP_LDPC_ENC) {
401 i = FPGA_NUM_DL_QUEUES;
402 range = FPGA_TOTAL_NUM_QUEUES;
405 for (; i < range; ++i) {
407 /* Check if index of queue is bound to current PF/VF */
408 if (d->q_bound_bit_map & q_idx)
409 /* Check if found queue was not already assigned */
410 if (!(d->q_assigned_bit_map & q_idx)) {
411 d->q_assigned_bit_map |= q_idx;
416 rte_bbdev_log(INFO, "Failed to find free queue on %s", dev->data->name);
422 fpga_queue_setup(struct rte_bbdev *dev, uint16_t queue_id,
423 const struct rte_bbdev_queue_conf *conf)
425 uint32_t address, ring_offset;
426 struct fpga_5gnr_fec_device *d = dev->data->dev_private;
427 struct fpga_queue *q;
430 /* Check if there is a free queue to assign */
431 q_idx = fpga_find_free_queue_idx(dev, conf);
435 /* Allocate the queue data structure. */
436 q = rte_zmalloc_socket(dev->device->driver->name, sizeof(*q),
437 RTE_CACHE_LINE_SIZE, conf->socket);
439 /* Mark queue as un-assigned */
440 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx));
441 rte_bbdev_log(ERR, "Failed to allocate queue memory");
448 /* Set ring_base_addr */
449 q->ring_addr = RTE_PTR_ADD(d->sw_rings, (d->sw_ring_size * queue_id));
450 q->ring_ctrl_reg.ring_base_addr = d->sw_rings_phys +
451 (d->sw_ring_size * queue_id);
453 /* Allocate memory for Completion Head variable*/
454 q->ring_head_addr = rte_zmalloc_socket(dev->device->driver->name,
455 sizeof(uint64_t), RTE_CACHE_LINE_SIZE, conf->socket);
456 if (q->ring_head_addr == NULL) {
457 /* Mark queue as un-assigned */
458 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx));
461 "Failed to allocate memory for %s:%u completion_head",
462 dev->device->driver->name, dev->data->dev_id);
465 /* Set ring_head_addr */
466 q->ring_ctrl_reg.ring_head_addr =
467 rte_malloc_virt2iova(q->ring_head_addr);
469 /* Clear shadow_completion_head */
470 q->shadow_completion_head = 0;
473 if (conf->queue_size > FPGA_RING_MAX_SIZE) {
474 /* Mark queue as un-assigned */
475 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx));
476 rte_free(q->ring_head_addr);
479 "Size of queue is too big %d (MAX: %d ) for %s:%u",
480 conf->queue_size, FPGA_RING_MAX_SIZE,
481 dev->device->driver->name, dev->data->dev_id);
484 q->ring_ctrl_reg.ring_size = conf->queue_size;
486 /* Set Miscellaneous FPGA register*/
487 /* Max iteration number for TTI mitigation - todo */
488 q->ring_ctrl_reg.max_ul_dec = 0;
489 /* Enable max iteration number for TTI - todo */
490 q->ring_ctrl_reg.max_ul_dec_en = 0;
492 /* Enable the ring */
493 q->ring_ctrl_reg.enable = 1;
495 /* Set FPGA head_point and tail registers */
496 q->ring_ctrl_reg.head_point = q->tail = 0;
498 /* Set FPGA shadow_tail register */
499 q->ring_ctrl_reg.shadow_tail = q->tail;
501 /* Calculates the ring offset for found queue */
502 ring_offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
503 (sizeof(struct fpga_ring_ctrl_reg) * q_idx);
505 /* Set FPGA Ring Control Registers */
506 fpga_ring_reg_write(d->mmio_base, ring_offset, q->ring_ctrl_reg);
508 /* Store MMIO register of shadow_tail */
509 address = ring_offset + FPGA_5GNR_FEC_RING_SHADOW_TAIL;
510 q->shadow_tail_addr = RTE_PTR_ADD(d->mmio_base, address);
512 q->head_free_desc = q->tail;
515 q->sw_ring_wrap_mask = conf->queue_size - 1;
517 rte_bbdev_log_debug("Setup dev%u q%u: queue_idx=%u",
518 dev->data->dev_id, queue_id, q->q_idx);
520 dev->data->queues[queue_id].queue_private = q;
522 rte_bbdev_log_debug("BBDEV queue[%d] set up for FPGA queue[%d]",
525 #ifdef RTE_LIBRTE_BBDEV_DEBUG
526 /* Read FPGA Ring Control Registers after configuration*/
527 print_ring_reg_debug_info(d->mmio_base, ring_offset);
533 fpga_queue_release(struct rte_bbdev *dev, uint16_t queue_id)
535 struct fpga_5gnr_fec_device *d = dev->data->dev_private;
536 struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
537 struct fpga_ring_ctrl_reg ring_reg;
540 rte_bbdev_log_debug("FPGA Queue[%d] released", queue_id);
543 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg));
544 offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
545 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx);
547 fpga_reg_write_8(d->mmio_base,
548 offset + FPGA_5GNR_FEC_RING_ENABLE, 0x00);
549 /* Clear queue registers */
550 fpga_ring_reg_write(d->mmio_base, offset, ring_reg);
552 /* Mark the Queue as un-assigned */
553 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q->q_idx));
554 rte_free(q->ring_head_addr);
556 dev->data->queues[queue_id].queue_private = NULL;
562 /* Function starts a device queue. */
564 fpga_queue_start(struct rte_bbdev *dev, uint16_t queue_id)
566 struct fpga_5gnr_fec_device *d = dev->data->dev_private;
567 #ifdef RTE_LIBRTE_BBDEV_DEBUG
569 rte_bbdev_log(ERR, "Invalid device pointer");
573 struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
574 uint32_t offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
575 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx);
576 uint8_t enable = 0x01;
577 uint16_t zero = 0x0000;
579 /* Clear queue head and tail variables */
580 q->tail = q->head_free_desc = 0;
582 /* Clear FPGA head_point and tail registers */
583 fpga_reg_write_16(d->mmio_base, offset + FPGA_5GNR_FEC_RING_HEAD_POINT,
585 fpga_reg_write_16(d->mmio_base, offset + FPGA_5GNR_FEC_RING_SHADOW_TAIL,
589 fpga_reg_write_8(d->mmio_base, offset + FPGA_5GNR_FEC_RING_ENABLE,
592 rte_bbdev_log_debug("FPGA Queue[%d] started", queue_id);
596 /* Function stops a device queue. */
598 fpga_queue_stop(struct rte_bbdev *dev, uint16_t queue_id)
600 struct fpga_5gnr_fec_device *d = dev->data->dev_private;
601 #ifdef RTE_LIBRTE_BBDEV_DEBUG
603 rte_bbdev_log(ERR, "Invalid device pointer");
607 struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
608 uint32_t offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
609 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx);
610 uint8_t payload = 0x01;
612 uint8_t timeout = FPGA_QUEUE_FLUSH_TIMEOUT_US /
613 FPGA_TIMEOUT_CHECK_INTERVAL;
615 /* Set flush_queue_en bit to trigger queue flushing */
616 fpga_reg_write_8(d->mmio_base,
617 offset + FPGA_5GNR_FEC_RING_FLUSH_QUEUE_EN, payload);
619 /** Check if queue flush is completed.
620 * FPGA will update the completion flag after queue flushing is
621 * completed. If completion flag is not updated within 1ms it is
622 * considered as a failure.
624 while (!(*((volatile uint8_t *)d->flush_queue_status + q->q_idx)
626 if (counter > timeout) {
627 rte_bbdev_log(ERR, "FPGA Queue Flush failed for queue %d",
631 usleep(FPGA_TIMEOUT_CHECK_INTERVAL);
637 fpga_reg_write_8(d->mmio_base, offset + FPGA_5GNR_FEC_RING_ENABLE,
640 rte_bbdev_log_debug("FPGA Queue[%d] stopped", queue_id);
644 static inline uint16_t
645 get_queue_id(struct rte_bbdev_data *data, uint8_t q_idx)
649 for (queue_id = 0; queue_id < data->num_queues; ++queue_id) {
650 struct fpga_queue *q = data->queues[queue_id].queue_private;
651 if (q != NULL && q->q_idx == q_idx)
658 /* Interrupt handler triggered by FPGA dev for handling specific interrupt */
660 fpga_dev_interrupt_handler(void *cb_arg)
662 struct rte_bbdev *dev = cb_arg;
663 struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private;
664 struct fpga_queue *q;
670 /* Scan queue assigned to this device */
671 for (i = 0; i < FPGA_TOTAL_NUM_QUEUES; ++i) {
673 if (fpga_dev->q_bound_bit_map & q_idx) {
674 queue_id = get_queue_id(dev->data, i);
675 if (queue_id == (uint16_t) -1)
678 /* Check if completion head was changed */
679 q = dev->data->queues[queue_id].queue_private;
680 ring_head = *q->ring_head_addr;
681 if (q->shadow_completion_head != ring_head &&
682 q->irq_enable == 1) {
683 q->shadow_completion_head = ring_head;
684 rte_bbdev_pmd_callback_process(
686 RTE_BBDEV_EVENT_DEQUEUE,
694 fpga_queue_intr_enable(struct rte_bbdev *dev, uint16_t queue_id)
696 struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
698 if (!rte_intr_cap_multiple(dev->intr_handle))
707 fpga_queue_intr_disable(struct rte_bbdev *dev, uint16_t queue_id)
709 struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
716 fpga_intr_enable(struct rte_bbdev *dev)
721 if (!rte_intr_cap_multiple(dev->intr_handle)) {
722 rte_bbdev_log(ERR, "Multiple intr vector is not supported by FPGA (%s)",
727 /* Create event file descriptors for each of 64 queue. Event fds will be
728 * mapped to FPGA IRQs in rte_intr_enable(). This is a 1:1 mapping where
729 * the IRQ number is a direct translation to the queue number.
731 * 63 (FPGA_NUM_INTR_VEC) event fds are created as rte_intr_enable()
732 * mapped the first IRQ to already created interrupt event file
733 * descriptor (intr_handle->fd).
735 if (rte_intr_efd_enable(dev->intr_handle, FPGA_NUM_INTR_VEC)) {
736 rte_bbdev_log(ERR, "Failed to create fds for %u queues",
737 dev->data->num_queues);
741 /* TODO Each event file descriptor is overwritten by interrupt event
742 * file descriptor. That descriptor is added to epoll observed list.
743 * It ensures that callback function assigned to that descriptor will
744 * invoked when any FPGA queue issues interrupt.
746 for (i = 0; i < FPGA_NUM_INTR_VEC; ++i) {
747 if (rte_intr_efds_index_set(dev->intr_handle, i,
748 rte_intr_fd_get(dev->intr_handle)))
752 if (rte_intr_vec_list_alloc(dev->intr_handle, "intr_vec",
753 dev->data->num_queues)) {
754 rte_bbdev_log(ERR, "Failed to allocate %u vectors",
755 dev->data->num_queues);
759 ret = rte_intr_enable(dev->intr_handle);
762 "Couldn't enable interrupts for device: %s",
767 ret = rte_intr_callback_register(dev->intr_handle,
768 fpga_dev_interrupt_handler, dev);
771 "Couldn't register interrupt callback for device: %s",
779 static const struct rte_bbdev_ops fpga_ops = {
780 .setup_queues = fpga_setup_queues,
781 .intr_enable = fpga_intr_enable,
782 .close = fpga_dev_close,
783 .info_get = fpga_dev_info_get,
784 .queue_setup = fpga_queue_setup,
785 .queue_stop = fpga_queue_stop,
786 .queue_start = fpga_queue_start,
787 .queue_release = fpga_queue_release,
788 .queue_intr_enable = fpga_queue_intr_enable,
789 .queue_intr_disable = fpga_queue_intr_disable
793 fpga_dma_enqueue(struct fpga_queue *q, uint16_t num_desc,
794 struct rte_bbdev_stats *queue_stats)
796 #ifdef RTE_BBDEV_OFFLOAD_COST
797 uint64_t start_time = 0;
798 queue_stats->acc_offload_cycles = 0;
800 RTE_SET_USED(queue_stats);
803 /* Update tail and shadow_tail register */
804 q->tail = (q->tail + num_desc) & q->sw_ring_wrap_mask;
808 #ifdef RTE_BBDEV_OFFLOAD_COST
809 /* Start time measurement for enqueue function offload. */
810 start_time = rte_rdtsc_precise();
812 mmio_write_16(q->shadow_tail_addr, q->tail);
814 #ifdef RTE_BBDEV_OFFLOAD_COST
816 queue_stats->acc_offload_cycles += rte_rdtsc_precise() - start_time;
820 /* Read flag value 0/1/ from bitmap */
822 check_bit(uint32_t bitmap, uint32_t bitmask)
824 return bitmap & bitmask;
827 /* Print an error if a descriptor error has occurred.
828 * Return 0 on success, 1 on failure
831 check_desc_error(uint32_t error_code) {
832 switch (error_code) {
833 case DESC_ERR_NO_ERR:
835 case DESC_ERR_K_P_OUT_OF_RANGE:
836 rte_bbdev_log(ERR, "Encode block size K' is out of range");
838 case DESC_ERR_Z_C_NOT_LEGAL:
839 rte_bbdev_log(ERR, "Zc is illegal");
841 case DESC_ERR_DESC_OFFSET_ERR:
843 "Queue offset does not meet the expectation in the FPGA"
846 case DESC_ERR_DESC_READ_FAIL:
847 rte_bbdev_log(ERR, "Unsuccessful completion for descriptor read");
849 case DESC_ERR_DESC_READ_TIMEOUT:
850 rte_bbdev_log(ERR, "Descriptor read time-out");
852 case DESC_ERR_DESC_READ_TLP_POISONED:
853 rte_bbdev_log(ERR, "Descriptor read TLP poisoned");
855 case DESC_ERR_CB_READ_FAIL:
856 rte_bbdev_log(ERR, "Unsuccessful completion for code block");
858 case DESC_ERR_CB_READ_TIMEOUT:
859 rte_bbdev_log(ERR, "Code block read time-out");
861 case DESC_ERR_CB_READ_TLP_POISONED:
862 rte_bbdev_log(ERR, "Code block read TLP poisoned");
864 case DESC_ERR_HBSTORE_ERR:
865 rte_bbdev_log(ERR, "Hbstroe exceeds HARQ buffer size.");
868 rte_bbdev_log(ERR, "Descriptor error unknown error code %u",
875 /* Compute value of k0.
876 * Based on 3GPP 38.212 Table 5.4.2.1-2
877 * Starting position of different redundancy versions, k0
879 static inline uint16_t
880 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index)
884 uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c;
887 return (bg == 1 ? K0_1_1 : K0_1_2) * z_c;
888 else if (rv_index == 2)
889 return (bg == 1 ? K0_2_1 : K0_2_2) * z_c;
891 return (bg == 1 ? K0_3_1 : K0_3_2) * z_c;
893 /* LBRM case - includes a division by N */
895 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb)
897 else if (rv_index == 2)
898 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb)
901 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb)
906 * Set DMA descriptor for encode operation (1 Code Block)
909 * Pointer to a single encode operation.
911 * Pointer to DMA descriptor.
913 * Pointer to pointer to input data which will be decoded.
915 * E value (length of output in bits).
917 * Ncb value (size of the soft buffer).
919 * Length of output buffer
921 * Input offset in rte_mbuf structure. It is used for calculating the point
922 * where data is starting.
924 * Output offset in rte_mbuf structure. It is used for calculating the point
925 * where hard output data will be stored.
927 * Number of CBs contained in one operation.
930 fpga_dma_desc_te_fill(struct rte_bbdev_enc_op *op,
931 struct fpga_dma_enc_desc *desc, struct rte_mbuf *input,
932 struct rte_mbuf *output, uint16_t k_, uint16_t e,
933 uint32_t in_offset, uint32_t out_offset, uint16_t desc_offset,
941 desc->desc_idx = desc_offset;
942 desc->zc = op->ldpc_enc.z_c;
943 desc->bg_idx = op->ldpc_enc.basegraph - 1;
944 desc->qm_idx = op->ldpc_enc.q_m / 2;
945 desc->crc_en = check_bit(op->ldpc_enc.op_flags,
946 RTE_BBDEV_LDPC_CRC_24B_ATTACH);
948 desc->k0 = get_k0(op->ldpc_enc.n_cb, op->ldpc_enc.z_c,
949 op->ldpc_enc.basegraph, op->ldpc_enc.rv_index);
950 desc->ncb = op->ldpc_enc.n_cb;
951 desc->num_null = op->ldpc_enc.n_filler;
952 /* Set inbound data buffer address */
953 desc->in_addr_hi = (uint32_t)(
954 rte_pktmbuf_iova_offset(input, in_offset) >> 32);
955 desc->in_addr_lw = (uint32_t)(
956 rte_pktmbuf_iova_offset(input, in_offset));
958 desc->out_addr_hi = (uint32_t)(
959 rte_pktmbuf_iova_offset(output, out_offset) >> 32);
960 desc->out_addr_lw = (uint32_t)(
961 rte_pktmbuf_iova_offset(output, out_offset));
962 /* Save software context needed for dequeue */
964 /* Set total number of CBs in an op */
965 desc->cbs_in_op = cbs_in_op;
970 * Set DMA descriptor for decode operation (1 Code Block)
973 * Pointer to a single encode operation.
975 * Pointer to DMA descriptor.
977 * Pointer to pointer to input data which will be decoded.
979 * Input offset in rte_mbuf structure. It is used for calculating the point
980 * where data is starting.
982 * Output offset in rte_mbuf structure. It is used for calculating the point
983 * where hard output data will be stored.
985 * Number of CBs contained in one operation.
988 fpga_dma_desc_ld_fill(struct rte_bbdev_dec_op *op,
989 struct fpga_dma_dec_desc *desc,
990 struct rte_mbuf *input, struct rte_mbuf *output,
991 uint16_t harq_in_length,
992 uint32_t in_offset, uint32_t out_offset,
993 uint32_t harq_offset,
994 uint16_t desc_offset,
1000 /* Set inbound data buffer address */
1001 desc->in_addr_hi = (uint32_t)(
1002 rte_pktmbuf_iova_offset(input, in_offset) >> 32);
1003 desc->in_addr_lw = (uint32_t)(
1004 rte_pktmbuf_iova_offset(input, in_offset));
1005 desc->rm_e = op->ldpc_dec.cb_params.e;
1006 desc->harq_input_length = harq_in_length;
1007 desc->et_dis = !check_bit(op->ldpc_dec.op_flags,
1008 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE);
1009 desc->rv = op->ldpc_dec.rv_index;
1010 desc->crc24b_ind = check_bit(op->ldpc_dec.op_flags,
1011 RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK);
1012 desc->drop_crc24b = check_bit(op->ldpc_dec.op_flags,
1013 RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP);
1014 desc->desc_idx = desc_offset;
1015 desc->ncb = op->ldpc_dec.n_cb;
1016 desc->num_null = op->ldpc_dec.n_filler;
1017 desc->hbstroe_offset = harq_offset >> 10;
1018 desc->zc = op->ldpc_dec.z_c;
1019 desc->harqin_en = check_bit(op->ldpc_dec.op_flags,
1020 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE);
1021 desc->bg_idx = op->ldpc_dec.basegraph - 1;
1022 desc->max_iter = op->ldpc_dec.iter_max;
1023 desc->qm_idx = op->ldpc_dec.q_m / 2;
1024 desc->out_addr_hi = (uint32_t)(
1025 rte_pktmbuf_iova_offset(output, out_offset) >> 32);
1026 desc->out_addr_lw = (uint32_t)(
1027 rte_pktmbuf_iova_offset(output, out_offset));
1028 /* Save software context needed for dequeue */
1030 /* Set total number of CBs in an op */
1031 desc->cbs_in_op = cbs_in_op;
1036 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1037 /* Validates LDPC encoder parameters */
1039 validate_enc_op(struct rte_bbdev_enc_op *op __rte_unused)
1041 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &op->ldpc_enc;
1042 struct rte_bbdev_op_enc_ldpc_cb_params *cb = NULL;
1043 struct rte_bbdev_op_enc_ldpc_tb_params *tb = NULL;
1046 if (ldpc_enc->input.length >
1047 RTE_BBDEV_LDPC_MAX_CB_SIZE >> 3) {
1048 rte_bbdev_log(ERR, "CB size (%u) is too big, max: %d",
1049 ldpc_enc->input.length,
1050 RTE_BBDEV_LDPC_MAX_CB_SIZE);
1054 if (op->mempool == NULL) {
1055 rte_bbdev_log(ERR, "Invalid mempool pointer");
1058 if (ldpc_enc->input.data == NULL) {
1059 rte_bbdev_log(ERR, "Invalid input pointer");
1062 if (ldpc_enc->output.data == NULL) {
1063 rte_bbdev_log(ERR, "Invalid output pointer");
1066 if ((ldpc_enc->basegraph > 2) || (ldpc_enc->basegraph == 0)) {
1068 "basegraph (%u) is out of range 1 <= value <= 2",
1069 ldpc_enc->basegraph);
1072 if (ldpc_enc->code_block_mode > RTE_BBDEV_CODE_BLOCK) {
1074 "code_block_mode (%u) is out of range 0:Tb 1:CB",
1075 ldpc_enc->code_block_mode);
1079 if (ldpc_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
1080 tb = &ldpc_enc->tb_params;
1083 "c (%u) is out of range 1 <= value <= %u",
1084 tb->c, RTE_BBDEV_LDPC_MAX_CODE_BLOCKS);
1087 if (tb->cab > tb->c) {
1089 "cab (%u) is greater than c (%u)",
1093 if ((tb->ea < RTE_BBDEV_LDPC_MIN_CB_SIZE)
1094 && tb->r < tb->cab) {
1096 "ea (%u) is less than %u or it is not even",
1097 tb->ea, RTE_BBDEV_LDPC_MIN_CB_SIZE);
1100 if ((tb->eb < RTE_BBDEV_LDPC_MIN_CB_SIZE)
1101 && tb->c > tb->cab) {
1103 "eb (%u) is less than %u",
1104 tb->eb, RTE_BBDEV_LDPC_MIN_CB_SIZE);
1107 if (tb->r > (tb->c - 1)) {
1109 "r (%u) is greater than c - 1 (%u)",
1114 cb = &ldpc_enc->cb_params;
1115 if (cb->e < RTE_BBDEV_LDPC_MIN_CB_SIZE) {
1117 "e (%u) is less than %u or it is not even",
1118 cb->e, RTE_BBDEV_LDPC_MIN_CB_SIZE);
1126 static inline char *
1127 mbuf_append(struct rte_mbuf *m_head, struct rte_mbuf *m, uint16_t len)
1129 if (unlikely(len > rte_pktmbuf_tailroom(m)))
1132 char *tail = (char *)m->buf_addr + m->data_off + m->data_len;
1133 m->data_len = (uint16_t)(m->data_len + len);
1134 m_head->pkt_len = (m_head->pkt_len + len);
1138 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1139 /* Validates LDPC decoder parameters */
1141 validate_dec_op(struct rte_bbdev_dec_op *op __rte_unused)
1143 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &op->ldpc_dec;
1144 struct rte_bbdev_op_dec_ldpc_cb_params *cb = NULL;
1145 struct rte_bbdev_op_dec_ldpc_tb_params *tb = NULL;
1147 if (op->mempool == NULL) {
1148 rte_bbdev_log(ERR, "Invalid mempool pointer");
1151 if (ldpc_dec->rv_index > 3) {
1153 "rv_index (%u) is out of range 0 <= value <= 3",
1154 ldpc_dec->rv_index);
1158 if (ldpc_dec->iter_max == 0) {
1160 "iter_max (%u) is equal to 0",
1161 ldpc_dec->iter_max);
1165 if (ldpc_dec->code_block_mode > RTE_BBDEV_CODE_BLOCK) {
1167 "code_block_mode (%u) is out of range 0 <= value <= 1",
1168 ldpc_dec->code_block_mode);
1172 if (ldpc_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
1173 tb = &ldpc_dec->tb_params;
1176 "c (%u) is out of range 1 <= value <= %u",
1177 tb->c, RTE_BBDEV_LDPC_MAX_CODE_BLOCKS);
1180 if (tb->cab > tb->c) {
1182 "cab (%u) is greater than c (%u)",
1187 cb = &ldpc_dec->cb_params;
1188 if (cb->e < RTE_BBDEV_LDPC_MIN_CB_SIZE) {
1190 "e (%u) is out of range %u <= value <= %u",
1191 cb->e, RTE_BBDEV_LDPC_MIN_CB_SIZE,
1192 RTE_BBDEV_LDPC_MAX_CB_SIZE);
1202 fpga_harq_write_loopback(struct fpga_5gnr_fec_device *fpga_dev,
1203 struct rte_mbuf *harq_input, uint16_t harq_in_length,
1204 uint32_t harq_in_offset, uint32_t harq_out_offset)
1206 uint32_t out_offset = harq_out_offset;
1207 uint32_t in_offset = harq_in_offset;
1208 uint32_t left_length = harq_in_length;
1209 uint32_t reg_32, increment = 0;
1210 uint64_t *input = NULL;
1211 uint32_t last_transaction = left_length
1212 % FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
1215 if (last_transaction > 0)
1216 left_length -= last_transaction;
1219 * Get HARQ buffer size for each VF/PF: When 0x00, there is no
1220 * available DDR space for the corresponding VF/PF.
1222 reg_32 = fpga_reg_read_32(fpga_dev->mmio_base,
1223 FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS);
1224 if (reg_32 < harq_in_length) {
1225 left_length = reg_32;
1226 rte_bbdev_log(ERR, "HARQ in length > HARQ buffer size\n");
1229 input = (uint64_t *)rte_pktmbuf_mtod_offset(harq_input,
1230 uint8_t *, in_offset);
1232 while (left_length > 0) {
1233 if (fpga_reg_read_8(fpga_dev->mmio_base,
1234 FPGA_5GNR_FEC_DDR4_ADDR_RDY_REGS) == 1) {
1235 fpga_reg_write_32(fpga_dev->mmio_base,
1236 FPGA_5GNR_FEC_DDR4_WR_ADDR_REGS,
1238 fpga_reg_write_64(fpga_dev->mmio_base,
1239 FPGA_5GNR_FEC_DDR4_WR_DATA_REGS,
1241 left_length -= FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
1242 out_offset += FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
1244 fpga_reg_write_8(fpga_dev->mmio_base,
1245 FPGA_5GNR_FEC_DDR4_WR_DONE_REGS, 1);
1248 while (last_transaction > 0) {
1249 if (fpga_reg_read_8(fpga_dev->mmio_base,
1250 FPGA_5GNR_FEC_DDR4_ADDR_RDY_REGS) == 1) {
1251 fpga_reg_write_32(fpga_dev->mmio_base,
1252 FPGA_5GNR_FEC_DDR4_WR_ADDR_REGS,
1254 last_word = input[increment];
1255 last_word &= (uint64_t)(1 << (last_transaction * 4))
1257 fpga_reg_write_64(fpga_dev->mmio_base,
1258 FPGA_5GNR_FEC_DDR4_WR_DATA_REGS,
1260 fpga_reg_write_8(fpga_dev->mmio_base,
1261 FPGA_5GNR_FEC_DDR4_WR_DONE_REGS, 1);
1262 last_transaction = 0;
1269 fpga_harq_read_loopback(struct fpga_5gnr_fec_device *fpga_dev,
1270 struct rte_mbuf *harq_output, uint16_t harq_in_length,
1271 uint32_t harq_in_offset, uint32_t harq_out_offset)
1273 uint32_t left_length, in_offset = harq_in_offset;
1275 uint32_t increment = 0;
1276 uint64_t *input = NULL;
1277 uint32_t last_transaction = harq_in_length
1278 % FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
1280 if (last_transaction > 0)
1281 harq_in_length += (8 - last_transaction);
1283 reg = fpga_reg_read_32(fpga_dev->mmio_base,
1284 FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS);
1285 if (reg < harq_in_length) {
1286 harq_in_length = reg;
1287 rte_bbdev_log(ERR, "HARQ in length > HARQ buffer size\n");
1290 if (!mbuf_append(harq_output, harq_output, harq_in_length)) {
1291 rte_bbdev_log(ERR, "HARQ output buffer warning %d %d\n",
1292 harq_output->buf_len -
1293 rte_pktmbuf_headroom(harq_output),
1295 harq_in_length = harq_output->buf_len -
1296 rte_pktmbuf_headroom(harq_output);
1297 if (!mbuf_append(harq_output, harq_output, harq_in_length)) {
1298 rte_bbdev_log(ERR, "HARQ output buffer issue %d %d\n",
1299 harq_output->buf_len, harq_in_length);
1303 left_length = harq_in_length;
1305 input = (uint64_t *)rte_pktmbuf_mtod_offset(harq_output,
1306 uint8_t *, harq_out_offset);
1308 while (left_length > 0) {
1309 fpga_reg_write_32(fpga_dev->mmio_base,
1310 FPGA_5GNR_FEC_DDR4_RD_ADDR_REGS, in_offset);
1311 fpga_reg_write_8(fpga_dev->mmio_base,
1312 FPGA_5GNR_FEC_DDR4_RD_DONE_REGS, 1);
1313 reg = fpga_reg_read_8(fpga_dev->mmio_base,
1314 FPGA_5GNR_FEC_DDR4_RD_RDY_REGS);
1316 reg = fpga_reg_read_8(fpga_dev->mmio_base,
1317 FPGA_5GNR_FEC_DDR4_RD_RDY_REGS);
1318 if (reg == FPGA_DDR_OVERFLOW) {
1320 "Read address is overflow!\n");
1324 input[increment] = fpga_reg_read_64(fpga_dev->mmio_base,
1325 FPGA_5GNR_FEC_DDR4_RD_DATA_REGS);
1326 left_length -= FPGA_5GNR_FEC_DDR_RD_DATA_LEN_IN_BYTES;
1327 in_offset += FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
1329 fpga_reg_write_8(fpga_dev->mmio_base,
1330 FPGA_5GNR_FEC_DDR4_RD_DONE_REGS, 0);
1336 enqueue_ldpc_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op *op,
1337 uint16_t desc_offset)
1339 union fpga_dma_desc *desc;
1341 uint8_t c, crc24_bits = 0;
1342 struct rte_bbdev_op_ldpc_enc *enc = &op->ldpc_enc;
1343 uint16_t in_offset = enc->input.offset;
1344 uint16_t out_offset = enc->output.offset;
1345 struct rte_mbuf *m_in = enc->input.data;
1346 struct rte_mbuf *m_out = enc->output.data;
1347 struct rte_mbuf *m_out_head = enc->output.data;
1348 uint32_t in_length, out_length, e;
1349 uint16_t total_left = enc->input.length;
1350 uint16_t ring_offset;
1353 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1354 /* Validate op structure */
1356 if (validate_enc_op(op) == -1) {
1357 rte_bbdev_log(ERR, "LDPC encoder validation failed");
1362 /* Clear op status */
1365 if (m_in == NULL || m_out == NULL) {
1366 rte_bbdev_log(ERR, "Invalid mbuf pointer");
1367 op->status = 1 << RTE_BBDEV_DATA_ERROR;
1371 if (enc->op_flags & RTE_BBDEV_LDPC_CRC_24B_ATTACH)
1374 if (enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
1375 /* For Transport Block mode */
1377 c = enc->tb_params.c;
1378 e = enc->tb_params.ea;
1379 } else { /* For Code Block mode */
1381 e = enc->cb_params.e;
1384 /* Update total_left */
1385 K = (enc->basegraph == 1 ? 22 : 10) * enc->z_c;
1386 k_ = K - enc->n_filler;
1387 in_length = (k_ - crc24_bits) >> 3;
1388 out_length = (e + 7) >> 3;
1390 total_left = rte_pktmbuf_data_len(m_in) - in_offset;
1392 /* Update offsets */
1393 if (total_left != in_length) {
1394 op->status |= 1 << RTE_BBDEV_DATA_ERROR;
1396 "Mismatch between mbuf length and included CBs sizes %d",
1400 mbuf_append(m_out_head, m_out, out_length);
1402 /* Offset into the ring */
1403 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask);
1404 /* Setup DMA Descriptor */
1405 desc = q->ring_addr + ring_offset;
1407 ret = fpga_dma_desc_te_fill(op, &desc->enc_req, m_in, m_out,
1408 k_, e, in_offset, out_offset, ring_offset, c);
1409 if (unlikely(ret < 0))
1412 /* Update lengths */
1413 total_left -= in_length;
1414 op->ldpc_enc.output.length += out_length;
1416 if (total_left > 0) {
1418 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u",
1419 total_left, in_length);
1423 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1424 print_dma_enc_desc_debug_info(desc);
1430 enqueue_ldpc_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op *op,
1431 uint16_t desc_offset)
1433 union fpga_dma_desc *desc;
1435 uint16_t ring_offset;
1437 uint16_t e, in_length, out_length, k0, l, seg_total_left, sys_cols;
1438 uint16_t K, parity_offset, harq_in_length = 0, harq_out_length = 0;
1439 uint16_t crc24_overlap = 0;
1440 struct rte_bbdev_op_ldpc_dec *dec = &op->ldpc_dec;
1441 struct rte_mbuf *m_in = dec->input.data;
1442 struct rte_mbuf *m_out = dec->hard_output.data;
1443 struct rte_mbuf *m_out_head = dec->hard_output.data;
1444 uint16_t in_offset = dec->input.offset;
1445 uint16_t out_offset = dec->hard_output.offset;
1446 uint32_t harq_offset = 0;
1448 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1449 /* Validate op structure */
1450 if (validate_dec_op(op) == -1) {
1451 rte_bbdev_log(ERR, "LDPC decoder validation failed");
1456 /* Clear op status */
1459 /* Setup DMA Descriptor */
1460 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask);
1461 desc = q->ring_addr + ring_offset;
1463 if (check_bit(dec->op_flags,
1464 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1465 struct rte_mbuf *harq_in = dec->harq_combined_input.data;
1466 struct rte_mbuf *harq_out = dec->harq_combined_output.data;
1467 harq_in_length = dec->harq_combined_input.length;
1468 uint32_t harq_in_offset = dec->harq_combined_input.offset;
1469 uint32_t harq_out_offset = dec->harq_combined_output.offset;
1471 if (check_bit(dec->op_flags,
1472 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE
1474 ret = fpga_harq_write_loopback(q->d, harq_in,
1475 harq_in_length, harq_in_offset,
1477 } else if (check_bit(dec->op_flags,
1478 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE
1480 ret = fpga_harq_read_loopback(q->d, harq_out,
1481 harq_in_length, harq_in_offset,
1483 dec->harq_combined_output.length = harq_in_length;
1485 rte_bbdev_log(ERR, "OP flag Err!");
1488 /* Set descriptor for dequeue */
1489 desc->dec_req.done = 1;
1490 desc->dec_req.error = 0;
1491 desc->dec_req.op_addr = op;
1492 desc->dec_req.cbs_in_op = 1;
1493 /* Mark this dummy descriptor to be dropped by HW */
1494 desc->dec_req.desc_idx = (ring_offset + 1)
1495 & q->sw_ring_wrap_mask;
1496 return ret; /* Error or number of CB */
1499 if (m_in == NULL || m_out == NULL) {
1500 rte_bbdev_log(ERR, "Invalid mbuf pointer");
1501 op->status = 1 << RTE_BBDEV_DATA_ERROR;
1506 e = dec->cb_params.e;
1508 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP))
1511 sys_cols = (dec->basegraph == 1) ? 22 : 10;
1512 K = sys_cols * dec->z_c;
1513 parity_offset = K - 2 * dec->z_c;
1515 out_length = ((K - crc24_overlap - dec->n_filler) >> 3);
1517 seg_total_left = dec->input.length;
1519 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) {
1520 harq_in_length = RTE_MIN(dec->harq_combined_input.length,
1521 (uint32_t)dec->n_cb);
1524 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE)) {
1525 k0 = get_k0(dec->n_cb, dec->z_c,
1526 dec->basegraph, dec->rv_index);
1527 if (k0 > parity_offset)
1530 l = k0 + e + dec->n_filler;
1531 harq_out_length = RTE_MIN(RTE_MAX(harq_in_length, l),
1532 dec->n_cb - dec->n_filler);
1533 dec->harq_combined_output.length = harq_out_length;
1536 mbuf_append(m_out_head, m_out, out_length);
1537 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE))
1538 harq_offset = dec->harq_combined_input.offset;
1539 else if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE))
1540 harq_offset = dec->harq_combined_output.offset;
1542 if ((harq_offset & 0x3FF) > 0) {
1543 rte_bbdev_log(ERR, "Invalid HARQ offset %d", harq_offset);
1544 op->status = 1 << RTE_BBDEV_DATA_ERROR;
1548 ret = fpga_dma_desc_ld_fill(op, &desc->dec_req, m_in, m_out,
1549 harq_in_length, in_offset, out_offset, harq_offset,
1551 if (unlikely(ret < 0))
1553 /* Update lengths */
1554 seg_total_left -= in_length;
1555 op->ldpc_dec.hard_output.length += out_length;
1556 if (seg_total_left > 0) {
1558 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u",
1559 seg_total_left, in_length);
1563 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1564 print_dma_dec_desc_debug_info(desc);
1571 fpga_enqueue_ldpc_enc(struct rte_bbdev_queue_data *q_data,
1572 struct rte_bbdev_enc_op **ops, uint16_t num)
1574 uint16_t i, total_enqueued_cbs = 0;
1577 struct fpga_queue *q = q_data->queue_private;
1578 union fpga_dma_desc *desc;
1580 /* Check if queue is not full */
1581 if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) ==
1585 /* Calculates available space */
1586 avail = (q->head_free_desc > q->tail) ?
1587 q->head_free_desc - q->tail - 1 :
1588 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1;
1590 for (i = 0; i < num; ++i) {
1592 /* Check if there is available space for further
1595 if (unlikely(avail - 1 < 0))
1598 enqueued_cbs = enqueue_ldpc_enc_one_op_cb(q, ops[i],
1599 total_enqueued_cbs);
1601 if (enqueued_cbs < 0)
1604 total_enqueued_cbs += enqueued_cbs;
1606 rte_bbdev_log_debug("enqueuing enc ops [%d/%d] | head %d | tail %d",
1607 total_enqueued_cbs, num,
1608 q->head_free_desc, q->tail);
1611 /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt
1612 * only when all previous CBs were already processed.
1614 desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1)
1615 & q->sw_ring_wrap_mask);
1616 desc->enc_req.irq_en = q->irq_enable;
1618 fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats);
1621 q_data->queue_stats.enqueued_count += i;
1622 q_data->queue_stats.enqueue_err_count += num - i;
1628 fpga_enqueue_ldpc_dec(struct rte_bbdev_queue_data *q_data,
1629 struct rte_bbdev_dec_op **ops, uint16_t num)
1631 uint16_t i, total_enqueued_cbs = 0;
1634 struct fpga_queue *q = q_data->queue_private;
1635 union fpga_dma_desc *desc;
1637 /* Check if queue is not full */
1638 if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) ==
1642 /* Calculates available space */
1643 avail = (q->head_free_desc > q->tail) ?
1644 q->head_free_desc - q->tail - 1 :
1645 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1;
1647 for (i = 0; i < num; ++i) {
1649 /* Check if there is available space for further
1652 if (unlikely(avail - 1 < 0))
1655 enqueued_cbs = enqueue_ldpc_dec_one_op_cb(q, ops[i],
1656 total_enqueued_cbs);
1658 if (enqueued_cbs < 0)
1661 total_enqueued_cbs += enqueued_cbs;
1663 rte_bbdev_log_debug("enqueuing dec ops [%d/%d] | head %d | tail %d",
1664 total_enqueued_cbs, num,
1665 q->head_free_desc, q->tail);
1669 q_data->queue_stats.enqueued_count += i;
1670 q_data->queue_stats.enqueue_err_count += num - i;
1672 /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt
1673 * only when all previous CBs were already processed.
1675 desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1)
1676 & q->sw_ring_wrap_mask);
1677 desc->enc_req.irq_en = q->irq_enable;
1678 fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats);
1684 dequeue_ldpc_enc_one_op_cb(struct fpga_queue *q,
1685 struct rte_bbdev_enc_op **op,
1686 uint16_t desc_offset)
1688 union fpga_dma_desc *desc;
1690 /* Set current desc */
1691 desc = q->ring_addr + ((q->head_free_desc + desc_offset)
1692 & q->sw_ring_wrap_mask);
1695 if (desc->enc_req.done == 0)
1698 /* make sure the response is read atomically */
1701 rte_bbdev_log_debug("DMA response desc %p", desc);
1703 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1704 print_dma_enc_desc_debug_info(desc);
1707 *op = desc->enc_req.op_addr;
1708 /* Check the descriptor error field, return 1 on error */
1709 desc_error = check_desc_error(desc->enc_req.error);
1710 (*op)->status = desc_error << RTE_BBDEV_DATA_ERROR;
1717 dequeue_ldpc_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op **op,
1718 uint16_t desc_offset)
1720 union fpga_dma_desc *desc;
1722 /* Set descriptor */
1723 desc = q->ring_addr + ((q->head_free_desc + desc_offset)
1724 & q->sw_ring_wrap_mask);
1726 /* Verify done bit is set */
1727 if (desc->dec_req.done == 0)
1730 /* make sure the response is read atomically */
1733 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1734 print_dma_dec_desc_debug_info(desc);
1737 *op = desc->dec_req.op_addr;
1739 if (check_bit((*op)->ldpc_dec.op_flags,
1740 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1745 /* FPGA reports iterations based on round-up minus 1 */
1746 (*op)->ldpc_dec.iter_count = desc->dec_req.iter + 1;
1747 /* CRC Check criteria */
1748 if (desc->dec_req.crc24b_ind && !(desc->dec_req.crcb_pass))
1749 (*op)->status = 1 << RTE_BBDEV_CRC_ERROR;
1750 /* et_pass = 0 when decoder fails */
1751 (*op)->status |= !(desc->dec_req.et_pass) << RTE_BBDEV_SYNDROME_ERROR;
1752 /* Check the descriptor error field, return 1 on error */
1753 desc_error = check_desc_error(desc->dec_req.error);
1754 (*op)->status |= desc_error << RTE_BBDEV_DATA_ERROR;
1759 fpga_dequeue_ldpc_enc(struct rte_bbdev_queue_data *q_data,
1760 struct rte_bbdev_enc_op **ops, uint16_t num)
1762 struct fpga_queue *q = q_data->queue_private;
1763 uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask;
1765 uint16_t dequeued_cbs = 0;
1768 for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) {
1769 ret = dequeue_ldpc_enc_one_op_cb(q, &ops[i], dequeued_cbs);
1774 dequeued_cbs += ret;
1776 rte_bbdev_log_debug("dequeuing enc ops [%d/%d] | head %d | tail %d",
1777 dequeued_cbs, num, q->head_free_desc, q->tail);
1781 q->head_free_desc = (q->head_free_desc + dequeued_cbs) &
1782 q->sw_ring_wrap_mask;
1785 q_data->queue_stats.dequeued_count += i;
1791 fpga_dequeue_ldpc_dec(struct rte_bbdev_queue_data *q_data,
1792 struct rte_bbdev_dec_op **ops, uint16_t num)
1794 struct fpga_queue *q = q_data->queue_private;
1795 uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask;
1797 uint16_t dequeued_cbs = 0;
1800 for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) {
1801 ret = dequeue_ldpc_dec_one_op_cb(q, &ops[i], dequeued_cbs);
1806 dequeued_cbs += ret;
1808 rte_bbdev_log_debug("dequeuing dec ops [%d/%d] | head %d | tail %d",
1809 dequeued_cbs, num, q->head_free_desc, q->tail);
1813 q->head_free_desc = (q->head_free_desc + dequeued_cbs) &
1814 q->sw_ring_wrap_mask;
1817 q_data->queue_stats.dequeued_count += i;
1823 /* Initialization Function */
1825 fpga_5gnr_fec_init(struct rte_bbdev *dev, struct rte_pci_driver *drv)
1827 struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device);
1829 dev->dev_ops = &fpga_ops;
1830 dev->enqueue_ldpc_enc_ops = fpga_enqueue_ldpc_enc;
1831 dev->enqueue_ldpc_dec_ops = fpga_enqueue_ldpc_dec;
1832 dev->dequeue_ldpc_enc_ops = fpga_dequeue_ldpc_enc;
1833 dev->dequeue_ldpc_dec_ops = fpga_dequeue_ldpc_dec;
1835 ((struct fpga_5gnr_fec_device *) dev->data->dev_private)->pf_device =
1836 !strcmp(drv->driver.name,
1837 RTE_STR(FPGA_5GNR_FEC_PF_DRIVER_NAME));
1838 ((struct fpga_5gnr_fec_device *) dev->data->dev_private)->mmio_base =
1839 pci_dev->mem_resource[0].addr;
1841 rte_bbdev_log_debug(
1842 "Init device %s [%s] @ virtaddr %p phyaddr %#"PRIx64,
1843 drv->driver.name, dev->data->name,
1844 (void *)pci_dev->mem_resource[0].addr,
1845 pci_dev->mem_resource[0].phys_addr);
1849 fpga_5gnr_fec_probe(struct rte_pci_driver *pci_drv,
1850 struct rte_pci_device *pci_dev)
1852 struct rte_bbdev *bbdev = NULL;
1853 char dev_name[RTE_BBDEV_NAME_MAX_LEN];
1855 if (pci_dev == NULL) {
1856 rte_bbdev_log(ERR, "NULL PCI device");
1860 rte_pci_device_name(&pci_dev->addr, dev_name, sizeof(dev_name));
1862 /* Allocate memory to be used privately by drivers */
1863 bbdev = rte_bbdev_allocate(pci_dev->device.name);
1867 /* allocate device private memory */
1868 bbdev->data->dev_private = rte_zmalloc_socket(dev_name,
1869 sizeof(struct fpga_5gnr_fec_device),
1870 RTE_CACHE_LINE_SIZE,
1871 pci_dev->device.numa_node);
1873 if (bbdev->data->dev_private == NULL) {
1875 "Allocate of %zu bytes for device \"%s\" failed",
1876 sizeof(struct fpga_5gnr_fec_device), dev_name);
1877 rte_bbdev_release(bbdev);
1881 /* Fill HW specific part of device structure */
1882 bbdev->device = &pci_dev->device;
1883 bbdev->intr_handle = pci_dev->intr_handle;
1884 bbdev->data->socket_id = pci_dev->device.numa_node;
1886 /* Invoke FEC FPGA device initialization function */
1887 fpga_5gnr_fec_init(bbdev, pci_drv);
1889 rte_bbdev_log_debug("bbdev id = %u [%s]",
1890 bbdev->data->dev_id, dev_name);
1892 struct fpga_5gnr_fec_device *d = bbdev->data->dev_private;
1893 uint32_t version_id = fpga_reg_read_32(d->mmio_base,
1894 FPGA_5GNR_FEC_VERSION_ID);
1895 rte_bbdev_log(INFO, "FEC FPGA RTL v%u.%u",
1896 ((uint16_t)(version_id >> 16)), ((uint16_t)version_id));
1898 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1899 if (!strcmp(pci_drv->driver.name,
1900 RTE_STR(FPGA_5GNR_FEC_PF_DRIVER_NAME)))
1901 print_static_reg_debug_info(d->mmio_base);
1907 fpga_5gnr_fec_remove(struct rte_pci_device *pci_dev)
1909 struct rte_bbdev *bbdev;
1913 if (pci_dev == NULL)
1917 bbdev = rte_bbdev_get_named_dev(pci_dev->device.name);
1918 if (bbdev == NULL) {
1920 "Couldn't find HW dev \"%s\" to uninitialise it",
1921 pci_dev->device.name);
1924 dev_id = bbdev->data->dev_id;
1926 /* free device private memory before close */
1927 rte_free(bbdev->data->dev_private);
1930 ret = rte_bbdev_close(dev_id);
1933 "Device %i failed to close during uninit: %i",
1936 /* release bbdev from library */
1937 ret = rte_bbdev_release(bbdev);
1939 rte_bbdev_log(ERR, "Device %i failed to uninit: %i", dev_id,
1942 rte_bbdev_log_debug("Destroyed bbdev = %u", dev_id);
1948 set_default_fpga_conf(struct rte_fpga_5gnr_fec_conf *def_conf)
1950 /* clear default configuration before initialization */
1951 memset(def_conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf));
1952 /* Set pf mode to true */
1953 def_conf->pf_mode_en = true;
1955 /* Set ratio between UL and DL to 1:1 (unit of weight is 3 CBs) */
1956 def_conf->ul_bandwidth = 3;
1957 def_conf->dl_bandwidth = 3;
1959 /* Set Load Balance Factor to 64 */
1960 def_conf->dl_load_balance = 64;
1961 def_conf->ul_load_balance = 64;
1964 /* Initial configuration of FPGA 5GNR FEC device */
1966 rte_fpga_5gnr_fec_configure(const char *dev_name,
1967 const struct rte_fpga_5gnr_fec_conf *conf)
1969 uint32_t payload_32, address;
1970 uint16_t payload_16;
1972 uint16_t q_id, vf_id, total_q_id, total_ul_q_id, total_dl_q_id;
1973 struct rte_bbdev *bbdev = rte_bbdev_get_named_dev(dev_name);
1974 struct rte_fpga_5gnr_fec_conf def_conf;
1976 if (bbdev == NULL) {
1978 "Invalid dev_name (%s), or device is not yet initialised",
1983 struct fpga_5gnr_fec_device *d = bbdev->data->dev_private;
1987 "FPGA Configuration was not provided. Default configuration will be loaded.");
1988 set_default_fpga_conf(&def_conf);
1993 * Configure UL:DL ratio.
1997 payload_16 = (conf->dl_bandwidth << 8) | conf->ul_bandwidth;
1998 address = FPGA_5GNR_FEC_CONFIGURATION;
1999 fpga_reg_write_16(d->mmio_base, address, payload_16);
2001 /* Clear all queues registers */
2002 payload_32 = FPGA_INVALID_HW_QUEUE_ID;
2003 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) {
2004 address = (q_id << 2) + FPGA_5GNR_FEC_QUEUE_MAP;
2005 fpga_reg_write_32(d->mmio_base, address, payload_32);
2009 * If PF mode is enabled allocate all queues for PF only.
2011 * For VF mode each VF can have different number of UL and DL queues.
2012 * Total number of queues to configure cannot exceed FPGA
2013 * capabilities - 64 queues - 32 queues for UL and 32 queues for DL.
2014 * Queues mapping is done according to configuration:
2020 * | conf->vf_dl_queues_number[0] - 1 | 0 |
2021 * | conf->vf_dl_queues_number[0] | 1 |
2023 * | conf->vf_dl_queues_number[1] - 1 | 1 |
2025 * | conf->vf_dl_queues_number[7] - 1 | 7 |
2031 * | conf->vf_ul_queues_number[0] - 1 | 0 |
2032 * | conf->vf_ul_queues_number[0] | 1 |
2034 * | conf->vf_ul_queues_number[1] - 1 | 1 |
2036 * | conf->vf_ul_queues_number[7] - 1 | 7 |
2038 * Example of configuration:
2039 * conf->vf_ul_queues_number[0] = 4; -> 4 UL queues for VF0
2040 * conf->vf_dl_queues_number[0] = 4; -> 4 DL queues for VF0
2041 * conf->vf_ul_queues_number[1] = 2; -> 2 UL queues for VF1
2042 * conf->vf_dl_queues_number[1] = 2; -> 2 DL queues for VF1
2062 if (conf->pf_mode_en) {
2064 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) {
2065 address = (q_id << 2) + FPGA_5GNR_FEC_QUEUE_MAP;
2066 fpga_reg_write_32(d->mmio_base, address, payload_32);
2069 /* Calculate total number of UL and DL queues to configure */
2070 total_ul_q_id = total_dl_q_id = 0;
2071 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) {
2072 total_ul_q_id += conf->vf_ul_queues_number[vf_id];
2073 total_dl_q_id += conf->vf_dl_queues_number[vf_id];
2075 total_q_id = total_dl_q_id + total_ul_q_id;
2077 * Check if total number of queues to configure does not exceed
2078 * FPGA capabilities (64 queues - 32 UL and 32 DL queues)
2080 if ((total_ul_q_id > FPGA_NUM_UL_QUEUES) ||
2081 (total_dl_q_id > FPGA_NUM_DL_QUEUES) ||
2082 (total_q_id > FPGA_TOTAL_NUM_QUEUES)) {
2084 "FPGA Configuration failed. Too many queues to configure: UL_Q %u, DL_Q %u, FPGA_Q %u",
2085 total_ul_q_id, total_dl_q_id,
2086 FPGA_TOTAL_NUM_QUEUES);
2090 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) {
2091 for (q_id = 0; q_id < conf->vf_ul_queues_number[vf_id];
2092 ++q_id, ++total_ul_q_id) {
2093 address = (total_ul_q_id << 2) +
2094 FPGA_5GNR_FEC_QUEUE_MAP;
2095 payload_32 = ((0x80 + vf_id) << 16) | 0x1;
2096 fpga_reg_write_32(d->mmio_base, address,
2101 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) {
2102 for (q_id = 0; q_id < conf->vf_dl_queues_number[vf_id];
2103 ++q_id, ++total_dl_q_id) {
2104 address = ((total_dl_q_id + FPGA_NUM_UL_QUEUES)
2105 << 2) + FPGA_5GNR_FEC_QUEUE_MAP;
2106 payload_32 = ((0x80 + vf_id) << 16) | 0x1;
2107 fpga_reg_write_32(d->mmio_base, address,
2113 /* Setting Load Balance Factor */
2114 payload_16 = (conf->dl_load_balance << 8) | (conf->ul_load_balance);
2115 address = FPGA_5GNR_FEC_LOAD_BALANCE_FACTOR;
2116 fpga_reg_write_16(d->mmio_base, address, payload_16);
2118 /* Setting length of ring descriptor entry */
2119 payload_16 = FPGA_RING_DESC_ENTRY_LENGTH;
2120 address = FPGA_5GNR_FEC_RING_DESC_LEN;
2121 fpga_reg_write_16(d->mmio_base, address, payload_16);
2123 /* Setting FLR timeout value */
2124 payload_16 = conf->flr_time_out;
2125 address = FPGA_5GNR_FEC_FLR_TIME_OUT;
2126 fpga_reg_write_16(d->mmio_base, address, payload_16);
2128 /* Queue PF/VF mapping table is ready */
2130 address = FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE;
2131 fpga_reg_write_8(d->mmio_base, address, payload_8);
2133 rte_bbdev_log_debug("PF FPGA 5GNR FEC configuration complete for %s",
2136 #ifdef RTE_LIBRTE_BBDEV_DEBUG
2137 print_static_reg_debug_info(d->mmio_base);
2142 /* FPGA 5GNR FEC PCI PF address map */
2143 static struct rte_pci_id pci_id_fpga_5gnr_fec_pf_map[] = {
2145 RTE_PCI_DEVICE(FPGA_5GNR_FEC_VENDOR_ID,
2146 FPGA_5GNR_FEC_PF_DEVICE_ID)
2151 static struct rte_pci_driver fpga_5gnr_fec_pci_pf_driver = {
2152 .probe = fpga_5gnr_fec_probe,
2153 .remove = fpga_5gnr_fec_remove,
2154 .id_table = pci_id_fpga_5gnr_fec_pf_map,
2155 .drv_flags = RTE_PCI_DRV_NEED_MAPPING
2158 /* FPGA 5GNR FEC PCI VF address map */
2159 static struct rte_pci_id pci_id_fpga_5gnr_fec_vf_map[] = {
2161 RTE_PCI_DEVICE(FPGA_5GNR_FEC_VENDOR_ID,
2162 FPGA_5GNR_FEC_VF_DEVICE_ID)
2167 static struct rte_pci_driver fpga_5gnr_fec_pci_vf_driver = {
2168 .probe = fpga_5gnr_fec_probe,
2169 .remove = fpga_5gnr_fec_remove,
2170 .id_table = pci_id_fpga_5gnr_fec_vf_map,
2171 .drv_flags = RTE_PCI_DRV_NEED_MAPPING
2175 RTE_PMD_REGISTER_PCI(FPGA_5GNR_FEC_PF_DRIVER_NAME, fpga_5gnr_fec_pci_pf_driver);
2176 RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_PF_DRIVER_NAME,
2177 pci_id_fpga_5gnr_fec_pf_map);
2178 RTE_PMD_REGISTER_PCI(FPGA_5GNR_FEC_VF_DRIVER_NAME, fpga_5gnr_fec_pci_vf_driver);
2179 RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_VF_DRIVER_NAME,
2180 pci_id_fpga_5gnr_fec_vf_map);