drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4
   5 #include <unistd.h>
   6
   7 #include <rte_common.h>
   8 #include <rte_log.h>
   9 #include <rte_dev.h>
  10 #include <rte_malloc.h>
  11 #include <rte_mempool.h>
  12 #include <rte_errno.h>
  13 #include <rte_pci.h>
  14 #include <rte_bus_pci.h>
  15 #include <rte_byteorder.h>
  16 #ifdef RTE_BBDEV_OFFLOAD_COST
  17 #include <rte_cycles.h>
  18 #endif
  19
  20 #include <rte_bbdev.h>
  21 #include <rte_bbdev_pmd.h>
  22
  23 #include "fpga_5gnr_fec.h"
  24 #include "rte_pmd_fpga_5gnr_fec.h"
  25
  26 /* 5GNR SW PMD logging ID */
  27 static int fpga_5gnr_fec_logtype;
  28
  29 #ifdef RTE_LIBRTE_BBDEV_DEBUG
  30
  31 /* Read Ring Control Register of FPGA 5GNR FEC device */
  32 static inline void
  33 print_ring_reg_debug_info(void *mmio_base, uint32_t offset)
  34 {
  35         rte_bbdev_log_debug(
  36                 "FPGA MMIO base address @ %p | Ring Control Register @ offset = 0x%08"
  37                 PRIx32, mmio_base, offset);
  38         rte_bbdev_log_debug(
  39                 "RING_BASE_ADDR = 0x%016"PRIx64,
  40                 fpga_reg_read_64(mmio_base, offset));
  41         rte_bbdev_log_debug(
  42                 "RING_HEAD_ADDR = 0x%016"PRIx64,
  43                 fpga_reg_read_64(mmio_base, offset +
  44                                 FPGA_5GNR_FEC_RING_HEAD_ADDR));
  45         rte_bbdev_log_debug(
  46                 "RING_SIZE = 0x%04"PRIx16,
  47                 fpga_reg_read_16(mmio_base, offset +
  48                                 FPGA_5GNR_FEC_RING_SIZE));
  49         rte_bbdev_log_debug(
  50                 "RING_MISC = 0x%02"PRIx8,
  51                 fpga_reg_read_8(mmio_base, offset +
  52                                 FPGA_5GNR_FEC_RING_MISC));
  53         rte_bbdev_log_debug(
  54                 "RING_ENABLE = 0x%02"PRIx8,
  55                 fpga_reg_read_8(mmio_base, offset +
  56                                 FPGA_5GNR_FEC_RING_ENABLE));
  57         rte_bbdev_log_debug(
  58                 "RING_FLUSH_QUEUE_EN = 0x%02"PRIx8,
  59                 fpga_reg_read_8(mmio_base, offset +
  60                                 FPGA_5GNR_FEC_RING_FLUSH_QUEUE_EN));
  61         rte_bbdev_log_debug(
  62                 "RING_SHADOW_TAIL = 0x%04"PRIx16,
  63                 fpga_reg_read_16(mmio_base, offset +
  64                                 FPGA_5GNR_FEC_RING_SHADOW_TAIL));
  65         rte_bbdev_log_debug(
  66                 "RING_HEAD_POINT = 0x%04"PRIx16,
  67                 fpga_reg_read_16(mmio_base, offset +
  68                                 FPGA_5GNR_FEC_RING_HEAD_POINT));
  69 }
  70
  71 /* Read Static Register of FPGA 5GNR FEC device */
  72 static inline void
  73 print_static_reg_debug_info(void *mmio_base)
  74 {
  75         uint16_t config = fpga_reg_read_16(mmio_base,
  76                         FPGA_5GNR_FEC_CONFIGURATION);
  77         uint8_t qmap_done = fpga_reg_read_8(mmio_base,
  78                         FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE);
  79         uint16_t lb_factor = fpga_reg_read_16(mmio_base,
  80                         FPGA_5GNR_FEC_LOAD_BALANCE_FACTOR);
  81         uint16_t ring_desc_len = fpga_reg_read_16(mmio_base,
  82                         FPGA_5GNR_FEC_RING_DESC_LEN);
  83         uint16_t flr_time_out = fpga_reg_read_16(mmio_base,
  84                         FPGA_5GNR_FEC_FLR_TIME_OUT);
  85
  86         rte_bbdev_log_debug("UL.DL Weights = %u.%u",
  87                         ((uint8_t)config), ((uint8_t)(config >> 8)));
  88         rte_bbdev_log_debug("UL.DL Load Balance = %u.%u",
  89                         ((uint8_t)lb_factor), ((uint8_t)(lb_factor >> 8)));
  90         rte_bbdev_log_debug("Queue-PF/VF Mapping Table = %s",
  91                         (qmap_done > 0) ? "READY" : "NOT-READY");
  92         rte_bbdev_log_debug("Ring Descriptor Size = %u bytes",
  93                         ring_desc_len*FPGA_RING_DESC_LEN_UNIT_BYTES);
  94         rte_bbdev_log_debug("FLR Timeout = %f usec",
  95                         (float)flr_time_out*FPGA_FLR_TIMEOUT_UNIT);
  96 }
  97
  98 /* Print decode DMA Descriptor of FPGA 5GNR Decoder device */
  99 static void
 100 print_dma_dec_desc_debug_info(union fpga_dma_desc *desc)
 101 {
 102         rte_bbdev_log_debug("DMA response desc %p\n"
 103                 "\t-- done(%"PRIu32") | iter(%"PRIu32") | et_pass(%"PRIu32")"
 104                 " | crcb_pass (%"PRIu32") | error(%"PRIu32")\n"
 105                 "\t-- qm_idx(%"PRIu32") | max_iter(%"PRIu32") | "
 106                 "bg_idx (%"PRIu32") | harqin_en(%"PRIu32") | zc(%"PRIu32")\n"
 107                 "\t-- hbstroe_offset(%"PRIu32") | num_null (%"PRIu32") "
 108                 "| irq_en(%"PRIu32")\n"
 109                 "\t-- ncb(%"PRIu32") | desc_idx (%"PRIu32") | "
 110                 "drop_crc24b(%"PRIu32") | RV (%"PRIu32")\n"
 111                 "\t-- crc24b_ind(%"PRIu32") | et_dis (%"PRIu32")\n"
 112                 "\t-- harq_input_length(%"PRIu32") | rm_e(%"PRIu32")\n"
 113                 "\t-- cbs_in_op(%"PRIu32") | in_add (0x%08"PRIx32"%08"PRIx32")"
 114                 "| out_add (0x%08"PRIx32"%08"PRIx32")",
 115                 desc,
 116                 (uint32_t)desc->dec_req.done,
 117                 (uint32_t)desc->dec_req.iter,
 118                 (uint32_t)desc->dec_req.et_pass,
 119                 (uint32_t)desc->dec_req.crcb_pass,
 120                 (uint32_t)desc->dec_req.error,
 121                 (uint32_t)desc->dec_req.qm_idx,
 122                 (uint32_t)desc->dec_req.max_iter,
 123                 (uint32_t)desc->dec_req.bg_idx,
 124                 (uint32_t)desc->dec_req.harqin_en,
 125                 (uint32_t)desc->dec_req.zc,
 126                 (uint32_t)desc->dec_req.hbstroe_offset,
 127                 (uint32_t)desc->dec_req.num_null,
 128                 (uint32_t)desc->dec_req.irq_en,
 129                 (uint32_t)desc->dec_req.ncb,
 130                 (uint32_t)desc->dec_req.desc_idx,
 131                 (uint32_t)desc->dec_req.drop_crc24b,
 132                 (uint32_t)desc->dec_req.rv,
 133                 (uint32_t)desc->dec_req.crc24b_ind,
 134                 (uint32_t)desc->dec_req.et_dis,
 135                 (uint32_t)desc->dec_req.harq_input_length,
 136                 (uint32_t)desc->dec_req.rm_e,
 137                 (uint32_t)desc->dec_req.cbs_in_op,
 138                 (uint32_t)desc->dec_req.in_addr_hi,
 139                 (uint32_t)desc->dec_req.in_addr_lw,
 140                 (uint32_t)desc->dec_req.out_addr_hi,
 141                 (uint32_t)desc->dec_req.out_addr_lw);
 142         uint32_t *word = (uint32_t *) desc;
 143         rte_bbdev_log_debug("%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n"
 144                         "%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n",
 145                         word[0], word[1], word[2], word[3],
 146                         word[4], word[5], word[6], word[7]);
 147 }
 148
 149 /* Print decode DMA Descriptor of FPGA 5GNR encoder device */
 150 static void
 151 print_dma_enc_desc_debug_info(union fpga_dma_desc *desc)
 152 {
 153         rte_bbdev_log_debug("DMA response desc %p\n"
 154                         "%"PRIu32" %"PRIu32"\n"
 155                         "K' %"PRIu32" E %"PRIu32" desc %"PRIu32" Z %"PRIu32"\n"
 156                         "BG %"PRIu32" Qm %"PRIu32" CRC %"PRIu32" IRQ %"PRIu32"\n"
 157                         "k0 %"PRIu32" Ncb %"PRIu32" F %"PRIu32"\n",
 158                         desc,
 159                         (uint32_t)desc->enc_req.done,
 160                         (uint32_t)desc->enc_req.error,
 161
 162                         (uint32_t)desc->enc_req.k_,
 163                         (uint32_t)desc->enc_req.rm_e,
 164                         (uint32_t)desc->enc_req.desc_idx,
 165                         (uint32_t)desc->enc_req.zc,
 166
 167                         (uint32_t)desc->enc_req.bg_idx,
 168                         (uint32_t)desc->enc_req.qm_idx,
 169                         (uint32_t)desc->enc_req.crc_en,
 170                         (uint32_t)desc->enc_req.irq_en,
 171
 172                         (uint32_t)desc->enc_req.k0,
 173                         (uint32_t)desc->enc_req.ncb,
 174                         (uint32_t)desc->enc_req.num_null);
 175         uint32_t *word = (uint32_t *) desc;
 176         rte_bbdev_log_debug("%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n"
 177                         "%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n",
 178                         word[0], word[1], word[2], word[3],
 179                         word[4], word[5], word[6], word[7]);
 180 }
 181
 182 #endif
 183
 184 static int
 185 fpga_setup_queues(struct rte_bbdev *dev, uint16_t num_queues, int socket_id)
 186 {
 187         /* Number of queues bound to a PF/VF */
 188         uint32_t hw_q_num = 0;
 189         uint32_t ring_size, payload, address, q_id, offset;
 190         rte_iova_t phys_addr;
 191         struct fpga_ring_ctrl_reg ring_reg;
 192         struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private;
 193
 194         address = FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE;
 195         if (!(fpga_reg_read_32(fpga_dev->mmio_base, address) & 0x1)) {
 196                 rte_bbdev_log(ERR,
 197                                 "Queue-PF/VF mapping is not set! Was PF configured for device (%s) ?",
 198                                 dev->data->name);
 199                 return -EPERM;
 200         }
 201
 202         /* Clear queue registers structure */
 203         memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg));
 204
 205         /* Scan queue map.
 206          * If a queue is valid and mapped to a calling PF/VF the read value is
 207          * replaced with a queue ID and if it's not then
 208          * FPGA_INVALID_HW_QUEUE_ID is returned.
 209          */
 210         for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) {
 211                 uint32_t hw_q_id = fpga_reg_read_32(fpga_dev->mmio_base,
 212                                 FPGA_5GNR_FEC_QUEUE_MAP + (q_id << 2));
 213
 214                 rte_bbdev_log_debug("%s: queue ID: %u, registry queue ID: %u",
 215                                 dev->device->name, q_id, hw_q_id);
 216
 217                 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID) {
 218                         fpga_dev->q_bound_bit_map |= (1ULL << q_id);
 219                         /* Clear queue register of found queue */
 220                         offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
 221                                 (sizeof(struct fpga_ring_ctrl_reg) * q_id);
 222                         fpga_ring_reg_write(fpga_dev->mmio_base,
 223                                         offset, ring_reg);
 224                         ++hw_q_num;
 225                 }
 226         }
 227         if (hw_q_num == 0) {
 228                 rte_bbdev_log(ERR,
 229                         "No HW queues assigned to this device. Probably this is a VF configured for PF mode. Check device configuration!");
 230                 return -ENODEV;
 231         }
 232
 233         if (num_queues > hw_q_num) {
 234                 rte_bbdev_log(ERR,
 235                         "Not enough queues for device %s! Requested: %u, available: %u",
 236                         dev->device->name, num_queues, hw_q_num);
 237                 return -EINVAL;
 238         }
 239
 240         ring_size = FPGA_RING_MAX_SIZE * sizeof(struct fpga_dma_dec_desc);
 241
 242         /* Enforce 32 byte alignment */
 243         RTE_BUILD_BUG_ON((RTE_CACHE_LINE_SIZE % 32) != 0);
 244
 245         /* Allocate memory for SW descriptor rings */
 246         fpga_dev->sw_rings = rte_zmalloc_socket(dev->device->driver->name,
 247                         num_queues * ring_size, RTE_CACHE_LINE_SIZE,
 248                         socket_id);
 249         if (fpga_dev->sw_rings == NULL) {
 250                 rte_bbdev_log(ERR,
 251                                 "Failed to allocate memory for %s:%u sw_rings",
 252                                 dev->device->driver->name, dev->data->dev_id);
 253                 return -ENOMEM;
 254         }
 255
 256         fpga_dev->sw_rings_phys = rte_malloc_virt2iova(fpga_dev->sw_rings);
 257         fpga_dev->sw_ring_size = ring_size;
 258         fpga_dev->sw_ring_max_depth = FPGA_RING_MAX_SIZE;
 259
 260         /* Allocate memory for ring flush status */
 261         fpga_dev->flush_queue_status = rte_zmalloc_socket(NULL,
 262                         sizeof(uint64_t), RTE_CACHE_LINE_SIZE, socket_id);
 263         if (fpga_dev->flush_queue_status == NULL) {
 264                 rte_bbdev_log(ERR,
 265                                 "Failed to allocate memory for %s:%u flush_queue_status",
 266                                 dev->device->driver->name, dev->data->dev_id);
 267                 return -ENOMEM;
 268         }
 269
 270         /* Set the flush status address registers */
 271         phys_addr = rte_malloc_virt2iova(fpga_dev->flush_queue_status);
 272
 273         address = FPGA_5GNR_FEC_VFQ_FLUSH_STATUS_LW;
 274         payload = (uint32_t)(phys_addr);
 275         fpga_reg_write_32(fpga_dev->mmio_base, address, payload);
 276
 277         address = FPGA_5GNR_FEC_VFQ_FLUSH_STATUS_HI;
 278         payload = (uint32_t)(phys_addr >> 32);
 279         fpga_reg_write_32(fpga_dev->mmio_base, address, payload);
 280
 281         return 0;
 282 }
 283
 284 static int
 285 fpga_dev_close(struct rte_bbdev *dev)
 286 {
 287         struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private;
 288
 289         rte_free(fpga_dev->sw_rings);
 290         rte_free(fpga_dev->flush_queue_status);
 291
 292         return 0;
 293 }
 294
 295 static void
 296 fpga_dev_info_get(struct rte_bbdev *dev,
 297                 struct rte_bbdev_driver_info *dev_info)
 298 {
 299         struct fpga_5gnr_fec_device *d = dev->data->dev_private;
 300         uint32_t q_id = 0;
 301
 302         static const struct rte_bbdev_op_cap bbdev_capabilities[] = {
 303                 {
 304                         .type   = RTE_BBDEV_OP_LDPC_ENC,
 305                         .cap.ldpc_enc = {
 306                                 .capability_flags =
 307                                                 RTE_BBDEV_LDPC_RATE_MATCH |
 308                                                 RTE_BBDEV_LDPC_ENC_INTERRUPTS |
 309                                                 RTE_BBDEV_LDPC_CRC_24B_ATTACH,
 310                                 .num_buffers_src =
 311                                                 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
 312                                 .num_buffers_dst =
 313                                                 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
 314                         }
 315                 },
 316                 {
 317                 .type   = RTE_BBDEV_OP_LDPC_DEC,
 318                 .cap.ldpc_dec = {
 319                         .capability_flags =
 320                                 RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK |
 321                                 RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP |
 322                                 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE |
 323                                 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE |
 324                                 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE |
 325                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE |
 326                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE |
 327                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS,
 328                         .llr_size = 6,
 329                         .llr_decimals = 2,
 330                         .num_buffers_src =
 331                                         RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
 332                         .num_buffers_hard_out =
 333                                         RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
 334                         .num_buffers_soft_out = 0,
 335                 }
 336                 },
 337                 RTE_BBDEV_END_OF_CAPABILITIES_LIST()
 338         };
 339
 340         /* Check the HARQ DDR size available */
 341         uint8_t timeout_counter = 0;
 342         uint32_t harq_buf_ready = fpga_reg_read_32(d->mmio_base,
 343                         FPGA_5GNR_FEC_HARQ_BUF_SIZE_RDY_REGS);
 344         while (harq_buf_ready != 1) {
 345                 usleep(FPGA_TIMEOUT_CHECK_INTERVAL);
 346                 timeout_counter++;
 347                 harq_buf_ready = fpga_reg_read_32(d->mmio_base,
 348                                 FPGA_5GNR_FEC_HARQ_BUF_SIZE_RDY_REGS);
 349                 if (timeout_counter > FPGA_HARQ_RDY_TIMEOUT) {
 350                         rte_bbdev_log(ERR, "HARQ Buffer not ready %d",
 351                                         harq_buf_ready);
 352                         harq_buf_ready = 1;
 353                 }
 354         }
 355         uint32_t harq_buf_size = fpga_reg_read_32(d->mmio_base,
 356                         FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS);
 357
 358         static struct rte_bbdev_queue_conf default_queue_conf;
 359         default_queue_conf.socket = dev->data->socket_id;
 360         default_queue_conf.queue_size = FPGA_RING_MAX_SIZE;
 361
 362         dev_info->driver_name = dev->device->driver->name;
 363         dev_info->queue_size_lim = FPGA_RING_MAX_SIZE;
 364         dev_info->hardware_accelerated = true;
 365         dev_info->min_alignment = 64;
 366         dev_info->harq_buffer_size = (harq_buf_size >> 10) + 1;
 367         dev_info->default_queue_conf = default_queue_conf;
 368         dev_info->capabilities = bbdev_capabilities;
 369         dev_info->cpu_flag_reqs = NULL;
 370
 371         /* Calculates number of queues assigned to device */
 372         dev_info->max_num_queues = 0;
 373         for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) {
 374                 uint32_t hw_q_id = fpga_reg_read_32(d->mmio_base,
 375                                 FPGA_5GNR_FEC_QUEUE_MAP + (q_id << 2));
 376                 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID)
 377                         dev_info->max_num_queues++;
 378         }
 379 }
 380
 381 /**
 382  * Find index of queue bound to current PF/VF which is unassigned. Return -1
 383  * when there is no available queue
 384  */
 385 static inline int
 386 fpga_find_free_queue_idx(struct rte_bbdev *dev,
 387                 const struct rte_bbdev_queue_conf *conf)
 388 {
 389         struct fpga_5gnr_fec_device *d = dev->data->dev_private;
 390         uint64_t q_idx;
 391         uint8_t i = 0;
 392         uint8_t range = FPGA_TOTAL_NUM_QUEUES >> 1;
 393
 394         if (conf->op_type == RTE_BBDEV_OP_LDPC_ENC) {
 395                 i = FPGA_NUM_DL_QUEUES;
 396                 range = FPGA_TOTAL_NUM_QUEUES;
 397         }
 398
 399         for (; i < range; ++i) {
 400                 q_idx = 1ULL << i;
 401                 /* Check if index of queue is bound to current PF/VF */
 402                 if (d->q_bound_bit_map & q_idx)
 403                         /* Check if found queue was not already assigned */
 404                         if (!(d->q_assigned_bit_map & q_idx)) {
 405                                 d->q_assigned_bit_map |= q_idx;
 406                                 return i;
 407                         }
 408         }
 409
 410         rte_bbdev_log(INFO, "Failed to find free queue on %s", dev->data->name);
 411
 412         return -1;
 413 }
 414
 415 static int
 416 fpga_queue_setup(struct rte_bbdev *dev, uint16_t queue_id,
 417                 const struct rte_bbdev_queue_conf *conf)
 418 {
 419         uint32_t address, ring_offset;
 420         struct fpga_5gnr_fec_device *d = dev->data->dev_private;
 421         struct fpga_queue *q;
 422         int8_t q_idx;
 423
 424         /* Check if there is a free queue to assign */
 425         q_idx = fpga_find_free_queue_idx(dev, conf);
 426         if (q_idx == -1)
 427                 return -1;
 428
 429         /* Allocate the queue data structure. */
 430         q = rte_zmalloc_socket(dev->device->driver->name, sizeof(*q),
 431                         RTE_CACHE_LINE_SIZE, conf->socket);
 432         if (q == NULL) {
 433                 /* Mark queue as un-assigned */
 434                 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx));
 435                 rte_bbdev_log(ERR, "Failed to allocate queue memory");
 436                 return -ENOMEM;
 437         }
 438
 439         q->d = d;
 440         q->q_idx = q_idx;
 441
 442         /* Set ring_base_addr */
 443         q->ring_addr = RTE_PTR_ADD(d->sw_rings, (d->sw_ring_size * queue_id));
 444         q->ring_ctrl_reg.ring_base_addr = d->sw_rings_phys +
 445                         (d->sw_ring_size * queue_id);
 446
 447         /* Allocate memory for Completion Head variable*/
 448         q->ring_head_addr = rte_zmalloc_socket(dev->device->driver->name,
 449                         sizeof(uint64_t), RTE_CACHE_LINE_SIZE, conf->socket);
 450         if (q->ring_head_addr == NULL) {
 451                 /* Mark queue as un-assigned */
 452                 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx));
 453                 rte_free(q);
 454                 rte_bbdev_log(ERR,
 455                                 "Failed to allocate memory for %s:%u completion_head",
 456                                 dev->device->driver->name, dev->data->dev_id);
 457                 return -ENOMEM;
 458         }
 459         /* Set ring_head_addr */
 460         q->ring_ctrl_reg.ring_head_addr =
 461                         rte_malloc_virt2iova(q->ring_head_addr);
 462
 463         /* Clear shadow_completion_head */
 464         q->shadow_completion_head = 0;
 465
 466         /* Set ring_size */
 467         if (conf->queue_size > FPGA_RING_MAX_SIZE) {
 468                 /* Mark queue as un-assigned */
 469                 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx));
 470                 rte_free(q->ring_head_addr);
 471                 rte_free(q);
 472                 rte_bbdev_log(ERR,
 473                                 "Size of queue is too big %d (MAX: %d ) for %s:%u",
 474                                 conf->queue_size, FPGA_RING_MAX_SIZE,
 475                                 dev->device->driver->name, dev->data->dev_id);
 476                 return -EINVAL;
 477         }
 478         q->ring_ctrl_reg.ring_size = conf->queue_size;
 479
 480         /* Set Miscellaneous FPGA register*/
 481         /* Max iteration number for TTI mitigation - todo */
 482         q->ring_ctrl_reg.max_ul_dec = 0;
 483         /* Enable max iteration number for TTI - todo */
 484         q->ring_ctrl_reg.max_ul_dec_en = 0;
 485
 486         /* Enable the ring */
 487         q->ring_ctrl_reg.enable = 1;
 488
 489         /* Set FPGA head_point and tail registers */
 490         q->ring_ctrl_reg.head_point = q->tail = 0;
 491
 492         /* Set FPGA shadow_tail register */
 493         q->ring_ctrl_reg.shadow_tail = q->tail;
 494
 495         /* Calculates the ring offset for found queue */
 496         ring_offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
 497                         (sizeof(struct fpga_ring_ctrl_reg) * q_idx);
 498
 499         /* Set FPGA Ring Control Registers */
 500         fpga_ring_reg_write(d->mmio_base, ring_offset, q->ring_ctrl_reg);
 501
 502         /* Store MMIO register of shadow_tail */
 503         address = ring_offset + FPGA_5GNR_FEC_RING_SHADOW_TAIL;
 504         q->shadow_tail_addr = RTE_PTR_ADD(d->mmio_base, address);
 505
 506         q->head_free_desc = q->tail;
 507
 508         /* Set wrap mask */
 509         q->sw_ring_wrap_mask = conf->queue_size - 1;
 510
 511         rte_bbdev_log_debug("Setup dev%u q%u: queue_idx=%u",
 512                         dev->data->dev_id, queue_id, q->q_idx);
 513
 514         dev->data->queues[queue_id].queue_private = q;
 515
 516         rte_bbdev_log_debug("BBDEV queue[%d] set up for FPGA queue[%d]",
 517                         queue_id, q_idx);
 518
 519 #ifdef RTE_LIBRTE_BBDEV_DEBUG
 520         /* Read FPGA Ring Control Registers after configuration*/
 521         print_ring_reg_debug_info(d->mmio_base, ring_offset);
 522 #endif
 523         return 0;
 524 }
 525
 526 static int
 527 fpga_queue_release(struct rte_bbdev *dev, uint16_t queue_id)
 528 {
 529         struct fpga_5gnr_fec_device *d = dev->data->dev_private;
 530         struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
 531         struct fpga_ring_ctrl_reg ring_reg;
 532         uint32_t offset;
 533
 534         rte_bbdev_log_debug("FPGA Queue[%d] released", queue_id);
 535
 536         if (q != NULL) {
 537                 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg));
 538                 offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
 539                         (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx);
 540                 /* Disable queue */
 541                 fpga_reg_write_8(d->mmio_base,
 542                                 offset + FPGA_5GNR_FEC_RING_ENABLE, 0x00);
 543                 /* Clear queue registers */
 544                 fpga_ring_reg_write(d->mmio_base, offset, ring_reg);
 545
 546                 /* Mark the Queue as un-assigned */
 547                 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q->q_idx));
 548                 rte_free(q->ring_head_addr);
 549                 rte_free(q);
 550                 dev->data->queues[queue_id].queue_private = NULL;
 551         }
 552
 553         return 0;
 554 }
 555
 556 /* Function starts a device queue. */
 557 static int
 558 fpga_queue_start(struct rte_bbdev *dev, uint16_t queue_id)
 559 {
 560         struct fpga_5gnr_fec_device *d = dev->data->dev_private;
 561 #ifdef RTE_LIBRTE_BBDEV_DEBUG
 562         if (d == NULL) {
 563                 rte_bbdev_log(ERR, "Invalid device pointer");
 564                 return -1;
 565         }
 566 #endif
 567         struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
 568         uint32_t offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
 569                         (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx);
 570         uint8_t enable = 0x01;
 571         uint16_t zero = 0x0000;
 572
 573         /* Clear queue head and tail variables */
 574         q->tail = q->head_free_desc = 0;
 575
 576         /* Clear FPGA head_point and tail registers */
 577         fpga_reg_write_16(d->mmio_base, offset + FPGA_5GNR_FEC_RING_HEAD_POINT,
 578                         zero);
 579         fpga_reg_write_16(d->mmio_base, offset + FPGA_5GNR_FEC_RING_SHADOW_TAIL,
 580                         zero);
 581
 582         /* Enable queue */
 583         fpga_reg_write_8(d->mmio_base, offset + FPGA_5GNR_FEC_RING_ENABLE,
 584                         enable);
 585
 586         rte_bbdev_log_debug("FPGA Queue[%d] started", queue_id);
 587         return 0;
 588 }
 589
 590 /* Function stops a device queue. */
 591 static int
 592 fpga_queue_stop(struct rte_bbdev *dev, uint16_t queue_id)
 593 {
 594         struct fpga_5gnr_fec_device *d = dev->data->dev_private;
 595 #ifdef RTE_LIBRTE_BBDEV_DEBUG
 596         if (d == NULL) {
 597                 rte_bbdev_log(ERR, "Invalid device pointer");
 598                 return -1;
 599         }
 600 #endif
 601         struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
 602         uint32_t offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
 603                         (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx);
 604         uint8_t payload = 0x01;
 605         uint8_t counter = 0;
 606         uint8_t timeout = FPGA_QUEUE_FLUSH_TIMEOUT_US /
 607                         FPGA_TIMEOUT_CHECK_INTERVAL;
 608
 609         /* Set flush_queue_en bit to trigger queue flushing */
 610         fpga_reg_write_8(d->mmio_base,
 611                         offset + FPGA_5GNR_FEC_RING_FLUSH_QUEUE_EN, payload);
 612
 613         /** Check if queue flush is completed.
 614          * FPGA will update the completion flag after queue flushing is
 615          * completed. If completion flag is not updated within 1ms it is
 616          * considered as a failure.
 617          */
 618         while (!(*((volatile uint8_t *)d->flush_queue_status + q->q_idx)
 619                         & payload)) {
 620                 if (counter > timeout) {
 621                         rte_bbdev_log(ERR, "FPGA Queue Flush failed for queue %d",
 622                                         queue_id);
 623                         return -1;
 624                 }
 625                 usleep(FPGA_TIMEOUT_CHECK_INTERVAL);
 626                 counter++;
 627         }
 628
 629         /* Disable queue */
 630         payload = 0x00;
 631         fpga_reg_write_8(d->mmio_base, offset + FPGA_5GNR_FEC_RING_ENABLE,
 632                         payload);
 633
 634         rte_bbdev_log_debug("FPGA Queue[%d] stopped", queue_id);
 635         return 0;
 636 }
 637
 638 static const struct rte_bbdev_ops fpga_ops = {
 639         .setup_queues = fpga_setup_queues,
 640         .close = fpga_dev_close,
 641         .info_get = fpga_dev_info_get,
 642         .queue_setup = fpga_queue_setup,
 643         .queue_stop = fpga_queue_stop,
 644         .queue_start = fpga_queue_start,
 645         .queue_release = fpga_queue_release,
 646 };
 647
 648 static inline void
 649 fpga_dma_enqueue(struct fpga_queue *q, uint16_t num_desc,
 650                 struct rte_bbdev_stats *queue_stats)
 651 {
 652 #ifdef RTE_BBDEV_OFFLOAD_COST
 653         uint64_t start_time = 0;
 654         queue_stats->acc_offload_cycles = 0;
 655 #else
 656         RTE_SET_USED(queue_stats);
 657 #endif
 658
 659         /* Update tail and shadow_tail register */
 660         q->tail = (q->tail + num_desc) & q->sw_ring_wrap_mask;
 661
 662         rte_wmb();
 663
 664 #ifdef RTE_BBDEV_OFFLOAD_COST
 665         /* Start time measurement for enqueue function offload. */
 666         start_time = rte_rdtsc_precise();
 667 #endif
 668         mmio_write_16(q->shadow_tail_addr, q->tail);
 669
 670 #ifdef RTE_BBDEV_OFFLOAD_COST
 671         rte_wmb();
 672         queue_stats->acc_offload_cycles += rte_rdtsc_precise() - start_time;
 673 #endif
 674 }
 675
 676 /* Read flag value 0/1/ from bitmap */
 677 static inline bool
 678 check_bit(uint32_t bitmap, uint32_t bitmask)
 679 {
 680         return bitmap & bitmask;
 681 }
 682
 683 /* Print an error if a descriptor error has occurred.
 684  *  Return 0 on success, 1 on failure
 685  */
 686 static inline int
 687 check_desc_error(uint32_t error_code) {
 688         switch (error_code) {
 689         case DESC_ERR_NO_ERR:
 690                 return 0;
 691         case DESC_ERR_K_P_OUT_OF_RANGE:
 692                 rte_bbdev_log(ERR, "Encode block size K' is out of range");
 693                 break;
 694         case DESC_ERR_Z_C_NOT_LEGAL:
 695                 rte_bbdev_log(ERR, "Zc is illegal");
 696                 break;
 697         case DESC_ERR_DESC_OFFSET_ERR:
 698                 rte_bbdev_log(ERR,
 699                                 "Queue offset does not meet the expectation in the FPGA"
 700                                 );
 701                 break;
 702         case DESC_ERR_DESC_READ_FAIL:
 703                 rte_bbdev_log(ERR, "Unsuccessful completion for descriptor read");
 704                 break;
 705         case DESC_ERR_DESC_READ_TIMEOUT:
 706                 rte_bbdev_log(ERR, "Descriptor read time-out");
 707                 break;
 708         case DESC_ERR_DESC_READ_TLP_POISONED:
 709                 rte_bbdev_log(ERR, "Descriptor read TLP poisoned");
 710                 break;
 711         case DESC_ERR_CB_READ_FAIL:
 712                 rte_bbdev_log(ERR, "Unsuccessful completion for code block");
 713                 break;
 714         case DESC_ERR_CB_READ_TIMEOUT:
 715                 rte_bbdev_log(ERR, "Code block read time-out");
 716                 break;
 717         case DESC_ERR_CB_READ_TLP_POISONED:
 718                 rte_bbdev_log(ERR, "Code block read TLP poisoned");
 719                 break;
 720         case DESC_ERR_HBSTORE_ERR:
 721                 rte_bbdev_log(ERR, "Hbstroe exceeds HARQ buffer size.");
 722                 break;
 723         default:
 724                 rte_bbdev_log(ERR, "Descriptor error unknown error code %u",
 725                                 error_code);
 726                 break;
 727         }
 728         return 1;
 729 }
 730
 731 /* Compute value of k0.
 732  * Based on 3GPP 38.212 Table 5.4.2.1-2
 733  * Starting position of different redundancy versions, k0
 734  */
 735 static inline uint16_t
 736 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index)
 737 {
 738         if (rv_index == 0)
 739                 return 0;
 740         uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c;
 741         if (n_cb == n) {
 742                 if (rv_index == 1)
 743                         return (bg == 1 ? K0_1_1 : K0_1_2) * z_c;
 744                 else if (rv_index == 2)
 745                         return (bg == 1 ? K0_2_1 : K0_2_2) * z_c;
 746                 else
 747                         return (bg == 1 ? K0_3_1 : K0_3_2) * z_c;
 748         }
 749         /* LBRM case - includes a division by N */
 750         if (rv_index == 1)
 751                 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb)
 752                                 / n) * z_c;
 753         else if (rv_index == 2)
 754                 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb)
 755                                 / n) * z_c;
 756         else
 757                 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb)
 758                                 / n) * z_c;
 759 }
 760
 761 /**
 762  * Set DMA descriptor for encode operation (1 Code Block)
 763  *
 764  * @param op
 765  *   Pointer to a single encode operation.
 766  * @param desc
 767  *   Pointer to DMA descriptor.
 768  * @param input
 769  *   Pointer to pointer to input data which will be decoded.
 770  * @param e
 771  *   E value (length of output in bits).
 772  * @param ncb
 773  *   Ncb value (size of the soft buffer).
 774  * @param out_length
 775  *   Length of output buffer
 776  * @param in_offset
 777  *   Input offset in rte_mbuf structure. It is used for calculating the point
 778  *   where data is starting.
 779  * @param out_offset
 780  *   Output offset in rte_mbuf structure. It is used for calculating the point
 781  *   where hard output data will be stored.
 782  * @param cbs_in_op
 783  *   Number of CBs contained in one operation.
 784  */
 785 static inline int
 786 fpga_dma_desc_te_fill(struct rte_bbdev_enc_op *op,
 787                 struct fpga_dma_enc_desc *desc, struct rte_mbuf *input,
 788                 struct rte_mbuf *output, uint16_t k_,  uint16_t e,
 789                 uint32_t in_offset, uint32_t out_offset, uint16_t desc_offset,
 790                 uint8_t cbs_in_op)
 791 {
 792         /* reset */
 793         desc->done = 0;
 794         desc->error = 0;
 795         desc->k_ = k_;
 796         desc->rm_e = e;
 797         desc->desc_idx = desc_offset;
 798         desc->zc = op->ldpc_enc.z_c;
 799         desc->bg_idx = op->ldpc_enc.basegraph - 1;
 800         desc->qm_idx = op->ldpc_enc.q_m / 2;
 801         desc->crc_en = check_bit(op->ldpc_enc.op_flags,
 802                         RTE_BBDEV_LDPC_CRC_24B_ATTACH);
 803         desc->irq_en = 0;
 804         desc->k0 = get_k0(op->ldpc_enc.n_cb, op->ldpc_enc.z_c,
 805                         op->ldpc_enc.basegraph, op->ldpc_enc.rv_index);
 806         desc->ncb = op->ldpc_enc.n_cb;
 807         desc->num_null = op->ldpc_enc.n_filler;
 808         /* Set inbound data buffer address */
 809         desc->in_addr_hi = (uint32_t)(
 810                         rte_pktmbuf_mtophys_offset(input, in_offset) >> 32);
 811         desc->in_addr_lw = (uint32_t)(
 812                         rte_pktmbuf_mtophys_offset(input, in_offset));
 813
 814         desc->out_addr_hi = (uint32_t)(
 815                         rte_pktmbuf_mtophys_offset(output, out_offset) >> 32);
 816         desc->out_addr_lw = (uint32_t)(
 817                         rte_pktmbuf_mtophys_offset(output, out_offset));
 818         /* Save software context needed for dequeue */
 819         desc->op_addr = op;
 820         /* Set total number of CBs in an op */
 821         desc->cbs_in_op = cbs_in_op;
 822         return 0;
 823 }
 824
 825 /**
 826  * Set DMA descriptor for decode operation (1 Code Block)
 827  *
 828  * @param op
 829  *   Pointer to a single encode operation.
 830  * @param desc
 831  *   Pointer to DMA descriptor.
 832  * @param input
 833  *   Pointer to pointer to input data which will be decoded.
 834  * @param in_offset
 835  *   Input offset in rte_mbuf structure. It is used for calculating the point
 836  *   where data is starting.
 837  * @param out_offset
 838  *   Output offset in rte_mbuf structure. It is used for calculating the point
 839  *   where hard output data will be stored.
 840  * @param cbs_in_op
 841  *   Number of CBs contained in one operation.
 842  */
 843 static inline int
 844 fpga_dma_desc_ld_fill(struct rte_bbdev_dec_op *op,
 845                 struct fpga_dma_dec_desc *desc,
 846                 struct rte_mbuf *input, struct rte_mbuf *output,
 847                 uint16_t harq_in_length,
 848                 uint32_t in_offset, uint32_t out_offset,
 849                 uint32_t harq_offset,
 850                 uint16_t desc_offset,
 851                 uint8_t cbs_in_op)
 852 {
 853         /* reset */
 854         desc->done = 0;
 855         desc->error = 0;
 856         /* Set inbound data buffer address */
 857         desc->in_addr_hi = (uint32_t)(
 858                         rte_pktmbuf_mtophys_offset(input, in_offset) >> 32);
 859         desc->in_addr_lw = (uint32_t)(
 860                         rte_pktmbuf_mtophys_offset(input, in_offset));
 861         desc->rm_e = op->ldpc_dec.cb_params.e;
 862         desc->harq_input_length = harq_in_length;
 863         desc->et_dis = !check_bit(op->ldpc_dec.op_flags,
 864                         RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE);
 865         desc->rv = op->ldpc_dec.rv_index;
 866         desc->crc24b_ind = check_bit(op->ldpc_dec.op_flags,
 867                         RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK);
 868         desc->drop_crc24b = check_bit(op->ldpc_dec.op_flags,
 869                         RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP);
 870         desc->desc_idx = desc_offset;
 871         desc->ncb = op->ldpc_dec.n_cb;
 872         desc->num_null = op->ldpc_dec.n_filler;
 873         desc->hbstroe_offset = harq_offset >> 10;
 874         desc->zc = op->ldpc_dec.z_c;
 875         desc->harqin_en = check_bit(op->ldpc_dec.op_flags,
 876                         RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE);
 877         desc->bg_idx = op->ldpc_dec.basegraph - 1;
 878         desc->max_iter = op->ldpc_dec.iter_max;
 879         desc->qm_idx = op->ldpc_dec.q_m / 2;
 880         desc->out_addr_hi = (uint32_t)(
 881                         rte_pktmbuf_mtophys_offset(output, out_offset) >> 32);
 882         desc->out_addr_lw = (uint32_t)(
 883                         rte_pktmbuf_mtophys_offset(output, out_offset));
 884         /* Save software context needed for dequeue */
 885         desc->op_addr = op;
 886         /* Set total number of CBs in an op */
 887         desc->cbs_in_op = cbs_in_op;
 888
 889         return 0;
 890 }
 891
 892 #ifdef RTE_LIBRTE_BBDEV_DEBUG
 893 /* Validates LDPC encoder parameters */
 894 static int
 895 validate_enc_op(struct rte_bbdev_enc_op *op __rte_unused)
 896 {
 897         struct rte_bbdev_op_ldpc_enc *ldpc_enc = &op->ldpc_enc;
 898         struct rte_bbdev_op_enc_ldpc_cb_params *cb = NULL;
 899         struct rte_bbdev_op_enc_ldpc_tb_params *tb = NULL;
 900
 901
 902         if (ldpc_enc->input.length >
 903                         RTE_BBDEV_LDPC_MAX_CB_SIZE >> 3) {
 904                 rte_bbdev_log(ERR, "CB size (%u) is too big, max: %d",
 905                                 ldpc_enc->input.length,
 906                                 RTE_BBDEV_LDPC_MAX_CB_SIZE);
 907                 return -1;
 908         }
 909
 910         if (op->mempool == NULL) {
 911                 rte_bbdev_log(ERR, "Invalid mempool pointer");
 912                 return -1;
 913         }
 914         if (ldpc_enc->input.data == NULL) {
 915                 rte_bbdev_log(ERR, "Invalid input pointer");
 916                 return -1;
 917         }
 918         if (ldpc_enc->output.data == NULL) {
 919                 rte_bbdev_log(ERR, "Invalid output pointer");
 920                 return -1;
 921         }
 922         if ((ldpc_enc->basegraph > 2) || (ldpc_enc->basegraph == 0)) {
 923                 rte_bbdev_log(ERR,
 924                                 "basegraph (%u) is out of range 1 <= value <= 2",
 925                                 ldpc_enc->basegraph);
 926                 return -1;
 927         }
 928         if (ldpc_enc->code_block_mode > 1) {
 929                 rte_bbdev_log(ERR,
 930                                 "code_block_mode (%u) is out of range 0:Tb 1:CB",
 931                                 ldpc_enc->code_block_mode);
 932                 return -1;
 933         }
 934
 935         if (ldpc_enc->code_block_mode == 0) {
 936                 tb = &ldpc_enc->tb_params;
 937                 if (tb->c == 0) {
 938                         rte_bbdev_log(ERR,
 939                                         "c (%u) is out of range 1 <= value <= %u",
 940                                         tb->c, RTE_BBDEV_LDPC_MAX_CODE_BLOCKS);
 941                         return -1;
 942                 }
 943                 if (tb->cab > tb->c) {
 944                         rte_bbdev_log(ERR,
 945                                         "cab (%u) is greater than c (%u)",
 946                                         tb->cab, tb->c);
 947                         return -1;
 948                 }
 949                 if ((tb->ea < RTE_BBDEV_LDPC_MIN_CB_SIZE)
 950                                 && tb->r < tb->cab) {
 951                         rte_bbdev_log(ERR,
 952                                         "ea (%u) is less than %u or it is not even",
 953                                         tb->ea, RTE_BBDEV_LDPC_MIN_CB_SIZE);
 954                         return -1;
 955                 }
 956                 if ((tb->eb < RTE_BBDEV_LDPC_MIN_CB_SIZE)
 957                                 && tb->c > tb->cab) {
 958                         rte_bbdev_log(ERR,
 959                                         "eb (%u) is less than %u",
 960                                         tb->eb, RTE_BBDEV_LDPC_MIN_CB_SIZE);
 961                         return -1;
 962                 }
 963                 if (tb->r > (tb->c - 1)) {
 964                         rte_bbdev_log(ERR,
 965                                         "r (%u) is greater than c - 1 (%u)",
 966                                         tb->r, tb->c - 1);
 967                         return -1;
 968                 }
 969         } else {
 970                 cb = &ldpc_enc->cb_params;
 971                 if (cb->e < RTE_BBDEV_LDPC_MIN_CB_SIZE) {
 972                         rte_bbdev_log(ERR,
 973                                         "e (%u) is less than %u or it is not even",
 974                                         cb->e, RTE_BBDEV_LDPC_MIN_CB_SIZE);
 975                         return -1;
 976                 }
 977         }
 978         return 0;
 979 }
 980 #endif
 981
 982 static inline char *
 983 mbuf_append(struct rte_mbuf *m_head, struct rte_mbuf *m, uint16_t len)
 984 {
 985         if (unlikely(len > rte_pktmbuf_tailroom(m)))
 986                 return NULL;
 987
 988         char *tail = (char *)m->buf_addr + m->data_off + m->data_len;
 989         m->data_len = (uint16_t)(m->data_len + len);
 990         m_head->pkt_len  = (m_head->pkt_len + len);
 991         return tail;
 992 }
 993
 994 #ifdef RTE_LIBRTE_BBDEV_DEBUG
 995 /* Validates LDPC decoder parameters */
 996 static int
 997 validate_dec_op(struct rte_bbdev_dec_op *op __rte_unused)
 998 {
 999         struct rte_bbdev_op_ldpc_dec *ldpc_dec = &op->ldpc_dec;
1000         struct rte_bbdev_op_dec_ldpc_cb_params *cb = NULL;
1001         struct rte_bbdev_op_dec_ldpc_tb_params *tb = NULL;
1002
1003         if (op->mempool == NULL) {
1004                 rte_bbdev_log(ERR, "Invalid mempool pointer");
1005                 return -1;
1006         }
1007         if (ldpc_dec->rv_index > 3) {
1008                 rte_bbdev_log(ERR,
1009                                 "rv_index (%u) is out of range 0 <= value <= 3",
1010                                 ldpc_dec->rv_index);
1011                 return -1;
1012         }
1013
1014         if (ldpc_dec->iter_max == 0) {
1015                 rte_bbdev_log(ERR,
1016                                 "iter_max (%u) is equal to 0",
1017                                 ldpc_dec->iter_max);
1018                 return -1;
1019         }
1020
1021         if (ldpc_dec->code_block_mode > 1) {
1022                 rte_bbdev_log(ERR,
1023                                 "code_block_mode (%u) is out of range 0 <= value <= 1",
1024                                 ldpc_dec->code_block_mode);
1025                 return -1;
1026         }
1027
1028         if (ldpc_dec->code_block_mode == 0) {
1029                 tb = &ldpc_dec->tb_params;
1030                 if (tb->c < 1) {
1031                         rte_bbdev_log(ERR,
1032                                         "c (%u) is out of range 1 <= value <= %u",
1033                                         tb->c, RTE_BBDEV_LDPC_MAX_CODE_BLOCKS);
1034                         return -1;
1035                 }
1036                 if (tb->cab > tb->c) {
1037                         rte_bbdev_log(ERR,
1038                                         "cab (%u) is greater than c (%u)",
1039                                         tb->cab, tb->c);
1040                         return -1;
1041                 }
1042         } else {
1043                 cb = &ldpc_dec->cb_params;
1044                 if (cb->e < RTE_BBDEV_LDPC_MIN_CB_SIZE) {
1045                         rte_bbdev_log(ERR,
1046                                         "e (%u) is out of range %u <= value <= %u",
1047                                         cb->e, RTE_BBDEV_LDPC_MIN_CB_SIZE,
1048                                         RTE_BBDEV_LDPC_MAX_CB_SIZE);
1049                         return -1;
1050                 }
1051         }
1052
1053         return 0;
1054 }
1055 #endif
1056
1057 static inline int
1058 enqueue_ldpc_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op *op,
1059                 uint16_t desc_offset)
1060 {
1061         union fpga_dma_desc *desc;
1062         int ret;
1063         uint8_t c, crc24_bits = 0;
1064         struct rte_bbdev_op_ldpc_enc *enc = &op->ldpc_enc;
1065         uint16_t in_offset = enc->input.offset;
1066         uint16_t out_offset = enc->output.offset;
1067         struct rte_mbuf *m_in = enc->input.data;
1068         struct rte_mbuf *m_out = enc->output.data;
1069         struct rte_mbuf *m_out_head = enc->output.data;
1070         uint32_t in_length, out_length, e;
1071         uint16_t total_left = enc->input.length;
1072         uint16_t ring_offset;
1073         uint16_t K, k_;
1074
1075 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1076         /* Validate op structure */
1077         /* FIXME */
1078         if (validate_enc_op(op) == -1) {
1079                 rte_bbdev_log(ERR, "LDPC encoder validation failed");
1080                 return -EINVAL;
1081         }
1082 #endif
1083
1084         /* Clear op status */
1085         op->status = 0;
1086
1087         if (m_in == NULL || m_out == NULL) {
1088                 rte_bbdev_log(ERR, "Invalid mbuf pointer");
1089                 op->status = 1 << RTE_BBDEV_DATA_ERROR;
1090                 return -EINVAL;
1091         }
1092
1093         if (enc->op_flags & RTE_BBDEV_LDPC_CRC_24B_ATTACH)
1094                 crc24_bits = 24;
1095
1096         if (enc->code_block_mode == 0) {
1097                 /* For Transport Block mode */
1098                 /* FIXME */
1099                 c = enc->tb_params.c;
1100                 e = enc->tb_params.ea;
1101         } else { /* For Code Block mode */
1102                 c = 1;
1103                 e = enc->cb_params.e;
1104         }
1105
1106         /* Update total_left */
1107         K = (enc->basegraph == 1 ? 22 : 10) * enc->z_c;
1108         k_ = K - enc->n_filler;
1109         in_length = (k_ - crc24_bits) >> 3;
1110         out_length = (e + 7) >> 3;
1111
1112         total_left = rte_pktmbuf_data_len(m_in) - in_offset;
1113
1114         /* Update offsets */
1115         if (total_left != in_length) {
1116                 op->status |= 1 << RTE_BBDEV_DATA_ERROR;
1117                 rte_bbdev_log(ERR,
1118                                 "Mismatch between mbuf length and included CBs sizes %d",
1119                                 total_left);
1120         }
1121
1122         mbuf_append(m_out_head, m_out, out_length);
1123
1124         /* Offset into the ring */
1125         ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask);
1126         /* Setup DMA Descriptor */
1127         desc = q->ring_addr + ring_offset;
1128
1129         ret = fpga_dma_desc_te_fill(op, &desc->enc_req, m_in, m_out,
1130                         k_, e, in_offset, out_offset, ring_offset, c);
1131         if (unlikely(ret < 0))
1132                 return ret;
1133
1134         /* Update lengths */
1135         total_left -= in_length;
1136         op->ldpc_enc.output.length += out_length;
1137
1138         if (total_left > 0) {
1139                 rte_bbdev_log(ERR,
1140                         "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u",
1141                                 total_left, in_length);
1142                 return -1;
1143         }
1144
1145 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1146         print_dma_enc_desc_debug_info(desc);
1147 #endif
1148         return 1;
1149 }
1150
1151 static inline int
1152 enqueue_ldpc_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op *op,
1153                 uint16_t desc_offset)
1154 {
1155         union fpga_dma_desc *desc;
1156         int ret;
1157         uint16_t ring_offset;
1158         uint8_t c;
1159         uint16_t e, in_length, out_length, k0, l, seg_total_left, sys_cols;
1160         uint16_t K, parity_offset, harq_in_length = 0, harq_out_length = 0;
1161         uint16_t crc24_overlap = 0;
1162         struct rte_bbdev_op_ldpc_dec *dec = &op->ldpc_dec;
1163         struct rte_mbuf *m_in = dec->input.data;
1164         struct rte_mbuf *m_out = dec->hard_output.data;
1165         struct rte_mbuf *m_out_head = dec->hard_output.data;
1166         uint16_t in_offset = dec->input.offset;
1167         uint16_t out_offset = dec->hard_output.offset;
1168         uint32_t harq_offset = 0;
1169
1170 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1171                 /* Validate op structure */
1172                 if (validate_dec_op(op) == -1) {
1173                         rte_bbdev_log(ERR, "LDPC decoder validation failed");
1174                         return -EINVAL;
1175                 }
1176 #endif
1177
1178         /* Clear op status */
1179         op->status = 0;
1180
1181         /* Setup DMA Descriptor */
1182         ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask);
1183         desc = q->ring_addr + ring_offset;
1184
1185         if (m_in == NULL || m_out == NULL) {
1186                 rte_bbdev_log(ERR, "Invalid mbuf pointer");
1187                 op->status = 1 << RTE_BBDEV_DATA_ERROR;
1188                 return -1;
1189         }
1190
1191         c = 1;
1192         e = dec->cb_params.e;
1193
1194         if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP))
1195                 crc24_overlap = 24;
1196
1197         sys_cols = (dec->basegraph == 1) ? 22 : 10;
1198         K = sys_cols * dec->z_c;
1199         parity_offset = K - 2 * dec->z_c;
1200
1201         out_length = ((K - crc24_overlap - dec->n_filler) >> 3);
1202         in_length = e;
1203         seg_total_left = dec->input.length;
1204
1205         if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) {
1206                 harq_in_length = RTE_MIN(dec->harq_combined_input.length,
1207                                 (uint32_t)dec->n_cb);
1208         }
1209
1210         if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE)) {
1211                 k0 = get_k0(dec->n_cb, dec->z_c,
1212                                 dec->basegraph, dec->rv_index);
1213                 if (k0 > parity_offset)
1214                         l = k0 + e;
1215                 else
1216                         l = k0 + e + dec->n_filler;
1217                 harq_out_length = RTE_MIN(RTE_MAX(harq_in_length, l),
1218                                 dec->n_cb - dec->n_filler);
1219                 dec->harq_combined_output.length = harq_out_length;
1220         }
1221
1222         mbuf_append(m_out_head, m_out, out_length);
1223         if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE))
1224                 harq_offset = dec->harq_combined_input.offset;
1225         else if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE))
1226                 harq_offset = dec->harq_combined_output.offset;
1227
1228         if ((harq_offset & 0x3FF) > 0) {
1229                 rte_bbdev_log(ERR, "Invalid HARQ offset %d", harq_offset);
1230                 op->status = 1 << RTE_BBDEV_DATA_ERROR;
1231                 return -1;
1232         }
1233
1234         ret = fpga_dma_desc_ld_fill(op, &desc->dec_req, m_in, m_out,
1235                 harq_in_length, in_offset, out_offset, harq_offset,
1236                 ring_offset, c);
1237         if (unlikely(ret < 0))
1238                 return ret;
1239         /* Update lengths */
1240         seg_total_left -= in_length;
1241         op->ldpc_dec.hard_output.length += out_length;
1242         if (seg_total_left > 0) {
1243                 rte_bbdev_log(ERR,
1244                                 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u",
1245                                 seg_total_left, in_length);
1246                 return -1;
1247         }
1248
1249 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1250         print_dma_dec_desc_debug_info(desc);
1251 #endif
1252
1253         return 1;
1254 }
1255
1256 static uint16_t
1257 fpga_enqueue_ldpc_enc(struct rte_bbdev_queue_data *q_data,
1258                 struct rte_bbdev_enc_op **ops, uint16_t num)
1259 {
1260         uint16_t i, total_enqueued_cbs = 0;
1261         int32_t avail;
1262         int enqueued_cbs;
1263         struct fpga_queue *q = q_data->queue_private;
1264         union fpga_dma_desc *desc;
1265
1266         /* Check if queue is not full */
1267         if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) ==
1268                         q->head_free_desc))
1269                 return 0;
1270
1271         /* Calculates available space */
1272         avail = (q->head_free_desc > q->tail) ?
1273                 q->head_free_desc - q->tail - 1 :
1274                 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1;
1275
1276         for (i = 0; i < num; ++i) {
1277
1278                 /* Check if there is available space for further
1279                  * processing
1280                  */
1281                 if (unlikely(avail - 1 < 0))
1282                         break;
1283                 avail -= 1;
1284                 enqueued_cbs = enqueue_ldpc_enc_one_op_cb(q, ops[i],
1285                                 total_enqueued_cbs);
1286
1287                 if (enqueued_cbs < 0)
1288                         break;
1289
1290                 total_enqueued_cbs += enqueued_cbs;
1291
1292                 rte_bbdev_log_debug("enqueuing enc ops [%d/%d] | head %d | tail %d",
1293                                 total_enqueued_cbs, num,
1294                                 q->head_free_desc, q->tail);
1295         }
1296
1297         /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt
1298          * only when all previous CBs were already processed.
1299          */
1300         desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1)
1301                         & q->sw_ring_wrap_mask);
1302         desc->enc_req.irq_en = q->irq_enable;
1303
1304         fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats);
1305
1306         /* Update stats */
1307         q_data->queue_stats.enqueued_count += i;
1308         q_data->queue_stats.enqueue_err_count += num - i;
1309
1310         return i;
1311 }
1312
1313 static uint16_t
1314 fpga_enqueue_ldpc_dec(struct rte_bbdev_queue_data *q_data,
1315                 struct rte_bbdev_dec_op **ops, uint16_t num)
1316 {
1317         uint16_t i, total_enqueued_cbs = 0;
1318         int32_t avail;
1319         int enqueued_cbs;
1320         struct fpga_queue *q = q_data->queue_private;
1321         union fpga_dma_desc *desc;
1322
1323         /* Check if queue is not full */
1324         if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) ==
1325                         q->head_free_desc))
1326                 return 0;
1327
1328         /* Calculates available space */
1329         avail = (q->head_free_desc > q->tail) ?
1330                 q->head_free_desc - q->tail - 1 :
1331                 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1;
1332
1333         for (i = 0; i < num; ++i) {
1334
1335                 /* Check if there is available space for further
1336                  * processing
1337                  */
1338                 if (unlikely(avail - 1 < 0))
1339                         break;
1340                 avail -= 1;
1341                 enqueued_cbs = enqueue_ldpc_dec_one_op_cb(q, ops[i],
1342                                 total_enqueued_cbs);
1343
1344                 if (enqueued_cbs < 0)
1345                         break;
1346
1347                 total_enqueued_cbs += enqueued_cbs;
1348
1349                 rte_bbdev_log_debug("enqueuing dec ops [%d/%d] | head %d | tail %d",
1350                                 total_enqueued_cbs, num,
1351                                 q->head_free_desc, q->tail);
1352         }
1353
1354         /* Update stats */
1355         q_data->queue_stats.enqueued_count += i;
1356         q_data->queue_stats.enqueue_err_count += num - i;
1357
1358         /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt
1359          * only when all previous CBs were already processed.
1360          */
1361         desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1)
1362                         & q->sw_ring_wrap_mask);
1363         desc->enc_req.irq_en = q->irq_enable;
1364         fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats);
1365         return i;
1366 }
1367
1368
1369 static inline int
1370 dequeue_ldpc_enc_one_op_cb(struct fpga_queue *q,
1371                 struct rte_bbdev_enc_op **op,
1372                 uint16_t desc_offset)
1373 {
1374         union fpga_dma_desc *desc;
1375         int desc_error;
1376         /* Set current desc */
1377         desc = q->ring_addr + ((q->head_free_desc + desc_offset)
1378                         & q->sw_ring_wrap_mask);
1379
1380         /*check if done */
1381         if (desc->enc_req.done == 0)
1382                 return -1;
1383
1384         /* make sure the response is read atomically */
1385         rte_smp_rmb();
1386
1387         rte_bbdev_log_debug("DMA response desc %p", desc);
1388
1389 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1390         print_dma_enc_desc_debug_info(desc);
1391 #endif
1392
1393         *op = desc->enc_req.op_addr;
1394         /* Check the descriptor error field, return 1 on error */
1395         desc_error = check_desc_error(desc->enc_req.error);
1396         (*op)->status = desc_error << RTE_BBDEV_DATA_ERROR;
1397
1398         return 1;
1399 }
1400
1401
1402 static inline int
1403 dequeue_ldpc_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op **op,
1404                 uint16_t desc_offset)
1405 {
1406         union fpga_dma_desc *desc;
1407         int desc_error;
1408         /* Set descriptor */
1409         desc = q->ring_addr + ((q->head_free_desc + desc_offset)
1410                         & q->sw_ring_wrap_mask);
1411
1412         /* Verify done bit is set */
1413         if (desc->dec_req.done == 0)
1414                 return -1;
1415
1416         /* make sure the response is read atomically */
1417         rte_smp_rmb();
1418
1419 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1420         print_dma_dec_desc_debug_info(desc);
1421 #endif
1422
1423         *op = desc->dec_req.op_addr;
1424
1425         if (check_bit((*op)->ldpc_dec.op_flags,
1426                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1427                 (*op)->status = 0;
1428                 return 1;
1429         }
1430
1431         /* FPGA reports iterations based on round-up minus 1 */
1432         (*op)->ldpc_dec.iter_count = desc->dec_req.iter + 1;
1433         /* CRC Check criteria */
1434         if (desc->dec_req.crc24b_ind && !(desc->dec_req.crcb_pass))
1435                 (*op)->status = 1 << RTE_BBDEV_CRC_ERROR;
1436         /* et_pass = 0 when decoder fails */
1437         (*op)->status |= !(desc->dec_req.et_pass) << RTE_BBDEV_SYNDROME_ERROR;
1438         /* Check the descriptor error field, return 1 on error */
1439         desc_error = check_desc_error(desc->dec_req.error);
1440         (*op)->status |= desc_error << RTE_BBDEV_DATA_ERROR;
1441         return 1;
1442 }
1443
1444 static uint16_t
1445 fpga_dequeue_ldpc_enc(struct rte_bbdev_queue_data *q_data,
1446                 struct rte_bbdev_enc_op **ops, uint16_t num)
1447 {
1448         struct fpga_queue *q = q_data->queue_private;
1449         uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask;
1450         uint16_t i;
1451         uint16_t dequeued_cbs = 0;
1452         int ret;
1453
1454         for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) {
1455                 ret = dequeue_ldpc_enc_one_op_cb(q, &ops[i], dequeued_cbs);
1456
1457                 if (ret < 0)
1458                         break;
1459
1460                 dequeued_cbs += ret;
1461
1462                 rte_bbdev_log_debug("dequeuing enc ops [%d/%d] | head %d | tail %d",
1463                                 dequeued_cbs, num, q->head_free_desc, q->tail);
1464         }
1465
1466         /* Update head */
1467         q->head_free_desc = (q->head_free_desc + dequeued_cbs) &
1468                         q->sw_ring_wrap_mask;
1469
1470         /* Update stats */
1471         q_data->queue_stats.dequeued_count += i;
1472
1473         return i;
1474 }
1475
1476 static uint16_t
1477 fpga_dequeue_ldpc_dec(struct rte_bbdev_queue_data *q_data,
1478                 struct rte_bbdev_dec_op **ops, uint16_t num)
1479 {
1480         struct fpga_queue *q = q_data->queue_private;
1481         uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask;
1482         uint16_t i;
1483         uint16_t dequeued_cbs = 0;
1484         int ret;
1485
1486         for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) {
1487                 ret = dequeue_ldpc_dec_one_op_cb(q, &ops[i], dequeued_cbs);
1488
1489                 if (ret < 0)
1490                         break;
1491
1492                 dequeued_cbs += ret;
1493
1494                 rte_bbdev_log_debug("dequeuing dec ops [%d/%d] | head %d | tail %d",
1495                                 dequeued_cbs, num, q->head_free_desc, q->tail);
1496         }
1497
1498         /* Update head */
1499         q->head_free_desc = (q->head_free_desc + dequeued_cbs) &
1500                         q->sw_ring_wrap_mask;
1501
1502         /* Update stats */
1503         q_data->queue_stats.dequeued_count += i;
1504
1505         return i;
1506 }
1507
1508
1509 /* Initialization Function */
1510 static void
1511 fpga_5gnr_fec_init(struct rte_bbdev *dev, struct rte_pci_driver *drv)
1512 {
1513         struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device);
1514
1515         dev->dev_ops = &fpga_ops;
1516         dev->enqueue_ldpc_enc_ops = fpga_enqueue_ldpc_enc;
1517         dev->enqueue_ldpc_dec_ops = fpga_enqueue_ldpc_dec;
1518         dev->dequeue_ldpc_enc_ops = fpga_dequeue_ldpc_enc;
1519         dev->dequeue_ldpc_dec_ops = fpga_dequeue_ldpc_dec;
1520
1521         ((struct fpga_5gnr_fec_device *) dev->data->dev_private)->pf_device =
1522                         !strcmp(drv->driver.name,
1523                                         RTE_STR(FPGA_5GNR_FEC_PF_DRIVER_NAME));
1524         ((struct fpga_5gnr_fec_device *) dev->data->dev_private)->mmio_base =
1525                         pci_dev->mem_resource[0].addr;
1526
1527         rte_bbdev_log_debug(
1528                         "Init device %s [%s] @ virtaddr %p phyaddr %#"PRIx64,
1529                         dev->device->driver->name, dev->data->name,
1530                         (void *)pci_dev->mem_resource[0].addr,
1531                         pci_dev->mem_resource[0].phys_addr);
1532 }
1533
1534 static int
1535 fpga_5gnr_fec_probe(struct rte_pci_driver *pci_drv,
1536         struct rte_pci_device *pci_dev)
1537 {
1538         struct rte_bbdev *bbdev = NULL;
1539         char dev_name[RTE_BBDEV_NAME_MAX_LEN];
1540
1541         if (pci_dev == NULL) {
1542                 rte_bbdev_log(ERR, "NULL PCI device");
1543                 return -EINVAL;
1544         }
1545
1546         rte_pci_device_name(&pci_dev->addr, dev_name, sizeof(dev_name));
1547
1548         /* Allocate memory to be used privately by drivers */
1549         bbdev = rte_bbdev_allocate(pci_dev->device.name);
1550         if (bbdev == NULL)
1551                 return -ENODEV;
1552
1553         /* allocate device private memory */
1554         bbdev->data->dev_private = rte_zmalloc_socket(dev_name,
1555                         sizeof(struct fpga_5gnr_fec_device),
1556                         RTE_CACHE_LINE_SIZE,
1557                         pci_dev->device.numa_node);
1558
1559         if (bbdev->data->dev_private == NULL) {
1560                 rte_bbdev_log(CRIT,
1561                                 "Allocate of %zu bytes for device \"%s\" failed",
1562                                 sizeof(struct fpga_5gnr_fec_device), dev_name);
1563                                 rte_bbdev_release(bbdev);
1564                         return -ENOMEM;
1565         }
1566
1567         /* Fill HW specific part of device structure */
1568         bbdev->device = &pci_dev->device;
1569         bbdev->intr_handle = &pci_dev->intr_handle;
1570         bbdev->data->socket_id = pci_dev->device.numa_node;
1571
1572         /* Invoke FEC FPGA device initialization function */
1573         fpga_5gnr_fec_init(bbdev, pci_drv);
1574
1575         rte_bbdev_log_debug("bbdev id = %u [%s]",
1576                         bbdev->data->dev_id, dev_name);
1577
1578         struct fpga_5gnr_fec_device *d = bbdev->data->dev_private;
1579         uint32_t version_id = fpga_reg_read_32(d->mmio_base,
1580                         FPGA_5GNR_FEC_VERSION_ID);
1581         rte_bbdev_log(INFO, "FEC FPGA RTL v%u.%u",
1582                 ((uint16_t)(version_id >> 16)), ((uint16_t)version_id));
1583
1584 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1585         if (!strcmp(bbdev->device->driver->name,
1586                         RTE_STR(FPGA_5GNR_FEC_PF_DRIVER_NAME)))
1587                 print_static_reg_debug_info(d->mmio_base);
1588 #endif
1589         return 0;
1590 }
1591
1592 static int
1593 fpga_5gnr_fec_remove(struct rte_pci_device *pci_dev)
1594 {
1595         struct rte_bbdev *bbdev;
1596         int ret;
1597         uint8_t dev_id;
1598
1599         if (pci_dev == NULL)
1600                 return -EINVAL;
1601
1602         /* Find device */
1603         bbdev = rte_bbdev_get_named_dev(pci_dev->device.name);
1604         if (bbdev == NULL) {
1605                 rte_bbdev_log(CRIT,
1606                                 "Couldn't find HW dev \"%s\" to uninitialise it",
1607                                 pci_dev->device.name);
1608                 return -ENODEV;
1609         }
1610         dev_id = bbdev->data->dev_id;
1611
1612         /* free device private memory before close */
1613         rte_free(bbdev->data->dev_private);
1614
1615         /* Close device */
1616         ret = rte_bbdev_close(dev_id);
1617         if (ret < 0)
1618                 rte_bbdev_log(ERR,
1619                                 "Device %i failed to close during uninit: %i",
1620                                 dev_id, ret);
1621
1622         /* release bbdev from library */
1623         ret = rte_bbdev_release(bbdev);
1624         if (ret)
1625                 rte_bbdev_log(ERR, "Device %i failed to uninit: %i", dev_id,
1626                                 ret);
1627
1628         rte_bbdev_log_debug("Destroyed bbdev = %u", dev_id);
1629
1630         return 0;
1631 }
1632
1633 static inline void
1634 set_default_fpga_conf(struct fpga_5gnr_fec_conf *def_conf)
1635 {
1636         /* clear default configuration before initialization */
1637         memset(def_conf, 0, sizeof(struct fpga_5gnr_fec_conf));
1638         /* Set pf mode to true */
1639         def_conf->pf_mode_en = true;
1640
1641         /* Set ratio between UL and DL to 1:1 (unit of weight is 3 CBs) */
1642         def_conf->ul_bandwidth = 3;
1643         def_conf->dl_bandwidth = 3;
1644
1645         /* Set Load Balance Factor to 64 */
1646         def_conf->dl_load_balance = 64;
1647         def_conf->ul_load_balance = 64;
1648 }
1649
1650 /* Initial configuration of FPGA 5GNR FEC device */
1651 int
1652 fpga_5gnr_fec_configure(const char *dev_name,
1653                 const struct fpga_5gnr_fec_conf *conf)
1654 {
1655         uint32_t payload_32, address;
1656         uint16_t payload_16;
1657         uint8_t payload_8;
1658         uint16_t q_id, vf_id, total_q_id, total_ul_q_id, total_dl_q_id;
1659         struct rte_bbdev *bbdev = rte_bbdev_get_named_dev(dev_name);
1660         struct fpga_5gnr_fec_conf def_conf;
1661
1662         if (bbdev == NULL) {
1663                 rte_bbdev_log(ERR,
1664                                 "Invalid dev_name (%s), or device is not yet initialised",
1665                                 dev_name);
1666                 return -ENODEV;
1667         }
1668
1669         struct fpga_5gnr_fec_device *d = bbdev->data->dev_private;
1670
1671         if (conf == NULL) {
1672                 rte_bbdev_log(ERR,
1673                                 "FPGA Configuration was not provided. Default configuration will be loaded.");
1674                 set_default_fpga_conf(&def_conf);
1675                 conf = &def_conf;
1676         }
1677
1678         /*
1679          * Configure UL:DL ratio.
1680          * [7:0]: UL weight
1681          * [15:8]: DL weight
1682          */
1683         payload_16 = (conf->dl_bandwidth << 8) | conf->ul_bandwidth;
1684         address = FPGA_5GNR_FEC_CONFIGURATION;
1685         fpga_reg_write_16(d->mmio_base, address, payload_16);
1686
1687         /* Clear all queues registers */
1688         payload_32 = FPGA_INVALID_HW_QUEUE_ID;
1689         for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) {
1690                 address = (q_id << 2) + FPGA_5GNR_FEC_QUEUE_MAP;
1691                 fpga_reg_write_32(d->mmio_base, address, payload_32);
1692         }
1693
1694         /*
1695          * If PF mode is enabled allocate all queues for PF only.
1696          *
1697          * For VF mode each VF can have different number of UL and DL queues.
1698          * Total number of queues to configure cannot exceed FPGA
1699          * capabilities - 64 queues - 32 queues for UL and 32 queues for DL.
1700          * Queues mapping is done according to configuration:
1701          *
1702          * UL queues:
1703          * |                Q_ID              | VF_ID |
1704          * |                 0                |   0   |
1705          * |                ...               |   0   |
1706          * | conf->vf_dl_queues_number[0] - 1 |   0   |
1707          * | conf->vf_dl_queues_number[0]     |   1   |
1708          * |                ...               |   1   |
1709          * | conf->vf_dl_queues_number[1] - 1 |   1   |
1710          * |                ...               |  ...  |
1711          * | conf->vf_dl_queues_number[7] - 1 |   7   |
1712          *
1713          * DL queues:
1714          * |                Q_ID              | VF_ID |
1715          * |                 32               |   0   |
1716          * |                ...               |   0   |
1717          * | conf->vf_ul_queues_number[0] - 1 |   0   |
1718          * | conf->vf_ul_queues_number[0]     |   1   |
1719          * |                ...               |   1   |
1720          * | conf->vf_ul_queues_number[1] - 1 |   1   |
1721          * |                ...               |  ...  |
1722          * | conf->vf_ul_queues_number[7] - 1 |   7   |
1723          *
1724          * Example of configuration:
1725          * conf->vf_ul_queues_number[0] = 4;  -> 4 UL queues for VF0
1726          * conf->vf_dl_queues_number[0] = 4;  -> 4 DL queues for VF0
1727          * conf->vf_ul_queues_number[1] = 2;  -> 2 UL queues for VF1
1728          * conf->vf_dl_queues_number[1] = 2;  -> 2 DL queues for VF1
1729          *
1730          * UL:
1731          * | Q_ID | VF_ID |
1732          * |   0  |   0   |
1733          * |   1  |   0   |
1734          * |   2  |   0   |
1735          * |   3  |   0   |
1736          * |   4  |   1   |
1737          * |   5  |   1   |
1738          *
1739          * DL:
1740          * | Q_ID | VF_ID |
1741          * |  32  |   0   |
1742          * |  33  |   0   |
1743          * |  34  |   0   |
1744          * |  35  |   0   |
1745          * |  36  |   1   |
1746          * |  37  |   1   |
1747          */
1748         if (conf->pf_mode_en) {
1749                 payload_32 = 0x1;
1750                 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) {
1751                         address = (q_id << 2) + FPGA_5GNR_FEC_QUEUE_MAP;
1752                         fpga_reg_write_32(d->mmio_base, address, payload_32);
1753                 }
1754         } else {
1755                 /* Calculate total number of UL and DL queues to configure */
1756                 total_ul_q_id = total_dl_q_id = 0;
1757                 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) {
1758                         total_ul_q_id += conf->vf_ul_queues_number[vf_id];
1759                         total_dl_q_id += conf->vf_dl_queues_number[vf_id];
1760                 }
1761                 total_q_id = total_dl_q_id + total_ul_q_id;
1762                 /*
1763                  * Check if total number of queues to configure does not exceed
1764                  * FPGA capabilities (64 queues - 32 UL and 32 DL queues)
1765                  */
1766                 if ((total_ul_q_id > FPGA_NUM_UL_QUEUES) ||
1767                         (total_dl_q_id > FPGA_NUM_DL_QUEUES) ||
1768                         (total_q_id > FPGA_TOTAL_NUM_QUEUES)) {
1769                         rte_bbdev_log(ERR,
1770                                         "FPGA Configuration failed. Too many queues to configure: UL_Q %u, DL_Q %u, FPGA_Q %u",
1771                                         total_ul_q_id, total_dl_q_id,
1772                                         FPGA_TOTAL_NUM_QUEUES);
1773                         return -EINVAL;
1774                 }
1775                 total_ul_q_id = 0;
1776                 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) {
1777                         for (q_id = 0; q_id < conf->vf_ul_queues_number[vf_id];
1778                                         ++q_id, ++total_ul_q_id) {
1779                                 address = (total_ul_q_id << 2) +
1780                                                 FPGA_5GNR_FEC_QUEUE_MAP;
1781                                 payload_32 = ((0x80 + vf_id) << 16) | 0x1;
1782                                 fpga_reg_write_32(d->mmio_base, address,
1783                                                 payload_32);
1784                         }
1785                 }
1786                 total_dl_q_id = 0;
1787                 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) {
1788                         for (q_id = 0; q_id < conf->vf_dl_queues_number[vf_id];
1789                                         ++q_id, ++total_dl_q_id) {
1790                                 address = ((total_dl_q_id + FPGA_NUM_UL_QUEUES)
1791                                                 << 2) + FPGA_5GNR_FEC_QUEUE_MAP;
1792                                 payload_32 = ((0x80 + vf_id) << 16) | 0x1;
1793                                 fpga_reg_write_32(d->mmio_base, address,
1794                                                 payload_32);
1795                         }
1796                 }
1797         }
1798
1799         /* Setting Load Balance Factor */
1800         payload_16 = (conf->dl_load_balance << 8) | (conf->ul_load_balance);
1801         address = FPGA_5GNR_FEC_LOAD_BALANCE_FACTOR;
1802         fpga_reg_write_16(d->mmio_base, address, payload_16);
1803
1804         /* Setting length of ring descriptor entry */
1805         payload_16 = FPGA_RING_DESC_ENTRY_LENGTH;
1806         address = FPGA_5GNR_FEC_RING_DESC_LEN;
1807         fpga_reg_write_16(d->mmio_base, address, payload_16);
1808
1809         /* Setting FLR timeout value */
1810         payload_16 = conf->flr_time_out;
1811         address = FPGA_5GNR_FEC_FLR_TIME_OUT;
1812         fpga_reg_write_16(d->mmio_base, address, payload_16);
1813
1814         /* Queue PF/VF mapping table is ready */
1815         payload_8 = 0x1;
1816         address = FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE;
1817         fpga_reg_write_8(d->mmio_base, address, payload_8);
1818
1819         rte_bbdev_log_debug("PF FPGA 5GNR FEC configuration complete for %s",
1820                         dev_name);
1821
1822 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1823         print_static_reg_debug_info(d->mmio_base);
1824 #endif
1825         return 0;
1826 }
1827
1828 /* FPGA 5GNR FEC PCI PF address map */
1829 static struct rte_pci_id pci_id_fpga_5gnr_fec_pf_map[] = {
1830         {
1831                 RTE_PCI_DEVICE(FPGA_5GNR_FEC_VENDOR_ID,
1832                                 FPGA_5GNR_FEC_PF_DEVICE_ID)
1833         },
1834         {.device_id = 0},
1835 };
1836
1837 static struct rte_pci_driver fpga_5gnr_fec_pci_pf_driver = {
1838         .probe = fpga_5gnr_fec_probe,
1839         .remove = fpga_5gnr_fec_remove,
1840         .id_table = pci_id_fpga_5gnr_fec_pf_map,
1841         .drv_flags = RTE_PCI_DRV_NEED_MAPPING
1842 };
1843
1844 /* FPGA 5GNR FEC PCI VF address map */
1845 static struct rte_pci_id pci_id_fpga_5gnr_fec_vf_map[] = {
1846         {
1847                 RTE_PCI_DEVICE(FPGA_5GNR_FEC_VENDOR_ID,
1848                                 FPGA_5GNR_FEC_VF_DEVICE_ID)
1849         },
1850         {.device_id = 0},
1851 };
1852
1853 static struct rte_pci_driver fpga_5gnr_fec_pci_vf_driver = {
1854         .probe = fpga_5gnr_fec_probe,
1855         .remove = fpga_5gnr_fec_remove,
1856         .id_table = pci_id_fpga_5gnr_fec_vf_map,
1857         .drv_flags = RTE_PCI_DRV_NEED_MAPPING
1858 };
1859
1860
1861 RTE_PMD_REGISTER_PCI(FPGA_5GNR_FEC_PF_DRIVER_NAME, fpga_5gnr_fec_pci_pf_driver);
1862 RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_PF_DRIVER_NAME,
1863                 pci_id_fpga_5gnr_fec_pf_map);
1864 RTE_PMD_REGISTER_PCI(FPGA_5GNR_FEC_VF_DRIVER_NAME, fpga_5gnr_fec_pci_vf_driver);
1865 RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_VF_DRIVER_NAME,
1866                 pci_id_fpga_5gnr_fec_vf_map);
1867
1868 RTE_INIT(fpga_5gnr_fec_init_log)
1869 {
1870         fpga_5gnr_fec_logtype = rte_log_register("pmd.bb.fpga_5gnr_fec");
1871         if (fpga_5gnr_fec_logtype >= 0)
1872 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1873                 rte_log_set_level(fpga_5gnr_fec_logtype, RTE_LOG_DEBUG);
1874 #else
1875                 rte_log_set_level(fpga_5gnr_fec_logtype, RTE_LOG_NOTICE);
1876 #endif
1877 }