drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4
   5 #include <unistd.h>
   6
   7 #include <rte_common.h>
   8 #include <rte_log.h>
   9 #include <rte_dev.h>
  10 #include <rte_malloc.h>
  11 #include <rte_mempool.h>
  12 #include <rte_errno.h>
  13 #include <rte_pci.h>
  14 #include <rte_bus_pci.h>
  15 #include <rte_byteorder.h>
  16 #ifdef RTE_BBDEV_OFFLOAD_COST
  17 #include <rte_cycles.h>
  18 #endif
  19
  20 #include <rte_bbdev.h>
  21 #include <rte_bbdev_pmd.h>
  22
  23 #include "fpga_5gnr_fec.h"
  24 #include "rte_pmd_fpga_5gnr_fec.h"
  25
  26 #ifdef RTE_LIBRTE_BBDEV_DEBUG
  27 RTE_LOG_REGISTER_DEFAULT(fpga_5gnr_fec_logtype, DEBUG);
  28 #else
  29 RTE_LOG_REGISTER_DEFAULT(fpga_5gnr_fec_logtype, NOTICE);
  30 #endif
  31
  32 #ifdef RTE_LIBRTE_BBDEV_DEBUG
  33
  34 /* Read Ring Control Register of FPGA 5GNR FEC device */
  35 static inline void
  36 print_ring_reg_debug_info(void *mmio_base, uint32_t offset)
  37 {
  38         rte_bbdev_log_debug(
  39                 "FPGA MMIO base address @ %p | Ring Control Register @ offset = 0x%08"
  40                 PRIx32, mmio_base, offset);
  41         rte_bbdev_log_debug(
  42                 "RING_BASE_ADDR = 0x%016"PRIx64,
  43                 fpga_reg_read_64(mmio_base, offset));
  44         rte_bbdev_log_debug(
  45                 "RING_HEAD_ADDR = 0x%016"PRIx64,
  46                 fpga_reg_read_64(mmio_base, offset +
  47                                 FPGA_5GNR_FEC_RING_HEAD_ADDR));
  48         rte_bbdev_log_debug(
  49                 "RING_SIZE = 0x%04"PRIx16,
  50                 fpga_reg_read_16(mmio_base, offset +
  51                                 FPGA_5GNR_FEC_RING_SIZE));
  52         rte_bbdev_log_debug(
  53                 "RING_MISC = 0x%02"PRIx8,
  54                 fpga_reg_read_8(mmio_base, offset +
  55                                 FPGA_5GNR_FEC_RING_MISC));
  56         rte_bbdev_log_debug(
  57                 "RING_ENABLE = 0x%02"PRIx8,
  58                 fpga_reg_read_8(mmio_base, offset +
  59                                 FPGA_5GNR_FEC_RING_ENABLE));
  60         rte_bbdev_log_debug(
  61                 "RING_FLUSH_QUEUE_EN = 0x%02"PRIx8,
  62                 fpga_reg_read_8(mmio_base, offset +
  63                                 FPGA_5GNR_FEC_RING_FLUSH_QUEUE_EN));
  64         rte_bbdev_log_debug(
  65                 "RING_SHADOW_TAIL = 0x%04"PRIx16,
  66                 fpga_reg_read_16(mmio_base, offset +
  67                                 FPGA_5GNR_FEC_RING_SHADOW_TAIL));
  68         rte_bbdev_log_debug(
  69                 "RING_HEAD_POINT = 0x%04"PRIx16,
  70                 fpga_reg_read_16(mmio_base, offset +
  71                                 FPGA_5GNR_FEC_RING_HEAD_POINT));
  72 }
  73
  74 /* Read Static Register of FPGA 5GNR FEC device */
  75 static inline void
  76 print_static_reg_debug_info(void *mmio_base)
  77 {
  78         uint16_t config = fpga_reg_read_16(mmio_base,
  79                         FPGA_5GNR_FEC_CONFIGURATION);
  80         uint8_t qmap_done = fpga_reg_read_8(mmio_base,
  81                         FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE);
  82         uint16_t lb_factor = fpga_reg_read_16(mmio_base,
  83                         FPGA_5GNR_FEC_LOAD_BALANCE_FACTOR);
  84         uint16_t ring_desc_len = fpga_reg_read_16(mmio_base,
  85                         FPGA_5GNR_FEC_RING_DESC_LEN);
  86
  87         rte_bbdev_log_debug("UL.DL Weights = %u.%u",
  88                         ((uint8_t)config), ((uint8_t)(config >> 8)));
  89         rte_bbdev_log_debug("UL.DL Load Balance = %u.%u",
  90                         ((uint8_t)lb_factor), ((uint8_t)(lb_factor >> 8)));
  91         rte_bbdev_log_debug("Queue-PF/VF Mapping Table = %s",
  92                         (qmap_done > 0) ? "READY" : "NOT-READY");
  93         rte_bbdev_log_debug("Ring Descriptor Size = %u bytes",
  94                         ring_desc_len*FPGA_RING_DESC_LEN_UNIT_BYTES);
  95 }
  96
  97 /* Print decode DMA Descriptor of FPGA 5GNR Decoder device */
  98 static void
  99 print_dma_dec_desc_debug_info(union fpga_dma_desc *desc)
 100 {
 101         rte_bbdev_log_debug("DMA response desc %p\n"
 102                 "\t-- done(%"PRIu32") | iter(%"PRIu32") | et_pass(%"PRIu32")"
 103                 " | crcb_pass (%"PRIu32") | error(%"PRIu32")\n"
 104                 "\t-- qm_idx(%"PRIu32") | max_iter(%"PRIu32") | "
 105                 "bg_idx (%"PRIu32") | harqin_en(%"PRIu32") | zc(%"PRIu32")\n"
 106                 "\t-- hbstroe_offset(%"PRIu32") | num_null (%"PRIu32") "
 107                 "| irq_en(%"PRIu32")\n"
 108                 "\t-- ncb(%"PRIu32") | desc_idx (%"PRIu32") | "
 109                 "drop_crc24b(%"PRIu32") | RV (%"PRIu32")\n"
 110                 "\t-- crc24b_ind(%"PRIu32") | et_dis (%"PRIu32")\n"
 111                 "\t-- harq_input_length(%"PRIu32") | rm_e(%"PRIu32")\n"
 112                 "\t-- cbs_in_op(%"PRIu32") | in_add (0x%08"PRIx32"%08"PRIx32")"
 113                 "| out_add (0x%08"PRIx32"%08"PRIx32")",
 114                 desc,
 115                 (uint32_t)desc->dec_req.done,
 116                 (uint32_t)desc->dec_req.iter,
 117                 (uint32_t)desc->dec_req.et_pass,
 118                 (uint32_t)desc->dec_req.crcb_pass,
 119                 (uint32_t)desc->dec_req.error,
 120                 (uint32_t)desc->dec_req.qm_idx,
 121                 (uint32_t)desc->dec_req.max_iter,
 122                 (uint32_t)desc->dec_req.bg_idx,
 123                 (uint32_t)desc->dec_req.harqin_en,
 124                 (uint32_t)desc->dec_req.zc,
 125                 (uint32_t)desc->dec_req.hbstroe_offset,
 126                 (uint32_t)desc->dec_req.num_null,
 127                 (uint32_t)desc->dec_req.irq_en,
 128                 (uint32_t)desc->dec_req.ncb,
 129                 (uint32_t)desc->dec_req.desc_idx,
 130                 (uint32_t)desc->dec_req.drop_crc24b,
 131                 (uint32_t)desc->dec_req.rv,
 132                 (uint32_t)desc->dec_req.crc24b_ind,
 133                 (uint32_t)desc->dec_req.et_dis,
 134                 (uint32_t)desc->dec_req.harq_input_length,
 135                 (uint32_t)desc->dec_req.rm_e,
 136                 (uint32_t)desc->dec_req.cbs_in_op,
 137                 (uint32_t)desc->dec_req.in_addr_hi,
 138                 (uint32_t)desc->dec_req.in_addr_lw,
 139                 (uint32_t)desc->dec_req.out_addr_hi,
 140                 (uint32_t)desc->dec_req.out_addr_lw);
 141         uint32_t *word = (uint32_t *) desc;
 142         rte_bbdev_log_debug("%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n"
 143                         "%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n",
 144                         word[0], word[1], word[2], word[3],
 145                         word[4], word[5], word[6], word[7]);
 146 }
 147
 148 /* Print decode DMA Descriptor of FPGA 5GNR encoder device */
 149 static void
 150 print_dma_enc_desc_debug_info(union fpga_dma_desc *desc)
 151 {
 152         rte_bbdev_log_debug("DMA response desc %p\n"
 153                         "%"PRIu32" %"PRIu32"\n"
 154                         "K' %"PRIu32" E %"PRIu32" desc %"PRIu32" Z %"PRIu32"\n"
 155                         "BG %"PRIu32" Qm %"PRIu32" CRC %"PRIu32" IRQ %"PRIu32"\n"
 156                         "k0 %"PRIu32" Ncb %"PRIu32" F %"PRIu32"\n",
 157                         desc,
 158                         (uint32_t)desc->enc_req.done,
 159                         (uint32_t)desc->enc_req.error,
 160
 161                         (uint32_t)desc->enc_req.k_,
 162                         (uint32_t)desc->enc_req.rm_e,
 163                         (uint32_t)desc->enc_req.desc_idx,
 164                         (uint32_t)desc->enc_req.zc,
 165
 166                         (uint32_t)desc->enc_req.bg_idx,
 167                         (uint32_t)desc->enc_req.qm_idx,
 168                         (uint32_t)desc->enc_req.crc_en,
 169                         (uint32_t)desc->enc_req.irq_en,
 170
 171                         (uint32_t)desc->enc_req.k0,
 172                         (uint32_t)desc->enc_req.ncb,
 173                         (uint32_t)desc->enc_req.num_null);
 174         uint32_t *word = (uint32_t *) desc;
 175         rte_bbdev_log_debug("%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n"
 176                         "%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n",
 177                         word[0], word[1], word[2], word[3],
 178                         word[4], word[5], word[6], word[7]);
 179 }
 180
 181 #endif
 182
 183 static int
 184 fpga_setup_queues(struct rte_bbdev *dev, uint16_t num_queues, int socket_id)
 185 {
 186         /* Number of queues bound to a PF/VF */
 187         uint32_t hw_q_num = 0;
 188         uint32_t ring_size, payload, address, q_id, offset;
 189         rte_iova_t phys_addr;
 190         struct fpga_ring_ctrl_reg ring_reg;
 191         struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private;
 192
 193         address = FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE;
 194         if (!(fpga_reg_read_32(fpga_dev->mmio_base, address) & 0x1)) {
 195                 rte_bbdev_log(ERR,
 196                                 "Queue-PF/VF mapping is not set! Was PF configured for device (%s) ?",
 197                                 dev->data->name);
 198                 return -EPERM;
 199         }
 200
 201         /* Clear queue registers structure */
 202         memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg));
 203
 204         /* Scan queue map.
 205          * If a queue is valid and mapped to a calling PF/VF the read value is
 206          * replaced with a queue ID and if it's not then
 207          * FPGA_INVALID_HW_QUEUE_ID is returned.
 208          */
 209         for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) {
 210                 uint32_t hw_q_id = fpga_reg_read_32(fpga_dev->mmio_base,
 211                                 FPGA_5GNR_FEC_QUEUE_MAP + (q_id << 2));
 212
 213                 rte_bbdev_log_debug("%s: queue ID: %u, registry queue ID: %u",
 214                                 dev->device->name, q_id, hw_q_id);
 215
 216                 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID) {
 217                         fpga_dev->q_bound_bit_map |= (1ULL << q_id);
 218                         /* Clear queue register of found queue */
 219                         offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
 220                                 (sizeof(struct fpga_ring_ctrl_reg) * q_id);
 221                         fpga_ring_reg_write(fpga_dev->mmio_base,
 222                                         offset, ring_reg);
 223                         ++hw_q_num;
 224                 }
 225         }
 226         if (hw_q_num == 0) {
 227                 rte_bbdev_log(ERR,
 228                         "No HW queues assigned to this device. Probably this is a VF configured for PF mode. Check device configuration!");
 229                 return -ENODEV;
 230         }
 231
 232         if (num_queues > hw_q_num) {
 233                 rte_bbdev_log(ERR,
 234                         "Not enough queues for device %s! Requested: %u, available: %u",
 235                         dev->device->name, num_queues, hw_q_num);
 236                 return -EINVAL;
 237         }
 238
 239         ring_size = FPGA_RING_MAX_SIZE * sizeof(struct fpga_dma_dec_desc);
 240
 241         /* Enforce 32 byte alignment */
 242         RTE_BUILD_BUG_ON((RTE_CACHE_LINE_SIZE % 32) != 0);
 243
 244         /* Allocate memory for SW descriptor rings */
 245         fpga_dev->sw_rings = rte_zmalloc_socket(dev->device->driver->name,
 246                         num_queues * ring_size, RTE_CACHE_LINE_SIZE,
 247                         socket_id);
 248         if (fpga_dev->sw_rings == NULL) {
 249                 rte_bbdev_log(ERR,
 250                                 "Failed to allocate memory for %s:%u sw_rings",
 251                                 dev->device->driver->name, dev->data->dev_id);
 252                 return -ENOMEM;
 253         }
 254
 255         fpga_dev->sw_rings_phys = rte_malloc_virt2iova(fpga_dev->sw_rings);
 256         fpga_dev->sw_ring_size = ring_size;
 257         fpga_dev->sw_ring_max_depth = FPGA_RING_MAX_SIZE;
 258
 259         /* Allocate memory for ring flush status */
 260         fpga_dev->flush_queue_status = rte_zmalloc_socket(NULL,
 261                         sizeof(uint64_t), RTE_CACHE_LINE_SIZE, socket_id);
 262         if (fpga_dev->flush_queue_status == NULL) {
 263                 rte_bbdev_log(ERR,
 264                                 "Failed to allocate memory for %s:%u flush_queue_status",
 265                                 dev->device->driver->name, dev->data->dev_id);
 266                 return -ENOMEM;
 267         }
 268
 269         /* Set the flush status address registers */
 270         phys_addr = rte_malloc_virt2iova(fpga_dev->flush_queue_status);
 271
 272         address = FPGA_5GNR_FEC_VFQ_FLUSH_STATUS_LW;
 273         payload = (uint32_t)(phys_addr);
 274         fpga_reg_write_32(fpga_dev->mmio_base, address, payload);
 275
 276         address = FPGA_5GNR_FEC_VFQ_FLUSH_STATUS_HI;
 277         payload = (uint32_t)(phys_addr >> 32);
 278         fpga_reg_write_32(fpga_dev->mmio_base, address, payload);
 279
 280         return 0;
 281 }
 282
 283 static int
 284 fpga_dev_close(struct rte_bbdev *dev)
 285 {
 286         struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private;
 287
 288         rte_free(fpga_dev->sw_rings);
 289         rte_free(fpga_dev->flush_queue_status);
 290
 291         return 0;
 292 }
 293
 294 static void
 295 fpga_dev_info_get(struct rte_bbdev *dev,
 296                 struct rte_bbdev_driver_info *dev_info)
 297 {
 298         struct fpga_5gnr_fec_device *d = dev->data->dev_private;
 299         uint32_t q_id = 0;
 300
 301         static const struct rte_bbdev_op_cap bbdev_capabilities[] = {
 302                 {
 303                         .type   = RTE_BBDEV_OP_LDPC_ENC,
 304                         .cap.ldpc_enc = {
 305                                 .capability_flags =
 306                                                 RTE_BBDEV_LDPC_RATE_MATCH |
 307                                                 RTE_BBDEV_LDPC_ENC_INTERRUPTS |
 308                                                 RTE_BBDEV_LDPC_CRC_24B_ATTACH,
 309                                 .num_buffers_src =
 310                                                 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
 311                                 .num_buffers_dst =
 312                                                 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
 313                         }
 314                 },
 315                 {
 316                 .type   = RTE_BBDEV_OP_LDPC_DEC,
 317                 .cap.ldpc_dec = {
 318                         .capability_flags =
 319                                 RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK |
 320                                 RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP |
 321                                 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE |
 322                                 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE |
 323                                 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE |
 324                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE |
 325                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE |
 326                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK |
 327                                 RTE_BBDEV_LDPC_DEC_INTERRUPTS |
 328                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS,
 329                         .llr_size = 6,
 330                         .llr_decimals = 2,
 331                         .num_buffers_src =
 332                                         RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
 333                         .num_buffers_hard_out =
 334                                         RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
 335                         .num_buffers_soft_out = 0,
 336                 }
 337                 },
 338                 RTE_BBDEV_END_OF_CAPABILITIES_LIST()
 339         };
 340
 341         /* Check the HARQ DDR size available */
 342         uint8_t timeout_counter = 0;
 343         uint32_t harq_buf_ready = fpga_reg_read_32(d->mmio_base,
 344                         FPGA_5GNR_FEC_HARQ_BUF_SIZE_RDY_REGS);
 345         while (harq_buf_ready != 1) {
 346                 usleep(FPGA_TIMEOUT_CHECK_INTERVAL);
 347                 timeout_counter++;
 348                 harq_buf_ready = fpga_reg_read_32(d->mmio_base,
 349                                 FPGA_5GNR_FEC_HARQ_BUF_SIZE_RDY_REGS);
 350                 if (timeout_counter > FPGA_HARQ_RDY_TIMEOUT) {
 351                         rte_bbdev_log(ERR, "HARQ Buffer not ready %d",
 352                                         harq_buf_ready);
 353                         harq_buf_ready = 1;
 354                 }
 355         }
 356         uint32_t harq_buf_size = fpga_reg_read_32(d->mmio_base,
 357                         FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS);
 358
 359         static struct rte_bbdev_queue_conf default_queue_conf;
 360         default_queue_conf.socket = dev->data->socket_id;
 361         default_queue_conf.queue_size = FPGA_RING_MAX_SIZE;
 362
 363         dev_info->driver_name = dev->device->driver->name;
 364         dev_info->queue_size_lim = FPGA_RING_MAX_SIZE;
 365         dev_info->hardware_accelerated = true;
 366         dev_info->min_alignment = 64;
 367         dev_info->harq_buffer_size = (harq_buf_size >> 10) + 1;
 368         dev_info->default_queue_conf = default_queue_conf;
 369         dev_info->capabilities = bbdev_capabilities;
 370         dev_info->cpu_flag_reqs = NULL;
 371         dev_info->data_endianness = RTE_LITTLE_ENDIAN;
 372
 373         /* Calculates number of queues assigned to device */
 374         dev_info->max_num_queues = 0;
 375         for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) {
 376                 uint32_t hw_q_id = fpga_reg_read_32(d->mmio_base,
 377                                 FPGA_5GNR_FEC_QUEUE_MAP + (q_id << 2));
 378                 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID)
 379                         dev_info->max_num_queues++;
 380         }
 381 }
 382
 383 /**
 384  * Find index of queue bound to current PF/VF which is unassigned. Return -1
 385  * when there is no available queue
 386  */
 387 static inline int
 388 fpga_find_free_queue_idx(struct rte_bbdev *dev,
 389                 const struct rte_bbdev_queue_conf *conf)
 390 {
 391         struct fpga_5gnr_fec_device *d = dev->data->dev_private;
 392         uint64_t q_idx;
 393         uint8_t i = 0;
 394         uint8_t range = FPGA_TOTAL_NUM_QUEUES >> 1;
 395
 396         if (conf->op_type == RTE_BBDEV_OP_LDPC_ENC) {
 397                 i = FPGA_NUM_DL_QUEUES;
 398                 range = FPGA_TOTAL_NUM_QUEUES;
 399         }
 400
 401         for (; i < range; ++i) {
 402                 q_idx = 1ULL << i;
 403                 /* Check if index of queue is bound to current PF/VF */
 404                 if (d->q_bound_bit_map & q_idx)
 405                         /* Check if found queue was not already assigned */
 406                         if (!(d->q_assigned_bit_map & q_idx)) {
 407                                 d->q_assigned_bit_map |= q_idx;
 408                                 return i;
 409                         }
 410         }
 411
 412         rte_bbdev_log(INFO, "Failed to find free queue on %s", dev->data->name);
 413
 414         return -1;
 415 }
 416
 417 static int
 418 fpga_queue_setup(struct rte_bbdev *dev, uint16_t queue_id,
 419                 const struct rte_bbdev_queue_conf *conf)
 420 {
 421         uint32_t address, ring_offset;
 422         struct fpga_5gnr_fec_device *d = dev->data->dev_private;
 423         struct fpga_queue *q;
 424         int8_t q_idx;
 425
 426         /* Check if there is a free queue to assign */
 427         q_idx = fpga_find_free_queue_idx(dev, conf);
 428         if (q_idx == -1)
 429                 return -1;
 430
 431         /* Allocate the queue data structure. */
 432         q = rte_zmalloc_socket(dev->device->driver->name, sizeof(*q),
 433                         RTE_CACHE_LINE_SIZE, conf->socket);
 434         if (q == NULL) {
 435                 /* Mark queue as un-assigned */
 436                 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx));
 437                 rte_bbdev_log(ERR, "Failed to allocate queue memory");
 438                 return -ENOMEM;
 439         }
 440
 441         q->d = d;
 442         q->q_idx = q_idx;
 443
 444         /* Set ring_base_addr */
 445         q->ring_addr = RTE_PTR_ADD(d->sw_rings, (d->sw_ring_size * queue_id));
 446         q->ring_ctrl_reg.ring_base_addr = d->sw_rings_phys +
 447                         (d->sw_ring_size * queue_id);
 448
 449         /* Allocate memory for Completion Head variable*/
 450         q->ring_head_addr = rte_zmalloc_socket(dev->device->driver->name,
 451                         sizeof(uint64_t), RTE_CACHE_LINE_SIZE, conf->socket);
 452         if (q->ring_head_addr == NULL) {
 453                 /* Mark queue as un-assigned */
 454                 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx));
 455                 rte_free(q);
 456                 rte_bbdev_log(ERR,
 457                                 "Failed to allocate memory for %s:%u completion_head",
 458                                 dev->device->driver->name, dev->data->dev_id);
 459                 return -ENOMEM;
 460         }
 461         /* Set ring_head_addr */
 462         q->ring_ctrl_reg.ring_head_addr =
 463                         rte_malloc_virt2iova(q->ring_head_addr);
 464
 465         /* Clear shadow_completion_head */
 466         q->shadow_completion_head = 0;
 467
 468         /* Set ring_size */
 469         if (conf->queue_size > FPGA_RING_MAX_SIZE) {
 470                 /* Mark queue as un-assigned */
 471                 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx));
 472                 rte_free(q->ring_head_addr);
 473                 rte_free(q);
 474                 rte_bbdev_log(ERR,
 475                                 "Size of queue is too big %d (MAX: %d ) for %s:%u",
 476                                 conf->queue_size, FPGA_RING_MAX_SIZE,
 477                                 dev->device->driver->name, dev->data->dev_id);
 478                 return -EINVAL;
 479         }
 480         q->ring_ctrl_reg.ring_size = conf->queue_size;
 481
 482         /* Set Miscellaneous FPGA register*/
 483         /* Max iteration number for TTI mitigation - todo */
 484         q->ring_ctrl_reg.max_ul_dec = 0;
 485         /* Enable max iteration number for TTI - todo */
 486         q->ring_ctrl_reg.max_ul_dec_en = 0;
 487
 488         /* Enable the ring */
 489         q->ring_ctrl_reg.enable = 1;
 490
 491         /* Set FPGA head_point and tail registers */
 492         q->ring_ctrl_reg.head_point = q->tail = 0;
 493
 494         /* Set FPGA shadow_tail register */
 495         q->ring_ctrl_reg.shadow_tail = q->tail;
 496
 497         /* Calculates the ring offset for found queue */
 498         ring_offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
 499                         (sizeof(struct fpga_ring_ctrl_reg) * q_idx);
 500
 501         /* Set FPGA Ring Control Registers */
 502         fpga_ring_reg_write(d->mmio_base, ring_offset, q->ring_ctrl_reg);
 503
 504         /* Store MMIO register of shadow_tail */
 505         address = ring_offset + FPGA_5GNR_FEC_RING_SHADOW_TAIL;
 506         q->shadow_tail_addr = RTE_PTR_ADD(d->mmio_base, address);
 507
 508         q->head_free_desc = q->tail;
 509
 510         /* Set wrap mask */
 511         q->sw_ring_wrap_mask = conf->queue_size - 1;
 512
 513         rte_bbdev_log_debug("Setup dev%u q%u: queue_idx=%u",
 514                         dev->data->dev_id, queue_id, q->q_idx);
 515
 516         dev->data->queues[queue_id].queue_private = q;
 517
 518         rte_bbdev_log_debug("BBDEV queue[%d] set up for FPGA queue[%d]",
 519                         queue_id, q_idx);
 520
 521 #ifdef RTE_LIBRTE_BBDEV_DEBUG
 522         /* Read FPGA Ring Control Registers after configuration*/
 523         print_ring_reg_debug_info(d->mmio_base, ring_offset);
 524 #endif
 525         return 0;
 526 }
 527
 528 static int
 529 fpga_queue_release(struct rte_bbdev *dev, uint16_t queue_id)
 530 {
 531         struct fpga_5gnr_fec_device *d = dev->data->dev_private;
 532         struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
 533         struct fpga_ring_ctrl_reg ring_reg;
 534         uint32_t offset;
 535
 536         rte_bbdev_log_debug("FPGA Queue[%d] released", queue_id);
 537
 538         if (q != NULL) {
 539                 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg));
 540                 offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
 541                         (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx);
 542                 /* Disable queue */
 543                 fpga_reg_write_8(d->mmio_base,
 544                                 offset + FPGA_5GNR_FEC_RING_ENABLE, 0x00);
 545                 /* Clear queue registers */
 546                 fpga_ring_reg_write(d->mmio_base, offset, ring_reg);
 547
 548                 /* Mark the Queue as un-assigned */
 549                 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q->q_idx));
 550                 rte_free(q->ring_head_addr);
 551                 rte_free(q);
 552                 dev->data->queues[queue_id].queue_private = NULL;
 553         }
 554
 555         return 0;
 556 }
 557
 558 /* Function starts a device queue. */
 559 static int
 560 fpga_queue_start(struct rte_bbdev *dev, uint16_t queue_id)
 561 {
 562         struct fpga_5gnr_fec_device *d = dev->data->dev_private;
 563 #ifdef RTE_LIBRTE_BBDEV_DEBUG
 564         if (d == NULL) {
 565                 rte_bbdev_log(ERR, "Invalid device pointer");
 566                 return -1;
 567         }
 568 #endif
 569         struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
 570         uint32_t offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
 571                         (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx);
 572         uint8_t enable = 0x01;
 573         uint16_t zero = 0x0000;
 574
 575         /* Clear queue head and tail variables */
 576         q->tail = q->head_free_desc = 0;
 577
 578         /* Clear FPGA head_point and tail registers */
 579         fpga_reg_write_16(d->mmio_base, offset + FPGA_5GNR_FEC_RING_HEAD_POINT,
 580                         zero);
 581         fpga_reg_write_16(d->mmio_base, offset + FPGA_5GNR_FEC_RING_SHADOW_TAIL,
 582                         zero);
 583
 584         /* Enable queue */
 585         fpga_reg_write_8(d->mmio_base, offset + FPGA_5GNR_FEC_RING_ENABLE,
 586                         enable);
 587
 588         rte_bbdev_log_debug("FPGA Queue[%d] started", queue_id);
 589         return 0;
 590 }
 591
 592 /* Function stops a device queue. */
 593 static int
 594 fpga_queue_stop(struct rte_bbdev *dev, uint16_t queue_id)
 595 {
 596         struct fpga_5gnr_fec_device *d = dev->data->dev_private;
 597 #ifdef RTE_LIBRTE_BBDEV_DEBUG
 598         if (d == NULL) {
 599                 rte_bbdev_log(ERR, "Invalid device pointer");
 600                 return -1;
 601         }
 602 #endif
 603         struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
 604         uint32_t offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
 605                         (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx);
 606         uint8_t payload = 0x01;
 607         uint8_t counter = 0;
 608         uint8_t timeout = FPGA_QUEUE_FLUSH_TIMEOUT_US /
 609                         FPGA_TIMEOUT_CHECK_INTERVAL;
 610
 611         /* Set flush_queue_en bit to trigger queue flushing */
 612         fpga_reg_write_8(d->mmio_base,
 613                         offset + FPGA_5GNR_FEC_RING_FLUSH_QUEUE_EN, payload);
 614
 615         /** Check if queue flush is completed.
 616          * FPGA will update the completion flag after queue flushing is
 617          * completed. If completion flag is not updated within 1ms it is
 618          * considered as a failure.
 619          */
 620         while (!(*((volatile uint8_t *)d->flush_queue_status + q->q_idx)
 621                         & payload)) {
 622                 if (counter > timeout) {
 623                         rte_bbdev_log(ERR, "FPGA Queue Flush failed for queue %d",
 624                                         queue_id);
 625                         return -1;
 626                 }
 627                 usleep(FPGA_TIMEOUT_CHECK_INTERVAL);
 628                 counter++;
 629         }
 630
 631         /* Disable queue */
 632         payload = 0x00;
 633         fpga_reg_write_8(d->mmio_base, offset + FPGA_5GNR_FEC_RING_ENABLE,
 634                         payload);
 635
 636         rte_bbdev_log_debug("FPGA Queue[%d] stopped", queue_id);
 637         return 0;
 638 }
 639
 640 static inline uint16_t
 641 get_queue_id(struct rte_bbdev_data *data, uint8_t q_idx)
 642 {
 643         uint16_t queue_id;
 644
 645         for (queue_id = 0; queue_id < data->num_queues; ++queue_id) {
 646                 struct fpga_queue *q = data->queues[queue_id].queue_private;
 647                 if (q != NULL && q->q_idx == q_idx)
 648                         return queue_id;
 649         }
 650
 651         return -1;
 652 }
 653
 654 /* Interrupt handler triggered by FPGA dev for handling specific interrupt */
 655 static void
 656 fpga_dev_interrupt_handler(void *cb_arg)
 657 {
 658         struct rte_bbdev *dev = cb_arg;
 659         struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private;
 660         struct fpga_queue *q;
 661         uint64_t ring_head;
 662         uint64_t q_idx;
 663         uint16_t queue_id;
 664         uint8_t i;
 665
 666         /* Scan queue assigned to this device */
 667         for (i = 0; i < FPGA_TOTAL_NUM_QUEUES; ++i) {
 668                 q_idx = 1ULL << i;
 669                 if (fpga_dev->q_bound_bit_map & q_idx) {
 670                         queue_id = get_queue_id(dev->data, i);
 671                         if (queue_id == (uint16_t) -1)
 672                                 continue;
 673
 674                         /* Check if completion head was changed */
 675                         q = dev->data->queues[queue_id].queue_private;
 676                         ring_head = *q->ring_head_addr;
 677                         if (q->shadow_completion_head != ring_head &&
 678                                 q->irq_enable == 1) {
 679                                 q->shadow_completion_head = ring_head;
 680                                 rte_bbdev_pmd_callback_process(
 681                                                 dev,
 682                                                 RTE_BBDEV_EVENT_DEQUEUE,
 683                                                 &queue_id);
 684                         }
 685                 }
 686         }
 687 }
 688
 689 static int
 690 fpga_queue_intr_enable(struct rte_bbdev *dev, uint16_t queue_id)
 691 {
 692         struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
 693
 694         if (!rte_intr_cap_multiple(dev->intr_handle))
 695                 return -ENOTSUP;
 696
 697         q->irq_enable = 1;
 698
 699         return 0;
 700 }
 701
 702 static int
 703 fpga_queue_intr_disable(struct rte_bbdev *dev, uint16_t queue_id)
 704 {
 705         struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
 706         q->irq_enable = 0;
 707
 708         return 0;
 709 }
 710
 711 static int
 712 fpga_intr_enable(struct rte_bbdev *dev)
 713 {
 714         int ret;
 715         uint8_t i;
 716
 717         if (!rte_intr_cap_multiple(dev->intr_handle)) {
 718                 rte_bbdev_log(ERR, "Multiple intr vector is not supported by FPGA (%s)",
 719                                 dev->data->name);
 720                 return -ENOTSUP;
 721         }
 722
 723         /* Create event file descriptors for each of 64 queue. Event fds will be
 724          * mapped to FPGA IRQs in rte_intr_enable(). This is a 1:1 mapping where
 725          * the IRQ number is a direct translation to the queue number.
 726          *
 727          * 63 (FPGA_NUM_INTR_VEC) event fds are created as rte_intr_enable()
 728          * mapped the first IRQ to already created interrupt event file
 729          * descriptor (intr_handle->fd).
 730          */
 731         if (rte_intr_efd_enable(dev->intr_handle, FPGA_NUM_INTR_VEC)) {
 732                 rte_bbdev_log(ERR, "Failed to create fds for %u queues",
 733                                 dev->data->num_queues);
 734                 return -1;
 735         }
 736
 737         /* TODO Each event file descriptor is overwritten by interrupt event
 738          * file descriptor. That descriptor is added to epoll observed list.
 739          * It ensures that callback function assigned to that descriptor will
 740          * invoked when any FPGA queue issues interrupt.
 741          */
 742         for (i = 0; i < FPGA_NUM_INTR_VEC; ++i) {
 743                 if (rte_intr_efds_index_set(dev->intr_handle, i,
 744                                 rte_intr_fd_get(dev->intr_handle)))
 745                         return -rte_errno;
 746         }
 747
 748         if (rte_intr_vec_list_alloc(dev->intr_handle, "intr_vec",
 749                         dev->data->num_queues)) {
 750                 rte_bbdev_log(ERR, "Failed to allocate %u vectors",
 751                                 dev->data->num_queues);
 752                 return -ENOMEM;
 753         }
 754
 755         ret = rte_intr_enable(dev->intr_handle);
 756         if (ret < 0) {
 757                 rte_bbdev_log(ERR,
 758                                 "Couldn't enable interrupts for device: %s",
 759                                 dev->data->name);
 760                 return ret;
 761         }
 762
 763         ret = rte_intr_callback_register(dev->intr_handle,
 764                         fpga_dev_interrupt_handler, dev);
 765         if (ret < 0) {
 766                 rte_bbdev_log(ERR,
 767                                 "Couldn't register interrupt callback for device: %s",
 768                                 dev->data->name);
 769                 return ret;
 770         }
 771
 772         return 0;
 773 }
 774
 775 static const struct rte_bbdev_ops fpga_ops = {
 776         .setup_queues = fpga_setup_queues,
 777         .intr_enable = fpga_intr_enable,
 778         .close = fpga_dev_close,
 779         .info_get = fpga_dev_info_get,
 780         .queue_setup = fpga_queue_setup,
 781         .queue_stop = fpga_queue_stop,
 782         .queue_start = fpga_queue_start,
 783         .queue_release = fpga_queue_release,
 784         .queue_intr_enable = fpga_queue_intr_enable,
 785         .queue_intr_disable = fpga_queue_intr_disable
 786 };
 787
 788 static inline void
 789 fpga_dma_enqueue(struct fpga_queue *q, uint16_t num_desc,
 790                 struct rte_bbdev_stats *queue_stats)
 791 {
 792 #ifdef RTE_BBDEV_OFFLOAD_COST
 793         uint64_t start_time = 0;
 794         queue_stats->acc_offload_cycles = 0;
 795 #else
 796         RTE_SET_USED(queue_stats);
 797 #endif
 798
 799         /* Update tail and shadow_tail register */
 800         q->tail = (q->tail + num_desc) & q->sw_ring_wrap_mask;
 801
 802         rte_wmb();
 803
 804 #ifdef RTE_BBDEV_OFFLOAD_COST
 805         /* Start time measurement for enqueue function offload. */
 806         start_time = rte_rdtsc_precise();
 807 #endif
 808         mmio_write_16(q->shadow_tail_addr, q->tail);
 809
 810 #ifdef RTE_BBDEV_OFFLOAD_COST
 811         rte_wmb();
 812         queue_stats->acc_offload_cycles += rte_rdtsc_precise() - start_time;
 813 #endif
 814 }
 815
 816 /* Read flag value 0/1/ from bitmap */
 817 static inline bool
 818 check_bit(uint32_t bitmap, uint32_t bitmask)
 819 {
 820         return bitmap & bitmask;
 821 }
 822
 823 /* Print an error if a descriptor error has occurred.
 824  *  Return 0 on success, 1 on failure
 825  */
 826 static inline int
 827 check_desc_error(uint32_t error_code) {
 828         switch (error_code) {
 829         case DESC_ERR_NO_ERR:
 830                 return 0;
 831         case DESC_ERR_K_P_OUT_OF_RANGE:
 832                 rte_bbdev_log(ERR, "Encode block size K' is out of range");
 833                 break;
 834         case DESC_ERR_Z_C_NOT_LEGAL:
 835                 rte_bbdev_log(ERR, "Zc is illegal");
 836                 break;
 837         case DESC_ERR_DESC_OFFSET_ERR:
 838                 rte_bbdev_log(ERR,
 839                                 "Queue offset does not meet the expectation in the FPGA"
 840                                 );
 841                 break;
 842         case DESC_ERR_DESC_READ_FAIL:
 843                 rte_bbdev_log(ERR, "Unsuccessful completion for descriptor read");
 844                 break;
 845         case DESC_ERR_DESC_READ_TIMEOUT:
 846                 rte_bbdev_log(ERR, "Descriptor read time-out");
 847                 break;
 848         case DESC_ERR_DESC_READ_TLP_POISONED:
 849                 rte_bbdev_log(ERR, "Descriptor read TLP poisoned");
 850                 break;
 851         case DESC_ERR_HARQ_INPUT_LEN:
 852                 rte_bbdev_log(ERR, "HARQ input length is invalid");
 853                 break;
 854         case DESC_ERR_CB_READ_FAIL:
 855                 rte_bbdev_log(ERR, "Unsuccessful completion for code block");
 856                 break;
 857         case DESC_ERR_CB_READ_TIMEOUT:
 858                 rte_bbdev_log(ERR, "Code block read time-out");
 859                 break;
 860         case DESC_ERR_CB_READ_TLP_POISONED:
 861                 rte_bbdev_log(ERR, "Code block read TLP poisoned");
 862                 break;
 863         case DESC_ERR_HBSTORE_ERR:
 864                 rte_bbdev_log(ERR, "Hbstroe exceeds HARQ buffer size.");
 865                 break;
 866         default:
 867                 rte_bbdev_log(ERR, "Descriptor error unknown error code %u",
 868                                 error_code);
 869                 break;
 870         }
 871         return 1;
 872 }
 873
 874 /* Compute value of k0.
 875  * Based on 3GPP 38.212 Table 5.4.2.1-2
 876  * Starting position of different redundancy versions, k0
 877  */
 878 static inline uint16_t
 879 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index)
 880 {
 881         if (rv_index == 0)
 882                 return 0;
 883         uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c;
 884         if (n_cb == n) {
 885                 if (rv_index == 1)
 886                         return (bg == 1 ? K0_1_1 : K0_1_2) * z_c;
 887                 else if (rv_index == 2)
 888                         return (bg == 1 ? K0_2_1 : K0_2_2) * z_c;
 889                 else
 890                         return (bg == 1 ? K0_3_1 : K0_3_2) * z_c;
 891         }
 892         /* LBRM case - includes a division by N */
 893         if (rv_index == 1)
 894                 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb)
 895                                 / n) * z_c;
 896         else if (rv_index == 2)
 897                 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb)
 898                                 / n) * z_c;
 899         else
 900                 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb)
 901                                 / n) * z_c;
 902 }
 903
 904 /**
 905  * Set DMA descriptor for encode operation (1 Code Block)
 906  *
 907  * @param op
 908  *   Pointer to a single encode operation.
 909  * @param desc
 910  *   Pointer to DMA descriptor.
 911  * @param input
 912  *   Pointer to pointer to input data which will be decoded.
 913  * @param e
 914  *   E value (length of output in bits).
 915  * @param ncb
 916  *   Ncb value (size of the soft buffer).
 917  * @param out_length
 918  *   Length of output buffer
 919  * @param in_offset
 920  *   Input offset in rte_mbuf structure. It is used for calculating the point
 921  *   where data is starting.
 922  * @param out_offset
 923  *   Output offset in rte_mbuf structure. It is used for calculating the point
 924  *   where hard output data will be stored.
 925  * @param cbs_in_op
 926  *   Number of CBs contained in one operation.
 927  */
 928 static inline int
 929 fpga_dma_desc_te_fill(struct rte_bbdev_enc_op *op,
 930                 struct fpga_dma_enc_desc *desc, struct rte_mbuf *input,
 931                 struct rte_mbuf *output, uint16_t k_,  uint16_t e,
 932                 uint32_t in_offset, uint32_t out_offset, uint16_t desc_offset,
 933                 uint8_t cbs_in_op)
 934 {
 935         /* reset */
 936         desc->done = 0;
 937         desc->error = 0;
 938         desc->k_ = k_;
 939         desc->rm_e = e;
 940         desc->desc_idx = desc_offset;
 941         desc->zc = op->ldpc_enc.z_c;
 942         desc->bg_idx = op->ldpc_enc.basegraph - 1;
 943         desc->qm_idx = op->ldpc_enc.q_m / 2;
 944         desc->crc_en = check_bit(op->ldpc_enc.op_flags,
 945                         RTE_BBDEV_LDPC_CRC_24B_ATTACH);
 946         desc->irq_en = 0;
 947         desc->k0 = get_k0(op->ldpc_enc.n_cb, op->ldpc_enc.z_c,
 948                         op->ldpc_enc.basegraph, op->ldpc_enc.rv_index);
 949         desc->ncb = op->ldpc_enc.n_cb;
 950         desc->num_null = op->ldpc_enc.n_filler;
 951         /* Set inbound data buffer address */
 952         desc->in_addr_hi = (uint32_t)(
 953                         rte_pktmbuf_iova_offset(input, in_offset) >> 32);
 954         desc->in_addr_lw = (uint32_t)(
 955                         rte_pktmbuf_iova_offset(input, in_offset));
 956
 957         desc->out_addr_hi = (uint32_t)(
 958                         rte_pktmbuf_iova_offset(output, out_offset) >> 32);
 959         desc->out_addr_lw = (uint32_t)(
 960                         rte_pktmbuf_iova_offset(output, out_offset));
 961         /* Save software context needed for dequeue */
 962         desc->op_addr = op;
 963         /* Set total number of CBs in an op */
 964         desc->cbs_in_op = cbs_in_op;
 965         return 0;
 966 }
 967
 968 /**
 969  * Set DMA descriptor for decode operation (1 Code Block)
 970  *
 971  * @param op
 972  *   Pointer to a single encode operation.
 973  * @param desc
 974  *   Pointer to DMA descriptor.
 975  * @param input
 976  *   Pointer to pointer to input data which will be decoded.
 977  * @param in_offset
 978  *   Input offset in rte_mbuf structure. It is used for calculating the point
 979  *   where data is starting.
 980  * @param out_offset
 981  *   Output offset in rte_mbuf structure. It is used for calculating the point
 982  *   where hard output data will be stored.
 983  * @param cbs_in_op
 984  *   Number of CBs contained in one operation.
 985  */
 986 static inline int
 987 fpga_dma_desc_ld_fill(struct rte_bbdev_dec_op *op,
 988                 struct fpga_dma_dec_desc *desc,
 989                 struct rte_mbuf *input, struct rte_mbuf *output,
 990                 uint16_t harq_in_length,
 991                 uint32_t in_offset, uint32_t out_offset,
 992                 uint32_t harq_offset,
 993                 uint16_t desc_offset,
 994                 uint8_t cbs_in_op)
 995 {
 996         /* reset */
 997         desc->done = 0;
 998         desc->error = 0;
 999         /* Set inbound data buffer address */
1000         desc->in_addr_hi = (uint32_t)(
1001                         rte_pktmbuf_iova_offset(input, in_offset) >> 32);
1002         desc->in_addr_lw = (uint32_t)(
1003                         rte_pktmbuf_iova_offset(input, in_offset));
1004         desc->rm_e = op->ldpc_dec.cb_params.e;
1005         desc->harq_input_length = harq_in_length;
1006         desc->et_dis = !check_bit(op->ldpc_dec.op_flags,
1007                         RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE);
1008         desc->rv = op->ldpc_dec.rv_index;
1009         desc->crc24b_ind = check_bit(op->ldpc_dec.op_flags,
1010                         RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK);
1011         desc->drop_crc24b = check_bit(op->ldpc_dec.op_flags,
1012                         RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP);
1013         desc->desc_idx = desc_offset;
1014         desc->ncb = op->ldpc_dec.n_cb;
1015         desc->num_null = op->ldpc_dec.n_filler;
1016         desc->hbstroe_offset = harq_offset >> 10;
1017         desc->zc = op->ldpc_dec.z_c;
1018         desc->harqin_en = check_bit(op->ldpc_dec.op_flags,
1019                         RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE);
1020         desc->bg_idx = op->ldpc_dec.basegraph - 1;
1021         desc->max_iter = op->ldpc_dec.iter_max;
1022         desc->qm_idx = op->ldpc_dec.q_m / 2;
1023         desc->out_addr_hi = (uint32_t)(
1024                         rte_pktmbuf_iova_offset(output, out_offset) >> 32);
1025         desc->out_addr_lw = (uint32_t)(
1026                         rte_pktmbuf_iova_offset(output, out_offset));
1027         /* Save software context needed for dequeue */
1028         desc->op_addr = op;
1029         /* Set total number of CBs in an op */
1030         desc->cbs_in_op = cbs_in_op;
1031
1032         return 0;
1033 }
1034
1035 /* Validates LDPC encoder parameters */
1036 static inline int
1037 validate_ldpc_enc_op(struct rte_bbdev_enc_op *op)
1038 {
1039         struct rte_bbdev_op_ldpc_enc *ldpc_enc = &op->ldpc_enc;
1040
1041         if (op->mempool == NULL) {
1042                 rte_bbdev_log(ERR, "Invalid mempool pointer");
1043                 return -1;
1044         }
1045         if (ldpc_enc->input.data == NULL) {
1046                 rte_bbdev_log(ERR, "Invalid input pointer");
1047                 return -1;
1048         }
1049         if (ldpc_enc->output.data == NULL) {
1050                 rte_bbdev_log(ERR, "Invalid output pointer");
1051                 return -1;
1052         }
1053         if (ldpc_enc->input.length == 0) {
1054                 rte_bbdev_log(ERR, "CB size (%u) is null",
1055                                 ldpc_enc->input.length);
1056                 return -1;
1057         }
1058         if ((ldpc_enc->basegraph > 2) || (ldpc_enc->basegraph == 0)) {
1059                 rte_bbdev_log(ERR,
1060                                 "BG (%u) is out of range 1 <= value <= 2",
1061                                 ldpc_enc->basegraph);
1062                 return -1;
1063         }
1064         if (ldpc_enc->rv_index > 3) {
1065                 rte_bbdev_log(ERR,
1066                                 "rv_index (%u) is out of range 0 <= value <= 3",
1067                                 ldpc_enc->rv_index);
1068                 return -1;
1069         }
1070         if (ldpc_enc->code_block_mode > RTE_BBDEV_CODE_BLOCK) {
1071                 rte_bbdev_log(ERR,
1072                                 "code_block_mode (%u) is out of range 0 <= value <= 1",
1073                                 ldpc_enc->code_block_mode);
1074                 return -1;
1075         }
1076
1077         if (ldpc_enc->input.length >
1078                 RTE_BBDEV_LDPC_MAX_CB_SIZE >> 3) {
1079                 rte_bbdev_log(ERR, "CB size (%u) is too big, max: %d",
1080                                 ldpc_enc->input.length,
1081                                 RTE_BBDEV_LDPC_MAX_CB_SIZE);
1082                 return -1;
1083         }
1084         int z_c = ldpc_enc->z_c;
1085         /* Check Zc is valid value */
1086         if ((z_c > 384) || (z_c < 4)) {
1087                 rte_bbdev_log(ERR, "Zc (%u) is out of range", z_c);
1088                 return -1;
1089         }
1090         if (z_c > 256) {
1091                 if ((z_c % 32) != 0) {
1092                         rte_bbdev_log(ERR, "Invalid Zc %d", z_c);
1093                         return -1;
1094                 }
1095         } else if (z_c > 128) {
1096                 if ((z_c % 16) != 0) {
1097                         rte_bbdev_log(ERR, "Invalid Zc %d", z_c);
1098                         return -1;
1099                 }
1100         } else if (z_c > 64) {
1101                 if ((z_c % 8) != 0) {
1102                         rte_bbdev_log(ERR, "Invalid Zc %d", z_c);
1103                         return -1;
1104                 }
1105         } else if (z_c > 32) {
1106                 if ((z_c % 4) != 0) {
1107                         rte_bbdev_log(ERR, "Invalid Zc %d", z_c);
1108                         return -1;
1109                 }
1110         } else if (z_c > 16) {
1111                 if ((z_c % 2) != 0) {
1112                         rte_bbdev_log(ERR, "Invalid Zc %d", z_c);
1113                         return -1;
1114                 }
1115         }
1116
1117         int n_filler = ldpc_enc->n_filler;
1118         int K = (ldpc_enc->basegraph == 1 ? 22 : 10) * ldpc_enc->z_c;
1119         int Kp = K - n_filler;
1120         int q_m = ldpc_enc->q_m;
1121         int n_cb = ldpc_enc->n_cb;
1122         int N = (ldpc_enc->basegraph == 1 ? N_ZC_1 : N_ZC_2) * z_c;
1123         int k0 = get_k0(n_cb, z_c, ldpc_enc->basegraph,
1124                         ldpc_enc->rv_index);
1125         int crc24 = 0;
1126         int32_t L, Lcb, cw, cw_rm;
1127         int32_t e = ldpc_enc->cb_params.e;
1128         if (check_bit(op->ldpc_enc.op_flags,
1129                         RTE_BBDEV_LDPC_CRC_24B_ATTACH))
1130                 crc24 = 24;
1131
1132         if (K < (int) (ldpc_enc->input.length * 8 + n_filler) + crc24) {
1133                 rte_bbdev_log(ERR, "K and F not matching input size %u %u %u",
1134                                 K, n_filler, ldpc_enc->input.length);
1135                 return -1;
1136         }
1137         if (ldpc_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
1138                 rte_bbdev_log(ERR, "TB mode not supported");
1139                 return -1;
1140
1141         }
1142
1143         /* K' range check */
1144         if (Kp % 8 > 0) {
1145                 rte_bbdev_log(ERR, "K' not byte aligned %u", Kp);
1146                 return -1;
1147         }
1148         if ((crc24 > 0) && (Kp < 292)) {
1149                 rte_bbdev_log(ERR, "Invalid CRC24 for small block %u", Kp);
1150                 return -1;
1151         }
1152         if (Kp < 24) {
1153                 rte_bbdev_log(ERR, "K' too small %u", Kp);
1154                 return -1;
1155         }
1156         if (n_filler >= (K - 2 * z_c)) {
1157                 rte_bbdev_log(ERR, "K - F invalid %u %u", K, n_filler);
1158                 return -1;
1159         }
1160         /* Ncb range check */
1161         if ((n_cb > N) || (n_cb < 32) || (n_cb <= (Kp - crc24))) {
1162                 rte_bbdev_log(ERR, "Ncb (%u) is out of range K  %d N %d", n_cb, K, N);
1163                 return -1;
1164         }
1165         /* Qm range check */
1166         if (!check_bit(op->ldpc_enc.op_flags, RTE_BBDEV_LDPC_INTERLEAVER_BYPASS) &&
1167                         ((q_m == 0) || ((q_m > 2) && ((q_m % 2) == 1)) || (q_m > 8))) {
1168                 rte_bbdev_log(ERR, "Qm (%u) is out of range", q_m);
1169                 return -1;
1170         }
1171         /* K0 range check */
1172         if (((k0 % z_c) > 0) || (k0 >= n_cb) || ((k0 >= (Kp - 2 * z_c))
1173                         && (k0 < (K - 2 * z_c)))) {
1174                 rte_bbdev_log(ERR, "K0 (%u) is out of range", k0);
1175                 return -1;
1176         }
1177         /* E range check */
1178         if (e <= RTE_MAX(32, z_c)) {
1179                 rte_bbdev_log(ERR, "E is too small %"PRIu32"", e);
1180                 return -1;
1181         }
1182         if ((e > 0xFFFF)) {
1183                 rte_bbdev_log(ERR, "E is too large for N3000 %"PRIu32" > 64k", e);
1184                 return -1;
1185         }
1186         if (q_m > 0) {
1187                 if (e % q_m > 0) {
1188                         rte_bbdev_log(ERR, "E %"PRIu32" not multiple of qm %d", e, q_m);
1189                         return -1;
1190                 }
1191         }
1192         /* Code word in RM range check */
1193         if (k0 > (Kp - 2 * z_c))
1194                 L = k0 + e;
1195         else
1196                 L = k0 + e + n_filler;
1197         Lcb = RTE_MIN(L, n_cb);
1198         if (ldpc_enc->basegraph == 1) {
1199                 if (Lcb <= 25 * z_c)
1200                         cw = 25 * z_c;
1201                 else if (Lcb <= 27 * z_c)
1202                         cw = 27 * z_c;
1203                 else if (Lcb <= 30 * z_c)
1204                         cw = 30 * z_c;
1205                 else if (Lcb <= 33 * z_c)
1206                         cw = 33 * z_c;
1207                 else if (Lcb <= 44 * z_c)
1208                         cw = 44 * z_c;
1209                 else if (Lcb <= 55 * z_c)
1210                         cw = 55 * z_c;
1211                 else
1212                         cw = 66 * z_c;
1213         } else {
1214                 if (Lcb <= 15 * z_c)
1215                         cw = 15 * z_c;
1216                 else if (Lcb <= 20 * z_c)
1217                         cw = 20 * z_c;
1218                 else if (Lcb <= 25 * z_c)
1219                         cw = 25 * z_c;
1220                 else if (Lcb <= 30 * z_c)
1221                         cw = 30 * z_c;
1222                 else
1223                         cw = 50 * z_c;
1224         }
1225         if (n_cb < Kp - 2 * z_c)
1226                 cw_rm = n_cb;
1227         else if ((Kp - 2 * z_c <= n_cb) && (n_cb < K - 2 * z_c))
1228                 cw_rm = Kp - 2 * z_c;
1229         else if ((K - 2 * z_c <= n_cb) && (n_cb < cw))
1230                 cw_rm = n_cb - n_filler;
1231         else
1232                 cw_rm = cw - n_filler;
1233         if (cw_rm <= 32) {
1234                 rte_bbdev_log(ERR,
1235                                 "Invalid Ratematching");
1236                 return -1;
1237         }
1238         return 0;
1239 }
1240
1241 /* Validates LDPC decoder parameters */
1242 static inline int
1243 validate_ldpc_dec_op(struct rte_bbdev_dec_op *op)
1244 {
1245         struct rte_bbdev_op_ldpc_dec *ldpc_dec = &op->ldpc_dec;
1246         if (check_bit(ldpc_dec->op_flags,
1247                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK))
1248                 return 0;
1249         if (ldpc_dec->input.data == NULL) {
1250                 rte_bbdev_log(ERR, "Invalid input pointer");
1251                 return -1;
1252         }
1253         if (ldpc_dec->hard_output.data == NULL) {
1254                 rte_bbdev_log(ERR, "Invalid output pointer");
1255                 return -1;
1256         }
1257         if (ldpc_dec->input.length == 0) {
1258                 rte_bbdev_log(ERR, "input is null");
1259                 return -1;
1260         }
1261         if ((ldpc_dec->basegraph > 2) || (ldpc_dec->basegraph == 0)) {
1262                 rte_bbdev_log(ERR,
1263                                 "BG (%u) is out of range 1 <= value <= 2",
1264                                 ldpc_dec->basegraph);
1265                 return -1;
1266         }
1267         if (ldpc_dec->iter_max == 0) {
1268                 rte_bbdev_log(ERR,
1269                                 "iter_max (%u) is equal to 0",
1270                                 ldpc_dec->iter_max);
1271                 return -1;
1272         }
1273         if (ldpc_dec->rv_index > 3) {
1274                 rte_bbdev_log(ERR,
1275                                 "rv_index (%u) is out of range 0 <= value <= 3",
1276                                 ldpc_dec->rv_index);
1277                 return -1;
1278         }
1279         if (ldpc_dec->code_block_mode > RTE_BBDEV_CODE_BLOCK) {
1280                 rte_bbdev_log(ERR,
1281                                 "code_block_mode (%u) is out of range 0 <= value <= 1",
1282                                 ldpc_dec->code_block_mode);
1283                 return -1;
1284         }
1285         if (check_bit(op->ldpc_dec.op_flags,
1286                         RTE_BBDEV_LDPC_DECODE_BYPASS)) {
1287                 rte_bbdev_log(ERR, "Avoid LDPC Decode bypass");
1288                 return -1;
1289         }
1290         int z_c = ldpc_dec->z_c;
1291         /* Check Zc is valid value */
1292         if ((z_c > 384) || (z_c < 4)) {
1293                 rte_bbdev_log(ERR,
1294                                 "Zc (%u) is out of range",
1295                                 z_c);
1296                 return -1;
1297         }
1298         if (z_c > 256) {
1299                 if ((z_c % 32) != 0) {
1300                         rte_bbdev_log(ERR, "Invalid Zc %d", z_c);
1301                         return -1;
1302                 }
1303         } else if (z_c > 128) {
1304                 if ((z_c % 16) != 0) {
1305                         rte_bbdev_log(ERR, "Invalid Zc %d", z_c);
1306                         return -1;
1307                 }
1308         } else if (z_c > 64) {
1309                 if ((z_c % 8) != 0) {
1310                         rte_bbdev_log(ERR, "Invalid Zc %d", z_c);
1311                         return -1;
1312                 }
1313         } else if (z_c > 32) {
1314                 if ((z_c % 4) != 0) {
1315                         rte_bbdev_log(ERR, "Invalid Zc %d", z_c);
1316                         return -1;
1317                 }
1318         } else if (z_c > 16) {
1319                 if ((z_c % 2) != 0) {
1320                         rte_bbdev_log(ERR, "Invalid Zc %d", z_c);
1321                         return -1;
1322                 }
1323         }
1324
1325         int n_filler = ldpc_dec->n_filler;
1326         int K = (ldpc_dec->basegraph == 1 ? 22 : 10) * ldpc_dec->z_c;
1327         int Kp = K - n_filler;
1328         int q_m = ldpc_dec->q_m;
1329         int n_cb = ldpc_dec->n_cb;
1330         int N = (ldpc_dec->basegraph == 1 ? N_ZC_1 : N_ZC_2) * z_c;
1331         int k0 = get_k0(n_cb, z_c, ldpc_dec->basegraph,
1332                         ldpc_dec->rv_index);
1333         int crc24 = 0;
1334         int32_t L, Lcb, cw, cw_rm;
1335         int32_t e = ldpc_dec->cb_params.e;
1336         if (check_bit(op->ldpc_dec.op_flags,
1337                         RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK))
1338                 crc24 = 24;
1339
1340         if (ldpc_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
1341                 rte_bbdev_log(ERR,
1342                                 "TB mode not supported");
1343                 return -1;
1344         }
1345         /* Enforce HARQ input length */
1346         ldpc_dec->harq_combined_input.length = RTE_MIN((uint32_t) n_cb,
1347                         ldpc_dec->harq_combined_input.length);
1348         if ((ldpc_dec->harq_combined_input.length == 0) &&
1349                         check_bit(ldpc_dec->op_flags,
1350                         RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) {
1351                 rte_bbdev_log(ERR,
1352                                 "HARQ input length (%u) should not be null",
1353                                 ldpc_dec->harq_combined_input.length);
1354                 return -1;
1355         }
1356         if ((ldpc_dec->harq_combined_input.length > 0) &&
1357                         !check_bit(ldpc_dec->op_flags,
1358                         RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) {
1359                 ldpc_dec->harq_combined_input.length = 0;
1360         }
1361
1362         /* K' range check */
1363         if (Kp % 8 > 0) {
1364                 rte_bbdev_log(ERR,
1365                                 "K' not byte aligned %u",
1366                                 Kp);
1367                 return -1;
1368         }
1369         if ((crc24 > 0) && (Kp < 292)) {
1370                 rte_bbdev_log(ERR,
1371                                 "Invalid CRC24 for small block %u",
1372                                 Kp);
1373                 return -1;
1374         }
1375         if (Kp < 24) {
1376                 rte_bbdev_log(ERR,
1377                                 "K' too small %u",
1378                                 Kp);
1379                 return -1;
1380         }
1381         if (n_filler >= (K - 2 * z_c)) {
1382                 rte_bbdev_log(ERR,
1383                                 "K - F invalid %u %u",
1384                                 K, n_filler);
1385                 return -1;
1386         }
1387         /* Ncb range check */
1388         if (n_cb != N) {
1389                 rte_bbdev_log(ERR,
1390                                 "Ncb (%u) is out of range K  %d N %d",
1391                                 n_cb, K, N);
1392                 return -1;
1393         }
1394         /* Qm range check */
1395         if (!check_bit(op->ldpc_dec.op_flags,
1396                         RTE_BBDEV_LDPC_INTERLEAVER_BYPASS) &&
1397                         ((q_m == 0) || ((q_m > 2) && ((q_m % 2) == 1))
1398                         || (q_m > 8))) {
1399                 rte_bbdev_log(ERR,
1400                                 "Qm (%u) is out of range",
1401                                 q_m);
1402                 return -1;
1403         }
1404         /* K0 range check */
1405         if (((k0 % z_c) > 0) || (k0 >= n_cb) || ((k0 >= (Kp - 2 * z_c))
1406                         && (k0 < (K - 2 * z_c)))) {
1407                 rte_bbdev_log(ERR,
1408                                 "K0 (%u) is out of range",
1409                                 k0);
1410                 return -1;
1411         }
1412         /* E range check */
1413         if (e <= RTE_MAX(32, z_c)) {
1414                 rte_bbdev_log(ERR,
1415                                 "E is too small");
1416                 return -1;
1417         }
1418         if ((e > 0xFFFF)) {
1419                 rte_bbdev_log(ERR,
1420                                 "E is too large");
1421                 return -1;
1422         }
1423         if (q_m > 0) {
1424                 if (e % q_m > 0) {
1425                         rte_bbdev_log(ERR,
1426                                         "E not multiple of qm %d", q_m);
1427                         return -1;
1428                 }
1429         }
1430         /* Code word in RM range check */
1431         if (k0 > (Kp - 2 * z_c))
1432                 L = k0 + e;
1433         else
1434                 L = k0 + e + n_filler;
1435         Lcb = RTE_MIN(n_cb, RTE_MAX(L,
1436                         (int32_t) ldpc_dec->harq_combined_input.length));
1437         if (ldpc_dec->basegraph == 1) {
1438                 if (Lcb <= 25 * z_c)
1439                         cw = 25 * z_c;
1440                 else if (Lcb <= 27 * z_c)
1441                         cw = 27 * z_c;
1442                 else if (Lcb <= 30 * z_c)
1443                         cw = 30 * z_c;
1444                 else if (Lcb <= 33 * z_c)
1445                         cw = 33 * z_c;
1446                 else if (Lcb <= 44 * z_c)
1447                         cw = 44 * z_c;
1448                 else if (Lcb <= 55 * z_c)
1449                         cw = 55 * z_c;
1450                 else
1451                         cw = 66 * z_c;
1452         } else {
1453                 if (Lcb <= 15 * z_c)
1454                         cw = 15 * z_c;
1455                 else if (Lcb <= 20 * z_c)
1456                         cw = 20 * z_c;
1457                 else if (Lcb <= 25 * z_c)
1458                         cw = 25 * z_c;
1459                 else if (Lcb <= 30 * z_c)
1460                         cw = 30 * z_c;
1461                 else
1462                         cw = 50 * z_c;
1463         }
1464         cw_rm = cw - n_filler;
1465         if (cw_rm <= 32) {
1466                 rte_bbdev_log(ERR,
1467                                 "Invalid Ratematching");
1468                 return -1;
1469         }
1470         return 0;
1471 }
1472
1473 static inline char *
1474 mbuf_append(struct rte_mbuf *m_head, struct rte_mbuf *m, uint16_t len)
1475 {
1476         if (unlikely(len > rte_pktmbuf_tailroom(m)))
1477                 return NULL;
1478
1479         char *tail = (char *)m->buf_addr + m->data_off + m->data_len;
1480         m->data_len = (uint16_t)(m->data_len + len);
1481         m_head->pkt_len  = (m_head->pkt_len + len);
1482         return tail;
1483 }
1484
1485 static inline void
1486 fpga_mutex_acquisition(struct fpga_queue *q)
1487 {
1488         uint32_t mutex_ctrl, mutex_read, cnt = 0;
1489         /* Assign a unique id for the duration of the DDR access */
1490         q->ddr_mutex_uuid = rand();
1491         /* Request and wait for acquisition of the mutex */
1492         mutex_ctrl = (q->ddr_mutex_uuid << 16) + 1;
1493         do {
1494                 if (cnt > 0)
1495                         usleep(FPGA_TIMEOUT_CHECK_INTERVAL);
1496                 rte_bbdev_log_debug("Acquiring Mutex for %x\n",
1497                                 q->ddr_mutex_uuid);
1498                 fpga_reg_write_32(q->d->mmio_base,
1499                                 FPGA_5GNR_FEC_MUTEX,
1500                                 mutex_ctrl);
1501                 mutex_read = fpga_reg_read_32(q->d->mmio_base,
1502                                 FPGA_5GNR_FEC_MUTEX);
1503                 rte_bbdev_log_debug("Mutex %x cnt %d owner %x\n",
1504                                 mutex_read, cnt, q->ddr_mutex_uuid);
1505                 cnt++;
1506         } while ((mutex_read >> 16) != q->ddr_mutex_uuid);
1507 }
1508
1509 static inline void
1510 fpga_mutex_free(struct fpga_queue *q)
1511 {
1512         uint32_t mutex_ctrl = q->ddr_mutex_uuid << 16;
1513         fpga_reg_write_32(q->d->mmio_base,
1514                         FPGA_5GNR_FEC_MUTEX,
1515                         mutex_ctrl);
1516 }
1517
1518 static inline int
1519 fpga_harq_write_loopback(struct fpga_queue *q,
1520                 struct rte_mbuf *harq_input, uint16_t harq_in_length,
1521                 uint32_t harq_in_offset, uint32_t harq_out_offset)
1522 {
1523         fpga_mutex_acquisition(q);
1524         uint32_t out_offset = harq_out_offset;
1525         uint32_t in_offset = harq_in_offset;
1526         uint32_t left_length = harq_in_length;
1527         uint32_t reg_32, increment = 0;
1528         uint64_t *input = NULL;
1529         uint32_t last_transaction = left_length
1530                         % FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
1531         uint64_t last_word;
1532
1533         if (last_transaction > 0)
1534                 left_length -= last_transaction;
1535
1536         /*
1537          * Get HARQ buffer size for each VF/PF: When 0x00, there is no
1538          * available DDR space for the corresponding VF/PF.
1539          */
1540         reg_32 = fpga_reg_read_32(q->d->mmio_base,
1541                         FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS);
1542         if (reg_32 < harq_in_length) {
1543                 left_length = reg_32;
1544                 rte_bbdev_log(ERR, "HARQ in length > HARQ buffer size\n");
1545         }
1546
1547         input = (uint64_t *)rte_pktmbuf_mtod_offset(harq_input,
1548                         uint8_t *, in_offset);
1549
1550         while (left_length > 0) {
1551                 if (fpga_reg_read_8(q->d->mmio_base,
1552                                 FPGA_5GNR_FEC_DDR4_ADDR_RDY_REGS) ==  1) {
1553                         fpga_reg_write_32(q->d->mmio_base,
1554                                         FPGA_5GNR_FEC_DDR4_WR_ADDR_REGS,
1555                                         out_offset);
1556                         fpga_reg_write_64(q->d->mmio_base,
1557                                         FPGA_5GNR_FEC_DDR4_WR_DATA_REGS,
1558                                         input[increment]);
1559                         left_length -= FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
1560                         out_offset += FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
1561                         increment++;
1562                         fpga_reg_write_8(q->d->mmio_base,
1563                                         FPGA_5GNR_FEC_DDR4_WR_DONE_REGS, 1);
1564                 }
1565         }
1566         while (last_transaction > 0) {
1567                 if (fpga_reg_read_8(q->d->mmio_base,
1568                                 FPGA_5GNR_FEC_DDR4_ADDR_RDY_REGS) ==  1) {
1569                         fpga_reg_write_32(q->d->mmio_base,
1570                                         FPGA_5GNR_FEC_DDR4_WR_ADDR_REGS,
1571                                         out_offset);
1572                         last_word = input[increment];
1573                         last_word &= (uint64_t)(1 << (last_transaction * 4))
1574                                         - 1;
1575                         fpga_reg_write_64(q->d->mmio_base,
1576                                         FPGA_5GNR_FEC_DDR4_WR_DATA_REGS,
1577                                         last_word);
1578                         fpga_reg_write_8(q->d->mmio_base,
1579                                         FPGA_5GNR_FEC_DDR4_WR_DONE_REGS, 1);
1580                         last_transaction = 0;
1581                 }
1582         }
1583         fpga_mutex_free(q);
1584         return 1;
1585 }
1586
1587 static inline int
1588 fpga_harq_read_loopback(struct fpga_queue *q,
1589                 struct rte_mbuf *harq_output, uint16_t harq_in_length,
1590                 uint32_t harq_in_offset, uint32_t harq_out_offset)
1591 {
1592         fpga_mutex_acquisition(q);
1593         uint32_t left_length, in_offset = harq_in_offset;
1594         uint64_t reg;
1595         uint32_t increment = 0;
1596         uint64_t *input = NULL;
1597         uint32_t last_transaction = harq_in_length
1598                         % FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
1599
1600         if (last_transaction > 0)
1601                 harq_in_length += (8 - last_transaction);
1602
1603         reg = fpga_reg_read_32(q->d->mmio_base,
1604                         FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS);
1605         if (reg < harq_in_length) {
1606                 harq_in_length = reg;
1607                 rte_bbdev_log(ERR, "HARQ in length > HARQ buffer size\n");
1608         }
1609
1610         if (!mbuf_append(harq_output, harq_output, harq_in_length)) {
1611                 rte_bbdev_log(ERR, "HARQ output buffer warning %d %d\n",
1612                                 harq_output->buf_len -
1613                                 rte_pktmbuf_headroom(harq_output),
1614                                 harq_in_length);
1615                 harq_in_length = harq_output->buf_len -
1616                                 rte_pktmbuf_headroom(harq_output);
1617                 if (!mbuf_append(harq_output, harq_output, harq_in_length)) {
1618                         rte_bbdev_log(ERR, "HARQ output buffer issue %d %d\n",
1619                                         harq_output->buf_len, harq_in_length);
1620                         return -1;
1621                 }
1622         }
1623         left_length = harq_in_length;
1624
1625         input = (uint64_t *)rte_pktmbuf_mtod_offset(harq_output,
1626                         uint8_t *, harq_out_offset);
1627
1628         while (left_length > 0) {
1629                 fpga_reg_write_32(q->d->mmio_base,
1630                         FPGA_5GNR_FEC_DDR4_RD_ADDR_REGS, in_offset);
1631                 fpga_reg_write_8(q->d->mmio_base,
1632                                 FPGA_5GNR_FEC_DDR4_RD_DONE_REGS, 1);
1633                 reg = fpga_reg_read_8(q->d->mmio_base,
1634                         FPGA_5GNR_FEC_DDR4_RD_RDY_REGS);
1635                 while (reg != 1) {
1636                         reg = fpga_reg_read_8(q->d->mmio_base,
1637                                 FPGA_5GNR_FEC_DDR4_RD_RDY_REGS);
1638                         if (reg == FPGA_DDR_OVERFLOW) {
1639                                 rte_bbdev_log(ERR,
1640                                                 "Read address is overflow!\n");
1641                                 return -1;
1642                         }
1643                 }
1644                 input[increment] = fpga_reg_read_64(q->d->mmio_base,
1645                         FPGA_5GNR_FEC_DDR4_RD_DATA_REGS);
1646                 left_length -= FPGA_5GNR_FEC_DDR_RD_DATA_LEN_IN_BYTES;
1647                 in_offset += FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
1648                 increment++;
1649                 fpga_reg_write_8(q->d->mmio_base,
1650                                 FPGA_5GNR_FEC_DDR4_RD_DONE_REGS, 0);
1651         }
1652         fpga_mutex_free(q);
1653         return 1;
1654 }
1655
1656 static inline int
1657 enqueue_ldpc_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op *op,
1658                 uint16_t desc_offset)
1659 {
1660         union fpga_dma_desc *desc;
1661         int ret;
1662         uint8_t c, crc24_bits = 0;
1663         struct rte_bbdev_op_ldpc_enc *enc = &op->ldpc_enc;
1664         uint16_t in_offset = enc->input.offset;
1665         uint16_t out_offset = enc->output.offset;
1666         struct rte_mbuf *m_in = enc->input.data;
1667         struct rte_mbuf *m_out = enc->output.data;
1668         struct rte_mbuf *m_out_head = enc->output.data;
1669         uint32_t in_length, out_length, e;
1670         uint16_t total_left = enc->input.length;
1671         uint16_t ring_offset;
1672         uint16_t K, k_;
1673
1674
1675         if (validate_ldpc_enc_op(op) == -1) {
1676                 rte_bbdev_log(ERR, "LDPC encoder validation rejected");
1677                 return -EINVAL;
1678         }
1679
1680         /* Clear op status */
1681         op->status = 0;
1682
1683         if (m_in == NULL || m_out == NULL) {
1684                 rte_bbdev_log(ERR, "Invalid mbuf pointer");
1685                 op->status = 1 << RTE_BBDEV_DATA_ERROR;
1686                 return -EINVAL;
1687         }
1688
1689         if (enc->op_flags & RTE_BBDEV_LDPC_CRC_24B_ATTACH)
1690                 crc24_bits = 24;
1691
1692         if (enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) {
1693                 /* For Transport Block mode */
1694                 /* FIXME */
1695                 c = enc->tb_params.c;
1696                 e = enc->tb_params.ea;
1697         } else { /* For Code Block mode */
1698                 c = 1;
1699                 e = enc->cb_params.e;
1700         }
1701
1702         /* Update total_left */
1703         K = (enc->basegraph == 1 ? 22 : 10) * enc->z_c;
1704         k_ = K - enc->n_filler;
1705         in_length = (k_ - crc24_bits) >> 3;
1706         out_length = (e + 7) >> 3;
1707
1708         total_left = rte_pktmbuf_data_len(m_in) - in_offset;
1709
1710         /* Update offsets */
1711         if (total_left != in_length) {
1712                 op->status |= 1 << RTE_BBDEV_DATA_ERROR;
1713                 rte_bbdev_log(ERR,
1714                                 "Mismatch between mbuf length and included CBs sizes %d",
1715                                 total_left);
1716         }
1717
1718         mbuf_append(m_out_head, m_out, out_length);
1719
1720         /* Offset into the ring */
1721         ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask);
1722         /* Setup DMA Descriptor */
1723         desc = q->ring_addr + ring_offset;
1724
1725         ret = fpga_dma_desc_te_fill(op, &desc->enc_req, m_in, m_out,
1726                         k_, e, in_offset, out_offset, ring_offset, c);
1727         if (unlikely(ret < 0))
1728                 return ret;
1729
1730         /* Update lengths */
1731         total_left -= in_length;
1732         op->ldpc_enc.output.length += out_length;
1733
1734         if (total_left > 0) {
1735                 rte_bbdev_log(ERR,
1736                         "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u",
1737                                 total_left, in_length);
1738                 return -1;
1739         }
1740
1741 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1742         print_dma_enc_desc_debug_info(desc);
1743 #endif
1744         return 1;
1745 }
1746
1747 static inline int
1748 enqueue_ldpc_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op *op,
1749                 uint16_t desc_offset)
1750 {
1751         union fpga_dma_desc *desc;
1752         int ret;
1753         uint16_t ring_offset;
1754         uint8_t c;
1755         uint16_t e, in_length, out_length, k0, l, seg_total_left, sys_cols;
1756         uint16_t K, parity_offset, harq_in_length = 0, harq_out_length = 0;
1757         uint16_t crc24_overlap = 0;
1758         struct rte_bbdev_op_ldpc_dec *dec = &op->ldpc_dec;
1759         struct rte_mbuf *m_in = dec->input.data;
1760         struct rte_mbuf *m_out = dec->hard_output.data;
1761         struct rte_mbuf *m_out_head = dec->hard_output.data;
1762         uint16_t in_offset = dec->input.offset;
1763         uint16_t out_offset = dec->hard_output.offset;
1764         uint32_t harq_offset = 0;
1765
1766         if (validate_ldpc_dec_op(op) == -1) {
1767                 rte_bbdev_log(ERR, "LDPC decoder validation rejected");
1768                 return -EINVAL;
1769         }
1770
1771         /* Clear op status */
1772         op->status = 0;
1773
1774         /* Setup DMA Descriptor */
1775         ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask);
1776         desc = q->ring_addr + ring_offset;
1777
1778         if (check_bit(dec->op_flags,
1779                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1780                 struct rte_mbuf *harq_in = dec->harq_combined_input.data;
1781                 struct rte_mbuf *harq_out = dec->harq_combined_output.data;
1782                 harq_in_length = dec->harq_combined_input.length;
1783                 uint32_t harq_in_offset = dec->harq_combined_input.offset;
1784                 uint32_t harq_out_offset = dec->harq_combined_output.offset;
1785
1786                 if (check_bit(dec->op_flags,
1787                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE
1788                                 )) {
1789                         ret = fpga_harq_write_loopback(q, harq_in,
1790                                         harq_in_length, harq_in_offset,
1791                                         harq_out_offset);
1792                 } else if (check_bit(dec->op_flags,
1793                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE
1794                                 )) {
1795                         ret = fpga_harq_read_loopback(q, harq_out,
1796                                 harq_in_length, harq_in_offset,
1797                                 harq_out_offset);
1798                         dec->harq_combined_output.length = harq_in_length;
1799                 } else {
1800                         rte_bbdev_log(ERR, "OP flag Err!");
1801                         ret = -1;
1802                 }
1803                 /* Set descriptor for dequeue */
1804                 desc->dec_req.done = 1;
1805                 desc->dec_req.error = 0;
1806                 desc->dec_req.op_addr = op;
1807                 desc->dec_req.cbs_in_op = 1;
1808                 /* Mark this dummy descriptor to be dropped by HW */
1809                 desc->dec_req.desc_idx = (ring_offset + 1)
1810                                 & q->sw_ring_wrap_mask;
1811                 return ret; /* Error or number of CB */
1812         }
1813
1814         if (m_in == NULL || m_out == NULL) {
1815                 rte_bbdev_log(ERR, "Invalid mbuf pointer");
1816                 op->status = 1 << RTE_BBDEV_DATA_ERROR;
1817                 return -1;
1818         }
1819
1820         c = 1;
1821         e = dec->cb_params.e;
1822
1823         if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP))
1824                 crc24_overlap = 24;
1825
1826         sys_cols = (dec->basegraph == 1) ? 22 : 10;
1827         K = sys_cols * dec->z_c;
1828         parity_offset = K - 2 * dec->z_c;
1829
1830         out_length = ((K - crc24_overlap - dec->n_filler) >> 3);
1831         in_length = e;
1832         seg_total_left = dec->input.length;
1833
1834         if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) {
1835                 harq_in_length = RTE_MIN(dec->harq_combined_input.length,
1836                                 (uint32_t)dec->n_cb);
1837         }
1838
1839         if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE)) {
1840                 k0 = get_k0(dec->n_cb, dec->z_c,
1841                                 dec->basegraph, dec->rv_index);
1842                 if (k0 > parity_offset)
1843                         l = k0 + e;
1844                 else
1845                         l = k0 + e + dec->n_filler;
1846                 harq_out_length = RTE_MIN(RTE_MAX(harq_in_length, l),
1847                                 dec->n_cb);
1848                 dec->harq_combined_output.length = harq_out_length;
1849         }
1850
1851         mbuf_append(m_out_head, m_out, out_length);
1852         if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE))
1853                 harq_offset = dec->harq_combined_input.offset;
1854         else if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE))
1855                 harq_offset = dec->harq_combined_output.offset;
1856
1857         if ((harq_offset & 0x3FF) > 0) {
1858                 rte_bbdev_log(ERR, "Invalid HARQ offset %d", harq_offset);
1859                 op->status = 1 << RTE_BBDEV_DATA_ERROR;
1860                 return -1;
1861         }
1862
1863         ret = fpga_dma_desc_ld_fill(op, &desc->dec_req, m_in, m_out,
1864                 harq_in_length, in_offset, out_offset, harq_offset,
1865                 ring_offset, c);
1866         if (unlikely(ret < 0))
1867                 return ret;
1868         /* Update lengths */
1869         seg_total_left -= in_length;
1870         op->ldpc_dec.hard_output.length += out_length;
1871         if (seg_total_left > 0) {
1872                 rte_bbdev_log(ERR,
1873                                 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u",
1874                                 seg_total_left, in_length);
1875                 return -1;
1876         }
1877
1878 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1879         print_dma_dec_desc_debug_info(desc);
1880 #endif
1881
1882         return 1;
1883 }
1884
1885 static uint16_t
1886 fpga_enqueue_ldpc_enc(struct rte_bbdev_queue_data *q_data,
1887                 struct rte_bbdev_enc_op **ops, uint16_t num)
1888 {
1889         uint16_t i, total_enqueued_cbs = 0;
1890         int32_t avail;
1891         int enqueued_cbs;
1892         struct fpga_queue *q = q_data->queue_private;
1893         union fpga_dma_desc *desc;
1894
1895         /* Check if queue is not full */
1896         if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) ==
1897                         q->head_free_desc))
1898                 return 0;
1899
1900         /* Calculates available space */
1901         avail = (q->head_free_desc > q->tail) ?
1902                 q->head_free_desc - q->tail - 1 :
1903                 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1;
1904
1905         for (i = 0; i < num; ++i) {
1906
1907                 /* Check if there is available space for further
1908                  * processing
1909                  */
1910                 if (unlikely(avail - 1 < 0))
1911                         break;
1912                 avail -= 1;
1913                 enqueued_cbs = enqueue_ldpc_enc_one_op_cb(q, ops[i],
1914                                 total_enqueued_cbs);
1915
1916                 if (enqueued_cbs < 0)
1917                         break;
1918
1919                 total_enqueued_cbs += enqueued_cbs;
1920
1921                 rte_bbdev_log_debug("enqueuing enc ops [%d/%d] | head %d | tail %d",
1922                                 total_enqueued_cbs, num,
1923                                 q->head_free_desc, q->tail);
1924         }
1925
1926         /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt
1927          * only when all previous CBs were already processed.
1928          */
1929         desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1)
1930                         & q->sw_ring_wrap_mask);
1931         desc->enc_req.irq_en = q->irq_enable;
1932
1933         fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats);
1934
1935         /* Update stats */
1936         q_data->queue_stats.enqueued_count += i;
1937         q_data->queue_stats.enqueue_err_count += num - i;
1938
1939         return i;
1940 }
1941
1942 static uint16_t
1943 fpga_enqueue_ldpc_dec(struct rte_bbdev_queue_data *q_data,
1944                 struct rte_bbdev_dec_op **ops, uint16_t num)
1945 {
1946         uint16_t i, total_enqueued_cbs = 0;
1947         int32_t avail;
1948         int enqueued_cbs;
1949         struct fpga_queue *q = q_data->queue_private;
1950         union fpga_dma_desc *desc;
1951
1952         /* Check if queue is not full */
1953         if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) ==
1954                         q->head_free_desc))
1955                 return 0;
1956
1957         /* Calculates available space */
1958         avail = (q->head_free_desc > q->tail) ?
1959                 q->head_free_desc - q->tail - 1 :
1960                 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1;
1961
1962         for (i = 0; i < num; ++i) {
1963
1964                 /* Check if there is available space for further
1965                  * processing
1966                  */
1967                 if (unlikely(avail - 1 < 0))
1968                         break;
1969                 avail -= 1;
1970                 enqueued_cbs = enqueue_ldpc_dec_one_op_cb(q, ops[i],
1971                                 total_enqueued_cbs);
1972
1973                 if (enqueued_cbs < 0)
1974                         break;
1975
1976                 total_enqueued_cbs += enqueued_cbs;
1977
1978                 rte_bbdev_log_debug("enqueuing dec ops [%d/%d] | head %d | tail %d",
1979                                 total_enqueued_cbs, num,
1980                                 q->head_free_desc, q->tail);
1981         }
1982
1983         /* Update stats */
1984         q_data->queue_stats.enqueued_count += i;
1985         q_data->queue_stats.enqueue_err_count += num - i;
1986
1987         /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt
1988          * only when all previous CBs were already processed.
1989          */
1990         desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1)
1991                         & q->sw_ring_wrap_mask);
1992         desc->enc_req.irq_en = q->irq_enable;
1993         fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats);
1994         return i;
1995 }
1996
1997
1998 static inline int
1999 dequeue_ldpc_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op **op,
2000                 uint16_t desc_offset)
2001 {
2002         union fpga_dma_desc *desc;
2003         int desc_error;
2004         /* Set current desc */
2005         desc = q->ring_addr + ((q->head_free_desc + desc_offset)
2006                         & q->sw_ring_wrap_mask);
2007
2008         /*check if done */
2009         if (desc->enc_req.done == 0)
2010                 return -1;
2011
2012         /* make sure the response is read atomically */
2013         rte_smp_rmb();
2014
2015         rte_bbdev_log_debug("DMA response desc %p", desc);
2016
2017 #ifdef RTE_LIBRTE_BBDEV_DEBUG
2018         print_dma_enc_desc_debug_info(desc);
2019 #endif
2020
2021         *op = desc->enc_req.op_addr;
2022         /* Check the descriptor error field, return 1 on error */
2023         desc_error = check_desc_error(desc->enc_req.error);
2024         (*op)->status = desc_error << RTE_BBDEV_DATA_ERROR;
2025
2026         return 1;
2027 }
2028
2029
2030 static inline int
2031 dequeue_ldpc_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op **op,
2032                 uint16_t desc_offset)
2033 {
2034         union fpga_dma_desc *desc;
2035         int desc_error;
2036         /* Set descriptor */
2037         desc = q->ring_addr + ((q->head_free_desc + desc_offset)
2038                         & q->sw_ring_wrap_mask);
2039
2040         /* Verify done bit is set */
2041         if (desc->dec_req.done == 0)
2042                 return -1;
2043
2044         /* make sure the response is read atomically */
2045         rte_smp_rmb();
2046
2047 #ifdef RTE_LIBRTE_BBDEV_DEBUG
2048         print_dma_dec_desc_debug_info(desc);
2049 #endif
2050
2051         *op = desc->dec_req.op_addr;
2052
2053         if (check_bit((*op)->ldpc_dec.op_flags,
2054                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
2055                 (*op)->status = 0;
2056                 return 1;
2057         }
2058
2059         /* FPGA reports iterations based on round-up minus 1 */
2060         (*op)->ldpc_dec.iter_count = desc->dec_req.iter + 1;
2061         /* CRC Check criteria */
2062         if (desc->dec_req.crc24b_ind && !(desc->dec_req.crcb_pass))
2063                 (*op)->status = 1 << RTE_BBDEV_CRC_ERROR;
2064         /* et_pass = 0 when decoder fails */
2065         (*op)->status |= !(desc->dec_req.et_pass) << RTE_BBDEV_SYNDROME_ERROR;
2066         /* Check the descriptor error field, return 1 on error */
2067         desc_error = check_desc_error(desc->dec_req.error);
2068         (*op)->status |= desc_error << RTE_BBDEV_DATA_ERROR;
2069         return 1;
2070 }
2071
2072 static uint16_t
2073 fpga_dequeue_ldpc_enc(struct rte_bbdev_queue_data *q_data,
2074                 struct rte_bbdev_enc_op **ops, uint16_t num)
2075 {
2076         struct fpga_queue *q = q_data->queue_private;
2077         uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask;
2078         uint16_t i;
2079         uint16_t dequeued_cbs = 0;
2080         int ret;
2081
2082         for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) {
2083                 ret = dequeue_ldpc_enc_one_op_cb(q, &ops[i], dequeued_cbs);
2084
2085                 if (ret < 0)
2086                         break;
2087
2088                 dequeued_cbs += ret;
2089
2090                 rte_bbdev_log_debug("dequeuing enc ops [%d/%d] | head %d | tail %d",
2091                                 dequeued_cbs, num, q->head_free_desc, q->tail);
2092         }
2093
2094         /* Update head */
2095         q->head_free_desc = (q->head_free_desc + dequeued_cbs) &
2096                         q->sw_ring_wrap_mask;
2097
2098         /* Update stats */
2099         q_data->queue_stats.dequeued_count += i;
2100
2101         return i;
2102 }
2103
2104 static uint16_t
2105 fpga_dequeue_ldpc_dec(struct rte_bbdev_queue_data *q_data,
2106                 struct rte_bbdev_dec_op **ops, uint16_t num)
2107 {
2108         struct fpga_queue *q = q_data->queue_private;
2109         uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask;
2110         uint16_t i;
2111         uint16_t dequeued_cbs = 0;
2112         int ret;
2113
2114         for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) {
2115                 ret = dequeue_ldpc_dec_one_op_cb(q, &ops[i], dequeued_cbs);
2116
2117                 if (ret < 0)
2118                         break;
2119
2120                 dequeued_cbs += ret;
2121
2122                 rte_bbdev_log_debug("dequeuing dec ops [%d/%d] | head %d | tail %d",
2123                                 dequeued_cbs, num, q->head_free_desc, q->tail);
2124         }
2125
2126         /* Update head */
2127         q->head_free_desc = (q->head_free_desc + dequeued_cbs) &
2128                         q->sw_ring_wrap_mask;
2129
2130         /* Update stats */
2131         q_data->queue_stats.dequeued_count += i;
2132
2133         return i;
2134 }
2135
2136
2137 /* Initialization Function */
2138 static void
2139 fpga_5gnr_fec_init(struct rte_bbdev *dev, struct rte_pci_driver *drv)
2140 {
2141         struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device);
2142
2143         dev->dev_ops = &fpga_ops;
2144         dev->enqueue_ldpc_enc_ops = fpga_enqueue_ldpc_enc;
2145         dev->enqueue_ldpc_dec_ops = fpga_enqueue_ldpc_dec;
2146         dev->dequeue_ldpc_enc_ops = fpga_dequeue_ldpc_enc;
2147         dev->dequeue_ldpc_dec_ops = fpga_dequeue_ldpc_dec;
2148
2149         ((struct fpga_5gnr_fec_device *) dev->data->dev_private)->pf_device =
2150                         !strcmp(drv->driver.name,
2151                                         RTE_STR(FPGA_5GNR_FEC_PF_DRIVER_NAME));
2152         ((struct fpga_5gnr_fec_device *) dev->data->dev_private)->mmio_base =
2153                         pci_dev->mem_resource[0].addr;
2154
2155         rte_bbdev_log_debug(
2156                         "Init device %s [%s] @ virtaddr %p phyaddr %#"PRIx64,
2157                         drv->driver.name, dev->data->name,
2158                         (void *)pci_dev->mem_resource[0].addr,
2159                         pci_dev->mem_resource[0].phys_addr);
2160 }
2161
2162 static int
2163 fpga_5gnr_fec_probe(struct rte_pci_driver *pci_drv,
2164         struct rte_pci_device *pci_dev)
2165 {
2166         struct rte_bbdev *bbdev = NULL;
2167         char dev_name[RTE_BBDEV_NAME_MAX_LEN];
2168
2169         if (pci_dev == NULL) {
2170                 rte_bbdev_log(ERR, "NULL PCI device");
2171                 return -EINVAL;
2172         }
2173
2174         rte_pci_device_name(&pci_dev->addr, dev_name, sizeof(dev_name));
2175
2176         /* Allocate memory to be used privately by drivers */
2177         bbdev = rte_bbdev_allocate(pci_dev->device.name);
2178         if (bbdev == NULL)
2179                 return -ENODEV;
2180
2181         /* allocate device private memory */
2182         bbdev->data->dev_private = rte_zmalloc_socket(dev_name,
2183                         sizeof(struct fpga_5gnr_fec_device),
2184                         RTE_CACHE_LINE_SIZE,
2185                         pci_dev->device.numa_node);
2186
2187         if (bbdev->data->dev_private == NULL) {
2188                 rte_bbdev_log(CRIT,
2189                                 "Allocate of %zu bytes for device \"%s\" failed",
2190                                 sizeof(struct fpga_5gnr_fec_device), dev_name);
2191                                 rte_bbdev_release(bbdev);
2192                         return -ENOMEM;
2193         }
2194
2195         /* Fill HW specific part of device structure */
2196         bbdev->device = &pci_dev->device;
2197         bbdev->intr_handle = pci_dev->intr_handle;
2198         bbdev->data->socket_id = pci_dev->device.numa_node;
2199
2200         /* Invoke FEC FPGA device initialization function */
2201         fpga_5gnr_fec_init(bbdev, pci_drv);
2202
2203         rte_bbdev_log_debug("bbdev id = %u [%s]",
2204                         bbdev->data->dev_id, dev_name);
2205
2206         struct fpga_5gnr_fec_device *d = bbdev->data->dev_private;
2207         uint32_t version_id = fpga_reg_read_32(d->mmio_base,
2208                         FPGA_5GNR_FEC_VERSION_ID);
2209         rte_bbdev_log(INFO, "FEC FPGA RTL v%u.%u",
2210                 ((uint16_t)(version_id >> 16)), ((uint16_t)version_id));
2211
2212 #ifdef RTE_LIBRTE_BBDEV_DEBUG
2213         if (!strcmp(pci_drv->driver.name,
2214                         RTE_STR(FPGA_5GNR_FEC_PF_DRIVER_NAME)))
2215                 print_static_reg_debug_info(d->mmio_base);
2216 #endif
2217         return 0;
2218 }
2219
2220 static int
2221 fpga_5gnr_fec_remove(struct rte_pci_device *pci_dev)
2222 {
2223         struct rte_bbdev *bbdev;
2224         int ret;
2225         uint8_t dev_id;
2226
2227         if (pci_dev == NULL)
2228                 return -EINVAL;
2229
2230         /* Find device */
2231         bbdev = rte_bbdev_get_named_dev(pci_dev->device.name);
2232         if (bbdev == NULL) {
2233                 rte_bbdev_log(CRIT,
2234                                 "Couldn't find HW dev \"%s\" to uninitialise it",
2235                                 pci_dev->device.name);
2236                 return -ENODEV;
2237         }
2238         dev_id = bbdev->data->dev_id;
2239
2240         /* free device private memory before close */
2241         rte_free(bbdev->data->dev_private);
2242
2243         /* Close device */
2244         ret = rte_bbdev_close(dev_id);
2245         if (ret < 0)
2246                 rte_bbdev_log(ERR,
2247                                 "Device %i failed to close during uninit: %i",
2248                                 dev_id, ret);
2249
2250         /* release bbdev from library */
2251         ret = rte_bbdev_release(bbdev);
2252         if (ret)
2253                 rte_bbdev_log(ERR, "Device %i failed to uninit: %i", dev_id,
2254                                 ret);
2255
2256         rte_bbdev_log_debug("Destroyed bbdev = %u", dev_id);
2257
2258         return 0;
2259 }
2260
2261 static inline void
2262 set_default_fpga_conf(struct rte_fpga_5gnr_fec_conf *def_conf)
2263 {
2264         /* clear default configuration before initialization */
2265         memset(def_conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf));
2266         /* Set pf mode to true */
2267         def_conf->pf_mode_en = true;
2268
2269         /* Set ratio between UL and DL to 1:1 (unit of weight is 3 CBs) */
2270         def_conf->ul_bandwidth = 3;
2271         def_conf->dl_bandwidth = 3;
2272
2273         /* Set Load Balance Factor to 64 */
2274         def_conf->dl_load_balance = 64;
2275         def_conf->ul_load_balance = 64;
2276 }
2277
2278 /* Initial configuration of FPGA 5GNR FEC device */
2279 int
2280 rte_fpga_5gnr_fec_configure(const char *dev_name,
2281                 const struct rte_fpga_5gnr_fec_conf *conf)
2282 {
2283         uint32_t payload_32, address;
2284         uint16_t payload_16;
2285         uint8_t payload_8;
2286         uint16_t q_id, vf_id, total_q_id, total_ul_q_id, total_dl_q_id;
2287         struct rte_bbdev *bbdev = rte_bbdev_get_named_dev(dev_name);
2288         struct rte_fpga_5gnr_fec_conf def_conf;
2289
2290         if (bbdev == NULL) {
2291                 rte_bbdev_log(ERR,
2292                                 "Invalid dev_name (%s), or device is not yet initialised",
2293                                 dev_name);
2294                 return -ENODEV;
2295         }
2296
2297         struct fpga_5gnr_fec_device *d = bbdev->data->dev_private;
2298
2299         if (conf == NULL) {
2300                 rte_bbdev_log(ERR,
2301                                 "FPGA Configuration was not provided. Default configuration will be loaded.");
2302                 set_default_fpga_conf(&def_conf);
2303                 conf = &def_conf;
2304         }
2305
2306         /*
2307          * Configure UL:DL ratio.
2308          * [7:0]: UL weight
2309          * [15:8]: DL weight
2310          */
2311         payload_16 = (conf->dl_bandwidth << 8) | conf->ul_bandwidth;
2312         address = FPGA_5GNR_FEC_CONFIGURATION;
2313         fpga_reg_write_16(d->mmio_base, address, payload_16);
2314
2315         /* Clear all queues registers */
2316         payload_32 = FPGA_INVALID_HW_QUEUE_ID;
2317         for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) {
2318                 address = (q_id << 2) + FPGA_5GNR_FEC_QUEUE_MAP;
2319                 fpga_reg_write_32(d->mmio_base, address, payload_32);
2320         }
2321
2322         /*
2323          * If PF mode is enabled allocate all queues for PF only.
2324          *
2325          * For VF mode each VF can have different number of UL and DL queues.
2326          * Total number of queues to configure cannot exceed FPGA
2327          * capabilities - 64 queues - 32 queues for UL and 32 queues for DL.
2328          * Queues mapping is done according to configuration:
2329          *
2330          * UL queues:
2331          * |                Q_ID              | VF_ID |
2332          * |                 0                |   0   |
2333          * |                ...               |   0   |
2334          * | conf->vf_dl_queues_number[0] - 1 |   0   |
2335          * | conf->vf_dl_queues_number[0]     |   1   |
2336          * |                ...               |   1   |
2337          * | conf->vf_dl_queues_number[1] - 1 |   1   |
2338          * |                ...               |  ...  |
2339          * | conf->vf_dl_queues_number[7] - 1 |   7   |
2340          *
2341          * DL queues:
2342          * |                Q_ID              | VF_ID |
2343          * |                 32               |   0   |
2344          * |                ...               |   0   |
2345          * | conf->vf_ul_queues_number[0] - 1 |   0   |
2346          * | conf->vf_ul_queues_number[0]     |   1   |
2347          * |                ...               |   1   |
2348          * | conf->vf_ul_queues_number[1] - 1 |   1   |
2349          * |                ...               |  ...  |
2350          * | conf->vf_ul_queues_number[7] - 1 |   7   |
2351          *
2352          * Example of configuration:
2353          * conf->vf_ul_queues_number[0] = 4;  -> 4 UL queues for VF0
2354          * conf->vf_dl_queues_number[0] = 4;  -> 4 DL queues for VF0
2355          * conf->vf_ul_queues_number[1] = 2;  -> 2 UL queues for VF1
2356          * conf->vf_dl_queues_number[1] = 2;  -> 2 DL queues for VF1
2357          *
2358          * UL:
2359          * | Q_ID | VF_ID |
2360          * |   0  |   0   |
2361          * |   1  |   0   |
2362          * |   2  |   0   |
2363          * |   3  |   0   |
2364          * |   4  |   1   |
2365          * |   5  |   1   |
2366          *
2367          * DL:
2368          * | Q_ID | VF_ID |
2369          * |  32  |   0   |
2370          * |  33  |   0   |
2371          * |  34  |   0   |
2372          * |  35  |   0   |
2373          * |  36  |   1   |
2374          * |  37  |   1   |
2375          */
2376         if (conf->pf_mode_en) {
2377                 payload_32 = 0x1;
2378                 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) {
2379                         address = (q_id << 2) + FPGA_5GNR_FEC_QUEUE_MAP;
2380                         fpga_reg_write_32(d->mmio_base, address, payload_32);
2381                 }
2382         } else {
2383                 /* Calculate total number of UL and DL queues to configure */
2384                 total_ul_q_id = total_dl_q_id = 0;
2385                 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) {
2386                         total_ul_q_id += conf->vf_ul_queues_number[vf_id];
2387                         total_dl_q_id += conf->vf_dl_queues_number[vf_id];
2388                 }
2389                 total_q_id = total_dl_q_id + total_ul_q_id;
2390                 /*
2391                  * Check if total number of queues to configure does not exceed
2392                  * FPGA capabilities (64 queues - 32 UL and 32 DL queues)
2393                  */
2394                 if ((total_ul_q_id > FPGA_NUM_UL_QUEUES) ||
2395                         (total_dl_q_id > FPGA_NUM_DL_QUEUES) ||
2396                         (total_q_id > FPGA_TOTAL_NUM_QUEUES)) {
2397                         rte_bbdev_log(ERR,
2398                                         "FPGA Configuration failed. Too many queues to configure: UL_Q %u, DL_Q %u, FPGA_Q %u",
2399                                         total_ul_q_id, total_dl_q_id,
2400                                         FPGA_TOTAL_NUM_QUEUES);
2401                         return -EINVAL;
2402                 }
2403                 total_ul_q_id = 0;
2404                 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) {
2405                         for (q_id = 0; q_id < conf->vf_ul_queues_number[vf_id];
2406                                         ++q_id, ++total_ul_q_id) {
2407                                 address = (total_ul_q_id << 2) +
2408                                                 FPGA_5GNR_FEC_QUEUE_MAP;
2409                                 payload_32 = ((0x80 + vf_id) << 16) | 0x1;
2410                                 fpga_reg_write_32(d->mmio_base, address,
2411                                                 payload_32);
2412                         }
2413                 }
2414                 total_dl_q_id = 0;
2415                 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) {
2416                         for (q_id = 0; q_id < conf->vf_dl_queues_number[vf_id];
2417                                         ++q_id, ++total_dl_q_id) {
2418                                 address = ((total_dl_q_id + FPGA_NUM_UL_QUEUES)
2419                                                 << 2) + FPGA_5GNR_FEC_QUEUE_MAP;
2420                                 payload_32 = ((0x80 + vf_id) << 16) | 0x1;
2421                                 fpga_reg_write_32(d->mmio_base, address,
2422                                                 payload_32);
2423                         }
2424                 }
2425         }
2426
2427         /* Setting Load Balance Factor */
2428         payload_16 = (conf->dl_load_balance << 8) | (conf->ul_load_balance);
2429         address = FPGA_5GNR_FEC_LOAD_BALANCE_FACTOR;
2430         fpga_reg_write_16(d->mmio_base, address, payload_16);
2431
2432         /* Setting length of ring descriptor entry */
2433         payload_16 = FPGA_RING_DESC_ENTRY_LENGTH;
2434         address = FPGA_5GNR_FEC_RING_DESC_LEN;
2435         fpga_reg_write_16(d->mmio_base, address, payload_16);
2436
2437         /* Queue PF/VF mapping table is ready */
2438         payload_8 = 0x1;
2439         address = FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE;
2440         fpga_reg_write_8(d->mmio_base, address, payload_8);
2441
2442         rte_bbdev_log_debug("PF FPGA 5GNR FEC configuration complete for %s",
2443                         dev_name);
2444
2445 #ifdef RTE_LIBRTE_BBDEV_DEBUG
2446         print_static_reg_debug_info(d->mmio_base);
2447 #endif
2448         return 0;
2449 }
2450
2451 /* FPGA 5GNR FEC PCI PF address map */
2452 static struct rte_pci_id pci_id_fpga_5gnr_fec_pf_map[] = {
2453         {
2454                 RTE_PCI_DEVICE(FPGA_5GNR_FEC_VENDOR_ID,
2455                                 FPGA_5GNR_FEC_PF_DEVICE_ID)
2456         },
2457         {.device_id = 0},
2458 };
2459
2460 static struct rte_pci_driver fpga_5gnr_fec_pci_pf_driver = {
2461         .probe = fpga_5gnr_fec_probe,
2462         .remove = fpga_5gnr_fec_remove,
2463         .id_table = pci_id_fpga_5gnr_fec_pf_map,
2464         .drv_flags = RTE_PCI_DRV_NEED_MAPPING
2465 };
2466
2467 /* FPGA 5GNR FEC PCI VF address map */
2468 static struct rte_pci_id pci_id_fpga_5gnr_fec_vf_map[] = {
2469         {
2470                 RTE_PCI_DEVICE(FPGA_5GNR_FEC_VENDOR_ID,
2471                                 FPGA_5GNR_FEC_VF_DEVICE_ID)
2472         },
2473         {.device_id = 0},
2474 };
2475
2476 static struct rte_pci_driver fpga_5gnr_fec_pci_vf_driver = {
2477         .probe = fpga_5gnr_fec_probe,
2478         .remove = fpga_5gnr_fec_remove,
2479         .id_table = pci_id_fpga_5gnr_fec_vf_map,
2480         .drv_flags = RTE_PCI_DRV_NEED_MAPPING
2481 };
2482
2483
2484 RTE_PMD_REGISTER_PCI(FPGA_5GNR_FEC_PF_DRIVER_NAME, fpga_5gnr_fec_pci_pf_driver);
2485 RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_PF_DRIVER_NAME,
2486                 pci_id_fpga_5gnr_fec_pf_map);
2487 RTE_PMD_REGISTER_PCI(FPGA_5GNR_FEC_VF_DRIVER_NAME, fpga_5gnr_fec_pci_vf_driver);
2488 RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_VF_DRIVER_NAME,
2489                 pci_id_fpga_5gnr_fec_vf_map);