drivers/baseband/fpga_5gnr_fec/rte_fpga_5gnr_fec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4
   5 #include <unistd.h>
   6
   7 #include <rte_common.h>
   8 #include <rte_log.h>
   9 #include <rte_dev.h>
  10 #include <rte_malloc.h>
  11 #include <rte_mempool.h>
  12 #include <rte_errno.h>
  13 #include <rte_pci.h>
  14 #include <rte_bus_pci.h>
  15 #include <rte_byteorder.h>
  16 #ifdef RTE_BBDEV_OFFLOAD_COST
  17 #include <rte_cycles.h>
  18 #endif
  19
  20 #include <rte_bbdev.h>
  21 #include <rte_bbdev_pmd.h>
  22
  23 #include "fpga_5gnr_fec.h"
  24 #include "rte_pmd_fpga_5gnr_fec.h"
  25
  26 /* 5GNR SW PMD logging ID */
  27 static int fpga_5gnr_fec_logtype;
  28
  29 #ifdef RTE_LIBRTE_BBDEV_DEBUG
  30
  31 /* Read Ring Control Register of FPGA 5GNR FEC device */
  32 static inline void
  33 print_ring_reg_debug_info(void *mmio_base, uint32_t offset)
  34 {
  35         rte_bbdev_log_debug(
  36                 "FPGA MMIO base address @ %p | Ring Control Register @ offset = 0x%08"
  37                 PRIx32, mmio_base, offset);
  38         rte_bbdev_log_debug(
  39                 "RING_BASE_ADDR = 0x%016"PRIx64,
  40                 fpga_reg_read_64(mmio_base, offset));
  41         rte_bbdev_log_debug(
  42                 "RING_HEAD_ADDR = 0x%016"PRIx64,
  43                 fpga_reg_read_64(mmio_base, offset +
  44                                 FPGA_5GNR_FEC_RING_HEAD_ADDR));
  45         rte_bbdev_log_debug(
  46                 "RING_SIZE = 0x%04"PRIx16,
  47                 fpga_reg_read_16(mmio_base, offset +
  48                                 FPGA_5GNR_FEC_RING_SIZE));
  49         rte_bbdev_log_debug(
  50                 "RING_MISC = 0x%02"PRIx8,
  51                 fpga_reg_read_8(mmio_base, offset +
  52                                 FPGA_5GNR_FEC_RING_MISC));
  53         rte_bbdev_log_debug(
  54                 "RING_ENABLE = 0x%02"PRIx8,
  55                 fpga_reg_read_8(mmio_base, offset +
  56                                 FPGA_5GNR_FEC_RING_ENABLE));
  57         rte_bbdev_log_debug(
  58                 "RING_FLUSH_QUEUE_EN = 0x%02"PRIx8,
  59                 fpga_reg_read_8(mmio_base, offset +
  60                                 FPGA_5GNR_FEC_RING_FLUSH_QUEUE_EN));
  61         rte_bbdev_log_debug(
  62                 "RING_SHADOW_TAIL = 0x%04"PRIx16,
  63                 fpga_reg_read_16(mmio_base, offset +
  64                                 FPGA_5GNR_FEC_RING_SHADOW_TAIL));
  65         rte_bbdev_log_debug(
  66                 "RING_HEAD_POINT = 0x%04"PRIx16,
  67                 fpga_reg_read_16(mmio_base, offset +
  68                                 FPGA_5GNR_FEC_RING_HEAD_POINT));
  69 }
  70
  71 /* Read Static Register of FPGA 5GNR FEC device */
  72 static inline void
  73 print_static_reg_debug_info(void *mmio_base)
  74 {
  75         uint16_t config = fpga_reg_read_16(mmio_base,
  76                         FPGA_5GNR_FEC_CONFIGURATION);
  77         uint8_t qmap_done = fpga_reg_read_8(mmio_base,
  78                         FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE);
  79         uint16_t lb_factor = fpga_reg_read_16(mmio_base,
  80                         FPGA_5GNR_FEC_LOAD_BALANCE_FACTOR);
  81         uint16_t ring_desc_len = fpga_reg_read_16(mmio_base,
  82                         FPGA_5GNR_FEC_RING_DESC_LEN);
  83         uint16_t flr_time_out = fpga_reg_read_16(mmio_base,
  84                         FPGA_5GNR_FEC_FLR_TIME_OUT);
  85
  86         rte_bbdev_log_debug("UL.DL Weights = %u.%u",
  87                         ((uint8_t)config), ((uint8_t)(config >> 8)));
  88         rte_bbdev_log_debug("UL.DL Load Balance = %u.%u",
  89                         ((uint8_t)lb_factor), ((uint8_t)(lb_factor >> 8)));
  90         rte_bbdev_log_debug("Queue-PF/VF Mapping Table = %s",
  91                         (qmap_done > 0) ? "READY" : "NOT-READY");
  92         rte_bbdev_log_debug("Ring Descriptor Size = %u bytes",
  93                         ring_desc_len*FPGA_RING_DESC_LEN_UNIT_BYTES);
  94         rte_bbdev_log_debug("FLR Timeout = %f usec",
  95                         (float)flr_time_out*FPGA_FLR_TIMEOUT_UNIT);
  96 }
  97
  98 /* Print decode DMA Descriptor of FPGA 5GNR Decoder device */
  99 static void
 100 print_dma_dec_desc_debug_info(union fpga_dma_desc *desc)
 101 {
 102         rte_bbdev_log_debug("DMA response desc %p\n"
 103                 "\t-- done(%"PRIu32") | iter(%"PRIu32") | et_pass(%"PRIu32")"
 104                 " | crcb_pass (%"PRIu32") | error(%"PRIu32")\n"
 105                 "\t-- qm_idx(%"PRIu32") | max_iter(%"PRIu32") | "
 106                 "bg_idx (%"PRIu32") | harqin_en(%"PRIu32") | zc(%"PRIu32")\n"
 107                 "\t-- hbstroe_offset(%"PRIu32") | num_null (%"PRIu32") "
 108                 "| irq_en(%"PRIu32")\n"
 109                 "\t-- ncb(%"PRIu32") | desc_idx (%"PRIu32") | "
 110                 "drop_crc24b(%"PRIu32") | RV (%"PRIu32")\n"
 111                 "\t-- crc24b_ind(%"PRIu32") | et_dis (%"PRIu32")\n"
 112                 "\t-- harq_input_length(%"PRIu32") | rm_e(%"PRIu32")\n"
 113                 "\t-- cbs_in_op(%"PRIu32") | in_add (0x%08"PRIx32"%08"PRIx32")"
 114                 "| out_add (0x%08"PRIx32"%08"PRIx32")",
 115                 desc,
 116                 (uint32_t)desc->dec_req.done,
 117                 (uint32_t)desc->dec_req.iter,
 118                 (uint32_t)desc->dec_req.et_pass,
 119                 (uint32_t)desc->dec_req.crcb_pass,
 120                 (uint32_t)desc->dec_req.error,
 121                 (uint32_t)desc->dec_req.qm_idx,
 122                 (uint32_t)desc->dec_req.max_iter,
 123                 (uint32_t)desc->dec_req.bg_idx,
 124                 (uint32_t)desc->dec_req.harqin_en,
 125                 (uint32_t)desc->dec_req.zc,
 126                 (uint32_t)desc->dec_req.hbstroe_offset,
 127                 (uint32_t)desc->dec_req.num_null,
 128                 (uint32_t)desc->dec_req.irq_en,
 129                 (uint32_t)desc->dec_req.ncb,
 130                 (uint32_t)desc->dec_req.desc_idx,
 131                 (uint32_t)desc->dec_req.drop_crc24b,
 132                 (uint32_t)desc->dec_req.rv,
 133                 (uint32_t)desc->dec_req.crc24b_ind,
 134                 (uint32_t)desc->dec_req.et_dis,
 135                 (uint32_t)desc->dec_req.harq_input_length,
 136                 (uint32_t)desc->dec_req.rm_e,
 137                 (uint32_t)desc->dec_req.cbs_in_op,
 138                 (uint32_t)desc->dec_req.in_addr_hi,
 139                 (uint32_t)desc->dec_req.in_addr_lw,
 140                 (uint32_t)desc->dec_req.out_addr_hi,
 141                 (uint32_t)desc->dec_req.out_addr_lw);
 142         uint32_t *word = (uint32_t *) desc;
 143         rte_bbdev_log_debug("%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n"
 144                         "%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n",
 145                         word[0], word[1], word[2], word[3],
 146                         word[4], word[5], word[6], word[7]);
 147 }
 148
 149 /* Print decode DMA Descriptor of FPGA 5GNR encoder device */
 150 static void
 151 print_dma_enc_desc_debug_info(union fpga_dma_desc *desc)
 152 {
 153         rte_bbdev_log_debug("DMA response desc %p\n"
 154                         "%"PRIu32" %"PRIu32"\n"
 155                         "K' %"PRIu32" E %"PRIu32" desc %"PRIu32" Z %"PRIu32"\n"
 156                         "BG %"PRIu32" Qm %"PRIu32" CRC %"PRIu32" IRQ %"PRIu32"\n"
 157                         "k0 %"PRIu32" Ncb %"PRIu32" F %"PRIu32"\n",
 158                         desc,
 159                         (uint32_t)desc->enc_req.done,
 160                         (uint32_t)desc->enc_req.error,
 161
 162                         (uint32_t)desc->enc_req.k_,
 163                         (uint32_t)desc->enc_req.rm_e,
 164                         (uint32_t)desc->enc_req.desc_idx,
 165                         (uint32_t)desc->enc_req.zc,
 166
 167                         (uint32_t)desc->enc_req.bg_idx,
 168                         (uint32_t)desc->enc_req.qm_idx,
 169                         (uint32_t)desc->enc_req.crc_en,
 170                         (uint32_t)desc->enc_req.irq_en,
 171
 172                         (uint32_t)desc->enc_req.k0,
 173                         (uint32_t)desc->enc_req.ncb,
 174                         (uint32_t)desc->enc_req.num_null);
 175         uint32_t *word = (uint32_t *) desc;
 176         rte_bbdev_log_debug("%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n"
 177                         "%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n",
 178                         word[0], word[1], word[2], word[3],
 179                         word[4], word[5], word[6], word[7]);
 180 }
 181
 182 #endif
 183
 184 static int
 185 fpga_setup_queues(struct rte_bbdev *dev, uint16_t num_queues, int socket_id)
 186 {
 187         /* Number of queues bound to a PF/VF */
 188         uint32_t hw_q_num = 0;
 189         uint32_t ring_size, payload, address, q_id, offset;
 190         rte_iova_t phys_addr;
 191         struct fpga_ring_ctrl_reg ring_reg;
 192         struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private;
 193
 194         address = FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE;
 195         if (!(fpga_reg_read_32(fpga_dev->mmio_base, address) & 0x1)) {
 196                 rte_bbdev_log(ERR,
 197                                 "Queue-PF/VF mapping is not set! Was PF configured for device (%s) ?",
 198                                 dev->data->name);
 199                 return -EPERM;
 200         }
 201
 202         /* Clear queue registers structure */
 203         memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg));
 204
 205         /* Scan queue map.
 206          * If a queue is valid and mapped to a calling PF/VF the read value is
 207          * replaced with a queue ID and if it's not then
 208          * FPGA_INVALID_HW_QUEUE_ID is returned.
 209          */
 210         for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) {
 211                 uint32_t hw_q_id = fpga_reg_read_32(fpga_dev->mmio_base,
 212                                 FPGA_5GNR_FEC_QUEUE_MAP + (q_id << 2));
 213
 214                 rte_bbdev_log_debug("%s: queue ID: %u, registry queue ID: %u",
 215                                 dev->device->name, q_id, hw_q_id);
 216
 217                 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID) {
 218                         fpga_dev->q_bound_bit_map |= (1ULL << q_id);
 219                         /* Clear queue register of found queue */
 220                         offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
 221                                 (sizeof(struct fpga_ring_ctrl_reg) * q_id);
 222                         fpga_ring_reg_write(fpga_dev->mmio_base,
 223                                         offset, ring_reg);
 224                         ++hw_q_num;
 225                 }
 226         }
 227         if (hw_q_num == 0) {
 228                 rte_bbdev_log(ERR,
 229                         "No HW queues assigned to this device. Probably this is a VF configured for PF mode. Check device configuration!");
 230                 return -ENODEV;
 231         }
 232
 233         if (num_queues > hw_q_num) {
 234                 rte_bbdev_log(ERR,
 235                         "Not enough queues for device %s! Requested: %u, available: %u",
 236                         dev->device->name, num_queues, hw_q_num);
 237                 return -EINVAL;
 238         }
 239
 240         ring_size = FPGA_RING_MAX_SIZE * sizeof(struct fpga_dma_dec_desc);
 241
 242         /* Enforce 32 byte alignment */
 243         RTE_BUILD_BUG_ON((RTE_CACHE_LINE_SIZE % 32) != 0);
 244
 245         /* Allocate memory for SW descriptor rings */
 246         fpga_dev->sw_rings = rte_zmalloc_socket(dev->device->driver->name,
 247                         num_queues * ring_size, RTE_CACHE_LINE_SIZE,
 248                         socket_id);
 249         if (fpga_dev->sw_rings == NULL) {
 250                 rte_bbdev_log(ERR,
 251                                 "Failed to allocate memory for %s:%u sw_rings",
 252                                 dev->device->driver->name, dev->data->dev_id);
 253                 return -ENOMEM;
 254         }
 255
 256         fpga_dev->sw_rings_phys = rte_malloc_virt2iova(fpga_dev->sw_rings);
 257         fpga_dev->sw_ring_size = ring_size;
 258         fpga_dev->sw_ring_max_depth = FPGA_RING_MAX_SIZE;
 259
 260         /* Allocate memory for ring flush status */
 261         fpga_dev->flush_queue_status = rte_zmalloc_socket(NULL,
 262                         sizeof(uint64_t), RTE_CACHE_LINE_SIZE, socket_id);
 263         if (fpga_dev->flush_queue_status == NULL) {
 264                 rte_bbdev_log(ERR,
 265                                 "Failed to allocate memory for %s:%u flush_queue_status",
 266                                 dev->device->driver->name, dev->data->dev_id);
 267                 return -ENOMEM;
 268         }
 269
 270         /* Set the flush status address registers */
 271         phys_addr = rte_malloc_virt2iova(fpga_dev->flush_queue_status);
 272
 273         address = FPGA_5GNR_FEC_VFQ_FLUSH_STATUS_LW;
 274         payload = (uint32_t)(phys_addr);
 275         fpga_reg_write_32(fpga_dev->mmio_base, address, payload);
 276
 277         address = FPGA_5GNR_FEC_VFQ_FLUSH_STATUS_HI;
 278         payload = (uint32_t)(phys_addr >> 32);
 279         fpga_reg_write_32(fpga_dev->mmio_base, address, payload);
 280
 281         return 0;
 282 }
 283
 284 static int
 285 fpga_dev_close(struct rte_bbdev *dev)
 286 {
 287         struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private;
 288
 289         rte_free(fpga_dev->sw_rings);
 290         rte_free(fpga_dev->flush_queue_status);
 291
 292         return 0;
 293 }
 294
 295 static void
 296 fpga_dev_info_get(struct rte_bbdev *dev,
 297                 struct rte_bbdev_driver_info *dev_info)
 298 {
 299         struct fpga_5gnr_fec_device *d = dev->data->dev_private;
 300         uint32_t q_id = 0;
 301
 302         static const struct rte_bbdev_op_cap bbdev_capabilities[] = {
 303                 {
 304                         .type   = RTE_BBDEV_OP_LDPC_ENC,
 305                         .cap.ldpc_enc = {
 306                                 .capability_flags =
 307                                                 RTE_BBDEV_LDPC_RATE_MATCH |
 308                                                 RTE_BBDEV_LDPC_ENC_INTERRUPTS |
 309                                                 RTE_BBDEV_LDPC_CRC_24B_ATTACH,
 310                                 .num_buffers_src =
 311                                                 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
 312                                 .num_buffers_dst =
 313                                                 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
 314                         }
 315                 },
 316                 {
 317                 .type   = RTE_BBDEV_OP_LDPC_DEC,
 318                 .cap.ldpc_dec = {
 319                         .capability_flags =
 320                                 RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK |
 321                                 RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP |
 322                                 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE |
 323                                 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE |
 324                                 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE |
 325                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE |
 326                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE |
 327                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK |
 328                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS,
 329                         .llr_size = 6,
 330                         .llr_decimals = 2,
 331                         .num_buffers_src =
 332                                         RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
 333                         .num_buffers_hard_out =
 334                                         RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
 335                         .num_buffers_soft_out = 0,
 336                 }
 337                 },
 338                 RTE_BBDEV_END_OF_CAPABILITIES_LIST()
 339         };
 340
 341         /* Check the HARQ DDR size available */
 342         uint8_t timeout_counter = 0;
 343         uint32_t harq_buf_ready = fpga_reg_read_32(d->mmio_base,
 344                         FPGA_5GNR_FEC_HARQ_BUF_SIZE_RDY_REGS);
 345         while (harq_buf_ready != 1) {
 346                 usleep(FPGA_TIMEOUT_CHECK_INTERVAL);
 347                 timeout_counter++;
 348                 harq_buf_ready = fpga_reg_read_32(d->mmio_base,
 349                                 FPGA_5GNR_FEC_HARQ_BUF_SIZE_RDY_REGS);
 350                 if (timeout_counter > FPGA_HARQ_RDY_TIMEOUT) {
 351                         rte_bbdev_log(ERR, "HARQ Buffer not ready %d",
 352                                         harq_buf_ready);
 353                         harq_buf_ready = 1;
 354                 }
 355         }
 356         uint32_t harq_buf_size = fpga_reg_read_32(d->mmio_base,
 357                         FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS);
 358
 359         static struct rte_bbdev_queue_conf default_queue_conf;
 360         default_queue_conf.socket = dev->data->socket_id;
 361         default_queue_conf.queue_size = FPGA_RING_MAX_SIZE;
 362
 363         dev_info->driver_name = dev->device->driver->name;
 364         dev_info->queue_size_lim = FPGA_RING_MAX_SIZE;
 365         dev_info->hardware_accelerated = true;
 366         dev_info->min_alignment = 64;
 367         dev_info->harq_buffer_size = (harq_buf_size >> 10) + 1;
 368         dev_info->default_queue_conf = default_queue_conf;
 369         dev_info->capabilities = bbdev_capabilities;
 370         dev_info->cpu_flag_reqs = NULL;
 371
 372         /* Calculates number of queues assigned to device */
 373         dev_info->max_num_queues = 0;
 374         for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) {
 375                 uint32_t hw_q_id = fpga_reg_read_32(d->mmio_base,
 376                                 FPGA_5GNR_FEC_QUEUE_MAP + (q_id << 2));
 377                 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID)
 378                         dev_info->max_num_queues++;
 379         }
 380 }
 381
 382 /**
 383  * Find index of queue bound to current PF/VF which is unassigned. Return -1
 384  * when there is no available queue
 385  */
 386 static inline int
 387 fpga_find_free_queue_idx(struct rte_bbdev *dev,
 388                 const struct rte_bbdev_queue_conf *conf)
 389 {
 390         struct fpga_5gnr_fec_device *d = dev->data->dev_private;
 391         uint64_t q_idx;
 392         uint8_t i = 0;
 393         uint8_t range = FPGA_TOTAL_NUM_QUEUES >> 1;
 394
 395         if (conf->op_type == RTE_BBDEV_OP_LDPC_ENC) {
 396                 i = FPGA_NUM_DL_QUEUES;
 397                 range = FPGA_TOTAL_NUM_QUEUES;
 398         }
 399
 400         for (; i < range; ++i) {
 401                 q_idx = 1ULL << i;
 402                 /* Check if index of queue is bound to current PF/VF */
 403                 if (d->q_bound_bit_map & q_idx)
 404                         /* Check if found queue was not already assigned */
 405                         if (!(d->q_assigned_bit_map & q_idx)) {
 406                                 d->q_assigned_bit_map |= q_idx;
 407                                 return i;
 408                         }
 409         }
 410
 411         rte_bbdev_log(INFO, "Failed to find free queue on %s", dev->data->name);
 412
 413         return -1;
 414 }
 415
 416 static int
 417 fpga_queue_setup(struct rte_bbdev *dev, uint16_t queue_id,
 418                 const struct rte_bbdev_queue_conf *conf)
 419 {
 420         uint32_t address, ring_offset;
 421         struct fpga_5gnr_fec_device *d = dev->data->dev_private;
 422         struct fpga_queue *q;
 423         int8_t q_idx;
 424
 425         /* Check if there is a free queue to assign */
 426         q_idx = fpga_find_free_queue_idx(dev, conf);
 427         if (q_idx == -1)
 428                 return -1;
 429
 430         /* Allocate the queue data structure. */
 431         q = rte_zmalloc_socket(dev->device->driver->name, sizeof(*q),
 432                         RTE_CACHE_LINE_SIZE, conf->socket);
 433         if (q == NULL) {
 434                 /* Mark queue as un-assigned */
 435                 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx));
 436                 rte_bbdev_log(ERR, "Failed to allocate queue memory");
 437                 return -ENOMEM;
 438         }
 439
 440         q->d = d;
 441         q->q_idx = q_idx;
 442
 443         /* Set ring_base_addr */
 444         q->ring_addr = RTE_PTR_ADD(d->sw_rings, (d->sw_ring_size * queue_id));
 445         q->ring_ctrl_reg.ring_base_addr = d->sw_rings_phys +
 446                         (d->sw_ring_size * queue_id);
 447
 448         /* Allocate memory for Completion Head variable*/
 449         q->ring_head_addr = rte_zmalloc_socket(dev->device->driver->name,
 450                         sizeof(uint64_t), RTE_CACHE_LINE_SIZE, conf->socket);
 451         if (q->ring_head_addr == NULL) {
 452                 /* Mark queue as un-assigned */
 453                 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx));
 454                 rte_free(q);
 455                 rte_bbdev_log(ERR,
 456                                 "Failed to allocate memory for %s:%u completion_head",
 457                                 dev->device->driver->name, dev->data->dev_id);
 458                 return -ENOMEM;
 459         }
 460         /* Set ring_head_addr */
 461         q->ring_ctrl_reg.ring_head_addr =
 462                         rte_malloc_virt2iova(q->ring_head_addr);
 463
 464         /* Clear shadow_completion_head */
 465         q->shadow_completion_head = 0;
 466
 467         /* Set ring_size */
 468         if (conf->queue_size > FPGA_RING_MAX_SIZE) {
 469                 /* Mark queue as un-assigned */
 470                 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx));
 471                 rte_free(q->ring_head_addr);
 472                 rte_free(q);
 473                 rte_bbdev_log(ERR,
 474                                 "Size of queue is too big %d (MAX: %d ) for %s:%u",
 475                                 conf->queue_size, FPGA_RING_MAX_SIZE,
 476                                 dev->device->driver->name, dev->data->dev_id);
 477                 return -EINVAL;
 478         }
 479         q->ring_ctrl_reg.ring_size = conf->queue_size;
 480
 481         /* Set Miscellaneous FPGA register*/
 482         /* Max iteration number for TTI mitigation - todo */
 483         q->ring_ctrl_reg.max_ul_dec = 0;
 484         /* Enable max iteration number for TTI - todo */
 485         q->ring_ctrl_reg.max_ul_dec_en = 0;
 486
 487         /* Enable the ring */
 488         q->ring_ctrl_reg.enable = 1;
 489
 490         /* Set FPGA head_point and tail registers */
 491         q->ring_ctrl_reg.head_point = q->tail = 0;
 492
 493         /* Set FPGA shadow_tail register */
 494         q->ring_ctrl_reg.shadow_tail = q->tail;
 495
 496         /* Calculates the ring offset for found queue */
 497         ring_offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
 498                         (sizeof(struct fpga_ring_ctrl_reg) * q_idx);
 499
 500         /* Set FPGA Ring Control Registers */
 501         fpga_ring_reg_write(d->mmio_base, ring_offset, q->ring_ctrl_reg);
 502
 503         /* Store MMIO register of shadow_tail */
 504         address = ring_offset + FPGA_5GNR_FEC_RING_SHADOW_TAIL;
 505         q->shadow_tail_addr = RTE_PTR_ADD(d->mmio_base, address);
 506
 507         q->head_free_desc = q->tail;
 508
 509         /* Set wrap mask */
 510         q->sw_ring_wrap_mask = conf->queue_size - 1;
 511
 512         rte_bbdev_log_debug("Setup dev%u q%u: queue_idx=%u",
 513                         dev->data->dev_id, queue_id, q->q_idx);
 514
 515         dev->data->queues[queue_id].queue_private = q;
 516
 517         rte_bbdev_log_debug("BBDEV queue[%d] set up for FPGA queue[%d]",
 518                         queue_id, q_idx);
 519
 520 #ifdef RTE_LIBRTE_BBDEV_DEBUG
 521         /* Read FPGA Ring Control Registers after configuration*/
 522         print_ring_reg_debug_info(d->mmio_base, ring_offset);
 523 #endif
 524         return 0;
 525 }
 526
 527 static int
 528 fpga_queue_release(struct rte_bbdev *dev, uint16_t queue_id)
 529 {
 530         struct fpga_5gnr_fec_device *d = dev->data->dev_private;
 531         struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
 532         struct fpga_ring_ctrl_reg ring_reg;
 533         uint32_t offset;
 534
 535         rte_bbdev_log_debug("FPGA Queue[%d] released", queue_id);
 536
 537         if (q != NULL) {
 538                 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg));
 539                 offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
 540                         (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx);
 541                 /* Disable queue */
 542                 fpga_reg_write_8(d->mmio_base,
 543                                 offset + FPGA_5GNR_FEC_RING_ENABLE, 0x00);
 544                 /* Clear queue registers */
 545                 fpga_ring_reg_write(d->mmio_base, offset, ring_reg);
 546
 547                 /* Mark the Queue as un-assigned */
 548                 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q->q_idx));
 549                 rte_free(q->ring_head_addr);
 550                 rte_free(q);
 551                 dev->data->queues[queue_id].queue_private = NULL;
 552         }
 553
 554         return 0;
 555 }
 556
 557 /* Function starts a device queue. */
 558 static int
 559 fpga_queue_start(struct rte_bbdev *dev, uint16_t queue_id)
 560 {
 561         struct fpga_5gnr_fec_device *d = dev->data->dev_private;
 562 #ifdef RTE_LIBRTE_BBDEV_DEBUG
 563         if (d == NULL) {
 564                 rte_bbdev_log(ERR, "Invalid device pointer");
 565                 return -1;
 566         }
 567 #endif
 568         struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
 569         uint32_t offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
 570                         (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx);
 571         uint8_t enable = 0x01;
 572         uint16_t zero = 0x0000;
 573
 574         /* Clear queue head and tail variables */
 575         q->tail = q->head_free_desc = 0;
 576
 577         /* Clear FPGA head_point and tail registers */
 578         fpga_reg_write_16(d->mmio_base, offset + FPGA_5GNR_FEC_RING_HEAD_POINT,
 579                         zero);
 580         fpga_reg_write_16(d->mmio_base, offset + FPGA_5GNR_FEC_RING_SHADOW_TAIL,
 581                         zero);
 582
 583         /* Enable queue */
 584         fpga_reg_write_8(d->mmio_base, offset + FPGA_5GNR_FEC_RING_ENABLE,
 585                         enable);
 586
 587         rte_bbdev_log_debug("FPGA Queue[%d] started", queue_id);
 588         return 0;
 589 }
 590
 591 /* Function stops a device queue. */
 592 static int
 593 fpga_queue_stop(struct rte_bbdev *dev, uint16_t queue_id)
 594 {
 595         struct fpga_5gnr_fec_device *d = dev->data->dev_private;
 596 #ifdef RTE_LIBRTE_BBDEV_DEBUG
 597         if (d == NULL) {
 598                 rte_bbdev_log(ERR, "Invalid device pointer");
 599                 return -1;
 600         }
 601 #endif
 602         struct fpga_queue *q = dev->data->queues[queue_id].queue_private;
 603         uint32_t offset = FPGA_5GNR_FEC_RING_CTRL_REGS +
 604                         (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx);
 605         uint8_t payload = 0x01;
 606         uint8_t counter = 0;
 607         uint8_t timeout = FPGA_QUEUE_FLUSH_TIMEOUT_US /
 608                         FPGA_TIMEOUT_CHECK_INTERVAL;
 609
 610         /* Set flush_queue_en bit to trigger queue flushing */
 611         fpga_reg_write_8(d->mmio_base,
 612                         offset + FPGA_5GNR_FEC_RING_FLUSH_QUEUE_EN, payload);
 613
 614         /** Check if queue flush is completed.
 615          * FPGA will update the completion flag after queue flushing is
 616          * completed. If completion flag is not updated within 1ms it is
 617          * considered as a failure.
 618          */
 619         while (!(*((volatile uint8_t *)d->flush_queue_status + q->q_idx)
 620                         & payload)) {
 621                 if (counter > timeout) {
 622                         rte_bbdev_log(ERR, "FPGA Queue Flush failed for queue %d",
 623                                         queue_id);
 624                         return -1;
 625                 }
 626                 usleep(FPGA_TIMEOUT_CHECK_INTERVAL);
 627                 counter++;
 628         }
 629
 630         /* Disable queue */
 631         payload = 0x00;
 632         fpga_reg_write_8(d->mmio_base, offset + FPGA_5GNR_FEC_RING_ENABLE,
 633                         payload);
 634
 635         rte_bbdev_log_debug("FPGA Queue[%d] stopped", queue_id);
 636         return 0;
 637 }
 638
 639 static const struct rte_bbdev_ops fpga_ops = {
 640         .setup_queues = fpga_setup_queues,
 641         .close = fpga_dev_close,
 642         .info_get = fpga_dev_info_get,
 643         .queue_setup = fpga_queue_setup,
 644         .queue_stop = fpga_queue_stop,
 645         .queue_start = fpga_queue_start,
 646         .queue_release = fpga_queue_release,
 647 };
 648
 649 static inline void
 650 fpga_dma_enqueue(struct fpga_queue *q, uint16_t num_desc,
 651                 struct rte_bbdev_stats *queue_stats)
 652 {
 653 #ifdef RTE_BBDEV_OFFLOAD_COST
 654         uint64_t start_time = 0;
 655         queue_stats->acc_offload_cycles = 0;
 656 #else
 657         RTE_SET_USED(queue_stats);
 658 #endif
 659
 660         /* Update tail and shadow_tail register */
 661         q->tail = (q->tail + num_desc) & q->sw_ring_wrap_mask;
 662
 663         rte_wmb();
 664
 665 #ifdef RTE_BBDEV_OFFLOAD_COST
 666         /* Start time measurement for enqueue function offload. */
 667         start_time = rte_rdtsc_precise();
 668 #endif
 669         mmio_write_16(q->shadow_tail_addr, q->tail);
 670
 671 #ifdef RTE_BBDEV_OFFLOAD_COST
 672         rte_wmb();
 673         queue_stats->acc_offload_cycles += rte_rdtsc_precise() - start_time;
 674 #endif
 675 }
 676
 677 /* Read flag value 0/1/ from bitmap */
 678 static inline bool
 679 check_bit(uint32_t bitmap, uint32_t bitmask)
 680 {
 681         return bitmap & bitmask;
 682 }
 683
 684 /* Print an error if a descriptor error has occurred.
 685  *  Return 0 on success, 1 on failure
 686  */
 687 static inline int
 688 check_desc_error(uint32_t error_code) {
 689         switch (error_code) {
 690         case DESC_ERR_NO_ERR:
 691                 return 0;
 692         case DESC_ERR_K_P_OUT_OF_RANGE:
 693                 rte_bbdev_log(ERR, "Encode block size K' is out of range");
 694                 break;
 695         case DESC_ERR_Z_C_NOT_LEGAL:
 696                 rte_bbdev_log(ERR, "Zc is illegal");
 697                 break;
 698         case DESC_ERR_DESC_OFFSET_ERR:
 699                 rte_bbdev_log(ERR,
 700                                 "Queue offset does not meet the expectation in the FPGA"
 701                                 );
 702                 break;
 703         case DESC_ERR_DESC_READ_FAIL:
 704                 rte_bbdev_log(ERR, "Unsuccessful completion for descriptor read");
 705                 break;
 706         case DESC_ERR_DESC_READ_TIMEOUT:
 707                 rte_bbdev_log(ERR, "Descriptor read time-out");
 708                 break;
 709         case DESC_ERR_DESC_READ_TLP_POISONED:
 710                 rte_bbdev_log(ERR, "Descriptor read TLP poisoned");
 711                 break;
 712         case DESC_ERR_CB_READ_FAIL:
 713                 rte_bbdev_log(ERR, "Unsuccessful completion for code block");
 714                 break;
 715         case DESC_ERR_CB_READ_TIMEOUT:
 716                 rte_bbdev_log(ERR, "Code block read time-out");
 717                 break;
 718         case DESC_ERR_CB_READ_TLP_POISONED:
 719                 rte_bbdev_log(ERR, "Code block read TLP poisoned");
 720                 break;
 721         case DESC_ERR_HBSTORE_ERR:
 722                 rte_bbdev_log(ERR, "Hbstroe exceeds HARQ buffer size.");
 723                 break;
 724         default:
 725                 rte_bbdev_log(ERR, "Descriptor error unknown error code %u",
 726                                 error_code);
 727                 break;
 728         }
 729         return 1;
 730 }
 731
 732 /* Compute value of k0.
 733  * Based on 3GPP 38.212 Table 5.4.2.1-2
 734  * Starting position of different redundancy versions, k0
 735  */
 736 static inline uint16_t
 737 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index)
 738 {
 739         if (rv_index == 0)
 740                 return 0;
 741         uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c;
 742         if (n_cb == n) {
 743                 if (rv_index == 1)
 744                         return (bg == 1 ? K0_1_1 : K0_1_2) * z_c;
 745                 else if (rv_index == 2)
 746                         return (bg == 1 ? K0_2_1 : K0_2_2) * z_c;
 747                 else
 748                         return (bg == 1 ? K0_3_1 : K0_3_2) * z_c;
 749         }
 750         /* LBRM case - includes a division by N */
 751         if (rv_index == 1)
 752                 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb)
 753                                 / n) * z_c;
 754         else if (rv_index == 2)
 755                 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb)
 756                                 / n) * z_c;
 757         else
 758                 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb)
 759                                 / n) * z_c;
 760 }
 761
 762 /**
 763  * Set DMA descriptor for encode operation (1 Code Block)
 764  *
 765  * @param op
 766  *   Pointer to a single encode operation.
 767  * @param desc
 768  *   Pointer to DMA descriptor.
 769  * @param input
 770  *   Pointer to pointer to input data which will be decoded.
 771  * @param e
 772  *   E value (length of output in bits).
 773  * @param ncb
 774  *   Ncb value (size of the soft buffer).
 775  * @param out_length
 776  *   Length of output buffer
 777  * @param in_offset
 778  *   Input offset in rte_mbuf structure. It is used for calculating the point
 779  *   where data is starting.
 780  * @param out_offset
 781  *   Output offset in rte_mbuf structure. It is used for calculating the point
 782  *   where hard output data will be stored.
 783  * @param cbs_in_op
 784  *   Number of CBs contained in one operation.
 785  */
 786 static inline int
 787 fpga_dma_desc_te_fill(struct rte_bbdev_enc_op *op,
 788                 struct fpga_dma_enc_desc *desc, struct rte_mbuf *input,
 789                 struct rte_mbuf *output, uint16_t k_,  uint16_t e,
 790                 uint32_t in_offset, uint32_t out_offset, uint16_t desc_offset,
 791                 uint8_t cbs_in_op)
 792 {
 793         /* reset */
 794         desc->done = 0;
 795         desc->error = 0;
 796         desc->k_ = k_;
 797         desc->rm_e = e;
 798         desc->desc_idx = desc_offset;
 799         desc->zc = op->ldpc_enc.z_c;
 800         desc->bg_idx = op->ldpc_enc.basegraph - 1;
 801         desc->qm_idx = op->ldpc_enc.q_m / 2;
 802         desc->crc_en = check_bit(op->ldpc_enc.op_flags,
 803                         RTE_BBDEV_LDPC_CRC_24B_ATTACH);
 804         desc->irq_en = 0;
 805         desc->k0 = get_k0(op->ldpc_enc.n_cb, op->ldpc_enc.z_c,
 806                         op->ldpc_enc.basegraph, op->ldpc_enc.rv_index);
 807         desc->ncb = op->ldpc_enc.n_cb;
 808         desc->num_null = op->ldpc_enc.n_filler;
 809         /* Set inbound data buffer address */
 810         desc->in_addr_hi = (uint32_t)(
 811                         rte_pktmbuf_mtophys_offset(input, in_offset) >> 32);
 812         desc->in_addr_lw = (uint32_t)(
 813                         rte_pktmbuf_mtophys_offset(input, in_offset));
 814
 815         desc->out_addr_hi = (uint32_t)(
 816                         rte_pktmbuf_mtophys_offset(output, out_offset) >> 32);
 817         desc->out_addr_lw = (uint32_t)(
 818                         rte_pktmbuf_mtophys_offset(output, out_offset));
 819         /* Save software context needed for dequeue */
 820         desc->op_addr = op;
 821         /* Set total number of CBs in an op */
 822         desc->cbs_in_op = cbs_in_op;
 823         return 0;
 824 }
 825
 826 /**
 827  * Set DMA descriptor for decode operation (1 Code Block)
 828  *
 829  * @param op
 830  *   Pointer to a single encode operation.
 831  * @param desc
 832  *   Pointer to DMA descriptor.
 833  * @param input
 834  *   Pointer to pointer to input data which will be decoded.
 835  * @param in_offset
 836  *   Input offset in rte_mbuf structure. It is used for calculating the point
 837  *   where data is starting.
 838  * @param out_offset
 839  *   Output offset in rte_mbuf structure. It is used for calculating the point
 840  *   where hard output data will be stored.
 841  * @param cbs_in_op
 842  *   Number of CBs contained in one operation.
 843  */
 844 static inline int
 845 fpga_dma_desc_ld_fill(struct rte_bbdev_dec_op *op,
 846                 struct fpga_dma_dec_desc *desc,
 847                 struct rte_mbuf *input, struct rte_mbuf *output,
 848                 uint16_t harq_in_length,
 849                 uint32_t in_offset, uint32_t out_offset,
 850                 uint32_t harq_offset,
 851                 uint16_t desc_offset,
 852                 uint8_t cbs_in_op)
 853 {
 854         /* reset */
 855         desc->done = 0;
 856         desc->error = 0;
 857         /* Set inbound data buffer address */
 858         desc->in_addr_hi = (uint32_t)(
 859                         rte_pktmbuf_mtophys_offset(input, in_offset) >> 32);
 860         desc->in_addr_lw = (uint32_t)(
 861                         rte_pktmbuf_mtophys_offset(input, in_offset));
 862         desc->rm_e = op->ldpc_dec.cb_params.e;
 863         desc->harq_input_length = harq_in_length;
 864         desc->et_dis = !check_bit(op->ldpc_dec.op_flags,
 865                         RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE);
 866         desc->rv = op->ldpc_dec.rv_index;
 867         desc->crc24b_ind = check_bit(op->ldpc_dec.op_flags,
 868                         RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK);
 869         desc->drop_crc24b = check_bit(op->ldpc_dec.op_flags,
 870                         RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP);
 871         desc->desc_idx = desc_offset;
 872         desc->ncb = op->ldpc_dec.n_cb;
 873         desc->num_null = op->ldpc_dec.n_filler;
 874         desc->hbstroe_offset = harq_offset >> 10;
 875         desc->zc = op->ldpc_dec.z_c;
 876         desc->harqin_en = check_bit(op->ldpc_dec.op_flags,
 877                         RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE);
 878         desc->bg_idx = op->ldpc_dec.basegraph - 1;
 879         desc->max_iter = op->ldpc_dec.iter_max;
 880         desc->qm_idx = op->ldpc_dec.q_m / 2;
 881         desc->out_addr_hi = (uint32_t)(
 882                         rte_pktmbuf_mtophys_offset(output, out_offset) >> 32);
 883         desc->out_addr_lw = (uint32_t)(
 884                         rte_pktmbuf_mtophys_offset(output, out_offset));
 885         /* Save software context needed for dequeue */
 886         desc->op_addr = op;
 887         /* Set total number of CBs in an op */
 888         desc->cbs_in_op = cbs_in_op;
 889
 890         return 0;
 891 }
 892
 893 #ifdef RTE_LIBRTE_BBDEV_DEBUG
 894 /* Validates LDPC encoder parameters */
 895 static int
 896 validate_enc_op(struct rte_bbdev_enc_op *op __rte_unused)
 897 {
 898         struct rte_bbdev_op_ldpc_enc *ldpc_enc = &op->ldpc_enc;
 899         struct rte_bbdev_op_enc_ldpc_cb_params *cb = NULL;
 900         struct rte_bbdev_op_enc_ldpc_tb_params *tb = NULL;
 901
 902
 903         if (ldpc_enc->input.length >
 904                         RTE_BBDEV_LDPC_MAX_CB_SIZE >> 3) {
 905                 rte_bbdev_log(ERR, "CB size (%u) is too big, max: %d",
 906                                 ldpc_enc->input.length,
 907                                 RTE_BBDEV_LDPC_MAX_CB_SIZE);
 908                 return -1;
 909         }
 910
 911         if (op->mempool == NULL) {
 912                 rte_bbdev_log(ERR, "Invalid mempool pointer");
 913                 return -1;
 914         }
 915         if (ldpc_enc->input.data == NULL) {
 916                 rte_bbdev_log(ERR, "Invalid input pointer");
 917                 return -1;
 918         }
 919         if (ldpc_enc->output.data == NULL) {
 920                 rte_bbdev_log(ERR, "Invalid output pointer");
 921                 return -1;
 922         }
 923         if ((ldpc_enc->basegraph > 2) || (ldpc_enc->basegraph == 0)) {
 924                 rte_bbdev_log(ERR,
 925                                 "basegraph (%u) is out of range 1 <= value <= 2",
 926                                 ldpc_enc->basegraph);
 927                 return -1;
 928         }
 929         if (ldpc_enc->code_block_mode > 1) {
 930                 rte_bbdev_log(ERR,
 931                                 "code_block_mode (%u) is out of range 0:Tb 1:CB",
 932                                 ldpc_enc->code_block_mode);
 933                 return -1;
 934         }
 935
 936         if (ldpc_enc->code_block_mode == 0) {
 937                 tb = &ldpc_enc->tb_params;
 938                 if (tb->c == 0) {
 939                         rte_bbdev_log(ERR,
 940                                         "c (%u) is out of range 1 <= value <= %u",
 941                                         tb->c, RTE_BBDEV_LDPC_MAX_CODE_BLOCKS);
 942                         return -1;
 943                 }
 944                 if (tb->cab > tb->c) {
 945                         rte_bbdev_log(ERR,
 946                                         "cab (%u) is greater than c (%u)",
 947                                         tb->cab, tb->c);
 948                         return -1;
 949                 }
 950                 if ((tb->ea < RTE_BBDEV_LDPC_MIN_CB_SIZE)
 951                                 && tb->r < tb->cab) {
 952                         rte_bbdev_log(ERR,
 953                                         "ea (%u) is less than %u or it is not even",
 954                                         tb->ea, RTE_BBDEV_LDPC_MIN_CB_SIZE);
 955                         return -1;
 956                 }
 957                 if ((tb->eb < RTE_BBDEV_LDPC_MIN_CB_SIZE)
 958                                 && tb->c > tb->cab) {
 959                         rte_bbdev_log(ERR,
 960                                         "eb (%u) is less than %u",
 961                                         tb->eb, RTE_BBDEV_LDPC_MIN_CB_SIZE);
 962                         return -1;
 963                 }
 964                 if (tb->r > (tb->c - 1)) {
 965                         rte_bbdev_log(ERR,
 966                                         "r (%u) is greater than c - 1 (%u)",
 967                                         tb->r, tb->c - 1);
 968                         return -1;
 969                 }
 970         } else {
 971                 cb = &ldpc_enc->cb_params;
 972                 if (cb->e < RTE_BBDEV_LDPC_MIN_CB_SIZE) {
 973                         rte_bbdev_log(ERR,
 974                                         "e (%u) is less than %u or it is not even",
 975                                         cb->e, RTE_BBDEV_LDPC_MIN_CB_SIZE);
 976                         return -1;
 977                 }
 978         }
 979         return 0;
 980 }
 981 #endif
 982
 983 static inline char *
 984 mbuf_append(struct rte_mbuf *m_head, struct rte_mbuf *m, uint16_t len)
 985 {
 986         if (unlikely(len > rte_pktmbuf_tailroom(m)))
 987                 return NULL;
 988
 989         char *tail = (char *)m->buf_addr + m->data_off + m->data_len;
 990         m->data_len = (uint16_t)(m->data_len + len);
 991         m_head->pkt_len  = (m_head->pkt_len + len);
 992         return tail;
 993 }
 994
 995 #ifdef RTE_LIBRTE_BBDEV_DEBUG
 996 /* Validates LDPC decoder parameters */
 997 static int
 998 validate_dec_op(struct rte_bbdev_dec_op *op __rte_unused)
 999 {
1000         struct rte_bbdev_op_ldpc_dec *ldpc_dec = &op->ldpc_dec;
1001         struct rte_bbdev_op_dec_ldpc_cb_params *cb = NULL;
1002         struct rte_bbdev_op_dec_ldpc_tb_params *tb = NULL;
1003
1004         if (op->mempool == NULL) {
1005                 rte_bbdev_log(ERR, "Invalid mempool pointer");
1006                 return -1;
1007         }
1008         if (ldpc_dec->rv_index > 3) {
1009                 rte_bbdev_log(ERR,
1010                                 "rv_index (%u) is out of range 0 <= value <= 3",
1011                                 ldpc_dec->rv_index);
1012                 return -1;
1013         }
1014
1015         if (ldpc_dec->iter_max == 0) {
1016                 rte_bbdev_log(ERR,
1017                                 "iter_max (%u) is equal to 0",
1018                                 ldpc_dec->iter_max);
1019                 return -1;
1020         }
1021
1022         if (ldpc_dec->code_block_mode > 1) {
1023                 rte_bbdev_log(ERR,
1024                                 "code_block_mode (%u) is out of range 0 <= value <= 1",
1025                                 ldpc_dec->code_block_mode);
1026                 return -1;
1027         }
1028
1029         if (ldpc_dec->code_block_mode == 0) {
1030                 tb = &ldpc_dec->tb_params;
1031                 if (tb->c < 1) {
1032                         rte_bbdev_log(ERR,
1033                                         "c (%u) is out of range 1 <= value <= %u",
1034                                         tb->c, RTE_BBDEV_LDPC_MAX_CODE_BLOCKS);
1035                         return -1;
1036                 }
1037                 if (tb->cab > tb->c) {
1038                         rte_bbdev_log(ERR,
1039                                         "cab (%u) is greater than c (%u)",
1040                                         tb->cab, tb->c);
1041                         return -1;
1042                 }
1043         } else {
1044                 cb = &ldpc_dec->cb_params;
1045                 if (cb->e < RTE_BBDEV_LDPC_MIN_CB_SIZE) {
1046                         rte_bbdev_log(ERR,
1047                                         "e (%u) is out of range %u <= value <= %u",
1048                                         cb->e, RTE_BBDEV_LDPC_MIN_CB_SIZE,
1049                                         RTE_BBDEV_LDPC_MAX_CB_SIZE);
1050                         return -1;
1051                 }
1052         }
1053
1054         return 0;
1055 }
1056 #endif
1057
1058 static inline int
1059 fpga_harq_write_loopback(struct fpga_5gnr_fec_device *fpga_dev,
1060                 struct rte_mbuf *harq_input, uint16_t harq_in_length,
1061                 uint32_t harq_in_offset, uint32_t harq_out_offset)
1062 {
1063         uint32_t out_offset = harq_out_offset;
1064         uint32_t in_offset = harq_in_offset;
1065         uint32_t left_length = harq_in_length;
1066         uint32_t reg_32, increment = 0;
1067         uint64_t *input = NULL;
1068         uint32_t last_transaction = left_length
1069                         % FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
1070         uint64_t last_word;
1071
1072         if (last_transaction > 0)
1073                 left_length -= last_transaction;
1074
1075         /*
1076          * Get HARQ buffer size for each VF/PF: When 0x00, there is no
1077          * available DDR space for the corresponding VF/PF.
1078          */
1079         reg_32 = fpga_reg_read_32(fpga_dev->mmio_base,
1080                         FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS);
1081         if (reg_32 < harq_in_length) {
1082                 left_length = reg_32;
1083                 rte_bbdev_log(ERR, "HARQ in length > HARQ buffer size\n");
1084         }
1085
1086         input = (uint64_t *)rte_pktmbuf_mtod_offset(harq_input,
1087                         uint8_t *, in_offset);
1088
1089         while (left_length > 0) {
1090                 if (fpga_reg_read_8(fpga_dev->mmio_base,
1091                                 FPGA_5GNR_FEC_DDR4_ADDR_RDY_REGS) ==  1) {
1092                         fpga_reg_write_32(fpga_dev->mmio_base,
1093                                         FPGA_5GNR_FEC_DDR4_WR_ADDR_REGS,
1094                                         out_offset);
1095                         fpga_reg_write_64(fpga_dev->mmio_base,
1096                                         FPGA_5GNR_FEC_DDR4_WR_DATA_REGS,
1097                                         input[increment]);
1098                         left_length -= FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
1099                         out_offset += FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
1100                         increment++;
1101                         fpga_reg_write_8(fpga_dev->mmio_base,
1102                                         FPGA_5GNR_FEC_DDR4_WR_DONE_REGS, 1);
1103                 }
1104         }
1105         while (last_transaction > 0) {
1106                 if (fpga_reg_read_8(fpga_dev->mmio_base,
1107                                 FPGA_5GNR_FEC_DDR4_ADDR_RDY_REGS) ==  1) {
1108                         fpga_reg_write_32(fpga_dev->mmio_base,
1109                                         FPGA_5GNR_FEC_DDR4_WR_ADDR_REGS,
1110                                         out_offset);
1111                         last_word = input[increment];
1112                         last_word &= (uint64_t)(1 << (last_transaction * 4))
1113                                         - 1;
1114                         fpga_reg_write_64(fpga_dev->mmio_base,
1115                                         FPGA_5GNR_FEC_DDR4_WR_DATA_REGS,
1116                                         last_word);
1117                         fpga_reg_write_8(fpga_dev->mmio_base,
1118                                         FPGA_5GNR_FEC_DDR4_WR_DONE_REGS, 1);
1119                         last_transaction = 0;
1120                 }
1121         }
1122         return 1;
1123 }
1124
1125 static inline int
1126 fpga_harq_read_loopback(struct fpga_5gnr_fec_device *fpga_dev,
1127                 struct rte_mbuf *harq_output, uint16_t harq_in_length,
1128                 uint32_t harq_in_offset, uint32_t harq_out_offset)
1129 {
1130         uint32_t left_length, in_offset = harq_in_offset;
1131         uint64_t reg;
1132         uint32_t increment = 0;
1133         uint64_t *input = NULL;
1134         uint32_t last_transaction = harq_in_length
1135                         % FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
1136
1137         if (last_transaction > 0)
1138                 harq_in_length += (8 - last_transaction);
1139
1140         reg = fpga_reg_read_32(fpga_dev->mmio_base,
1141                         FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS);
1142         if (reg < harq_in_length) {
1143                 harq_in_length = reg;
1144                 rte_bbdev_log(ERR, "HARQ in length > HARQ buffer size\n");
1145         }
1146
1147         if (!mbuf_append(harq_output, harq_output, harq_in_length)) {
1148                 rte_bbdev_log(ERR, "HARQ output buffer warning %d %d\n",
1149                                 harq_output->buf_len -
1150                                 rte_pktmbuf_headroom(harq_output),
1151                                 harq_in_length);
1152                 harq_in_length = harq_output->buf_len -
1153                                 rte_pktmbuf_headroom(harq_output);
1154                 if (!mbuf_append(harq_output, harq_output, harq_in_length)) {
1155                         rte_bbdev_log(ERR, "HARQ output buffer issue %d %d\n",
1156                                         harq_output->buf_len, harq_in_length);
1157                         return -1;
1158                 }
1159         }
1160         left_length = harq_in_length;
1161
1162         input = (uint64_t *)rte_pktmbuf_mtod_offset(harq_output,
1163                         uint8_t *, harq_out_offset);
1164
1165         while (left_length > 0) {
1166                 fpga_reg_write_32(fpga_dev->mmio_base,
1167                         FPGA_5GNR_FEC_DDR4_RD_ADDR_REGS, in_offset);
1168                 fpga_reg_write_8(fpga_dev->mmio_base,
1169                                 FPGA_5GNR_FEC_DDR4_RD_DONE_REGS, 1);
1170                 reg = fpga_reg_read_8(fpga_dev->mmio_base,
1171                         FPGA_5GNR_FEC_DDR4_RD_RDY_REGS);
1172                 while (reg != 1) {
1173                         reg = fpga_reg_read_8(fpga_dev->mmio_base,
1174                                 FPGA_5GNR_FEC_DDR4_RD_RDY_REGS);
1175                         if (reg == FPGA_DDR_OVERFLOW) {
1176                                 rte_bbdev_log(ERR,
1177                                                 "Read address is overflow!\n");
1178                                 return -1;
1179                         }
1180                 }
1181                 input[increment] = fpga_reg_read_64(fpga_dev->mmio_base,
1182                         FPGA_5GNR_FEC_DDR4_RD_DATA_REGS);
1183                 left_length -= FPGA_5GNR_FEC_DDR_RD_DATA_LEN_IN_BYTES;
1184                 in_offset += FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES;
1185                 increment++;
1186                 fpga_reg_write_8(fpga_dev->mmio_base,
1187                                 FPGA_5GNR_FEC_DDR4_RD_DONE_REGS, 0);
1188         }
1189         return 1;
1190 }
1191
1192 static inline int
1193 enqueue_ldpc_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op *op,
1194                 uint16_t desc_offset)
1195 {
1196         union fpga_dma_desc *desc;
1197         int ret;
1198         uint8_t c, crc24_bits = 0;
1199         struct rte_bbdev_op_ldpc_enc *enc = &op->ldpc_enc;
1200         uint16_t in_offset = enc->input.offset;
1201         uint16_t out_offset = enc->output.offset;
1202         struct rte_mbuf *m_in = enc->input.data;
1203         struct rte_mbuf *m_out = enc->output.data;
1204         struct rte_mbuf *m_out_head = enc->output.data;
1205         uint32_t in_length, out_length, e;
1206         uint16_t total_left = enc->input.length;
1207         uint16_t ring_offset;
1208         uint16_t K, k_;
1209
1210 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1211         /* Validate op structure */
1212         /* FIXME */
1213         if (validate_enc_op(op) == -1) {
1214                 rte_bbdev_log(ERR, "LDPC encoder validation failed");
1215                 return -EINVAL;
1216         }
1217 #endif
1218
1219         /* Clear op status */
1220         op->status = 0;
1221
1222         if (m_in == NULL || m_out == NULL) {
1223                 rte_bbdev_log(ERR, "Invalid mbuf pointer");
1224                 op->status = 1 << RTE_BBDEV_DATA_ERROR;
1225                 return -EINVAL;
1226         }
1227
1228         if (enc->op_flags & RTE_BBDEV_LDPC_CRC_24B_ATTACH)
1229                 crc24_bits = 24;
1230
1231         if (enc->code_block_mode == 0) {
1232                 /* For Transport Block mode */
1233                 /* FIXME */
1234                 c = enc->tb_params.c;
1235                 e = enc->tb_params.ea;
1236         } else { /* For Code Block mode */
1237                 c = 1;
1238                 e = enc->cb_params.e;
1239         }
1240
1241         /* Update total_left */
1242         K = (enc->basegraph == 1 ? 22 : 10) * enc->z_c;
1243         k_ = K - enc->n_filler;
1244         in_length = (k_ - crc24_bits) >> 3;
1245         out_length = (e + 7) >> 3;
1246
1247         total_left = rte_pktmbuf_data_len(m_in) - in_offset;
1248
1249         /* Update offsets */
1250         if (total_left != in_length) {
1251                 op->status |= 1 << RTE_BBDEV_DATA_ERROR;
1252                 rte_bbdev_log(ERR,
1253                                 "Mismatch between mbuf length and included CBs sizes %d",
1254                                 total_left);
1255         }
1256
1257         mbuf_append(m_out_head, m_out, out_length);
1258
1259         /* Offset into the ring */
1260         ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask);
1261         /* Setup DMA Descriptor */
1262         desc = q->ring_addr + ring_offset;
1263
1264         ret = fpga_dma_desc_te_fill(op, &desc->enc_req, m_in, m_out,
1265                         k_, e, in_offset, out_offset, ring_offset, c);
1266         if (unlikely(ret < 0))
1267                 return ret;
1268
1269         /* Update lengths */
1270         total_left -= in_length;
1271         op->ldpc_enc.output.length += out_length;
1272
1273         if (total_left > 0) {
1274                 rte_bbdev_log(ERR,
1275                         "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u",
1276                                 total_left, in_length);
1277                 return -1;
1278         }
1279
1280 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1281         print_dma_enc_desc_debug_info(desc);
1282 #endif
1283         return 1;
1284 }
1285
1286 static inline int
1287 enqueue_ldpc_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op *op,
1288                 uint16_t desc_offset)
1289 {
1290         union fpga_dma_desc *desc;
1291         int ret;
1292         uint16_t ring_offset;
1293         uint8_t c;
1294         uint16_t e, in_length, out_length, k0, l, seg_total_left, sys_cols;
1295         uint16_t K, parity_offset, harq_in_length = 0, harq_out_length = 0;
1296         uint16_t crc24_overlap = 0;
1297         struct rte_bbdev_op_ldpc_dec *dec = &op->ldpc_dec;
1298         struct rte_mbuf *m_in = dec->input.data;
1299         struct rte_mbuf *m_out = dec->hard_output.data;
1300         struct rte_mbuf *m_out_head = dec->hard_output.data;
1301         uint16_t in_offset = dec->input.offset;
1302         uint16_t out_offset = dec->hard_output.offset;
1303         uint32_t harq_offset = 0;
1304
1305 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1306                 /* Validate op structure */
1307                 if (validate_dec_op(op) == -1) {
1308                         rte_bbdev_log(ERR, "LDPC decoder validation failed");
1309                         return -EINVAL;
1310                 }
1311 #endif
1312
1313         /* Clear op status */
1314         op->status = 0;
1315
1316         /* Setup DMA Descriptor */
1317         ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask);
1318         desc = q->ring_addr + ring_offset;
1319
1320         if (check_bit(dec->op_flags,
1321                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1322                 struct rte_mbuf *harq_in = dec->harq_combined_input.data;
1323                 struct rte_mbuf *harq_out = dec->harq_combined_output.data;
1324                 harq_in_length = dec->harq_combined_input.length;
1325                 uint32_t harq_in_offset = dec->harq_combined_input.offset;
1326                 uint32_t harq_out_offset = dec->harq_combined_output.offset;
1327
1328                 if (check_bit(dec->op_flags,
1329                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE
1330                                 )) {
1331                         ret = fpga_harq_write_loopback(q->d, harq_in,
1332                                         harq_in_length, harq_in_offset,
1333                                         harq_out_offset);
1334                 } else if (check_bit(dec->op_flags,
1335                                 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE
1336                                 )) {
1337                         ret = fpga_harq_read_loopback(q->d, harq_out,
1338                                 harq_in_length, harq_in_offset,
1339                                 harq_out_offset);
1340                         dec->harq_combined_output.length = harq_in_length;
1341                 } else {
1342                         rte_bbdev_log(ERR, "OP flag Err!");
1343                         ret = -1;
1344                 }
1345                 /* Set descriptor for dequeue */
1346                 desc->dec_req.done = 1;
1347                 desc->dec_req.error = 0;
1348                 desc->dec_req.op_addr = op;
1349                 desc->dec_req.cbs_in_op = 1;
1350                 /* Mark this dummy descriptor to be dropped by HW */
1351                 desc->dec_req.desc_idx = (ring_offset + 1)
1352                                 & q->sw_ring_wrap_mask;
1353                 return ret; /* Error or number of CB */
1354         }
1355
1356         if (m_in == NULL || m_out == NULL) {
1357                 rte_bbdev_log(ERR, "Invalid mbuf pointer");
1358                 op->status = 1 << RTE_BBDEV_DATA_ERROR;
1359                 return -1;
1360         }
1361
1362         c = 1;
1363         e = dec->cb_params.e;
1364
1365         if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP))
1366                 crc24_overlap = 24;
1367
1368         sys_cols = (dec->basegraph == 1) ? 22 : 10;
1369         K = sys_cols * dec->z_c;
1370         parity_offset = K - 2 * dec->z_c;
1371
1372         out_length = ((K - crc24_overlap - dec->n_filler) >> 3);
1373         in_length = e;
1374         seg_total_left = dec->input.length;
1375
1376         if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) {
1377                 harq_in_length = RTE_MIN(dec->harq_combined_input.length,
1378                                 (uint32_t)dec->n_cb);
1379         }
1380
1381         if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE)) {
1382                 k0 = get_k0(dec->n_cb, dec->z_c,
1383                                 dec->basegraph, dec->rv_index);
1384                 if (k0 > parity_offset)
1385                         l = k0 + e;
1386                 else
1387                         l = k0 + e + dec->n_filler;
1388                 harq_out_length = RTE_MIN(RTE_MAX(harq_in_length, l),
1389                                 dec->n_cb - dec->n_filler);
1390                 dec->harq_combined_output.length = harq_out_length;
1391         }
1392
1393         mbuf_append(m_out_head, m_out, out_length);
1394         if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE))
1395                 harq_offset = dec->harq_combined_input.offset;
1396         else if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE))
1397                 harq_offset = dec->harq_combined_output.offset;
1398
1399         if ((harq_offset & 0x3FF) > 0) {
1400                 rte_bbdev_log(ERR, "Invalid HARQ offset %d", harq_offset);
1401                 op->status = 1 << RTE_BBDEV_DATA_ERROR;
1402                 return -1;
1403         }
1404
1405         ret = fpga_dma_desc_ld_fill(op, &desc->dec_req, m_in, m_out,
1406                 harq_in_length, in_offset, out_offset, harq_offset,
1407                 ring_offset, c);
1408         if (unlikely(ret < 0))
1409                 return ret;
1410         /* Update lengths */
1411         seg_total_left -= in_length;
1412         op->ldpc_dec.hard_output.length += out_length;
1413         if (seg_total_left > 0) {
1414                 rte_bbdev_log(ERR,
1415                                 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u",
1416                                 seg_total_left, in_length);
1417                 return -1;
1418         }
1419
1420 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1421         print_dma_dec_desc_debug_info(desc);
1422 #endif
1423
1424         return 1;
1425 }
1426
1427 static uint16_t
1428 fpga_enqueue_ldpc_enc(struct rte_bbdev_queue_data *q_data,
1429                 struct rte_bbdev_enc_op **ops, uint16_t num)
1430 {
1431         uint16_t i, total_enqueued_cbs = 0;
1432         int32_t avail;
1433         int enqueued_cbs;
1434         struct fpga_queue *q = q_data->queue_private;
1435         union fpga_dma_desc *desc;
1436
1437         /* Check if queue is not full */
1438         if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) ==
1439                         q->head_free_desc))
1440                 return 0;
1441
1442         /* Calculates available space */
1443         avail = (q->head_free_desc > q->tail) ?
1444                 q->head_free_desc - q->tail - 1 :
1445                 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1;
1446
1447         for (i = 0; i < num; ++i) {
1448
1449                 /* Check if there is available space for further
1450                  * processing
1451                  */
1452                 if (unlikely(avail - 1 < 0))
1453                         break;
1454                 avail -= 1;
1455                 enqueued_cbs = enqueue_ldpc_enc_one_op_cb(q, ops[i],
1456                                 total_enqueued_cbs);
1457
1458                 if (enqueued_cbs < 0)
1459                         break;
1460
1461                 total_enqueued_cbs += enqueued_cbs;
1462
1463                 rte_bbdev_log_debug("enqueuing enc ops [%d/%d] | head %d | tail %d",
1464                                 total_enqueued_cbs, num,
1465                                 q->head_free_desc, q->tail);
1466         }
1467
1468         /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt
1469          * only when all previous CBs were already processed.
1470          */
1471         desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1)
1472                         & q->sw_ring_wrap_mask);
1473         desc->enc_req.irq_en = q->irq_enable;
1474
1475         fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats);
1476
1477         /* Update stats */
1478         q_data->queue_stats.enqueued_count += i;
1479         q_data->queue_stats.enqueue_err_count += num - i;
1480
1481         return i;
1482 }
1483
1484 static uint16_t
1485 fpga_enqueue_ldpc_dec(struct rte_bbdev_queue_data *q_data,
1486                 struct rte_bbdev_dec_op **ops, uint16_t num)
1487 {
1488         uint16_t i, total_enqueued_cbs = 0;
1489         int32_t avail;
1490         int enqueued_cbs;
1491         struct fpga_queue *q = q_data->queue_private;
1492         union fpga_dma_desc *desc;
1493
1494         /* Check if queue is not full */
1495         if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) ==
1496                         q->head_free_desc))
1497                 return 0;
1498
1499         /* Calculates available space */
1500         avail = (q->head_free_desc > q->tail) ?
1501                 q->head_free_desc - q->tail - 1 :
1502                 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1;
1503
1504         for (i = 0; i < num; ++i) {
1505
1506                 /* Check if there is available space for further
1507                  * processing
1508                  */
1509                 if (unlikely(avail - 1 < 0))
1510                         break;
1511                 avail -= 1;
1512                 enqueued_cbs = enqueue_ldpc_dec_one_op_cb(q, ops[i],
1513                                 total_enqueued_cbs);
1514
1515                 if (enqueued_cbs < 0)
1516                         break;
1517
1518                 total_enqueued_cbs += enqueued_cbs;
1519
1520                 rte_bbdev_log_debug("enqueuing dec ops [%d/%d] | head %d | tail %d",
1521                                 total_enqueued_cbs, num,
1522                                 q->head_free_desc, q->tail);
1523         }
1524
1525         /* Update stats */
1526         q_data->queue_stats.enqueued_count += i;
1527         q_data->queue_stats.enqueue_err_count += num - i;
1528
1529         /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt
1530          * only when all previous CBs were already processed.
1531          */
1532         desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1)
1533                         & q->sw_ring_wrap_mask);
1534         desc->enc_req.irq_en = q->irq_enable;
1535         fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats);
1536         return i;
1537 }
1538
1539
1540 static inline int
1541 dequeue_ldpc_enc_one_op_cb(struct fpga_queue *q,
1542                 struct rte_bbdev_enc_op **op,
1543                 uint16_t desc_offset)
1544 {
1545         union fpga_dma_desc *desc;
1546         int desc_error;
1547         /* Set current desc */
1548         desc = q->ring_addr + ((q->head_free_desc + desc_offset)
1549                         & q->sw_ring_wrap_mask);
1550
1551         /*check if done */
1552         if (desc->enc_req.done == 0)
1553                 return -1;
1554
1555         /* make sure the response is read atomically */
1556         rte_smp_rmb();
1557
1558         rte_bbdev_log_debug("DMA response desc %p", desc);
1559
1560 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1561         print_dma_enc_desc_debug_info(desc);
1562 #endif
1563
1564         *op = desc->enc_req.op_addr;
1565         /* Check the descriptor error field, return 1 on error */
1566         desc_error = check_desc_error(desc->enc_req.error);
1567         (*op)->status = desc_error << RTE_BBDEV_DATA_ERROR;
1568
1569         return 1;
1570 }
1571
1572
1573 static inline int
1574 dequeue_ldpc_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op **op,
1575                 uint16_t desc_offset)
1576 {
1577         union fpga_dma_desc *desc;
1578         int desc_error;
1579         /* Set descriptor */
1580         desc = q->ring_addr + ((q->head_free_desc + desc_offset)
1581                         & q->sw_ring_wrap_mask);
1582
1583         /* Verify done bit is set */
1584         if (desc->dec_req.done == 0)
1585                 return -1;
1586
1587         /* make sure the response is read atomically */
1588         rte_smp_rmb();
1589
1590 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1591         print_dma_dec_desc_debug_info(desc);
1592 #endif
1593
1594         *op = desc->dec_req.op_addr;
1595
1596         if (check_bit((*op)->ldpc_dec.op_flags,
1597                         RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) {
1598                 (*op)->status = 0;
1599                 return 1;
1600         }
1601
1602         /* FPGA reports iterations based on round-up minus 1 */
1603         (*op)->ldpc_dec.iter_count = desc->dec_req.iter + 1;
1604         /* CRC Check criteria */
1605         if (desc->dec_req.crc24b_ind && !(desc->dec_req.crcb_pass))
1606                 (*op)->status = 1 << RTE_BBDEV_CRC_ERROR;
1607         /* et_pass = 0 when decoder fails */
1608         (*op)->status |= !(desc->dec_req.et_pass) << RTE_BBDEV_SYNDROME_ERROR;
1609         /* Check the descriptor error field, return 1 on error */
1610         desc_error = check_desc_error(desc->dec_req.error);
1611         (*op)->status |= desc_error << RTE_BBDEV_DATA_ERROR;
1612         return 1;
1613 }
1614
1615 static uint16_t
1616 fpga_dequeue_ldpc_enc(struct rte_bbdev_queue_data *q_data,
1617                 struct rte_bbdev_enc_op **ops, uint16_t num)
1618 {
1619         struct fpga_queue *q = q_data->queue_private;
1620         uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask;
1621         uint16_t i;
1622         uint16_t dequeued_cbs = 0;
1623         int ret;
1624
1625         for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) {
1626                 ret = dequeue_ldpc_enc_one_op_cb(q, &ops[i], dequeued_cbs);
1627
1628                 if (ret < 0)
1629                         break;
1630
1631                 dequeued_cbs += ret;
1632
1633                 rte_bbdev_log_debug("dequeuing enc ops [%d/%d] | head %d | tail %d",
1634                                 dequeued_cbs, num, q->head_free_desc, q->tail);
1635         }
1636
1637         /* Update head */
1638         q->head_free_desc = (q->head_free_desc + dequeued_cbs) &
1639                         q->sw_ring_wrap_mask;
1640
1641         /* Update stats */
1642         q_data->queue_stats.dequeued_count += i;
1643
1644         return i;
1645 }
1646
1647 static uint16_t
1648 fpga_dequeue_ldpc_dec(struct rte_bbdev_queue_data *q_data,
1649                 struct rte_bbdev_dec_op **ops, uint16_t num)
1650 {
1651         struct fpga_queue *q = q_data->queue_private;
1652         uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask;
1653         uint16_t i;
1654         uint16_t dequeued_cbs = 0;
1655         int ret;
1656
1657         for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) {
1658                 ret = dequeue_ldpc_dec_one_op_cb(q, &ops[i], dequeued_cbs);
1659
1660                 if (ret < 0)
1661                         break;
1662
1663                 dequeued_cbs += ret;
1664
1665                 rte_bbdev_log_debug("dequeuing dec ops [%d/%d] | head %d | tail %d",
1666                                 dequeued_cbs, num, q->head_free_desc, q->tail);
1667         }
1668
1669         /* Update head */
1670         q->head_free_desc = (q->head_free_desc + dequeued_cbs) &
1671                         q->sw_ring_wrap_mask;
1672
1673         /* Update stats */
1674         q_data->queue_stats.dequeued_count += i;
1675
1676         return i;
1677 }
1678
1679
1680 /* Initialization Function */
1681 static void
1682 fpga_5gnr_fec_init(struct rte_bbdev *dev, struct rte_pci_driver *drv)
1683 {
1684         struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device);
1685
1686         dev->dev_ops = &fpga_ops;
1687         dev->enqueue_ldpc_enc_ops = fpga_enqueue_ldpc_enc;
1688         dev->enqueue_ldpc_dec_ops = fpga_enqueue_ldpc_dec;
1689         dev->dequeue_ldpc_enc_ops = fpga_dequeue_ldpc_enc;
1690         dev->dequeue_ldpc_dec_ops = fpga_dequeue_ldpc_dec;
1691
1692         ((struct fpga_5gnr_fec_device *) dev->data->dev_private)->pf_device =
1693                         !strcmp(drv->driver.name,
1694                                         RTE_STR(FPGA_5GNR_FEC_PF_DRIVER_NAME));
1695         ((struct fpga_5gnr_fec_device *) dev->data->dev_private)->mmio_base =
1696                         pci_dev->mem_resource[0].addr;
1697
1698         rte_bbdev_log_debug(
1699                         "Init device %s [%s] @ virtaddr %p phyaddr %#"PRIx64,
1700                         dev->device->driver->name, dev->data->name,
1701                         (void *)pci_dev->mem_resource[0].addr,
1702                         pci_dev->mem_resource[0].phys_addr);
1703 }
1704
1705 static int
1706 fpga_5gnr_fec_probe(struct rte_pci_driver *pci_drv,
1707         struct rte_pci_device *pci_dev)
1708 {
1709         struct rte_bbdev *bbdev = NULL;
1710         char dev_name[RTE_BBDEV_NAME_MAX_LEN];
1711
1712         if (pci_dev == NULL) {
1713                 rte_bbdev_log(ERR, "NULL PCI device");
1714                 return -EINVAL;
1715         }
1716
1717         rte_pci_device_name(&pci_dev->addr, dev_name, sizeof(dev_name));
1718
1719         /* Allocate memory to be used privately by drivers */
1720         bbdev = rte_bbdev_allocate(pci_dev->device.name);
1721         if (bbdev == NULL)
1722                 return -ENODEV;
1723
1724         /* allocate device private memory */
1725         bbdev->data->dev_private = rte_zmalloc_socket(dev_name,
1726                         sizeof(struct fpga_5gnr_fec_device),
1727                         RTE_CACHE_LINE_SIZE,
1728                         pci_dev->device.numa_node);
1729
1730         if (bbdev->data->dev_private == NULL) {
1731                 rte_bbdev_log(CRIT,
1732                                 "Allocate of %zu bytes for device \"%s\" failed",
1733                                 sizeof(struct fpga_5gnr_fec_device), dev_name);
1734                                 rte_bbdev_release(bbdev);
1735                         return -ENOMEM;
1736         }
1737
1738         /* Fill HW specific part of device structure */
1739         bbdev->device = &pci_dev->device;
1740         bbdev->intr_handle = &pci_dev->intr_handle;
1741         bbdev->data->socket_id = pci_dev->device.numa_node;
1742
1743         /* Invoke FEC FPGA device initialization function */
1744         fpga_5gnr_fec_init(bbdev, pci_drv);
1745
1746         rte_bbdev_log_debug("bbdev id = %u [%s]",
1747                         bbdev->data->dev_id, dev_name);
1748
1749         struct fpga_5gnr_fec_device *d = bbdev->data->dev_private;
1750         uint32_t version_id = fpga_reg_read_32(d->mmio_base,
1751                         FPGA_5GNR_FEC_VERSION_ID);
1752         rte_bbdev_log(INFO, "FEC FPGA RTL v%u.%u",
1753                 ((uint16_t)(version_id >> 16)), ((uint16_t)version_id));
1754
1755 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1756         if (!strcmp(bbdev->device->driver->name,
1757                         RTE_STR(FPGA_5GNR_FEC_PF_DRIVER_NAME)))
1758                 print_static_reg_debug_info(d->mmio_base);
1759 #endif
1760         return 0;
1761 }
1762
1763 static int
1764 fpga_5gnr_fec_remove(struct rte_pci_device *pci_dev)
1765 {
1766         struct rte_bbdev *bbdev;
1767         int ret;
1768         uint8_t dev_id;
1769
1770         if (pci_dev == NULL)
1771                 return -EINVAL;
1772
1773         /* Find device */
1774         bbdev = rte_bbdev_get_named_dev(pci_dev->device.name);
1775         if (bbdev == NULL) {
1776                 rte_bbdev_log(CRIT,
1777                                 "Couldn't find HW dev \"%s\" to uninitialise it",
1778                                 pci_dev->device.name);
1779                 return -ENODEV;
1780         }
1781         dev_id = bbdev->data->dev_id;
1782
1783         /* free device private memory before close */
1784         rte_free(bbdev->data->dev_private);
1785
1786         /* Close device */
1787         ret = rte_bbdev_close(dev_id);
1788         if (ret < 0)
1789                 rte_bbdev_log(ERR,
1790                                 "Device %i failed to close during uninit: %i",
1791                                 dev_id, ret);
1792
1793         /* release bbdev from library */
1794         ret = rte_bbdev_release(bbdev);
1795         if (ret)
1796                 rte_bbdev_log(ERR, "Device %i failed to uninit: %i", dev_id,
1797                                 ret);
1798
1799         rte_bbdev_log_debug("Destroyed bbdev = %u", dev_id);
1800
1801         return 0;
1802 }
1803
1804 static inline void
1805 set_default_fpga_conf(struct fpga_5gnr_fec_conf *def_conf)
1806 {
1807         /* clear default configuration before initialization */
1808         memset(def_conf, 0, sizeof(struct fpga_5gnr_fec_conf));
1809         /* Set pf mode to true */
1810         def_conf->pf_mode_en = true;
1811
1812         /* Set ratio between UL and DL to 1:1 (unit of weight is 3 CBs) */
1813         def_conf->ul_bandwidth = 3;
1814         def_conf->dl_bandwidth = 3;
1815
1816         /* Set Load Balance Factor to 64 */
1817         def_conf->dl_load_balance = 64;
1818         def_conf->ul_load_balance = 64;
1819 }
1820
1821 /* Initial configuration of FPGA 5GNR FEC device */
1822 int
1823 fpga_5gnr_fec_configure(const char *dev_name,
1824                 const struct fpga_5gnr_fec_conf *conf)
1825 {
1826         uint32_t payload_32, address;
1827         uint16_t payload_16;
1828         uint8_t payload_8;
1829         uint16_t q_id, vf_id, total_q_id, total_ul_q_id, total_dl_q_id;
1830         struct rte_bbdev *bbdev = rte_bbdev_get_named_dev(dev_name);
1831         struct fpga_5gnr_fec_conf def_conf;
1832
1833         if (bbdev == NULL) {
1834                 rte_bbdev_log(ERR,
1835                                 "Invalid dev_name (%s), or device is not yet initialised",
1836                                 dev_name);
1837                 return -ENODEV;
1838         }
1839
1840         struct fpga_5gnr_fec_device *d = bbdev->data->dev_private;
1841
1842         if (conf == NULL) {
1843                 rte_bbdev_log(ERR,
1844                                 "FPGA Configuration was not provided. Default configuration will be loaded.");
1845                 set_default_fpga_conf(&def_conf);
1846                 conf = &def_conf;
1847         }
1848
1849         /*
1850          * Configure UL:DL ratio.
1851          * [7:0]: UL weight
1852          * [15:8]: DL weight
1853          */
1854         payload_16 = (conf->dl_bandwidth << 8) | conf->ul_bandwidth;
1855         address = FPGA_5GNR_FEC_CONFIGURATION;
1856         fpga_reg_write_16(d->mmio_base, address, payload_16);
1857
1858         /* Clear all queues registers */
1859         payload_32 = FPGA_INVALID_HW_QUEUE_ID;
1860         for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) {
1861                 address = (q_id << 2) + FPGA_5GNR_FEC_QUEUE_MAP;
1862                 fpga_reg_write_32(d->mmio_base, address, payload_32);
1863         }
1864
1865         /*
1866          * If PF mode is enabled allocate all queues for PF only.
1867          *
1868          * For VF mode each VF can have different number of UL and DL queues.
1869          * Total number of queues to configure cannot exceed FPGA
1870          * capabilities - 64 queues - 32 queues for UL and 32 queues for DL.
1871          * Queues mapping is done according to configuration:
1872          *
1873          * UL queues:
1874          * |                Q_ID              | VF_ID |
1875          * |                 0                |   0   |
1876          * |                ...               |   0   |
1877          * | conf->vf_dl_queues_number[0] - 1 |   0   |
1878          * | conf->vf_dl_queues_number[0]     |   1   |
1879          * |                ...               |   1   |
1880          * | conf->vf_dl_queues_number[1] - 1 |   1   |
1881          * |                ...               |  ...  |
1882          * | conf->vf_dl_queues_number[7] - 1 |   7   |
1883          *
1884          * DL queues:
1885          * |                Q_ID              | VF_ID |
1886          * |                 32               |   0   |
1887          * |                ...               |   0   |
1888          * | conf->vf_ul_queues_number[0] - 1 |   0   |
1889          * | conf->vf_ul_queues_number[0]     |   1   |
1890          * |                ...               |   1   |
1891          * | conf->vf_ul_queues_number[1] - 1 |   1   |
1892          * |                ...               |  ...  |
1893          * | conf->vf_ul_queues_number[7] - 1 |   7   |
1894          *
1895          * Example of configuration:
1896          * conf->vf_ul_queues_number[0] = 4;  -> 4 UL queues for VF0
1897          * conf->vf_dl_queues_number[0] = 4;  -> 4 DL queues for VF0
1898          * conf->vf_ul_queues_number[1] = 2;  -> 2 UL queues for VF1
1899          * conf->vf_dl_queues_number[1] = 2;  -> 2 DL queues for VF1
1900          *
1901          * UL:
1902          * | Q_ID | VF_ID |
1903          * |   0  |   0   |
1904          * |   1  |   0   |
1905          * |   2  |   0   |
1906          * |   3  |   0   |
1907          * |   4  |   1   |
1908          * |   5  |   1   |
1909          *
1910          * DL:
1911          * | Q_ID | VF_ID |
1912          * |  32  |   0   |
1913          * |  33  |   0   |
1914          * |  34  |   0   |
1915          * |  35  |   0   |
1916          * |  36  |   1   |
1917          * |  37  |   1   |
1918          */
1919         if (conf->pf_mode_en) {
1920                 payload_32 = 0x1;
1921                 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) {
1922                         address = (q_id << 2) + FPGA_5GNR_FEC_QUEUE_MAP;
1923                         fpga_reg_write_32(d->mmio_base, address, payload_32);
1924                 }
1925         } else {
1926                 /* Calculate total number of UL and DL queues to configure */
1927                 total_ul_q_id = total_dl_q_id = 0;
1928                 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) {
1929                         total_ul_q_id += conf->vf_ul_queues_number[vf_id];
1930                         total_dl_q_id += conf->vf_dl_queues_number[vf_id];
1931                 }
1932                 total_q_id = total_dl_q_id + total_ul_q_id;
1933                 /*
1934                  * Check if total number of queues to configure does not exceed
1935                  * FPGA capabilities (64 queues - 32 UL and 32 DL queues)
1936                  */
1937                 if ((total_ul_q_id > FPGA_NUM_UL_QUEUES) ||
1938                         (total_dl_q_id > FPGA_NUM_DL_QUEUES) ||
1939                         (total_q_id > FPGA_TOTAL_NUM_QUEUES)) {
1940                         rte_bbdev_log(ERR,
1941                                         "FPGA Configuration failed. Too many queues to configure: UL_Q %u, DL_Q %u, FPGA_Q %u",
1942                                         total_ul_q_id, total_dl_q_id,
1943                                         FPGA_TOTAL_NUM_QUEUES);
1944                         return -EINVAL;
1945                 }
1946                 total_ul_q_id = 0;
1947                 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) {
1948                         for (q_id = 0; q_id < conf->vf_ul_queues_number[vf_id];
1949                                         ++q_id, ++total_ul_q_id) {
1950                                 address = (total_ul_q_id << 2) +
1951                                                 FPGA_5GNR_FEC_QUEUE_MAP;
1952                                 payload_32 = ((0x80 + vf_id) << 16) | 0x1;
1953                                 fpga_reg_write_32(d->mmio_base, address,
1954                                                 payload_32);
1955                         }
1956                 }
1957                 total_dl_q_id = 0;
1958                 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) {
1959                         for (q_id = 0; q_id < conf->vf_dl_queues_number[vf_id];
1960                                         ++q_id, ++total_dl_q_id) {
1961                                 address = ((total_dl_q_id + FPGA_NUM_UL_QUEUES)
1962                                                 << 2) + FPGA_5GNR_FEC_QUEUE_MAP;
1963                                 payload_32 = ((0x80 + vf_id) << 16) | 0x1;
1964                                 fpga_reg_write_32(d->mmio_base, address,
1965                                                 payload_32);
1966                         }
1967                 }
1968         }
1969
1970         /* Setting Load Balance Factor */
1971         payload_16 = (conf->dl_load_balance << 8) | (conf->ul_load_balance);
1972         address = FPGA_5GNR_FEC_LOAD_BALANCE_FACTOR;
1973         fpga_reg_write_16(d->mmio_base, address, payload_16);
1974
1975         /* Setting length of ring descriptor entry */
1976         payload_16 = FPGA_RING_DESC_ENTRY_LENGTH;
1977         address = FPGA_5GNR_FEC_RING_DESC_LEN;
1978         fpga_reg_write_16(d->mmio_base, address, payload_16);
1979
1980         /* Setting FLR timeout value */
1981         payload_16 = conf->flr_time_out;
1982         address = FPGA_5GNR_FEC_FLR_TIME_OUT;
1983         fpga_reg_write_16(d->mmio_base, address, payload_16);
1984
1985         /* Queue PF/VF mapping table is ready */
1986         payload_8 = 0x1;
1987         address = FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE;
1988         fpga_reg_write_8(d->mmio_base, address, payload_8);
1989
1990         rte_bbdev_log_debug("PF FPGA 5GNR FEC configuration complete for %s",
1991                         dev_name);
1992
1993 #ifdef RTE_LIBRTE_BBDEV_DEBUG
1994         print_static_reg_debug_info(d->mmio_base);
1995 #endif
1996         return 0;
1997 }
1998
1999 /* FPGA 5GNR FEC PCI PF address map */
2000 static struct rte_pci_id pci_id_fpga_5gnr_fec_pf_map[] = {
2001         {
2002                 RTE_PCI_DEVICE(FPGA_5GNR_FEC_VENDOR_ID,
2003                                 FPGA_5GNR_FEC_PF_DEVICE_ID)
2004         },
2005         {.device_id = 0},
2006 };
2007
2008 static struct rte_pci_driver fpga_5gnr_fec_pci_pf_driver = {
2009         .probe = fpga_5gnr_fec_probe,
2010         .remove = fpga_5gnr_fec_remove,
2011         .id_table = pci_id_fpga_5gnr_fec_pf_map,
2012         .drv_flags = RTE_PCI_DRV_NEED_MAPPING
2013 };
2014
2015 /* FPGA 5GNR FEC PCI VF address map */
2016 static struct rte_pci_id pci_id_fpga_5gnr_fec_vf_map[] = {
2017         {
2018                 RTE_PCI_DEVICE(FPGA_5GNR_FEC_VENDOR_ID,
2019                                 FPGA_5GNR_FEC_VF_DEVICE_ID)
2020         },
2021         {.device_id = 0},
2022 };
2023
2024 static struct rte_pci_driver fpga_5gnr_fec_pci_vf_driver = {
2025         .probe = fpga_5gnr_fec_probe,
2026         .remove = fpga_5gnr_fec_remove,
2027         .id_table = pci_id_fpga_5gnr_fec_vf_map,
2028         .drv_flags = RTE_PCI_DRV_NEED_MAPPING
2029 };
2030
2031
2032 RTE_PMD_REGISTER_PCI(FPGA_5GNR_FEC_PF_DRIVER_NAME, fpga_5gnr_fec_pci_pf_driver);
2033 RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_PF_DRIVER_NAME,
2034                 pci_id_fpga_5gnr_fec_pf_map);
2035 RTE_PMD_REGISTER_PCI(FPGA_5GNR_FEC_VF_DRIVER_NAME, fpga_5gnr_fec_pci_vf_driver);
2036 RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_VF_DRIVER_NAME,
2037                 pci_id_fpga_5gnr_fec_vf_map);
2038
2039 RTE_INIT(fpga_5gnr_fec_init_log)
2040 {
2041         fpga_5gnr_fec_logtype = rte_log_register("pmd.bb.fpga_5gnr_fec");
2042         if (fpga_5gnr_fec_logtype >= 0)
2043 #ifdef RTE_LIBRTE_BBDEV_DEBUG
2044                 rte_log_set_level(fpga_5gnr_fec_logtype, RTE_LOG_DEBUG);
2045 #else
2046                 rte_log_set_level(fpga_5gnr_fec_logtype, RTE_LOG_NOTICE);
2047 #endif
2048 }