drivers/baseband/la12xx/bbdev_la12xx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2020-2021 NXP
   3  */
   4
   5 #include <string.h>
   6 #include <unistd.h>
   7 #include <fcntl.h>
   8 #include <sys/ioctl.h>
   9 #include <sys/mman.h>
  10 #include <dirent.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_bus_vdev.h>
  14 #include <rte_malloc.h>
  15 #include <rte_ring.h>
  16 #include <rte_kvargs.h>
  17
  18 #include <rte_bbdev.h>
  19 #include <rte_bbdev_pmd.h>
  20
  21 #include <bbdev_la12xx_pmd_logs.h>
  22 #include <bbdev_la12xx_ipc.h>
  23 #include <bbdev_la12xx.h>
  24
  25 #define DRIVER_NAME baseband_la12xx
  26
  27 /*  Initialisation params structure that can be used by LA12xx BBDEV driver */
  28 struct bbdev_la12xx_params {
  29         uint8_t queues_num; /*< LA12xx BBDEV queues number */
  30         int8_t modem_id; /*< LA12xx modem instance id */
  31 };
  32
  33 #define LA12XX_MAX_NB_QUEUES_ARG        "max_nb_queues"
  34 #define LA12XX_VDEV_MODEM_ID_ARG        "modem"
  35 #define LA12XX_MAX_MODEM 4
  36
  37 #define LA12XX_MAX_CORES        4
  38 #define LA12XX_LDPC_ENC_CORE    0
  39 #define LA12XX_LDPC_DEC_CORE    1
  40
  41 #define LA12XX_MAX_LDPC_ENC_QUEUES      4
  42 #define LA12XX_MAX_LDPC_DEC_QUEUES      4
  43
  44 static const char * const bbdev_la12xx_valid_params[] = {
  45         LA12XX_MAX_NB_QUEUES_ARG,
  46         LA12XX_VDEV_MODEM_ID_ARG,
  47 };
  48
  49 static const struct rte_bbdev_op_cap bbdev_capabilities[] = {
  50         {
  51                 .type   = RTE_BBDEV_OP_LDPC_ENC,
  52                 .cap.ldpc_enc = {
  53                         .capability_flags =
  54                                         RTE_BBDEV_LDPC_RATE_MATCH |
  55                                         RTE_BBDEV_LDPC_CRC_24A_ATTACH |
  56                                         RTE_BBDEV_LDPC_CRC_24B_ATTACH,
  57                         .num_buffers_src =
  58                                         RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
  59                         .num_buffers_dst =
  60                                         RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
  61                 }
  62         },
  63         {
  64                 .type   = RTE_BBDEV_OP_LDPC_DEC,
  65                 .cap.ldpc_dec = {
  66                         .capability_flags =
  67                                 RTE_BBDEV_LDPC_CRC_TYPE_24A_CHECK |
  68                                         RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK |
  69                                         RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP,
  70                         .num_buffers_src =
  71                                         RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
  72                         .num_buffers_hard_out =
  73                                         RTE_BBDEV_LDPC_MAX_CODE_BLOCKS,
  74                         .llr_size = 8,
  75                         .llr_decimals = 1,
  76                 }
  77         },
  78         RTE_BBDEV_END_OF_CAPABILITIES_LIST()
  79 };
  80
  81 static struct rte_bbdev_queue_conf default_queue_conf = {
  82         .queue_size = MAX_CHANNEL_DEPTH,
  83 };
  84
  85 /* Get device info */
  86 static void
  87 la12xx_info_get(struct rte_bbdev *dev __rte_unused,
  88                 struct rte_bbdev_driver_info *dev_info)
  89 {
  90         PMD_INIT_FUNC_TRACE();
  91
  92         dev_info->driver_name = RTE_STR(DRIVER_NAME);
  93         dev_info->max_num_queues = LA12XX_MAX_QUEUES;
  94         dev_info->queue_size_lim = MAX_CHANNEL_DEPTH;
  95         dev_info->hardware_accelerated = true;
  96         dev_info->max_dl_queue_priority = 0;
  97         dev_info->max_ul_queue_priority = 0;
  98         dev_info->data_endianness = RTE_BIG_ENDIAN;
  99         dev_info->default_queue_conf = default_queue_conf;
 100         dev_info->capabilities = bbdev_capabilities;
 101         dev_info->cpu_flag_reqs = NULL;
 102         dev_info->min_alignment = 64;
 103
 104         rte_bbdev_log_debug("got device info from %u", dev->data->dev_id);
 105 }
 106
 107 /* Release queue */
 108 static int
 109 la12xx_queue_release(struct rte_bbdev *dev, uint16_t q_id)
 110 {
 111         RTE_SET_USED(dev);
 112         RTE_SET_USED(q_id);
 113
 114         PMD_INIT_FUNC_TRACE();
 115
 116         return 0;
 117 }
 118
 119 #define HUGEPG_OFFSET(A) \
 120                 ((uint64_t) ((unsigned long) (A) \
 121                 - ((uint64_t)ipc_priv->hugepg_start.host_vaddr)))
 122
 123 #define MODEM_P2V(A) \
 124         ((uint64_t) ((unsigned long) (A) \
 125                 + (unsigned long)(ipc_priv->peb_start.host_vaddr)))
 126
 127 static int
 128 ipc_queue_configure(uint32_t channel_id,
 129                     ipc_t instance,
 130                     struct bbdev_la12xx_q_priv *q_priv)
 131 {
 132         ipc_userspace_t *ipc_priv = (ipc_userspace_t *)instance;
 133         ipc_instance_t *ipc_instance = ipc_priv->instance;
 134         ipc_ch_t *ch;
 135         void *vaddr;
 136         uint32_t i = 0;
 137         uint32_t msg_size = sizeof(struct bbdev_ipc_enqueue_op);
 138
 139         PMD_INIT_FUNC_TRACE();
 140
 141         rte_bbdev_log_debug("%x %p", ipc_instance->initialized,
 142                 ipc_priv->instance);
 143         ch = &(ipc_instance->ch_list[channel_id]);
 144
 145         rte_bbdev_log_debug("channel: %u, depth: %u, msg size: %u",
 146                 channel_id, q_priv->queue_size, msg_size);
 147
 148         /* Start init of channel */
 149         ch->md.ring_size = rte_cpu_to_be_32(q_priv->queue_size);
 150         ch->md.pi = 0;
 151         ch->md.ci = 0;
 152         ch->md.msg_size = msg_size;
 153         for (i = 0; i < q_priv->queue_size; i++) {
 154                 vaddr = rte_malloc(NULL, msg_size, RTE_CACHE_LINE_SIZE);
 155                 if (!vaddr)
 156                         return IPC_HOST_BUF_ALLOC_FAIL;
 157                 /* Only offset now */
 158                 ch->bd_h[i].modem_ptr =
 159                         rte_cpu_to_be_32(HUGEPG_OFFSET(vaddr));
 160                 ch->bd_h[i].host_virt_l = lower_32_bits(vaddr);
 161                 ch->bd_h[i].host_virt_h = upper_32_bits(vaddr);
 162                 q_priv->msg_ch_vaddr[i] = vaddr;
 163                 /* Not sure use of this len may be for CRC*/
 164                 ch->bd_h[i].len = 0;
 165         }
 166         ch->host_ipc_params =
 167                 rte_cpu_to_be_32(HUGEPG_OFFSET(q_priv->host_params));
 168
 169         rte_bbdev_log_debug("Channel configured");
 170         return IPC_SUCCESS;
 171 }
 172
 173 static int
 174 la12xx_e200_queue_setup(struct rte_bbdev *dev,
 175                 struct bbdev_la12xx_q_priv *q_priv)
 176 {
 177         struct bbdev_la12xx_private *priv = dev->data->dev_private;
 178         ipc_userspace_t *ipc_priv = priv->ipc_priv;
 179         struct gul_hif *mhif;
 180         ipc_metadata_t *ipc_md;
 181         ipc_ch_t *ch;
 182         int instance_id = 0, i;
 183         int ret;
 184
 185         PMD_INIT_FUNC_TRACE();
 186
 187         switch (q_priv->op_type) {
 188         case RTE_BBDEV_OP_LDPC_ENC:
 189                 q_priv->la12xx_core_id = LA12XX_LDPC_ENC_CORE;
 190                 break;
 191         case RTE_BBDEV_OP_LDPC_DEC:
 192                 q_priv->la12xx_core_id = LA12XX_LDPC_DEC_CORE;
 193                 break;
 194         default:
 195                 rte_bbdev_log(ERR, "Unsupported op type\n");
 196                 return -1;
 197         }
 198
 199         mhif = (struct gul_hif *)ipc_priv->mhif_start.host_vaddr;
 200         /* offset is from start of PEB */
 201         ipc_md = (ipc_metadata_t *)((uintptr_t)ipc_priv->peb_start.host_vaddr +
 202                 mhif->ipc_regs.ipc_mdata_offset);
 203         ch = &ipc_md->instance_list[instance_id].ch_list[q_priv->q_id];
 204
 205         if (q_priv->q_id < priv->num_valid_queues) {
 206                 ipc_br_md_t *md = &(ch->md);
 207
 208                 q_priv->feca_blk_id = rte_cpu_to_be_32(ch->feca_blk_id);
 209                 q_priv->feca_blk_id_be32 = ch->feca_blk_id;
 210                 q_priv->host_pi = rte_be_to_cpu_32(md->pi);
 211                 q_priv->host_ci = rte_be_to_cpu_32(md->ci);
 212                 q_priv->host_params = (host_ipc_params_t *)(uintptr_t)
 213                         (rte_be_to_cpu_32(ch->host_ipc_params) +
 214                         ((uint64_t)ipc_priv->hugepg_start.host_vaddr));
 215
 216                 for (i = 0; i < q_priv->queue_size; i++) {
 217                         uint32_t h, l;
 218
 219                         h = ch->bd_h[i].host_virt_h;
 220                         l = ch->bd_h[i].host_virt_l;
 221                         q_priv->msg_ch_vaddr[i] = (void *)join_32_bits(h, l);
 222                 }
 223
 224                 rte_bbdev_log(WARNING,
 225                         "Queue [%d] already configured, not configuring again",
 226                         q_priv->q_id);
 227                 return 0;
 228         }
 229
 230         rte_bbdev_log_debug("setting up queue %d", q_priv->q_id);
 231
 232         /* Call ipc_configure_channel */
 233         ret = ipc_queue_configure(q_priv->q_id, ipc_priv, q_priv);
 234         if (ret) {
 235                 rte_bbdev_log(ERR, "Unable to setup queue (%d) (err=%d)",
 236                        q_priv->q_id, ret);
 237                 return ret;
 238         }
 239
 240         /* Set queue properties for LA12xx device */
 241         switch (q_priv->op_type) {
 242         case RTE_BBDEV_OP_LDPC_ENC:
 243                 if (priv->num_ldpc_enc_queues >= LA12XX_MAX_LDPC_ENC_QUEUES) {
 244                         rte_bbdev_log(ERR,
 245                                 "num_ldpc_enc_queues reached max value");
 246                         return -1;
 247                 }
 248                 ch->la12xx_core_id =
 249                         rte_cpu_to_be_32(LA12XX_LDPC_ENC_CORE);
 250                 ch->feca_blk_id = rte_cpu_to_be_32(priv->num_ldpc_enc_queues++);
 251                 break;
 252         case RTE_BBDEV_OP_LDPC_DEC:
 253                 if (priv->num_ldpc_dec_queues >= LA12XX_MAX_LDPC_DEC_QUEUES) {
 254                         rte_bbdev_log(ERR,
 255                                 "num_ldpc_dec_queues reached max value");
 256                         return -1;
 257                 }
 258                 ch->la12xx_core_id =
 259                         rte_cpu_to_be_32(LA12XX_LDPC_DEC_CORE);
 260                 ch->feca_blk_id = rte_cpu_to_be_32(priv->num_ldpc_dec_queues++);
 261                 break;
 262         default:
 263                 rte_bbdev_log(ERR, "Not supported op type\n");
 264                 return -1;
 265         }
 266         ch->op_type = rte_cpu_to_be_32(q_priv->op_type);
 267         ch->depth = rte_cpu_to_be_32(q_priv->queue_size);
 268
 269         /* Store queue config here */
 270         q_priv->feca_blk_id = rte_cpu_to_be_32(ch->feca_blk_id);
 271         q_priv->feca_blk_id_be32 = ch->feca_blk_id;
 272
 273         return 0;
 274 }
 275
 276 /* Setup a queue */
 277 static int
 278 la12xx_queue_setup(struct rte_bbdev *dev, uint16_t q_id,
 279                 const struct rte_bbdev_queue_conf *queue_conf)
 280 {
 281         struct bbdev_la12xx_private *priv = dev->data->dev_private;
 282         struct rte_bbdev_queue_data *q_data;
 283         struct bbdev_la12xx_q_priv *q_priv;
 284         int ret;
 285
 286         PMD_INIT_FUNC_TRACE();
 287
 288         /* Move to setup_queues callback */
 289         q_data = &dev->data->queues[q_id];
 290         q_data->queue_private = rte_zmalloc(NULL,
 291                 sizeof(struct bbdev_la12xx_q_priv), 0);
 292         if (!q_data->queue_private) {
 293                 rte_bbdev_log(ERR, "Memory allocation failed for qpriv");
 294                 return -ENOMEM;
 295         }
 296         q_priv = q_data->queue_private;
 297         q_priv->q_id = q_id;
 298         q_priv->bbdev_priv = dev->data->dev_private;
 299         q_priv->queue_size = queue_conf->queue_size;
 300         q_priv->op_type = queue_conf->op_type;
 301
 302         ret = la12xx_e200_queue_setup(dev, q_priv);
 303         if (ret) {
 304                 rte_bbdev_log(ERR, "e200_queue_setup failed for qid: %d",
 305                                      q_id);
 306                 return ret;
 307         }
 308
 309         /* Store queue config here */
 310         priv->num_valid_queues++;
 311
 312         return 0;
 313 }
 314
 315 static int
 316 la12xx_start(struct rte_bbdev *dev)
 317 {
 318         struct bbdev_la12xx_private *priv = dev->data->dev_private;
 319         ipc_userspace_t *ipc_priv = priv->ipc_priv;
 320         int ready = 0;
 321         struct gul_hif *hif_start;
 322
 323         PMD_INIT_FUNC_TRACE();
 324
 325         hif_start = (struct gul_hif *)ipc_priv->mhif_start.host_vaddr;
 326
 327         /* Set Host Read bit */
 328         SET_HIF_HOST_RDY(hif_start, HIF_HOST_READY_IPC_APP);
 329
 330         /* Now wait for modem ready bit */
 331         while (!ready)
 332                 ready = CHK_HIF_MOD_RDY(hif_start, HIF_MOD_READY_IPC_APP);
 333
 334         return 0;
 335 }
 336
 337 static const struct rte_bbdev_ops pmd_ops = {
 338         .info_get = la12xx_info_get,
 339         .queue_setup = la12xx_queue_setup,
 340         .queue_release = la12xx_queue_release,
 341         .start = la12xx_start
 342 };
 343
 344 static inline int
 345 is_bd_ring_full(uint32_t ci, uint32_t ci_flag,
 346                 uint32_t pi, uint32_t pi_flag)
 347 {
 348         if (pi == ci) {
 349                 if (pi_flag != ci_flag)
 350                         return 1; /* Ring is Full */
 351         }
 352         return 0;
 353 }
 354
 355 static inline int
 356 prepare_ldpc_enc_op(struct rte_bbdev_enc_op *bbdev_enc_op,
 357                     struct bbdev_la12xx_q_priv *q_priv __rte_unused,
 358                     struct rte_bbdev_op_data *in_op_data __rte_unused,
 359                     struct rte_bbdev_op_data *out_op_data)
 360 {
 361         struct rte_bbdev_op_ldpc_enc *ldpc_enc = &bbdev_enc_op->ldpc_enc;
 362         uint32_t total_out_bits;
 363
 364         total_out_bits = (ldpc_enc->tb_params.cab *
 365                 ldpc_enc->tb_params.ea) + (ldpc_enc->tb_params.c -
 366                 ldpc_enc->tb_params.cab) * ldpc_enc->tb_params.eb;
 367
 368         ldpc_enc->output.length = (total_out_bits + 7)/8;
 369
 370         rte_pktmbuf_append(out_op_data->data, ldpc_enc->output.length);
 371
 372         return 0;
 373 }
 374
 375 static inline int
 376 prepare_ldpc_dec_op(struct rte_bbdev_dec_op *bbdev_dec_op,
 377                     struct bbdev_ipc_dequeue_op *bbdev_ipc_op,
 378                     struct bbdev_la12xx_q_priv *q_priv  __rte_unused,
 379                     struct rte_bbdev_op_data *out_op_data  __rte_unused)
 380 {
 381         struct rte_bbdev_op_ldpc_dec *ldpc_dec = &bbdev_dec_op->ldpc_dec;
 382         uint32_t total_out_bits;
 383         uint32_t num_code_blocks = 0;
 384         uint16_t sys_cols;
 385
 386         sys_cols = (ldpc_dec->basegraph == 1) ? 22 : 10;
 387         if (ldpc_dec->tb_params.c == 1) {
 388                 total_out_bits = ((sys_cols * ldpc_dec->z_c) -
 389                                 ldpc_dec->n_filler);
 390                 /* 5G-NR protocol uses 16 bit CRC when output packet
 391                  * size <= 3824 (bits). Otherwise 24 bit CRC is used.
 392                  * Adjust the output bits accordingly
 393                  */
 394                 if (total_out_bits - 16 <= 3824)
 395                         total_out_bits -= 16;
 396                 else
 397                         total_out_bits -= 24;
 398                 ldpc_dec->hard_output.length = (total_out_bits / 8);
 399         } else {
 400                 total_out_bits = (((sys_cols * ldpc_dec->z_c) -
 401                                 ldpc_dec->n_filler - 24) *
 402                                 ldpc_dec->tb_params.c);
 403                 ldpc_dec->hard_output.length = (total_out_bits / 8) - 3;
 404         }
 405
 406         num_code_blocks = ldpc_dec->tb_params.c;
 407
 408         bbdev_ipc_op->num_code_blocks = rte_cpu_to_be_32(num_code_blocks);
 409
 410         return 0;
 411 }
 412
 413 static int
 414 enqueue_single_op(struct bbdev_la12xx_q_priv *q_priv, void *bbdev_op)
 415 {
 416         struct bbdev_la12xx_private *priv = q_priv->bbdev_priv;
 417         ipc_userspace_t *ipc_priv = priv->ipc_priv;
 418         ipc_instance_t *ipc_instance = ipc_priv->instance;
 419         struct bbdev_ipc_dequeue_op *bbdev_ipc_op;
 420         struct rte_bbdev_op_ldpc_enc *ldpc_enc;
 421         struct rte_bbdev_op_ldpc_dec *ldpc_dec;
 422         uint32_t q_id = q_priv->q_id;
 423         uint32_t ci, ci_flag, pi, pi_flag;
 424         ipc_ch_t *ch = &(ipc_instance->ch_list[q_id]);
 425         ipc_br_md_t *md = &(ch->md);
 426         size_t virt;
 427         char *huge_start_addr =
 428                 (char *)q_priv->bbdev_priv->ipc_priv->hugepg_start.host_vaddr;
 429         struct rte_bbdev_op_data *in_op_data, *out_op_data;
 430         char *data_ptr;
 431         uint32_t l1_pcie_addr;
 432         int ret;
 433
 434         ci = IPC_GET_CI_INDEX(q_priv->host_ci);
 435         ci_flag = IPC_GET_CI_FLAG(q_priv->host_ci);
 436
 437         pi = IPC_GET_PI_INDEX(q_priv->host_pi);
 438         pi_flag = IPC_GET_PI_FLAG(q_priv->host_pi);
 439
 440         rte_bbdev_dp_log(DEBUG, "before bd_ring_full: pi: %u, ci: %u,"
 441                 "pi_flag: %u, ci_flag: %u, ring size: %u",
 442                 pi, ci, pi_flag, ci_flag, q_priv->queue_size);
 443
 444         if (is_bd_ring_full(ci, ci_flag, pi, pi_flag)) {
 445                 rte_bbdev_dp_log(DEBUG, "bd ring full for queue id: %d", q_id);
 446                 return IPC_CH_FULL;
 447         }
 448
 449         virt = MODEM_P2V(q_priv->host_params->bd_m_modem_ptr[pi]);
 450         bbdev_ipc_op = (struct bbdev_ipc_dequeue_op *)virt;
 451         q_priv->bbdev_op[pi] = bbdev_op;
 452
 453         switch (q_priv->op_type) {
 454         case RTE_BBDEV_OP_LDPC_ENC:
 455                 ldpc_enc = &(((struct rte_bbdev_enc_op *)bbdev_op)->ldpc_enc);
 456                 in_op_data = &ldpc_enc->input;
 457                 out_op_data = &ldpc_enc->output;
 458
 459                 ret = prepare_ldpc_enc_op(bbdev_op, q_priv,
 460                                           in_op_data, out_op_data);
 461                 if (ret) {
 462                         rte_bbdev_log(ERR, "process_ldpc_enc_op fail, ret: %d",
 463                                 ret);
 464                         return ret;
 465                 }
 466                 break;
 467
 468         case RTE_BBDEV_OP_LDPC_DEC:
 469                 ldpc_dec = &(((struct rte_bbdev_dec_op *)bbdev_op)->ldpc_dec);
 470                 in_op_data = &ldpc_dec->input;
 471
 472                 out_op_data = &ldpc_dec->hard_output;
 473
 474                 ret = prepare_ldpc_dec_op(bbdev_op, bbdev_ipc_op,
 475                                           q_priv, out_op_data);
 476                 if (ret) {
 477                         rte_bbdev_log(ERR, "process_ldpc_dec_op fail, ret: %d",
 478                                 ret);
 479                         return ret;
 480                 }
 481                 break;
 482
 483         default:
 484                 rte_bbdev_log(ERR, "unsupported bbdev_ipc op type");
 485                 return -1;
 486         }
 487
 488         if (in_op_data->data) {
 489                 data_ptr = rte_pktmbuf_mtod(in_op_data->data, char *);
 490                 l1_pcie_addr = (uint32_t)GUL_USER_HUGE_PAGE_ADDR +
 491                                data_ptr - huge_start_addr;
 492                 bbdev_ipc_op->in_addr = l1_pcie_addr;
 493                 bbdev_ipc_op->in_len = in_op_data->length;
 494         }
 495
 496         if (out_op_data->data) {
 497                 data_ptr = rte_pktmbuf_mtod(out_op_data->data, char *);
 498                 l1_pcie_addr = (uint32_t)GUL_USER_HUGE_PAGE_ADDR +
 499                                 data_ptr - huge_start_addr;
 500                 bbdev_ipc_op->out_addr = rte_cpu_to_be_32(l1_pcie_addr);
 501                 bbdev_ipc_op->out_len = rte_cpu_to_be_32(out_op_data->length);
 502         }
 503
 504         /* Move Producer Index forward */
 505         pi++;
 506         /* Flip the PI flag, if wrapping */
 507         if (unlikely(q_priv->queue_size == pi)) {
 508                 pi = 0;
 509                 pi_flag = pi_flag ? 0 : 1;
 510         }
 511
 512         if (pi_flag)
 513                 IPC_SET_PI_FLAG(pi);
 514         else
 515                 IPC_RESET_PI_FLAG(pi);
 516         q_priv->host_pi = pi;
 517
 518         /* Wait for Data Copy & pi_flag update to complete before updating pi */
 519         rte_mb();
 520         /* now update pi */
 521         md->pi = rte_cpu_to_be_32(pi);
 522
 523         rte_bbdev_dp_log(DEBUG, "enter: pi: %u, ci: %u,"
 524                         "pi_flag: %u, ci_flag: %u, ring size: %u",
 525                         pi, ci, pi_flag, ci_flag, q_priv->queue_size);
 526
 527         return 0;
 528 }
 529
 530 /* Enqueue decode burst */
 531 static uint16_t
 532 enqueue_dec_ops(struct rte_bbdev_queue_data *q_data,
 533                 struct rte_bbdev_dec_op **ops, uint16_t nb_ops)
 534 {
 535         struct bbdev_la12xx_q_priv *q_priv = q_data->queue_private;
 536         int nb_enqueued, ret;
 537
 538         for (nb_enqueued = 0; nb_enqueued < nb_ops; nb_enqueued++) {
 539                 ret = enqueue_single_op(q_priv, ops[nb_enqueued]);
 540                 if (ret)
 541                         break;
 542         }
 543
 544         q_data->queue_stats.enqueue_err_count += nb_ops - nb_enqueued;
 545         q_data->queue_stats.enqueued_count += nb_enqueued;
 546
 547         return nb_enqueued;
 548 }
 549
 550 /* Enqueue encode burst */
 551 static uint16_t
 552 enqueue_enc_ops(struct rte_bbdev_queue_data *q_data,
 553                 struct rte_bbdev_enc_op **ops, uint16_t nb_ops)
 554 {
 555         struct bbdev_la12xx_q_priv *q_priv = q_data->queue_private;
 556         int nb_enqueued, ret;
 557
 558         for (nb_enqueued = 0; nb_enqueued < nb_ops; nb_enqueued++) {
 559                 ret = enqueue_single_op(q_priv, ops[nb_enqueued]);
 560                 if (ret)
 561                         break;
 562         }
 563
 564         q_data->queue_stats.enqueue_err_count += nb_ops - nb_enqueued;
 565         q_data->queue_stats.enqueued_count += nb_enqueued;
 566
 567         return nb_enqueued;
 568 }
 569
 570 /* Dequeue encode burst */
 571 static void *
 572 dequeue_single_op(struct bbdev_la12xx_q_priv *q_priv, void *dst)
 573 {
 574         void *op;
 575         uint32_t ci, ci_flag;
 576         uint32_t temp_ci;
 577
 578         temp_ci = q_priv->host_params->ci;
 579         if (temp_ci == q_priv->host_ci)
 580                 return NULL;
 581
 582         ci = IPC_GET_CI_INDEX(q_priv->host_ci);
 583         ci_flag = IPC_GET_CI_FLAG(q_priv->host_ci);
 584
 585         rte_bbdev_dp_log(DEBUG,
 586                 "ci: %u, ci_flag: %u, ring size: %u",
 587                 ci, ci_flag, q_priv->queue_size);
 588
 589         op = q_priv->bbdev_op[ci];
 590
 591         rte_memcpy(dst, q_priv->msg_ch_vaddr[ci],
 592                 sizeof(struct bbdev_ipc_enqueue_op));
 593
 594         /* Move Consumer Index forward */
 595         ci++;
 596         /* Flip the CI flag, if wrapping */
 597         if (q_priv->queue_size == ci) {
 598                 ci = 0;
 599                 ci_flag = ci_flag ? 0 : 1;
 600         }
 601         if (ci_flag)
 602                 IPC_SET_CI_FLAG(ci);
 603         else
 604                 IPC_RESET_CI_FLAG(ci);
 605
 606         q_priv->host_ci = ci;
 607
 608         rte_bbdev_dp_log(DEBUG,
 609                 "exit: ci: %u, ci_flag: %u, ring size: %u",
 610                 ci, ci_flag, q_priv->queue_size);
 611
 612         return op;
 613 }
 614
 615 /* Dequeue decode burst */
 616 static uint16_t
 617 dequeue_dec_ops(struct rte_bbdev_queue_data *q_data,
 618                 struct rte_bbdev_dec_op **ops, uint16_t nb_ops)
 619 {
 620         struct bbdev_la12xx_q_priv *q_priv = q_data->queue_private;
 621         struct bbdev_ipc_enqueue_op bbdev_ipc_op;
 622         int nb_dequeued;
 623
 624         for (nb_dequeued = 0; nb_dequeued < nb_ops; nb_dequeued++) {
 625                 ops[nb_dequeued] = dequeue_single_op(q_priv, &bbdev_ipc_op);
 626                 if (!ops[nb_dequeued])
 627                         break;
 628                 ops[nb_dequeued]->status = bbdev_ipc_op.status;
 629         }
 630         q_data->queue_stats.dequeued_count += nb_dequeued;
 631
 632         return nb_dequeued;
 633 }
 634
 635 /* Dequeue encode burst */
 636 static uint16_t
 637 dequeue_enc_ops(struct rte_bbdev_queue_data *q_data,
 638                 struct rte_bbdev_enc_op **ops, uint16_t nb_ops)
 639 {
 640         struct bbdev_la12xx_q_priv *q_priv = q_data->queue_private;
 641         struct bbdev_ipc_enqueue_op bbdev_ipc_op;
 642         int nb_enqueued;
 643
 644         for (nb_enqueued = 0; nb_enqueued < nb_ops; nb_enqueued++) {
 645                 ops[nb_enqueued] = dequeue_single_op(q_priv, &bbdev_ipc_op);
 646                 if (!ops[nb_enqueued])
 647                         break;
 648                 ops[nb_enqueued]->status = bbdev_ipc_op.status;
 649         }
 650         q_data->queue_stats.enqueued_count += nb_enqueued;
 651
 652         return nb_enqueued;
 653 }
 654
 655 static struct hugepage_info *
 656 get_hugepage_info(void)
 657 {
 658         struct hugepage_info *hp_info;
 659         struct rte_memseg *mseg;
 660
 661         PMD_INIT_FUNC_TRACE();
 662
 663         /* TODO - find a better way */
 664         hp_info = rte_malloc(NULL, sizeof(struct hugepage_info), 0);
 665         if (!hp_info) {
 666                 rte_bbdev_log(ERR, "Unable to allocate on local heap");
 667                 return NULL;
 668         }
 669
 670         mseg = rte_mem_virt2memseg(hp_info, NULL);
 671         hp_info->vaddr = mseg->addr;
 672         hp_info->paddr = rte_mem_virt2phy(mseg->addr);
 673         hp_info->len = mseg->len;
 674
 675         return hp_info;
 676 }
 677
 678 static int
 679 open_ipc_dev(int modem_id)
 680 {
 681         char dev_initials[16], dev_path[PATH_MAX];
 682         struct dirent *entry;
 683         int dev_ipc = 0;
 684         DIR *dir;
 685
 686         dir = opendir("/dev/");
 687         if (!dir) {
 688                 rte_bbdev_log(ERR, "Unable to open /dev/");
 689                 return -1;
 690         }
 691
 692         sprintf(dev_initials, "gulipcgul%d", modem_id);
 693
 694         while ((entry = readdir(dir)) != NULL) {
 695                 if (!strncmp(dev_initials, entry->d_name,
 696                     sizeof(dev_initials) - 1))
 697                         break;
 698         }
 699
 700         if (!entry) {
 701                 rte_bbdev_log(ERR, "Error: No gulipcgul%d device", modem_id);
 702                 return -1;
 703         }
 704
 705         sprintf(dev_path, "/dev/%s", entry->d_name);
 706         dev_ipc = open(dev_path, O_RDWR);
 707         if (dev_ipc  < 0) {
 708                 rte_bbdev_log(ERR, "Error: Cannot open %s", dev_path);
 709                 return -errno;
 710         }
 711
 712         return dev_ipc;
 713 }
 714
 715 static int
 716 setup_la12xx_dev(struct rte_bbdev *dev)
 717 {
 718         struct bbdev_la12xx_private *priv = dev->data->dev_private;
 719         ipc_userspace_t *ipc_priv = priv->ipc_priv;
 720         struct hugepage_info *hp = NULL;
 721         ipc_channel_us_t *ipc_priv_ch = NULL;
 722         int dev_ipc = 0, dev_mem = 0, i;
 723         ipc_metadata_t *ipc_md;
 724         struct gul_hif *mhif;
 725         uint32_t phy_align = 0;
 726         int ret;
 727
 728         PMD_INIT_FUNC_TRACE();
 729
 730         if (!ipc_priv) {
 731                 /* TODO - get a better way */
 732                 /* Get the hugepage info against it */
 733                 hp = get_hugepage_info();
 734                 if (!hp) {
 735                         rte_bbdev_log(ERR, "Unable to get hugepage info");
 736                         ret = -ENOMEM;
 737                         goto err;
 738                 }
 739
 740                 rte_bbdev_log_debug("0x%" PRIx64 " %p 0x%" PRIx64,
 741                                 hp->paddr, hp->vaddr, hp->len);
 742
 743                 ipc_priv = rte_zmalloc(0, sizeof(ipc_userspace_t), 0);
 744                 if (ipc_priv == NULL) {
 745                         rte_bbdev_log(ERR,
 746                                 "Unable to allocate memory for ipc priv");
 747                         ret = -ENOMEM;
 748                         goto err;
 749                 }
 750
 751                 for (i = 0; i < IPC_MAX_CHANNEL_COUNT; i++) {
 752                         ipc_priv_ch = rte_zmalloc(0,
 753                                 sizeof(ipc_channel_us_t), 0);
 754                         if (ipc_priv_ch == NULL) {
 755                                 rte_bbdev_log(ERR,
 756                                         "Unable to allocate memory for channels");
 757                                 ret = -ENOMEM;
 758                         }
 759                         ipc_priv->channels[i] = ipc_priv_ch;
 760                 }
 761
 762                 dev_mem = open("/dev/mem", O_RDWR);
 763                 if (dev_mem < 0) {
 764                         rte_bbdev_log(ERR, "Error: Cannot open /dev/mem");
 765                         ret = -errno;
 766                         goto err;
 767                 }
 768
 769                 ipc_priv->instance_id = 0;
 770                 ipc_priv->dev_mem = dev_mem;
 771
 772                 rte_bbdev_log_debug("hugepg input 0x%" PRIx64 "%p 0x%" PRIx64,
 773                         hp->paddr, hp->vaddr, hp->len);
 774
 775                 ipc_priv->sys_map.hugepg_start.host_phys = hp->paddr;
 776                 ipc_priv->sys_map.hugepg_start.size = hp->len;
 777
 778                 ipc_priv->hugepg_start.host_phys = hp->paddr;
 779                 ipc_priv->hugepg_start.host_vaddr = hp->vaddr;
 780                 ipc_priv->hugepg_start.size = hp->len;
 781
 782                 rte_free(hp);
 783         }
 784
 785         dev_ipc = open_ipc_dev(priv->modem_id);
 786         if (dev_ipc < 0) {
 787                 rte_bbdev_log(ERR, "Error: open_ipc_dev failed");
 788                 goto err;
 789         }
 790         ipc_priv->dev_ipc = dev_ipc;
 791
 792         ret = ioctl(ipc_priv->dev_ipc, IOCTL_GUL_IPC_GET_SYS_MAP,
 793                     &ipc_priv->sys_map);
 794         if (ret) {
 795                 rte_bbdev_log(ERR,
 796                         "IOCTL_GUL_IPC_GET_SYS_MAP ioctl failed");
 797                 goto err;
 798         }
 799
 800         phy_align = (ipc_priv->sys_map.mhif_start.host_phys % 0x1000);
 801         ipc_priv->mhif_start.host_vaddr =
 802                 mmap(0, ipc_priv->sys_map.mhif_start.size + phy_align,
 803                      (PROT_READ | PROT_WRITE), MAP_SHARED, ipc_priv->dev_mem,
 804                      (ipc_priv->sys_map.mhif_start.host_phys - phy_align));
 805         if (ipc_priv->mhif_start.host_vaddr == MAP_FAILED) {
 806                 rte_bbdev_log(ERR, "MAP failed:");
 807                 ret = -errno;
 808                 goto err;
 809         }
 810
 811         ipc_priv->mhif_start.host_vaddr = (void *) ((uintptr_t)
 812                 (ipc_priv->mhif_start.host_vaddr) + phy_align);
 813
 814         phy_align = (ipc_priv->sys_map.peb_start.host_phys % 0x1000);
 815         ipc_priv->peb_start.host_vaddr =
 816                 mmap(0, ipc_priv->sys_map.peb_start.size + phy_align,
 817                      (PROT_READ | PROT_WRITE), MAP_SHARED, ipc_priv->dev_mem,
 818                      (ipc_priv->sys_map.peb_start.host_phys - phy_align));
 819         if (ipc_priv->peb_start.host_vaddr == MAP_FAILED) {
 820                 rte_bbdev_log(ERR, "MAP failed:");
 821                 ret = -errno;
 822                 goto err;
 823         }
 824
 825         ipc_priv->peb_start.host_vaddr = (void *)((uintptr_t)
 826                 (ipc_priv->peb_start.host_vaddr) + phy_align);
 827
 828         phy_align = (ipc_priv->sys_map.modem_ccsrbar.host_phys % 0x1000);
 829         ipc_priv->modem_ccsrbar.host_vaddr =
 830                 mmap(0, ipc_priv->sys_map.modem_ccsrbar.size + phy_align,
 831                      (PROT_READ | PROT_WRITE), MAP_SHARED, ipc_priv->dev_mem,
 832                      (ipc_priv->sys_map.modem_ccsrbar.host_phys - phy_align));
 833         if (ipc_priv->modem_ccsrbar.host_vaddr == MAP_FAILED) {
 834                 rte_bbdev_log(ERR, "MAP failed:");
 835                 ret = -errno;
 836                 goto err;
 837         }
 838
 839         ipc_priv->modem_ccsrbar.host_vaddr = (void *)((uintptr_t)
 840                 (ipc_priv->modem_ccsrbar.host_vaddr) + phy_align);
 841
 842         ipc_priv->hugepg_start.modem_phys =
 843                 ipc_priv->sys_map.hugepg_start.modem_phys;
 844
 845         ipc_priv->mhif_start.host_phys =
 846                 ipc_priv->sys_map.mhif_start.host_phys;
 847         ipc_priv->mhif_start.size = ipc_priv->sys_map.mhif_start.size;
 848         ipc_priv->peb_start.host_phys = ipc_priv->sys_map.peb_start.host_phys;
 849         ipc_priv->peb_start.size = ipc_priv->sys_map.peb_start.size;
 850
 851         rte_bbdev_log(INFO, "peb 0x%" PRIx64 "%p 0x%" PRIx32,
 852                         ipc_priv->peb_start.host_phys,
 853                         ipc_priv->peb_start.host_vaddr,
 854                         ipc_priv->peb_start.size);
 855         rte_bbdev_log(INFO, "hugepg 0x%" PRIx64 "%p 0x%" PRIx32,
 856                         ipc_priv->hugepg_start.host_phys,
 857                         ipc_priv->hugepg_start.host_vaddr,
 858                         ipc_priv->hugepg_start.size);
 859         rte_bbdev_log(INFO, "mhif 0x%" PRIx64 "%p 0x%" PRIx32,
 860                         ipc_priv->mhif_start.host_phys,
 861                         ipc_priv->mhif_start.host_vaddr,
 862                         ipc_priv->mhif_start.size);
 863         mhif = (struct gul_hif *)ipc_priv->mhif_start.host_vaddr;
 864
 865         /* offset is from start of PEB */
 866         ipc_md = (ipc_metadata_t *)((uintptr_t)ipc_priv->peb_start.host_vaddr +
 867                         mhif->ipc_regs.ipc_mdata_offset);
 868
 869         if (sizeof(ipc_metadata_t) != mhif->ipc_regs.ipc_mdata_size) {
 870                 rte_bbdev_log(ERR,
 871                         "ipc_metadata_t =0x%" PRIx64
 872                         ", mhif->ipc_regs.ipc_mdata_size=0x%" PRIx32,
 873                         (uint64_t)(sizeof(ipc_metadata_t)),
 874                         mhif->ipc_regs.ipc_mdata_size);
 875                 rte_bbdev_log(ERR, "--> mhif->ipc_regs.ipc_mdata_offset= 0x%"
 876                         PRIx32, mhif->ipc_regs.ipc_mdata_offset);
 877                 rte_bbdev_log(ERR, "gul_hif size=0x%" PRIx64,
 878                         (uint64_t)(sizeof(struct gul_hif)));
 879                 return IPC_MD_SZ_MISS_MATCH;
 880         }
 881
 882         ipc_priv->instance = (ipc_instance_t *)
 883                 (&ipc_md->instance_list[ipc_priv->instance_id]);
 884
 885         rte_bbdev_log_debug("finish host init");
 886
 887         priv->ipc_priv = ipc_priv;
 888
 889         return 0;
 890
 891 err:
 892         rte_free(hp);
 893         rte_free(ipc_priv);
 894         rte_free(ipc_priv_ch);
 895         if (dev_mem)
 896                 close(dev_mem);
 897         if (dev_ipc)
 898                 close(dev_ipc);
 899
 900         return ret;
 901 }
 902
 903 static inline int
 904 parse_u16_arg(const char *key, const char *value, void *extra_args)
 905 {
 906         uint16_t *u16 = extra_args;
 907
 908         uint64_t result;
 909         if ((value == NULL) || (extra_args == NULL))
 910                 return -EINVAL;
 911         errno = 0;
 912         result = strtoul(value, NULL, 0);
 913         if ((result >= (1 << 16)) || (errno != 0)) {
 914                 rte_bbdev_log(ERR, "Invalid value %" PRIu64 " for %s",
 915                               result, key);
 916                 return -ERANGE;
 917         }
 918         *u16 = (uint16_t)result;
 919         return 0;
 920 }
 921
 922 /* Parse integer from integer argument */
 923 static int
 924 parse_integer_arg(const char *key __rte_unused,
 925                 const char *value, void *extra_args)
 926 {
 927         int i;
 928         char *end;
 929
 930         errno = 0;
 931
 932         i = strtol(value, &end, 10);
 933         if (*end != 0 || errno != 0 || i < 0 || i > LA12XX_MAX_MODEM) {
 934                 rte_bbdev_log(ERR, "Supported Port IDS are 0 to %d",
 935                         LA12XX_MAX_MODEM - 1);
 936                 return -EINVAL;
 937         }
 938
 939         *((uint32_t *)extra_args) = i;
 940
 941         return 0;
 942 }
 943
 944 /* Parse parameters used to create device */
 945 static int
 946 parse_bbdev_la12xx_params(struct bbdev_la12xx_params *params,
 947                 const char *input_args)
 948 {
 949         struct rte_kvargs *kvlist = NULL;
 950         int ret = 0;
 951
 952         if (params == NULL)
 953                 return -EINVAL;
 954         if (input_args) {
 955                 kvlist = rte_kvargs_parse(input_args,
 956                                 bbdev_la12xx_valid_params);
 957                 if (kvlist == NULL)
 958                         return -EFAULT;
 959
 960                 ret = rte_kvargs_process(kvlist, bbdev_la12xx_valid_params[0],
 961                                         &parse_u16_arg, &params->queues_num);
 962                 if (ret < 0)
 963                         goto exit;
 964
 965                 ret = rte_kvargs_process(kvlist,
 966                                         bbdev_la12xx_valid_params[1],
 967                                         &parse_integer_arg,
 968                                         &params->modem_id);
 969
 970                 if (params->modem_id >= LA12XX_MAX_MODEM) {
 971                         rte_bbdev_log(ERR, "Invalid modem id, must be < %u",
 972                                         LA12XX_MAX_MODEM);
 973                         goto exit;
 974                 }
 975         }
 976
 977 exit:
 978         if (kvlist)
 979                 rte_kvargs_free(kvlist);
 980         return ret;
 981 }
 982
 983 /* Create device */
 984 static int
 985 la12xx_bbdev_create(struct rte_vdev_device *vdev,
 986                 struct bbdev_la12xx_params *init_params)
 987 {
 988         struct rte_bbdev *bbdev;
 989         const char *name = rte_vdev_device_name(vdev);
 990         struct bbdev_la12xx_private *priv;
 991         int ret;
 992
 993         PMD_INIT_FUNC_TRACE();
 994
 995         bbdev = rte_bbdev_allocate(name);
 996         if (bbdev == NULL)
 997                 return -ENODEV;
 998
 999         bbdev->data->dev_private = rte_zmalloc(name,
1000                         sizeof(struct bbdev_la12xx_private),
1001                         RTE_CACHE_LINE_SIZE);
1002         if (bbdev->data->dev_private == NULL) {
1003                 rte_bbdev_release(bbdev);
1004                 return -ENOMEM;
1005         }
1006
1007         priv = bbdev->data->dev_private;
1008         priv->modem_id = init_params->modem_id;
1009         /* if modem id is not configured */
1010         if (priv->modem_id == -1)
1011                 priv->modem_id = bbdev->data->dev_id;
1012
1013         /* Reset Global variables */
1014         priv->num_ldpc_enc_queues = 0;
1015         priv->num_ldpc_dec_queues = 0;
1016         priv->num_valid_queues = 0;
1017         priv->max_nb_queues = init_params->queues_num;
1018
1019         rte_bbdev_log(INFO, "Setting Up %s: DevId=%d, ModemId=%d",
1020                                 name, bbdev->data->dev_id, priv->modem_id);
1021         ret = setup_la12xx_dev(bbdev);
1022         if (ret) {
1023                 rte_bbdev_log(ERR, "IPC Setup failed for %s", name);
1024                 rte_free(bbdev->data->dev_private);
1025                 return ret;
1026         }
1027         bbdev->dev_ops = &pmd_ops;
1028         bbdev->device = &vdev->device;
1029         bbdev->data->socket_id = 0;
1030         bbdev->intr_handle = NULL;
1031
1032         /* register rx/tx burst functions for data path */
1033         bbdev->dequeue_enc_ops = NULL;
1034         bbdev->dequeue_dec_ops = NULL;
1035         bbdev->enqueue_enc_ops = NULL;
1036         bbdev->enqueue_dec_ops = NULL;
1037         bbdev->dequeue_ldpc_enc_ops = dequeue_enc_ops;
1038         bbdev->dequeue_ldpc_dec_ops = dequeue_dec_ops;
1039         bbdev->enqueue_ldpc_enc_ops = enqueue_enc_ops;
1040         bbdev->enqueue_ldpc_dec_ops = enqueue_dec_ops;
1041
1042         return 0;
1043 }
1044
1045 /* Initialise device */
1046 static int
1047 la12xx_bbdev_probe(struct rte_vdev_device *vdev)
1048 {
1049         struct bbdev_la12xx_params init_params = {
1050                 8, -1,
1051         };
1052         const char *name;
1053         const char *input_args;
1054
1055         PMD_INIT_FUNC_TRACE();
1056
1057         if (vdev == NULL)
1058                 return -EINVAL;
1059
1060         name = rte_vdev_device_name(vdev);
1061         if (name == NULL)
1062                 return -EINVAL;
1063
1064         input_args = rte_vdev_device_args(vdev);
1065         parse_bbdev_la12xx_params(&init_params, input_args);
1066
1067         return la12xx_bbdev_create(vdev, &init_params);
1068 }
1069
1070 /* Uninitialise device */
1071 static int
1072 la12xx_bbdev_remove(struct rte_vdev_device *vdev)
1073 {
1074         struct rte_bbdev *bbdev;
1075         const char *name;
1076
1077         PMD_INIT_FUNC_TRACE();
1078
1079         if (vdev == NULL)
1080                 return -EINVAL;
1081
1082         name = rte_vdev_device_name(vdev);
1083         if (name == NULL)
1084                 return -EINVAL;
1085
1086         bbdev = rte_bbdev_get_named_dev(name);
1087         if (bbdev == NULL)
1088                 return -EINVAL;
1089
1090         rte_free(bbdev->data->dev_private);
1091
1092         return rte_bbdev_release(bbdev);
1093 }
1094
1095 static struct rte_vdev_driver bbdev_la12xx_pmd_drv = {
1096         .probe = la12xx_bbdev_probe,
1097         .remove = la12xx_bbdev_remove
1098 };
1099
1100 RTE_PMD_REGISTER_VDEV(DRIVER_NAME, bbdev_la12xx_pmd_drv);
1101 RTE_PMD_REGISTER_PARAM_STRING(DRIVER_NAME,
1102         LA12XX_MAX_NB_QUEUES_ARG"=<int>"
1103         LA12XX_VDEV_MODEM_ID_ARG "=<int> ");
1104 RTE_LOG_REGISTER_DEFAULT(bbdev_la12xx_logtype, NOTICE);