drivers/crypto/octeontx/otx_cryptodev_ops.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Cavium, Inc
   3  */
   4
   5 #include <rte_alarm.h>
   6 #include <rte_bus_pci.h>
   7 #include <rte_cryptodev.h>
   8 #include <rte_cryptodev_pmd.h>
   9 #include <rte_errno.h>
  10 #include <rte_malloc.h>
  11 #include <rte_mempool.h>
  12
  13 #include "otx_cryptodev.h"
  14 #include "otx_cryptodev_capabilities.h"
  15 #include "otx_cryptodev_hw_access.h"
  16 #include "otx_cryptodev_mbox.h"
  17 #include "otx_cryptodev_ops.h"
  18
  19 #include "cpt_pmd_logs.h"
  20 #include "cpt_pmd_ops_helper.h"
  21 #include "cpt_ucode.h"
  22 #include "cpt_ucode_asym.h"
  23
  24 static uint64_t otx_fpm_iova[CPT_EC_ID_PMAX];
  25
  26 /* Forward declarations */
  27
  28 static int
  29 otx_cpt_que_pair_release(struct rte_cryptodev *dev, uint16_t que_pair_id);
  30
  31 /* Alarm routines */
  32
  33 static void
  34 otx_cpt_alarm_cb(void *arg)
  35 {
  36         struct cpt_vf *cptvf = arg;
  37         otx_cpt_poll_misc(cptvf);
  38         rte_eal_alarm_set(CPT_INTR_POLL_INTERVAL_MS * 1000,
  39                           otx_cpt_alarm_cb, cptvf);
  40 }
  41
  42 static int
  43 otx_cpt_periodic_alarm_start(void *arg)
  44 {
  45         return rte_eal_alarm_set(CPT_INTR_POLL_INTERVAL_MS * 1000,
  46                                  otx_cpt_alarm_cb, arg);
  47 }
  48
  49 static int
  50 otx_cpt_periodic_alarm_stop(void *arg)
  51 {
  52         return rte_eal_alarm_cancel(otx_cpt_alarm_cb, arg);
  53 }
  54
  55 /* PMD ops */
  56
  57 static int
  58 otx_cpt_dev_config(struct rte_cryptodev *dev,
  59                    struct rte_cryptodev_config *config __rte_unused)
  60 {
  61         int ret = 0;
  62
  63         CPT_PMD_INIT_FUNC_TRACE();
  64
  65         if (dev->feature_flags & RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO)
  66                 /* Initialize shared FPM table */
  67                 ret = cpt_fpm_init(otx_fpm_iova);
  68
  69         return ret;
  70 }
  71
  72 static int
  73 otx_cpt_dev_start(struct rte_cryptodev *c_dev)
  74 {
  75         void *cptvf = c_dev->data->dev_private;
  76
  77         CPT_PMD_INIT_FUNC_TRACE();
  78
  79         return otx_cpt_start_device(cptvf);
  80 }
  81
  82 static void
  83 otx_cpt_dev_stop(struct rte_cryptodev *c_dev)
  84 {
  85         void *cptvf = c_dev->data->dev_private;
  86
  87         CPT_PMD_INIT_FUNC_TRACE();
  88
  89         if (c_dev->feature_flags & RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO)
  90                 cpt_fpm_clear();
  91
  92         otx_cpt_stop_device(cptvf);
  93 }
  94
  95 static int
  96 otx_cpt_dev_close(struct rte_cryptodev *c_dev)
  97 {
  98         void *cptvf = c_dev->data->dev_private;
  99         int i, ret;
 100
 101         CPT_PMD_INIT_FUNC_TRACE();
 102
 103         for (i = 0; i < c_dev->data->nb_queue_pairs; i++) {
 104                 ret = otx_cpt_que_pair_release(c_dev, i);
 105                 if (ret)
 106                         return ret;
 107         }
 108
 109         otx_cpt_periodic_alarm_stop(cptvf);
 110         otx_cpt_deinit_device(cptvf);
 111
 112         return 0;
 113 }
 114
 115 static void
 116 otx_cpt_dev_info_get(struct rte_cryptodev *dev, struct rte_cryptodev_info *info)
 117 {
 118         CPT_PMD_INIT_FUNC_TRACE();
 119         if (info != NULL) {
 120                 info->max_nb_queue_pairs = CPT_NUM_QS_PER_VF;
 121                 info->feature_flags = dev->feature_flags;
 122                 info->capabilities = otx_get_capabilities(info->feature_flags);
 123                 info->sym.max_nb_sessions = 0;
 124                 info->driver_id = otx_cryptodev_driver_id;
 125                 info->min_mbuf_headroom_req = OTX_CPT_MIN_HEADROOM_REQ;
 126                 info->min_mbuf_tailroom_req = OTX_CPT_MIN_TAILROOM_REQ;
 127         }
 128 }
 129
 130 static void
 131 otx_cpt_stats_get(struct rte_cryptodev *dev __rte_unused,
 132                   struct rte_cryptodev_stats *stats __rte_unused)
 133 {
 134         CPT_PMD_INIT_FUNC_TRACE();
 135 }
 136
 137 static void
 138 otx_cpt_stats_reset(struct rte_cryptodev *dev __rte_unused)
 139 {
 140         CPT_PMD_INIT_FUNC_TRACE();
 141 }
 142
 143 static int
 144 otx_cpt_que_pair_setup(struct rte_cryptodev *dev,
 145                        uint16_t que_pair_id,
 146                        const struct rte_cryptodev_qp_conf *qp_conf,
 147                        int socket_id __rte_unused)
 148 {
 149         struct cpt_instance *instance = NULL;
 150         struct rte_pci_device *pci_dev;
 151         int ret = -1;
 152
 153         CPT_PMD_INIT_FUNC_TRACE();
 154
 155         if (dev->data->queue_pairs[que_pair_id] != NULL) {
 156                 ret = otx_cpt_que_pair_release(dev, que_pair_id);
 157                 if (ret)
 158                         return ret;
 159         }
 160
 161         if (qp_conf->nb_descriptors > DEFAULT_CMD_QLEN) {
 162                 CPT_LOG_INFO("Number of descriptors too big %d, using default "
 163                              "queue length of %d", qp_conf->nb_descriptors,
 164                              DEFAULT_CMD_QLEN);
 165         }
 166
 167         pci_dev = RTE_DEV_TO_PCI(dev->device);
 168
 169         if (pci_dev->mem_resource[0].addr == NULL) {
 170                 CPT_LOG_ERR("PCI mem address null");
 171                 return -EIO;
 172         }
 173
 174         ret = otx_cpt_get_resource(dev, 0, &instance, que_pair_id);
 175         if (ret != 0 || instance == NULL) {
 176                 CPT_LOG_ERR("Error getting instance handle from device %s : "
 177                             "ret = %d", dev->data->name, ret);
 178                 return ret;
 179         }
 180
 181         instance->queue_id = que_pair_id;
 182         instance->sess_mp = qp_conf->mp_session;
 183         instance->sess_mp_priv = qp_conf->mp_session_private;
 184         dev->data->queue_pairs[que_pair_id] = instance;
 185
 186         return 0;
 187 }
 188
 189 static int
 190 otx_cpt_que_pair_release(struct rte_cryptodev *dev, uint16_t que_pair_id)
 191 {
 192         struct cpt_instance *instance = dev->data->queue_pairs[que_pair_id];
 193         int ret;
 194
 195         CPT_PMD_INIT_FUNC_TRACE();
 196
 197         ret = otx_cpt_put_resource(instance);
 198         if (ret != 0) {
 199                 CPT_LOG_ERR("Error putting instance handle of device %s : "
 200                             "ret = %d", dev->data->name, ret);
 201                 return ret;
 202         }
 203
 204         dev->data->queue_pairs[que_pair_id] = NULL;
 205
 206         return 0;
 207 }
 208
 209 static unsigned int
 210 otx_cpt_get_session_size(struct rte_cryptodev *dev __rte_unused)
 211 {
 212         return cpt_get_session_size();
 213 }
 214
 215 static int
 216 sym_xform_verify(struct rte_crypto_sym_xform *xform)
 217 {
 218         if (xform->next) {
 219                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 220                     xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 221                     xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 222                         return -ENOTSUP;
 223
 224                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 225                     xform->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT &&
 226                     xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 227                         return -ENOTSUP;
 228
 229                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 230                     xform->cipher.algo == RTE_CRYPTO_CIPHER_3DES_CBC &&
 231                     xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 232                     xform->next->auth.algo == RTE_CRYPTO_AUTH_SHA1)
 233                         return -ENOTSUP;
 234
 235                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 236                     xform->auth.algo == RTE_CRYPTO_AUTH_SHA1 &&
 237                     xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 238                     xform->next->cipher.algo == RTE_CRYPTO_CIPHER_3DES_CBC)
 239                         return -ENOTSUP;
 240
 241         } else {
 242                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 243                     xform->auth.algo == RTE_CRYPTO_AUTH_NULL &&
 244                     xform->auth.op == RTE_CRYPTO_AUTH_OP_VERIFY)
 245                         return -ENOTSUP;
 246         }
 247         return 0;
 248 }
 249
 250 static int
 251 sym_session_configure(int driver_id, struct rte_crypto_sym_xform *xform,
 252                       struct rte_cryptodev_sym_session *sess,
 253                       struct rte_mempool *pool)
 254 {
 255         struct cpt_sess_misc *misc;
 256         void *priv;
 257         int ret;
 258
 259         ret = sym_xform_verify(xform);
 260         if (unlikely(ret))
 261                 return ret;
 262
 263         if (unlikely(rte_mempool_get(pool, &priv))) {
 264                 CPT_LOG_ERR("Could not allocate session private data");
 265                 return -ENOMEM;
 266         }
 267
 268         misc = priv;
 269
 270         for ( ; xform != NULL; xform = xform->next) {
 271                 switch (xform->type) {
 272                 case RTE_CRYPTO_SYM_XFORM_AEAD:
 273                         ret = fill_sess_aead(xform, misc);
 274                         break;
 275                 case RTE_CRYPTO_SYM_XFORM_CIPHER:
 276                         ret = fill_sess_cipher(xform, misc);
 277                         break;
 278                 case RTE_CRYPTO_SYM_XFORM_AUTH:
 279                         if (xform->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC)
 280                                 ret = fill_sess_gmac(xform, misc);
 281                         else
 282                                 ret = fill_sess_auth(xform, misc);
 283                         break;
 284                 default:
 285                         ret = -1;
 286                 }
 287
 288                 if (ret)
 289                         goto priv_put;
 290         }
 291
 292         set_sym_session_private_data(sess, driver_id, priv);
 293
 294         misc->ctx_dma_addr = rte_mempool_virt2iova(misc) +
 295                              sizeof(struct cpt_sess_misc);
 296
 297         return 0;
 298
 299 priv_put:
 300         if (priv)
 301                 rte_mempool_put(pool, priv);
 302         return -ENOTSUP;
 303 }
 304
 305 static void
 306 sym_session_clear(int driver_id, struct rte_cryptodev_sym_session *sess)
 307 {
 308         void *priv = get_sym_session_private_data(sess, driver_id);
 309         struct rte_mempool *pool;
 310
 311         if (priv == NULL)
 312                 return;
 313
 314         memset(priv, 0, cpt_get_session_size());
 315
 316         pool = rte_mempool_from_obj(priv);
 317
 318         set_sym_session_private_data(sess, driver_id, NULL);
 319
 320         rte_mempool_put(pool, priv);
 321 }
 322
 323 static int
 324 otx_cpt_session_cfg(struct rte_cryptodev *dev,
 325                     struct rte_crypto_sym_xform *xform,
 326                     struct rte_cryptodev_sym_session *sess,
 327                     struct rte_mempool *pool)
 328 {
 329         CPT_PMD_INIT_FUNC_TRACE();
 330
 331         return sym_session_configure(dev->driver_id, xform, sess, pool);
 332 }
 333
 334
 335 static void
 336 otx_cpt_session_clear(struct rte_cryptodev *dev,
 337                   struct rte_cryptodev_sym_session *sess)
 338 {
 339         CPT_PMD_INIT_FUNC_TRACE();
 340
 341         return sym_session_clear(dev->driver_id, sess);
 342 }
 343
 344 static unsigned int
 345 otx_cpt_asym_session_size_get(struct rte_cryptodev *dev __rte_unused)
 346 {
 347         return sizeof(struct cpt_asym_sess_misc);
 348 }
 349
 350 static int
 351 otx_cpt_asym_session_cfg(struct rte_cryptodev *dev,
 352                          struct rte_crypto_asym_xform *xform __rte_unused,
 353                          struct rte_cryptodev_asym_session *sess,
 354                          struct rte_mempool *pool)
 355 {
 356         struct cpt_asym_sess_misc *priv;
 357         int ret;
 358
 359         CPT_PMD_INIT_FUNC_TRACE();
 360
 361         if (rte_mempool_get(pool, (void **)&priv)) {
 362                 CPT_LOG_ERR("Could not allocate session private data");
 363                 return -ENOMEM;
 364         }
 365
 366         memset(priv, 0, sizeof(struct cpt_asym_sess_misc));
 367
 368         ret = cpt_fill_asym_session_parameters(priv, xform);
 369         if (ret) {
 370                 CPT_LOG_ERR("Could not configure session parameters");
 371
 372                 /* Return session to mempool */
 373                 rte_mempool_put(pool, priv);
 374                 return ret;
 375         }
 376
 377         set_asym_session_private_data(sess, dev->driver_id, priv);
 378         return 0;
 379 }
 380
 381 static void
 382 otx_cpt_asym_session_clear(struct rte_cryptodev *dev,
 383                            struct rte_cryptodev_asym_session *sess)
 384 {
 385         struct cpt_asym_sess_misc *priv;
 386         struct rte_mempool *sess_mp;
 387
 388         CPT_PMD_INIT_FUNC_TRACE();
 389
 390         priv = get_asym_session_private_data(sess, dev->driver_id);
 391
 392         if (priv == NULL)
 393                 return;
 394
 395         /* Free resources allocated during session configure */
 396         cpt_free_asym_session_parameters(priv);
 397         memset(priv, 0, otx_cpt_asym_session_size_get(dev));
 398         sess_mp = rte_mempool_from_obj(priv);
 399         set_asym_session_private_data(sess, dev->driver_id, NULL);
 400         rte_mempool_put(sess_mp, priv);
 401 }
 402
 403 static __rte_always_inline int32_t __rte_hot
 404 otx_cpt_request_enqueue(struct cpt_instance *instance,
 405                         struct pending_queue *pqueue,
 406                         void *req)
 407 {
 408         struct cpt_request_info *user_req = (struct cpt_request_info *)req;
 409
 410         if (unlikely(pqueue->pending_count >= DEFAULT_CMD_QLEN))
 411                 return -EAGAIN;
 412
 413         fill_cpt_inst(instance, req);
 414
 415         CPT_LOG_DP_DEBUG("req: %p op: %p ", req, user_req->op);
 416
 417         /* Fill time_out cycles */
 418         user_req->time_out = rte_get_timer_cycles() +
 419                         DEFAULT_COMMAND_TIMEOUT * rte_get_timer_hz();
 420         user_req->extra_time = 0;
 421
 422         /* Default mode of software queue */
 423         mark_cpt_inst(instance);
 424
 425         pqueue->rid_queue[pqueue->enq_tail].rid = (uintptr_t)user_req;
 426
 427         /* We will use soft queue length here to limit requests */
 428         MOD_INC(pqueue->enq_tail, DEFAULT_CMD_QLEN);
 429         pqueue->pending_count += 1;
 430
 431         CPT_LOG_DP_DEBUG("Submitted NB cmd with request: %p "
 432                          "op: %p", user_req, user_req->op);
 433         return 0;
 434 }
 435
 436 static __rte_always_inline int __rte_hot
 437 otx_cpt_enq_single_asym(struct cpt_instance *instance,
 438                         struct rte_crypto_op *op,
 439                         struct pending_queue *pqueue)
 440 {
 441         struct cpt_qp_meta_info *minfo = &instance->meta_info;
 442         struct rte_crypto_asym_op *asym_op = op->asym;
 443         struct asym_op_params params = {0};
 444         struct cpt_asym_sess_misc *sess;
 445         uintptr_t *cop;
 446         void *mdata;
 447         int ret;
 448
 449         if (unlikely(rte_mempool_get(minfo->pool, &mdata) < 0)) {
 450                 CPT_LOG_DP_ERR("Could not allocate meta buffer for request");
 451                 return -ENOMEM;
 452         }
 453
 454         sess = get_asym_session_private_data(asym_op->session,
 455                                              otx_cryptodev_driver_id);
 456
 457         /* Store phys_addr of the mdata to meta_buf */
 458         params.meta_buf = rte_mempool_virt2iova(mdata);
 459
 460         cop = mdata;
 461         cop[0] = (uintptr_t)mdata;
 462         cop[1] = (uintptr_t)op;
 463         cop[2] = cop[3] = 0ULL;
 464
 465         params.req = RTE_PTR_ADD(cop, 4 * sizeof(uintptr_t));
 466         params.req->op = cop;
 467
 468         /* Adjust meta_buf by crypto_op data  and request_info struct */
 469         params.meta_buf += (4 * sizeof(uintptr_t)) +
 470                            sizeof(struct cpt_request_info);
 471
 472         switch (sess->xfrm_type) {
 473         case RTE_CRYPTO_ASYM_XFORM_MODEX:
 474                 ret = cpt_modex_prep(&params, &sess->mod_ctx);
 475                 if (unlikely(ret))
 476                         goto req_fail;
 477                 break;
 478         case RTE_CRYPTO_ASYM_XFORM_RSA:
 479                 ret = cpt_enqueue_rsa_op(op, &params, sess);
 480                 if (unlikely(ret))
 481                         goto req_fail;
 482                 break;
 483         case RTE_CRYPTO_ASYM_XFORM_ECDSA:
 484                 ret = cpt_enqueue_ecdsa_op(op, &params, sess, otx_fpm_iova);
 485                 if (unlikely(ret))
 486                         goto req_fail;
 487                 break;
 488         case RTE_CRYPTO_ASYM_XFORM_ECPM:
 489                 ret = cpt_ecpm_prep(&asym_op->ecpm, &params,
 490                                     sess->ec_ctx.curveid);
 491                 if (unlikely(ret))
 492                         goto req_fail;
 493                 break;
 494
 495         default:
 496                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 497                 ret = -EINVAL;
 498                 goto req_fail;
 499         }
 500
 501         ret = otx_cpt_request_enqueue(instance, pqueue, params.req);
 502
 503         if (unlikely(ret)) {
 504                 CPT_LOG_DP_ERR("Could not enqueue crypto req");
 505                 goto req_fail;
 506         }
 507
 508         return 0;
 509
 510 req_fail:
 511         free_op_meta(mdata, minfo->pool);
 512
 513         return ret;
 514 }
 515
 516 static __rte_always_inline int __rte_hot
 517 otx_cpt_enq_single_sym(struct cpt_instance *instance,
 518                        struct rte_crypto_op *op,
 519                        struct pending_queue *pqueue)
 520 {
 521         struct cpt_sess_misc *sess;
 522         struct rte_crypto_sym_op *sym_op = op->sym;
 523         void *prep_req, *mdata = NULL;
 524         int ret = 0;
 525         uint64_t cpt_op;
 526
 527         sess = (struct cpt_sess_misc *)
 528                         get_sym_session_private_data(sym_op->session,
 529                                                      otx_cryptodev_driver_id);
 530
 531         cpt_op = sess->cpt_op;
 532
 533         if (likely(cpt_op & CPT_OP_CIPHER_MASK))
 534                 ret = fill_fc_params(op, sess, &instance->meta_info, &mdata,
 535                                      &prep_req);
 536         else
 537                 ret = fill_digest_params(op, sess, &instance->meta_info,
 538                                          &mdata, &prep_req);
 539
 540         if (unlikely(ret)) {
 541                 CPT_LOG_DP_ERR("prep cryto req : op %p, cpt_op 0x%x "
 542                                "ret 0x%x", op, (unsigned int)cpt_op, ret);
 543                 return ret;
 544         }
 545
 546         /* Enqueue prepared instruction to h/w */
 547         ret = otx_cpt_request_enqueue(instance, pqueue, prep_req);
 548
 549         if (unlikely(ret)) {
 550                 /* Buffer allocated for request preparation need to be freed */
 551                 free_op_meta(mdata, instance->meta_info.pool);
 552                 return ret;
 553         }
 554
 555         return 0;
 556 }
 557
 558 static __rte_always_inline int __rte_hot
 559 otx_cpt_enq_single_sym_sessless(struct cpt_instance *instance,
 560                                 struct rte_crypto_op *op,
 561                                 struct pending_queue *pend_q)
 562 {
 563         const int driver_id = otx_cryptodev_driver_id;
 564         struct rte_crypto_sym_op *sym_op = op->sym;
 565         struct rte_cryptodev_sym_session *sess;
 566         int ret;
 567
 568         /* Create temporary session */
 569
 570         if (rte_mempool_get(instance->sess_mp, (void **)&sess))
 571                 return -ENOMEM;
 572
 573         ret = sym_session_configure(driver_id, sym_op->xform, sess,
 574                                     instance->sess_mp_priv);
 575         if (ret)
 576                 goto sess_put;
 577
 578         sym_op->session = sess;
 579
 580         ret = otx_cpt_enq_single_sym(instance, op, pend_q);
 581
 582         if (unlikely(ret))
 583                 goto priv_put;
 584
 585         return 0;
 586
 587 priv_put:
 588         sym_session_clear(driver_id, sess);
 589 sess_put:
 590         rte_mempool_put(instance->sess_mp, sess);
 591         return ret;
 592 }
 593
 594 #define OP_TYPE_SYM             0
 595 #define OP_TYPE_ASYM            1
 596
 597 static __rte_always_inline int __rte_hot
 598 otx_cpt_enq_single(struct cpt_instance *inst,
 599                    struct rte_crypto_op *op,
 600                    struct pending_queue *pqueue,
 601                    const uint8_t op_type)
 602 {
 603         /* Check for the type */
 604
 605         if (op_type == OP_TYPE_SYM) {
 606                 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION)
 607                         return otx_cpt_enq_single_sym(inst, op, pqueue);
 608                 else
 609                         return otx_cpt_enq_single_sym_sessless(inst, op,
 610                                                                pqueue);
 611         }
 612
 613         if (op_type == OP_TYPE_ASYM) {
 614                 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION)
 615                         return otx_cpt_enq_single_asym(inst, op, pqueue);
 616         }
 617
 618         /* Should not reach here */
 619         return -ENOTSUP;
 620 }
 621
 622 static  __rte_always_inline uint16_t __rte_hot
 623 otx_cpt_pkt_enqueue(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops,
 624                     const uint8_t op_type)
 625 {
 626         struct cpt_instance *instance = (struct cpt_instance *)qptr;
 627         uint16_t count;
 628         int ret;
 629         struct cpt_vf *cptvf = (struct cpt_vf *)instance;
 630         struct pending_queue *pqueue = &cptvf->pqueue;
 631
 632         count = DEFAULT_CMD_QLEN - pqueue->pending_count;
 633         if (nb_ops > count)
 634                 nb_ops = count;
 635
 636         count = 0;
 637         while (likely(count < nb_ops)) {
 638
 639                 /* Enqueue single op */
 640                 ret = otx_cpt_enq_single(instance, ops[count], pqueue, op_type);
 641
 642                 if (unlikely(ret))
 643                         break;
 644                 count++;
 645         }
 646         otx_cpt_ring_dbell(instance, count);
 647         return count;
 648 }
 649
 650 static uint16_t
 651 otx_cpt_enqueue_asym(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
 652 {
 653         return otx_cpt_pkt_enqueue(qptr, ops, nb_ops, OP_TYPE_ASYM);
 654 }
 655
 656 static uint16_t
 657 otx_cpt_enqueue_sym(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
 658 {
 659         return otx_cpt_pkt_enqueue(qptr, ops, nb_ops, OP_TYPE_SYM);
 660 }
 661
 662 static inline void
 663 otx_cpt_asym_rsa_op(struct rte_crypto_op *cop, struct cpt_request_info *req,
 664                     struct rte_crypto_rsa_xform *rsa_ctx)
 665
 666 {
 667         struct rte_crypto_rsa_op_param *rsa = &cop->asym->rsa;
 668
 669         switch (rsa->op_type) {
 670         case RTE_CRYPTO_ASYM_OP_ENCRYPT:
 671                 rsa->cipher.length = rsa_ctx->n.length;
 672                 memcpy(rsa->cipher.data, req->rptr, rsa->cipher.length);
 673                 break;
 674         case RTE_CRYPTO_ASYM_OP_DECRYPT:
 675                 if (rsa->pad == RTE_CRYPTO_RSA_PADDING_NONE)
 676                         rsa->message.length = rsa_ctx->n.length;
 677                 else {
 678                         /* Get length of decrypted output */
 679                         rsa->message.length = rte_cpu_to_be_16
 680                                         (*((uint16_t *)req->rptr));
 681
 682                         /* Offset data pointer by length fields */
 683                         req->rptr += 2;
 684                 }
 685                 memcpy(rsa->message.data, req->rptr, rsa->message.length);
 686                 break;
 687         case RTE_CRYPTO_ASYM_OP_SIGN:
 688                 rsa->sign.length = rsa_ctx->n.length;
 689                 memcpy(rsa->sign.data, req->rptr, rsa->sign.length);
 690                 break;
 691         case RTE_CRYPTO_ASYM_OP_VERIFY:
 692                 if (rsa->pad == RTE_CRYPTO_RSA_PADDING_NONE)
 693                         rsa->sign.length = rsa_ctx->n.length;
 694                 else {
 695                         /* Get length of decrypted output */
 696                         rsa->sign.length = rte_cpu_to_be_16
 697                                         (*((uint16_t *)req->rptr));
 698
 699                         /* Offset data pointer by length fields */
 700                         req->rptr += 2;
 701                 }
 702                 memcpy(rsa->sign.data, req->rptr, rsa->sign.length);
 703
 704                 if (memcmp(rsa->sign.data, rsa->message.data,
 705                            rsa->message.length)) {
 706                         CPT_LOG_DP_ERR("RSA verification failed");
 707                         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
 708                 }
 709                 break;
 710         default:
 711                 CPT_LOG_DP_DEBUG("Invalid RSA operation type");
 712                 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 713                 break;
 714         }
 715 }
 716
 717 static __rte_always_inline void
 718 otx_cpt_asym_dequeue_ecdsa_op(struct rte_crypto_ecdsa_op_param *ecdsa,
 719                             struct cpt_request_info *req,
 720                             struct cpt_asym_ec_ctx *ec)
 721
 722 {
 723         int prime_len = ec_grp[ec->curveid].prime.length;
 724
 725         if (ecdsa->op_type == RTE_CRYPTO_ASYM_OP_VERIFY)
 726                 return;
 727
 728         /* Separate out sign r and s components */
 729         memcpy(ecdsa->r.data, req->rptr, prime_len);
 730         memcpy(ecdsa->s.data, req->rptr + ROUNDUP8(prime_len), prime_len);
 731         ecdsa->r.length = prime_len;
 732         ecdsa->s.length = prime_len;
 733 }
 734
 735 static __rte_always_inline void
 736 otx_cpt_asym_dequeue_ecpm_op(struct rte_crypto_ecpm_op_param *ecpm,
 737                              struct cpt_request_info *req,
 738                              struct cpt_asym_ec_ctx *ec)
 739 {
 740         int prime_len = ec_grp[ec->curveid].prime.length;
 741
 742         memcpy(ecpm->r.x.data, req->rptr, prime_len);
 743         memcpy(ecpm->r.y.data, req->rptr + ROUNDUP8(prime_len), prime_len);
 744         ecpm->r.x.length = prime_len;
 745         ecpm->r.y.length = prime_len;
 746 }
 747
 748 static __rte_always_inline void __rte_hot
 749 otx_cpt_asym_post_process(struct rte_crypto_op *cop,
 750                           struct cpt_request_info *req)
 751 {
 752         struct rte_crypto_asym_op *op = cop->asym;
 753         struct cpt_asym_sess_misc *sess;
 754
 755         sess = get_asym_session_private_data(op->session,
 756                                              otx_cryptodev_driver_id);
 757
 758         switch (sess->xfrm_type) {
 759         case RTE_CRYPTO_ASYM_XFORM_RSA:
 760                 otx_cpt_asym_rsa_op(cop, req, &sess->rsa_ctx);
 761                 break;
 762         case RTE_CRYPTO_ASYM_XFORM_MODEX:
 763                 op->modex.result.length = sess->mod_ctx.modulus.length;
 764                 memcpy(op->modex.result.data, req->rptr,
 765                        op->modex.result.length);
 766                 break;
 767         case RTE_CRYPTO_ASYM_XFORM_ECDSA:
 768                 otx_cpt_asym_dequeue_ecdsa_op(&op->ecdsa, req, &sess->ec_ctx);
 769                 break;
 770         case RTE_CRYPTO_ASYM_XFORM_ECPM:
 771                 otx_cpt_asym_dequeue_ecpm_op(&op->ecpm, req, &sess->ec_ctx);
 772                 break;
 773         default:
 774                 CPT_LOG_DP_DEBUG("Invalid crypto xform type");
 775                 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 776                 break;
 777         }
 778 }
 779
 780 static __rte_always_inline void __rte_hot
 781 otx_cpt_dequeue_post_process(struct rte_crypto_op *cop, uintptr_t *rsp,
 782                              const uint8_t op_type)
 783 {
 784         /* H/w has returned success */
 785         cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 786
 787         /* Perform further post processing */
 788
 789         if ((op_type == OP_TYPE_SYM) &&
 790             (cop->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC)) {
 791                 /* Check if auth verify need to be completed */
 792                 if (unlikely(rsp[2]))
 793                         compl_auth_verify(cop, (uint8_t *)rsp[2], rsp[3]);
 794                 return;
 795         }
 796
 797         if ((op_type == OP_TYPE_ASYM) &&
 798             (cop->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC)) {
 799                 rsp = RTE_PTR_ADD(rsp, 4 * sizeof(uintptr_t));
 800                 otx_cpt_asym_post_process(cop, (struct cpt_request_info *)rsp);
 801         }
 802
 803         return;
 804 }
 805
 806 static __rte_always_inline uint16_t __rte_hot
 807 otx_cpt_pkt_dequeue(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops,
 808                     const uint8_t op_type)
 809 {
 810         struct cpt_instance *instance = (struct cpt_instance *)qptr;
 811         struct cpt_request_info *user_req;
 812         struct cpt_vf *cptvf = (struct cpt_vf *)instance;
 813         struct rid *rid_e;
 814         uint8_t cc[nb_ops];
 815         int i, count, pcount;
 816         uint8_t ret;
 817         int nb_completed;
 818         struct pending_queue *pqueue = &cptvf->pqueue;
 819         struct rte_crypto_op *cop;
 820         void *metabuf;
 821         uintptr_t *rsp;
 822
 823         pcount = pqueue->pending_count;
 824         count = (nb_ops > pcount) ? pcount : nb_ops;
 825
 826         for (i = 0; i < count; i++) {
 827                 rid_e = &pqueue->rid_queue[pqueue->deq_head];
 828                 user_req = (struct cpt_request_info *)(rid_e->rid);
 829
 830                 if (likely((i+1) < count))
 831                         rte_prefetch_non_temporal((void *)rid_e[1].rid);
 832
 833                 ret = check_nb_command_id(user_req, instance);
 834
 835                 if (unlikely(ret == ERR_REQ_PENDING)) {
 836                         /* Stop checking for completions */
 837                         break;
 838                 }
 839
 840                 /* Return completion code and op handle */
 841                 cc[i] = ret;
 842                 ops[i] = user_req->op;
 843
 844                 CPT_LOG_DP_DEBUG("Request %p Op %p completed with code %d",
 845                                  user_req, user_req->op, ret);
 846
 847                 MOD_INC(pqueue->deq_head, DEFAULT_CMD_QLEN);
 848                 pqueue->pending_count -= 1;
 849         }
 850
 851         nb_completed = i;
 852
 853         for (i = 0; i < nb_completed; i++) {
 854
 855                 rsp = (void *)ops[i];
 856
 857                 if (likely((i + 1) < nb_completed))
 858                         rte_prefetch0(ops[i+1]);
 859
 860                 metabuf = (void *)rsp[0];
 861                 cop = (void *)rsp[1];
 862
 863                 ops[i] = cop;
 864
 865                 /* Check completion code */
 866
 867                 if (likely(cc[i] == 0)) {
 868                         /* H/w success pkt. Post process */
 869                         otx_cpt_dequeue_post_process(cop, rsp, op_type);
 870                 } else if (cc[i] == ERR_GC_ICV_MISCOMPARE) {
 871                         /* auth data mismatch */
 872                         cop->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 873                 } else {
 874                         /* Error */
 875                         cop->status = RTE_CRYPTO_OP_STATUS_ERROR;
 876                 }
 877
 878                 if (unlikely(cop->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
 879                         void *sess_private_data_t =
 880                                 get_sym_session_private_data(cop->sym->session,
 881                                                 otx_cryptodev_driver_id);
 882                         memset(sess_private_data_t, 0,
 883                                         cpt_get_session_size());
 884                         memset(cop->sym->session, 0,
 885                         rte_cryptodev_sym_get_existing_header_session_size(
 886                                         cop->sym->session));
 887                         rte_mempool_put(instance->sess_mp_priv,
 888                                         sess_private_data_t);
 889                         rte_mempool_put(instance->sess_mp, cop->sym->session);
 890                         cop->sym->session = NULL;
 891                 }
 892                 free_op_meta(metabuf, instance->meta_info.pool);
 893         }
 894
 895         return nb_completed;
 896 }
 897
 898 static uint16_t
 899 otx_cpt_dequeue_asym(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
 900 {
 901         return otx_cpt_pkt_dequeue(qptr, ops, nb_ops, OP_TYPE_ASYM);
 902 }
 903
 904 static uint16_t
 905 otx_cpt_dequeue_sym(void *qptr, struct rte_crypto_op **ops, uint16_t nb_ops)
 906 {
 907         return otx_cpt_pkt_dequeue(qptr, ops, nb_ops, OP_TYPE_SYM);
 908 }
 909
 910 static struct rte_cryptodev_ops cptvf_ops = {
 911         /* Device related operations */
 912         .dev_configure = otx_cpt_dev_config,
 913         .dev_start = otx_cpt_dev_start,
 914         .dev_stop = otx_cpt_dev_stop,
 915         .dev_close = otx_cpt_dev_close,
 916         .dev_infos_get = otx_cpt_dev_info_get,
 917
 918         .stats_get = otx_cpt_stats_get,
 919         .stats_reset = otx_cpt_stats_reset,
 920         .queue_pair_setup = otx_cpt_que_pair_setup,
 921         .queue_pair_release = otx_cpt_que_pair_release,
 922
 923         /* Crypto related operations */
 924         .sym_session_get_size = otx_cpt_get_session_size,
 925         .sym_session_configure = otx_cpt_session_cfg,
 926         .sym_session_clear = otx_cpt_session_clear,
 927
 928         .asym_session_get_size = otx_cpt_asym_session_size_get,
 929         .asym_session_configure = otx_cpt_asym_session_cfg,
 930         .asym_session_clear = otx_cpt_asym_session_clear,
 931 };
 932
 933 int
 934 otx_cpt_dev_create(struct rte_cryptodev *c_dev)
 935 {
 936         struct rte_pci_device *pdev = RTE_DEV_TO_PCI(c_dev->device);
 937         struct cpt_vf *cptvf = NULL;
 938         void *reg_base;
 939         char dev_name[32];
 940         int ret;
 941
 942         if (pdev->mem_resource[0].phys_addr == 0ULL)
 943                 return -EIO;
 944
 945         /* for secondary processes, we don't initialise any further as primary
 946          * has already done this work.
 947          */
 948         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 949                 return 0;
 950
 951         cptvf = rte_zmalloc_socket("otx_cryptodev_private_mem",
 952                         sizeof(struct cpt_vf), RTE_CACHE_LINE_SIZE,
 953                         rte_socket_id());
 954
 955         if (cptvf == NULL) {
 956                 CPT_LOG_ERR("Cannot allocate memory for device private data");
 957                 return -ENOMEM;
 958         }
 959
 960         snprintf(dev_name, 32, "%02x:%02x.%x",
 961                         pdev->addr.bus, pdev->addr.devid, pdev->addr.function);
 962
 963         reg_base = pdev->mem_resource[0].addr;
 964         if (!reg_base) {
 965                 CPT_LOG_ERR("Failed to map BAR0 of %s", dev_name);
 966                 ret = -ENODEV;
 967                 goto fail;
 968         }
 969
 970         ret = otx_cpt_hw_init(cptvf, pdev, reg_base, dev_name);
 971         if (ret) {
 972                 CPT_LOG_ERR("Failed to init cptvf %s", dev_name);
 973                 ret = -EIO;
 974                 goto fail;
 975         }
 976
 977         switch (cptvf->vftype) {
 978         case OTX_CPT_VF_TYPE_AE:
 979                 /* Set asymmetric cpt feature flags */
 980                 c_dev->feature_flags = RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO |
 981                                 RTE_CRYPTODEV_FF_HW_ACCELERATED |
 982                                 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT;
 983                 break;
 984         case OTX_CPT_VF_TYPE_SE:
 985                 /* Set symmetric cpt feature flags */
 986                 c_dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
 987                                 RTE_CRYPTODEV_FF_HW_ACCELERATED |
 988                                 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
 989                                 RTE_CRYPTODEV_FF_IN_PLACE_SGL |
 990                                 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
 991                                 RTE_CRYPTODEV_FF_OOP_SGL_IN_SGL_OUT |
 992                                 RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA |
 993                                 RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
 994                 break;
 995         default:
 996                 /* Feature not supported. Abort */
 997                 CPT_LOG_ERR("VF type not supported by %s", dev_name);
 998                 ret = -EIO;
 999                 goto deinit_dev;
1000         }
1001
1002         /* Start off timer for mailbox interrupts */
1003         otx_cpt_periodic_alarm_start(cptvf);
1004
1005         c_dev->dev_ops = &cptvf_ops;
1006
1007         if (c_dev->feature_flags & RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO) {
1008                 c_dev->enqueue_burst = otx_cpt_enqueue_sym;
1009                 c_dev->dequeue_burst = otx_cpt_dequeue_sym;
1010         } else {
1011                 c_dev->enqueue_burst = otx_cpt_enqueue_asym;
1012                 c_dev->dequeue_burst = otx_cpt_dequeue_asym;
1013         }
1014
1015         /* Save dev private data */
1016         c_dev->data->dev_private = cptvf;
1017
1018         return 0;
1019
1020 deinit_dev:
1021         otx_cpt_deinit_device(cptvf);
1022
1023 fail:
1024         if (cptvf) {
1025                 /* Free private data allocated */
1026                 rte_free(cptvf);
1027         }
1028
1029         return ret;
1030 }