drivers/crypto/bcmfs/bcmfs_sym_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Broadcom
   3  * All rights reserved.
   4  */
   5
   6 #include <rte_common.h>
   7 #include <rte_dev.h>
   8 #include <rte_errno.h>
   9 #include <rte_malloc.h>
  10 #include <rte_cryptodev_pmd.h>
  11
  12 #include "bcmfs_device.h"
  13 #include "bcmfs_logs.h"
  14 #include "bcmfs_qp.h"
  15 #include "bcmfs_sym_pmd.h"
  16 #include "bcmfs_sym_req.h"
  17 #include "bcmfs_sym_session.h"
  18 #include "bcmfs_sym_capabilities.h"
  19
  20 uint8_t cryptodev_bcmfs_driver_id;
  21
  22 static int bcmfs_sym_qp_release(struct rte_cryptodev *dev,
  23                                 uint16_t queue_pair_id);
  24
  25 static int
  26 bcmfs_sym_dev_config(__rte_unused struct rte_cryptodev *dev,
  27                      __rte_unused struct rte_cryptodev_config *config)
  28 {
  29         return 0;
  30 }
  31
  32 static int
  33 bcmfs_sym_dev_start(__rte_unused struct rte_cryptodev *dev)
  34 {
  35         return 0;
  36 }
  37
  38 static void
  39 bcmfs_sym_dev_stop(__rte_unused struct rte_cryptodev *dev)
  40 {
  41 }
  42
  43 static int
  44 bcmfs_sym_dev_close(struct rte_cryptodev *dev)
  45 {
  46         int i, ret;
  47
  48         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
  49                 ret = bcmfs_sym_qp_release(dev, i);
  50                 if (ret < 0)
  51                         return ret;
  52         }
  53
  54         return 0;
  55 }
  56
  57 static void
  58 bcmfs_sym_dev_info_get(struct rte_cryptodev *dev,
  59                        struct rte_cryptodev_info *dev_info)
  60 {
  61         struct bcmfs_sym_dev_private *internals = dev->data->dev_private;
  62         struct bcmfs_device *fsdev = internals->fsdev;
  63
  64         if (dev_info != NULL) {
  65                 dev_info->driver_id = cryptodev_bcmfs_driver_id;
  66                 dev_info->feature_flags = dev->feature_flags;
  67                 dev_info->max_nb_queue_pairs = fsdev->max_hw_qps;
  68                 /* No limit of number of sessions */
  69                 dev_info->sym.max_nb_sessions = 0;
  70                 dev_info->capabilities = bcmfs_sym_get_capabilities();
  71         }
  72 }
  73
  74 static void
  75 bcmfs_sym_stats_get(struct rte_cryptodev *dev,
  76                     struct rte_cryptodev_stats *stats)
  77 {
  78         struct bcmfs_qp_stats bcmfs_stats = {0};
  79         struct bcmfs_sym_dev_private *bcmfs_priv;
  80         struct bcmfs_device *fsdev;
  81
  82         if (stats == NULL || dev == NULL) {
  83                 BCMFS_LOG(ERR, "invalid ptr: stats %p, dev %p", stats, dev);
  84                 return;
  85         }
  86         bcmfs_priv = dev->data->dev_private;
  87         fsdev = bcmfs_priv->fsdev;
  88
  89         bcmfs_qp_stats_get(fsdev->qps_in_use, fsdev->max_hw_qps, &bcmfs_stats);
  90
  91         stats->enqueued_count = bcmfs_stats.enqueued_count;
  92         stats->dequeued_count = bcmfs_stats.dequeued_count;
  93         stats->enqueue_err_count = bcmfs_stats.enqueue_err_count;
  94         stats->dequeue_err_count = bcmfs_stats.dequeue_err_count;
  95 }
  96
  97 static void
  98 bcmfs_sym_stats_reset(struct rte_cryptodev *dev)
  99 {
 100         struct bcmfs_sym_dev_private *bcmfs_priv;
 101         struct bcmfs_device *fsdev;
 102
 103         if (dev == NULL) {
 104                 BCMFS_LOG(ERR, "invalid cryptodev ptr %p", dev);
 105                 return;
 106         }
 107         bcmfs_priv = dev->data->dev_private;
 108         fsdev = bcmfs_priv->fsdev;
 109
 110         bcmfs_qp_stats_reset(fsdev->qps_in_use, fsdev->max_hw_qps);
 111 }
 112
 113 static int
 114 bcmfs_sym_qp_release(struct rte_cryptodev *dev, uint16_t queue_pair_id)
 115 {
 116         struct bcmfs_sym_dev_private *bcmfs_private = dev->data->dev_private;
 117         struct bcmfs_qp *qp = (struct bcmfs_qp *)
 118                               (dev->data->queue_pairs[queue_pair_id]);
 119
 120         BCMFS_LOG(DEBUG, "Release sym qp %u on device %d",
 121                   queue_pair_id, dev->data->dev_id);
 122
 123         rte_mempool_free(qp->sr_mp);
 124
 125         bcmfs_private->fsdev->qps_in_use[queue_pair_id] = NULL;
 126
 127         return bcmfs_qp_release((struct bcmfs_qp **)
 128                                 &dev->data->queue_pairs[queue_pair_id]);
 129 }
 130
 131 static void
 132 spu_req_init(struct bcmfs_sym_request *sr, rte_iova_t iova __rte_unused)
 133 {
 134         memset(sr, 0, sizeof(*sr));
 135         sr->fptr = iova;
 136         sr->cptr = iova + offsetof(struct bcmfs_sym_request, cipher_key);
 137         sr->aptr = iova + offsetof(struct bcmfs_sym_request, auth_key);
 138         sr->iptr = iova + offsetof(struct bcmfs_sym_request, iv);
 139         sr->dptr = iova + offsetof(struct bcmfs_sym_request, digest);
 140         sr->rptr = iova + offsetof(struct bcmfs_sym_request, resp);
 141 }
 142
 143 static void
 144 req_pool_obj_init(__rte_unused struct rte_mempool *mp,
 145                   __rte_unused void *opaque, void *obj,
 146                   __rte_unused unsigned int obj_idx)
 147 {
 148         spu_req_init(obj, rte_mempool_virt2iova(obj));
 149 }
 150
 151 static struct rte_mempool *
 152 bcmfs_sym_req_pool_create(struct rte_cryptodev *cdev __rte_unused,
 153                           uint32_t nobjs, uint16_t qp_id,
 154                           int socket_id)
 155 {
 156         char softreq_pool_name[RTE_RING_NAMESIZE];
 157         struct rte_mempool *mp;
 158
 159         snprintf(softreq_pool_name, RTE_RING_NAMESIZE, "%s_%d",
 160                  "bcm_sym", qp_id);
 161
 162         mp = rte_mempool_create(softreq_pool_name,
 163                                 RTE_ALIGN_MUL_CEIL(nobjs, 64),
 164                                 sizeof(struct bcmfs_sym_request),
 165                                 64, 0, NULL, NULL, req_pool_obj_init, NULL,
 166                                 socket_id, 0);
 167         if (mp == NULL)
 168                 BCMFS_LOG(ERR, "Failed to create req pool, qid %d, err %d",
 169                                 qp_id, rte_errno);
 170
 171         return mp;
 172 }
 173
 174 static int
 175 bcmfs_sym_qp_setup(struct rte_cryptodev *cdev, uint16_t qp_id,
 176                    const struct rte_cryptodev_qp_conf *qp_conf,
 177                    int socket_id)
 178 {
 179         int ret = 0;
 180         struct bcmfs_qp *qp = NULL;
 181         struct bcmfs_qp_config bcmfs_qp_conf;
 182
 183         struct bcmfs_qp **qp_addr =
 184                         (struct bcmfs_qp **)&cdev->data->queue_pairs[qp_id];
 185         struct bcmfs_sym_dev_private *bcmfs_private = cdev->data->dev_private;
 186         struct bcmfs_device *fsdev = bcmfs_private->fsdev;
 187
 188
 189         /* If qp is already in use free ring memory and qp metadata. */
 190         if (*qp_addr != NULL) {
 191                 ret = bcmfs_sym_qp_release(cdev, qp_id);
 192                 if (ret < 0)
 193                         return ret;
 194         }
 195
 196         if (qp_id >= fsdev->max_hw_qps) {
 197                 BCMFS_LOG(ERR, "qp_id %u invalid for this device", qp_id);
 198                 return -EINVAL;
 199         }
 200
 201         bcmfs_qp_conf.nb_descriptors = qp_conf->nb_descriptors;
 202         bcmfs_qp_conf.socket_id = socket_id;
 203         bcmfs_qp_conf.max_descs_req = BCMFS_CRYPTO_MAX_HW_DESCS_PER_REQ;
 204         bcmfs_qp_conf.iobase = BCMFS_QP_IOBASE_XLATE(fsdev->mmap_addr, qp_id);
 205         bcmfs_qp_conf.ops = fsdev->sym_hw_qp_ops;
 206
 207         ret = bcmfs_qp_setup(qp_addr, qp_id, &bcmfs_qp_conf);
 208         if (ret != 0)
 209                 return ret;
 210
 211         qp = (struct bcmfs_qp *)*qp_addr;
 212
 213         qp->sr_mp = bcmfs_sym_req_pool_create(cdev, qp_conf->nb_descriptors,
 214                                               qp_id, socket_id);
 215         if (qp->sr_mp == NULL)
 216                 return -ENOMEM;
 217
 218         /* store a link to the qp in the bcmfs_device */
 219         bcmfs_private->fsdev->qps_in_use[qp_id] = *qp_addr;
 220
 221         cdev->data->queue_pairs[qp_id] = qp;
 222         BCMFS_LOG(NOTICE, "queue %d setup done\n", qp_id);
 223
 224         return 0;
 225 }
 226
 227 static struct rte_cryptodev_ops crypto_bcmfs_ops = {
 228         /* Device related operations */
 229         .dev_configure          = bcmfs_sym_dev_config,
 230         .dev_start              = bcmfs_sym_dev_start,
 231         .dev_stop               = bcmfs_sym_dev_stop,
 232         .dev_close              = bcmfs_sym_dev_close,
 233         .dev_infos_get          = bcmfs_sym_dev_info_get,
 234         /* Stats Collection */
 235         .stats_get              = bcmfs_sym_stats_get,
 236         .stats_reset            = bcmfs_sym_stats_reset,
 237         /* Queue-Pair management */
 238         .queue_pair_setup       = bcmfs_sym_qp_setup,
 239         .queue_pair_release     = bcmfs_sym_qp_release,
 240         /* Crypto session related operations */
 241         .sym_session_get_size   = bcmfs_sym_session_get_private_size,
 242         .sym_session_configure  = bcmfs_sym_session_configure,
 243         .sym_session_clear      = bcmfs_sym_session_clear
 244 };
 245
 246 /** Enqueue burst */
 247 static uint16_t
 248 bcmfs_sym_pmd_enqueue_op_burst(void *queue_pair,
 249                                struct rte_crypto_op **ops,
 250                                uint16_t nb_ops)
 251 {
 252         int i, j;
 253         int retval;
 254         uint16_t enq = 0;
 255         struct bcmfs_sym_request *sreq;
 256         struct bcmfs_sym_session *sess;
 257         struct bcmfs_qp *qp = (struct bcmfs_qp *)queue_pair;
 258
 259         if (nb_ops == 0)
 260                 return 0;
 261
 262         if (nb_ops > BCMFS_MAX_REQS_BUFF)
 263                 nb_ops = BCMFS_MAX_REQS_BUFF;
 264
 265          /* We do not process more than available space */
 266         if (nb_ops >  (qp->nb_descriptors - qp->nb_pending_requests))
 267                 nb_ops = qp->nb_descriptors - qp->nb_pending_requests;
 268
 269         for (i = 0; i < nb_ops; i++) {
 270                 sess = bcmfs_sym_get_session(ops[i]);
 271                 if (unlikely(sess == NULL))
 272                         goto enqueue_err;
 273
 274                 if (rte_mempool_get(qp->sr_mp, (void **)&sreq))
 275                         goto enqueue_err;
 276
 277                 /* save rte_crypto_op */
 278                 sreq->op = ops[i];
 279
 280                 /* save context */
 281                 qp->infl_msgs[i] = &sreq->msgs;
 282                 qp->infl_msgs[i]->ctx = (void *)sreq;
 283
 284                 /* pre process the request crypto h/w acceleration */
 285                 retval = bcmfs_process_sym_crypto_op(ops[i], sess, sreq);
 286                 if (unlikely(retval < 0))
 287                         goto enqueue_err;
 288         }
 289         /* Send burst request to hw QP */
 290         enq = bcmfs_enqueue_op_burst(qp, (void **)qp->infl_msgs, i);
 291
 292         for (j = enq; j < i; j++)
 293                 rte_mempool_put(qp->sr_mp, qp->infl_msgs[j]->ctx);
 294
 295         return enq;
 296
 297 enqueue_err:
 298         for (j = 0; j < i; j++)
 299                 rte_mempool_put(qp->sr_mp, qp->infl_msgs[j]->ctx);
 300
 301         return enq;
 302 }
 303
 304 static void bcmfs_sym_set_request_status(struct rte_crypto_op *op,
 305                                          struct bcmfs_sym_request *out)
 306 {
 307         if (*out->resp == BCMFS_SYM_RESPONSE_SUCCESS)
 308                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 309         else if (*out->resp == BCMFS_SYM_RESPONSE_HASH_TAG_ERROR)
 310                 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 311         else
 312                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 313 }
 314
 315 static uint16_t
 316 bcmfs_sym_pmd_dequeue_op_burst(void *queue_pair,
 317                                struct rte_crypto_op **ops,
 318                                uint16_t nb_ops)
 319 {
 320         int i;
 321         uint16_t deq = 0;
 322         unsigned int pkts = 0;
 323         struct bcmfs_sym_request *sreq;
 324         struct bcmfs_qp *qp = queue_pair;
 325
 326         if (nb_ops > BCMFS_MAX_REQS_BUFF)
 327                 nb_ops = BCMFS_MAX_REQS_BUFF;
 328
 329         deq = bcmfs_dequeue_op_burst(qp, (void **)qp->infl_msgs, nb_ops);
 330         /* get rte_crypto_ops */
 331         for (i = 0; i < deq; i++) {
 332                 sreq = (struct bcmfs_sym_request *)qp->infl_msgs[i]->ctx;
 333
 334                 /* set the status based on the response from the crypto h/w */
 335                 bcmfs_sym_set_request_status(sreq->op, sreq);
 336
 337                 ops[pkts++] = sreq->op;
 338
 339                 rte_mempool_put(qp->sr_mp, sreq);
 340         }
 341
 342         return pkts;
 343 }
 344
 345 /*
 346  * An rte_driver is needed in the registration of both the
 347  * device and the driver with cryptodev.
 348  */
 349 static const char bcmfs_sym_drv_name[] = RTE_STR(CRYPTODEV_NAME_BCMFS_SYM_PMD);
 350 static const struct rte_driver cryptodev_bcmfs_sym_driver = {
 351         .name = bcmfs_sym_drv_name,
 352         .alias = bcmfs_sym_drv_name
 353 };
 354
 355 int
 356 bcmfs_sym_dev_create(struct bcmfs_device *fsdev)
 357 {
 358         struct rte_cryptodev_pmd_init_params init_params = {
 359                 .name = "",
 360                 .socket_id = rte_socket_id(),
 361                 .private_data_size = sizeof(struct bcmfs_sym_dev_private)
 362         };
 363         char name[RTE_CRYPTODEV_NAME_MAX_LEN];
 364         struct rte_cryptodev *cryptodev;
 365         struct bcmfs_sym_dev_private *internals;
 366
 367         snprintf(name, RTE_CRYPTODEV_NAME_MAX_LEN, "%s_%s",
 368                  fsdev->name, "sym");
 369
 370         /* Populate subset device to use in cryptodev device creation */
 371         fsdev->sym_rte_dev.driver = &cryptodev_bcmfs_sym_driver;
 372         fsdev->sym_rte_dev.numa_node = 0;
 373         fsdev->sym_rte_dev.devargs = NULL;
 374
 375         cryptodev = rte_cryptodev_pmd_create(name,
 376                                              &fsdev->sym_rte_dev,
 377                                              &init_params);
 378         if (cryptodev == NULL)
 379                 return -ENODEV;
 380
 381         fsdev->sym_rte_dev.name = cryptodev->data->name;
 382         cryptodev->driver_id = cryptodev_bcmfs_driver_id;
 383         cryptodev->dev_ops = &crypto_bcmfs_ops;
 384
 385         cryptodev->enqueue_burst = bcmfs_sym_pmd_enqueue_op_burst;
 386         cryptodev->dequeue_burst = bcmfs_sym_pmd_dequeue_op_burst;
 387
 388         cryptodev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
 389                                    RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
 390                                    RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
 391
 392         internals = cryptodev->data->dev_private;
 393         internals->fsdev = fsdev;
 394         fsdev->sym_dev = internals;
 395
 396         internals->sym_dev_id = cryptodev->data->dev_id;
 397         internals->fsdev_capabilities = bcmfs_sym_get_capabilities();
 398
 399         BCMFS_LOG(DEBUG, "Created bcmfs-sym device %s as cryptodev instance %d",
 400                   cryptodev->data->name, internals->sym_dev_id);
 401         return 0;
 402 }
 403
 404 int
 405 bcmfs_sym_dev_destroy(struct bcmfs_device *fsdev)
 406 {
 407         struct rte_cryptodev *cryptodev;
 408
 409         if (fsdev == NULL)
 410                 return -ENODEV;
 411         if (fsdev->sym_dev == NULL)
 412                 return 0;
 413
 414         /* free crypto device */
 415         cryptodev = rte_cryptodev_pmd_get_dev(fsdev->sym_dev->sym_dev_id);
 416         rte_cryptodev_pmd_destroy(cryptodev);
 417         fsdev->sym_rte_dev.name = NULL;
 418         fsdev->sym_dev = NULL;
 419
 420         return 0;
 421 }
 422
 423 static struct cryptodev_driver bcmfs_crypto_drv;
 424 RTE_PMD_REGISTER_CRYPTO_DRIVER(bcmfs_crypto_drv,
 425                                cryptodev_bcmfs_sym_driver,
 426                                cryptodev_bcmfs_driver_id);