drivers/crypto/ccp/rte_ccp_pmd.c

   1 /*   SPDX-License-Identifier: BSD-3-Clause
   2  *   Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved.
   3  */
   4
   5 #include <rte_bus_pci.h>
   6 #include <rte_bus_vdev.h>
   7 #include <rte_common.h>
   8 #include <rte_config.h>
   9 #include <rte_cryptodev.h>
  10 #include <rte_cryptodev_pmd.h>
  11 #include <rte_pci.h>
  12 #include <rte_dev.h>
  13 #include <rte_malloc.h>
  14
  15 #include "ccp_crypto.h"
  16 #include "ccp_dev.h"
  17 #include "ccp_pmd_private.h"
  18
  19 /**
  20  * Global static parameter used to find if CCP device is already initialized.
  21  */
  22 static unsigned int ccp_pmd_init_done;
  23 uint8_t ccp_cryptodev_driver_id;
  24
  25 struct ccp_pmd_init_params {
  26         struct rte_cryptodev_pmd_init_params def_p;
  27         bool auth_opt;
  28 };
  29
  30 #define CCP_CRYPTODEV_PARAM_NAME                ("name")
  31 #define CCP_CRYPTODEV_PARAM_SOCKET_ID           ("socket_id")
  32 #define CCP_CRYPTODEV_PARAM_MAX_NB_QP           ("max_nb_queue_pairs")
  33 #define CCP_CRYPTODEV_PARAM_AUTH_OPT            ("ccp_auth_opt")
  34
  35 const char *ccp_pmd_valid_params[] = {
  36         CCP_CRYPTODEV_PARAM_NAME,
  37         CCP_CRYPTODEV_PARAM_SOCKET_ID,
  38         CCP_CRYPTODEV_PARAM_MAX_NB_QP,
  39         CCP_CRYPTODEV_PARAM_AUTH_OPT,
  40 };
  41
  42 /** ccp pmd auth option */
  43 enum ccp_pmd_auth_opt {
  44         CCP_PMD_AUTH_OPT_CCP = 0,
  45         CCP_PMD_AUTH_OPT_CPU,
  46 };
  47
  48 /** parse integer from integer argument */
  49 static int
  50 parse_integer_arg(const char *key __rte_unused,
  51                   const char *value, void *extra_args)
  52 {
  53         int *i = (int *) extra_args;
  54
  55         *i = atoi(value);
  56         if (*i < 0) {
  57                 CCP_LOG_ERR("Argument has to be positive.\n");
  58                 return -EINVAL;
  59         }
  60
  61         return 0;
  62 }
  63
  64 /** parse name argument */
  65 static int
  66 parse_name_arg(const char *key __rte_unused,
  67                const char *value, void *extra_args)
  68 {
  69         struct rte_cryptodev_pmd_init_params *params = extra_args;
  70
  71         if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
  72                 CCP_LOG_ERR("Invalid name %s, should be less than "
  73                             "%u bytes.\n", value,
  74                             RTE_CRYPTODEV_NAME_MAX_LEN - 1);
  75                 return -EINVAL;
  76         }
  77
  78         strncpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN);
  79
  80         return 0;
  81 }
  82
  83 /** parse authentication operation option */
  84 static int
  85 parse_auth_opt_arg(const char *key __rte_unused,
  86                    const char *value, void *extra_args)
  87 {
  88         struct ccp_pmd_init_params *params = extra_args;
  89         int i;
  90
  91         i = atoi(value);
  92         if (i < CCP_PMD_AUTH_OPT_CCP || i > CCP_PMD_AUTH_OPT_CPU) {
  93                 CCP_LOG_ERR("Invalid ccp pmd auth option. "
  94                             "0->auth on CCP(default), "
  95                             "1->auth on CPU\n");
  96                 return -EINVAL;
  97         }
  98         params->auth_opt = i;
  99         return 0;
 100 }
 101
 102 static int
 103 ccp_pmd_parse_input_args(struct ccp_pmd_init_params *params,
 104                          const char *input_args)
 105 {
 106         struct rte_kvargs *kvlist = NULL;
 107         int ret = 0;
 108
 109         if (params == NULL)
 110                 return -EINVAL;
 111
 112         if (input_args) {
 113                 kvlist = rte_kvargs_parse(input_args,
 114                                           ccp_pmd_valid_params);
 115                 if (kvlist == NULL)
 116                         return -1;
 117
 118                 ret = rte_kvargs_process(kvlist,
 119                                          CCP_CRYPTODEV_PARAM_MAX_NB_QP,
 120                                          &parse_integer_arg,
 121                                          &params->def_p.max_nb_queue_pairs);
 122                 if (ret < 0)
 123                         goto free_kvlist;
 124
 125                 ret = rte_kvargs_process(kvlist,
 126                                          CCP_CRYPTODEV_PARAM_SOCKET_ID,
 127                                          &parse_integer_arg,
 128                                          &params->def_p.socket_id);
 129                 if (ret < 0)
 130                         goto free_kvlist;
 131
 132                 ret = rte_kvargs_process(kvlist,
 133                                          CCP_CRYPTODEV_PARAM_NAME,
 134                                          &parse_name_arg,
 135                                          &params->def_p);
 136                 if (ret < 0)
 137                         goto free_kvlist;
 138
 139                 ret = rte_kvargs_process(kvlist,
 140                                          CCP_CRYPTODEV_PARAM_AUTH_OPT,
 141                                          &parse_auth_opt_arg,
 142                                          params);
 143                 if (ret < 0)
 144                         goto free_kvlist;
 145
 146         }
 147
 148 free_kvlist:
 149         rte_kvargs_free(kvlist);
 150         return ret;
 151 }
 152
 153 static struct ccp_session *
 154 get_ccp_session(struct ccp_qp *qp, struct rte_crypto_op *op)
 155 {
 156         struct ccp_session *sess = NULL;
 157
 158         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 159                 if (unlikely(op->sym->session == NULL))
 160                         return NULL;
 161
 162                 sess = (struct ccp_session *)
 163                         get_sym_session_private_data(
 164                                 op->sym->session,
 165                                 ccp_cryptodev_driver_id);
 166         } else if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
 167                 void *_sess;
 168                 void *_sess_private_data = NULL;
 169                 struct ccp_private *internals;
 170
 171                 if (rte_mempool_get(qp->sess_mp, &_sess))
 172                         return NULL;
 173                 if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data))
 174                         return NULL;
 175
 176                 sess = (struct ccp_session *)_sess_private_data;
 177
 178                 internals = (struct ccp_private *)qp->dev->data->dev_private;
 179                 if (unlikely(ccp_set_session_parameters(sess, op->sym->xform,
 180                                                         internals) != 0)) {
 181                         rte_mempool_put(qp->sess_mp, _sess);
 182                         rte_mempool_put(qp->sess_mp, _sess_private_data);
 183                         sess = NULL;
 184                 }
 185                 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
 186                 set_sym_session_private_data(op->sym->session,
 187                                          ccp_cryptodev_driver_id,
 188                                          _sess_private_data);
 189         }
 190
 191         return sess;
 192 }
 193
 194 static uint16_t
 195 ccp_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 196                       uint16_t nb_ops)
 197 {
 198         struct ccp_session *sess = NULL;
 199         struct ccp_qp *qp = queue_pair;
 200         struct ccp_queue *cmd_q;
 201         struct rte_cryptodev *dev = qp->dev;
 202         uint16_t i, enq_cnt = 0, slots_req = 0;
 203
 204         if (nb_ops == 0)
 205                 return 0;
 206
 207         if (unlikely(rte_ring_full(qp->processed_pkts) != 0))
 208                 return 0;
 209
 210         for (i = 0; i < nb_ops; i++) {
 211                 sess = get_ccp_session(qp, ops[i]);
 212                 if (unlikely(sess == NULL) && (i == 0)) {
 213                         qp->qp_stats.enqueue_err_count++;
 214                         return 0;
 215                 } else if (sess == NULL) {
 216                         nb_ops = i;
 217                         break;
 218                 }
 219                 slots_req += ccp_compute_slot_count(sess);
 220         }
 221
 222         cmd_q = ccp_allot_queue(dev, slots_req);
 223         if (unlikely(cmd_q == NULL))
 224                 return 0;
 225
 226         enq_cnt = process_ops_to_enqueue(qp, ops, cmd_q, nb_ops, slots_req);
 227         qp->qp_stats.enqueued_count += enq_cnt;
 228         return enq_cnt;
 229 }
 230
 231 static uint16_t
 232 ccp_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 233                 uint16_t nb_ops)
 234 {
 235         struct ccp_qp *qp = queue_pair;
 236         uint16_t nb_dequeued = 0, i;
 237
 238         nb_dequeued = process_ops_to_dequeue(qp, ops, nb_ops);
 239
 240         /* Free session if a session-less crypto op */
 241         for (i = 0; i < nb_dequeued; i++)
 242                 if (unlikely(ops[i]->sess_type ==
 243                              RTE_CRYPTO_OP_SESSIONLESS)) {
 244                         rte_mempool_put(qp->sess_mp,
 245                                         ops[i]->sym->session);
 246                         ops[i]->sym->session = NULL;
 247                 }
 248         qp->qp_stats.dequeued_count += nb_dequeued;
 249
 250         return nb_dequeued;
 251 }
 252
 253 /*
 254  * The set of PCI devices this driver supports
 255  */
 256 static struct rte_pci_id ccp_pci_id[] = {
 257         {
 258                 RTE_PCI_DEVICE(0x1022, 0x1456), /* AMD CCP-5a */
 259         },
 260         {
 261                 RTE_PCI_DEVICE(0x1022, 0x1468), /* AMD CCP-5b */
 262         },
 263         {.device_id = 0},
 264 };
 265
 266 /** Remove ccp pmd */
 267 static int
 268 cryptodev_ccp_remove(struct rte_vdev_device *dev)
 269 {
 270         const char *name;
 271
 272         ccp_pmd_init_done = 0;
 273         name = rte_vdev_device_name(dev);
 274         if (name == NULL)
 275                 return -EINVAL;
 276
 277         RTE_LOG(INFO, PMD, "Closing ccp device %s on numa socket %u\n",
 278                         name, rte_socket_id());
 279
 280         return 0;
 281 }
 282
 283 /** Create crypto device */
 284 static int
 285 cryptodev_ccp_create(const char *name,
 286                      struct rte_vdev_device *vdev,
 287                      struct ccp_pmd_init_params *init_params)
 288 {
 289         struct rte_cryptodev *dev;
 290         struct ccp_private *internals;
 291         uint8_t cryptodev_cnt = 0;
 292
 293         if (init_params->def_p.name[0] == '\0')
 294                 snprintf(init_params->def_p.name,
 295                          sizeof(init_params->def_p.name),
 296                          "%s", name);
 297
 298         dev = rte_cryptodev_pmd_create(init_params->def_p.name,
 299                                        &vdev->device,
 300                                        &init_params->def_p);
 301         if (dev == NULL) {
 302                 CCP_LOG_ERR("failed to create cryptodev vdev");
 303                 goto init_error;
 304         }
 305
 306         cryptodev_cnt = ccp_probe_devices(ccp_pci_id);
 307
 308         if (cryptodev_cnt == 0) {
 309                 CCP_LOG_ERR("failed to detect CCP crypto device");
 310                 goto init_error;
 311         }
 312
 313         printf("CCP : Crypto device count = %d\n", cryptodev_cnt);
 314         dev->driver_id = ccp_cryptodev_driver_id;
 315
 316         /* register rx/tx burst functions for data path */
 317         dev->dev_ops = ccp_pmd_ops;
 318         dev->enqueue_burst = ccp_pmd_enqueue_burst;
 319         dev->dequeue_burst = ccp_pmd_dequeue_burst;
 320
 321         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
 322                         RTE_CRYPTODEV_FF_HW_ACCELERATED |
 323                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING;
 324
 325         internals = dev->data->dev_private;
 326
 327         internals->max_nb_qpairs = init_params->def_p.max_nb_queue_pairs;
 328         internals->auth_opt = init_params->auth_opt;
 329         internals->crypto_num_dev = cryptodev_cnt;
 330
 331         return 0;
 332
 333 init_error:
 334         CCP_LOG_ERR("driver %s: %s() failed",
 335                     init_params->def_p.name, __func__);
 336         cryptodev_ccp_remove(vdev);
 337
 338         return -EFAULT;
 339 }
 340
 341 /** Probe ccp pmd */
 342 static int
 343 cryptodev_ccp_probe(struct rte_vdev_device *vdev)
 344 {
 345         int rc = 0;
 346         const char *name;
 347         struct ccp_pmd_init_params init_params = {
 348                 .def_p = {
 349                         "",
 350                         sizeof(struct ccp_private),
 351                         rte_socket_id(),
 352                         CCP_PMD_MAX_QUEUE_PAIRS
 353                 },
 354                 .auth_opt = CCP_PMD_AUTH_OPT_CCP,
 355         };
 356         const char *input_args;
 357
 358         if (ccp_pmd_init_done) {
 359                 RTE_LOG(INFO, PMD, "CCP PMD already initialized\n");
 360                 return -EFAULT;
 361         }
 362         name = rte_vdev_device_name(vdev);
 363         if (name == NULL)
 364                 return -EINVAL;
 365
 366         input_args = rte_vdev_device_args(vdev);
 367         ccp_pmd_parse_input_args(&init_params, input_args);
 368         init_params.def_p.max_nb_queue_pairs = CCP_PMD_MAX_QUEUE_PAIRS;
 369
 370         RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name,
 371                 init_params.def_p.socket_id);
 372         RTE_LOG(INFO, PMD, "Max number of queue pairs = %d\n",
 373                 init_params.def_p.max_nb_queue_pairs);
 374         RTE_LOG(INFO, PMD, "Authentication offload to %s\n",
 375                 ((init_params.auth_opt == 0) ? "CCP" : "CPU"));
 376
 377         rc = cryptodev_ccp_create(name, vdev, &init_params);
 378         if (rc)
 379                 return rc;
 380         ccp_pmd_init_done = 1;
 381         return 0;
 382 }
 383
 384 static struct rte_vdev_driver cryptodev_ccp_pmd_drv = {
 385         .probe = cryptodev_ccp_probe,
 386         .remove = cryptodev_ccp_remove
 387 };
 388
 389 static struct cryptodev_driver ccp_crypto_drv;
 390
 391 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_CCP_PMD, cryptodev_ccp_pmd_drv);
 392 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_CCP_PMD,
 393         "max_nb_queue_pairs=<int> "
 394         "socket_id=<int> "
 395         "ccp_auth_opt=<int>");
 396 RTE_PMD_REGISTER_CRYPTO_DRIVER(ccp_crypto_drv, cryptodev_ccp_pmd_drv.driver,
 397                                ccp_cryptodev_driver_id);