drivers/crypto/ccp/rte_ccp_pmd.c

   1 /*   SPDX-License-Identifier: BSD-3-Clause
   2  *   Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved.
   3  */
   4
   5 #include <rte_string_fns.h>
   6 #include <rte_bus_pci.h>
   7 #include <rte_bus_vdev.h>
   8 #include <rte_common.h>
   9 #include <rte_config.h>
  10 #include <rte_cryptodev.h>
  11 #include <rte_cryptodev_pmd.h>
  12 #include <rte_pci.h>
  13 #include <rte_dev.h>
  14 #include <rte_malloc.h>
  15
  16 #include "ccp_crypto.h"
  17 #include "ccp_dev.h"
  18 #include "ccp_pmd_private.h"
  19
  20 /**
  21  * Global static parameter used to find if CCP device is already initialized.
  22  */
  23 static unsigned int ccp_pmd_init_done;
  24 uint8_t ccp_cryptodev_driver_id;
  25 uint8_t cryptodev_cnt;
  26
  27 struct ccp_pmd_init_params {
  28         struct rte_cryptodev_pmd_init_params def_p;
  29         bool auth_opt;
  30 };
  31
  32 #define CCP_CRYPTODEV_PARAM_NAME                ("name")
  33 #define CCP_CRYPTODEV_PARAM_SOCKET_ID           ("socket_id")
  34 #define CCP_CRYPTODEV_PARAM_MAX_NB_QP           ("max_nb_queue_pairs")
  35 #define CCP_CRYPTODEV_PARAM_AUTH_OPT            ("ccp_auth_opt")
  36
  37 const char *ccp_pmd_valid_params[] = {
  38         CCP_CRYPTODEV_PARAM_NAME,
  39         CCP_CRYPTODEV_PARAM_SOCKET_ID,
  40         CCP_CRYPTODEV_PARAM_MAX_NB_QP,
  41         CCP_CRYPTODEV_PARAM_AUTH_OPT,
  42 };
  43
  44 /** ccp pmd auth option */
  45 enum ccp_pmd_auth_opt {
  46         CCP_PMD_AUTH_OPT_CCP = 0,
  47         CCP_PMD_AUTH_OPT_CPU,
  48 };
  49
  50 /** parse integer from integer argument */
  51 static int
  52 parse_integer_arg(const char *key __rte_unused,
  53                   const char *value, void *extra_args)
  54 {
  55         int *i = (int *) extra_args;
  56
  57         *i = atoi(value);
  58         if (*i < 0) {
  59                 CCP_LOG_ERR("Argument has to be positive.\n");
  60                 return -EINVAL;
  61         }
  62
  63         return 0;
  64 }
  65
  66 /** parse name argument */
  67 static int
  68 parse_name_arg(const char *key __rte_unused,
  69                const char *value, void *extra_args)
  70 {
  71         struct rte_cryptodev_pmd_init_params *params = extra_args;
  72
  73         if (strlen(value) >= RTE_CRYPTODEV_NAME_MAX_LEN - 1) {
  74                 CCP_LOG_ERR("Invalid name %s, should be less than "
  75                             "%u bytes.\n", value,
  76                             RTE_CRYPTODEV_NAME_MAX_LEN - 1);
  77                 return -EINVAL;
  78         }
  79
  80         strncpy(params->name, value, RTE_CRYPTODEV_NAME_MAX_LEN);
  81
  82         return 0;
  83 }
  84
  85 /** parse authentication operation option */
  86 static int
  87 parse_auth_opt_arg(const char *key __rte_unused,
  88                    const char *value, void *extra_args)
  89 {
  90         struct ccp_pmd_init_params *params = extra_args;
  91         int i;
  92
  93         i = atoi(value);
  94         if (i < CCP_PMD_AUTH_OPT_CCP || i > CCP_PMD_AUTH_OPT_CPU) {
  95                 CCP_LOG_ERR("Invalid ccp pmd auth option. "
  96                             "0->auth on CCP(default), "
  97                             "1->auth on CPU\n");
  98                 return -EINVAL;
  99         }
 100         params->auth_opt = i;
 101         return 0;
 102 }
 103
 104 static int
 105 ccp_pmd_parse_input_args(struct ccp_pmd_init_params *params,
 106                          const char *input_args)
 107 {
 108         struct rte_kvargs *kvlist = NULL;
 109         int ret = 0;
 110
 111         if (params == NULL)
 112                 return -EINVAL;
 113
 114         if (input_args) {
 115                 kvlist = rte_kvargs_parse(input_args,
 116                                           ccp_pmd_valid_params);
 117                 if (kvlist == NULL)
 118                         return -1;
 119
 120                 ret = rte_kvargs_process(kvlist,
 121                                          CCP_CRYPTODEV_PARAM_MAX_NB_QP,
 122                                          &parse_integer_arg,
 123                                          &params->def_p.max_nb_queue_pairs);
 124                 if (ret < 0)
 125                         goto free_kvlist;
 126
 127                 ret = rte_kvargs_process(kvlist,
 128                                          CCP_CRYPTODEV_PARAM_SOCKET_ID,
 129                                          &parse_integer_arg,
 130                                          &params->def_p.socket_id);
 131                 if (ret < 0)
 132                         goto free_kvlist;
 133
 134                 ret = rte_kvargs_process(kvlist,
 135                                          CCP_CRYPTODEV_PARAM_NAME,
 136                                          &parse_name_arg,
 137                                          &params->def_p);
 138                 if (ret < 0)
 139                         goto free_kvlist;
 140
 141                 ret = rte_kvargs_process(kvlist,
 142                                          CCP_CRYPTODEV_PARAM_AUTH_OPT,
 143                                          &parse_auth_opt_arg,
 144                                          params);
 145                 if (ret < 0)
 146                         goto free_kvlist;
 147
 148         }
 149
 150 free_kvlist:
 151         rte_kvargs_free(kvlist);
 152         return ret;
 153 }
 154
 155 static struct ccp_session *
 156 get_ccp_session(struct ccp_qp *qp, struct rte_crypto_op *op)
 157 {
 158         struct ccp_session *sess = NULL;
 159
 160         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 161                 if (unlikely(op->sym->session == NULL))
 162                         return NULL;
 163
 164                 sess = (struct ccp_session *)
 165                         get_sym_session_private_data(
 166                                 op->sym->session,
 167                                 ccp_cryptodev_driver_id);
 168         } else if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
 169                 void *_sess;
 170                 void *_sess_private_data = NULL;
 171                 struct ccp_private *internals;
 172
 173                 if (rte_mempool_get(qp->sess_mp, &_sess))
 174                         return NULL;
 175                 if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data))
 176                         return NULL;
 177
 178                 sess = (struct ccp_session *)_sess_private_data;
 179
 180                 internals = (struct ccp_private *)qp->dev->data->dev_private;
 181                 if (unlikely(ccp_set_session_parameters(sess, op->sym->xform,
 182                                                         internals) != 0)) {
 183                         rte_mempool_put(qp->sess_mp, _sess);
 184                         rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
 185                         sess = NULL;
 186                 }
 187                 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
 188                 set_sym_session_private_data(op->sym->session,
 189                                          ccp_cryptodev_driver_id,
 190                                          _sess_private_data);
 191         }
 192
 193         return sess;
 194 }
 195
 196 static uint16_t
 197 ccp_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 198                       uint16_t nb_ops)
 199 {
 200         struct ccp_session *sess = NULL;
 201         struct ccp_qp *qp = queue_pair;
 202         struct ccp_queue *cmd_q;
 203         struct rte_cryptodev *dev = qp->dev;
 204         uint16_t i, enq_cnt = 0, slots_req = 0;
 205         uint16_t tmp_ops = nb_ops, b_idx, cur_ops = 0;
 206
 207         if (nb_ops == 0)
 208                 return 0;
 209
 210         if (unlikely(rte_ring_full(qp->processed_pkts) != 0))
 211                 return 0;
 212         if (tmp_ops >= cryptodev_cnt)
 213                 cur_ops = nb_ops / cryptodev_cnt + (nb_ops)%cryptodev_cnt;
 214         else
 215                 cur_ops = tmp_ops;
 216         while (tmp_ops) {
 217                 b_idx = nb_ops - tmp_ops;
 218                 slots_req = 0;
 219                 if (cur_ops <= tmp_ops) {
 220                         tmp_ops -= cur_ops;
 221                 } else {
 222                         cur_ops = tmp_ops;
 223                         tmp_ops = 0;
 224                 }
 225                 for (i = 0; i < cur_ops; i++) {
 226                         sess = get_ccp_session(qp, ops[i + b_idx]);
 227                         if (unlikely(sess == NULL) && (i == 0)) {
 228                                 qp->qp_stats.enqueue_err_count++;
 229                                 return 0;
 230                         } else if (sess == NULL) {
 231                                 cur_ops = i;
 232                                 break;
 233                         }
 234                         slots_req += ccp_compute_slot_count(sess);
 235                 }
 236
 237                 cmd_q = ccp_allot_queue(dev, slots_req);
 238                 if (unlikely(cmd_q == NULL))
 239                         return 0;
 240                 enq_cnt += process_ops_to_enqueue(qp, ops, cmd_q, cur_ops,
 241                                 nb_ops, slots_req, b_idx);
 242                 i++;
 243         }
 244
 245         qp->qp_stats.enqueued_count += enq_cnt;
 246         return enq_cnt;
 247 }
 248
 249 static uint16_t
 250 ccp_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 251                 uint16_t nb_ops)
 252 {
 253         struct ccp_qp *qp = queue_pair;
 254         uint16_t nb_dequeued = 0, i, total_nb_ops;
 255
 256         nb_dequeued = process_ops_to_dequeue(qp, ops, nb_ops, &total_nb_ops);
 257
 258         if (total_nb_ops) {
 259                 while (nb_dequeued != total_nb_ops) {
 260                         nb_dequeued = process_ops_to_dequeue(qp,
 261                                         ops, nb_ops, &total_nb_ops);
 262                 }
 263         }
 264
 265         /* Free session if a session-less crypto op */
 266         for (i = 0; i < nb_dequeued; i++)
 267                 if (unlikely(ops[i]->sess_type ==
 268                              RTE_CRYPTO_OP_SESSIONLESS)) {
 269                         struct ccp_session *sess = (struct ccp_session *)
 270                                         get_sym_session_private_data(
 271                                                 ops[i]->sym->session,
 272                                                 ccp_cryptodev_driver_id);
 273
 274                         rte_mempool_put(qp->sess_mp_priv,
 275                                         sess);
 276                         rte_mempool_put(qp->sess_mp,
 277                                         ops[i]->sym->session);
 278                         ops[i]->sym->session = NULL;
 279                 }
 280         qp->qp_stats.dequeued_count += nb_dequeued;
 281
 282         return nb_dequeued;
 283 }
 284
 285 /*
 286  * The set of PCI devices this driver supports
 287  */
 288 static struct rte_pci_id ccp_pci_id[] = {
 289         {
 290                 RTE_PCI_DEVICE(0x1022, 0x1456), /* AMD CCP-5a */
 291         },
 292         {
 293                 RTE_PCI_DEVICE(0x1022, 0x1468), /* AMD CCP-5b */
 294         },
 295         {.device_id = 0},
 296 };
 297
 298 /** Remove ccp pmd */
 299 static int
 300 cryptodev_ccp_remove(struct rte_vdev_device *dev)
 301 {
 302         const char *name;
 303
 304         ccp_pmd_init_done = 0;
 305         name = rte_vdev_device_name(dev);
 306         if (name == NULL)
 307                 return -EINVAL;
 308
 309         RTE_LOG(INFO, PMD, "Closing ccp device %s on numa socket %u\n",
 310                         name, rte_socket_id());
 311
 312         return 0;
 313 }
 314
 315 /** Create crypto device */
 316 static int
 317 cryptodev_ccp_create(const char *name,
 318                      struct rte_vdev_device *vdev,
 319                      struct ccp_pmd_init_params *init_params)
 320 {
 321         struct rte_cryptodev *dev;
 322         struct ccp_private *internals;
 323
 324         if (init_params->def_p.name[0] == '\0')
 325                 strlcpy(init_params->def_p.name, name,
 326                         sizeof(init_params->def_p.name));
 327
 328         dev = rte_cryptodev_pmd_create(init_params->def_p.name,
 329                                        &vdev->device,
 330                                        &init_params->def_p);
 331         if (dev == NULL) {
 332                 CCP_LOG_ERR("failed to create cryptodev vdev");
 333                 goto init_error;
 334         }
 335
 336         cryptodev_cnt = ccp_probe_devices(ccp_pci_id);
 337
 338         if (cryptodev_cnt == 0) {
 339                 CCP_LOG_ERR("failed to detect CCP crypto device");
 340                 goto init_error;
 341         }
 342
 343         printf("CCP : Crypto device count = %d\n", cryptodev_cnt);
 344         dev->driver_id = ccp_cryptodev_driver_id;
 345
 346         /* register rx/tx burst functions for data path */
 347         dev->dev_ops = ccp_pmd_ops;
 348         dev->enqueue_burst = ccp_pmd_enqueue_burst;
 349         dev->dequeue_burst = ccp_pmd_dequeue_burst;
 350
 351         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
 352                         RTE_CRYPTODEV_FF_HW_ACCELERATED |
 353                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING;
 354
 355         internals = dev->data->dev_private;
 356
 357         internals->max_nb_qpairs = init_params->def_p.max_nb_queue_pairs;
 358         internals->auth_opt = init_params->auth_opt;
 359         internals->crypto_num_dev = cryptodev_cnt;
 360
 361         return 0;
 362
 363 init_error:
 364         CCP_LOG_ERR("driver %s: %s() failed",
 365                     init_params->def_p.name, __func__);
 366         cryptodev_ccp_remove(vdev);
 367
 368         return -EFAULT;
 369 }
 370
 371 /** Probe ccp pmd */
 372 static int
 373 cryptodev_ccp_probe(struct rte_vdev_device *vdev)
 374 {
 375         int rc = 0;
 376         const char *name;
 377         struct ccp_pmd_init_params init_params = {
 378                 .def_p = {
 379                         "",
 380                         sizeof(struct ccp_private),
 381                         rte_socket_id(),
 382                         CCP_PMD_MAX_QUEUE_PAIRS
 383                 },
 384                 .auth_opt = CCP_PMD_AUTH_OPT_CCP,
 385         };
 386         const char *input_args;
 387
 388         if (ccp_pmd_init_done) {
 389                 RTE_LOG(INFO, PMD, "CCP PMD already initialized\n");
 390                 return -EFAULT;
 391         }
 392         name = rte_vdev_device_name(vdev);
 393         if (name == NULL)
 394                 return -EINVAL;
 395
 396         input_args = rte_vdev_device_args(vdev);
 397         ccp_pmd_parse_input_args(&init_params, input_args);
 398         init_params.def_p.max_nb_queue_pairs = CCP_PMD_MAX_QUEUE_PAIRS;
 399
 400         RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name,
 401                 init_params.def_p.socket_id);
 402         RTE_LOG(INFO, PMD, "Max number of queue pairs = %d\n",
 403                 init_params.def_p.max_nb_queue_pairs);
 404         RTE_LOG(INFO, PMD, "Authentication offload to %s\n",
 405                 ((init_params.auth_opt == 0) ? "CCP" : "CPU"));
 406
 407         rc = cryptodev_ccp_create(name, vdev, &init_params);
 408         if (rc)
 409                 return rc;
 410         ccp_pmd_init_done = 1;
 411         return 0;
 412 }
 413
 414 static struct rte_vdev_driver cryptodev_ccp_pmd_drv = {
 415         .probe = cryptodev_ccp_probe,
 416         .remove = cryptodev_ccp_remove
 417 };
 418
 419 static struct cryptodev_driver ccp_crypto_drv;
 420
 421 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_CCP_PMD, cryptodev_ccp_pmd_drv);
 422 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_CCP_PMD,
 423         "max_nb_queue_pairs=<int> "
 424         "socket_id=<int> "
 425         "ccp_auth_opt=<int>");
 426 RTE_PMD_REGISTER_CRYPTO_DRIVER(ccp_crypto_drv, cryptodev_ccp_pmd_drv.driver,
 427                                ccp_cryptodev_driver_id);