drivers/crypto/ccp/rte_ccp_pmd.c

   1 /*   SPDX-License-Identifier: BSD-3-Clause
   2  *   Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved.
   3  */
   4
   5 #include <rte_string_fns.h>
   6 #include <rte_bus_pci.h>
   7 #include <rte_bus_vdev.h>
   8 #include <rte_common.h>
   9 #include <rte_cryptodev.h>
  10 #include <cryptodev_pmd.h>
  11 #include <rte_pci.h>
  12 #include <rte_dev.h>
  13 #include <rte_malloc.h>
  14
  15 #include "ccp_crypto.h"
  16 #include "ccp_dev.h"
  17 #include "ccp_pmd_private.h"
  18
  19 /**
  20  * Global static parameter used to find if CCP device is already initialized.
  21  */
  22 static unsigned int ccp_pmd_init_done;
  23 uint8_t ccp_cryptodev_driver_id;
  24 uint8_t cryptodev_cnt;
  25 extern void *sha_ctx;
  26
  27 struct ccp_pmd_init_params {
  28         struct rte_cryptodev_pmd_init_params def_p;
  29         bool auth_opt;
  30 };
  31
  32 #define CCP_CRYPTODEV_PARAM_NAME                ("name")
  33 #define CCP_CRYPTODEV_PARAM_SOCKET_ID           ("socket_id")
  34 #define CCP_CRYPTODEV_PARAM_MAX_NB_QP           ("max_nb_queue_pairs")
  35 #define CCP_CRYPTODEV_PARAM_AUTH_OPT            ("ccp_auth_opt")
  36
  37 const char *ccp_pmd_valid_params[] = {
  38         CCP_CRYPTODEV_PARAM_NAME,
  39         CCP_CRYPTODEV_PARAM_SOCKET_ID,
  40         CCP_CRYPTODEV_PARAM_MAX_NB_QP,
  41         CCP_CRYPTODEV_PARAM_AUTH_OPT,
  42 };
  43
  44 /** ccp pmd auth option */
  45 enum ccp_pmd_auth_opt {
  46         CCP_PMD_AUTH_OPT_CCP = 0,
  47         CCP_PMD_AUTH_OPT_CPU,
  48 };
  49
  50 static struct ccp_session *
  51 get_ccp_session(struct ccp_qp *qp, struct rte_crypto_op *op)
  52 {
  53         struct ccp_session *sess = NULL;
  54
  55         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
  56                 if (unlikely(op->sym->session == NULL))
  57                         return NULL;
  58
  59                 sess = (struct ccp_session *)
  60                         get_sym_session_private_data(
  61                                 op->sym->session,
  62                                 ccp_cryptodev_driver_id);
  63         } else if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
  64                 void *_sess;
  65                 void *_sess_private_data = NULL;
  66                 struct ccp_private *internals;
  67
  68                 if (rte_mempool_get(qp->sess_mp, &_sess))
  69                         return NULL;
  70                 if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data))
  71                         return NULL;
  72
  73                 sess = (struct ccp_session *)_sess_private_data;
  74
  75                 internals = (struct ccp_private *)qp->dev->data->dev_private;
  76                 if (unlikely(ccp_set_session_parameters(sess, op->sym->xform,
  77                                                         internals) != 0)) {
  78                         rte_mempool_put(qp->sess_mp, _sess);
  79                         rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
  80                         sess = NULL;
  81                 }
  82                 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
  83                 set_sym_session_private_data(op->sym->session,
  84                                          ccp_cryptodev_driver_id,
  85                                          _sess_private_data);
  86         }
  87
  88         return sess;
  89 }
  90
  91 static uint16_t
  92 ccp_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
  93                       uint16_t nb_ops)
  94 {
  95         struct ccp_session *sess = NULL;
  96         struct ccp_qp *qp = queue_pair;
  97         struct ccp_queue *cmd_q;
  98         struct rte_cryptodev *dev = qp->dev;
  99         uint16_t i, enq_cnt = 0, slots_req = 0;
 100         uint16_t tmp_ops = nb_ops, b_idx, cur_ops = 0;
 101
 102         if (nb_ops == 0)
 103                 return 0;
 104
 105         if (unlikely(rte_ring_full(qp->processed_pkts) != 0))
 106                 return 0;
 107         if (tmp_ops >= cryptodev_cnt)
 108                 cur_ops = nb_ops / cryptodev_cnt + (nb_ops)%cryptodev_cnt;
 109         else
 110                 cur_ops = tmp_ops;
 111         while (tmp_ops) {
 112                 b_idx = nb_ops - tmp_ops;
 113                 slots_req = 0;
 114                 if (cur_ops <= tmp_ops) {
 115                         tmp_ops -= cur_ops;
 116                 } else {
 117                         cur_ops = tmp_ops;
 118                         tmp_ops = 0;
 119                 }
 120                 for (i = 0; i < cur_ops; i++) {
 121                         sess = get_ccp_session(qp, ops[i + b_idx]);
 122                         if (unlikely(sess == NULL) && (i == 0)) {
 123                                 qp->qp_stats.enqueue_err_count++;
 124                                 return 0;
 125                         } else if (sess == NULL) {
 126                                 cur_ops = i;
 127                                 break;
 128                         }
 129                         slots_req += ccp_compute_slot_count(sess);
 130                 }
 131
 132                 cmd_q = ccp_allot_queue(dev, slots_req);
 133                 if (unlikely(cmd_q == NULL))
 134                         return 0;
 135                 enq_cnt += process_ops_to_enqueue(qp, ops, cmd_q, cur_ops,
 136                                 nb_ops, slots_req, b_idx);
 137                 i++;
 138         }
 139
 140         qp->qp_stats.enqueued_count += enq_cnt;
 141         return enq_cnt;
 142 }
 143
 144 static uint16_t
 145 ccp_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 146                 uint16_t nb_ops)
 147 {
 148         struct ccp_qp *qp = queue_pair;
 149         uint16_t nb_dequeued = 0, i, total_nb_ops;
 150
 151         nb_dequeued = process_ops_to_dequeue(qp, ops, nb_ops, &total_nb_ops);
 152
 153         if (total_nb_ops) {
 154                 while (nb_dequeued != total_nb_ops) {
 155                         nb_dequeued = process_ops_to_dequeue(qp,
 156                                         ops, nb_ops, &total_nb_ops);
 157                 }
 158         }
 159
 160         /* Free session if a session-less crypto op */
 161         for (i = 0; i < nb_dequeued; i++)
 162                 if (unlikely(ops[i]->sess_type ==
 163                              RTE_CRYPTO_OP_SESSIONLESS)) {
 164                         struct ccp_session *sess = (struct ccp_session *)
 165                                         get_sym_session_private_data(
 166                                                 ops[i]->sym->session,
 167                                                 ccp_cryptodev_driver_id);
 168
 169                         rte_mempool_put(qp->sess_mp_priv,
 170                                         sess);
 171                         rte_mempool_put(qp->sess_mp,
 172                                         ops[i]->sym->session);
 173                         ops[i]->sym->session = NULL;
 174                 }
 175         qp->qp_stats.dequeued_count += nb_dequeued;
 176
 177         return nb_dequeued;
 178 }
 179
 180 /*
 181  * The set of PCI devices this driver supports
 182  */
 183 static struct rte_pci_id ccp_pci_id[] = {
 184         {
 185                 RTE_PCI_DEVICE(0x1022, 0x1456), /* AMD CCP-5a */
 186         },
 187         {
 188                 RTE_PCI_DEVICE(0x1022, 0x1468), /* AMD CCP-5b */
 189         },
 190         {
 191                 RTE_PCI_DEVICE(0x1022, 0x15df), /* AMD CCP RV */
 192         },
 193         {.device_id = 0},
 194 };
 195
 196 /** Remove ccp pmd */
 197 static int
 198 cryptodev_ccp_remove(struct rte_pci_device *pci_dev)
 199 {
 200         char name[RTE_CRYPTODEV_NAME_MAX_LEN];
 201         struct rte_cryptodev *dev;
 202
 203         if (pci_dev == NULL)
 204                 return -EINVAL;
 205
 206         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 207
 208         if (name[0] == '\0')
 209                 return -EINVAL;
 210
 211         dev = rte_cryptodev_pmd_get_named_dev(name);
 212         if (dev == NULL)
 213                 return -ENODEV;
 214
 215         ccp_pmd_init_done = 0;
 216         rte_free(sha_ctx);
 217
 218         RTE_LOG(INFO, PMD, "Closing ccp device %s on numa socket %u\n",
 219                         name, rte_socket_id());
 220
 221         return rte_cryptodev_pmd_destroy(dev);
 222 }
 223
 224 /** Create crypto device */
 225 static int
 226 cryptodev_ccp_create(const char *name,
 227                      struct rte_pci_device *pci_dev,
 228                      struct ccp_pmd_init_params *init_params,
 229                      struct rte_pci_driver *pci_drv)
 230 {
 231         struct rte_cryptodev *dev;
 232         struct ccp_private *internals;
 233
 234         if (init_params->def_p.name[0] == '\0')
 235                 strlcpy(init_params->def_p.name, name,
 236                         sizeof(init_params->def_p.name));
 237
 238         dev = rte_cryptodev_pmd_create(init_params->def_p.name,
 239                                        &pci_dev->device,
 240                                        &init_params->def_p);
 241         if (dev == NULL) {
 242                 CCP_LOG_ERR("failed to create cryptodev vdev");
 243                 goto init_error;
 244         }
 245
 246         cryptodev_cnt = ccp_probe_devices(pci_dev, ccp_pci_id);
 247
 248         if (cryptodev_cnt == 0) {
 249                 CCP_LOG_ERR("failed to detect CCP crypto device");
 250                 goto init_error;
 251         }
 252
 253         printf("CCP : Crypto device count = %d\n", cryptodev_cnt);
 254         dev->device = &pci_dev->device;
 255         dev->device->driver = &pci_drv->driver;
 256         dev->driver_id = ccp_cryptodev_driver_id;
 257
 258         /* register rx/tx burst functions for data path */
 259         dev->dev_ops = ccp_pmd_ops;
 260         dev->enqueue_burst = ccp_pmd_enqueue_burst;
 261         dev->dequeue_burst = ccp_pmd_dequeue_burst;
 262
 263         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
 264                         RTE_CRYPTODEV_FF_HW_ACCELERATED |
 265                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
 266                         RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
 267
 268         internals = dev->data->dev_private;
 269
 270         internals->max_nb_qpairs = init_params->def_p.max_nb_queue_pairs;
 271         internals->auth_opt = init_params->auth_opt;
 272         internals->crypto_num_dev = cryptodev_cnt;
 273
 274         return 0;
 275
 276 init_error:
 277         CCP_LOG_ERR("driver %s: %s() failed",
 278                     init_params->def_p.name, __func__);
 279         cryptodev_ccp_remove(pci_dev);
 280
 281         return -EFAULT;
 282 }
 283
 284 /** Probe ccp pmd */
 285 static int
 286 cryptodev_ccp_probe(struct rte_pci_driver *pci_drv __rte_unused,
 287         struct rte_pci_device *pci_dev)
 288 {
 289         int rc = 0;
 290         char name[RTE_CRYPTODEV_NAME_MAX_LEN];
 291         struct ccp_pmd_init_params init_params = {
 292                 .def_p = {
 293                         "",
 294                         sizeof(struct ccp_private),
 295                         rte_socket_id(),
 296                         CCP_PMD_MAX_QUEUE_PAIRS
 297                 },
 298                 .auth_opt = CCP_PMD_AUTH_OPT_CCP,
 299         };
 300
 301         sha_ctx = (void *)rte_malloc(NULL, SHA512_DIGEST_SIZE, 64);
 302         if (ccp_pmd_init_done) {
 303                 RTE_LOG(INFO, PMD, "CCP PMD already initialized\n");
 304                 return -EFAULT;
 305         }
 306         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 307         if (name[0] == '\0')
 308                 return -EINVAL;
 309
 310         init_params.def_p.max_nb_queue_pairs = CCP_PMD_MAX_QUEUE_PAIRS;
 311
 312         RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name,
 313                 init_params.def_p.socket_id);
 314         RTE_LOG(INFO, PMD, "Max number of queue pairs = %d\n",
 315                 init_params.def_p.max_nb_queue_pairs);
 316         RTE_LOG(INFO, PMD, "Authentication offload to %s\n",
 317                 ((init_params.auth_opt == 0) ? "CCP" : "CPU"));
 318
 319         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 320
 321         rc = cryptodev_ccp_create(name, pci_dev, &init_params, pci_drv);
 322         if (rc)
 323                 return rc;
 324         ccp_pmd_init_done = 1;
 325         return 0;
 326 }
 327
 328 static struct rte_pci_driver cryptodev_ccp_pmd_drv = {
 329         .id_table = ccp_pci_id,
 330         .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
 331         .probe = cryptodev_ccp_probe,
 332         .remove = cryptodev_ccp_remove
 333 };
 334
 335 static struct cryptodev_driver ccp_crypto_drv;
 336
 337 RTE_PMD_REGISTER_PCI(CRYPTODEV_NAME_CCP_PMD, cryptodev_ccp_pmd_drv);
 338 RTE_PMD_REGISTER_KMOD_DEP(CRYPTODEV_NAME_CCP_PMD, "* igb_uio | uio_pci_generic | vfio-pci");
 339 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_CCP_PMD,
 340         "max_nb_queue_pairs=<int> "
 341         "socket_id=<int> "
 342         "ccp_auth_opt=<int>");
 343 RTE_PMD_REGISTER_CRYPTO_DRIVER(ccp_crypto_drv, cryptodev_ccp_pmd_drv.driver,
 344                                ccp_cryptodev_driver_id);