lib/eventdev/rte_event_crypto_adapter.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation.
   3  * All rights reserved.
   4  */
   5
   6 #include <string.h>
   7 #include <stdbool.h>
   8 #include <rte_common.h>
   9 #include <rte_dev.h>
  10 #include <rte_errno.h>
  11 #include <rte_cryptodev.h>
  12 #include <rte_cryptodev_pmd.h>
  13 #include <rte_log.h>
  14 #include <rte_malloc.h>
  15 #include <rte_service_component.h>
  16
  17 #include "rte_eventdev.h"
  18 #include "eventdev_pmd.h"
  19 #include "rte_eventdev_trace.h"
  20 #include "rte_event_crypto_adapter.h"
  21
  22 #define BATCH_SIZE 32
  23 #define DEFAULT_MAX_NB 128
  24 #define CRYPTO_ADAPTER_NAME_LEN 32
  25 #define CRYPTO_ADAPTER_MEM_NAME_LEN 32
  26 #define CRYPTO_ADAPTER_MAX_EV_ENQ_RETRIES 100
  27
  28 /* Flush an instance's enqueue buffers every CRYPTO_ENQ_FLUSH_THRESHOLD
  29  * iterations of eca_crypto_adapter_enq_run()
  30  */
  31 #define CRYPTO_ENQ_FLUSH_THRESHOLD 1024
  32
  33 struct rte_event_crypto_adapter {
  34         /* Event device identifier */
  35         uint8_t eventdev_id;
  36         /* Event port identifier */
  37         uint8_t event_port_id;
  38         /* Store event device's implicit release capability */
  39         uint8_t implicit_release_disabled;
  40         /* Max crypto ops processed in any service function invocation */
  41         uint32_t max_nb;
  42         /* Lock to serialize config updates with service function */
  43         rte_spinlock_t lock;
  44         /* Next crypto device to be processed */
  45         uint16_t next_cdev_id;
  46         /* Per crypto device structure */
  47         struct crypto_device_info *cdevs;
  48         /* Loop counter to flush crypto ops */
  49         uint16_t transmit_loop_count;
  50         /* Per instance stats structure */
  51         struct rte_event_crypto_adapter_stats crypto_stats;
  52         /* Configuration callback for rte_service configuration */
  53         rte_event_crypto_adapter_conf_cb conf_cb;
  54         /* Configuration callback argument */
  55         void *conf_arg;
  56         /* Set if  default_cb is being used */
  57         int default_cb_arg;
  58         /* Service initialization state */
  59         uint8_t service_inited;
  60         /* Memory allocation name */
  61         char mem_name[CRYPTO_ADAPTER_MEM_NAME_LEN];
  62         /* Socket identifier cached from eventdev */
  63         int socket_id;
  64         /* Per adapter EAL service */
  65         uint32_t service_id;
  66         /* No. of queue pairs configured */
  67         uint16_t nb_qps;
  68         /* Adapter mode */
  69         enum rte_event_crypto_adapter_mode mode;
  70 } __rte_cache_aligned;
  71
  72 /* Per crypto device information */
  73 struct crypto_device_info {
  74         /* Pointer to cryptodev */
  75         struct rte_cryptodev *dev;
  76         /* Pointer to queue pair info */
  77         struct crypto_queue_pair_info *qpairs;
  78         /* Next queue pair to be processed */
  79         uint16_t next_queue_pair_id;
  80         /* Set to indicate cryptodev->eventdev packet
  81          * transfer uses a hardware mechanism
  82          */
  83         uint8_t internal_event_port;
  84         /* Set to indicate processing has been started */
  85         uint8_t dev_started;
  86         /* If num_qpairs > 0, the start callback will
  87          * be invoked if not already invoked
  88          */
  89         uint16_t num_qpairs;
  90 } __rte_cache_aligned;
  91
  92 /* Per queue pair information */
  93 struct crypto_queue_pair_info {
  94         /* Set to indicate queue pair is enabled */
  95         bool qp_enabled;
  96         /* Pointer to hold rte_crypto_ops for batching */
  97         struct rte_crypto_op **op_buffer;
  98         /* No of crypto ops accumulated */
  99         uint8_t len;
 100 } __rte_cache_aligned;
 101
 102 static struct rte_event_crypto_adapter **event_crypto_adapter;
 103
 104 /* Macros to check for valid adapter */
 105 #define EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, retval) do { \
 106         if (!eca_valid_id(id)) { \
 107                 RTE_EDEV_LOG_ERR("Invalid crypto adapter id = %d\n", id); \
 108                 return retval; \
 109         } \
 110 } while (0)
 111
 112 static inline int
 113 eca_valid_id(uint8_t id)
 114 {
 115         return id < RTE_EVENT_CRYPTO_ADAPTER_MAX_INSTANCE;
 116 }
 117
 118 static int
 119 eca_init(void)
 120 {
 121         const char *name = "crypto_adapter_array";
 122         const struct rte_memzone *mz;
 123         unsigned int sz;
 124
 125         sz = sizeof(*event_crypto_adapter) *
 126             RTE_EVENT_CRYPTO_ADAPTER_MAX_INSTANCE;
 127         sz = RTE_ALIGN(sz, RTE_CACHE_LINE_SIZE);
 128
 129         mz = rte_memzone_lookup(name);
 130         if (mz == NULL) {
 131                 mz = rte_memzone_reserve_aligned(name, sz, rte_socket_id(), 0,
 132                                                  RTE_CACHE_LINE_SIZE);
 133                 if (mz == NULL) {
 134                         RTE_EDEV_LOG_ERR("failed to reserve memzone err = %"
 135                                         PRId32, rte_errno);
 136                         return -rte_errno;
 137                 }
 138         }
 139
 140         event_crypto_adapter = mz->addr;
 141         return 0;
 142 }
 143
 144 static inline struct rte_event_crypto_adapter *
 145 eca_id_to_adapter(uint8_t id)
 146 {
 147         return event_crypto_adapter ?
 148                 event_crypto_adapter[id] : NULL;
 149 }
 150
 151 static int
 152 eca_default_config_cb(uint8_t id, uint8_t dev_id,
 153                         struct rte_event_crypto_adapter_conf *conf, void *arg)
 154 {
 155         struct rte_event_dev_config dev_conf;
 156         struct rte_eventdev *dev;
 157         uint8_t port_id;
 158         int started;
 159         int ret;
 160         struct rte_event_port_conf *port_conf = arg;
 161         struct rte_event_crypto_adapter *adapter = eca_id_to_adapter(id);
 162
 163         if (adapter == NULL)
 164                 return -EINVAL;
 165
 166         dev = &rte_eventdevs[adapter->eventdev_id];
 167         dev_conf = dev->data->dev_conf;
 168
 169         started = dev->data->dev_started;
 170         if (started)
 171                 rte_event_dev_stop(dev_id);
 172         port_id = dev_conf.nb_event_ports;
 173         dev_conf.nb_event_ports += 1;
 174         ret = rte_event_dev_configure(dev_id, &dev_conf);
 175         if (ret) {
 176                 RTE_EDEV_LOG_ERR("failed to configure event dev %u\n", dev_id);
 177                 if (started) {
 178                         if (rte_event_dev_start(dev_id))
 179                                 return -EIO;
 180                 }
 181                 return ret;
 182         }
 183
 184         ret = rte_event_port_setup(dev_id, port_id, port_conf);
 185         if (ret) {
 186                 RTE_EDEV_LOG_ERR("failed to setup event port %u\n", port_id);
 187                 return ret;
 188         }
 189
 190         conf->event_port_id = port_id;
 191         conf->max_nb = DEFAULT_MAX_NB;
 192         if (started)
 193                 ret = rte_event_dev_start(dev_id);
 194
 195         adapter->default_cb_arg = 1;
 196         return ret;
 197 }
 198
 199 int
 200 rte_event_crypto_adapter_create_ext(uint8_t id, uint8_t dev_id,
 201                                 rte_event_crypto_adapter_conf_cb conf_cb,
 202                                 enum rte_event_crypto_adapter_mode mode,
 203                                 void *conf_arg)
 204 {
 205         struct rte_event_crypto_adapter *adapter;
 206         char mem_name[CRYPTO_ADAPTER_NAME_LEN];
 207         struct rte_event_dev_info dev_info;
 208         int socket_id;
 209         uint8_t i;
 210         int ret;
 211
 212         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 213         RTE_EVENTDEV_VALID_DEVID_OR_ERR_RET(dev_id, -EINVAL);
 214         if (conf_cb == NULL)
 215                 return -EINVAL;
 216
 217         if (event_crypto_adapter == NULL) {
 218                 ret = eca_init();
 219                 if (ret)
 220                         return ret;
 221         }
 222
 223         adapter = eca_id_to_adapter(id);
 224         if (adapter != NULL) {
 225                 RTE_EDEV_LOG_ERR("Crypto adapter id %u already exists!", id);
 226                 return -EEXIST;
 227         }
 228
 229         socket_id = rte_event_dev_socket_id(dev_id);
 230         snprintf(mem_name, CRYPTO_ADAPTER_MEM_NAME_LEN,
 231                  "rte_event_crypto_adapter_%d", id);
 232
 233         adapter = rte_zmalloc_socket(mem_name, sizeof(*adapter),
 234                         RTE_CACHE_LINE_SIZE, socket_id);
 235         if (adapter == NULL) {
 236                 RTE_EDEV_LOG_ERR("Failed to get mem for event crypto adapter!");
 237                 return -ENOMEM;
 238         }
 239
 240         ret = rte_event_dev_info_get(dev_id, &dev_info);
 241         if (ret < 0) {
 242                 RTE_EDEV_LOG_ERR("Failed to get info for eventdev %d: %s!",
 243                                  dev_id, dev_info.driver_name);
 244                 rte_free(adapter);
 245                 return ret;
 246         }
 247
 248         adapter->implicit_release_disabled = (dev_info.event_dev_cap &
 249                         RTE_EVENT_DEV_CAP_IMPLICIT_RELEASE_DISABLE);
 250         adapter->eventdev_id = dev_id;
 251         adapter->socket_id = socket_id;
 252         adapter->conf_cb = conf_cb;
 253         adapter->conf_arg = conf_arg;
 254         adapter->mode = mode;
 255         strcpy(adapter->mem_name, mem_name);
 256         adapter->cdevs = rte_zmalloc_socket(adapter->mem_name,
 257                                         rte_cryptodev_count() *
 258                                         sizeof(struct crypto_device_info), 0,
 259                                         socket_id);
 260         if (adapter->cdevs == NULL) {
 261                 RTE_EDEV_LOG_ERR("Failed to get mem for crypto devices\n");
 262                 rte_free(adapter);
 263                 return -ENOMEM;
 264         }
 265
 266         rte_spinlock_init(&adapter->lock);
 267         for (i = 0; i < rte_cryptodev_count(); i++)
 268                 adapter->cdevs[i].dev = rte_cryptodev_pmd_get_dev(i);
 269
 270         event_crypto_adapter[id] = adapter;
 271
 272         rte_eventdev_trace_crypto_adapter_create(id, dev_id, adapter, conf_arg,
 273                 mode);
 274         return 0;
 275 }
 276
 277
 278 int
 279 rte_event_crypto_adapter_create(uint8_t id, uint8_t dev_id,
 280                                 struct rte_event_port_conf *port_config,
 281                                 enum rte_event_crypto_adapter_mode mode)
 282 {
 283         struct rte_event_port_conf *pc;
 284         int ret;
 285
 286         if (port_config == NULL)
 287                 return -EINVAL;
 288         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 289
 290         pc = rte_malloc(NULL, sizeof(*pc), 0);
 291         if (pc == NULL)
 292                 return -ENOMEM;
 293         *pc = *port_config;
 294         ret = rte_event_crypto_adapter_create_ext(id, dev_id,
 295                                                   eca_default_config_cb,
 296                                                   mode,
 297                                                   pc);
 298         if (ret)
 299                 rte_free(pc);
 300
 301         return ret;
 302 }
 303
 304 int
 305 rte_event_crypto_adapter_free(uint8_t id)
 306 {
 307         struct rte_event_crypto_adapter *adapter;
 308
 309         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 310
 311         adapter = eca_id_to_adapter(id);
 312         if (adapter == NULL)
 313                 return -EINVAL;
 314
 315         if (adapter->nb_qps) {
 316                 RTE_EDEV_LOG_ERR("%" PRIu16 "Queue pairs not deleted",
 317                                 adapter->nb_qps);
 318                 return -EBUSY;
 319         }
 320
 321         rte_eventdev_trace_crypto_adapter_free(id, adapter);
 322         if (adapter->default_cb_arg)
 323                 rte_free(adapter->conf_arg);
 324         rte_free(adapter->cdevs);
 325         rte_free(adapter);
 326         event_crypto_adapter[id] = NULL;
 327
 328         return 0;
 329 }
 330
 331 static inline unsigned int
 332 eca_enq_to_cryptodev(struct rte_event_crypto_adapter *adapter,
 333                  struct rte_event *ev, unsigned int cnt)
 334 {
 335         struct rte_event_crypto_adapter_stats *stats = &adapter->crypto_stats;
 336         union rte_event_crypto_metadata *m_data = NULL;
 337         struct crypto_queue_pair_info *qp_info = NULL;
 338         struct rte_crypto_op *crypto_op;
 339         unsigned int i, n;
 340         uint16_t qp_id, len, ret;
 341         uint8_t cdev_id;
 342
 343         len = 0;
 344         ret = 0;
 345         n = 0;
 346         stats->event_deq_count += cnt;
 347
 348         for (i = 0; i < cnt; i++) {
 349                 crypto_op = ev[i].event_ptr;
 350                 if (crypto_op == NULL)
 351                         continue;
 352                 if (crypto_op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 353                         m_data = rte_cryptodev_sym_session_get_user_data(
 354                                         crypto_op->sym->session);
 355                         if (m_data == NULL) {
 356                                 rte_pktmbuf_free(crypto_op->sym->m_src);
 357                                 rte_crypto_op_free(crypto_op);
 358                                 continue;
 359                         }
 360
 361                         cdev_id = m_data->request_info.cdev_id;
 362                         qp_id = m_data->request_info.queue_pair_id;
 363                         qp_info = &adapter->cdevs[cdev_id].qpairs[qp_id];
 364                         if (!qp_info->qp_enabled) {
 365                                 rte_pktmbuf_free(crypto_op->sym->m_src);
 366                                 rte_crypto_op_free(crypto_op);
 367                                 continue;
 368                         }
 369                         len = qp_info->len;
 370                         qp_info->op_buffer[len] = crypto_op;
 371                         len++;
 372                 } else if (crypto_op->sess_type == RTE_CRYPTO_OP_SESSIONLESS &&
 373                                 crypto_op->private_data_offset) {
 374                         m_data = (union rte_event_crypto_metadata *)
 375                                  ((uint8_t *)crypto_op +
 376                                         crypto_op->private_data_offset);
 377                         cdev_id = m_data->request_info.cdev_id;
 378                         qp_id = m_data->request_info.queue_pair_id;
 379                         qp_info = &adapter->cdevs[cdev_id].qpairs[qp_id];
 380                         if (!qp_info->qp_enabled) {
 381                                 rte_pktmbuf_free(crypto_op->sym->m_src);
 382                                 rte_crypto_op_free(crypto_op);
 383                                 continue;
 384                         }
 385                         len = qp_info->len;
 386                         qp_info->op_buffer[len] = crypto_op;
 387                         len++;
 388                 } else {
 389                         rte_pktmbuf_free(crypto_op->sym->m_src);
 390                         rte_crypto_op_free(crypto_op);
 391                         continue;
 392                 }
 393
 394                 if (len == BATCH_SIZE) {
 395                         struct rte_crypto_op **op_buffer = qp_info->op_buffer;
 396                         ret = rte_cryptodev_enqueue_burst(cdev_id,
 397                                                           qp_id,
 398                                                           op_buffer,
 399                                                           BATCH_SIZE);
 400
 401                         stats->crypto_enq_count += ret;
 402
 403                         while (ret < len) {
 404                                 struct rte_crypto_op *op;
 405                                 op = op_buffer[ret++];
 406                                 stats->crypto_enq_fail++;
 407                                 rte_pktmbuf_free(op->sym->m_src);
 408                                 rte_crypto_op_free(op);
 409                         }
 410
 411                         len = 0;
 412                 }
 413
 414                 if (qp_info)
 415                         qp_info->len = len;
 416                 n += ret;
 417         }
 418
 419         return n;
 420 }
 421
 422 static unsigned int
 423 eca_crypto_enq_flush(struct rte_event_crypto_adapter *adapter)
 424 {
 425         struct rte_event_crypto_adapter_stats *stats = &adapter->crypto_stats;
 426         struct crypto_device_info *curr_dev;
 427         struct crypto_queue_pair_info *curr_queue;
 428         struct rte_crypto_op **op_buffer;
 429         struct rte_cryptodev *dev;
 430         uint8_t cdev_id;
 431         uint16_t qp;
 432         uint16_t ret;
 433         uint16_t num_cdev = rte_cryptodev_count();
 434
 435         ret = 0;
 436         for (cdev_id = 0; cdev_id < num_cdev; cdev_id++) {
 437                 curr_dev = &adapter->cdevs[cdev_id];
 438                 dev = curr_dev->dev;
 439                 if (dev == NULL)
 440                         continue;
 441                 for (qp = 0; qp < dev->data->nb_queue_pairs; qp++) {
 442
 443                         curr_queue = &curr_dev->qpairs[qp];
 444                         if (!curr_queue->qp_enabled)
 445                                 continue;
 446
 447                         op_buffer = curr_queue->op_buffer;
 448                         ret = rte_cryptodev_enqueue_burst(cdev_id,
 449                                                           qp,
 450                                                           op_buffer,
 451                                                           curr_queue->len);
 452                         stats->crypto_enq_count += ret;
 453
 454                         while (ret < curr_queue->len) {
 455                                 struct rte_crypto_op *op;
 456                                 op = op_buffer[ret++];
 457                                 stats->crypto_enq_fail++;
 458                                 rte_pktmbuf_free(op->sym->m_src);
 459                                 rte_crypto_op_free(op);
 460                         }
 461                         curr_queue->len = 0;
 462                 }
 463         }
 464
 465         return ret;
 466 }
 467
 468 static int
 469 eca_crypto_adapter_enq_run(struct rte_event_crypto_adapter *adapter,
 470                         unsigned int max_enq)
 471 {
 472         struct rte_event_crypto_adapter_stats *stats = &adapter->crypto_stats;
 473         struct rte_event ev[BATCH_SIZE];
 474         unsigned int nb_enq, nb_enqueued;
 475         uint16_t n;
 476         uint8_t event_dev_id = adapter->eventdev_id;
 477         uint8_t event_port_id = adapter->event_port_id;
 478
 479         nb_enqueued = 0;
 480         if (adapter->mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW)
 481                 return 0;
 482
 483         for (nb_enq = 0; nb_enq < max_enq; nb_enq += n) {
 484                 stats->event_poll_count++;
 485                 n = rte_event_dequeue_burst(event_dev_id,
 486                                             event_port_id, ev, BATCH_SIZE, 0);
 487
 488                 if (!n)
 489                         break;
 490
 491                 nb_enqueued += eca_enq_to_cryptodev(adapter, ev, n);
 492         }
 493
 494         if ((++adapter->transmit_loop_count &
 495                 (CRYPTO_ENQ_FLUSH_THRESHOLD - 1)) == 0) {
 496                 nb_enqueued += eca_crypto_enq_flush(adapter);
 497         }
 498
 499         return nb_enqueued;
 500 }
 501
 502 static inline void
 503 eca_ops_enqueue_burst(struct rte_event_crypto_adapter *adapter,
 504                   struct rte_crypto_op **ops, uint16_t num)
 505 {
 506         struct rte_event_crypto_adapter_stats *stats = &adapter->crypto_stats;
 507         union rte_event_crypto_metadata *m_data = NULL;
 508         uint8_t event_dev_id = adapter->eventdev_id;
 509         uint8_t event_port_id = adapter->event_port_id;
 510         struct rte_event events[BATCH_SIZE];
 511         uint16_t nb_enqueued, nb_ev;
 512         uint8_t retry;
 513         uint8_t i;
 514
 515         nb_ev = 0;
 516         retry = 0;
 517         nb_enqueued = 0;
 518         num = RTE_MIN(num, BATCH_SIZE);
 519         for (i = 0; i < num; i++) {
 520                 struct rte_event *ev = &events[nb_ev++];
 521                 if (ops[i]->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 522                         m_data = rte_cryptodev_sym_session_get_user_data(
 523                                         ops[i]->sym->session);
 524                 } else if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS &&
 525                                 ops[i]->private_data_offset) {
 526                         m_data = (union rte_event_crypto_metadata *)
 527                                  ((uint8_t *)ops[i] +
 528                                   ops[i]->private_data_offset);
 529                 }
 530
 531                 if (unlikely(m_data == NULL)) {
 532                         rte_pktmbuf_free(ops[i]->sym->m_src);
 533                         rte_crypto_op_free(ops[i]);
 534                         continue;
 535                 }
 536
 537                 rte_memcpy(ev, &m_data->response_info, sizeof(*ev));
 538                 ev->event_ptr = ops[i];
 539                 ev->event_type = RTE_EVENT_TYPE_CRYPTODEV;
 540                 if (adapter->implicit_release_disabled)
 541                         ev->op = RTE_EVENT_OP_FORWARD;
 542                 else
 543                         ev->op = RTE_EVENT_OP_NEW;
 544         }
 545
 546         do {
 547                 nb_enqueued += rte_event_enqueue_burst(event_dev_id,
 548                                                   event_port_id,
 549                                                   &events[nb_enqueued],
 550                                                   nb_ev - nb_enqueued);
 551         } while (retry++ < CRYPTO_ADAPTER_MAX_EV_ENQ_RETRIES &&
 552                  nb_enqueued < nb_ev);
 553
 554         /* Free mbufs and rte_crypto_ops for failed events */
 555         for (i = nb_enqueued; i < nb_ev; i++) {
 556                 struct rte_crypto_op *op = events[i].event_ptr;
 557                 rte_pktmbuf_free(op->sym->m_src);
 558                 rte_crypto_op_free(op);
 559         }
 560
 561         stats->event_enq_fail_count += nb_ev - nb_enqueued;
 562         stats->event_enq_count += nb_enqueued;
 563         stats->event_enq_retry_count += retry - 1;
 564 }
 565
 566 static inline unsigned int
 567 eca_crypto_adapter_deq_run(struct rte_event_crypto_adapter *adapter,
 568                         unsigned int max_deq)
 569 {
 570         struct rte_event_crypto_adapter_stats *stats = &adapter->crypto_stats;
 571         struct crypto_device_info *curr_dev;
 572         struct crypto_queue_pair_info *curr_queue;
 573         struct rte_crypto_op *ops[BATCH_SIZE];
 574         uint16_t n, nb_deq;
 575         struct rte_cryptodev *dev;
 576         uint8_t cdev_id;
 577         uint16_t qp, dev_qps;
 578         bool done;
 579         uint16_t num_cdev = rte_cryptodev_count();
 580
 581         nb_deq = 0;
 582         do {
 583                 uint16_t queues = 0;
 584                 done = true;
 585
 586                 for (cdev_id = adapter->next_cdev_id;
 587                         cdev_id < num_cdev; cdev_id++) {
 588                         curr_dev = &adapter->cdevs[cdev_id];
 589                         dev = curr_dev->dev;
 590                         if (dev == NULL)
 591                                 continue;
 592                         dev_qps = dev->data->nb_queue_pairs;
 593
 594                         for (qp = curr_dev->next_queue_pair_id;
 595                                 queues < dev_qps; qp = (qp + 1) % dev_qps,
 596                                 queues++) {
 597
 598                                 curr_queue = &curr_dev->qpairs[qp];
 599                                 if (!curr_queue->qp_enabled)
 600                                         continue;
 601
 602                                 n = rte_cryptodev_dequeue_burst(cdev_id, qp,
 603                                         ops, BATCH_SIZE);
 604                                 if (!n)
 605                                         continue;
 606
 607                                 done = false;
 608                                 stats->crypto_deq_count += n;
 609                                 eca_ops_enqueue_burst(adapter, ops, n);
 610                                 nb_deq += n;
 611
 612                                 if (nb_deq > max_deq) {
 613                                         if ((qp + 1) == dev_qps) {
 614                                                 adapter->next_cdev_id =
 615                                                         (cdev_id + 1)
 616                                                         % num_cdev;
 617                                         }
 618                                         curr_dev->next_queue_pair_id = (qp + 1)
 619                                                 % dev->data->nb_queue_pairs;
 620
 621                                         return nb_deq;
 622                                 }
 623                         }
 624                 }
 625         } while (done == false);
 626         return nb_deq;
 627 }
 628
 629 static void
 630 eca_crypto_adapter_run(struct rte_event_crypto_adapter *adapter,
 631                         unsigned int max_ops)
 632 {
 633         while (max_ops) {
 634                 unsigned int e_cnt, d_cnt;
 635
 636                 e_cnt = eca_crypto_adapter_deq_run(adapter, max_ops);
 637                 max_ops -= RTE_MIN(max_ops, e_cnt);
 638
 639                 d_cnt = eca_crypto_adapter_enq_run(adapter, max_ops);
 640                 max_ops -= RTE_MIN(max_ops, d_cnt);
 641
 642                 if (e_cnt == 0 && d_cnt == 0)
 643                         break;
 644
 645         }
 646 }
 647
 648 static int
 649 eca_service_func(void *args)
 650 {
 651         struct rte_event_crypto_adapter *adapter = args;
 652
 653         if (rte_spinlock_trylock(&adapter->lock) == 0)
 654                 return 0;
 655         eca_crypto_adapter_run(adapter, adapter->max_nb);
 656         rte_spinlock_unlock(&adapter->lock);
 657
 658         return 0;
 659 }
 660
 661 static int
 662 eca_init_service(struct rte_event_crypto_adapter *adapter, uint8_t id)
 663 {
 664         struct rte_event_crypto_adapter_conf adapter_conf;
 665         struct rte_service_spec service;
 666         int ret;
 667
 668         if (adapter->service_inited)
 669                 return 0;
 670
 671         memset(&service, 0, sizeof(service));
 672         snprintf(service.name, CRYPTO_ADAPTER_NAME_LEN,
 673                 "rte_event_crypto_adapter_%d", id);
 674         service.socket_id = adapter->socket_id;
 675         service.callback = eca_service_func;
 676         service.callback_userdata = adapter;
 677         /* Service function handles locking for queue add/del updates */
 678         service.capabilities = RTE_SERVICE_CAP_MT_SAFE;
 679         ret = rte_service_component_register(&service, &adapter->service_id);
 680         if (ret) {
 681                 RTE_EDEV_LOG_ERR("failed to register service %s err = %" PRId32,
 682                         service.name, ret);
 683                 return ret;
 684         }
 685
 686         ret = adapter->conf_cb(id, adapter->eventdev_id,
 687                 &adapter_conf, adapter->conf_arg);
 688         if (ret) {
 689                 RTE_EDEV_LOG_ERR("configuration callback failed err = %" PRId32,
 690                         ret);
 691                 return ret;
 692         }
 693
 694         adapter->max_nb = adapter_conf.max_nb;
 695         adapter->event_port_id = adapter_conf.event_port_id;
 696         adapter->service_inited = 1;
 697
 698         return ret;
 699 }
 700
 701 static void
 702 eca_update_qp_info(struct rte_event_crypto_adapter *adapter,
 703                         struct crypto_device_info *dev_info,
 704                         int32_t queue_pair_id,
 705                         uint8_t add)
 706 {
 707         struct crypto_queue_pair_info *qp_info;
 708         int enabled;
 709         uint16_t i;
 710
 711         if (dev_info->qpairs == NULL)
 712                 return;
 713
 714         if (queue_pair_id == -1) {
 715                 for (i = 0; i < dev_info->dev->data->nb_queue_pairs; i++)
 716                         eca_update_qp_info(adapter, dev_info, i, add);
 717         } else {
 718                 qp_info = &dev_info->qpairs[queue_pair_id];
 719                 enabled = qp_info->qp_enabled;
 720                 if (add) {
 721                         adapter->nb_qps += !enabled;
 722                         dev_info->num_qpairs += !enabled;
 723                 } else {
 724                         adapter->nb_qps -= enabled;
 725                         dev_info->num_qpairs -= enabled;
 726                 }
 727                 qp_info->qp_enabled = !!add;
 728         }
 729 }
 730
 731 static int
 732 eca_add_queue_pair(struct rte_event_crypto_adapter *adapter,
 733                 uint8_t cdev_id,
 734                 int queue_pair_id)
 735 {
 736         struct crypto_device_info *dev_info = &adapter->cdevs[cdev_id];
 737         struct crypto_queue_pair_info *qpairs;
 738         uint32_t i;
 739
 740         if (dev_info->qpairs == NULL) {
 741                 dev_info->qpairs =
 742                     rte_zmalloc_socket(adapter->mem_name,
 743                                         dev_info->dev->data->nb_queue_pairs *
 744                                         sizeof(struct crypto_queue_pair_info),
 745                                         0, adapter->socket_id);
 746                 if (dev_info->qpairs == NULL)
 747                         return -ENOMEM;
 748
 749                 qpairs = dev_info->qpairs;
 750                 qpairs->op_buffer = rte_zmalloc_socket(adapter->mem_name,
 751                                         BATCH_SIZE *
 752                                         sizeof(struct rte_crypto_op *),
 753                                         0, adapter->socket_id);
 754                 if (!qpairs->op_buffer) {
 755                         rte_free(qpairs);
 756                         return -ENOMEM;
 757                 }
 758         }
 759
 760         if (queue_pair_id == -1) {
 761                 for (i = 0; i < dev_info->dev->data->nb_queue_pairs; i++)
 762                         eca_update_qp_info(adapter, dev_info, i, 1);
 763         } else
 764                 eca_update_qp_info(adapter, dev_info,
 765                                         (uint16_t)queue_pair_id, 1);
 766
 767         return 0;
 768 }
 769
 770 int
 771 rte_event_crypto_adapter_queue_pair_add(uint8_t id,
 772                         uint8_t cdev_id,
 773                         int32_t queue_pair_id,
 774                         const struct rte_event *event)
 775 {
 776         struct rte_event_crypto_adapter *adapter;
 777         struct rte_eventdev *dev;
 778         struct crypto_device_info *dev_info;
 779         uint32_t cap;
 780         int ret;
 781
 782         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 783
 784         if (!rte_cryptodev_pmd_is_valid_dev(cdev_id)) {
 785                 RTE_EDEV_LOG_ERR("Invalid dev_id=%" PRIu8, cdev_id);
 786                 return -EINVAL;
 787         }
 788
 789         adapter = eca_id_to_adapter(id);
 790         if (adapter == NULL)
 791                 return -EINVAL;
 792
 793         dev = &rte_eventdevs[adapter->eventdev_id];
 794         ret = rte_event_crypto_adapter_caps_get(adapter->eventdev_id,
 795                                                 cdev_id,
 796                                                 &cap);
 797         if (ret) {
 798                 RTE_EDEV_LOG_ERR("Failed to get adapter caps dev %" PRIu8
 799                         " cdev %" PRIu8, id, cdev_id);
 800                 return ret;
 801         }
 802
 803         if ((cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND) &&
 804             (event == NULL)) {
 805                 RTE_EDEV_LOG_ERR("Conf value can not be NULL for dev_id=%u",
 806                                   cdev_id);
 807                 return -EINVAL;
 808         }
 809
 810         dev_info = &adapter->cdevs[cdev_id];
 811
 812         if (queue_pair_id != -1 &&
 813             (uint16_t)queue_pair_id >= dev_info->dev->data->nb_queue_pairs) {
 814                 RTE_EDEV_LOG_ERR("Invalid queue_pair_id %" PRIu16,
 815                                  (uint16_t)queue_pair_id);
 816                 return -EINVAL;
 817         }
 818
 819         /* In case HW cap is RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD,
 820          * no need of service core as HW supports event forward capability.
 821          */
 822         if ((cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD) ||
 823             (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND &&
 824              adapter->mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW) ||
 825             (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW &&
 826              adapter->mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW)) {
 827                 RTE_FUNC_PTR_OR_ERR_RET(
 828                         *dev->dev_ops->crypto_adapter_queue_pair_add,
 829                         -ENOTSUP);
 830                 if (dev_info->qpairs == NULL) {
 831                         dev_info->qpairs =
 832                             rte_zmalloc_socket(adapter->mem_name,
 833                                         dev_info->dev->data->nb_queue_pairs *
 834                                         sizeof(struct crypto_queue_pair_info),
 835                                         0, adapter->socket_id);
 836                         if (dev_info->qpairs == NULL)
 837                                 return -ENOMEM;
 838                 }
 839
 840                 ret = (*dev->dev_ops->crypto_adapter_queue_pair_add)(dev,
 841                                 dev_info->dev,
 842                                 queue_pair_id,
 843                                 event);
 844                 if (ret)
 845                         return ret;
 846
 847                 else
 848                         eca_update_qp_info(adapter, &adapter->cdevs[cdev_id],
 849                                            queue_pair_id, 1);
 850         }
 851
 852         /* In case HW cap is RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW,
 853          * or SW adapter, initiate services so the application can choose
 854          * which ever way it wants to use the adapter.
 855          * Case 1: RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW
 856          *         Application may wants to use one of below two mode
 857          *          a. OP_FORWARD mode -> HW Dequeue + SW enqueue
 858          *          b. OP_NEW mode -> HW Dequeue
 859          * Case 2: No HW caps, use SW adapter
 860          *          a. OP_FORWARD mode -> SW enqueue & dequeue
 861          *          b. OP_NEW mode -> SW Dequeue
 862          */
 863         if ((cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW &&
 864              !(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD) &&
 865              adapter->mode == RTE_EVENT_CRYPTO_ADAPTER_OP_FORWARD) ||
 866              (!(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW) &&
 867               !(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD) &&
 868               !(cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_QP_EV_BIND) &&
 869                (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_SESSION_PRIVATE_DATA))) {
 870                 rte_spinlock_lock(&adapter->lock);
 871                 ret = eca_init_service(adapter, id);
 872                 if (ret == 0)
 873                         ret = eca_add_queue_pair(adapter, cdev_id,
 874                                                  queue_pair_id);
 875                 rte_spinlock_unlock(&adapter->lock);
 876
 877                 if (ret)
 878                         return ret;
 879
 880                 rte_service_component_runstate_set(adapter->service_id, 1);
 881         }
 882
 883         rte_eventdev_trace_crypto_adapter_queue_pair_add(id, cdev_id, event,
 884                 queue_pair_id);
 885         return 0;
 886 }
 887
 888 int
 889 rte_event_crypto_adapter_queue_pair_del(uint8_t id, uint8_t cdev_id,
 890                                         int32_t queue_pair_id)
 891 {
 892         struct rte_event_crypto_adapter *adapter;
 893         struct crypto_device_info *dev_info;
 894         struct rte_eventdev *dev;
 895         int ret;
 896         uint32_t cap;
 897         uint16_t i;
 898
 899         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 900
 901         if (!rte_cryptodev_pmd_is_valid_dev(cdev_id)) {
 902                 RTE_EDEV_LOG_ERR("Invalid dev_id=%" PRIu8, cdev_id);
 903                 return -EINVAL;
 904         }
 905
 906         adapter = eca_id_to_adapter(id);
 907         if (adapter == NULL)
 908                 return -EINVAL;
 909
 910         dev = &rte_eventdevs[adapter->eventdev_id];
 911         ret = rte_event_crypto_adapter_caps_get(adapter->eventdev_id,
 912                                                 cdev_id,
 913                                                 &cap);
 914         if (ret)
 915                 return ret;
 916
 917         dev_info = &adapter->cdevs[cdev_id];
 918
 919         if (queue_pair_id != -1 &&
 920             (uint16_t)queue_pair_id >= dev_info->dev->data->nb_queue_pairs) {
 921                 RTE_EDEV_LOG_ERR("Invalid queue_pair_id %" PRIu16,
 922                                  (uint16_t)queue_pair_id);
 923                 return -EINVAL;
 924         }
 925
 926         if ((cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_FWD) ||
 927             (cap & RTE_EVENT_CRYPTO_ADAPTER_CAP_INTERNAL_PORT_OP_NEW &&
 928              adapter->mode == RTE_EVENT_CRYPTO_ADAPTER_OP_NEW)) {
 929                 RTE_FUNC_PTR_OR_ERR_RET(
 930                         *dev->dev_ops->crypto_adapter_queue_pair_del,
 931                         -ENOTSUP);
 932                 ret = (*dev->dev_ops->crypto_adapter_queue_pair_del)(dev,
 933                                                 dev_info->dev,
 934                                                 queue_pair_id);
 935                 if (ret == 0) {
 936                         eca_update_qp_info(adapter,
 937                                         &adapter->cdevs[cdev_id],
 938                                         queue_pair_id,
 939                                         0);
 940                         if (dev_info->num_qpairs == 0) {
 941                                 rte_free(dev_info->qpairs);
 942                                 dev_info->qpairs = NULL;
 943                         }
 944                 }
 945         } else {
 946                 if (adapter->nb_qps == 0)
 947                         return 0;
 948
 949                 rte_spinlock_lock(&adapter->lock);
 950                 if (queue_pair_id == -1) {
 951                         for (i = 0; i < dev_info->dev->data->nb_queue_pairs;
 952                                 i++)
 953                                 eca_update_qp_info(adapter, dev_info,
 954                                                         queue_pair_id, 0);
 955                 } else {
 956                         eca_update_qp_info(adapter, dev_info,
 957                                                 (uint16_t)queue_pair_id, 0);
 958                 }
 959
 960                 if (dev_info->num_qpairs == 0) {
 961                         rte_free(dev_info->qpairs);
 962                         dev_info->qpairs = NULL;
 963                 }
 964
 965                 rte_spinlock_unlock(&adapter->lock);
 966                 rte_service_component_runstate_set(adapter->service_id,
 967                                 adapter->nb_qps);
 968         }
 969
 970         rte_eventdev_trace_crypto_adapter_queue_pair_del(id, cdev_id,
 971                 queue_pair_id, ret);
 972         return ret;
 973 }
 974
 975 static int
 976 eca_adapter_ctrl(uint8_t id, int start)
 977 {
 978         struct rte_event_crypto_adapter *adapter;
 979         struct crypto_device_info *dev_info;
 980         struct rte_eventdev *dev;
 981         uint32_t i;
 982         int use_service;
 983         int stop = !start;
 984
 985         use_service = 0;
 986         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
 987         adapter = eca_id_to_adapter(id);
 988         if (adapter == NULL)
 989                 return -EINVAL;
 990
 991         dev = &rte_eventdevs[adapter->eventdev_id];
 992
 993         for (i = 0; i < rte_cryptodev_count(); i++) {
 994                 dev_info = &adapter->cdevs[i];
 995                 /* if start  check for num queue pairs */
 996                 if (start && !dev_info->num_qpairs)
 997                         continue;
 998                 /* if stop check if dev has been started */
 999                 if (stop && !dev_info->dev_started)
1000                         continue;
1001                 use_service |= !dev_info->internal_event_port;
1002                 dev_info->dev_started = start;
1003                 if (dev_info->internal_event_port == 0)
1004                         continue;
1005                 start ? (*dev->dev_ops->crypto_adapter_start)(dev,
1006                                                 &dev_info->dev[i]) :
1007                         (*dev->dev_ops->crypto_adapter_stop)(dev,
1008                                                 &dev_info->dev[i]);
1009         }
1010
1011         if (use_service)
1012                 rte_service_runstate_set(adapter->service_id, start);
1013
1014         return 0;
1015 }
1016
1017 int
1018 rte_event_crypto_adapter_start(uint8_t id)
1019 {
1020         struct rte_event_crypto_adapter *adapter;
1021
1022         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
1023         adapter = eca_id_to_adapter(id);
1024         if (adapter == NULL)
1025                 return -EINVAL;
1026
1027         rte_eventdev_trace_crypto_adapter_start(id, adapter);
1028         return eca_adapter_ctrl(id, 1);
1029 }
1030
1031 int
1032 rte_event_crypto_adapter_stop(uint8_t id)
1033 {
1034         rte_eventdev_trace_crypto_adapter_stop(id);
1035         return eca_adapter_ctrl(id, 0);
1036 }
1037
1038 int
1039 rte_event_crypto_adapter_stats_get(uint8_t id,
1040                                 struct rte_event_crypto_adapter_stats *stats)
1041 {
1042         struct rte_event_crypto_adapter *adapter;
1043         struct rte_event_crypto_adapter_stats dev_stats_sum = { 0 };
1044         struct rte_event_crypto_adapter_stats dev_stats;
1045         struct rte_eventdev *dev;
1046         struct crypto_device_info *dev_info;
1047         uint32_t i;
1048         int ret;
1049
1050         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
1051
1052         adapter = eca_id_to_adapter(id);
1053         if (adapter == NULL || stats == NULL)
1054                 return -EINVAL;
1055
1056         dev = &rte_eventdevs[adapter->eventdev_id];
1057         memset(stats, 0, sizeof(*stats));
1058         for (i = 0; i < rte_cryptodev_count(); i++) {
1059                 dev_info = &adapter->cdevs[i];
1060                 if (dev_info->internal_event_port == 0 ||
1061                         dev->dev_ops->crypto_adapter_stats_get == NULL)
1062                         continue;
1063                 ret = (*dev->dev_ops->crypto_adapter_stats_get)(dev,
1064                                                 dev_info->dev,
1065                                                 &dev_stats);
1066                 if (ret)
1067                         continue;
1068
1069                 dev_stats_sum.crypto_deq_count += dev_stats.crypto_deq_count;
1070                 dev_stats_sum.event_enq_count +=
1071                         dev_stats.event_enq_count;
1072         }
1073
1074         if (adapter->service_inited)
1075                 *stats = adapter->crypto_stats;
1076
1077         stats->crypto_deq_count += dev_stats_sum.crypto_deq_count;
1078         stats->event_enq_count += dev_stats_sum.event_enq_count;
1079
1080         return 0;
1081 }
1082
1083 int
1084 rte_event_crypto_adapter_stats_reset(uint8_t id)
1085 {
1086         struct rte_event_crypto_adapter *adapter;
1087         struct crypto_device_info *dev_info;
1088         struct rte_eventdev *dev;
1089         uint32_t i;
1090
1091         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
1092
1093         adapter = eca_id_to_adapter(id);
1094         if (adapter == NULL)
1095                 return -EINVAL;
1096
1097         dev = &rte_eventdevs[adapter->eventdev_id];
1098         for (i = 0; i < rte_cryptodev_count(); i++) {
1099                 dev_info = &adapter->cdevs[i];
1100                 if (dev_info->internal_event_port == 0 ||
1101                         dev->dev_ops->crypto_adapter_stats_reset == NULL)
1102                         continue;
1103                 (*dev->dev_ops->crypto_adapter_stats_reset)(dev,
1104                                                 dev_info->dev);
1105         }
1106
1107         memset(&adapter->crypto_stats, 0, sizeof(adapter->crypto_stats));
1108         return 0;
1109 }
1110
1111 int
1112 rte_event_crypto_adapter_service_id_get(uint8_t id, uint32_t *service_id)
1113 {
1114         struct rte_event_crypto_adapter *adapter;
1115
1116         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
1117
1118         adapter = eca_id_to_adapter(id);
1119         if (adapter == NULL || service_id == NULL)
1120                 return -EINVAL;
1121
1122         if (adapter->service_inited)
1123                 *service_id = adapter->service_id;
1124
1125         return adapter->service_inited ? 0 : -ESRCH;
1126 }
1127
1128 int
1129 rte_event_crypto_adapter_event_port_get(uint8_t id, uint8_t *event_port_id)
1130 {
1131         struct rte_event_crypto_adapter *adapter;
1132
1133         EVENT_CRYPTO_ADAPTER_ID_VALID_OR_ERR_RET(id, -EINVAL);
1134
1135         adapter = eca_id_to_adapter(id);
1136         if (adapter == NULL || event_port_id == NULL)
1137                 return -EINVAL;
1138
1139         *event_port_id = adapter->event_port_id;
1140
1141         return 0;
1142 }