drivers/crypto/ipsec_mb/pmd_aesni_gcm.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016-2021 Intel Corporation
   3  */
   4
   5 #include "pmd_aesni_gcm_priv.h"
   6
   7 static void
   8 aesni_gcm_set_ops(struct aesni_gcm_ops *ops, IMB_MGR *mb_mgr)
   9 {
  10         /* Set 128 bit function pointers. */
  11         ops[GCM_KEY_128].pre = mb_mgr->gcm128_pre;
  12         ops[GCM_KEY_128].init = mb_mgr->gcm128_init;
  13
  14         ops[GCM_KEY_128].enc = mb_mgr->gcm128_enc;
  15         ops[GCM_KEY_128].update_enc = mb_mgr->gcm128_enc_update;
  16         ops[GCM_KEY_128].finalize_enc = mb_mgr->gcm128_enc_finalize;
  17
  18         ops[GCM_KEY_128].dec = mb_mgr->gcm128_dec;
  19         ops[GCM_KEY_128].update_dec = mb_mgr->gcm128_dec_update;
  20         ops[GCM_KEY_128].finalize_dec = mb_mgr->gcm128_dec_finalize;
  21
  22         ops[GCM_KEY_128].gmac_init = mb_mgr->gmac128_init;
  23         ops[GCM_KEY_128].gmac_update = mb_mgr->gmac128_update;
  24         ops[GCM_KEY_128].gmac_finalize = mb_mgr->gmac128_finalize;
  25
  26         /* Set 192 bit function pointers. */
  27         ops[GCM_KEY_192].pre = mb_mgr->gcm192_pre;
  28         ops[GCM_KEY_192].init = mb_mgr->gcm192_init;
  29
  30         ops[GCM_KEY_192].enc = mb_mgr->gcm192_enc;
  31         ops[GCM_KEY_192].update_enc = mb_mgr->gcm192_enc_update;
  32         ops[GCM_KEY_192].finalize_enc = mb_mgr->gcm192_enc_finalize;
  33
  34         ops[GCM_KEY_192].dec = mb_mgr->gcm192_dec;
  35         ops[GCM_KEY_192].update_dec = mb_mgr->gcm192_dec_update;
  36         ops[GCM_KEY_192].finalize_dec = mb_mgr->gcm192_dec_finalize;
  37
  38         ops[GCM_KEY_192].gmac_init = mb_mgr->gmac192_init;
  39         ops[GCM_KEY_192].gmac_update = mb_mgr->gmac192_update;
  40         ops[GCM_KEY_192].gmac_finalize = mb_mgr->gmac192_finalize;
  41
  42         /* Set 256 bit function pointers. */
  43         ops[GCM_KEY_256].pre = mb_mgr->gcm256_pre;
  44         ops[GCM_KEY_256].init = mb_mgr->gcm256_init;
  45
  46         ops[GCM_KEY_256].enc = mb_mgr->gcm256_enc;
  47         ops[GCM_KEY_256].update_enc = mb_mgr->gcm256_enc_update;
  48         ops[GCM_KEY_256].finalize_enc = mb_mgr->gcm256_enc_finalize;
  49
  50         ops[GCM_KEY_256].dec = mb_mgr->gcm256_dec;
  51         ops[GCM_KEY_256].update_dec = mb_mgr->gcm256_dec_update;
  52         ops[GCM_KEY_256].finalize_dec = mb_mgr->gcm256_dec_finalize;
  53
  54         ops[GCM_KEY_256].gmac_init = mb_mgr->gmac256_init;
  55         ops[GCM_KEY_256].gmac_update = mb_mgr->gmac256_update;
  56         ops[GCM_KEY_256].gmac_finalize = mb_mgr->gmac256_finalize;
  57 }
  58
  59 static int
  60 aesni_gcm_session_configure(IMB_MGR *mb_mgr, void *session,
  61                             const struct rte_crypto_sym_xform *xform)
  62 {
  63         struct aesni_gcm_session *sess = session;
  64         const struct rte_crypto_sym_xform *auth_xform;
  65         const struct rte_crypto_sym_xform *cipher_xform;
  66         const struct rte_crypto_sym_xform *aead_xform;
  67
  68         uint8_t key_length;
  69         const uint8_t *key;
  70         enum ipsec_mb_operation mode;
  71         int ret = 0;
  72
  73         ret = ipsec_mb_parse_xform(xform, &mode, &auth_xform,
  74                                 &cipher_xform, &aead_xform);
  75         if (ret)
  76                 return ret;
  77
  78         /**< GCM key type */
  79
  80         sess->op = mode;
  81
  82         switch (sess->op) {
  83         case IPSEC_MB_OP_HASH_GEN_ONLY:
  84         case IPSEC_MB_OP_HASH_VERIFY_ONLY:
  85                 /* AES-GMAC
  86                  * auth_xform = xform;
  87                  */
  88                 if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_AES_GMAC) {
  89                         IPSEC_MB_LOG(ERR,
  90         "Only AES GMAC is supported as an authentication only algorithm");
  91                         ret = -ENOTSUP;
  92                         goto error_exit;
  93                 }
  94                 /* Set IV parameters */
  95                 sess->iv.offset = auth_xform->auth.iv.offset;
  96                 sess->iv.length = auth_xform->auth.iv.length;
  97                 key_length = auth_xform->auth.key.length;
  98                 key = auth_xform->auth.key.data;
  99                 sess->req_digest_length = auth_xform->auth.digest_length;
 100                 break;
 101         case IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT:
 102         case IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT:
 103                 /* AES-GCM
 104                  * aead_xform = xform;
 105                  */
 106
 107                 if (aead_xform->aead.algo != RTE_CRYPTO_AEAD_AES_GCM) {
 108                         IPSEC_MB_LOG(ERR,
 109                         "The only combined operation supported is AES GCM");
 110                         ret = -ENOTSUP;
 111                         goto error_exit;
 112                 }
 113                 /* Set IV parameters */
 114                 sess->iv.offset = aead_xform->aead.iv.offset;
 115                 sess->iv.length = aead_xform->aead.iv.length;
 116                 key_length = aead_xform->aead.key.length;
 117                 key = aead_xform->aead.key.data;
 118                 sess->aad_length = aead_xform->aead.aad_length;
 119                 sess->req_digest_length = aead_xform->aead.digest_length;
 120                 break;
 121         default:
 122                 IPSEC_MB_LOG(
 123                     ERR, "Wrong xform type, has to be AEAD or authentication");
 124                 ret = -ENOTSUP;
 125                 goto error_exit;
 126         }
 127
 128         /* Check key length, and calculate GCM pre-compute. */
 129         switch (key_length) {
 130         case 16:
 131                 sess->key_length = GCM_KEY_128;
 132                 mb_mgr->gcm128_pre(key, &sess->gdata_key);
 133                 break;
 134         case 24:
 135                 sess->key_length = GCM_KEY_192;
 136                 mb_mgr->gcm192_pre(key, &sess->gdata_key);
 137                 break;
 138         case 32:
 139                 sess->key_length = GCM_KEY_256;
 140                 mb_mgr->gcm256_pre(key, &sess->gdata_key);
 141                 break;
 142         default:
 143                 IPSEC_MB_LOG(ERR, "Invalid key length");
 144                 ret = -EINVAL;
 145                 goto error_exit;
 146         }
 147
 148         /* Digest check */
 149         if (sess->req_digest_length > 16) {
 150                 IPSEC_MB_LOG(ERR, "Invalid digest length");
 151                 ret = -EINVAL;
 152                 goto error_exit;
 153         }
 154         /*
 155          * If size requested is different, generate the full digest
 156          * (16 bytes) in a temporary location and then memcpy
 157          * the requested number of bytes.
 158          */
 159         if (sess->req_digest_length < 4)
 160                 sess->gen_digest_length = 16;
 161         else
 162                 sess->gen_digest_length = sess->req_digest_length;
 163
 164 error_exit:
 165         return ret;
 166 }
 167
 168 /**
 169  * Process a completed job and return rte_mbuf which job processed
 170  *
 171  * @param job   IMB_JOB job to process
 172  *
 173  * @return
 174  * - Returns processed mbuf which is trimmed of output digest used in
 175  * verification of supplied digest in the case of a HASH_CIPHER operation
 176  * - Returns NULL on invalid job
 177  */
 178 static void
 179 post_process_gcm_crypto_op(struct ipsec_mb_qp *qp,
 180                 struct rte_crypto_op *op,
 181                 struct aesni_gcm_session *session)
 182 {
 183         struct aesni_gcm_qp_data *qp_data = ipsec_mb_get_qp_private_data(qp);
 184
 185         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 186         /* Verify digest if required */
 187         if (session->op == IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT ||
 188                         session->op == IPSEC_MB_OP_HASH_VERIFY_ONLY) {
 189                 uint8_t *digest;
 190
 191                 uint8_t *tag = qp_data->temp_digest;
 192
 193                 if (session->op == IPSEC_MB_OP_HASH_VERIFY_ONLY)
 194                         digest = op->sym->auth.digest.data;
 195                 else
 196                         digest = op->sym->aead.digest.data;
 197
 198 #ifdef RTE_LIBRTE_PMD_AESNI_GCM_DEBUG
 199                 rte_hexdump(stdout, "auth tag (orig):",
 200                                 digest, session->req_digest_length);
 201                 rte_hexdump(stdout, "auth tag (calc):",
 202                                 tag, session->req_digest_length);
 203 #endif
 204
 205                 if (memcmp(tag, digest, session->req_digest_length) != 0)
 206                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 207         } else {
 208                 if (session->req_digest_length != session->gen_digest_length) {
 209                         if (session->op ==
 210                                 IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT)
 211                                 memcpy(op->sym->aead.digest.data,
 212                                         qp_data->temp_digest,
 213                                         session->req_digest_length);
 214                         else
 215                                 memcpy(op->sym->auth.digest.data,
 216                                         qp_data->temp_digest,
 217                                         session->req_digest_length);
 218                 }
 219         }
 220 }
 221
 222 /**
 223  * Process a completed GCM request
 224  *
 225  * @param qp            Queue Pair to process
 226  * @param op            Crypto operation
 227  * @param sess          AESNI-GCM session
 228  *
 229  */
 230 static void
 231 handle_completed_gcm_crypto_op(struct ipsec_mb_qp *qp,
 232                 struct rte_crypto_op *op,
 233                 struct aesni_gcm_session *sess)
 234 {
 235         post_process_gcm_crypto_op(qp, op, sess);
 236
 237         /* Free session if a session-less crypto op */
 238         if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
 239                 memset(sess, 0, sizeof(struct aesni_gcm_session));
 240                 memset(op->sym->session, 0,
 241                         rte_cryptodev_sym_get_existing_header_session_size(
 242                                 op->sym->session));
 243                 rte_mempool_put(qp->sess_mp_priv, sess);
 244                 rte_mempool_put(qp->sess_mp, op->sym->session);
 245                 op->sym->session = NULL;
 246         }
 247 }
 248
 249 /**
 250  * Process a crypto operation, calling
 251  * the GCM API from the multi buffer library.
 252  *
 253  * @param       qp              queue pair
 254  * @param       op              symmetric crypto operation
 255  * @param       session         GCM session
 256  *
 257  * @return
 258  *  0 on success
 259  */
 260 static int
 261 process_gcm_crypto_op(struct ipsec_mb_qp *qp, struct rte_crypto_op *op,
 262                 struct aesni_gcm_session *session)
 263 {
 264         struct aesni_gcm_qp_data *qp_data = ipsec_mb_get_qp_private_data(qp);
 265         uint8_t *src, *dst;
 266         uint8_t *iv_ptr;
 267         struct rte_crypto_sym_op *sym_op = op->sym;
 268         struct rte_mbuf *m_src = sym_op->m_src;
 269         uint32_t offset, data_offset, data_length;
 270         uint32_t part_len, total_len, data_len;
 271         uint8_t *tag;
 272         unsigned int oop = 0;
 273         struct aesni_gcm_ops *ops = &qp_data->ops[session->key_length];
 274
 275         if (session->op == IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT ||
 276                         session->op == IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT) {
 277                 offset = sym_op->aead.data.offset;
 278                 data_offset = offset;
 279                 data_length = sym_op->aead.data.length;
 280         } else {
 281                 offset = sym_op->auth.data.offset;
 282                 data_offset = offset;
 283                 data_length = sym_op->auth.data.length;
 284         }
 285
 286         RTE_ASSERT(m_src != NULL);
 287
 288         while (offset >= m_src->data_len && data_length != 0) {
 289                 offset -= m_src->data_len;
 290                 m_src = m_src->next;
 291
 292                 RTE_ASSERT(m_src != NULL);
 293         }
 294
 295         src = rte_pktmbuf_mtod_offset(m_src, uint8_t *, offset);
 296
 297         data_len = m_src->data_len - offset;
 298         part_len = (data_len < data_length) ? data_len :
 299                         data_length;
 300
 301         RTE_ASSERT((sym_op->m_dst == NULL) ||
 302                         ((sym_op->m_dst != NULL) &&
 303                                 rte_pktmbuf_is_contiguous(sym_op->m_dst)));
 304
 305         /* In-place */
 306         if (sym_op->m_dst == NULL || (sym_op->m_dst == sym_op->m_src))
 307                 dst = src;
 308         /* Out-of-place */
 309         else {
 310                 oop = 1;
 311                 /* Segmented destination buffer is not supported
 312                  * if operation is Out-of-place
 313                  */
 314                 RTE_ASSERT(rte_pktmbuf_is_contiguous(sym_op->m_dst));
 315                 dst = rte_pktmbuf_mtod_offset(sym_op->m_dst, uint8_t *,
 316                                         data_offset);
 317         }
 318
 319         iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 320                                 session->iv.offset);
 321
 322         if (session->op == IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT) {
 323                 ops->init(&session->gdata_key, &qp_data->gcm_ctx_data, iv_ptr,
 324                                 sym_op->aead.aad.data,
 325                                 (uint64_t)session->aad_length);
 326
 327                 ops->update_enc(&session->gdata_key, &qp_data->gcm_ctx_data,
 328                                 dst, src, (uint64_t)part_len);
 329                 total_len = data_length - part_len;
 330
 331                 while (total_len) {
 332                         m_src = m_src->next;
 333
 334                         RTE_ASSERT(m_src != NULL);
 335
 336                         src = rte_pktmbuf_mtod(m_src, uint8_t *);
 337                         if (oop)
 338                                 dst += part_len;
 339                         else
 340                                 dst = src;
 341                         part_len = (m_src->data_len < total_len) ?
 342                                         m_src->data_len : total_len;
 343
 344                         ops->update_enc(&session->gdata_key,
 345                                         &qp_data->gcm_ctx_data,
 346                                         dst, src, (uint64_t)part_len);
 347                         total_len -= part_len;
 348                 }
 349
 350                 if (session->req_digest_length != session->gen_digest_length)
 351                         tag = qp_data->temp_digest;
 352                 else
 353                         tag = sym_op->aead.digest.data;
 354
 355                 ops->finalize_enc(&session->gdata_key, &qp_data->gcm_ctx_data,
 356                                 tag, session->gen_digest_length);
 357         } else if (session->op == IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT) {
 358                 ops->init(&session->gdata_key, &qp_data->gcm_ctx_data, iv_ptr,
 359                                 sym_op->aead.aad.data,
 360                                 (uint64_t)session->aad_length);
 361
 362                 ops->update_dec(&session->gdata_key, &qp_data->gcm_ctx_data,
 363                                 dst, src, (uint64_t)part_len);
 364                 total_len = data_length - part_len;
 365
 366                 while (total_len) {
 367                         m_src = m_src->next;
 368
 369                         RTE_ASSERT(m_src != NULL);
 370
 371                         src = rte_pktmbuf_mtod(m_src, uint8_t *);
 372                         if (oop)
 373                                 dst += part_len;
 374                         else
 375                                 dst = src;
 376                         part_len = (m_src->data_len < total_len) ?
 377                                         m_src->data_len : total_len;
 378
 379                         ops->update_dec(&session->gdata_key,
 380                                         &qp_data->gcm_ctx_data,
 381                                         dst, src, (uint64_t)part_len);
 382                         total_len -= part_len;
 383                 }
 384
 385                 tag = qp_data->temp_digest;
 386                 ops->finalize_dec(&session->gdata_key, &qp_data->gcm_ctx_data,
 387                                 tag, session->gen_digest_length);
 388         } else if (session->op == IPSEC_MB_OP_HASH_GEN_ONLY) {
 389                 ops->gmac_init(&session->gdata_key, &qp_data->gcm_ctx_data,
 390                                 iv_ptr, session->iv.length);
 391
 392                 ops->gmac_update(&session->gdata_key, &qp_data->gcm_ctx_data,
 393                                 src, (uint64_t)part_len);
 394                 total_len = data_length - part_len;
 395
 396                 while (total_len) {
 397                         m_src = m_src->next;
 398
 399                         RTE_ASSERT(m_src != NULL);
 400
 401                         src = rte_pktmbuf_mtod(m_src, uint8_t *);
 402                         part_len = (m_src->data_len < total_len) ?
 403                                         m_src->data_len : total_len;
 404
 405                         ops->gmac_update(&session->gdata_key,
 406                                         &qp_data->gcm_ctx_data, src,
 407                                         (uint64_t)part_len);
 408                         total_len -= part_len;
 409                 }
 410
 411                 if (session->req_digest_length != session->gen_digest_length)
 412                         tag = qp_data->temp_digest;
 413                 else
 414                         tag = sym_op->auth.digest.data;
 415
 416                 ops->gmac_finalize(&session->gdata_key, &qp_data->gcm_ctx_data,
 417                                 tag, session->gen_digest_length);
 418         } else { /* IPSEC_MB_OP_HASH_VERIFY_ONLY */
 419                 ops->gmac_init(&session->gdata_key, &qp_data->gcm_ctx_data,
 420                                 iv_ptr, session->iv.length);
 421
 422                 ops->gmac_update(&session->gdata_key, &qp_data->gcm_ctx_data,
 423                                 src, (uint64_t)part_len);
 424                 total_len = data_length - part_len;
 425
 426                 while (total_len) {
 427                         m_src = m_src->next;
 428
 429                         RTE_ASSERT(m_src != NULL);
 430
 431                         src = rte_pktmbuf_mtod(m_src, uint8_t *);
 432                         part_len = (m_src->data_len < total_len) ?
 433                                         m_src->data_len : total_len;
 434
 435                         ops->gmac_update(&session->gdata_key,
 436                                         &qp_data->gcm_ctx_data, src,
 437                                         (uint64_t)part_len);
 438                         total_len -= part_len;
 439                 }
 440
 441                 tag = qp_data->temp_digest;
 442
 443                 ops->gmac_finalize(&session->gdata_key, &qp_data->gcm_ctx_data,
 444                                 tag, session->gen_digest_length);
 445         }
 446         return 0;
 447 }
 448
 449 /** Get gcm session */
 450 static inline struct aesni_gcm_session *
 451 aesni_gcm_get_session(struct ipsec_mb_qp *qp,
 452              struct rte_crypto_op *op)
 453 {
 454         struct aesni_gcm_session *sess = NULL;
 455         uint32_t driver_id =
 456             ipsec_mb_get_driver_id(IPSEC_MB_PMD_TYPE_AESNI_GCM);
 457         struct rte_crypto_sym_op *sym_op = op->sym;
 458
 459         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 460                 if (likely(sym_op->session != NULL))
 461                         sess = (struct aesni_gcm_session *)
 462                             get_sym_session_private_data(sym_op->session,
 463                                                          driver_id);
 464         } else {
 465                 void *_sess;
 466                 void *_sess_private_data = NULL;
 467
 468                 if (rte_mempool_get(qp->sess_mp, (void **)&_sess))
 469                         return NULL;
 470
 471                 if (rte_mempool_get(qp->sess_mp_priv,
 472                                 (void **)&_sess_private_data))
 473                         return NULL;
 474
 475                 sess = (struct aesni_gcm_session *)_sess_private_data;
 476
 477                 if (unlikely(aesni_gcm_session_configure(qp->mb_mgr,
 478                                  _sess_private_data, sym_op->xform) != 0)) {
 479                         rte_mempool_put(qp->sess_mp, _sess);
 480                         rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
 481                         sess = NULL;
 482                 }
 483                 sym_op->session = (struct rte_cryptodev_sym_session *)_sess;
 484                 set_sym_session_private_data(sym_op->session, driver_id,
 485                                              _sess_private_data);
 486         }
 487
 488         if (unlikely(sess == NULL))
 489                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 490
 491         return sess;
 492 }
 493
 494 static uint16_t
 495 aesni_gcm_pmd_dequeue_burst(void *queue_pair,
 496                 struct rte_crypto_op **ops, uint16_t nb_ops)
 497 {
 498         struct aesni_gcm_session *sess;
 499         struct ipsec_mb_qp *qp = queue_pair;
 500
 501         int retval = 0;
 502         unsigned int i, nb_dequeued;
 503
 504         nb_dequeued = rte_ring_dequeue_burst(qp->ingress_queue,
 505                         (void **)ops, nb_ops, NULL);
 506
 507         for (i = 0; i < nb_dequeued; i++) {
 508
 509                 sess = aesni_gcm_get_session(qp, ops[i]);
 510                 if (unlikely(sess == NULL)) {
 511                         ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 512                         qp->stats.dequeue_err_count++;
 513                         break;
 514                 }
 515
 516                 retval = process_gcm_crypto_op(qp, ops[i], sess);
 517                 if (retval < 0) {
 518                         ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 519                         qp->stats.dequeue_err_count++;
 520                         break;
 521                 }
 522
 523                 handle_completed_gcm_crypto_op(qp, ops[i], sess);
 524         }
 525
 526         qp->stats.dequeued_count += i;
 527
 528         return i;
 529 }
 530
 531 static inline void
 532 aesni_gcm_fill_error_code(struct rte_crypto_sym_vec *vec,
 533                           int32_t errnum)
 534 {
 535         uint32_t i;
 536
 537         for (i = 0; i < vec->num; i++)
 538                 vec->status[i] = errnum;
 539 }
 540
 541 static inline int32_t
 542 aesni_gcm_sgl_op_finalize_encryption(const struct aesni_gcm_session *s,
 543                                      struct gcm_context_data *gdata_ctx,
 544                                      uint8_t *digest, struct aesni_gcm_ops ops)
 545 {
 546         if (s->req_digest_length != s->gen_digest_length) {
 547                 uint8_t tmpdigest[s->gen_digest_length];
 548
 549                 ops.finalize_enc(&s->gdata_key, gdata_ctx, tmpdigest,
 550                                 s->gen_digest_length);
 551                 memcpy(digest, tmpdigest, s->req_digest_length);
 552         } else {
 553                 ops.finalize_enc(&s->gdata_key, gdata_ctx, digest,
 554                                 s->gen_digest_length);
 555         }
 556
 557         return 0;
 558 }
 559
 560 static inline int32_t
 561 aesni_gcm_sgl_op_finalize_decryption(const struct aesni_gcm_session *s,
 562                                      struct gcm_context_data *gdata_ctx,
 563                                      uint8_t *digest, struct aesni_gcm_ops ops)
 564 {
 565         uint8_t tmpdigest[s->gen_digest_length];
 566
 567         ops.finalize_dec(&s->gdata_key, gdata_ctx, tmpdigest,
 568                         s->gen_digest_length);
 569
 570         return memcmp(digest, tmpdigest, s->req_digest_length) == 0 ? 0
 571                                                                     : EBADMSG;
 572 }
 573
 574 static inline void
 575 aesni_gcm_process_gcm_sgl_op(const struct aesni_gcm_session *s,
 576                              struct gcm_context_data *gdata_ctx,
 577                              struct rte_crypto_sgl *sgl, void *iv, void *aad,
 578                              struct aesni_gcm_ops ops)
 579 {
 580         uint32_t i;
 581
 582         /* init crypto operation */
 583         ops.init(&s->gdata_key, gdata_ctx, iv, aad,
 584                     (uint64_t)s->aad_length);
 585
 586         /* update with sgl data */
 587         for (i = 0; i < sgl->num; i++) {
 588                 struct rte_crypto_vec *vec = &sgl->vec[i];
 589
 590                 switch (s->op) {
 591                 case IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT:
 592                         ops.update_enc(&s->gdata_key, gdata_ctx,
 593                               vec->base, vec->base, vec->len);
 594                         break;
 595                 case IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT:
 596                         ops.update_dec(&s->gdata_key, gdata_ctx,
 597                               vec->base, vec->base, vec->len);
 598                         break;
 599                 default:
 600                         IPSEC_MB_LOG(ERR, "Invalid session op");
 601                         break;
 602                 }
 603
 604         }
 605 }
 606
 607 static inline void
 608 aesni_gcm_process_gmac_sgl_op(const struct aesni_gcm_session *s,
 609                               struct gcm_context_data *gdata_ctx,
 610                               struct rte_crypto_sgl *sgl, void *iv,
 611                               struct aesni_gcm_ops ops)
 612 {
 613         ops.init(&s->gdata_key, gdata_ctx, iv, sgl->vec[0].base,
 614                     sgl->vec[0].len);
 615 }
 616
 617 static inline uint32_t
 618 aesni_gcm_sgl_encrypt(struct aesni_gcm_session *s,
 619                       struct gcm_context_data *gdata_ctx,
 620                       struct rte_crypto_sym_vec *vec,
 621                       struct aesni_gcm_ops ops)
 622 {
 623         uint32_t i, processed;
 624
 625         processed = 0;
 626         for (i = 0; i < vec->num; ++i) {
 627                 aesni_gcm_process_gcm_sgl_op(s, gdata_ctx, &vec->src_sgl[i],
 628                                              vec->iv[i].va, vec->aad[i].va,
 629                                              ops);
 630                 vec->status[i] = aesni_gcm_sgl_op_finalize_encryption(
 631                     s, gdata_ctx, vec->digest[i].va, ops);
 632                 processed += (vec->status[i] == 0);
 633         }
 634
 635         return processed;
 636 }
 637
 638 static inline uint32_t
 639 aesni_gcm_sgl_decrypt(struct aesni_gcm_session *s,
 640                       struct gcm_context_data *gdata_ctx,
 641                       struct rte_crypto_sym_vec *vec,
 642                       struct aesni_gcm_ops ops)
 643 {
 644         uint32_t i, processed;
 645
 646         processed = 0;
 647         for (i = 0; i < vec->num; ++i) {
 648                 aesni_gcm_process_gcm_sgl_op(s, gdata_ctx, &vec->src_sgl[i],
 649                                              vec->iv[i].va, vec->aad[i].va,
 650                                              ops);
 651                 vec->status[i] = aesni_gcm_sgl_op_finalize_decryption(
 652                     s, gdata_ctx, vec->digest[i].va, ops);
 653                 processed += (vec->status[i] == 0);
 654         }
 655
 656         return processed;
 657 }
 658
 659 static inline uint32_t
 660 aesni_gmac_sgl_generate(struct aesni_gcm_session *s,
 661                         struct gcm_context_data *gdata_ctx,
 662                         struct rte_crypto_sym_vec *vec,
 663                         struct aesni_gcm_ops ops)
 664 {
 665         uint32_t i, processed;
 666
 667         processed = 0;
 668         for (i = 0; i < vec->num; ++i) {
 669                 if (vec->src_sgl[i].num != 1) {
 670                         vec->status[i] = ENOTSUP;
 671                         continue;
 672                 }
 673
 674                 aesni_gcm_process_gmac_sgl_op(s, gdata_ctx, &vec->src_sgl[i],
 675                                               vec->iv[i].va, ops);
 676                 vec->status[i] = aesni_gcm_sgl_op_finalize_encryption(
 677                     s, gdata_ctx, vec->digest[i].va, ops);
 678                 processed += (vec->status[i] == 0);
 679         }
 680
 681         return processed;
 682 }
 683
 684 static inline uint32_t
 685 aesni_gmac_sgl_verify(struct aesni_gcm_session *s,
 686                       struct gcm_context_data *gdata_ctx,
 687                       struct rte_crypto_sym_vec *vec,
 688                       struct aesni_gcm_ops ops)
 689 {
 690         uint32_t i, processed;
 691
 692         processed = 0;
 693         for (i = 0; i < vec->num; ++i) {
 694                 if (vec->src_sgl[i].num != 1) {
 695                         vec->status[i] = ENOTSUP;
 696                         continue;
 697                 }
 698
 699                 aesni_gcm_process_gmac_sgl_op(s, gdata_ctx, &vec->src_sgl[i],
 700                                               vec->iv[i].va, ops);
 701                 vec->status[i] = aesni_gcm_sgl_op_finalize_decryption(
 702                     s, gdata_ctx, vec->digest[i].va, ops);
 703                 processed += (vec->status[i] == 0);
 704         }
 705
 706         return processed;
 707 }
 708
 709 /** Process CPU crypto bulk operations */
 710 static uint32_t
 711 aesni_gcm_process_bulk(struct rte_cryptodev *dev,
 712                         struct rte_cryptodev_sym_session *sess,
 713                         __rte_unused union rte_crypto_sym_ofs ofs,
 714                         struct rte_crypto_sym_vec *vec)
 715 {
 716         struct aesni_gcm_session *s;
 717         struct gcm_context_data gdata_ctx;
 718         IMB_MGR *mb_mgr;
 719
 720         s = (struct aesni_gcm_session *) get_sym_session_private_data(sess,
 721                 dev->driver_id);
 722         if (unlikely(s == NULL)) {
 723                 aesni_gcm_fill_error_code(vec, EINVAL);
 724                 return 0;
 725         }
 726
 727         /* get per-thread MB MGR, create one if needed */
 728         mb_mgr = get_per_thread_mb_mgr();
 729         if (unlikely(mb_mgr == NULL))
 730                 return 0;
 731
 732         /* Check if function pointers have been set for this thread ops. */
 733         if (unlikely(RTE_PER_LCORE(gcm_ops)[s->key_length].init == NULL))
 734                 aesni_gcm_set_ops(RTE_PER_LCORE(gcm_ops), mb_mgr);
 735
 736         switch (s->op) {
 737         case IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT:
 738                 return aesni_gcm_sgl_encrypt(s, &gdata_ctx, vec,
 739                                 RTE_PER_LCORE(gcm_ops)[s->key_length]);
 740         case IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT:
 741                 return aesni_gcm_sgl_decrypt(s, &gdata_ctx, vec,
 742                                 RTE_PER_LCORE(gcm_ops)[s->key_length]);
 743         case IPSEC_MB_OP_HASH_GEN_ONLY:
 744                 return aesni_gmac_sgl_generate(s, &gdata_ctx, vec,
 745                                 RTE_PER_LCORE(gcm_ops)[s->key_length]);
 746         case IPSEC_MB_OP_HASH_VERIFY_ONLY:
 747                 return aesni_gmac_sgl_verify(s, &gdata_ctx, vec,
 748                                 RTE_PER_LCORE(gcm_ops)[s->key_length]);
 749         default:
 750                 aesni_gcm_fill_error_code(vec, EINVAL);
 751                 return 0;
 752         }
 753 }
 754
 755 static int
 756 aesni_gcm_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
 757                                 const struct rte_cryptodev_qp_conf *qp_conf,
 758                                 int socket_id)
 759 {
 760         int ret = ipsec_mb_qp_setup(dev, qp_id, qp_conf, socket_id);
 761         if (ret < 0)
 762                 return ret;
 763
 764         struct ipsec_mb_qp *qp = dev->data->queue_pairs[qp_id];
 765         struct aesni_gcm_qp_data *qp_data = ipsec_mb_get_qp_private_data(qp);
 766         aesni_gcm_set_ops(qp_data->ops, qp->mb_mgr);
 767         return 0;
 768 }
 769
 770 struct rte_cryptodev_ops aesni_gcm_pmd_ops = {
 771         .dev_configure = ipsec_mb_config,
 772         .dev_start = ipsec_mb_start,
 773         .dev_stop = ipsec_mb_stop,
 774         .dev_close = ipsec_mb_close,
 775
 776         .stats_get = ipsec_mb_stats_get,
 777         .stats_reset = ipsec_mb_stats_reset,
 778
 779         .dev_infos_get = ipsec_mb_info_get,
 780
 781         .queue_pair_setup = aesni_gcm_qp_setup,
 782         .queue_pair_release = ipsec_mb_qp_release,
 783
 784         .sym_cpu_process = aesni_gcm_process_bulk,
 785
 786         .sym_session_get_size = ipsec_mb_sym_session_get_size,
 787         .sym_session_configure = ipsec_mb_sym_session_configure,
 788         .sym_session_clear = ipsec_mb_sym_session_clear
 789 };
 790
 791 static int
 792 aesni_gcm_probe(struct rte_vdev_device *vdev)
 793 {
 794         return ipsec_mb_create(vdev, IPSEC_MB_PMD_TYPE_AESNI_GCM);
 795 }
 796
 797 static struct rte_vdev_driver cryptodev_aesni_gcm_pmd_drv = {
 798         .probe = aesni_gcm_probe,
 799         .remove = ipsec_mb_remove
 800 };
 801
 802 static struct cryptodev_driver aesni_gcm_crypto_drv;
 803
 804 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_GCM_PMD,
 805                       cryptodev_aesni_gcm_pmd_drv);
 806 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_GCM_PMD, cryptodev_aesni_gcm_pmd);
 807 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_GCM_PMD,
 808                               "max_nb_queue_pairs=<int> socket_id=<int>");
 809 RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_gcm_crypto_drv,
 810                                cryptodev_aesni_gcm_pmd_drv.driver,
 811                                pmd_driver_id_aesni_gcm);
 812
 813 /* Constructor function to register aesni-gcm PMD */
 814 RTE_INIT(ipsec_mb_register_aesni_gcm)
 815 {
 816         struct ipsec_mb_internals *aesni_gcm_data =
 817                 &ipsec_mb_pmds[IPSEC_MB_PMD_TYPE_AESNI_GCM];
 818
 819         aesni_gcm_data->caps = aesni_gcm_capabilities;
 820         aesni_gcm_data->dequeue_burst = aesni_gcm_pmd_dequeue_burst;
 821         aesni_gcm_data->feature_flags =
 822                 RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
 823                 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
 824                 RTE_CRYPTODEV_FF_IN_PLACE_SGL |
 825                 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
 826                 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
 827                 RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO |
 828                 RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
 829         aesni_gcm_data->internals_priv_size = 0;
 830         aesni_gcm_data->ops = &aesni_gcm_pmd_ops;
 831         aesni_gcm_data->qp_priv_size = sizeof(struct aesni_gcm_qp_data);
 832         aesni_gcm_data->queue_pair_configure = NULL;
 833         aesni_gcm_data->session_configure = aesni_gcm_session_configure;
 834         aesni_gcm_data->session_priv_size = sizeof(struct aesni_gcm_session);
 835 }