drivers/crypto/aesni_gcm/aesni_gcm_pmd.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016-2020 Intel Corporation
   3  */
   4
   5 #include <rte_common.h>
   6 #include <rte_hexdump.h>
   7 #include <rte_cryptodev.h>
   8 #include <rte_cryptodev_pmd.h>
   9 #include <rte_bus_vdev.h>
  10 #include <rte_malloc.h>
  11 #include <rte_cpuflags.h>
  12 #include <rte_byteorder.h>
  13
  14 #include "aesni_gcm_pmd_private.h"
  15
  16 static uint8_t cryptodev_driver_id;
  17
  18 /* setup session handlers */
  19 static void
  20 set_func_ops(struct aesni_gcm_session *s, const struct aesni_gcm_ops *gcm_ops)
  21 {
  22         s->ops.pre = gcm_ops->pre;
  23         s->ops.init = gcm_ops->init;
  24
  25         switch (s->op) {
  26         case AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION:
  27                 s->ops.cipher = gcm_ops->enc;
  28                 s->ops.update = gcm_ops->update_enc;
  29                 s->ops.finalize = gcm_ops->finalize_enc;
  30                 break;
  31         case AESNI_GCM_OP_AUTHENTICATED_DECRYPTION:
  32                 s->ops.cipher = gcm_ops->dec;
  33                 s->ops.update = gcm_ops->update_dec;
  34                 s->ops.finalize = gcm_ops->finalize_dec;
  35                 break;
  36         case AESNI_GMAC_OP_GENERATE:
  37         case AESNI_GMAC_OP_VERIFY:
  38                 s->ops.finalize = gcm_ops->finalize_enc;
  39                 break;
  40         }
  41 }
  42
  43 /** Parse crypto xform chain and set private session parameters */
  44 int
  45 aesni_gcm_set_session_parameters(const struct aesni_gcm_ops *gcm_ops,
  46                 struct aesni_gcm_session *sess,
  47                 const struct rte_crypto_sym_xform *xform)
  48 {
  49         const struct rte_crypto_sym_xform *auth_xform;
  50         const struct rte_crypto_sym_xform *aead_xform;
  51         uint8_t key_length;
  52         const uint8_t *key;
  53
  54         /* AES-GMAC */
  55         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
  56                 auth_xform = xform;
  57                 if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_AES_GMAC) {
  58                         AESNI_GCM_LOG(ERR, "Only AES GMAC is supported as an "
  59                                 "authentication only algorithm");
  60                         return -ENOTSUP;
  61                 }
  62                 /* Set IV parameters */
  63                 sess->iv.offset = auth_xform->auth.iv.offset;
  64                 sess->iv.length = auth_xform->auth.iv.length;
  65
  66                 /* Select Crypto operation */
  67                 if (auth_xform->auth.op == RTE_CRYPTO_AUTH_OP_GENERATE)
  68                         sess->op = AESNI_GMAC_OP_GENERATE;
  69                 else
  70                         sess->op = AESNI_GMAC_OP_VERIFY;
  71
  72                 key_length = auth_xform->auth.key.length;
  73                 key = auth_xform->auth.key.data;
  74                 sess->req_digest_length = auth_xform->auth.digest_length;
  75
  76         /* AES-GCM */
  77         } else if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
  78                 aead_xform = xform;
  79
  80                 if (aead_xform->aead.algo != RTE_CRYPTO_AEAD_AES_GCM) {
  81                         AESNI_GCM_LOG(ERR, "The only combined operation "
  82                                                 "supported is AES GCM");
  83                         return -ENOTSUP;
  84                 }
  85
  86                 /* Set IV parameters */
  87                 sess->iv.offset = aead_xform->aead.iv.offset;
  88                 sess->iv.length = aead_xform->aead.iv.length;
  89
  90                 /* Select Crypto operation */
  91                 if (aead_xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
  92                         sess->op = AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION;
  93                 /* op == RTE_CRYPTO_AEAD_OP_DECRYPT */
  94                 else
  95                         sess->op = AESNI_GCM_OP_AUTHENTICATED_DECRYPTION;
  96
  97                 key_length = aead_xform->aead.key.length;
  98                 key = aead_xform->aead.key.data;
  99
 100                 sess->aad_length = aead_xform->aead.aad_length;
 101                 sess->req_digest_length = aead_xform->aead.digest_length;
 102         } else {
 103                 AESNI_GCM_LOG(ERR, "Wrong xform type, has to be AEAD or authentication");
 104                 return -ENOTSUP;
 105         }
 106
 107         /* IV check */
 108         if (sess->iv.length != 16 && sess->iv.length != 12 &&
 109                         sess->iv.length != 0) {
 110                 AESNI_GCM_LOG(ERR, "Wrong IV length");
 111                 return -EINVAL;
 112         }
 113
 114         /* Check key length and calculate GCM pre-compute. */
 115         switch (key_length) {
 116         case 16:
 117                 sess->key = GCM_KEY_128;
 118                 break;
 119         case 24:
 120                 sess->key = GCM_KEY_192;
 121                 break;
 122         case 32:
 123                 sess->key = GCM_KEY_256;
 124                 break;
 125         default:
 126                 AESNI_GCM_LOG(ERR, "Invalid key length");
 127                 return -EINVAL;
 128         }
 129
 130         /* setup session handlers */
 131         set_func_ops(sess, &gcm_ops[sess->key]);
 132
 133         /* pre-generate key */
 134         gcm_ops[sess->key].pre(key, &sess->gdata_key);
 135
 136         /* Digest check */
 137         if (sess->req_digest_length > 16) {
 138                 AESNI_GCM_LOG(ERR, "Invalid digest length");
 139                 return -EINVAL;
 140         }
 141         /*
 142          * Multi-buffer lib supports digest sizes from 4 to 16 bytes
 143          * in version 0.50 and sizes of 8, 12 and 16 bytes,
 144          * in version 0.49.
 145          * If size requested is different, generate the full digest
 146          * (16 bytes) in a temporary location and then memcpy
 147          * the requested number of bytes.
 148          */
 149 #if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
 150         if (sess->req_digest_length < 4)
 151 #else
 152         if (sess->req_digest_length != 16 &&
 153                         sess->req_digest_length != 12 &&
 154                         sess->req_digest_length != 8)
 155 #endif
 156                 sess->gen_digest_length = 16;
 157         else
 158                 sess->gen_digest_length = sess->req_digest_length;
 159
 160         return 0;
 161 }
 162
 163 /** Get gcm session */
 164 static struct aesni_gcm_session *
 165 aesni_gcm_get_session(struct aesni_gcm_qp *qp, struct rte_crypto_op *op)
 166 {
 167         struct aesni_gcm_session *sess = NULL;
 168         struct rte_crypto_sym_op *sym_op = op->sym;
 169
 170         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 171                 if (likely(sym_op->session != NULL))
 172                         sess = (struct aesni_gcm_session *)
 173                                         get_sym_session_private_data(
 174                                         sym_op->session,
 175                                         cryptodev_driver_id);
 176         } else  {
 177                 void *_sess;
 178                 void *_sess_private_data = NULL;
 179
 180                 if (rte_mempool_get(qp->sess_mp, (void **)&_sess))
 181                         return NULL;
 182
 183                 if (rte_mempool_get(qp->sess_mp_priv,
 184                                 (void **)&_sess_private_data))
 185                         return NULL;
 186
 187                 sess = (struct aesni_gcm_session *)_sess_private_data;
 188
 189                 if (unlikely(aesni_gcm_set_session_parameters(qp->ops,
 190                                 sess, sym_op->xform) != 0)) {
 191                         rte_mempool_put(qp->sess_mp, _sess);
 192                         rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
 193                         sess = NULL;
 194                 }
 195                 sym_op->session = (struct rte_cryptodev_sym_session *)_sess;
 196                 set_sym_session_private_data(sym_op->session,
 197                                 cryptodev_driver_id, _sess_private_data);
 198         }
 199
 200         if (unlikely(sess == NULL))
 201                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 202
 203         return sess;
 204 }
 205
 206 /**
 207  * Process a crypto operation, calling
 208  * the GCM API from the multi buffer library.
 209  *
 210  * @param       qp              queue pair
 211  * @param       op              symmetric crypto operation
 212  * @param       session         GCM session
 213  *
 214  * @return
 215  *
 216  */
 217 static int
 218 process_gcm_crypto_op(struct aesni_gcm_qp *qp, struct rte_crypto_op *op,
 219                 struct aesni_gcm_session *session)
 220 {
 221         uint8_t *src, *dst;
 222         uint8_t *iv_ptr;
 223         struct rte_crypto_sym_op *sym_op = op->sym;
 224         struct rte_mbuf *m_src = sym_op->m_src;
 225         uint32_t offset, data_offset, data_length;
 226         uint32_t part_len, total_len, data_len;
 227         uint8_t *tag;
 228         unsigned int oop = 0;
 229
 230         if (session->op == AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION ||
 231                         session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION) {
 232                 offset = sym_op->aead.data.offset;
 233                 data_offset = offset;
 234                 data_length = sym_op->aead.data.length;
 235         } else {
 236                 offset = sym_op->auth.data.offset;
 237                 data_offset = offset;
 238                 data_length = sym_op->auth.data.length;
 239         }
 240
 241         RTE_ASSERT(m_src != NULL);
 242
 243         while (offset >= m_src->data_len && data_length != 0) {
 244                 offset -= m_src->data_len;
 245                 m_src = m_src->next;
 246
 247                 RTE_ASSERT(m_src != NULL);
 248         }
 249
 250         src = rte_pktmbuf_mtod_offset(m_src, uint8_t *, offset);
 251
 252         data_len = m_src->data_len - offset;
 253         part_len = (data_len < data_length) ? data_len :
 254                         data_length;
 255
 256         RTE_ASSERT((sym_op->m_dst == NULL) ||
 257                         ((sym_op->m_dst != NULL) &&
 258                                         rte_pktmbuf_is_contiguous(sym_op->m_dst)));
 259
 260         /* In-place */
 261         if (sym_op->m_dst == NULL || (sym_op->m_dst == sym_op->m_src))
 262                 dst = src;
 263         /* Out-of-place */
 264         else {
 265                 oop = 1;
 266                 /* Segmented destination buffer is not supported if operation is
 267                  * Out-of-place */
 268                 RTE_ASSERT(rte_pktmbuf_is_contiguous(sym_op->m_dst));
 269                 dst = rte_pktmbuf_mtod_offset(sym_op->m_dst, uint8_t *,
 270                                         data_offset);
 271         }
 272
 273         iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 274                                 session->iv.offset);
 275
 276         if (session->op == AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION) {
 277                 qp->ops[session->key].init(&session->gdata_key,
 278                                 &qp->gdata_ctx,
 279                                 iv_ptr,
 280                                 sym_op->aead.aad.data,
 281                                 (uint64_t)session->aad_length);
 282
 283                 qp->ops[session->key].update_enc(&session->gdata_key,
 284                                 &qp->gdata_ctx, dst, src,
 285                                 (uint64_t)part_len);
 286                 total_len = data_length - part_len;
 287
 288                 while (total_len) {
 289                         m_src = m_src->next;
 290
 291                         RTE_ASSERT(m_src != NULL);
 292
 293                         src = rte_pktmbuf_mtod(m_src, uint8_t *);
 294                         if (oop)
 295                                 dst += part_len;
 296                         else
 297                                 dst = src;
 298                         part_len = (m_src->data_len < total_len) ?
 299                                         m_src->data_len : total_len;
 300
 301                         qp->ops[session->key].update_enc(&session->gdata_key,
 302                                         &qp->gdata_ctx, dst, src,
 303                                         (uint64_t)part_len);
 304                         total_len -= part_len;
 305                 }
 306
 307                 if (session->req_digest_length != session->gen_digest_length)
 308                         tag = qp->temp_digest;
 309                 else
 310                         tag = sym_op->aead.digest.data;
 311
 312                 qp->ops[session->key].finalize_enc(&session->gdata_key,
 313                                 &qp->gdata_ctx,
 314                                 tag,
 315                                 session->gen_digest_length);
 316         } else if (session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION) {
 317                 qp->ops[session->key].init(&session->gdata_key,
 318                                 &qp->gdata_ctx,
 319                                 iv_ptr,
 320                                 sym_op->aead.aad.data,
 321                                 (uint64_t)session->aad_length);
 322
 323                 qp->ops[session->key].update_dec(&session->gdata_key,
 324                                 &qp->gdata_ctx, dst, src,
 325                                 (uint64_t)part_len);
 326                 total_len = data_length - part_len;
 327
 328                 while (total_len) {
 329                         m_src = m_src->next;
 330
 331                         RTE_ASSERT(m_src != NULL);
 332
 333                         src = rte_pktmbuf_mtod(m_src, uint8_t *);
 334                         if (oop)
 335                                 dst += part_len;
 336                         else
 337                                 dst = src;
 338                         part_len = (m_src->data_len < total_len) ?
 339                                         m_src->data_len : total_len;
 340
 341                         qp->ops[session->key].update_dec(&session->gdata_key,
 342                                         &qp->gdata_ctx,
 343                                         dst, src,
 344                                         (uint64_t)part_len);
 345                         total_len -= part_len;
 346                 }
 347
 348                 tag = qp->temp_digest;
 349                 qp->ops[session->key].finalize_dec(&session->gdata_key,
 350                                 &qp->gdata_ctx,
 351                                 tag,
 352                                 session->gen_digest_length);
 353         } else if (session->op == AESNI_GMAC_OP_GENERATE) {
 354                 qp->ops[session->key].init(&session->gdata_key,
 355                                 &qp->gdata_ctx,
 356                                 iv_ptr,
 357                                 src,
 358                                 (uint64_t)data_length);
 359                 if (session->req_digest_length != session->gen_digest_length)
 360                         tag = qp->temp_digest;
 361                 else
 362                         tag = sym_op->auth.digest.data;
 363                 qp->ops[session->key].finalize_enc(&session->gdata_key,
 364                                 &qp->gdata_ctx,
 365                                 tag,
 366                                 session->gen_digest_length);
 367         } else { /* AESNI_GMAC_OP_VERIFY */
 368                 qp->ops[session->key].init(&session->gdata_key,
 369                                 &qp->gdata_ctx,
 370                                 iv_ptr,
 371                                 src,
 372                                 (uint64_t)data_length);
 373
 374                 /*
 375                  * Generate always 16 bytes and later compare only
 376                  * the bytes passed.
 377                  */
 378                 tag = qp->temp_digest;
 379                 qp->ops[session->key].finalize_enc(&session->gdata_key,
 380                                 &qp->gdata_ctx,
 381                                 tag,
 382                                 session->gen_digest_length);
 383         }
 384
 385         return 0;
 386 }
 387
 388 static inline void
 389 aesni_gcm_fill_error_code(struct rte_crypto_sym_vec *vec, int32_t errnum)
 390 {
 391         uint32_t i;
 392
 393         for (i = 0; i < vec->num; i++)
 394                 vec->status[i] = errnum;
 395 }
 396
 397
 398 static inline int32_t
 399 aesni_gcm_sgl_op_finalize_encryption(const struct aesni_gcm_session *s,
 400         struct gcm_context_data *gdata_ctx, uint8_t *digest)
 401 {
 402         if (s->req_digest_length != s->gen_digest_length) {
 403                 uint8_t tmpdigest[s->gen_digest_length];
 404
 405                 s->ops.finalize(&s->gdata_key, gdata_ctx, tmpdigest,
 406                         s->gen_digest_length);
 407                 memcpy(digest, tmpdigest, s->req_digest_length);
 408         } else {
 409                 s->ops.finalize(&s->gdata_key, gdata_ctx, digest,
 410                         s->gen_digest_length);
 411         }
 412
 413         return 0;
 414 }
 415
 416 static inline int32_t
 417 aesni_gcm_sgl_op_finalize_decryption(const struct aesni_gcm_session *s,
 418         struct gcm_context_data *gdata_ctx, uint8_t *digest)
 419 {
 420         uint8_t tmpdigest[s->gen_digest_length];
 421
 422         s->ops.finalize(&s->gdata_key, gdata_ctx, tmpdigest,
 423                 s->gen_digest_length);
 424
 425         return memcmp(digest, tmpdigest, s->req_digest_length) == 0 ? 0 :
 426                 EBADMSG;
 427 }
 428
 429 static inline void
 430 aesni_gcm_process_gcm_sgl_op(const struct aesni_gcm_session *s,
 431         struct gcm_context_data *gdata_ctx, struct rte_crypto_sgl *sgl,
 432         void *iv, void *aad)
 433 {
 434         uint32_t i;
 435
 436         /* init crypto operation */
 437         s->ops.init(&s->gdata_key, gdata_ctx, iv, aad,
 438                 (uint64_t)s->aad_length);
 439
 440         /* update with sgl data */
 441         for (i = 0; i < sgl->num; i++) {
 442                 struct rte_crypto_vec *vec = &sgl->vec[i];
 443
 444                 s->ops.update(&s->gdata_key, gdata_ctx, vec->base, vec->base,
 445                         vec->len);
 446         }
 447 }
 448
 449 static inline void
 450 aesni_gcm_process_gmac_sgl_op(const struct aesni_gcm_session *s,
 451         struct gcm_context_data *gdata_ctx, struct rte_crypto_sgl *sgl,
 452         void *iv)
 453 {
 454         s->ops.init(&s->gdata_key, gdata_ctx, iv, sgl->vec[0].base,
 455                 sgl->vec[0].len);
 456 }
 457
 458 static inline uint32_t
 459 aesni_gcm_sgl_encrypt(struct aesni_gcm_session *s,
 460         struct gcm_context_data *gdata_ctx, struct rte_crypto_sym_vec *vec)
 461 {
 462         uint32_t i, processed;
 463
 464         processed = 0;
 465         for (i = 0; i < vec->num; ++i) {
 466                 aesni_gcm_process_gcm_sgl_op(s, gdata_ctx,
 467                         &vec->sgl[i], vec->iv[i], vec->aad[i]);
 468                 vec->status[i] = aesni_gcm_sgl_op_finalize_encryption(s,
 469                         gdata_ctx, vec->digest[i]);
 470                 processed += (vec->status[i] == 0);
 471         }
 472
 473         return processed;
 474 }
 475
 476 static inline uint32_t
 477 aesni_gcm_sgl_decrypt(struct aesni_gcm_session *s,
 478         struct gcm_context_data *gdata_ctx, struct rte_crypto_sym_vec *vec)
 479 {
 480         uint32_t i, processed;
 481
 482         processed = 0;
 483         for (i = 0; i < vec->num; ++i) {
 484                 aesni_gcm_process_gcm_sgl_op(s, gdata_ctx,
 485                         &vec->sgl[i], vec->iv[i], vec->aad[i]);
 486                  vec->status[i] = aesni_gcm_sgl_op_finalize_decryption(s,
 487                         gdata_ctx, vec->digest[i]);
 488                 processed += (vec->status[i] == 0);
 489         }
 490
 491         return processed;
 492 }
 493
 494 static inline uint32_t
 495 aesni_gmac_sgl_generate(struct aesni_gcm_session *s,
 496         struct gcm_context_data *gdata_ctx, struct rte_crypto_sym_vec *vec)
 497 {
 498         uint32_t i, processed;
 499
 500         processed = 0;
 501         for (i = 0; i < vec->num; ++i) {
 502                 if (vec->sgl[i].num != 1) {
 503                         vec->status[i] = ENOTSUP;
 504                         continue;
 505                 }
 506
 507                 aesni_gcm_process_gmac_sgl_op(s, gdata_ctx,
 508                         &vec->sgl[i], vec->iv[i]);
 509                 vec->status[i] = aesni_gcm_sgl_op_finalize_encryption(s,
 510                         gdata_ctx, vec->digest[i]);
 511                 processed += (vec->status[i] == 0);
 512         }
 513
 514         return processed;
 515 }
 516
 517 static inline uint32_t
 518 aesni_gmac_sgl_verify(struct aesni_gcm_session *s,
 519         struct gcm_context_data *gdata_ctx, struct rte_crypto_sym_vec *vec)
 520 {
 521         uint32_t i, processed;
 522
 523         processed = 0;
 524         for (i = 0; i < vec->num; ++i) {
 525                 if (vec->sgl[i].num != 1) {
 526                         vec->status[i] = ENOTSUP;
 527                         continue;
 528                 }
 529
 530                 aesni_gcm_process_gmac_sgl_op(s, gdata_ctx,
 531                         &vec->sgl[i], vec->iv[i]);
 532                 vec->status[i] = aesni_gcm_sgl_op_finalize_decryption(s,
 533                         gdata_ctx, vec->digest[i]);
 534                 processed += (vec->status[i] == 0);
 535         }
 536
 537         return processed;
 538 }
 539
 540 /** Process CPU crypto bulk operations */
 541 uint32_t
 542 aesni_gcm_pmd_cpu_crypto_process(struct rte_cryptodev *dev,
 543         struct rte_cryptodev_sym_session *sess,
 544         __rte_unused union rte_crypto_sym_ofs ofs,
 545         struct rte_crypto_sym_vec *vec)
 546 {
 547         void *sess_priv;
 548         struct aesni_gcm_session *s;
 549         struct gcm_context_data gdata_ctx;
 550
 551         sess_priv = get_sym_session_private_data(sess, dev->driver_id);
 552         if (unlikely(sess_priv == NULL)) {
 553                 aesni_gcm_fill_error_code(vec, EINVAL);
 554                 return 0;
 555         }
 556
 557         s = sess_priv;
 558         switch (s->op) {
 559         case AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION:
 560                 return aesni_gcm_sgl_encrypt(s, &gdata_ctx, vec);
 561         case AESNI_GCM_OP_AUTHENTICATED_DECRYPTION:
 562                 return aesni_gcm_sgl_decrypt(s, &gdata_ctx, vec);
 563         case AESNI_GMAC_OP_GENERATE:
 564                 return aesni_gmac_sgl_generate(s, &gdata_ctx, vec);
 565         case AESNI_GMAC_OP_VERIFY:
 566                 return aesni_gmac_sgl_verify(s, &gdata_ctx, vec);
 567         default:
 568                 aesni_gcm_fill_error_code(vec, EINVAL);
 569                 return 0;
 570         }
 571 }
 572
 573 /**
 574  * Process a completed job and return rte_mbuf which job processed
 575  *
 576  * @param job   JOB_AES_HMAC job to process
 577  *
 578  * @return
 579  * - Returns processed mbuf which is trimmed of output digest used in
 580  * verification of supplied digest in the case of a HASH_CIPHER operation
 581  * - Returns NULL on invalid job
 582  */
 583 static void
 584 post_process_gcm_crypto_op(struct aesni_gcm_qp *qp,
 585                 struct rte_crypto_op *op,
 586                 struct aesni_gcm_session *session)
 587 {
 588         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 589
 590         /* Verify digest if required */
 591         if (session->op == AESNI_GCM_OP_AUTHENTICATED_DECRYPTION ||
 592                         session->op == AESNI_GMAC_OP_VERIFY) {
 593                 uint8_t *digest;
 594
 595                 uint8_t *tag = qp->temp_digest;
 596
 597                 if (session->op == AESNI_GMAC_OP_VERIFY)
 598                         digest = op->sym->auth.digest.data;
 599                 else
 600                         digest = op->sym->aead.digest.data;
 601
 602 #ifdef RTE_LIBRTE_PMD_AESNI_GCM_DEBUG
 603                 rte_hexdump(stdout, "auth tag (orig):",
 604                                 digest, session->req_digest_length);
 605                 rte_hexdump(stdout, "auth tag (calc):",
 606                                 tag, session->req_digest_length);
 607 #endif
 608
 609                 if (memcmp(tag, digest, session->req_digest_length) != 0)
 610                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 611         } else {
 612                 if (session->req_digest_length != session->gen_digest_length) {
 613                         if (session->op == AESNI_GCM_OP_AUTHENTICATED_ENCRYPTION)
 614                                 memcpy(op->sym->aead.digest.data, qp->temp_digest,
 615                                                 session->req_digest_length);
 616                         else
 617                                 memcpy(op->sym->auth.digest.data, qp->temp_digest,
 618                                                 session->req_digest_length);
 619                 }
 620         }
 621 }
 622
 623 /**
 624  * Process a completed GCM request
 625  *
 626  * @param qp            Queue Pair to process
 627  * @param op            Crypto operation
 628  * @param job           JOB_AES_HMAC job
 629  *
 630  * @return
 631  * - Number of processed jobs
 632  */
 633 static void
 634 handle_completed_gcm_crypto_op(struct aesni_gcm_qp *qp,
 635                 struct rte_crypto_op *op,
 636                 struct aesni_gcm_session *sess)
 637 {
 638         post_process_gcm_crypto_op(qp, op, sess);
 639
 640         /* Free session if a session-less crypto op */
 641         if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
 642                 memset(sess, 0, sizeof(struct aesni_gcm_session));
 643                 memset(op->sym->session, 0,
 644                         rte_cryptodev_sym_get_existing_header_session_size(
 645                                 op->sym->session));
 646                 rte_mempool_put(qp->sess_mp_priv, sess);
 647                 rte_mempool_put(qp->sess_mp, op->sym->session);
 648                 op->sym->session = NULL;
 649         }
 650 }
 651
 652 static uint16_t
 653 aesni_gcm_pmd_dequeue_burst(void *queue_pair,
 654                 struct rte_crypto_op **ops, uint16_t nb_ops)
 655 {
 656         struct aesni_gcm_session *sess;
 657         struct aesni_gcm_qp *qp = queue_pair;
 658
 659         int retval = 0;
 660         unsigned int i, nb_dequeued;
 661
 662         nb_dequeued = rte_ring_dequeue_burst(qp->processed_pkts,
 663                         (void **)ops, nb_ops, NULL);
 664
 665         for (i = 0; i < nb_dequeued; i++) {
 666
 667                 sess = aesni_gcm_get_session(qp, ops[i]);
 668                 if (unlikely(sess == NULL)) {
 669                         ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 670                         qp->qp_stats.dequeue_err_count++;
 671                         break;
 672                 }
 673
 674                 retval = process_gcm_crypto_op(qp, ops[i], sess);
 675                 if (retval < 0) {
 676                         ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 677                         qp->qp_stats.dequeue_err_count++;
 678                         break;
 679                 }
 680
 681                 handle_completed_gcm_crypto_op(qp, ops[i], sess);
 682         }
 683
 684         qp->qp_stats.dequeued_count += i;
 685
 686         return i;
 687 }
 688
 689 static uint16_t
 690 aesni_gcm_pmd_enqueue_burst(void *queue_pair,
 691                 struct rte_crypto_op **ops, uint16_t nb_ops)
 692 {
 693         struct aesni_gcm_qp *qp = queue_pair;
 694
 695         unsigned int nb_enqueued;
 696
 697         nb_enqueued = rte_ring_enqueue_burst(qp->processed_pkts,
 698                         (void **)ops, nb_ops, NULL);
 699         qp->qp_stats.enqueued_count += nb_enqueued;
 700
 701         return nb_enqueued;
 702 }
 703
 704 static int aesni_gcm_remove(struct rte_vdev_device *vdev);
 705
 706 static int
 707 aesni_gcm_create(const char *name,
 708                 struct rte_vdev_device *vdev,
 709                 struct rte_cryptodev_pmd_init_params *init_params)
 710 {
 711         struct rte_cryptodev *dev;
 712         struct aesni_gcm_private *internals;
 713         enum aesni_gcm_vector_mode vector_mode;
 714         MB_MGR *mb_mgr;
 715
 716         dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
 717         if (dev == NULL) {
 718                 AESNI_GCM_LOG(ERR, "driver %s: create failed",
 719                         init_params->name);
 720                 return -ENODEV;
 721         }
 722
 723         /* Check CPU for supported vector instruction set */
 724         if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX512F))
 725                 vector_mode = RTE_AESNI_GCM_AVX512;
 726         else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2))
 727                 vector_mode = RTE_AESNI_GCM_AVX2;
 728         else if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX))
 729                 vector_mode = RTE_AESNI_GCM_AVX;
 730         else
 731                 vector_mode = RTE_AESNI_GCM_SSE;
 732
 733         dev->driver_id = cryptodev_driver_id;
 734         dev->dev_ops = rte_aesni_gcm_pmd_ops;
 735
 736         /* register rx/tx burst functions for data path */
 737         dev->dequeue_burst = aesni_gcm_pmd_dequeue_burst;
 738         dev->enqueue_burst = aesni_gcm_pmd_enqueue_burst;
 739
 740         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
 741                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
 742                         RTE_CRYPTODEV_FF_IN_PLACE_SGL |
 743                         RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
 744                         RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
 745                         RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO |
 746                         RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
 747
 748         /* Check CPU for support for AES instruction set */
 749         if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES))
 750                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AESNI;
 751         else
 752                 AESNI_GCM_LOG(WARNING, "AES instructions not supported by CPU");
 753
 754         mb_mgr = alloc_mb_mgr(0);
 755         if (mb_mgr == NULL)
 756                 return -ENOMEM;
 757
 758         switch (vector_mode) {
 759         case RTE_AESNI_GCM_SSE:
 760                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_SSE;
 761                 init_mb_mgr_sse(mb_mgr);
 762                 break;
 763         case RTE_AESNI_GCM_AVX:
 764                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX;
 765                 init_mb_mgr_avx(mb_mgr);
 766                 break;
 767         case RTE_AESNI_GCM_AVX2:
 768                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
 769                 init_mb_mgr_avx2(mb_mgr);
 770                 break;
 771         case RTE_AESNI_GCM_AVX512:
 772                 dev->feature_flags |= RTE_CRYPTODEV_FF_CPU_AVX2;
 773                 init_mb_mgr_avx512(mb_mgr);
 774                 break;
 775         default:
 776                 AESNI_GCM_LOG(ERR, "Unsupported vector mode %u\n", vector_mode);
 777                 goto error_exit;
 778         }
 779
 780         internals = dev->data->dev_private;
 781
 782         internals->vector_mode = vector_mode;
 783         internals->mb_mgr = mb_mgr;
 784
 785         /* Set arch independent function pointers, based on key size */
 786         internals->ops[GCM_KEY_128].enc = mb_mgr->gcm128_enc;
 787         internals->ops[GCM_KEY_128].dec = mb_mgr->gcm128_dec;
 788         internals->ops[GCM_KEY_128].pre = mb_mgr->gcm128_pre;
 789         internals->ops[GCM_KEY_128].init = mb_mgr->gcm128_init;
 790         internals->ops[GCM_KEY_128].update_enc = mb_mgr->gcm128_enc_update;
 791         internals->ops[GCM_KEY_128].update_dec = mb_mgr->gcm128_dec_update;
 792         internals->ops[GCM_KEY_128].finalize_enc = mb_mgr->gcm128_enc_finalize;
 793         internals->ops[GCM_KEY_128].finalize_dec = mb_mgr->gcm128_dec_finalize;
 794
 795         internals->ops[GCM_KEY_192].enc = mb_mgr->gcm192_enc;
 796         internals->ops[GCM_KEY_192].dec = mb_mgr->gcm192_dec;
 797         internals->ops[GCM_KEY_192].pre = mb_mgr->gcm192_pre;
 798         internals->ops[GCM_KEY_192].init = mb_mgr->gcm192_init;
 799         internals->ops[GCM_KEY_192].update_enc = mb_mgr->gcm192_enc_update;
 800         internals->ops[GCM_KEY_192].update_dec = mb_mgr->gcm192_dec_update;
 801         internals->ops[GCM_KEY_192].finalize_enc = mb_mgr->gcm192_enc_finalize;
 802         internals->ops[GCM_KEY_192].finalize_dec = mb_mgr->gcm192_dec_finalize;
 803
 804         internals->ops[GCM_KEY_256].enc = mb_mgr->gcm256_enc;
 805         internals->ops[GCM_KEY_256].dec = mb_mgr->gcm256_dec;
 806         internals->ops[GCM_KEY_256].pre = mb_mgr->gcm256_pre;
 807         internals->ops[GCM_KEY_256].init = mb_mgr->gcm256_init;
 808         internals->ops[GCM_KEY_256].update_enc = mb_mgr->gcm256_enc_update;
 809         internals->ops[GCM_KEY_256].update_dec = mb_mgr->gcm256_dec_update;
 810         internals->ops[GCM_KEY_256].finalize_enc = mb_mgr->gcm256_enc_finalize;
 811         internals->ops[GCM_KEY_256].finalize_dec = mb_mgr->gcm256_dec_finalize;
 812
 813         internals->max_nb_queue_pairs = init_params->max_nb_queue_pairs;
 814
 815 #if IMB_VERSION_NUM >= IMB_VERSION(0, 50, 0)
 816         AESNI_GCM_LOG(INFO, "IPSec Multi-buffer library version used: %s\n",
 817                         imb_get_version_str());
 818 #else
 819         AESNI_GCM_LOG(INFO, "IPSec Multi-buffer library version used: 0.49.0\n");
 820 #endif
 821
 822         return 0;
 823
 824 error_exit:
 825         if (mb_mgr)
 826                 free_mb_mgr(mb_mgr);
 827
 828         rte_cryptodev_pmd_destroy(dev);
 829
 830         return -1;
 831 }
 832
 833 static int
 834 aesni_gcm_probe(struct rte_vdev_device *vdev)
 835 {
 836         struct rte_cryptodev_pmd_init_params init_params = {
 837                 "",
 838                 sizeof(struct aesni_gcm_private),
 839                 rte_socket_id(),
 840                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
 841         };
 842         const char *name;
 843         const char *input_args;
 844
 845         name = rte_vdev_device_name(vdev);
 846         if (name == NULL)
 847                 return -EINVAL;
 848         input_args = rte_vdev_device_args(vdev);
 849         rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
 850
 851         return aesni_gcm_create(name, vdev, &init_params);
 852 }
 853
 854 static int
 855 aesni_gcm_remove(struct rte_vdev_device *vdev)
 856 {
 857         struct rte_cryptodev *cryptodev;
 858         struct aesni_gcm_private *internals;
 859         const char *name;
 860
 861         name = rte_vdev_device_name(vdev);
 862         if (name == NULL)
 863                 return -EINVAL;
 864
 865         cryptodev = rte_cryptodev_pmd_get_named_dev(name);
 866         if (cryptodev == NULL)
 867                 return -ENODEV;
 868
 869         internals = cryptodev->data->dev_private;
 870
 871         free_mb_mgr(internals->mb_mgr);
 872
 873         return rte_cryptodev_pmd_destroy(cryptodev);
 874 }
 875
 876 static struct rte_vdev_driver aesni_gcm_pmd_drv = {
 877         .probe = aesni_gcm_probe,
 878         .remove = aesni_gcm_remove
 879 };
 880
 881 static struct cryptodev_driver aesni_gcm_crypto_drv;
 882
 883 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_GCM_PMD, aesni_gcm_pmd_drv);
 884 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_GCM_PMD, cryptodev_aesni_gcm_pmd);
 885 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_GCM_PMD,
 886         "max_nb_queue_pairs=<int> "
 887         "socket_id=<int>");
 888 RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_gcm_crypto_drv, aesni_gcm_pmd_drv.driver,
 889                 cryptodev_driver_id);
 890 RTE_LOG_REGISTER(aesni_gcm_logtype_driver, pmd.crypto.aesni_gcm, NOTICE);