1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016-2021 Intel Corporation
5 #include "pmd_aesni_gcm_priv.h"
8 aesni_gcm_set_ops(struct aesni_gcm_ops *ops, IMB_MGR *mb_mgr)
10 /* Set 128 bit function pointers. */
11 ops[GCM_KEY_128].pre = mb_mgr->gcm128_pre;
12 ops[GCM_KEY_128].init = mb_mgr->gcm128_init;
14 ops[GCM_KEY_128].enc = mb_mgr->gcm128_enc;
15 ops[GCM_KEY_128].update_enc = mb_mgr->gcm128_enc_update;
16 ops[GCM_KEY_128].finalize_enc = mb_mgr->gcm128_enc_finalize;
18 ops[GCM_KEY_128].dec = mb_mgr->gcm128_dec;
19 ops[GCM_KEY_128].update_dec = mb_mgr->gcm128_dec_update;
20 ops[GCM_KEY_128].finalize_dec = mb_mgr->gcm128_dec_finalize;
22 ops[GCM_KEY_128].gmac_init = mb_mgr->gmac128_init;
23 ops[GCM_KEY_128].gmac_update = mb_mgr->gmac128_update;
24 ops[GCM_KEY_128].gmac_finalize = mb_mgr->gmac128_finalize;
26 /* Set 192 bit function pointers. */
27 ops[GCM_KEY_192].pre = mb_mgr->gcm192_pre;
28 ops[GCM_KEY_192].init = mb_mgr->gcm192_init;
30 ops[GCM_KEY_192].enc = mb_mgr->gcm192_enc;
31 ops[GCM_KEY_192].update_enc = mb_mgr->gcm192_enc_update;
32 ops[GCM_KEY_192].finalize_enc = mb_mgr->gcm192_enc_finalize;
34 ops[GCM_KEY_192].dec = mb_mgr->gcm192_dec;
35 ops[GCM_KEY_192].update_dec = mb_mgr->gcm192_dec_update;
36 ops[GCM_KEY_192].finalize_dec = mb_mgr->gcm192_dec_finalize;
38 ops[GCM_KEY_192].gmac_init = mb_mgr->gmac192_init;
39 ops[GCM_KEY_192].gmac_update = mb_mgr->gmac192_update;
40 ops[GCM_KEY_192].gmac_finalize = mb_mgr->gmac192_finalize;
42 /* Set 256 bit function pointers. */
43 ops[GCM_KEY_256].pre = mb_mgr->gcm256_pre;
44 ops[GCM_KEY_256].init = mb_mgr->gcm256_init;
46 ops[GCM_KEY_256].enc = mb_mgr->gcm256_enc;
47 ops[GCM_KEY_256].update_enc = mb_mgr->gcm256_enc_update;
48 ops[GCM_KEY_256].finalize_enc = mb_mgr->gcm256_enc_finalize;
50 ops[GCM_KEY_256].dec = mb_mgr->gcm256_dec;
51 ops[GCM_KEY_256].update_dec = mb_mgr->gcm256_dec_update;
52 ops[GCM_KEY_256].finalize_dec = mb_mgr->gcm256_dec_finalize;
54 ops[GCM_KEY_256].gmac_init = mb_mgr->gmac256_init;
55 ops[GCM_KEY_256].gmac_update = mb_mgr->gmac256_update;
56 ops[GCM_KEY_256].gmac_finalize = mb_mgr->gmac256_finalize;
60 aesni_gcm_session_configure(IMB_MGR *mb_mgr, void *session,
61 const struct rte_crypto_sym_xform *xform)
63 struct aesni_gcm_session *sess = session;
64 const struct rte_crypto_sym_xform *auth_xform;
65 const struct rte_crypto_sym_xform *cipher_xform;
66 const struct rte_crypto_sym_xform *aead_xform;
70 enum ipsec_mb_operation mode;
73 ret = ipsec_mb_parse_xform(xform, &mode, &auth_xform,
74 &cipher_xform, &aead_xform);
83 case IPSEC_MB_OP_HASH_GEN_ONLY:
84 case IPSEC_MB_OP_HASH_VERIFY_ONLY:
88 if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_AES_GMAC) {
90 "Only AES GMAC is supported as an authentication only algorithm");
94 /* Set IV parameters */
95 sess->iv.offset = auth_xform->auth.iv.offset;
96 sess->iv.length = auth_xform->auth.iv.length;
97 key_length = auth_xform->auth.key.length;
98 key = auth_xform->auth.key.data;
99 sess->req_digest_length = auth_xform->auth.digest_length;
101 case IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT:
102 case IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT:
104 * aead_xform = xform;
107 if (aead_xform->aead.algo != RTE_CRYPTO_AEAD_AES_GCM) {
109 "The only combined operation supported is AES GCM");
113 /* Set IV parameters */
114 sess->iv.offset = aead_xform->aead.iv.offset;
115 sess->iv.length = aead_xform->aead.iv.length;
116 key_length = aead_xform->aead.key.length;
117 key = aead_xform->aead.key.data;
118 sess->aad_length = aead_xform->aead.aad_length;
119 sess->req_digest_length = aead_xform->aead.digest_length;
123 ERR, "Wrong xform type, has to be AEAD or authentication");
128 /* Check key length, and calculate GCM pre-compute. */
129 switch (key_length) {
131 sess->key_length = GCM_KEY_128;
132 mb_mgr->gcm128_pre(key, &sess->gdata_key);
135 sess->key_length = GCM_KEY_192;
136 mb_mgr->gcm192_pre(key, &sess->gdata_key);
139 sess->key_length = GCM_KEY_256;
140 mb_mgr->gcm256_pre(key, &sess->gdata_key);
143 IPSEC_MB_LOG(ERR, "Invalid key length");
149 if (sess->req_digest_length > 16) {
150 IPSEC_MB_LOG(ERR, "Invalid digest length");
155 * If size requested is different, generate the full digest
156 * (16 bytes) in a temporary location and then memcpy
157 * the requested number of bytes.
159 if (sess->req_digest_length < 4)
160 sess->gen_digest_length = 16;
162 sess->gen_digest_length = sess->req_digest_length;
169 * Process a completed job and return rte_mbuf which job processed
171 * @param job IMB_JOB job to process
174 * - Returns processed mbuf which is trimmed of output digest used in
175 * verification of supplied digest in the case of a HASH_CIPHER operation
176 * - Returns NULL on invalid job
179 post_process_gcm_crypto_op(struct ipsec_mb_qp *qp,
180 struct rte_crypto_op *op,
181 struct aesni_gcm_session *session)
183 struct aesni_gcm_qp_data *qp_data = ipsec_mb_get_qp_private_data(qp);
185 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
186 /* Verify digest if required */
187 if (session->op == IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT ||
188 session->op == IPSEC_MB_OP_HASH_VERIFY_ONLY) {
191 uint8_t *tag = qp_data->temp_digest;
193 if (session->op == IPSEC_MB_OP_HASH_VERIFY_ONLY)
194 digest = op->sym->auth.digest.data;
196 digest = op->sym->aead.digest.data;
198 #ifdef RTE_LIBRTE_PMD_AESNI_GCM_DEBUG
199 rte_hexdump(stdout, "auth tag (orig):",
200 digest, session->req_digest_length);
201 rte_hexdump(stdout, "auth tag (calc):",
202 tag, session->req_digest_length);
205 if (memcmp(tag, digest, session->req_digest_length) != 0)
206 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
208 if (session->req_digest_length != session->gen_digest_length) {
210 IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT)
211 memcpy(op->sym->aead.digest.data,
212 qp_data->temp_digest,
213 session->req_digest_length);
215 memcpy(op->sym->auth.digest.data,
216 qp_data->temp_digest,
217 session->req_digest_length);
223 * Process a completed GCM request
225 * @param qp Queue Pair to process
226 * @param op Crypto operation
227 * @param sess AESNI-GCM session
231 handle_completed_gcm_crypto_op(struct ipsec_mb_qp *qp,
232 struct rte_crypto_op *op,
233 struct aesni_gcm_session *sess)
235 post_process_gcm_crypto_op(qp, op, sess);
237 /* Free session if a session-less crypto op */
238 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
239 memset(sess, 0, sizeof(struct aesni_gcm_session));
240 memset(op->sym->session, 0,
241 rte_cryptodev_sym_get_existing_header_session_size(
243 rte_mempool_put(qp->sess_mp_priv, sess);
244 rte_mempool_put(qp->sess_mp, op->sym->session);
245 op->sym->session = NULL;
250 * Process a crypto operation, calling
251 * the GCM API from the multi buffer library.
253 * @param qp queue pair
254 * @param op symmetric crypto operation
255 * @param session GCM session
261 process_gcm_crypto_op(struct ipsec_mb_qp *qp, struct rte_crypto_op *op,
262 struct aesni_gcm_session *session)
264 struct aesni_gcm_qp_data *qp_data = ipsec_mb_get_qp_private_data(qp);
267 struct rte_crypto_sym_op *sym_op = op->sym;
268 struct rte_mbuf *m_src = sym_op->m_src;
269 uint32_t offset, data_offset, data_length;
270 uint32_t part_len, total_len, data_len;
272 unsigned int oop = 0;
273 struct aesni_gcm_ops *ops = &qp_data->ops[session->key_length];
275 if (session->op == IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT ||
276 session->op == IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT) {
277 offset = sym_op->aead.data.offset;
278 data_offset = offset;
279 data_length = sym_op->aead.data.length;
281 offset = sym_op->auth.data.offset;
282 data_offset = offset;
283 data_length = sym_op->auth.data.length;
286 RTE_ASSERT(m_src != NULL);
288 while (offset >= m_src->data_len && data_length != 0) {
289 offset -= m_src->data_len;
292 RTE_ASSERT(m_src != NULL);
295 src = rte_pktmbuf_mtod_offset(m_src, uint8_t *, offset);
297 data_len = m_src->data_len - offset;
298 part_len = (data_len < data_length) ? data_len :
301 RTE_ASSERT((sym_op->m_dst == NULL) ||
302 ((sym_op->m_dst != NULL) &&
303 rte_pktmbuf_is_contiguous(sym_op->m_dst)));
306 if (sym_op->m_dst == NULL || (sym_op->m_dst == sym_op->m_src))
311 /* Segmented destination buffer is not supported
312 * if operation is Out-of-place
314 RTE_ASSERT(rte_pktmbuf_is_contiguous(sym_op->m_dst));
315 dst = rte_pktmbuf_mtod_offset(sym_op->m_dst, uint8_t *,
319 iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
322 if (session->op == IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT) {
323 ops->init(&session->gdata_key, &qp_data->gcm_ctx_data, iv_ptr,
324 sym_op->aead.aad.data,
325 (uint64_t)session->aad_length);
327 ops->update_enc(&session->gdata_key, &qp_data->gcm_ctx_data,
328 dst, src, (uint64_t)part_len);
329 total_len = data_length - part_len;
334 RTE_ASSERT(m_src != NULL);
336 src = rte_pktmbuf_mtod(m_src, uint8_t *);
341 part_len = (m_src->data_len < total_len) ?
342 m_src->data_len : total_len;
344 ops->update_enc(&session->gdata_key,
345 &qp_data->gcm_ctx_data,
346 dst, src, (uint64_t)part_len);
347 total_len -= part_len;
350 if (session->req_digest_length != session->gen_digest_length)
351 tag = qp_data->temp_digest;
353 tag = sym_op->aead.digest.data;
355 ops->finalize_enc(&session->gdata_key, &qp_data->gcm_ctx_data,
356 tag, session->gen_digest_length);
357 } else if (session->op == IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT) {
358 ops->init(&session->gdata_key, &qp_data->gcm_ctx_data, iv_ptr,
359 sym_op->aead.aad.data,
360 (uint64_t)session->aad_length);
362 ops->update_dec(&session->gdata_key, &qp_data->gcm_ctx_data,
363 dst, src, (uint64_t)part_len);
364 total_len = data_length - part_len;
369 RTE_ASSERT(m_src != NULL);
371 src = rte_pktmbuf_mtod(m_src, uint8_t *);
376 part_len = (m_src->data_len < total_len) ?
377 m_src->data_len : total_len;
379 ops->update_dec(&session->gdata_key,
380 &qp_data->gcm_ctx_data,
381 dst, src, (uint64_t)part_len);
382 total_len -= part_len;
385 tag = qp_data->temp_digest;
386 ops->finalize_dec(&session->gdata_key, &qp_data->gcm_ctx_data,
387 tag, session->gen_digest_length);
388 } else if (session->op == IPSEC_MB_OP_HASH_GEN_ONLY) {
389 ops->gmac_init(&session->gdata_key, &qp_data->gcm_ctx_data,
390 iv_ptr, session->iv.length);
392 ops->gmac_update(&session->gdata_key, &qp_data->gcm_ctx_data,
393 src, (uint64_t)part_len);
394 total_len = data_length - part_len;
399 RTE_ASSERT(m_src != NULL);
401 src = rte_pktmbuf_mtod(m_src, uint8_t *);
402 part_len = (m_src->data_len < total_len) ?
403 m_src->data_len : total_len;
405 ops->gmac_update(&session->gdata_key,
406 &qp_data->gcm_ctx_data, src,
408 total_len -= part_len;
411 if (session->req_digest_length != session->gen_digest_length)
412 tag = qp_data->temp_digest;
414 tag = sym_op->auth.digest.data;
416 ops->gmac_finalize(&session->gdata_key, &qp_data->gcm_ctx_data,
417 tag, session->gen_digest_length);
418 } else { /* IPSEC_MB_OP_HASH_VERIFY_ONLY */
419 ops->gmac_init(&session->gdata_key, &qp_data->gcm_ctx_data,
420 iv_ptr, session->iv.length);
422 ops->gmac_update(&session->gdata_key, &qp_data->gcm_ctx_data,
423 src, (uint64_t)part_len);
424 total_len = data_length - part_len;
429 RTE_ASSERT(m_src != NULL);
431 src = rte_pktmbuf_mtod(m_src, uint8_t *);
432 part_len = (m_src->data_len < total_len) ?
433 m_src->data_len : total_len;
435 ops->gmac_update(&session->gdata_key,
436 &qp_data->gcm_ctx_data, src,
438 total_len -= part_len;
441 tag = qp_data->temp_digest;
443 ops->gmac_finalize(&session->gdata_key, &qp_data->gcm_ctx_data,
444 tag, session->gen_digest_length);
449 /** Get gcm session */
450 static inline struct aesni_gcm_session *
451 aesni_gcm_get_session(struct ipsec_mb_qp *qp,
452 struct rte_crypto_op *op)
454 struct aesni_gcm_session *sess = NULL;
456 ipsec_mb_get_driver_id(IPSEC_MB_PMD_TYPE_AESNI_GCM);
457 struct rte_crypto_sym_op *sym_op = op->sym;
459 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
460 if (likely(sym_op->session != NULL))
461 sess = (struct aesni_gcm_session *)
462 get_sym_session_private_data(sym_op->session,
466 void *_sess_private_data = NULL;
468 if (rte_mempool_get(qp->sess_mp, (void **)&_sess))
471 if (rte_mempool_get(qp->sess_mp_priv,
472 (void **)&_sess_private_data))
475 sess = (struct aesni_gcm_session *)_sess_private_data;
477 if (unlikely(aesni_gcm_session_configure(qp->mb_mgr,
478 _sess_private_data, sym_op->xform) != 0)) {
479 rte_mempool_put(qp->sess_mp, _sess);
480 rte_mempool_put(qp->sess_mp_priv, _sess_private_data);
483 sym_op->session = (struct rte_cryptodev_sym_session *)_sess;
484 set_sym_session_private_data(sym_op->session, driver_id,
488 if (unlikely(sess == NULL))
489 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
495 aesni_gcm_pmd_dequeue_burst(void *queue_pair,
496 struct rte_crypto_op **ops, uint16_t nb_ops)
498 struct aesni_gcm_session *sess;
499 struct ipsec_mb_qp *qp = queue_pair;
502 unsigned int i, nb_dequeued;
504 nb_dequeued = rte_ring_dequeue_burst(qp->ingress_queue,
505 (void **)ops, nb_ops, NULL);
507 for (i = 0; i < nb_dequeued; i++) {
509 sess = aesni_gcm_get_session(qp, ops[i]);
510 if (unlikely(sess == NULL)) {
511 ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
512 qp->stats.dequeue_err_count++;
516 retval = process_gcm_crypto_op(qp, ops[i], sess);
518 ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
519 qp->stats.dequeue_err_count++;
523 handle_completed_gcm_crypto_op(qp, ops[i], sess);
526 qp->stats.dequeued_count += i;
532 aesni_gcm_fill_error_code(struct rte_crypto_sym_vec *vec,
537 for (i = 0; i < vec->num; i++)
538 vec->status[i] = errnum;
541 static inline int32_t
542 aesni_gcm_sgl_op_finalize_encryption(const struct aesni_gcm_session *s,
543 struct gcm_context_data *gdata_ctx,
544 uint8_t *digest, struct aesni_gcm_ops ops)
546 if (s->req_digest_length != s->gen_digest_length) {
547 uint8_t tmpdigest[s->gen_digest_length];
549 ops.finalize_enc(&s->gdata_key, gdata_ctx, tmpdigest,
550 s->gen_digest_length);
551 memcpy(digest, tmpdigest, s->req_digest_length);
553 ops.finalize_enc(&s->gdata_key, gdata_ctx, digest,
554 s->gen_digest_length);
560 static inline int32_t
561 aesni_gcm_sgl_op_finalize_decryption(const struct aesni_gcm_session *s,
562 struct gcm_context_data *gdata_ctx,
563 uint8_t *digest, struct aesni_gcm_ops ops)
565 uint8_t tmpdigest[s->gen_digest_length];
567 ops.finalize_dec(&s->gdata_key, gdata_ctx, tmpdigest,
568 s->gen_digest_length);
570 return memcmp(digest, tmpdigest, s->req_digest_length) == 0 ? 0
575 aesni_gcm_process_gcm_sgl_op(const struct aesni_gcm_session *s,
576 struct gcm_context_data *gdata_ctx,
577 struct rte_crypto_sgl *sgl, void *iv, void *aad,
578 struct aesni_gcm_ops ops)
582 /* init crypto operation */
583 ops.init(&s->gdata_key, gdata_ctx, iv, aad,
584 (uint64_t)s->aad_length);
586 /* update with sgl data */
587 for (i = 0; i < sgl->num; i++) {
588 struct rte_crypto_vec *vec = &sgl->vec[i];
591 case IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT:
592 ops.update_enc(&s->gdata_key, gdata_ctx,
593 vec->base, vec->base, vec->len);
595 case IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT:
596 ops.update_dec(&s->gdata_key, gdata_ctx,
597 vec->base, vec->base, vec->len);
600 IPSEC_MB_LOG(ERR, "Invalid session op");
608 aesni_gcm_process_gmac_sgl_op(const struct aesni_gcm_session *s,
609 struct gcm_context_data *gdata_ctx,
610 struct rte_crypto_sgl *sgl, void *iv,
611 struct aesni_gcm_ops ops)
613 ops.init(&s->gdata_key, gdata_ctx, iv, sgl->vec[0].base,
617 static inline uint32_t
618 aesni_gcm_sgl_encrypt(struct aesni_gcm_session *s,
619 struct gcm_context_data *gdata_ctx,
620 struct rte_crypto_sym_vec *vec,
621 struct aesni_gcm_ops ops)
623 uint32_t i, processed;
626 for (i = 0; i < vec->num; ++i) {
627 aesni_gcm_process_gcm_sgl_op(s, gdata_ctx, &vec->src_sgl[i],
628 vec->iv[i].va, vec->aad[i].va,
630 vec->status[i] = aesni_gcm_sgl_op_finalize_encryption(
631 s, gdata_ctx, vec->digest[i].va, ops);
632 processed += (vec->status[i] == 0);
638 static inline uint32_t
639 aesni_gcm_sgl_decrypt(struct aesni_gcm_session *s,
640 struct gcm_context_data *gdata_ctx,
641 struct rte_crypto_sym_vec *vec,
642 struct aesni_gcm_ops ops)
644 uint32_t i, processed;
647 for (i = 0; i < vec->num; ++i) {
648 aesni_gcm_process_gcm_sgl_op(s, gdata_ctx, &vec->src_sgl[i],
649 vec->iv[i].va, vec->aad[i].va,
651 vec->status[i] = aesni_gcm_sgl_op_finalize_decryption(
652 s, gdata_ctx, vec->digest[i].va, ops);
653 processed += (vec->status[i] == 0);
659 static inline uint32_t
660 aesni_gmac_sgl_generate(struct aesni_gcm_session *s,
661 struct gcm_context_data *gdata_ctx,
662 struct rte_crypto_sym_vec *vec,
663 struct aesni_gcm_ops ops)
665 uint32_t i, processed;
668 for (i = 0; i < vec->num; ++i) {
669 if (vec->src_sgl[i].num != 1) {
670 vec->status[i] = ENOTSUP;
674 aesni_gcm_process_gmac_sgl_op(s, gdata_ctx, &vec->src_sgl[i],
676 vec->status[i] = aesni_gcm_sgl_op_finalize_encryption(
677 s, gdata_ctx, vec->digest[i].va, ops);
678 processed += (vec->status[i] == 0);
684 static inline uint32_t
685 aesni_gmac_sgl_verify(struct aesni_gcm_session *s,
686 struct gcm_context_data *gdata_ctx,
687 struct rte_crypto_sym_vec *vec,
688 struct aesni_gcm_ops ops)
690 uint32_t i, processed;
693 for (i = 0; i < vec->num; ++i) {
694 if (vec->src_sgl[i].num != 1) {
695 vec->status[i] = ENOTSUP;
699 aesni_gcm_process_gmac_sgl_op(s, gdata_ctx, &vec->src_sgl[i],
701 vec->status[i] = aesni_gcm_sgl_op_finalize_decryption(
702 s, gdata_ctx, vec->digest[i].va, ops);
703 processed += (vec->status[i] == 0);
709 /** Process CPU crypto bulk operations */
711 aesni_gcm_process_bulk(struct rte_cryptodev *dev,
712 struct rte_cryptodev_sym_session *sess,
713 __rte_unused union rte_crypto_sym_ofs ofs,
714 struct rte_crypto_sym_vec *vec)
717 struct aesni_gcm_session *s;
718 struct gcm_context_data gdata_ctx;
721 sess_priv = get_sym_session_private_data(sess, dev->driver_id);
722 if (unlikely(sess_priv == NULL)) {
723 aesni_gcm_fill_error_code(vec, EINVAL);
729 /* get per-thread MB MGR, create one if needed */
730 mb_mgr = get_per_thread_mb_mgr();
731 if (unlikely(mb_mgr == NULL))
734 /* Check if function pointers have been set for this thread ops. */
735 if (unlikely(RTE_PER_LCORE(gcm_ops)[s->key_length].init == NULL))
736 aesni_gcm_set_ops(RTE_PER_LCORE(gcm_ops), mb_mgr);
739 case IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT:
740 return aesni_gcm_sgl_encrypt(s, &gdata_ctx, vec,
741 RTE_PER_LCORE(gcm_ops)[s->key_length]);
742 case IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT:
743 return aesni_gcm_sgl_decrypt(s, &gdata_ctx, vec,
744 RTE_PER_LCORE(gcm_ops)[s->key_length]);
745 case IPSEC_MB_OP_HASH_GEN_ONLY:
746 return aesni_gmac_sgl_generate(s, &gdata_ctx, vec,
747 RTE_PER_LCORE(gcm_ops)[s->key_length]);
748 case IPSEC_MB_OP_HASH_VERIFY_ONLY:
749 return aesni_gmac_sgl_verify(s, &gdata_ctx, vec,
750 RTE_PER_LCORE(gcm_ops)[s->key_length]);
752 aesni_gcm_fill_error_code(vec, EINVAL);
758 aesni_gcm_qp_setup(struct rte_cryptodev *dev, uint16_t qp_id,
759 const struct rte_cryptodev_qp_conf *qp_conf,
762 int ret = ipsec_mb_qp_setup(dev, qp_id, qp_conf, socket_id);
766 struct ipsec_mb_qp *qp = dev->data->queue_pairs[qp_id];
767 struct aesni_gcm_qp_data *qp_data = ipsec_mb_get_qp_private_data(qp);
768 aesni_gcm_set_ops(qp_data->ops, qp->mb_mgr);
772 struct rte_cryptodev_ops aesni_gcm_pmd_ops = {
773 .dev_configure = ipsec_mb_config,
774 .dev_start = ipsec_mb_start,
775 .dev_stop = ipsec_mb_stop,
776 .dev_close = ipsec_mb_close,
778 .stats_get = ipsec_mb_stats_get,
779 .stats_reset = ipsec_mb_stats_reset,
781 .dev_infos_get = ipsec_mb_info_get,
783 .queue_pair_setup = aesni_gcm_qp_setup,
784 .queue_pair_release = ipsec_mb_qp_release,
786 .sym_cpu_process = aesni_gcm_process_bulk,
788 .sym_session_get_size = ipsec_mb_sym_session_get_size,
789 .sym_session_configure = ipsec_mb_sym_session_configure,
790 .sym_session_clear = ipsec_mb_sym_session_clear
794 aesni_gcm_probe(struct rte_vdev_device *vdev)
796 return ipsec_mb_create(vdev, IPSEC_MB_PMD_TYPE_AESNI_GCM);
799 static struct rte_vdev_driver cryptodev_aesni_gcm_pmd_drv = {
800 .probe = aesni_gcm_probe,
801 .remove = ipsec_mb_remove
804 static struct cryptodev_driver aesni_gcm_crypto_drv;
806 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_AESNI_GCM_PMD,
807 cryptodev_aesni_gcm_pmd_drv);
808 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_AESNI_GCM_PMD, cryptodev_aesni_gcm_pmd);
809 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_AESNI_GCM_PMD,
810 "max_nb_queue_pairs=<int> socket_id=<int>");
811 RTE_PMD_REGISTER_CRYPTO_DRIVER(aesni_gcm_crypto_drv,
812 cryptodev_aesni_gcm_pmd_drv.driver,
813 pmd_driver_id_aesni_gcm);
815 /* Constructor function to register aesni-gcm PMD */
816 RTE_INIT(ipsec_mb_register_aesni_gcm)
818 struct ipsec_mb_internals *aesni_gcm_data =
819 &ipsec_mb_pmds[IPSEC_MB_PMD_TYPE_AESNI_GCM];
821 aesni_gcm_data->caps = aesni_gcm_capabilities;
822 aesni_gcm_data->dequeue_burst = aesni_gcm_pmd_dequeue_burst;
823 aesni_gcm_data->feature_flags =
824 RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
825 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
826 RTE_CRYPTODEV_FF_IN_PLACE_SGL |
827 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
828 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
829 RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO |
830 RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
831 aesni_gcm_data->internals_priv_size = 0;
832 aesni_gcm_data->ops = &aesni_gcm_pmd_ops;
833 aesni_gcm_data->qp_priv_size = sizeof(struct aesni_gcm_qp_data);
834 aesni_gcm_data->queue_pair_configure = NULL;
835 aesni_gcm_data->session_configure = aesni_gcm_session_configure;
836 aesni_gcm_data->session_priv_size = sizeof(struct aesni_gcm_session);