1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2017-2022 Intel Corporation
5 #ifndef _QAT_CRYPTO_PMD_GENS_H_
6 #define _QAT_CRYPTO_PMD_GENS_H_
8 #include <rte_cryptodev.h>
9 #include "qat_crypto.h"
10 #include "qat_sym_session.h"
13 #define QAT_SYM_DP_GET_MAX_ENQ(q, c, n) \
14 RTE_MIN((q->max_inflights - q->enqueued + q->dequeued - c), n)
16 #define QAT_SYM_DP_IS_RESP_SUCCESS(resp) \
17 (ICP_QAT_FW_COMN_STATUS_FLAG_OK == \
18 ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(resp->comn_hdr.comn_status))
20 static __rte_always_inline int
21 op_bpi_cipher_decrypt(uint8_t *src, uint8_t *dst,
22 uint8_t *iv, int ivlen, int srclen,
25 EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
27 uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
28 uint8_t *encr = encrypted_iv;
30 /* ECB method: encrypt (not decrypt!) the IV, then XOR with plaintext */
31 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
33 goto cipher_decrypt_err;
35 for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
41 QAT_DP_LOG(ERR, "libcrypto ECB cipher decrypt for BPI IV failed");
45 static __rte_always_inline uint32_t
46 qat_bpicipher_preprocess(struct qat_sym_session *ctx,
47 struct rte_crypto_op *op)
49 int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
50 struct rte_crypto_sym_op *sym_op = op->sym;
51 uint8_t last_block_len = block_len > 0 ?
52 sym_op->cipher.data.length % block_len : 0;
54 if (last_block_len && ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT) {
55 /* Decrypt last block */
56 uint8_t *last_block, *dst, *iv;
57 uint32_t last_block_offset = sym_op->cipher.data.offset +
58 sym_op->cipher.data.length - last_block_len;
59 last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
60 uint8_t *, last_block_offset);
62 if (unlikely((sym_op->m_dst != NULL)
63 && (sym_op->m_dst != sym_op->m_src)))
64 /* out-of-place operation (OOP) */
65 dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
66 uint8_t *, last_block_offset);
70 if (last_block_len < sym_op->cipher.data.length)
71 /* use previous block ciphertext as IV */
72 iv = last_block - block_len;
74 /* runt block, i.e. less than one full block */
75 iv = rte_crypto_op_ctod_offset(op, uint8_t *,
76 ctx->cipher_iv.offset);
78 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
79 QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src before pre-process:",
80 last_block, last_block_len);
81 if (sym_op->m_dst != NULL)
82 QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst before pre-process:",
85 op_bpi_cipher_decrypt(last_block, dst, iv, block_len,
86 last_block_len, ctx->bpi_ctx);
87 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
88 QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src after pre-process:",
89 last_block, last_block_len);
90 if (sym_op->m_dst != NULL)
91 QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst after pre-process:",
96 return sym_op->cipher.data.length - last_block_len;
99 static __rte_always_inline int
100 qat_auth_is_len_in_bits(struct qat_sym_session *ctx,
101 struct rte_crypto_op *op)
103 if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 ||
104 ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 ||
105 ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) {
106 if (unlikely((op->sym->auth.data.offset % BYTE_LENGTH != 0) ||
107 (op->sym->auth.data.length % BYTE_LENGTH != 0)))
114 static __rte_always_inline int
115 qat_cipher_is_len_in_bits(struct qat_sym_session *ctx,
116 struct rte_crypto_op *op)
118 if (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
119 ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI ||
120 ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
121 if (unlikely((op->sym->cipher.data.length % BYTE_LENGTH != 0) ||
122 ((op->sym->cipher.data.offset %
130 static __rte_always_inline int32_t
131 qat_sym_build_req_set_data(struct icp_qat_fw_la_bulk_req *req,
132 void *opaque, struct qat_sym_op_cookie *cookie,
133 struct rte_crypto_vec *src_vec, uint16_t n_src,
134 struct rte_crypto_vec *dst_vec, uint16_t n_dst)
136 struct qat_sgl *list;
138 uint32_t tl_src = 0, total_len_src, total_len_dst;
139 uint64_t src_data_start = 0, dst_data_start = 0;
140 int is_sgl = n_src > 1 || n_dst > 1;
142 if (unlikely(n_src < 1 || n_src > QAT_SYM_SGL_MAX_NUMBER ||
143 n_dst > QAT_SYM_SGL_MAX_NUMBER))
146 if (likely(!is_sgl)) {
147 src_data_start = src_vec[0].iova;
148 tl_src = total_len_src =
150 if (unlikely(n_dst)) { /* oop */
151 total_len_dst = dst_vec[0].len;
153 dst_data_start = dst_vec[0].iova;
154 if (unlikely(total_len_src != total_len_dst))
157 dst_data_start = src_data_start;
158 total_len_dst = tl_src;
161 total_len_dst = total_len_src = 0;
163 ICP_QAT_FW_COMN_PTR_TYPE_SET(req->comn_hdr.comn_req_flags,
164 QAT_COMN_PTR_TYPE_SGL);
166 list = (struct qat_sgl *)&cookie->qat_sgl_src;
167 for (i = 0; i < n_src; i++) {
168 list->buffers[i].len = src_vec[i].len;
169 list->buffers[i].resrvd = 0;
170 list->buffers[i].addr = src_vec[i].iova;
171 if (tl_src + src_vec[i].len > UINT32_MAX) {
172 QAT_DP_LOG(ERR, "Message too long");
175 tl_src += src_vec[i].len;
179 src_data_start = cookie->qat_sgl_src_phys_addr;
181 if (unlikely(n_dst > 0)) { /* oop sgl */
184 list = (struct qat_sgl *)&cookie->qat_sgl_dst;
186 for (i = 0; i < n_dst; i++) {
187 list->buffers[i].len = dst_vec[i].len;
188 list->buffers[i].resrvd = 0;
189 list->buffers[i].addr = dst_vec[i].iova;
190 if (tl_dst + dst_vec[i].len > UINT32_MAX) {
191 QAT_DP_LOG(ERR, "Message too long");
195 tl_dst += dst_vec[i].len;
198 if (tl_src != tl_dst)
201 dst_data_start = cookie->qat_sgl_dst_phys_addr;
203 dst_data_start = src_data_start;
206 req->comn_mid.src_data_addr = src_data_start;
207 req->comn_mid.dest_data_addr = dst_data_start;
208 req->comn_mid.src_length = total_len_src;
209 req->comn_mid.dst_length = total_len_dst;
210 req->comn_mid.opaque_data = (uintptr_t)opaque;
215 static __rte_always_inline uint64_t
216 qat_sym_convert_op_to_vec_cipher(struct rte_crypto_op *op,
217 struct qat_sym_session *ctx,
218 struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
219 struct rte_crypto_va_iova_ptr *cipher_iv,
220 struct rte_crypto_va_iova_ptr *auth_iv_or_aad __rte_unused,
221 struct rte_crypto_va_iova_ptr *digest __rte_unused)
223 uint32_t cipher_len = 0, cipher_ofs = 0;
227 ret = qat_cipher_is_len_in_bits(ctx, op);
230 cipher_len = op->sym->cipher.data.length >> 3;
231 cipher_ofs = op->sym->cipher.data.offset >> 3;
235 /* DOCSIS - only send complete blocks to device.
236 * Process any partial block using CFB mode.
237 * Even if 0 complete blocks, still send this to device
238 * to get into rx queue for post-process and dequeuing
240 cipher_len = qat_bpicipher_preprocess(ctx, op);
241 cipher_ofs = op->sym->cipher.data.offset;
243 cipher_len = op->sym->cipher.data.length;
244 cipher_ofs = op->sym->cipher.data.offset;
249 "SNOW3G/KASUMI/ZUC in QAT PMD only supports byte aligned values");
250 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
254 cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
255 ctx->cipher_iv.offset);
256 cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
257 ctx->cipher_iv.offset);
259 n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, cipher_ofs,
260 cipher_len, in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
261 if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
262 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
268 /* Out-Of-Place operation */
269 if (unlikely((op->sym->m_dst != NULL) &&
270 (op->sym->m_dst != op->sym->m_src))) {
271 int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, cipher_ofs,
272 cipher_len, out_sgl->vec,
273 QAT_SYM_SGL_MAX_NUMBER);
275 if ((n_dst < 0) || (n_dst > op->sym->m_dst->nb_segs)) {
276 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
280 out_sgl->num = n_dst;
287 static __rte_always_inline uint64_t
288 qat_sym_convert_op_to_vec_auth(struct rte_crypto_op *op,
289 struct qat_sym_session *ctx,
290 struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
291 struct rte_crypto_va_iova_ptr *cipher_iv __rte_unused,
292 struct rte_crypto_va_iova_ptr *auth_iv,
293 struct rte_crypto_va_iova_ptr *digest)
295 uint32_t auth_ofs = 0, auth_len = 0;
298 ret = qat_auth_is_len_in_bits(ctx, op);
301 auth_ofs = op->sym->auth.data.offset >> 3;
302 auth_len = op->sym->auth.data.length >> 3;
303 auth_iv->va = rte_crypto_op_ctod_offset(op, void *,
304 ctx->auth_iv.offset);
305 auth_iv->iova = rte_crypto_op_ctophys_offset(op,
306 ctx->auth_iv.offset);
309 if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
310 ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
312 auth_ofs = op->sym->auth.data.offset;
313 auth_len = op->sym->auth.data.length;
314 auth_iv->va = rte_crypto_op_ctod_offset(op, void *,
315 ctx->auth_iv.offset);
316 auth_iv->iova = rte_crypto_op_ctophys_offset(op,
317 ctx->auth_iv.offset);
319 auth_ofs = op->sym->auth.data.offset;
320 auth_len = op->sym->auth.data.length;
327 "For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
328 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
332 n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, auth_ofs,
333 auth_ofs + auth_len, in_sgl->vec,
334 QAT_SYM_SGL_MAX_NUMBER);
335 if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
336 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
342 /* Out-Of-Place operation */
343 if (unlikely((op->sym->m_dst != NULL) &&
344 (op->sym->m_dst != op->sym->m_src))) {
345 int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, auth_ofs,
346 auth_ofs + auth_len, out_sgl->vec,
347 QAT_SYM_SGL_MAX_NUMBER);
349 if ((n_dst < 0) || (n_dst > op->sym->m_dst->nb_segs)) {
350 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
353 out_sgl->num = n_dst;
357 digest->va = (void *)op->sym->auth.digest.data;
358 digest->iova = op->sym->auth.digest.phys_addr;
363 static __rte_always_inline uint64_t
364 qat_sym_convert_op_to_vec_chain(struct rte_crypto_op *op,
365 struct qat_sym_session *ctx,
366 struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
367 struct rte_crypto_va_iova_ptr *cipher_iv,
368 struct rte_crypto_va_iova_ptr *auth_iv_or_aad,
369 struct rte_crypto_va_iova_ptr *digest)
371 union rte_crypto_sym_ofs ofs;
372 uint32_t min_ofs = 0, max_len = 0;
373 uint32_t cipher_len = 0, cipher_ofs = 0;
374 uint32_t auth_len = 0, auth_ofs = 0;
375 int is_oop = (op->sym->m_dst != NULL) &&
376 (op->sym->m_dst != op->sym->m_src);
377 int is_sgl = op->sym->m_src->nb_segs > 1;
381 if (unlikely(is_oop))
382 is_sgl |= op->sym->m_dst->nb_segs > 1;
384 cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
385 ctx->cipher_iv.offset);
386 cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
387 ctx->cipher_iv.offset);
388 auth_iv_or_aad->va = rte_crypto_op_ctod_offset(op, void *,
389 ctx->auth_iv.offset);
390 auth_iv_or_aad->iova = rte_crypto_op_ctophys_offset(op,
391 ctx->auth_iv.offset);
392 digest->va = (void *)op->sym->auth.digest.data;
393 digest->iova = op->sym->auth.digest.phys_addr;
395 ret = qat_cipher_is_len_in_bits(ctx, op);
398 cipher_len = op->sym->aead.data.length >> 3;
399 cipher_ofs = op->sym->aead.data.offset >> 3;
402 cipher_len = op->sym->aead.data.length;
403 cipher_ofs = op->sym->aead.data.offset;
407 "For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
408 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
412 ret = qat_auth_is_len_in_bits(ctx, op);
415 auth_len = op->sym->auth.data.length >> 3;
416 auth_ofs = op->sym->auth.data.offset >> 3;
419 auth_len = op->sym->auth.data.length;
420 auth_ofs = op->sym->auth.data.offset;
424 "For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
425 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
429 min_ofs = cipher_ofs < auth_ofs ? cipher_ofs : auth_ofs;
430 max_len = RTE_MAX(cipher_ofs + cipher_len, auth_ofs + auth_len);
432 /* digest in buffer check. Needed only for wireless algos */
434 /* Handle digest-encrypted cases, i.e.
435 * auth-gen-then-cipher-encrypt and
436 * cipher-decrypt-then-auth-verify
438 uint64_t auth_end_iova;
440 if (unlikely(is_sgl)) {
441 uint32_t remaining_off = auth_ofs + auth_len;
442 struct rte_mbuf *sgl_buf = (is_oop ? op->sym->m_dst :
445 while (remaining_off >= rte_pktmbuf_data_len(sgl_buf)
446 && sgl_buf->next != NULL) {
447 remaining_off -= rte_pktmbuf_data_len(sgl_buf);
448 sgl_buf = sgl_buf->next;
451 auth_end_iova = (uint64_t)rte_pktmbuf_iova_offset(
452 sgl_buf, remaining_off);
454 auth_end_iova = (is_oop ?
455 rte_pktmbuf_iova(op->sym->m_dst) :
456 rte_pktmbuf_iova(op->sym->m_src)) + auth_ofs +
459 /* Then check if digest-encrypted conditions are met */
460 if ((auth_ofs + auth_len < cipher_ofs + cipher_len) &&
461 (digest->iova == auth_end_iova))
462 max_len = RTE_MAX(max_len, auth_ofs + auth_len +
466 n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, min_ofs, max_len,
467 in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
468 if (unlikely(n_src < 0 || n_src > op->sym->m_src->nb_segs)) {
469 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
474 if (unlikely((op->sym->m_dst != NULL) &&
475 (op->sym->m_dst != op->sym->m_src))) {
476 int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, min_ofs,
477 max_len, out_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
479 if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
480 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
483 out_sgl->num = n_dst;
487 ofs.ofs.cipher.head = cipher_ofs;
488 ofs.ofs.cipher.tail = max_len - cipher_ofs - cipher_len;
489 ofs.ofs.auth.head = auth_ofs;
490 ofs.ofs.auth.tail = max_len - auth_ofs - auth_len;
495 static __rte_always_inline uint64_t
496 qat_sym_convert_op_to_vec_aead(struct rte_crypto_op *op,
497 struct qat_sym_session *ctx,
498 struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
499 struct rte_crypto_va_iova_ptr *cipher_iv,
500 struct rte_crypto_va_iova_ptr *auth_iv_or_aad,
501 struct rte_crypto_va_iova_ptr *digest)
503 uint32_t cipher_len = 0, cipher_ofs = 0;
506 cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
507 ctx->cipher_iv.offset);
508 cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
509 ctx->cipher_iv.offset);
510 auth_iv_or_aad->va = (void *)op->sym->aead.aad.data;
511 auth_iv_or_aad->iova = op->sym->aead.aad.phys_addr;
512 digest->va = (void *)op->sym->aead.digest.data;
513 digest->iova = op->sym->aead.digest.phys_addr;
515 cipher_len = op->sym->aead.data.length;
516 cipher_ofs = op->sym->aead.data.offset;
518 n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, cipher_ofs, cipher_len,
519 in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
520 if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
521 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
526 /* Out-Of-Place operation */
527 if (unlikely((op->sym->m_dst != NULL) &&
528 (op->sym->m_dst != op->sym->m_src))) {
529 int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, cipher_ofs,
530 cipher_len, out_sgl->vec,
531 QAT_SYM_SGL_MAX_NUMBER);
532 if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
533 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
537 out_sgl->num = n_dst;
544 static __rte_always_inline void
545 qat_set_cipher_iv(struct icp_qat_fw_la_cipher_req_params *cipher_param,
546 struct rte_crypto_va_iova_ptr *iv_ptr, uint32_t iv_len,
547 struct icp_qat_fw_la_bulk_req *qat_req)
549 /* copy IV into request if it fits */
550 if (iv_len <= sizeof(cipher_param->u.cipher_IV_array))
551 rte_memcpy(cipher_param->u.cipher_IV_array, iv_ptr->va,
554 ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
555 qat_req->comn_hdr.serv_specif_flags,
556 ICP_QAT_FW_CIPH_IV_64BIT_PTR);
557 cipher_param->u.s.cipher_IV_ptr = iv_ptr->iova;
561 static __rte_always_inline void
562 qat_sym_dp_fill_vec_status(int32_t *sta, int status, uint32_t n)
566 for (i = 0; i < n; i++)
570 static __rte_always_inline void
571 enqueue_one_cipher_job_gen1(struct qat_sym_session *ctx,
572 struct icp_qat_fw_la_bulk_req *req,
573 struct rte_crypto_va_iova_ptr *iv,
574 union rte_crypto_sym_ofs ofs, uint32_t data_len)
576 struct icp_qat_fw_la_cipher_req_params *cipher_param;
578 cipher_param = (void *)&req->serv_specif_rqpars;
581 qat_set_cipher_iv(cipher_param, iv, ctx->cipher_iv.length, req);
582 cipher_param->cipher_offset = ofs.ofs.cipher.head;
583 cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
587 static __rte_always_inline void
588 enqueue_one_auth_job_gen1(struct qat_sym_session *ctx,
589 struct icp_qat_fw_la_bulk_req *req,
590 struct rte_crypto_va_iova_ptr *digest,
591 struct rte_crypto_va_iova_ptr *auth_iv,
592 union rte_crypto_sym_ofs ofs, uint32_t data_len)
594 struct icp_qat_fw_la_cipher_req_params *cipher_param;
595 struct icp_qat_fw_la_auth_req_params *auth_param;
597 cipher_param = (void *)&req->serv_specif_rqpars;
598 auth_param = (void *)((uint8_t *)cipher_param +
599 ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
601 auth_param->auth_off = ofs.ofs.auth.head;
602 auth_param->auth_len = data_len - ofs.ofs.auth.head -
604 auth_param->auth_res_addr = digest->iova;
606 switch (ctx->qat_hash_alg) {
607 case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
608 case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
609 case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
610 auth_param->u1.aad_adr = auth_iv->iova;
612 case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
613 case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
614 ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
615 req->comn_hdr.serv_specif_flags,
616 ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
617 rte_memcpy(cipher_param->u.cipher_IV_array, auth_iv->va,
618 ctx->auth_iv.length);
625 static __rte_always_inline int
626 enqueue_one_chain_job_gen1(struct qat_sym_session *ctx,
627 struct icp_qat_fw_la_bulk_req *req,
628 struct rte_crypto_vec *src_vec,
630 struct rte_crypto_vec *dst_vec,
632 struct rte_crypto_va_iova_ptr *cipher_iv,
633 struct rte_crypto_va_iova_ptr *digest,
634 struct rte_crypto_va_iova_ptr *auth_iv,
635 union rte_crypto_sym_ofs ofs, uint32_t data_len)
637 struct icp_qat_fw_la_cipher_req_params *cipher_param;
638 struct icp_qat_fw_la_auth_req_params *auth_param;
639 struct rte_crypto_vec *cvec = n_dst_vecs > 0 ?
641 rte_iova_t auth_iova_end;
642 int cipher_len, auth_len;
643 int is_sgl = n_src_vecs > 1 || n_dst_vecs > 1;
645 cipher_param = (void *)&req->serv_specif_rqpars;
646 auth_param = (void *)((uint8_t *)cipher_param +
647 ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
649 cipher_len = data_len - ofs.ofs.cipher.head -
651 auth_len = data_len - ofs.ofs.auth.head - ofs.ofs.auth.tail;
653 if (unlikely(cipher_len < 0 || auth_len < 0))
656 cipher_param->cipher_offset = ofs.ofs.cipher.head;
657 cipher_param->cipher_length = cipher_len;
658 qat_set_cipher_iv(cipher_param, cipher_iv, ctx->cipher_iv.length, req);
660 auth_param->auth_off = ofs.ofs.auth.head;
661 auth_param->auth_len = auth_len;
662 auth_param->auth_res_addr = digest->iova;
664 switch (ctx->qat_hash_alg) {
665 case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
666 case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
667 case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
668 auth_param->u1.aad_adr = auth_iv->iova;
670 case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
671 case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
677 if (unlikely(is_sgl)) {
679 int i = n_dst_vecs ? n_dst_vecs : n_src_vecs;
680 uint32_t remaining_off = data_len - ofs.ofs.auth.tail;
682 while (remaining_off >= cvec->len && i >= 1) {
684 remaining_off -= cvec->len;
688 auth_iova_end = cvec->iova + remaining_off;
690 auth_iova_end = cvec[0].iova + auth_param->auth_off +
691 auth_param->auth_len;
693 /* Then check if digest-encrypted conditions are met */
694 if ((auth_param->auth_off + auth_param->auth_len <
695 cipher_param->cipher_offset + cipher_param->cipher_length) &&
696 (digest->iova == auth_iova_end)) {
697 /* Handle partial digest encryption */
698 if (cipher_param->cipher_offset + cipher_param->cipher_length <
699 auth_param->auth_off + auth_param->auth_len +
700 ctx->digest_length && !is_sgl)
701 req->comn_mid.dst_length = req->comn_mid.src_length =
702 auth_param->auth_off + auth_param->auth_len +
704 struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
705 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
706 ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
712 static __rte_always_inline void
713 enqueue_one_aead_job_gen1(struct qat_sym_session *ctx,
714 struct icp_qat_fw_la_bulk_req *req,
715 struct rte_crypto_va_iova_ptr *iv,
716 struct rte_crypto_va_iova_ptr *digest,
717 struct rte_crypto_va_iova_ptr *aad,
718 union rte_crypto_sym_ofs ofs, uint32_t data_len)
720 struct icp_qat_fw_la_cipher_req_params *cipher_param =
721 (void *)&req->serv_specif_rqpars;
722 struct icp_qat_fw_la_auth_req_params *auth_param =
723 (void *)((uint8_t *)&req->serv_specif_rqpars +
724 ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
726 uint8_t aad_ccm_real_len;
727 uint8_t aad_len_field_sz;
729 rte_iova_t aad_iova = 0;
732 switch (ctx->qat_hash_alg) {
733 case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
734 case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
735 ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
736 req->comn_hdr.serv_specif_flags,
737 ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
738 rte_memcpy(cipher_param->u.cipher_IV_array, iv->va,
739 ctx->cipher_iv.length);
740 aad_iova = aad->iova;
742 case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:
744 aad_iova = aad->iova;
745 aad_ccm_real_len = 0;
746 aad_len_field_sz = 0;
747 msg_len_be = rte_bswap32((uint32_t)data_len -
748 ofs.ofs.cipher.head);
750 if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {
751 aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;
752 aad_ccm_real_len = ctx->aad_len -
753 ICP_QAT_HW_CCM_AAD_B0_LEN -
754 ICP_QAT_HW_CCM_AAD_LEN_INFO;
760 q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length;
761 aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(
762 aad_len_field_sz, ctx->digest_length, q);
763 if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {
764 memcpy(aad_data + ctx->cipher_iv.length +
765 ICP_QAT_HW_CCM_NONCE_OFFSET + (q -
766 ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),
767 (uint8_t *)&msg_len_be,
768 ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);
770 memcpy(aad_data + ctx->cipher_iv.length +
771 ICP_QAT_HW_CCM_NONCE_OFFSET,
772 (uint8_t *)&msg_len_be +
773 (ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE
777 if (aad_len_field_sz > 0) {
778 *(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] =
779 rte_bswap16(aad_ccm_real_len);
781 if ((aad_ccm_real_len + aad_len_field_sz)
782 % ICP_QAT_HW_CCM_AAD_B0_LEN) {
786 pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -
789 ICP_QAT_HW_CCM_AAD_B0_LEN);
790 pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +
793 memset(&aad_data[pad_idx], 0, pad_len);
797 rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array)
798 + ICP_QAT_HW_CCM_NONCE_OFFSET,
800 ICP_QAT_HW_CCM_NONCE_OFFSET, ctx->cipher_iv.length);
801 *(uint8_t *)&cipher_param->u.cipher_IV_array[0] =
802 q - ICP_QAT_HW_CCM_NONCE_OFFSET;
804 rte_memcpy((uint8_t *)aad->va +
805 ICP_QAT_HW_CCM_NONCE_OFFSET,
806 (uint8_t *)iv->va + ICP_QAT_HW_CCM_NONCE_OFFSET,
807 ctx->cipher_iv.length);
813 cipher_param->cipher_offset = ofs.ofs.cipher.head;
814 cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
816 auth_param->auth_off = ofs.ofs.cipher.head;
817 auth_param->auth_len = cipher_param->cipher_length;
818 auth_param->auth_res_addr = digest->iova;
819 auth_param->u1.aad_adr = aad_iova;
822 extern struct rte_cryptodev_ops qat_sym_crypto_ops_gen1;
823 extern struct rte_cryptodev_ops qat_asym_crypto_ops_gen1;
825 /* -----------------GEN 1 sym crypto op data path APIs ---------------- */
827 qat_sym_build_op_cipher_gen1(void *in_op, struct qat_sym_session *ctx,
828 uint8_t *out_msg, void *op_cookie);
831 qat_sym_build_op_auth_gen1(void *in_op, struct qat_sym_session *ctx,
832 uint8_t *out_msg, void *op_cookie);
835 qat_sym_build_op_aead_gen1(void *in_op, struct qat_sym_session *ctx,
836 uint8_t *out_msg, void *op_cookie);
839 qat_sym_build_op_chain_gen1(void *in_op, struct qat_sym_session *ctx,
840 uint8_t *out_msg, void *op_cookie);
842 /* -----------------GEN 1 sym crypto raw data path APIs ---------------- */
844 qat_sym_dp_enqueue_single_cipher_gen1(void *qp_data, uint8_t *drv_ctx,
845 struct rte_crypto_vec *data, uint16_t n_data_vecs,
846 union rte_crypto_sym_ofs ofs,
847 struct rte_crypto_va_iova_ptr *iv,
848 struct rte_crypto_va_iova_ptr *digest __rte_unused,
849 struct rte_crypto_va_iova_ptr *aad __rte_unused,
853 qat_sym_dp_enqueue_cipher_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
854 struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
855 void *user_data[], int *status);
858 qat_sym_dp_enqueue_single_auth_gen1(void *qp_data, uint8_t *drv_ctx,
859 struct rte_crypto_vec *data, uint16_t n_data_vecs,
860 union rte_crypto_sym_ofs ofs,
861 struct rte_crypto_va_iova_ptr *iv __rte_unused,
862 struct rte_crypto_va_iova_ptr *digest,
863 struct rte_crypto_va_iova_ptr *auth_iv,
867 qat_sym_dp_enqueue_auth_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
868 struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
869 void *user_data[], int *status);
872 qat_sym_dp_enqueue_single_chain_gen1(void *qp_data, uint8_t *drv_ctx,
873 struct rte_crypto_vec *data, uint16_t n_data_vecs,
874 union rte_crypto_sym_ofs ofs,
875 struct rte_crypto_va_iova_ptr *cipher_iv,
876 struct rte_crypto_va_iova_ptr *digest,
877 struct rte_crypto_va_iova_ptr *auth_iv,
881 qat_sym_dp_enqueue_chain_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
882 struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
883 void *user_data[], int *status);
886 qat_sym_dp_enqueue_single_aead_gen1(void *qp_data, uint8_t *drv_ctx,
887 struct rte_crypto_vec *data, uint16_t n_data_vecs,
888 union rte_crypto_sym_ofs ofs,
889 struct rte_crypto_va_iova_ptr *iv,
890 struct rte_crypto_va_iova_ptr *digest,
891 struct rte_crypto_va_iova_ptr *aad,
895 qat_sym_dp_enqueue_aead_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
896 struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
897 void *user_data[], int *status);
900 qat_sym_dp_dequeue_single_gen1(void *qp_data, uint8_t *drv_ctx,
901 int *dequeue_status, enum rte_crypto_op_status *op_status);
904 qat_sym_dp_dequeue_burst_gen1(void *qp_data, uint8_t *drv_ctx,
905 rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,
906 uint32_t max_nb_to_dequeue,
907 rte_cryptodev_raw_post_dequeue_t post_dequeue,
908 void **out_user_data, uint8_t is_user_data_array,
909 uint32_t *n_success_jobs, int *return_status);
912 qat_sym_dp_enqueue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n);
915 qat_sym_dp_dequeue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n);
918 qat_sym_configure_raw_dp_ctx_gen1(void *_raw_dp_ctx, void *_ctx);
920 /* -----------------GENx control path APIs ---------------- */
922 qat_sym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
925 qat_sym_crypto_set_session_gen1(void *cryptodev, void *session);
928 qat_sym_session_set_ext_hash_flags_gen2(struct qat_sym_session *session,
931 struct qat_capabilities_info
932 qat_asym_crypto_cap_get_gen1(struct qat_pci_device *qat_dev);
935 qat_asym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
938 qat_asym_crypto_set_session_gen1(void *cryptodev, void *session);
940 #ifdef RTE_LIB_SECURITY
941 extern struct rte_security_ops security_qat_ops_gen1;
944 qat_sym_create_security_gen1(void *cryptodev);