1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2020 Intel Corporation
5 #include <cryptodev_pmd.h>
7 #include "adf_transport_access_macros.h"
8 #include "icp_qat_fw.h"
9 #include "icp_qat_fw_la.h"
12 #include "qat_sym_pmd.h"
13 #include "qat_sym_session.h"
16 struct qat_sym_dp_ctx {
17 struct qat_sym_session *session;
20 uint16_t cached_enqueue;
21 uint16_t cached_dequeue;
24 static __rte_always_inline int32_t
25 qat_sym_dp_parse_data_vec(struct qat_qp *qp, struct icp_qat_fw_la_bulk_req *req,
26 struct rte_crypto_vec *data, uint16_t n_data_vecs)
28 struct qat_queue *tx_queue;
29 struct qat_sym_op_cookie *cookie;
34 if (likely(n_data_vecs == 1)) {
35 req->comn_mid.src_data_addr = req->comn_mid.dest_data_addr =
37 req->comn_mid.src_length = req->comn_mid.dst_length =
42 if (n_data_vecs == 0 || n_data_vecs > QAT_SYM_SGL_MAX_NUMBER)
48 ICP_QAT_FW_COMN_PTR_TYPE_SET(req->comn_hdr.comn_req_flags,
49 QAT_COMN_PTR_TYPE_SGL);
50 cookie = qp->op_cookies[tx_queue->tail >> tx_queue->trailz];
51 list = (struct qat_sgl *)&cookie->qat_sgl_src;
53 for (i = 0; i < n_data_vecs; i++) {
54 list->buffers[i].len = data[i].len;
55 list->buffers[i].resrvd = 0;
56 list->buffers[i].addr = data[i].iova;
57 if (total_len + data[i].len > UINT32_MAX) {
58 QAT_DP_LOG(ERR, "Message too long");
61 total_len += data[i].len;
65 req->comn_mid.src_data_addr = req->comn_mid.dest_data_addr =
66 cookie->qat_sgl_src_phys_addr;
67 req->comn_mid.src_length = req->comn_mid.dst_length = 0;
71 static __rte_always_inline void
72 set_cipher_iv(struct icp_qat_fw_la_cipher_req_params *cipher_param,
73 struct rte_crypto_va_iova_ptr *iv_ptr, uint32_t iv_len,
74 struct icp_qat_fw_la_bulk_req *qat_req)
76 /* copy IV into request if it fits */
77 if (iv_len <= sizeof(cipher_param->u.cipher_IV_array))
78 rte_memcpy(cipher_param->u.cipher_IV_array, iv_ptr->va,
81 ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
82 qat_req->comn_hdr.serv_specif_flags,
83 ICP_QAT_FW_CIPH_IV_64BIT_PTR);
84 cipher_param->u.s.cipher_IV_ptr = iv_ptr->iova;
88 #define QAT_SYM_DP_IS_RESP_SUCCESS(resp) \
89 (ICP_QAT_FW_COMN_STATUS_FLAG_OK == \
90 ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(resp->comn_hdr.comn_status))
92 static __rte_always_inline void
93 qat_sym_dp_fill_vec_status(int32_t *sta, int status, uint32_t n)
97 for (i = 0; i < n; i++)
101 #define QAT_SYM_DP_GET_MAX_ENQ(q, c, n) \
102 RTE_MIN((q->max_inflights - q->enqueued + q->dequeued - c), n)
104 static __rte_always_inline void
105 enqueue_one_cipher_job(struct qat_sym_session *ctx,
106 struct icp_qat_fw_la_bulk_req *req,
107 struct rte_crypto_va_iova_ptr *iv,
108 union rte_crypto_sym_ofs ofs, uint32_t data_len)
110 struct icp_qat_fw_la_cipher_req_params *cipher_param;
112 cipher_param = (void *)&req->serv_specif_rqpars;
115 set_cipher_iv(cipher_param, iv, ctx->cipher_iv.length, req);
116 cipher_param->cipher_offset = ofs.ofs.cipher.head;
117 cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
121 static __rte_always_inline int
122 qat_sym_dp_enqueue_single_cipher(void *qp_data, uint8_t *drv_ctx,
123 struct rte_crypto_vec *data, uint16_t n_data_vecs,
124 union rte_crypto_sym_ofs ofs,
125 struct rte_crypto_va_iova_ptr *iv,
126 struct rte_crypto_va_iova_ptr *digest __rte_unused,
127 struct rte_crypto_va_iova_ptr *aad __rte_unused,
130 struct qat_qp *qp = qp_data;
131 struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
132 struct qat_queue *tx_queue = &qp->tx_q;
133 struct qat_sym_session *ctx = dp_ctx->session;
134 struct icp_qat_fw_la_bulk_req *req;
136 uint32_t tail = dp_ctx->tail;
138 req = (struct icp_qat_fw_la_bulk_req *)(
139 (uint8_t *)tx_queue->base_addr + tail);
140 tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
141 rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
142 rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
143 data_len = qat_sym_dp_parse_data_vec(qp, req, data, n_data_vecs);
144 if (unlikely(data_len < 0))
146 req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data;
148 enqueue_one_cipher_job(ctx, req, iv, ofs, (uint32_t)data_len);
151 dp_ctx->cached_enqueue++;
156 static __rte_always_inline uint32_t
157 qat_sym_dp_enqueue_cipher_jobs(void *qp_data, uint8_t *drv_ctx,
158 struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
159 void *user_data[], int *status)
161 struct qat_qp *qp = qp_data;
162 struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
163 struct qat_queue *tx_queue = &qp->tx_q;
164 struct qat_sym_session *ctx = dp_ctx->session;
167 struct icp_qat_fw_la_bulk_req *req;
170 n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
171 if (unlikely(n == 0)) {
172 qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
179 for (i = 0; i < n; i++) {
180 req = (struct icp_qat_fw_la_bulk_req *)(
181 (uint8_t *)tx_queue->base_addr + tail);
182 rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
184 data_len = qat_sym_dp_parse_data_vec(qp, req,
186 vec->src_sgl[i].num);
187 if (unlikely(data_len < 0))
189 req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data[i];
190 enqueue_one_cipher_job(ctx, req, &vec->iv[i], ofs,
192 tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
196 qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
199 dp_ctx->cached_enqueue += i;
204 static __rte_always_inline void
205 enqueue_one_auth_job(struct qat_sym_session *ctx,
206 struct icp_qat_fw_la_bulk_req *req,
207 struct rte_crypto_va_iova_ptr *digest,
208 struct rte_crypto_va_iova_ptr *auth_iv,
209 union rte_crypto_sym_ofs ofs, uint32_t data_len)
211 struct icp_qat_fw_la_cipher_req_params *cipher_param;
212 struct icp_qat_fw_la_auth_req_params *auth_param;
214 cipher_param = (void *)&req->serv_specif_rqpars;
215 auth_param = (void *)((uint8_t *)cipher_param +
216 ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
218 auth_param->auth_off = ofs.ofs.auth.head;
219 auth_param->auth_len = data_len - ofs.ofs.auth.head -
221 auth_param->auth_res_addr = digest->iova;
223 switch (ctx->qat_hash_alg) {
224 case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
225 case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
226 case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
227 auth_param->u1.aad_adr = auth_iv->iova;
229 case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
230 case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
231 ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
232 req->comn_hdr.serv_specif_flags,
233 ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
234 rte_memcpy(cipher_param->u.cipher_IV_array, auth_iv->va,
235 ctx->auth_iv.length);
242 static __rte_always_inline int
243 qat_sym_dp_enqueue_single_auth(void *qp_data, uint8_t *drv_ctx,
244 struct rte_crypto_vec *data, uint16_t n_data_vecs,
245 union rte_crypto_sym_ofs ofs,
246 struct rte_crypto_va_iova_ptr *iv __rte_unused,
247 struct rte_crypto_va_iova_ptr *digest,
248 struct rte_crypto_va_iova_ptr *auth_iv,
251 struct qat_qp *qp = qp_data;
252 struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
253 struct qat_queue *tx_queue = &qp->tx_q;
254 struct qat_sym_session *ctx = dp_ctx->session;
255 struct icp_qat_fw_la_bulk_req *req;
257 uint32_t tail = dp_ctx->tail;
259 req = (struct icp_qat_fw_la_bulk_req *)(
260 (uint8_t *)tx_queue->base_addr + tail);
261 tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
262 rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
263 rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
264 data_len = qat_sym_dp_parse_data_vec(qp, req, data, n_data_vecs);
265 if (unlikely(data_len < 0))
267 req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data;
269 enqueue_one_auth_job(ctx, req, digest, auth_iv, ofs,
273 dp_ctx->cached_enqueue++;
278 static __rte_always_inline uint32_t
279 qat_sym_dp_enqueue_auth_jobs(void *qp_data, uint8_t *drv_ctx,
280 struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
281 void *user_data[], int *status)
283 struct qat_qp *qp = qp_data;
284 struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
285 struct qat_queue *tx_queue = &qp->tx_q;
286 struct qat_sym_session *ctx = dp_ctx->session;
289 struct icp_qat_fw_la_bulk_req *req;
292 n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
293 if (unlikely(n == 0)) {
294 qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
301 for (i = 0; i < n; i++) {
302 req = (struct icp_qat_fw_la_bulk_req *)(
303 (uint8_t *)tx_queue->base_addr + tail);
304 rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
306 data_len = qat_sym_dp_parse_data_vec(qp, req,
308 vec->src_sgl[i].num);
309 if (unlikely(data_len < 0))
311 req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data[i];
312 enqueue_one_auth_job(ctx, req, &vec->digest[i],
313 &vec->auth_iv[i], ofs, (uint32_t)data_len);
314 tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
318 qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
321 dp_ctx->cached_enqueue += i;
326 static __rte_always_inline int
327 enqueue_one_chain_job(struct qat_sym_session *ctx,
328 struct icp_qat_fw_la_bulk_req *req,
329 struct rte_crypto_vec *data,
330 uint16_t n_data_vecs,
331 struct rte_crypto_va_iova_ptr *cipher_iv,
332 struct rte_crypto_va_iova_ptr *digest,
333 struct rte_crypto_va_iova_ptr *auth_iv,
334 union rte_crypto_sym_ofs ofs, uint32_t data_len)
336 struct icp_qat_fw_la_cipher_req_params *cipher_param;
337 struct icp_qat_fw_la_auth_req_params *auth_param;
338 rte_iova_t auth_iova_end;
339 int32_t cipher_len, auth_len;
341 cipher_param = (void *)&req->serv_specif_rqpars;
342 auth_param = (void *)((uint8_t *)cipher_param +
343 ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
345 cipher_len = data_len - ofs.ofs.cipher.head -
347 auth_len = data_len - ofs.ofs.auth.head - ofs.ofs.auth.tail;
349 if (unlikely(cipher_len < 0 || auth_len < 0))
352 cipher_param->cipher_offset = ofs.ofs.cipher.head;
353 cipher_param->cipher_length = cipher_len;
354 set_cipher_iv(cipher_param, cipher_iv, ctx->cipher_iv.length, req);
356 auth_param->auth_off = ofs.ofs.auth.head;
357 auth_param->auth_len = auth_len;
358 auth_param->auth_res_addr = digest->iova;
360 switch (ctx->qat_hash_alg) {
361 case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
362 case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
363 case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
364 auth_param->u1.aad_adr = auth_iv->iova;
366 case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
367 case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
373 if (unlikely(n_data_vecs > 1)) {
374 int auth_end_get = 0, i = n_data_vecs - 1;
375 struct rte_crypto_vec *cvec = &data[0];
378 len = data_len - ofs.ofs.auth.tail;
380 while (i >= 0 && len > 0) {
381 if (cvec->len >= len) {
382 auth_iova_end = cvec->iova + len;
392 if (unlikely(auth_end_get == 0))
395 auth_iova_end = data[0].iova + auth_param->auth_off +
396 auth_param->auth_len;
398 /* Then check if digest-encrypted conditions are met */
399 if ((auth_param->auth_off + auth_param->auth_len <
400 cipher_param->cipher_offset +
401 cipher_param->cipher_length) &&
402 (digest->iova == auth_iova_end)) {
403 /* Handle partial digest encryption */
404 if (cipher_param->cipher_offset +
405 cipher_param->cipher_length <
406 auth_param->auth_off +
407 auth_param->auth_len +
409 req->comn_mid.dst_length =
410 req->comn_mid.src_length =
411 auth_param->auth_off +
412 auth_param->auth_len +
414 struct icp_qat_fw_comn_req_hdr *header =
416 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(
417 header->serv_specif_flags,
418 ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
424 static __rte_always_inline int
425 qat_sym_dp_enqueue_single_chain(void *qp_data, uint8_t *drv_ctx,
426 struct rte_crypto_vec *data, uint16_t n_data_vecs,
427 union rte_crypto_sym_ofs ofs,
428 struct rte_crypto_va_iova_ptr *cipher_iv,
429 struct rte_crypto_va_iova_ptr *digest,
430 struct rte_crypto_va_iova_ptr *auth_iv,
433 struct qat_qp *qp = qp_data;
434 struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
435 struct qat_queue *tx_queue = &qp->tx_q;
436 struct qat_sym_session *ctx = dp_ctx->session;
437 struct icp_qat_fw_la_bulk_req *req;
439 uint32_t tail = dp_ctx->tail;
441 req = (struct icp_qat_fw_la_bulk_req *)(
442 (uint8_t *)tx_queue->base_addr + tail);
443 tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
444 rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
445 rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
446 data_len = qat_sym_dp_parse_data_vec(qp, req, data, n_data_vecs);
447 if (unlikely(data_len < 0))
449 req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data;
451 if (unlikely(enqueue_one_chain_job(ctx, req, data, n_data_vecs,
452 cipher_iv, digest, auth_iv, ofs, (uint32_t)data_len)))
456 dp_ctx->cached_enqueue++;
461 static __rte_always_inline uint32_t
462 qat_sym_dp_enqueue_chain_jobs(void *qp_data, uint8_t *drv_ctx,
463 struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
464 void *user_data[], int *status)
466 struct qat_qp *qp = qp_data;
467 struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
468 struct qat_queue *tx_queue = &qp->tx_q;
469 struct qat_sym_session *ctx = dp_ctx->session;
472 struct icp_qat_fw_la_bulk_req *req;
475 n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
476 if (unlikely(n == 0)) {
477 qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
484 for (i = 0; i < n; i++) {
485 req = (struct icp_qat_fw_la_bulk_req *)(
486 (uint8_t *)tx_queue->base_addr + tail);
487 rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
489 data_len = qat_sym_dp_parse_data_vec(qp, req,
491 vec->src_sgl[i].num);
492 if (unlikely(data_len < 0))
494 req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data[i];
495 if (unlikely(enqueue_one_chain_job(ctx, req,
496 vec->src_sgl[i].vec, vec->src_sgl[i].num,
497 &vec->iv[i], &vec->digest[i],
498 &vec->auth_iv[i], ofs, (uint32_t)data_len)))
501 tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
505 qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
508 dp_ctx->cached_enqueue += i;
513 static __rte_always_inline void
514 enqueue_one_aead_job(struct qat_sym_session *ctx,
515 struct icp_qat_fw_la_bulk_req *req,
516 struct rte_crypto_va_iova_ptr *iv,
517 struct rte_crypto_va_iova_ptr *digest,
518 struct rte_crypto_va_iova_ptr *aad,
519 union rte_crypto_sym_ofs ofs, uint32_t data_len)
521 struct icp_qat_fw_la_cipher_req_params *cipher_param =
522 (void *)&req->serv_specif_rqpars;
523 struct icp_qat_fw_la_auth_req_params *auth_param =
524 (void *)((uint8_t *)&req->serv_specif_rqpars +
525 ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
527 uint8_t aad_ccm_real_len;
528 uint8_t aad_len_field_sz;
530 rte_iova_t aad_iova = 0;
533 /* CPM 1.7 uses single pass to treat AEAD as cipher operation */
534 if (ctx->is_single_pass) {
535 enqueue_one_cipher_job(ctx, req, iv, ofs, data_len);
538 /* QAT GEN4 uses single pass to treat AEAD as cipher
541 struct icp_qat_fw_la_cipher_20_req_params *cipher_param_20 =
542 (void *)&req->serv_specif_rqpars;
543 cipher_param_20->spc_aad_addr = aad->iova;
544 cipher_param_20->spc_auth_res_addr = digest->iova;
546 cipher_param->spc_aad_addr = aad->iova;
547 cipher_param->spc_auth_res_addr = digest->iova;
553 switch (ctx->qat_hash_alg) {
554 case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
555 case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
556 ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
557 req->comn_hdr.serv_specif_flags,
558 ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
559 rte_memcpy(cipher_param->u.cipher_IV_array, iv->va,
560 ctx->cipher_iv.length);
561 aad_iova = aad->iova;
563 case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:
565 aad_iova = aad->iova;
566 aad_ccm_real_len = 0;
567 aad_len_field_sz = 0;
568 msg_len_be = rte_bswap32((uint32_t)data_len -
569 ofs.ofs.cipher.head);
571 if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {
572 aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;
573 aad_ccm_real_len = ctx->aad_len -
574 ICP_QAT_HW_CCM_AAD_B0_LEN -
575 ICP_QAT_HW_CCM_AAD_LEN_INFO;
581 q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length;
582 aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(
583 aad_len_field_sz, ctx->digest_length, q);
584 if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {
585 memcpy(aad_data + ctx->cipher_iv.length +
586 ICP_QAT_HW_CCM_NONCE_OFFSET + (q -
587 ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),
588 (uint8_t *)&msg_len_be,
589 ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);
591 memcpy(aad_data + ctx->cipher_iv.length +
592 ICP_QAT_HW_CCM_NONCE_OFFSET,
593 (uint8_t *)&msg_len_be +
594 (ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE
598 if (aad_len_field_sz > 0) {
599 *(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] =
600 rte_bswap16(aad_ccm_real_len);
602 if ((aad_ccm_real_len + aad_len_field_sz)
603 % ICP_QAT_HW_CCM_AAD_B0_LEN) {
607 pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -
610 ICP_QAT_HW_CCM_AAD_B0_LEN);
611 pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +
614 memset(&aad_data[pad_idx], 0, pad_len);
618 rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array)
619 + ICP_QAT_HW_CCM_NONCE_OFFSET,
621 ICP_QAT_HW_CCM_NONCE_OFFSET, ctx->cipher_iv.length);
622 *(uint8_t *)&cipher_param->u.cipher_IV_array[0] =
623 q - ICP_QAT_HW_CCM_NONCE_OFFSET;
625 rte_memcpy((uint8_t *)aad->va +
626 ICP_QAT_HW_CCM_NONCE_OFFSET,
627 (uint8_t *)iv->va + ICP_QAT_HW_CCM_NONCE_OFFSET,
628 ctx->cipher_iv.length);
634 cipher_param->cipher_offset = ofs.ofs.cipher.head;
635 cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
637 auth_param->auth_off = ofs.ofs.cipher.head;
638 auth_param->auth_len = cipher_param->cipher_length;
639 auth_param->auth_res_addr = digest->iova;
640 auth_param->u1.aad_adr = aad_iova;
643 static __rte_always_inline int
644 qat_sym_dp_enqueue_single_aead(void *qp_data, uint8_t *drv_ctx,
645 struct rte_crypto_vec *data, uint16_t n_data_vecs,
646 union rte_crypto_sym_ofs ofs,
647 struct rte_crypto_va_iova_ptr *iv,
648 struct rte_crypto_va_iova_ptr *digest,
649 struct rte_crypto_va_iova_ptr *aad,
652 struct qat_qp *qp = qp_data;
653 struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
654 struct qat_queue *tx_queue = &qp->tx_q;
655 struct qat_sym_session *ctx = dp_ctx->session;
656 struct icp_qat_fw_la_bulk_req *req;
658 uint32_t tail = dp_ctx->tail;
660 req = (struct icp_qat_fw_la_bulk_req *)(
661 (uint8_t *)tx_queue->base_addr + tail);
662 tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
663 rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
664 rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
665 data_len = qat_sym_dp_parse_data_vec(qp, req, data, n_data_vecs);
666 if (unlikely(data_len < 0))
668 req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data;
670 enqueue_one_aead_job(ctx, req, iv, digest, aad, ofs,
674 dp_ctx->cached_enqueue++;
679 static __rte_always_inline uint32_t
680 qat_sym_dp_enqueue_aead_jobs(void *qp_data, uint8_t *drv_ctx,
681 struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
682 void *user_data[], int *status)
684 struct qat_qp *qp = qp_data;
685 struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
686 struct qat_queue *tx_queue = &qp->tx_q;
687 struct qat_sym_session *ctx = dp_ctx->session;
690 struct icp_qat_fw_la_bulk_req *req;
693 n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
694 if (unlikely(n == 0)) {
695 qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
702 for (i = 0; i < n; i++) {
703 req = (struct icp_qat_fw_la_bulk_req *)(
704 (uint8_t *)tx_queue->base_addr + tail);
705 rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
707 data_len = qat_sym_dp_parse_data_vec(qp, req,
709 vec->src_sgl[i].num);
710 if (unlikely(data_len < 0))
712 req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data[i];
713 enqueue_one_aead_job(ctx, req, &vec->iv[i], &vec->digest[i],
714 &vec->aad[i], ofs, (uint32_t)data_len);
715 tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
719 qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
722 dp_ctx->cached_enqueue += i;
727 static __rte_always_inline uint32_t
728 qat_sym_dp_dequeue_burst(void *qp_data, uint8_t *drv_ctx,
729 rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,
730 uint32_t max_nb_to_dequeue,
731 rte_cryptodev_raw_post_dequeue_t post_dequeue,
732 void **out_user_data, uint8_t is_user_data_array,
733 uint32_t *n_success_jobs, int *return_status)
735 struct qat_qp *qp = qp_data;
736 struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
737 struct qat_queue *rx_queue = &qp->rx_q;
738 struct icp_qat_fw_comn_resp *resp;
740 uint32_t i, n, inflight;
748 inflight = qp->enqueued - qp->dequeued;
749 if (unlikely(inflight == 0))
752 resp = (struct icp_qat_fw_comn_resp *)((uint8_t *)rx_queue->base_addr +
754 /* no operation ready */
755 if (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))
758 resp_opaque = (void *)(uintptr_t)resp->opaque_data;
759 /* get the dequeue count */
760 if (get_dequeue_count) {
761 n = get_dequeue_count(resp_opaque);
762 if (unlikely(n == 0))
765 if (unlikely(max_nb_to_dequeue == 0))
767 n = max_nb_to_dequeue;
770 out_user_data[0] = resp_opaque;
771 status = QAT_SYM_DP_IS_RESP_SUCCESS(resp);
772 post_dequeue(resp_opaque, 0, status);
773 *n_success_jobs += status;
775 head = (head + rx_queue->msg_size) & rx_queue->modulo_mask;
777 /* we already finished dequeue when n == 1 */
778 if (unlikely(n == 1)) {
783 if (is_user_data_array) {
784 for (i = 1; i < n; i++) {
785 resp = (struct icp_qat_fw_comn_resp *)(
786 (uint8_t *)rx_queue->base_addr + head);
787 if (unlikely(*(uint32_t *)resp ==
790 out_user_data[i] = (void *)(uintptr_t)resp->opaque_data;
791 status = QAT_SYM_DP_IS_RESP_SUCCESS(resp);
792 *n_success_jobs += status;
793 post_dequeue(out_user_data[i], i, status);
794 head = (head + rx_queue->msg_size) &
795 rx_queue->modulo_mask;
801 /* opaque is not array */
802 for (i = 1; i < n; i++) {
803 resp = (struct icp_qat_fw_comn_resp *)(
804 (uint8_t *)rx_queue->base_addr + head);
805 status = QAT_SYM_DP_IS_RESP_SUCCESS(resp);
806 if (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))
808 head = (head + rx_queue->msg_size) &
809 rx_queue->modulo_mask;
810 post_dequeue(resp_opaque, i, status);
811 *n_success_jobs += status;
816 dp_ctx->cached_dequeue += i;
820 static __rte_always_inline void *
821 qat_sym_dp_dequeue(void *qp_data, uint8_t *drv_ctx, int *dequeue_status,
822 enum rte_crypto_op_status *op_status)
824 struct qat_qp *qp = qp_data;
825 struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
826 struct qat_queue *rx_queue = &qp->rx_q;
827 register struct icp_qat_fw_comn_resp *resp;
829 resp = (struct icp_qat_fw_comn_resp *)((uint8_t *)rx_queue->base_addr +
832 if (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))
835 dp_ctx->head = (dp_ctx->head + rx_queue->msg_size) &
836 rx_queue->modulo_mask;
837 dp_ctx->cached_dequeue++;
839 *op_status = QAT_SYM_DP_IS_RESP_SUCCESS(resp) ?
840 RTE_CRYPTO_OP_STATUS_SUCCESS :
841 RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
843 return (void *)(uintptr_t)resp->opaque_data;
846 static __rte_always_inline int
847 qat_sym_dp_kick_tail(void *qp_data, uint8_t *drv_ctx, uint32_t n)
849 struct qat_qp *qp = qp_data;
850 struct qat_queue *tx_queue = &qp->tx_q;
851 struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
853 if (unlikely(dp_ctx->cached_enqueue != n))
857 qp->stats.enqueued_count += n;
859 tx_queue->tail = dp_ctx->tail;
861 WRITE_CSR_RING_TAIL(qp->mmap_bar_addr,
862 tx_queue->hw_bundle_number,
863 tx_queue->hw_queue_number, tx_queue->tail);
864 tx_queue->csr_tail = tx_queue->tail;
865 dp_ctx->cached_enqueue = 0;
870 static __rte_always_inline int
871 qat_sym_dp_update_head(void *qp_data, uint8_t *drv_ctx, uint32_t n)
873 struct qat_qp *qp = qp_data;
874 struct qat_queue *rx_queue = &qp->rx_q;
875 struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
877 if (unlikely(dp_ctx->cached_dequeue != n))
880 rx_queue->head = dp_ctx->head;
881 rx_queue->nb_processed_responses += n;
883 qp->stats.dequeued_count += n;
884 if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH) {
885 uint32_t old_head, new_head;
888 old_head = rx_queue->csr_head;
889 new_head = rx_queue->head;
890 max_head = qp->nb_descriptors * rx_queue->msg_size;
892 /* write out free descriptors */
893 void *cur_desc = (uint8_t *)rx_queue->base_addr + old_head;
895 if (new_head < old_head) {
896 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE,
897 max_head - old_head);
898 memset(rx_queue->base_addr, ADF_RING_EMPTY_SIG_BYTE,
901 memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head -
904 rx_queue->nb_processed_responses = 0;
905 rx_queue->csr_head = new_head;
907 /* write current head to CSR */
908 WRITE_CSR_RING_HEAD(qp->mmap_bar_addr,
909 rx_queue->hw_bundle_number, rx_queue->hw_queue_number,
913 dp_ctx->cached_dequeue = 0;
918 qat_sym_configure_dp_ctx(struct rte_cryptodev *dev, uint16_t qp_id,
919 struct rte_crypto_raw_dp_ctx *raw_dp_ctx,
920 enum rte_crypto_op_sess_type sess_type,
921 union rte_cryptodev_session_ctx session_ctx, uint8_t is_update)
924 struct qat_sym_session *ctx;
925 struct qat_sym_dp_ctx *dp_ctx;
927 qp = dev->data->queue_pairs[qp_id];
928 dp_ctx = (struct qat_sym_dp_ctx *)raw_dp_ctx->drv_ctx_data;
931 memset(raw_dp_ctx, 0, sizeof(*raw_dp_ctx) +
932 sizeof(struct qat_sym_dp_ctx));
933 raw_dp_ctx->qp_data = dev->data->queue_pairs[qp_id];
934 dp_ctx->tail = qp->tx_q.tail;
935 dp_ctx->head = qp->rx_q.head;
936 dp_ctx->cached_enqueue = dp_ctx->cached_dequeue = 0;
939 if (sess_type != RTE_CRYPTO_OP_WITH_SESSION)
942 ctx = (struct qat_sym_session *)get_sym_session_private_data(
943 session_ctx.crypto_sess, qat_sym_driver_id);
945 dp_ctx->session = ctx;
947 raw_dp_ctx->enqueue_done = qat_sym_dp_kick_tail;
948 raw_dp_ctx->dequeue_burst = qat_sym_dp_dequeue_burst;
949 raw_dp_ctx->dequeue = qat_sym_dp_dequeue;
950 raw_dp_ctx->dequeue_done = qat_sym_dp_update_head;
952 if ((ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER ||
953 ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) &&
955 /* AES-GCM or AES-CCM */
956 if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
957 ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 ||
958 (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_AES128
959 && ctx->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE
960 && ctx->qat_hash_alg ==
961 ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) {
962 raw_dp_ctx->enqueue_burst =
963 qat_sym_dp_enqueue_aead_jobs;
964 raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_aead;
966 raw_dp_ctx->enqueue_burst =
967 qat_sym_dp_enqueue_chain_jobs;
968 raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_chain;
970 } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH || ctx->is_gmac) {
971 raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_auth_jobs;
972 raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_auth;
973 } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) {
974 if (ctx->qat_mode == ICP_QAT_HW_CIPHER_AEAD_MODE ||
975 ctx->qat_cipher_alg ==
976 ICP_QAT_HW_CIPHER_ALGO_CHACHA20_POLY1305) {
977 raw_dp_ctx->enqueue_burst =
978 qat_sym_dp_enqueue_aead_jobs;
979 raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_aead;
981 raw_dp_ctx->enqueue_burst =
982 qat_sym_dp_enqueue_cipher_jobs;
983 raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_cipher;
992 qat_sym_get_dp_ctx_size(__rte_unused struct rte_cryptodev *dev)
994 return sizeof(struct qat_sym_dp_ctx);