drivers/crypto/qat/qat_sym_hw_dp.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2020 Intel Corporation
   3  */
   4
   5 #include <rte_cryptodev_pmd.h>
   6
   7 #include "adf_transport_access_macros.h"
   8 #include "icp_qat_fw.h"
   9 #include "icp_qat_fw_la.h"
  10
  11 #include "qat_sym.h"
  12 #include "qat_sym_pmd.h"
  13 #include "qat_sym_session.h"
  14 #include "qat_qp.h"
  15
  16 struct qat_sym_dp_ctx {
  17         struct qat_sym_session *session;
  18         uint32_t tail;
  19         uint32_t head;
  20         uint16_t cached_enqueue;
  21         uint16_t cached_dequeue;
  22 };
  23
  24 static __rte_always_inline int32_t
  25 qat_sym_dp_parse_data_vec(struct qat_qp *qp, struct icp_qat_fw_la_bulk_req *req,
  26                 struct rte_crypto_vec *data, uint16_t n_data_vecs)
  27 {
  28         struct qat_queue *tx_queue;
  29         struct qat_sym_op_cookie *cookie;
  30         struct qat_sgl *list;
  31         uint32_t i;
  32         uint32_t total_len;
  33
  34         if (likely(n_data_vecs == 1)) {
  35                 req->comn_mid.src_data_addr = req->comn_mid.dest_data_addr =
  36                         data[0].iova;
  37                 req->comn_mid.src_length = req->comn_mid.dst_length =
  38                         data[0].len;
  39                 return data[0].len;
  40         }
  41
  42         if (n_data_vecs == 0 || n_data_vecs > QAT_SYM_SGL_MAX_NUMBER)
  43                 return -1;
  44
  45         total_len = 0;
  46         tx_queue = &qp->tx_q;
  47
  48         ICP_QAT_FW_COMN_PTR_TYPE_SET(req->comn_hdr.comn_req_flags,
  49                         QAT_COMN_PTR_TYPE_SGL);
  50         cookie = qp->op_cookies[tx_queue->tail >> tx_queue->trailz];
  51         list = (struct qat_sgl *)&cookie->qat_sgl_src;
  52
  53         for (i = 0; i < n_data_vecs; i++) {
  54                 list->buffers[i].len = data[i].len;
  55                 list->buffers[i].resrvd = 0;
  56                 list->buffers[i].addr = data[i].iova;
  57                 if (total_len + data[i].len > UINT32_MAX) {
  58                         QAT_DP_LOG(ERR, "Message too long");
  59                         return -1;
  60                 }
  61                 total_len += data[i].len;
  62         }
  63
  64         list->num_bufs = i;
  65         req->comn_mid.src_data_addr = req->comn_mid.dest_data_addr =
  66                         cookie->qat_sgl_src_phys_addr;
  67         req->comn_mid.src_length = req->comn_mid.dst_length = 0;
  68         return total_len;
  69 }
  70
  71 static __rte_always_inline void
  72 set_cipher_iv(struct icp_qat_fw_la_cipher_req_params *cipher_param,
  73                 struct rte_crypto_va_iova_ptr *iv_ptr, uint32_t iv_len,
  74                 struct icp_qat_fw_la_bulk_req *qat_req)
  75 {
  76         /* copy IV into request if it fits */
  77         if (iv_len <= sizeof(cipher_param->u.cipher_IV_array))
  78                 rte_memcpy(cipher_param->u.cipher_IV_array, iv_ptr->va,
  79                                 iv_len);
  80         else {
  81                 ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
  82                                 qat_req->comn_hdr.serv_specif_flags,
  83                                 ICP_QAT_FW_CIPH_IV_64BIT_PTR);
  84                 cipher_param->u.s.cipher_IV_ptr = iv_ptr->iova;
  85         }
  86 }
  87
  88 #define QAT_SYM_DP_IS_RESP_SUCCESS(resp) \
  89         (ICP_QAT_FW_COMN_STATUS_FLAG_OK == \
  90         ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(resp->comn_hdr.comn_status))
  91
  92 static __rte_always_inline void
  93 qat_sym_dp_fill_vec_status(int32_t *sta, int status, uint32_t n)
  94 {
  95         uint32_t i;
  96
  97         for (i = 0; i < n; i++)
  98                 sta[i] = status;
  99 }
 100
 101 #define QAT_SYM_DP_GET_MAX_ENQ(q, c, n) \
 102         RTE_MIN((q->max_inflights - q->enqueued + q->dequeued - c), n)
 103
 104 static __rte_always_inline void
 105 enqueue_one_aead_job(struct qat_sym_session *ctx,
 106         struct icp_qat_fw_la_bulk_req *req,
 107         struct rte_crypto_va_iova_ptr *iv,
 108         struct rte_crypto_va_iova_ptr *digest,
 109         struct rte_crypto_va_iova_ptr *aad,
 110         union rte_crypto_sym_ofs ofs, uint32_t data_len)
 111 {
 112         struct icp_qat_fw_la_cipher_req_params *cipher_param =
 113                 (void *)&req->serv_specif_rqpars;
 114         struct icp_qat_fw_la_auth_req_params *auth_param =
 115                 (void *)((uint8_t *)&req->serv_specif_rqpars +
 116                 ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
 117         uint8_t *aad_data;
 118         uint8_t aad_ccm_real_len;
 119         uint8_t aad_len_field_sz;
 120         uint32_t msg_len_be;
 121         rte_iova_t aad_iova = 0;
 122         uint8_t q;
 123
 124         switch (ctx->qat_hash_alg) {
 125         case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
 126         case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
 127                 ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
 128                         req->comn_hdr.serv_specif_flags,
 129                                 ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
 130                 rte_memcpy(cipher_param->u.cipher_IV_array, iv->va,
 131                                 ctx->cipher_iv.length);
 132                 aad_iova = aad->iova;
 133                 break;
 134         case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:
 135                 aad_data = aad->va;
 136                 aad_iova = aad->iova;
 137                 aad_ccm_real_len = 0;
 138                 aad_len_field_sz = 0;
 139                 msg_len_be = rte_bswap32((uint32_t)data_len -
 140                                 ofs.ofs.cipher.head);
 141
 142                 if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {
 143                         aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;
 144                         aad_ccm_real_len = ctx->aad_len -
 145                                 ICP_QAT_HW_CCM_AAD_B0_LEN -
 146                                 ICP_QAT_HW_CCM_AAD_LEN_INFO;
 147                 } else {
 148                         aad_data = iv->va;
 149                         aad_iova = iv->iova;
 150                 }
 151
 152                 q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length;
 153                 aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(
 154                         aad_len_field_sz, ctx->digest_length, q);
 155                 if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {
 156                         memcpy(aad_data + ctx->cipher_iv.length +
 157                                 ICP_QAT_HW_CCM_NONCE_OFFSET + (q -
 158                                 ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),
 159                                 (uint8_t *)&msg_len_be,
 160                                 ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);
 161                 } else {
 162                         memcpy(aad_data + ctx->cipher_iv.length +
 163                                 ICP_QAT_HW_CCM_NONCE_OFFSET,
 164                                 (uint8_t *)&msg_len_be +
 165                                 (ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE
 166                                 - q), q);
 167                 }
 168
 169                 if (aad_len_field_sz > 0) {
 170                         *(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] =
 171                                 rte_bswap16(aad_ccm_real_len);
 172
 173                         if ((aad_ccm_real_len + aad_len_field_sz)
 174                                 % ICP_QAT_HW_CCM_AAD_B0_LEN) {
 175                                 uint8_t pad_len = 0;
 176                                 uint8_t pad_idx = 0;
 177
 178                                 pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -
 179                                         ((aad_ccm_real_len +
 180                                         aad_len_field_sz) %
 181                                         ICP_QAT_HW_CCM_AAD_B0_LEN);
 182                                 pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +
 183                                         aad_ccm_real_len +
 184                                         aad_len_field_sz;
 185                                 memset(&aad_data[pad_idx], 0, pad_len);
 186                         }
 187                 }
 188
 189                 rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array)
 190                         + ICP_QAT_HW_CCM_NONCE_OFFSET,
 191                         (uint8_t *)iv->va +
 192                         ICP_QAT_HW_CCM_NONCE_OFFSET, ctx->cipher_iv.length);
 193                 *(uint8_t *)&cipher_param->u.cipher_IV_array[0] =
 194                         q - ICP_QAT_HW_CCM_NONCE_OFFSET;
 195
 196                 rte_memcpy((uint8_t *)aad->va +
 197                                 ICP_QAT_HW_CCM_NONCE_OFFSET,
 198                         (uint8_t *)iv->va + ICP_QAT_HW_CCM_NONCE_OFFSET,
 199                         ctx->cipher_iv.length);
 200                 break;
 201         default:
 202                 break;
 203         }
 204
 205         cipher_param->cipher_offset = ofs.ofs.cipher.head;
 206         cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
 207                         ofs.ofs.cipher.tail;
 208         auth_param->auth_off = ofs.ofs.cipher.head;
 209         auth_param->auth_len = cipher_param->cipher_length;
 210         auth_param->auth_res_addr = digest->iova;
 211         auth_param->u1.aad_adr = aad_iova;
 212
 213         if (ctx->is_single_pass) {
 214                 cipher_param->spc_aad_addr = aad_iova;
 215                 cipher_param->spc_auth_res_addr = digest->iova;
 216         }
 217 }
 218
 219 static __rte_always_inline int
 220 qat_sym_dp_enqueue_single_aead(void *qp_data, uint8_t *drv_ctx,
 221         struct rte_crypto_vec *data, uint16_t n_data_vecs,
 222         union rte_crypto_sym_ofs ofs,
 223         struct rte_crypto_va_iova_ptr *iv,
 224         struct rte_crypto_va_iova_ptr *digest,
 225         struct rte_crypto_va_iova_ptr *aad,
 226         void *user_data)
 227 {
 228         struct qat_qp *qp = qp_data;
 229         struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
 230         struct qat_queue *tx_queue = &qp->tx_q;
 231         struct qat_sym_session *ctx = dp_ctx->session;
 232         struct icp_qat_fw_la_bulk_req *req;
 233         int32_t data_len;
 234         uint32_t tail = dp_ctx->tail;
 235
 236         req = (struct icp_qat_fw_la_bulk_req *)(
 237                 (uint8_t *)tx_queue->base_addr + tail);
 238         tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
 239         rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 240         rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
 241         data_len = qat_sym_dp_parse_data_vec(qp, req, data, n_data_vecs);
 242         if (unlikely(data_len < 0))
 243                 return -1;
 244         req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data;
 245
 246         enqueue_one_aead_job(ctx, req, iv, digest, aad, ofs,
 247                 (uint32_t)data_len);
 248
 249         dp_ctx->tail = tail;
 250         dp_ctx->cached_enqueue++;
 251
 252         return 0;
 253 }
 254
 255 static __rte_always_inline uint32_t
 256 qat_sym_dp_enqueue_aead_jobs(void *qp_data, uint8_t *drv_ctx,
 257         struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
 258         void *user_data[], int *status)
 259 {
 260         struct qat_qp *qp = qp_data;
 261         struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
 262         struct qat_queue *tx_queue = &qp->tx_q;
 263         struct qat_sym_session *ctx = dp_ctx->session;
 264         uint32_t i, n;
 265         uint32_t tail;
 266         struct icp_qat_fw_la_bulk_req *req;
 267         int32_t data_len;
 268
 269         n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
 270         if (unlikely(n == 0)) {
 271                 qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
 272                 *status = 0;
 273                 return 0;
 274         }
 275
 276         tail = dp_ctx->tail;
 277
 278         for (i = 0; i < n; i++) {
 279                 req  = (struct icp_qat_fw_la_bulk_req *)(
 280                         (uint8_t *)tx_queue->base_addr + tail);
 281                 rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 282
 283                 data_len = qat_sym_dp_parse_data_vec(qp, req, vec->sgl[i].vec,
 284                         vec->sgl[i].num);
 285                 if (unlikely(data_len < 0))
 286                         break;
 287                 req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data[i];
 288                 enqueue_one_aead_job(ctx, req, &vec->iv[i], &vec->digest[i],
 289                         &vec->aad[i], ofs, (uint32_t)data_len);
 290                 tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
 291         }
 292
 293         if (unlikely(i < n))
 294                 qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
 295
 296         dp_ctx->tail = tail;
 297         dp_ctx->cached_enqueue += i;
 298         *status = 0;
 299         return i;
 300 }
 301
 302 static __rte_always_inline void
 303 enqueue_one_cipher_job(struct qat_sym_session *ctx,
 304         struct icp_qat_fw_la_bulk_req *req,
 305         struct rte_crypto_va_iova_ptr *iv,
 306         union rte_crypto_sym_ofs ofs, uint32_t data_len)
 307 {
 308         struct icp_qat_fw_la_cipher_req_params *cipher_param;
 309
 310         cipher_param = (void *)&req->serv_specif_rqpars;
 311
 312         /* cipher IV */
 313         set_cipher_iv(cipher_param, iv, ctx->cipher_iv.length, req);
 314         cipher_param->cipher_offset = ofs.ofs.cipher.head;
 315         cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
 316                         ofs.ofs.cipher.tail;
 317 }
 318
 319 static __rte_always_inline int
 320 qat_sym_dp_enqueue_single_cipher(void *qp_data, uint8_t *drv_ctx,
 321         struct rte_crypto_vec *data, uint16_t n_data_vecs,
 322         union rte_crypto_sym_ofs ofs,
 323         struct rte_crypto_va_iova_ptr *iv,
 324         struct rte_crypto_va_iova_ptr *digest __rte_unused,
 325         struct rte_crypto_va_iova_ptr *aad __rte_unused,
 326         void *user_data)
 327 {
 328         struct qat_qp *qp = qp_data;
 329         struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
 330         struct qat_queue *tx_queue = &qp->tx_q;
 331         struct qat_sym_session *ctx = dp_ctx->session;
 332         struct icp_qat_fw_la_bulk_req *req;
 333         int32_t data_len;
 334         uint32_t tail = dp_ctx->tail;
 335
 336         req = (struct icp_qat_fw_la_bulk_req *)(
 337                 (uint8_t *)tx_queue->base_addr + tail);
 338         tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
 339         rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 340         rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
 341         data_len = qat_sym_dp_parse_data_vec(qp, req, data, n_data_vecs);
 342         if (unlikely(data_len < 0))
 343                 return -1;
 344         req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data;
 345
 346         enqueue_one_cipher_job(ctx, req, iv, ofs, (uint32_t)data_len);
 347
 348         dp_ctx->tail = tail;
 349         dp_ctx->cached_enqueue++;
 350
 351         return 0;
 352 }
 353
 354 static __rte_always_inline uint32_t
 355 qat_sym_dp_enqueue_cipher_jobs(void *qp_data, uint8_t *drv_ctx,
 356         struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
 357         void *user_data[], int *status)
 358 {
 359         struct qat_qp *qp = qp_data;
 360         struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
 361         struct qat_queue *tx_queue = &qp->tx_q;
 362         struct qat_sym_session *ctx = dp_ctx->session;
 363         uint32_t i, n;
 364         uint32_t tail;
 365         struct icp_qat_fw_la_bulk_req *req;
 366         int32_t data_len;
 367
 368         n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
 369         if (unlikely(n == 0)) {
 370                 qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
 371                 *status = 0;
 372                 return 0;
 373         }
 374
 375         tail = dp_ctx->tail;
 376
 377         for (i = 0; i < n; i++) {
 378                 req  = (struct icp_qat_fw_la_bulk_req *)(
 379                         (uint8_t *)tx_queue->base_addr + tail);
 380                 rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 381
 382                 data_len = qat_sym_dp_parse_data_vec(qp, req, vec->sgl[i].vec,
 383                         vec->sgl[i].num);
 384                 if (unlikely(data_len < 0))
 385                         break;
 386                 req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data[i];
 387                 enqueue_one_cipher_job(ctx, req, &vec->iv[i], ofs,
 388                         (uint32_t)data_len);
 389                 tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
 390         }
 391
 392         if (unlikely(i < n))
 393                 qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
 394
 395         dp_ctx->tail = tail;
 396         dp_ctx->cached_enqueue += i;
 397         *status = 0;
 398         return i;
 399 }
 400
 401 static __rte_always_inline void
 402 enqueue_one_auth_job(struct qat_sym_session *ctx,
 403         struct icp_qat_fw_la_bulk_req *req,
 404         struct rte_crypto_va_iova_ptr *digest,
 405         struct rte_crypto_va_iova_ptr *auth_iv,
 406         union rte_crypto_sym_ofs ofs, uint32_t data_len)
 407 {
 408         struct icp_qat_fw_la_cipher_req_params *cipher_param;
 409         struct icp_qat_fw_la_auth_req_params *auth_param;
 410
 411         cipher_param = (void *)&req->serv_specif_rqpars;
 412         auth_param = (void *)((uint8_t *)cipher_param +
 413                         ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
 414
 415         auth_param->auth_off = ofs.ofs.auth.head;
 416         auth_param->auth_len = data_len - ofs.ofs.auth.head -
 417                         ofs.ofs.auth.tail;
 418         auth_param->auth_res_addr = digest->iova;
 419
 420         switch (ctx->qat_hash_alg) {
 421         case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
 422         case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
 423         case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
 424                 auth_param->u1.aad_adr = auth_iv->iova;
 425                 break;
 426         case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
 427         case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
 428                 ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
 429                         req->comn_hdr.serv_specif_flags,
 430                                 ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
 431                 rte_memcpy(cipher_param->u.cipher_IV_array, auth_iv->va,
 432                                 ctx->auth_iv.length);
 433                 break;
 434         default:
 435                 break;
 436         }
 437 }
 438
 439 static __rte_always_inline int
 440 qat_sym_dp_enqueue_single_auth(void *qp_data, uint8_t *drv_ctx,
 441         struct rte_crypto_vec *data, uint16_t n_data_vecs,
 442         union rte_crypto_sym_ofs ofs,
 443         struct rte_crypto_va_iova_ptr *iv __rte_unused,
 444         struct rte_crypto_va_iova_ptr *digest,
 445         struct rte_crypto_va_iova_ptr *auth_iv,
 446         void *user_data)
 447 {
 448         struct qat_qp *qp = qp_data;
 449         struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
 450         struct qat_queue *tx_queue = &qp->tx_q;
 451         struct qat_sym_session *ctx = dp_ctx->session;
 452         struct icp_qat_fw_la_bulk_req *req;
 453         int32_t data_len;
 454         uint32_t tail = dp_ctx->tail;
 455
 456         req = (struct icp_qat_fw_la_bulk_req *)(
 457                 (uint8_t *)tx_queue->base_addr + tail);
 458         tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
 459         rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 460         rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
 461         data_len = qat_sym_dp_parse_data_vec(qp, req, data, n_data_vecs);
 462         if (unlikely(data_len < 0))
 463                 return -1;
 464         req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data;
 465
 466         enqueue_one_auth_job(ctx, req, digest, auth_iv, ofs,
 467                         (uint32_t)data_len);
 468
 469         dp_ctx->tail = tail;
 470         dp_ctx->cached_enqueue++;
 471
 472         return 0;
 473 }
 474
 475 static __rte_always_inline uint32_t
 476 qat_sym_dp_enqueue_auth_jobs(void *qp_data, uint8_t *drv_ctx,
 477         struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
 478         void *user_data[], int *status)
 479 {
 480         struct qat_qp *qp = qp_data;
 481         struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
 482         struct qat_queue *tx_queue = &qp->tx_q;
 483         struct qat_sym_session *ctx = dp_ctx->session;
 484         uint32_t i, n;
 485         uint32_t tail;
 486         struct icp_qat_fw_la_bulk_req *req;
 487         int32_t data_len;
 488
 489         n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
 490         if (unlikely(n == 0)) {
 491                 qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
 492                 *status = 0;
 493                 return 0;
 494         }
 495
 496         tail = dp_ctx->tail;
 497
 498         for (i = 0; i < n; i++) {
 499                 req  = (struct icp_qat_fw_la_bulk_req *)(
 500                         (uint8_t *)tx_queue->base_addr + tail);
 501                 rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 502
 503                 data_len = qat_sym_dp_parse_data_vec(qp, req, vec->sgl[i].vec,
 504                         vec->sgl[i].num);
 505                 if (unlikely(data_len < 0))
 506                         break;
 507                 req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data[i];
 508                 enqueue_one_auth_job(ctx, req, &vec->digest[i],
 509                         &vec->auth_iv[i], ofs, (uint32_t)data_len);
 510                 tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
 511         }
 512
 513         if (unlikely(i < n))
 514                 qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
 515
 516         dp_ctx->tail = tail;
 517         dp_ctx->cached_enqueue += i;
 518         *status = 0;
 519         return i;
 520 }
 521
 522 static __rte_always_inline int
 523 enqueue_one_chain_job(struct qat_sym_session *ctx,
 524         struct icp_qat_fw_la_bulk_req *req,
 525         struct rte_crypto_vec *data,
 526         uint16_t n_data_vecs,
 527         struct rte_crypto_va_iova_ptr *cipher_iv,
 528         struct rte_crypto_va_iova_ptr *digest,
 529         struct rte_crypto_va_iova_ptr *auth_iv,
 530         union rte_crypto_sym_ofs ofs, uint32_t data_len)
 531 {
 532         struct icp_qat_fw_la_cipher_req_params *cipher_param;
 533         struct icp_qat_fw_la_auth_req_params *auth_param;
 534         rte_iova_t auth_iova_end;
 535         int32_t cipher_len, auth_len;
 536
 537         cipher_param = (void *)&req->serv_specif_rqpars;
 538         auth_param = (void *)((uint8_t *)cipher_param +
 539                         ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
 540
 541         cipher_len = data_len - ofs.ofs.cipher.head -
 542                         ofs.ofs.cipher.tail;
 543         auth_len = data_len - ofs.ofs.auth.head - ofs.ofs.auth.tail;
 544
 545         if (unlikely(cipher_len < 0 || auth_len < 0))
 546                 return -1;
 547
 548         cipher_param->cipher_offset = ofs.ofs.cipher.head;
 549         cipher_param->cipher_length = cipher_len;
 550         set_cipher_iv(cipher_param, cipher_iv, ctx->cipher_iv.length, req);
 551
 552         auth_param->auth_off = ofs.ofs.auth.head;
 553         auth_param->auth_len = auth_len;
 554         auth_param->auth_res_addr = digest->iova;
 555
 556         switch (ctx->qat_hash_alg) {
 557         case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
 558         case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
 559         case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
 560                 auth_param->u1.aad_adr = auth_iv->iova;
 561
 562                 if (unlikely(n_data_vecs > 1)) {
 563                         int auth_end_get = 0, i = n_data_vecs - 1;
 564                         struct rte_crypto_vec *cvec = &data[0];
 565                         uint32_t len;
 566
 567                         len = data_len - ofs.ofs.auth.tail;
 568
 569                         while (i >= 0 && len > 0) {
 570                                 if (cvec->len >= len) {
 571                                         auth_iova_end = cvec->iova +
 572                                                 (cvec->len - len);
 573                                         len = 0;
 574                                         auth_end_get = 1;
 575                                         break;
 576                                 }
 577                                 len -= cvec->len;
 578                                 i--;
 579                                 cvec++;
 580                         }
 581
 582                         if (unlikely(auth_end_get == 0))
 583                                 return -1;
 584                 } else
 585                         auth_iova_end = data[0].iova + auth_param->auth_off +
 586                                 auth_param->auth_len;
 587
 588                 /* Then check if digest-encrypted conditions are met */
 589                 if ((auth_param->auth_off + auth_param->auth_len <
 590                         cipher_param->cipher_offset +
 591                         cipher_param->cipher_length) &&
 592                         (digest->iova == auth_iova_end)) {
 593                         /* Handle partial digest encryption */
 594                         if (cipher_param->cipher_offset +
 595                                         cipher_param->cipher_length <
 596                                         auth_param->auth_off +
 597                                         auth_param->auth_len +
 598                                         ctx->digest_length)
 599                                 req->comn_mid.dst_length =
 600                                         req->comn_mid.src_length =
 601                                         auth_param->auth_off +
 602                                         auth_param->auth_len +
 603                                         ctx->digest_length;
 604                         struct icp_qat_fw_comn_req_hdr *header =
 605                                 &req->comn_hdr;
 606                         ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(
 607                                 header->serv_specif_flags,
 608                                 ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
 609                 }
 610                 break;
 611         case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
 612         case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
 613                 break;
 614         default:
 615                 break;
 616         }
 617
 618         return 0;
 619 }
 620
 621 static __rte_always_inline int
 622 qat_sym_dp_enqueue_single_chain(void *qp_data, uint8_t *drv_ctx,
 623         struct rte_crypto_vec *data, uint16_t n_data_vecs,
 624         union rte_crypto_sym_ofs ofs,
 625         struct rte_crypto_va_iova_ptr *cipher_iv,
 626         struct rte_crypto_va_iova_ptr *digest,
 627         struct rte_crypto_va_iova_ptr *auth_iv,
 628         void *user_data)
 629 {
 630         struct qat_qp *qp = qp_data;
 631         struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
 632         struct qat_queue *tx_queue = &qp->tx_q;
 633         struct qat_sym_session *ctx = dp_ctx->session;
 634         struct icp_qat_fw_la_bulk_req *req;
 635         int32_t data_len;
 636         uint32_t tail = dp_ctx->tail;
 637
 638         req = (struct icp_qat_fw_la_bulk_req *)(
 639                 (uint8_t *)tx_queue->base_addr + tail);
 640         tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
 641         rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 642         rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
 643         data_len = qat_sym_dp_parse_data_vec(qp, req, data, n_data_vecs);
 644         if (unlikely(data_len < 0))
 645                 return -1;
 646         req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data;
 647
 648         if (unlikely(enqueue_one_chain_job(ctx, req, data, n_data_vecs,
 649                         cipher_iv, digest, auth_iv, ofs, (uint32_t)data_len)))
 650                 return -1;
 651
 652         dp_ctx->tail = tail;
 653         dp_ctx->cached_enqueue++;
 654
 655         return 0;
 656 }
 657
 658 static __rte_always_inline uint32_t
 659 qat_sym_dp_enqueue_chain_jobs(void *qp_data, uint8_t *drv_ctx,
 660         struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
 661         void *user_data[], int *status)
 662 {
 663         struct qat_qp *qp = qp_data;
 664         struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
 665         struct qat_queue *tx_queue = &qp->tx_q;
 666         struct qat_sym_session *ctx = dp_ctx->session;
 667         uint32_t i, n;
 668         uint32_t tail;
 669         struct icp_qat_fw_la_bulk_req *req;
 670         int32_t data_len;
 671
 672         n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
 673         if (unlikely(n == 0)) {
 674                 qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
 675                 *status = 0;
 676                 return 0;
 677         }
 678
 679         tail = dp_ctx->tail;
 680
 681         for (i = 0; i < n; i++) {
 682                 req  = (struct icp_qat_fw_la_bulk_req *)(
 683                         (uint8_t *)tx_queue->base_addr + tail);
 684                 rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 685
 686                 data_len = qat_sym_dp_parse_data_vec(qp, req, vec->sgl[i].vec,
 687                         vec->sgl[i].num);
 688                 if (unlikely(data_len < 0))
 689                         break;
 690                 req->comn_mid.opaque_data = (uint64_t)(uintptr_t)user_data[i];
 691                 if (unlikely(enqueue_one_chain_job(ctx, req, vec->sgl[i].vec,
 692                         vec->sgl[i].num, &vec->iv[i], &vec->digest[i],
 693                                 &vec->auth_iv[i], ofs, (uint32_t)data_len)))
 694                         break;
 695
 696                 tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
 697         }
 698
 699         if (unlikely(i < n))
 700                 qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
 701
 702         dp_ctx->tail = tail;
 703         dp_ctx->cached_enqueue += i;
 704         *status = 0;
 705         return i;
 706 }
 707
 708 static __rte_always_inline uint32_t
 709 qat_sym_dp_dequeue_burst(void *qp_data, uint8_t *drv_ctx,
 710         rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,
 711         rte_cryptodev_raw_post_dequeue_t post_dequeue,
 712         void **out_user_data, uint8_t is_user_data_array,
 713         uint32_t *n_success_jobs, int *return_status)
 714 {
 715         struct qat_qp *qp = qp_data;
 716         struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
 717         struct qat_queue *rx_queue = &qp->rx_q;
 718         struct icp_qat_fw_comn_resp *resp;
 719         void *resp_opaque;
 720         uint32_t i, n, inflight;
 721         uint32_t head;
 722         uint8_t status;
 723
 724         *n_success_jobs = 0;
 725         *return_status = 0;
 726         head = dp_ctx->head;
 727
 728         inflight = qp->enqueued - qp->dequeued;
 729         if (unlikely(inflight == 0))
 730                 return 0;
 731
 732         resp = (struct icp_qat_fw_comn_resp *)((uint8_t *)rx_queue->base_addr +
 733                         head);
 734         /* no operation ready */
 735         if (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))
 736                 return 0;
 737
 738         resp_opaque = (void *)(uintptr_t)resp->opaque_data;
 739         /* get the dequeue count */
 740         n = get_dequeue_count(resp_opaque);
 741         if (unlikely(n == 0))
 742                 return 0;
 743
 744         out_user_data[0] = resp_opaque;
 745         status = QAT_SYM_DP_IS_RESP_SUCCESS(resp);
 746         post_dequeue(resp_opaque, 0, status);
 747         *n_success_jobs += status;
 748
 749         head = (head + rx_queue->msg_size) & rx_queue->modulo_mask;
 750
 751         /* we already finished dequeue when n == 1 */
 752         if (unlikely(n == 1)) {
 753                 i = 1;
 754                 goto end_deq;
 755         }
 756
 757         if (is_user_data_array) {
 758                 for (i = 1; i < n; i++) {
 759                         resp = (struct icp_qat_fw_comn_resp *)(
 760                                 (uint8_t *)rx_queue->base_addr + head);
 761                         if (unlikely(*(uint32_t *)resp ==
 762                                         ADF_RING_EMPTY_SIG))
 763                                 goto end_deq;
 764                         out_user_data[i] = (void *)(uintptr_t)resp->opaque_data;
 765                         status = QAT_SYM_DP_IS_RESP_SUCCESS(resp);
 766                         *n_success_jobs += status;
 767                         post_dequeue(out_user_data[i], i, status);
 768                         head = (head + rx_queue->msg_size) &
 769                                         rx_queue->modulo_mask;
 770                 }
 771
 772                 goto end_deq;
 773         }
 774
 775         /* opaque is not array */
 776         for (i = 1; i < n; i++) {
 777                 resp = (struct icp_qat_fw_comn_resp *)(
 778                         (uint8_t *)rx_queue->base_addr + head);
 779                 status = QAT_SYM_DP_IS_RESP_SUCCESS(resp);
 780                 if (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))
 781                         goto end_deq;
 782                 head = (head + rx_queue->msg_size) &
 783                                 rx_queue->modulo_mask;
 784                 post_dequeue(resp_opaque, i, status);
 785                 *n_success_jobs += status;
 786         }
 787
 788 end_deq:
 789         dp_ctx->head = head;
 790         dp_ctx->cached_dequeue += i;
 791         return i;
 792 }
 793
 794 static __rte_always_inline void *
 795 qat_sym_dp_dequeue(void *qp_data, uint8_t *drv_ctx, int *dequeue_status,
 796                 enum rte_crypto_op_status *op_status)
 797 {
 798         struct qat_qp *qp = qp_data;
 799         struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
 800         struct qat_queue *rx_queue = &qp->rx_q;
 801         register struct icp_qat_fw_comn_resp *resp;
 802
 803         resp = (struct icp_qat_fw_comn_resp *)((uint8_t *)rx_queue->base_addr +
 804                         dp_ctx->head);
 805
 806         if (unlikely(*(uint32_t *)resp == ADF_RING_EMPTY_SIG))
 807                 return NULL;
 808
 809         dp_ctx->head = (dp_ctx->head + rx_queue->msg_size) &
 810                         rx_queue->modulo_mask;
 811         dp_ctx->cached_dequeue++;
 812
 813         *op_status = QAT_SYM_DP_IS_RESP_SUCCESS(resp) ?
 814                         RTE_CRYPTO_OP_STATUS_SUCCESS :
 815                         RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 816         *dequeue_status = 0;
 817         return (void *)(uintptr_t)resp->opaque_data;
 818 }
 819
 820 static __rte_always_inline int
 821 qat_sym_dp_kick_tail(void *qp_data, uint8_t *drv_ctx, uint32_t n)
 822 {
 823         struct qat_qp *qp = qp_data;
 824         struct qat_queue *tx_queue = &qp->tx_q;
 825         struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
 826
 827         if (unlikely(dp_ctx->cached_enqueue != n))
 828                 return -1;
 829
 830         qp->enqueued += n;
 831         qp->stats.enqueued_count += n;
 832
 833         tx_queue->tail = dp_ctx->tail;
 834
 835         WRITE_CSR_RING_TAIL(qp->mmap_bar_addr,
 836                         tx_queue->hw_bundle_number,
 837                         tx_queue->hw_queue_number, tx_queue->tail);
 838         tx_queue->csr_tail = tx_queue->tail;
 839         dp_ctx->cached_enqueue = 0;
 840
 841         return 0;
 842 }
 843
 844 static __rte_always_inline int
 845 qat_sym_dp_update_head(void *qp_data, uint8_t *drv_ctx, uint32_t n)
 846 {
 847         struct qat_qp *qp = qp_data;
 848         struct qat_queue *rx_queue = &qp->rx_q;
 849         struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
 850
 851         if (unlikely(dp_ctx->cached_dequeue != n))
 852                 return -1;
 853
 854         rx_queue->head = dp_ctx->head;
 855         rx_queue->nb_processed_responses += n;
 856         qp->dequeued += n;
 857         qp->stats.dequeued_count += n;
 858         if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH) {
 859                 uint32_t old_head, new_head;
 860                 uint32_t max_head;
 861
 862                 old_head = rx_queue->csr_head;
 863                 new_head = rx_queue->head;
 864                 max_head = qp->nb_descriptors * rx_queue->msg_size;
 865
 866                 /* write out free descriptors */
 867                 void *cur_desc = (uint8_t *)rx_queue->base_addr + old_head;
 868
 869                 if (new_head < old_head) {
 870                         memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE,
 871                                         max_head - old_head);
 872                         memset(rx_queue->base_addr, ADF_RING_EMPTY_SIG_BYTE,
 873                                         new_head);
 874                 } else {
 875                         memset(cur_desc, ADF_RING_EMPTY_SIG_BYTE, new_head -
 876                                         old_head);
 877                 }
 878                 rx_queue->nb_processed_responses = 0;
 879                 rx_queue->csr_head = new_head;
 880
 881                 /* write current head to CSR */
 882                 WRITE_CSR_RING_HEAD(qp->mmap_bar_addr,
 883                         rx_queue->hw_bundle_number, rx_queue->hw_queue_number,
 884                         new_head);
 885         }
 886
 887         dp_ctx->cached_dequeue = 0;
 888         return 0;
 889 }
 890
 891 int
 892 qat_sym_configure_dp_ctx(struct rte_cryptodev *dev, uint16_t qp_id,
 893         struct rte_crypto_raw_dp_ctx *raw_dp_ctx,
 894         enum rte_crypto_op_sess_type sess_type,
 895         union rte_cryptodev_session_ctx session_ctx, uint8_t is_update)
 896 {
 897         struct qat_qp *qp;
 898         struct qat_sym_session *ctx;
 899         struct qat_sym_dp_ctx *dp_ctx;
 900
 901         qp = dev->data->queue_pairs[qp_id];
 902         dp_ctx = (struct qat_sym_dp_ctx *)raw_dp_ctx->drv_ctx_data;
 903
 904         if (!is_update) {
 905                 memset(raw_dp_ctx, 0, sizeof(*raw_dp_ctx) +
 906                                 sizeof(struct qat_sym_dp_ctx));
 907                 raw_dp_ctx->qp_data = dev->data->queue_pairs[qp_id];
 908                 dp_ctx->tail = qp->tx_q.tail;
 909                 dp_ctx->head = qp->rx_q.head;
 910                 dp_ctx->cached_enqueue = dp_ctx->cached_dequeue = 0;
 911         }
 912
 913         if (sess_type != RTE_CRYPTO_OP_WITH_SESSION)
 914                 return -EINVAL;
 915
 916         ctx = (struct qat_sym_session *)get_sym_session_private_data(
 917                         session_ctx.crypto_sess, qat_sym_driver_id);
 918
 919         dp_ctx->session = ctx;
 920
 921         raw_dp_ctx->enqueue_done = qat_sym_dp_kick_tail;
 922         raw_dp_ctx->dequeue_burst = qat_sym_dp_dequeue_burst;
 923         raw_dp_ctx->dequeue = qat_sym_dp_dequeue;
 924         raw_dp_ctx->dequeue_done = qat_sym_dp_update_head;
 925
 926         if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER ||
 927                         ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) {
 928                 /* AES-GCM or AES-CCM */
 929                 if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
 930                         ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 ||
 931                         (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_AES128
 932                         && ctx->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE
 933                         && ctx->qat_hash_alg ==
 934                                         ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) {
 935                         raw_dp_ctx->enqueue_burst =
 936                                         qat_sym_dp_enqueue_aead_jobs;
 937                         raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_aead;
 938                 } else {
 939                         raw_dp_ctx->enqueue_burst =
 940                                         qat_sym_dp_enqueue_chain_jobs;
 941                         raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_chain;
 942                 }
 943         } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH) {
 944                 raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_auth_jobs;
 945                 raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_auth;
 946         } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) {
 947                 raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_cipher_jobs;
 948                 raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_cipher;
 949         } else
 950                 return -1;
 951
 952         return 0;
 953 }
 954
 955 int
 956 qat_sym_get_dp_ctx_size(__rte_unused struct rte_cryptodev *dev)
 957 {
 958         return sizeof(struct qat_sym_dp_ctx);
 959 }