drivers/crypto/qat/dev/qat_crypto_pmd_gen3.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017-2022 Intel Corporation
   3  */
   4
   5 #include <rte_cryptodev.h>
   6 #include <cryptodev_pmd.h>
   7 #include "qat_sym_session.h"
   8 #include "qat_sym.h"
   9 #include "qat_asym.h"
  10 #include "qat_crypto.h"
  11 #include "qat_crypto_pmd_gens.h"
  12
  13 static struct rte_cryptodev_capabilities qat_sym_crypto_caps_gen3[] = {
  14         QAT_SYM_PLAIN_AUTH_CAP(SHA1,
  15                 CAP_SET(block_size, 64),
  16                 CAP_RNG(digest_size, 1, 20, 1)),
  17         QAT_SYM_AEAD_CAP(AES_GCM,
  18                 CAP_SET(block_size, 16),
  19                 CAP_RNG(key_size, 16, 32, 8), CAP_RNG(digest_size, 8, 16, 4),
  20                 CAP_RNG(aad_size, 0, 240, 1), CAP_RNG(iv_size, 0, 12, 12)),
  21         QAT_SYM_AEAD_CAP(AES_CCM,
  22                 CAP_SET(block_size, 16),
  23                 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 16, 2),
  24                 CAP_RNG(aad_size, 0, 224, 1), CAP_RNG(iv_size, 7, 13, 1)),
  25         QAT_SYM_AUTH_CAP(AES_GMAC,
  26                 CAP_SET(block_size, 16),
  27                 CAP_RNG(key_size, 16, 32, 8), CAP_RNG(digest_size, 8, 16, 4),
  28                 CAP_RNG_ZERO(aad_size), CAP_RNG(iv_size, 0, 12, 12)),
  29         QAT_SYM_AUTH_CAP(AES_CMAC,
  30                 CAP_SET(block_size, 16),
  31                 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 16, 4),
  32                         CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
  33         QAT_SYM_AUTH_CAP(SHA224,
  34                 CAP_SET(block_size, 64),
  35                 CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 28, 1),
  36                 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
  37         QAT_SYM_AUTH_CAP(SHA256,
  38                 CAP_SET(block_size, 64),
  39                 CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 32, 1),
  40                 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
  41         QAT_SYM_AUTH_CAP(SHA384,
  42                 CAP_SET(block_size, 128),
  43                 CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 48, 1),
  44                 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
  45         QAT_SYM_AUTH_CAP(SHA512,
  46                 CAP_SET(block_size, 128),
  47                 CAP_RNG_ZERO(key_size), CAP_RNG(digest_size, 1, 64, 1),
  48                 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
  49         QAT_SYM_AUTH_CAP(SHA1_HMAC,
  50                 CAP_SET(block_size, 64),
  51                 CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 20, 1),
  52                 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
  53         QAT_SYM_AUTH_CAP(SHA224_HMAC,
  54                 CAP_SET(block_size, 64),
  55                 CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 28, 1),
  56                 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
  57         QAT_SYM_AUTH_CAP(SHA256_HMAC,
  58                 CAP_SET(block_size, 64),
  59                 CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 32, 1),
  60                 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
  61         QAT_SYM_AUTH_CAP(SHA384_HMAC,
  62                 CAP_SET(block_size, 128),
  63                 CAP_RNG(key_size, 1, 128, 1), CAP_RNG(digest_size, 1, 48, 1),
  64                 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
  65         QAT_SYM_AUTH_CAP(SHA512_HMAC,
  66                 CAP_SET(block_size, 128),
  67                 CAP_RNG(key_size, 1, 128, 1), CAP_RNG(digest_size, 1, 64, 1),
  68                 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
  69         QAT_SYM_AUTH_CAP(MD5_HMAC,
  70                 CAP_SET(block_size, 64),
  71                 CAP_RNG(key_size, 1, 64, 1), CAP_RNG(digest_size, 1, 16, 1),
  72                 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
  73         QAT_SYM_AUTH_CAP(AES_XCBC_MAC,
  74                 CAP_SET(block_size, 16),
  75                 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 12, 12, 0),
  76                 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
  77         QAT_SYM_AUTH_CAP(SNOW3G_UIA2,
  78                 CAP_SET(block_size, 16),
  79                 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 4, 0),
  80                 CAP_RNG_ZERO(aad_size), CAP_RNG(iv_size, 16, 16, 0)),
  81         QAT_SYM_AUTH_CAP(KASUMI_F9,
  82                 CAP_SET(block_size, 8),
  83                 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 4, 0),
  84                 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
  85         QAT_SYM_AUTH_CAP(NULL,
  86                 CAP_SET(block_size, 1),
  87                 CAP_RNG_ZERO(key_size), CAP_RNG_ZERO(digest_size),
  88                 CAP_RNG_ZERO(aad_size), CAP_RNG_ZERO(iv_size)),
  89         QAT_SYM_CIPHER_CAP(AES_CBC,
  90                 CAP_SET(block_size, 16),
  91                 CAP_RNG(key_size, 16, 32, 8), CAP_RNG(iv_size, 16, 16, 0)),
  92         QAT_SYM_CIPHER_CAP(AES_CTR,
  93                 CAP_SET(block_size, 16),
  94                 CAP_RNG(key_size, 16, 32, 8), CAP_RNG(iv_size, 16, 16, 0)),
  95         QAT_SYM_CIPHER_CAP(AES_XTS,
  96                 CAP_SET(block_size, 16),
  97                 CAP_RNG(key_size, 32, 64, 32), CAP_RNG(iv_size, 16, 16, 0)),
  98         QAT_SYM_CIPHER_CAP(AES_DOCSISBPI,
  99                 CAP_SET(block_size, 16),
 100                 CAP_RNG(key_size, 16, 32, 16), CAP_RNG(iv_size, 16, 16, 0)),
 101         QAT_SYM_CIPHER_CAP(SNOW3G_UEA2,
 102                 CAP_SET(block_size, 16),
 103                 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 16, 16, 0)),
 104         QAT_SYM_CIPHER_CAP(KASUMI_F8,
 105                 CAP_SET(block_size, 8),
 106                 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 8, 8, 0)),
 107         QAT_SYM_CIPHER_CAP(NULL,
 108                 CAP_SET(block_size, 1),
 109                 CAP_RNG_ZERO(key_size), CAP_RNG_ZERO(iv_size)),
 110         QAT_SYM_CIPHER_CAP(3DES_CBC,
 111                 CAP_SET(block_size, 8),
 112                 CAP_RNG(key_size, 8, 24, 8), CAP_RNG(iv_size, 8, 8, 0)),
 113         QAT_SYM_CIPHER_CAP(3DES_CTR,
 114                 CAP_SET(block_size, 8),
 115                 CAP_RNG(key_size, 16, 24, 8), CAP_RNG(iv_size, 8, 8, 0)),
 116         QAT_SYM_CIPHER_CAP(DES_CBC,
 117                 CAP_SET(block_size, 8),
 118                 CAP_RNG(key_size, 8, 24, 8), CAP_RNG(iv_size, 8, 8, 0)),
 119         QAT_SYM_CIPHER_CAP(DES_DOCSISBPI,
 120                 CAP_SET(block_size, 8),
 121                 CAP_RNG(key_size, 8, 8, 0), CAP_RNG(iv_size, 8, 8, 0)),
 122         QAT_SYM_CIPHER_CAP(ZUC_EEA3,
 123                 CAP_SET(block_size, 16),
 124                 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(iv_size, 16, 16, 0)),
 125         QAT_SYM_AUTH_CAP(ZUC_EIA3,
 126                 CAP_SET(block_size, 16),
 127                 CAP_RNG(key_size, 16, 16, 0), CAP_RNG(digest_size, 4, 4, 0),
 128                 CAP_RNG_ZERO(aad_size), CAP_RNG(iv_size, 16, 16, 0)),
 129         QAT_SYM_AEAD_CAP(CHACHA20_POLY1305,
 130                 CAP_SET(block_size, 64),
 131                 CAP_RNG(key_size, 32, 32, 0),
 132                 CAP_RNG(digest_size, 16, 16, 0),
 133                 CAP_RNG(aad_size, 0, 240, 1), CAP_RNG(iv_size, 12, 12, 0)),
 134         RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 135 };
 136
 137 static struct qat_capabilities_info
 138 qat_sym_crypto_cap_get_gen3(struct qat_pci_device *qat_dev __rte_unused)
 139 {
 140         struct qat_capabilities_info capa_info;
 141         capa_info.data = qat_sym_crypto_caps_gen3;
 142         capa_info.size = sizeof(qat_sym_crypto_caps_gen3);
 143         return capa_info;
 144 }
 145
 146 static __rte_always_inline void
 147 enqueue_one_aead_job_gen3(struct qat_sym_session *ctx,
 148         struct icp_qat_fw_la_bulk_req *req,
 149         struct rte_crypto_va_iova_ptr *iv,
 150         struct rte_crypto_va_iova_ptr *digest,
 151         struct rte_crypto_va_iova_ptr *aad,
 152         union rte_crypto_sym_ofs ofs, uint32_t data_len)
 153 {
 154         if (ctx->is_single_pass) {
 155                 struct icp_qat_fw_la_cipher_req_params *cipher_param =
 156                         (void *)&req->serv_specif_rqpars;
 157
 158                 /* QAT GEN3 uses single pass to treat AEAD as
 159                  * cipher operation
 160                  */
 161                 cipher_param = (void *)&req->serv_specif_rqpars;
 162
 163                 qat_set_cipher_iv(cipher_param, iv, ctx->cipher_iv.length, req);
 164                 cipher_param->cipher_offset = ofs.ofs.cipher.head;
 165                 cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
 166                                 ofs.ofs.cipher.tail;
 167
 168                 cipher_param->spc_aad_addr = aad->iova;
 169                 cipher_param->spc_auth_res_addr = digest->iova;
 170
 171                 return;
 172         }
 173
 174         enqueue_one_aead_job_gen1(ctx, req, iv, digest, aad, ofs, data_len);
 175 }
 176
 177 static __rte_always_inline void
 178 enqueue_one_auth_job_gen3(struct qat_sym_session *ctx,
 179         struct qat_sym_op_cookie *cookie,
 180         struct icp_qat_fw_la_bulk_req *req,
 181         struct rte_crypto_va_iova_ptr *digest,
 182         struct rte_crypto_va_iova_ptr *auth_iv,
 183         union rte_crypto_sym_ofs ofs, uint32_t data_len)
 184 {
 185         struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl;
 186         struct icp_qat_fw_la_cipher_req_params *cipher_param;
 187         uint32_t ver_key_offset;
 188         uint32_t auth_data_len = data_len - ofs.ofs.auth.head -
 189                         ofs.ofs.auth.tail;
 190
 191         if (!ctx->is_single_pass_gmac ||
 192                         (auth_data_len > QAT_AES_GMAC_SPC_MAX_SIZE)) {
 193                 enqueue_one_auth_job_gen1(ctx, req, digest, auth_iv, ofs,
 194                                 data_len);
 195                 return;
 196         }
 197
 198         cipher_cd_ctrl = (void *) &req->cd_ctrl;
 199         cipher_param = (void *)&req->serv_specif_rqpars;
 200         ver_key_offset = sizeof(struct icp_qat_hw_auth_setup) +
 201                         ICP_QAT_HW_GALOIS_128_STATE1_SZ +
 202                         ICP_QAT_HW_GALOIS_H_SZ + ICP_QAT_HW_GALOIS_LEN_A_SZ +
 203                         ICP_QAT_HW_GALOIS_E_CTR0_SZ +
 204                         sizeof(struct icp_qat_hw_cipher_config);
 205
 206         if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
 207                 ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
 208                 /* AES-GMAC */
 209                 qat_set_cipher_iv(cipher_param, auth_iv, ctx->auth_iv.length,
 210                                 req);
 211         }
 212
 213         /* Fill separate Content Descriptor for this op */
 214         rte_memcpy(cookie->opt.spc_gmac.cd_cipher.key,
 215                         ctx->auth_op == ICP_QAT_HW_AUTH_GENERATE ?
 216                                 ctx->cd.cipher.key :
 217                                 RTE_PTR_ADD(&ctx->cd, ver_key_offset),
 218                         ctx->auth_key_length);
 219         cookie->opt.spc_gmac.cd_cipher.cipher_config.val =
 220                         ICP_QAT_HW_CIPHER_CONFIG_BUILD(
 221                                 ICP_QAT_HW_CIPHER_AEAD_MODE,
 222                                 ctx->qat_cipher_alg,
 223                                 ICP_QAT_HW_CIPHER_NO_CONVERT,
 224                                 (ctx->auth_op == ICP_QAT_HW_AUTH_GENERATE ?
 225                                         ICP_QAT_HW_CIPHER_ENCRYPT :
 226                                         ICP_QAT_HW_CIPHER_DECRYPT));
 227         QAT_FIELD_SET(cookie->opt.spc_gmac.cd_cipher.cipher_config.val,
 228                         ctx->digest_length,
 229                         QAT_CIPHER_AEAD_HASH_CMP_LEN_BITPOS,
 230                         QAT_CIPHER_AEAD_HASH_CMP_LEN_MASK);
 231         cookie->opt.spc_gmac.cd_cipher.cipher_config.reserved =
 232                         ICP_QAT_HW_CIPHER_CONFIG_BUILD_UPPER(auth_data_len);
 233
 234         /* Update the request */
 235         req->cd_pars.u.s.content_desc_addr =
 236                         cookie->opt.spc_gmac.cd_phys_addr;
 237         req->cd_pars.u.s.content_desc_params_sz = RTE_ALIGN_CEIL(
 238                         sizeof(struct icp_qat_hw_cipher_config) +
 239                         ctx->auth_key_length, 8) >> 3;
 240         req->comn_mid.src_length = data_len;
 241         req->comn_mid.dst_length = 0;
 242
 243         cipher_param->spc_aad_addr = 0;
 244         cipher_param->spc_auth_res_addr = digest->iova;
 245         cipher_param->spc_aad_sz = auth_data_len;
 246         cipher_param->reserved = 0;
 247         cipher_param->spc_auth_res_sz = ctx->digest_length;
 248
 249         req->comn_hdr.service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER;
 250         cipher_cd_ctrl->cipher_cfg_offset = 0;
 251         ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
 252         ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
 253         ICP_QAT_FW_LA_SINGLE_PASS_PROTO_FLAG_SET(
 254                         req->comn_hdr.serv_specif_flags,
 255                         ICP_QAT_FW_LA_SINGLE_PASS_PROTO);
 256         ICP_QAT_FW_LA_PROTO_SET(
 257                         req->comn_hdr.serv_specif_flags,
 258                         ICP_QAT_FW_LA_NO_PROTO);
 259 }
 260
 261 static int
 262 qat_sym_build_op_aead_gen3(void *in_op, struct qat_sym_session *ctx,
 263                 uint8_t *out_msg, void *op_cookie)
 264 {
 265         register struct icp_qat_fw_la_bulk_req *req;
 266         struct rte_crypto_op *op = in_op;
 267         struct qat_sym_op_cookie *cookie = op_cookie;
 268         struct rte_crypto_sgl in_sgl, out_sgl;
 269         struct rte_crypto_vec in_vec[QAT_SYM_SGL_MAX_NUMBER],
 270                         out_vec[QAT_SYM_SGL_MAX_NUMBER];
 271         struct rte_crypto_va_iova_ptr cipher_iv;
 272         struct rte_crypto_va_iova_ptr aad;
 273         struct rte_crypto_va_iova_ptr digest;
 274         union rte_crypto_sym_ofs ofs;
 275         int32_t total_len;
 276
 277         in_sgl.vec = in_vec;
 278         out_sgl.vec = out_vec;
 279
 280         req = (struct icp_qat_fw_la_bulk_req *)out_msg;
 281         rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 282
 283         ofs.raw = qat_sym_convert_op_to_vec_aead(op, ctx, &in_sgl, &out_sgl,
 284                         &cipher_iv, &aad, &digest);
 285         if (unlikely(ofs.raw == UINT64_MAX)) {
 286                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 287                 return -EINVAL;
 288         }
 289
 290         total_len = qat_sym_build_req_set_data(req, in_op, cookie,
 291                         in_sgl.vec, in_sgl.num, out_sgl.vec, out_sgl.num);
 292         if (unlikely(total_len < 0)) {
 293                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 294                 return -EINVAL;
 295         }
 296
 297         enqueue_one_aead_job_gen3(ctx, req, &cipher_iv, &digest, &aad, ofs,
 298                 total_len);
 299
 300 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 301         qat_sym_debug_log_dump(req, ctx, in_sgl.vec, in_sgl.num, &cipher_iv,
 302                         NULL, &aad, &digest);
 303 #endif
 304
 305         return 0;
 306 }
 307
 308 static int
 309 qat_sym_build_op_auth_gen3(void *in_op, struct qat_sym_session *ctx,
 310                 uint8_t *out_msg, void *op_cookie)
 311 {
 312         register struct icp_qat_fw_la_bulk_req *req;
 313         struct rte_crypto_op *op = in_op;
 314         struct qat_sym_op_cookie *cookie = op_cookie;
 315         struct rte_crypto_sgl in_sgl, out_sgl;
 316         struct rte_crypto_vec in_vec[QAT_SYM_SGL_MAX_NUMBER],
 317                         out_vec[QAT_SYM_SGL_MAX_NUMBER];
 318         struct rte_crypto_va_iova_ptr auth_iv;
 319         struct rte_crypto_va_iova_ptr digest;
 320         union rte_crypto_sym_ofs ofs;
 321         int32_t total_len;
 322
 323         in_sgl.vec = in_vec;
 324         out_sgl.vec = out_vec;
 325
 326         req = (struct icp_qat_fw_la_bulk_req *)out_msg;
 327         rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 328
 329         ofs.raw = qat_sym_convert_op_to_vec_auth(op, ctx, &in_sgl, &out_sgl,
 330                         NULL, &auth_iv, &digest);
 331         if (unlikely(ofs.raw == UINT64_MAX)) {
 332                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 333                 return -EINVAL;
 334         }
 335
 336         total_len = qat_sym_build_req_set_data(req, in_op, cookie,
 337                         in_sgl.vec, in_sgl.num, out_sgl.vec, out_sgl.num);
 338         if (unlikely(total_len < 0)) {
 339                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 340                 return -EINVAL;
 341         }
 342
 343         enqueue_one_auth_job_gen3(ctx, cookie, req, &digest, &auth_iv,
 344                         ofs, total_len);
 345
 346 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 347         qat_sym_debug_log_dump(req, ctx, in_sgl.vec, in_sgl.num, NULL,
 348                         &auth_iv, NULL, &digest);
 349 #endif
 350
 351         return 0;
 352 }
 353
 354 static int
 355 qat_sym_crypto_set_session_gen3(void *cdev __rte_unused, void *session)
 356 {
 357         struct qat_sym_session *ctx = session;
 358         enum rte_proc_type_t proc_type = rte_eal_process_type();
 359         int ret;
 360
 361         if (proc_type == RTE_PROC_AUTO || proc_type == RTE_PROC_INVALID)
 362                 return -EINVAL;
 363
 364         ret = qat_sym_crypto_set_session_gen1(cdev, session);
 365         /* special single pass build request for GEN3 */
 366         if (ctx->is_single_pass)
 367                 ctx->build_request[proc_type] = qat_sym_build_op_aead_gen3;
 368         else if (ctx->is_single_pass_gmac)
 369                 ctx->build_request[proc_type] = qat_sym_build_op_auth_gen3;
 370
 371         if (ret == -ENOTSUP) {
 372                 /* GEN1 returning -ENOTSUP as it cannot handle some mixed algo,
 373                  * this is addressed by GEN3
 374                  */
 375                 if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3 &&
 376                                 ctx->qat_cipher_alg !=
 377                                 ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
 378                         qat_sym_session_set_ext_hash_flags_gen2(ctx,
 379                                 1 << ICP_QAT_FW_AUTH_HDR_FLAG_ZUC_EIA3_BITPOS);
 380                 } else if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 &&
 381                                 ctx->qat_cipher_alg !=
 382                                 ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2) {
 383                         qat_sym_session_set_ext_hash_flags_gen2(ctx,
 384                                 1 << ICP_QAT_FW_AUTH_HDR_FLAG_SNOW3G_UIA2_BITPOS);
 385                 } else if ((ctx->aes_cmac ||
 386                                 ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_NULL) &&
 387                                 (ctx->qat_cipher_alg ==
 388                                 ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
 389                                 ctx->qat_cipher_alg ==
 390                                 ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3)) {
 391                         qat_sym_session_set_ext_hash_flags_gen2(ctx, 0);
 392                 }
 393
 394                 ret = 0;
 395         }
 396
 397         return ret;
 398 }
 399
 400 static int
 401 qat_sym_dp_enqueue_single_aead_gen3(void *qp_data, uint8_t *drv_ctx,
 402         struct rte_crypto_vec *data, uint16_t n_data_vecs,
 403         union rte_crypto_sym_ofs ofs,
 404         struct rte_crypto_va_iova_ptr *iv,
 405         struct rte_crypto_va_iova_ptr *digest,
 406         struct rte_crypto_va_iova_ptr *aad,
 407         void *user_data)
 408 {
 409         struct qat_qp *qp = qp_data;
 410         struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
 411         struct qat_queue *tx_queue = &qp->tx_q;
 412         struct qat_sym_op_cookie *cookie;
 413         struct qat_sym_session *ctx = dp_ctx->session;
 414         struct icp_qat_fw_la_bulk_req *req;
 415
 416         int32_t data_len;
 417         uint32_t tail = dp_ctx->tail;
 418
 419         req = (struct icp_qat_fw_la_bulk_req *)(
 420                 (uint8_t *)tx_queue->base_addr + tail);
 421         cookie = qp->op_cookies[tail >> tx_queue->trailz];
 422         tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
 423         rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 424         rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
 425         data_len = qat_sym_build_req_set_data(req, user_data, cookie,
 426                         data, n_data_vecs, NULL, 0);
 427         if (unlikely(data_len < 0))
 428                 return -1;
 429
 430         enqueue_one_aead_job_gen3(ctx, req, iv, digest, aad, ofs,
 431                 (uint32_t)data_len);
 432
 433         dp_ctx->tail = tail;
 434         dp_ctx->cached_enqueue++;
 435
 436 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 437         qat_sym_debug_log_dump(req, ctx, data, n_data_vecs, iv,
 438                         NULL, aad, digest);
 439 #endif
 440         return 0;
 441 }
 442
 443 static uint32_t
 444 qat_sym_dp_enqueue_aead_jobs_gen3(void *qp_data, uint8_t *drv_ctx,
 445         struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
 446         void *user_data[], int *status)
 447 {
 448         struct qat_qp *qp = qp_data;
 449         struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
 450         struct qat_queue *tx_queue = &qp->tx_q;
 451         struct qat_sym_session *ctx = dp_ctx->session;
 452         uint32_t i, n;
 453         uint32_t tail;
 454         struct icp_qat_fw_la_bulk_req *req;
 455         int32_t data_len;
 456
 457         n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
 458         if (unlikely(n == 0)) {
 459                 qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
 460                 *status = 0;
 461                 return 0;
 462         }
 463
 464         tail = dp_ctx->tail;
 465
 466         for (i = 0; i < n; i++) {
 467                 struct qat_sym_op_cookie *cookie =
 468                         qp->op_cookies[tail >> tx_queue->trailz];
 469
 470                 req  = (struct icp_qat_fw_la_bulk_req *)(
 471                         (uint8_t *)tx_queue->base_addr + tail);
 472                 rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 473
 474                 if (vec->dest_sgl) {
 475                         data_len = qat_sym_build_req_set_data(req,
 476                                 user_data[i], cookie,
 477                                 vec->src_sgl[i].vec, vec->src_sgl[i].num,
 478                                 vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
 479                 } else {
 480                         data_len = qat_sym_build_req_set_data(req,
 481                                 user_data[i], cookie,
 482                                 vec->src_sgl[i].vec,
 483                                 vec->src_sgl[i].num, NULL, 0);
 484                 }
 485
 486                 if (unlikely(data_len < 0))
 487                         break;
 488
 489                 enqueue_one_aead_job_gen3(ctx, req, &vec->iv[i],
 490                                 &vec->digest[i], &vec->aad[i], ofs,
 491                                 (uint32_t)data_len);
 492
 493                 tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
 494
 495 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
 496                 qat_sym_debug_log_dump(req, ctx, vec->src_sgl[i].vec,
 497                                 vec->src_sgl[i].num, &vec->iv[i], NULL,
 498                                 &vec->aad[i], &vec->digest[i]);
 499 #endif
 500         }
 501
 502         if (unlikely(i < n))
 503                 qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
 504
 505         dp_ctx->tail = tail;
 506         dp_ctx->cached_enqueue += i;
 507         *status = 0;
 508         return i;
 509 }
 510
 511 static int
 512 qat_sym_dp_enqueue_single_auth_gen3(void *qp_data, uint8_t *drv_ctx,
 513         struct rte_crypto_vec *data, uint16_t n_data_vecs,
 514         union rte_crypto_sym_ofs ofs,
 515         struct rte_crypto_va_iova_ptr *iv __rte_unused,
 516         struct rte_crypto_va_iova_ptr *digest,
 517         struct rte_crypto_va_iova_ptr *auth_iv,
 518         void *user_data)
 519 {
 520         struct qat_qp *qp = qp_data;
 521         struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
 522         struct qat_queue *tx_queue = &qp->tx_q;
 523         struct qat_sym_op_cookie *cookie;
 524         struct qat_sym_session *ctx = dp_ctx->session;
 525         struct icp_qat_fw_la_bulk_req *req;
 526         int32_t data_len;
 527         uint32_t tail = dp_ctx->tail;
 528
 529         req = (struct icp_qat_fw_la_bulk_req *)(
 530                 (uint8_t *)tx_queue->base_addr + tail);
 531         cookie = qp->op_cookies[tail >> tx_queue->trailz];
 532         tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
 533
 534         rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 535         rte_prefetch0((uint8_t *)tx_queue->base_addr + tail);
 536         data_len = qat_sym_build_req_set_data(req, user_data, cookie,
 537                         data, n_data_vecs, NULL, 0);
 538         if (unlikely(data_len < 0))
 539                 return -1;
 540
 541         enqueue_one_auth_job_gen3(ctx, cookie, req, digest, auth_iv, ofs,
 542                         (uint32_t)data_len);
 543
 544         dp_ctx->tail = tail;
 545         dp_ctx->cached_enqueue++;
 546
 547         return 0;
 548 }
 549
 550 static uint32_t
 551 qat_sym_dp_enqueue_auth_jobs_gen3(void *qp_data, uint8_t *drv_ctx,
 552         struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
 553         void *user_data[], int *status)
 554 {
 555         struct qat_qp *qp = qp_data;
 556         struct qat_sym_dp_ctx *dp_ctx = (void *)drv_ctx;
 557         struct qat_queue *tx_queue = &qp->tx_q;
 558         struct qat_sym_session *ctx = dp_ctx->session;
 559         uint32_t i, n;
 560         uint32_t tail;
 561         struct icp_qat_fw_la_bulk_req *req;
 562         int32_t data_len;
 563
 564         n = QAT_SYM_DP_GET_MAX_ENQ(qp, dp_ctx->cached_enqueue, vec->num);
 565         if (unlikely(n == 0)) {
 566                 qat_sym_dp_fill_vec_status(vec->status, -1, vec->num);
 567                 *status = 0;
 568                 return 0;
 569         }
 570
 571         tail = dp_ctx->tail;
 572
 573         for (i = 0; i < n; i++) {
 574                 struct qat_sym_op_cookie *cookie =
 575                         qp->op_cookies[tail >> tx_queue->trailz];
 576
 577                 req  = (struct icp_qat_fw_la_bulk_req *)(
 578                         (uint8_t *)tx_queue->base_addr + tail);
 579                 rte_mov128((uint8_t *)req, (const uint8_t *)&(ctx->fw_req));
 580
 581                 if (vec->dest_sgl) {
 582                         data_len = qat_sym_build_req_set_data(req,
 583                                 user_data[i], cookie,
 584                                 vec->src_sgl[i].vec, vec->src_sgl[i].num,
 585                                 vec->dest_sgl[i].vec, vec->dest_sgl[i].num);
 586                 } else {
 587                         data_len = qat_sym_build_req_set_data(req,
 588                                 user_data[i], cookie,
 589                                 vec->src_sgl[i].vec,
 590                                 vec->src_sgl[i].num, NULL, 0);
 591                 }
 592
 593                 if (unlikely(data_len < 0))
 594                         break;
 595                 enqueue_one_auth_job_gen3(ctx, cookie, req, &vec->digest[i],
 596                         &vec->auth_iv[i], ofs, (uint32_t)data_len);
 597                 tail = (tail + tx_queue->msg_size) & tx_queue->modulo_mask;
 598         }
 599
 600         if (unlikely(i < n))
 601                 qat_sym_dp_fill_vec_status(vec->status + i, -1, n - i);
 602
 603         dp_ctx->tail = tail;
 604         dp_ctx->cached_enqueue += i;
 605         *status = 0;
 606         return i;
 607 }
 608
 609 static int
 610 qat_sym_configure_raw_dp_ctx_gen3(void *_raw_dp_ctx, void *_ctx)
 611 {
 612         struct rte_crypto_raw_dp_ctx *raw_dp_ctx = _raw_dp_ctx;
 613         struct qat_sym_session *ctx = _ctx;
 614         int ret;
 615
 616         ret = qat_sym_configure_raw_dp_ctx_gen1(_raw_dp_ctx, _ctx);
 617         if (ret < 0)
 618                 return ret;
 619
 620         if (ctx->is_single_pass) {
 621                 raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_aead_jobs_gen3;
 622                 raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_aead_gen3;
 623         } else if (ctx->is_single_pass_gmac) {
 624                 raw_dp_ctx->enqueue_burst = qat_sym_dp_enqueue_auth_jobs_gen3;
 625                 raw_dp_ctx->enqueue = qat_sym_dp_enqueue_single_auth_gen3;
 626         }
 627
 628         return 0;
 629 }
 630
 631
 632 RTE_INIT(qat_sym_crypto_gen3_init)
 633 {
 634         qat_sym_gen_dev_ops[QAT_GEN3].cryptodev_ops = &qat_sym_crypto_ops_gen1;
 635         qat_sym_gen_dev_ops[QAT_GEN3].get_capabilities =
 636                         qat_sym_crypto_cap_get_gen3;
 637         qat_sym_gen_dev_ops[QAT_GEN3].get_feature_flags =
 638                         qat_sym_crypto_feature_flags_get_gen1;
 639         qat_sym_gen_dev_ops[QAT_GEN3].set_session =
 640                         qat_sym_crypto_set_session_gen3;
 641         qat_sym_gen_dev_ops[QAT_GEN3].set_raw_dp_ctx =
 642                         qat_sym_configure_raw_dp_ctx_gen3;
 643 #ifdef RTE_LIB_SECURITY
 644         qat_sym_gen_dev_ops[QAT_GEN3].create_security_ctx =
 645                         qat_sym_create_security_gen1;
 646 #endif
 647 }
 648
 649 RTE_INIT(qat_asym_crypto_gen3_init)
 650 {
 651         qat_asym_gen_dev_ops[QAT_GEN3].cryptodev_ops = NULL;
 652         qat_asym_gen_dev_ops[QAT_GEN3].get_capabilities = NULL;
 653         qat_asym_gen_dev_ops[QAT_GEN3].get_feature_flags = NULL;
 654         qat_asym_gen_dev_ops[QAT_GEN3].set_session = NULL;
 655 }