drivers/crypto/qat/dev/qat_crypto_pmd_gens.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017-2022 Intel Corporation
   3  */
   4
   5 #ifndef _QAT_CRYPTO_PMD_GENS_H_
   6 #define _QAT_CRYPTO_PMD_GENS_H_
   7
   8 #include <rte_cryptodev.h>
   9 #include "qat_crypto.h"
  10 #include "qat_sym_session.h"
  11 #include "qat_sym.h"
  12
  13 #define QAT_SYM_DP_GET_MAX_ENQ(q, c, n) \
  14         RTE_MIN((q->max_inflights - q->enqueued + q->dequeued - c), n)
  15
  16 #define QAT_SYM_DP_IS_RESP_SUCCESS(resp) \
  17         (ICP_QAT_FW_COMN_STATUS_FLAG_OK == \
  18         ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(resp->comn_hdr.comn_status))
  19
  20 static __rte_always_inline int
  21 op_bpi_cipher_decrypt(uint8_t *src, uint8_t *dst,
  22                 uint8_t *iv, int ivlen, int srclen,
  23                 void *bpi_ctx)
  24 {
  25         EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
  26         int encrypted_ivlen;
  27         uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
  28         uint8_t *encr = encrypted_iv;
  29
  30         /* ECB method: encrypt (not decrypt!) the IV, then XOR with plaintext */
  31         if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
  32                                                                 <= 0)
  33                 goto cipher_decrypt_err;
  34
  35         for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
  36                 *dst = *src ^ *encr;
  37
  38         return 0;
  39
  40 cipher_decrypt_err:
  41         QAT_DP_LOG(ERR, "libcrypto ECB cipher decrypt for BPI IV failed");
  42         return -EINVAL;
  43 }
  44
  45 static __rte_always_inline uint32_t
  46 qat_bpicipher_preprocess(struct qat_sym_session *ctx,
  47                                 struct rte_crypto_op *op)
  48 {
  49         int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
  50         struct rte_crypto_sym_op *sym_op = op->sym;
  51         uint8_t last_block_len = block_len > 0 ?
  52                         sym_op->cipher.data.length % block_len : 0;
  53
  54         if (last_block_len && ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT) {
  55                 /* Decrypt last block */
  56                 uint8_t *last_block, *dst, *iv;
  57                 uint32_t last_block_offset = sym_op->cipher.data.offset +
  58                                 sym_op->cipher.data.length - last_block_len;
  59                 last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
  60                                 uint8_t *, last_block_offset);
  61
  62                 if (unlikely((sym_op->m_dst != NULL)
  63                                 && (sym_op->m_dst != sym_op->m_src)))
  64                         /* out-of-place operation (OOP) */
  65                         dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
  66                                                 uint8_t *, last_block_offset);
  67                 else
  68                         dst = last_block;
  69
  70                 if (last_block_len < sym_op->cipher.data.length)
  71                         /* use previous block ciphertext as IV */
  72                         iv = last_block - block_len;
  73                 else
  74                         /* runt block, i.e. less than one full block */
  75                         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
  76                                         ctx->cipher_iv.offset);
  77
  78 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
  79                 QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src before pre-process:",
  80                         last_block, last_block_len);
  81                 if (sym_op->m_dst != NULL)
  82                         QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst before pre-process:",
  83                         dst, last_block_len);
  84 #endif
  85                 op_bpi_cipher_decrypt(last_block, dst, iv, block_len,
  86                                 last_block_len, ctx->bpi_ctx);
  87 #if RTE_LOG_DP_LEVEL >= RTE_LOG_DEBUG
  88                 QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: src after pre-process:",
  89                         last_block, last_block_len);
  90                 if (sym_op->m_dst != NULL)
  91                         QAT_DP_HEXDUMP_LOG(DEBUG, "BPI: dst after pre-process:",
  92                         dst, last_block_len);
  93 #endif
  94         }
  95
  96         return sym_op->cipher.data.length - last_block_len;
  97 }
  98
  99 static __rte_always_inline int
 100 qat_auth_is_len_in_bits(struct qat_sym_session *ctx,
 101                 struct rte_crypto_op *op)
 102 {
 103         if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 ||
 104                 ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 ||
 105                 ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) {
 106                 if (unlikely((op->sym->auth.data.offset % BYTE_LENGTH != 0) ||
 107                                 (op->sym->auth.data.length % BYTE_LENGTH != 0)))
 108                         return -EINVAL;
 109                 return 1;
 110         }
 111         return 0;
 112 }
 113
 114 static __rte_always_inline int
 115 qat_cipher_is_len_in_bits(struct qat_sym_session *ctx,
 116                 struct rte_crypto_op *op)
 117 {
 118         if (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
 119                 ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI ||
 120                 ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
 121                 if (unlikely((op->sym->cipher.data.length % BYTE_LENGTH != 0) ||
 122                         ((op->sym->cipher.data.offset %
 123                         BYTE_LENGTH) != 0)))
 124                         return -EINVAL;
 125                 return 1;
 126         }
 127         return 0;
 128 }
 129
 130 static __rte_always_inline int32_t
 131 qat_sym_build_req_set_data(struct icp_qat_fw_la_bulk_req *req,
 132                 void *opaque, struct qat_sym_op_cookie *cookie,
 133                 struct rte_crypto_vec *src_vec, uint16_t n_src,
 134                 struct rte_crypto_vec *dst_vec, uint16_t n_dst)
 135 {
 136         struct qat_sgl *list;
 137         uint32_t i;
 138         uint32_t tl_src = 0, total_len_src, total_len_dst;
 139         uint64_t src_data_start = 0, dst_data_start = 0;
 140         int is_sgl = n_src > 1 || n_dst > 1;
 141
 142         if (unlikely(n_src < 1 || n_src > QAT_SYM_SGL_MAX_NUMBER ||
 143                         n_dst > QAT_SYM_SGL_MAX_NUMBER))
 144                 return -1;
 145
 146         if (likely(!is_sgl)) {
 147                 src_data_start = src_vec[0].iova;
 148                 tl_src = total_len_src =
 149                                 src_vec[0].len;
 150                 if (unlikely(n_dst)) { /* oop */
 151                         total_len_dst = dst_vec[0].len;
 152
 153                         dst_data_start = dst_vec[0].iova;
 154                         if (unlikely(total_len_src != total_len_dst))
 155                                 return -EINVAL;
 156                 } else {
 157                         dst_data_start = src_data_start;
 158                         total_len_dst = tl_src;
 159                 }
 160         } else { /* sgl */
 161                 total_len_dst = total_len_src = 0;
 162
 163                 ICP_QAT_FW_COMN_PTR_TYPE_SET(req->comn_hdr.comn_req_flags,
 164                         QAT_COMN_PTR_TYPE_SGL);
 165
 166                 list = (struct qat_sgl *)&cookie->qat_sgl_src;
 167                 for (i = 0; i < n_src; i++) {
 168                         list->buffers[i].len = src_vec[i].len;
 169                         list->buffers[i].resrvd = 0;
 170                         list->buffers[i].addr = src_vec[i].iova;
 171                         if (tl_src + src_vec[i].len > UINT32_MAX) {
 172                                 QAT_DP_LOG(ERR, "Message too long");
 173                                 return -1;
 174                         }
 175                         tl_src += src_vec[i].len;
 176                 }
 177
 178                 list->num_bufs = i;
 179                 src_data_start = cookie->qat_sgl_src_phys_addr;
 180
 181                 if (unlikely(n_dst > 0)) { /* oop sgl */
 182                         uint32_t tl_dst = 0;
 183
 184                         list = (struct qat_sgl *)&cookie->qat_sgl_dst;
 185
 186                         for (i = 0; i < n_dst; i++) {
 187                                 list->buffers[i].len = dst_vec[i].len;
 188                                 list->buffers[i].resrvd = 0;
 189                                 list->buffers[i].addr = dst_vec[i].iova;
 190                                 if (tl_dst + dst_vec[i].len > UINT32_MAX) {
 191                                         QAT_DP_LOG(ERR, "Message too long");
 192                                         return -ENOTSUP;
 193                                 }
 194
 195                                 tl_dst += dst_vec[i].len;
 196                         }
 197
 198                         if (tl_src != tl_dst)
 199                                 return -EINVAL;
 200                         list->num_bufs = i;
 201                         dst_data_start = cookie->qat_sgl_dst_phys_addr;
 202                 } else
 203                         dst_data_start = src_data_start;
 204         }
 205
 206         req->comn_mid.src_data_addr = src_data_start;
 207         req->comn_mid.dest_data_addr = dst_data_start;
 208         req->comn_mid.src_length = total_len_src;
 209         req->comn_mid.dst_length = total_len_dst;
 210         req->comn_mid.opaque_data = (uintptr_t)opaque;
 211
 212         return tl_src;
 213 }
 214
 215 static __rte_always_inline uint64_t
 216 qat_sym_convert_op_to_vec_cipher(struct rte_crypto_op *op,
 217                 struct qat_sym_session *ctx,
 218                 struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
 219                 struct rte_crypto_va_iova_ptr *cipher_iv,
 220                 struct rte_crypto_va_iova_ptr *auth_iv_or_aad __rte_unused,
 221                 struct rte_crypto_va_iova_ptr *digest __rte_unused)
 222 {
 223         uint32_t cipher_len = 0, cipher_ofs = 0;
 224         int n_src = 0;
 225         int ret;
 226
 227         ret = qat_cipher_is_len_in_bits(ctx, op);
 228         switch (ret) {
 229         case 1:
 230                 cipher_len = op->sym->cipher.data.length >> 3;
 231                 cipher_ofs = op->sym->cipher.data.offset >> 3;
 232                 break;
 233         case 0:
 234                 if (ctx->bpi_ctx) {
 235                         /* DOCSIS - only send complete blocks to device.
 236                          * Process any partial block using CFB mode.
 237                          * Even if 0 complete blocks, still send this to device
 238                          * to get into rx queue for post-process and dequeuing
 239                          */
 240                         cipher_len = qat_bpicipher_preprocess(ctx, op);
 241                         cipher_ofs = op->sym->cipher.data.offset;
 242                 } else {
 243                         cipher_len = op->sym->cipher.data.length;
 244                         cipher_ofs = op->sym->cipher.data.offset;
 245                 }
 246                 break;
 247         default:
 248                 QAT_DP_LOG(ERR,
 249           "SNOW3G/KASUMI/ZUC in QAT PMD only supports byte aligned values");
 250                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 251                 return UINT64_MAX;
 252         }
 253
 254         cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
 255                         ctx->cipher_iv.offset);
 256         cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
 257                         ctx->cipher_iv.offset);
 258
 259         n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, cipher_ofs,
 260                         cipher_len, in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
 261         if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
 262                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 263                 return UINT64_MAX;
 264         }
 265
 266         in_sgl->num = n_src;
 267
 268         /* Out-Of-Place operation */
 269         if (unlikely((op->sym->m_dst != NULL) &&
 270                         (op->sym->m_dst != op->sym->m_src))) {
 271                 int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, cipher_ofs,
 272                                 cipher_len, out_sgl->vec,
 273                                 QAT_SYM_SGL_MAX_NUMBER);
 274
 275                 if ((n_dst < 0) || (n_dst > op->sym->m_dst->nb_segs)) {
 276                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 277                         return UINT64_MAX;
 278                 }
 279
 280                 out_sgl->num = n_dst;
 281         } else
 282                 out_sgl->num = 0;
 283
 284         return 0;
 285 }
 286
 287 static __rte_always_inline uint64_t
 288 qat_sym_convert_op_to_vec_auth(struct rte_crypto_op *op,
 289                 struct qat_sym_session *ctx,
 290                 struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
 291                 struct rte_crypto_va_iova_ptr *cipher_iv __rte_unused,
 292                 struct rte_crypto_va_iova_ptr *auth_iv,
 293                 struct rte_crypto_va_iova_ptr *digest)
 294 {
 295         uint32_t auth_ofs = 0, auth_len = 0;
 296         int n_src, ret;
 297
 298         ret = qat_auth_is_len_in_bits(ctx, op);
 299         switch (ret) {
 300         case 1:
 301                 auth_ofs = op->sym->auth.data.offset >> 3;
 302                 auth_len = op->sym->auth.data.length >> 3;
 303                 auth_iv->va = rte_crypto_op_ctod_offset(op, void *,
 304                                 ctx->auth_iv.offset);
 305                 auth_iv->iova = rte_crypto_op_ctophys_offset(op,
 306                                 ctx->auth_iv.offset);
 307                 break;
 308         case 0:
 309                 if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
 310                         ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
 311                         /* AES-GMAC */
 312                         auth_ofs = op->sym->auth.data.offset;
 313                         auth_len = op->sym->auth.data.length;
 314                         auth_iv->va = rte_crypto_op_ctod_offset(op, void *,
 315                                         ctx->auth_iv.offset);
 316                         auth_iv->iova = rte_crypto_op_ctophys_offset(op,
 317                                         ctx->auth_iv.offset);
 318                 } else {
 319                         auth_ofs = op->sym->auth.data.offset;
 320                         auth_len = op->sym->auth.data.length;
 321                         auth_iv->va = NULL;
 322                         auth_iv->iova = 0;
 323                 }
 324                 break;
 325         default:
 326                 QAT_DP_LOG(ERR,
 327         "For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
 328                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 329                 return UINT64_MAX;
 330         }
 331
 332         n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, auth_ofs,
 333                         auth_len, in_sgl->vec,
 334                         QAT_SYM_SGL_MAX_NUMBER);
 335         if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
 336                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 337                 return UINT64_MAX;
 338         }
 339
 340         in_sgl->num = n_src;
 341
 342         /* Out-Of-Place operation */
 343         if (unlikely((op->sym->m_dst != NULL) &&
 344                         (op->sym->m_dst != op->sym->m_src))) {
 345                 int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, auth_ofs,
 346                                 auth_len, out_sgl->vec,
 347                                 QAT_SYM_SGL_MAX_NUMBER);
 348
 349                 if ((n_dst < 0) || (n_dst > op->sym->m_dst->nb_segs)) {
 350                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 351                         return UINT64_MAX;
 352                 }
 353                 out_sgl->num = n_dst;
 354         } else
 355                 out_sgl->num = 0;
 356
 357         digest->va = (void *)op->sym->auth.digest.data;
 358         digest->iova = op->sym->auth.digest.phys_addr;
 359
 360         return 0;
 361 }
 362
 363 static __rte_always_inline uint64_t
 364 qat_sym_convert_op_to_vec_chain(struct rte_crypto_op *op,
 365                 struct qat_sym_session *ctx,
 366                 struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
 367                 struct rte_crypto_va_iova_ptr *cipher_iv,
 368                 struct rte_crypto_va_iova_ptr *auth_iv_or_aad,
 369                 struct rte_crypto_va_iova_ptr *digest)
 370 {
 371         union rte_crypto_sym_ofs ofs;
 372         uint32_t max_len = 0;
 373         uint32_t cipher_len = 0, cipher_ofs = 0;
 374         uint32_t auth_len = 0, auth_ofs = 0;
 375         int is_oop = (op->sym->m_dst != NULL) &&
 376                         (op->sym->m_dst != op->sym->m_src);
 377         int is_sgl = op->sym->m_src->nb_segs > 1;
 378         int n_src;
 379         int ret;
 380
 381         if (unlikely(is_oop))
 382                 is_sgl |= op->sym->m_dst->nb_segs > 1;
 383
 384         cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
 385                         ctx->cipher_iv.offset);
 386         cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
 387                         ctx->cipher_iv.offset);
 388         auth_iv_or_aad->va = rte_crypto_op_ctod_offset(op, void *,
 389                         ctx->auth_iv.offset);
 390         auth_iv_or_aad->iova = rte_crypto_op_ctophys_offset(op,
 391                         ctx->auth_iv.offset);
 392         digest->va = (void *)op->sym->auth.digest.data;
 393         digest->iova = op->sym->auth.digest.phys_addr;
 394
 395         ret = qat_cipher_is_len_in_bits(ctx, op);
 396         switch (ret) {
 397         case 1:
 398                 cipher_len = op->sym->cipher.data.length >> 3;
 399                 cipher_ofs = op->sym->cipher.data.offset >> 3;
 400                 break;
 401         case 0:
 402                 cipher_len = op->sym->cipher.data.length;
 403                 cipher_ofs = op->sym->cipher.data.offset;
 404                 break;
 405         default:
 406                 QAT_DP_LOG(ERR,
 407         "For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
 408                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 409                 return -EINVAL;
 410         }
 411
 412         ret = qat_auth_is_len_in_bits(ctx, op);
 413         switch (ret) {
 414         case 1:
 415                 auth_len = op->sym->auth.data.length >> 3;
 416                 auth_ofs = op->sym->auth.data.offset >> 3;
 417                 break;
 418         case 0:
 419                 auth_len = op->sym->auth.data.length;
 420                 auth_ofs = op->sym->auth.data.offset;
 421                 break;
 422         default:
 423                 QAT_DP_LOG(ERR,
 424         "For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
 425                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 426                 return -EINVAL;
 427         }
 428
 429         max_len = RTE_MAX(cipher_ofs + cipher_len, auth_ofs + auth_len);
 430
 431         /* digest in buffer check. Needed only for wireless algos */
 432         if (ret == 1) {
 433                 /* Handle digest-encrypted cases, i.e.
 434                  * auth-gen-then-cipher-encrypt and
 435                  * cipher-decrypt-then-auth-verify
 436                  */
 437                 uint64_t auth_end_iova;
 438
 439                 if (unlikely(is_sgl)) {
 440                         uint32_t remaining_off = auth_ofs + auth_len;
 441                         struct rte_mbuf *sgl_buf = (is_oop ? op->sym->m_dst :
 442                                 op->sym->m_src);
 443
 444                         while (remaining_off >= rte_pktmbuf_data_len(sgl_buf)
 445                                         && sgl_buf->next != NULL) {
 446                                 remaining_off -= rte_pktmbuf_data_len(sgl_buf);
 447                                 sgl_buf = sgl_buf->next;
 448                         }
 449
 450                         auth_end_iova = (uint64_t)rte_pktmbuf_iova_offset(
 451                                 sgl_buf, remaining_off);
 452                 } else
 453                         auth_end_iova = (is_oop ?
 454                                 rte_pktmbuf_iova(op->sym->m_dst) :
 455                                 rte_pktmbuf_iova(op->sym->m_src)) + auth_ofs +
 456                                         auth_len;
 457
 458                 /* Then check if digest-encrypted conditions are met */
 459                 if ((auth_ofs + auth_len < cipher_ofs + cipher_len) &&
 460                                 (digest->iova == auth_end_iova))
 461                         max_len = RTE_MAX(max_len, auth_ofs + auth_len +
 462                                         ctx->digest_length);
 463         }
 464
 465         /* Passing 0 as cipher & auth offsets are assigned into ofs later */
 466         n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, 0, max_len,
 467                         in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
 468         if (unlikely(n_src < 0 || n_src > op->sym->m_src->nb_segs)) {
 469                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 470                 return -1;
 471         }
 472         in_sgl->num = n_src;
 473
 474         if (unlikely((op->sym->m_dst != NULL) &&
 475                         (op->sym->m_dst != op->sym->m_src))) {
 476                 int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, 0,
 477                                 max_len, out_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
 478
 479                 if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
 480                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 481                         return -1;
 482                 }
 483                 out_sgl->num = n_dst;
 484         } else
 485                 out_sgl->num = 0;
 486
 487         ofs.ofs.cipher.head = cipher_ofs;
 488         ofs.ofs.cipher.tail = max_len - cipher_ofs - cipher_len;
 489         ofs.ofs.auth.head = auth_ofs;
 490         ofs.ofs.auth.tail = max_len - auth_ofs - auth_len;
 491
 492         return ofs.raw;
 493 }
 494
 495 static __rte_always_inline uint64_t
 496 qat_sym_convert_op_to_vec_aead(struct rte_crypto_op *op,
 497                 struct qat_sym_session *ctx,
 498                 struct rte_crypto_sgl *in_sgl, struct rte_crypto_sgl *out_sgl,
 499                 struct rte_crypto_va_iova_ptr *cipher_iv,
 500                 struct rte_crypto_va_iova_ptr *auth_iv_or_aad,
 501                 struct rte_crypto_va_iova_ptr *digest)
 502 {
 503         uint32_t cipher_len = 0, cipher_ofs = 0;
 504         int32_t n_src = 0;
 505
 506         cipher_iv->va = rte_crypto_op_ctod_offset(op, void *,
 507                         ctx->cipher_iv.offset);
 508         cipher_iv->iova = rte_crypto_op_ctophys_offset(op,
 509                         ctx->cipher_iv.offset);
 510         auth_iv_or_aad->va = (void *)op->sym->aead.aad.data;
 511         auth_iv_or_aad->iova = op->sym->aead.aad.phys_addr;
 512         digest->va = (void *)op->sym->aead.digest.data;
 513         digest->iova = op->sym->aead.digest.phys_addr;
 514
 515         cipher_len = op->sym->aead.data.length;
 516         cipher_ofs = op->sym->aead.data.offset;
 517
 518         n_src = rte_crypto_mbuf_to_vec(op->sym->m_src, cipher_ofs, cipher_len,
 519                         in_sgl->vec, QAT_SYM_SGL_MAX_NUMBER);
 520         if (n_src < 0 || n_src > op->sym->m_src->nb_segs) {
 521                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 522                 return UINT64_MAX;
 523         }
 524         in_sgl->num = n_src;
 525
 526         /* Out-Of-Place operation */
 527         if (unlikely((op->sym->m_dst != NULL) &&
 528                         (op->sym->m_dst != op->sym->m_src))) {
 529                 int n_dst = rte_crypto_mbuf_to_vec(op->sym->m_dst, cipher_ofs,
 530                                 cipher_len, out_sgl->vec,
 531                                 QAT_SYM_SGL_MAX_NUMBER);
 532                 if (n_dst < 0 || n_dst > op->sym->m_dst->nb_segs) {
 533                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 534                         return UINT64_MAX;
 535                 }
 536
 537                 out_sgl->num = n_dst;
 538         } else
 539                 out_sgl->num = 0;
 540
 541         return 0;
 542 }
 543
 544 static __rte_always_inline void
 545 qat_set_cipher_iv(struct icp_qat_fw_la_cipher_req_params *cipher_param,
 546                 struct rte_crypto_va_iova_ptr *iv_ptr, uint32_t iv_len,
 547                 struct icp_qat_fw_la_bulk_req *qat_req)
 548 {
 549         /* copy IV into request if it fits */
 550         if (iv_len <= sizeof(cipher_param->u.cipher_IV_array))
 551                 rte_memcpy(cipher_param->u.cipher_IV_array, iv_ptr->va,
 552                                 iv_len);
 553         else {
 554                 ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
 555                                 qat_req->comn_hdr.serv_specif_flags,
 556                                 ICP_QAT_FW_CIPH_IV_64BIT_PTR);
 557                 cipher_param->u.s.cipher_IV_ptr = iv_ptr->iova;
 558         }
 559 }
 560
 561 static __rte_always_inline void
 562 qat_sym_dp_fill_vec_status(int32_t *sta, int status, uint32_t n)
 563 {
 564         uint32_t i;
 565
 566         for (i = 0; i < n; i++)
 567                 sta[i] = status;
 568 }
 569
 570 static __rte_always_inline void
 571 enqueue_one_cipher_job_gen1(struct qat_sym_session *ctx,
 572         struct icp_qat_fw_la_bulk_req *req,
 573         struct rte_crypto_va_iova_ptr *iv,
 574         union rte_crypto_sym_ofs ofs, uint32_t data_len)
 575 {
 576         struct icp_qat_fw_la_cipher_req_params *cipher_param;
 577
 578         cipher_param = (void *)&req->serv_specif_rqpars;
 579
 580         /* cipher IV */
 581         qat_set_cipher_iv(cipher_param, iv, ctx->cipher_iv.length, req);
 582         cipher_param->cipher_offset = ofs.ofs.cipher.head;
 583         cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
 584                         ofs.ofs.cipher.tail;
 585 }
 586
 587 static __rte_always_inline void
 588 enqueue_one_auth_job_gen1(struct qat_sym_session *ctx,
 589         struct icp_qat_fw_la_bulk_req *req,
 590         struct rte_crypto_va_iova_ptr *digest,
 591         struct rte_crypto_va_iova_ptr *auth_iv,
 592         union rte_crypto_sym_ofs ofs, uint32_t data_len)
 593 {
 594         struct icp_qat_fw_la_cipher_req_params *cipher_param;
 595         struct icp_qat_fw_la_auth_req_params *auth_param;
 596
 597         cipher_param = (void *)&req->serv_specif_rqpars;
 598         auth_param = (void *)((uint8_t *)cipher_param +
 599                         ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
 600
 601         auth_param->auth_off = ofs.ofs.auth.head;
 602         auth_param->auth_len = data_len - ofs.ofs.auth.head -
 603                         ofs.ofs.auth.tail;
 604         auth_param->auth_res_addr = digest->iova;
 605
 606         switch (ctx->qat_hash_alg) {
 607         case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
 608         case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
 609         case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
 610                 auth_param->u1.aad_adr = auth_iv->iova;
 611                 break;
 612         case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
 613         case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
 614                 ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
 615                         req->comn_hdr.serv_specif_flags,
 616                                 ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
 617                 rte_memcpy(cipher_param->u.cipher_IV_array, auth_iv->va,
 618                                 ctx->auth_iv.length);
 619                 break;
 620         default:
 621                 break;
 622         }
 623 }
 624
 625 static __rte_always_inline int
 626 enqueue_one_chain_job_gen1(struct qat_sym_session *ctx,
 627         struct icp_qat_fw_la_bulk_req *req,
 628         struct rte_crypto_vec *src_vec,
 629         uint16_t n_src_vecs,
 630         struct rte_crypto_vec *dst_vec,
 631         uint16_t n_dst_vecs,
 632         struct rte_crypto_va_iova_ptr *cipher_iv,
 633         struct rte_crypto_va_iova_ptr *digest,
 634         struct rte_crypto_va_iova_ptr *auth_iv,
 635         union rte_crypto_sym_ofs ofs, uint32_t data_len)
 636 {
 637         struct icp_qat_fw_la_cipher_req_params *cipher_param;
 638         struct icp_qat_fw_la_auth_req_params *auth_param;
 639         struct rte_crypto_vec *cvec = n_dst_vecs > 0 ?
 640                         dst_vec : src_vec;
 641         rte_iova_t auth_iova_end;
 642         int cipher_len, auth_len;
 643         int is_sgl = n_src_vecs > 1 || n_dst_vecs > 1;
 644
 645         cipher_param = (void *)&req->serv_specif_rqpars;
 646         auth_param = (void *)((uint8_t *)cipher_param +
 647                         ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
 648
 649         cipher_len = data_len - ofs.ofs.cipher.head -
 650                         ofs.ofs.cipher.tail;
 651         auth_len = data_len - ofs.ofs.auth.head - ofs.ofs.auth.tail;
 652
 653         if (unlikely(cipher_len < 0 || auth_len < 0))
 654                 return -1;
 655
 656         cipher_param->cipher_offset = ofs.ofs.cipher.head;
 657         cipher_param->cipher_length = cipher_len;
 658         qat_set_cipher_iv(cipher_param, cipher_iv, ctx->cipher_iv.length, req);
 659
 660         auth_param->auth_off = ofs.ofs.auth.head;
 661         auth_param->auth_len = auth_len;
 662         auth_param->auth_res_addr = digest->iova;
 663
 664         switch (ctx->qat_hash_alg) {
 665         case ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2:
 666         case ICP_QAT_HW_AUTH_ALGO_KASUMI_F9:
 667         case ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3:
 668                 auth_param->u1.aad_adr = auth_iv->iova;
 669                 break;
 670         case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
 671         case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
 672                 break;
 673         default:
 674                 break;
 675         }
 676
 677         if (unlikely(is_sgl)) {
 678                 /* sgl */
 679                 int i = n_dst_vecs ? n_dst_vecs : n_src_vecs;
 680                 uint32_t remaining_off = data_len - ofs.ofs.auth.tail;
 681
 682                 while (remaining_off >= cvec->len && i >= 1) {
 683                         i--;
 684                         remaining_off -= cvec->len;
 685                         cvec++;
 686                 }
 687
 688                 auth_iova_end = cvec->iova + remaining_off;
 689         } else
 690                 auth_iova_end = cvec[0].iova + auth_param->auth_off +
 691                         auth_param->auth_len;
 692
 693         /* Then check if digest-encrypted conditions are met */
 694         if ((auth_param->auth_off + auth_param->auth_len <
 695                 cipher_param->cipher_offset + cipher_param->cipher_length) &&
 696                         (digest->iova == auth_iova_end)) {
 697                 /* Handle partial digest encryption */
 698                 if (cipher_param->cipher_offset + cipher_param->cipher_length <
 699                         auth_param->auth_off + auth_param->auth_len +
 700                                 ctx->digest_length && !is_sgl)
 701                         req->comn_mid.dst_length = req->comn_mid.src_length =
 702                                 auth_param->auth_off + auth_param->auth_len +
 703                                         ctx->digest_length;
 704                 struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
 705                 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
 706                         ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
 707         }
 708
 709         return 0;
 710 }
 711
 712 static __rte_always_inline void
 713 enqueue_one_aead_job_gen1(struct qat_sym_session *ctx,
 714         struct icp_qat_fw_la_bulk_req *req,
 715         struct rte_crypto_va_iova_ptr *iv,
 716         struct rte_crypto_va_iova_ptr *digest,
 717         struct rte_crypto_va_iova_ptr *aad,
 718         union rte_crypto_sym_ofs ofs, uint32_t data_len)
 719 {
 720         struct icp_qat_fw_la_cipher_req_params *cipher_param =
 721                 (void *)&req->serv_specif_rqpars;
 722         struct icp_qat_fw_la_auth_req_params *auth_param =
 723                 (void *)((uint8_t *)&req->serv_specif_rqpars +
 724                 ICP_QAT_FW_HASH_REQUEST_PARAMETERS_OFFSET);
 725         uint8_t *aad_data;
 726         uint8_t aad_ccm_real_len;
 727         uint8_t aad_len_field_sz;
 728         uint32_t msg_len_be;
 729         rte_iova_t aad_iova = 0;
 730         uint8_t q;
 731
 732         switch (ctx->qat_hash_alg) {
 733         case ICP_QAT_HW_AUTH_ALGO_GALOIS_128:
 734         case ICP_QAT_HW_AUTH_ALGO_GALOIS_64:
 735                 ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
 736                         req->comn_hdr.serv_specif_flags,
 737                                 ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
 738                 rte_memcpy(cipher_param->u.cipher_IV_array, iv->va,
 739                                 ctx->cipher_iv.length);
 740                 aad_iova = aad->iova;
 741                 break;
 742         case ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC:
 743                 aad_data = aad->va;
 744                 aad_iova = aad->iova;
 745                 aad_ccm_real_len = 0;
 746                 aad_len_field_sz = 0;
 747                 msg_len_be = rte_bswap32((uint32_t)data_len -
 748                                 ofs.ofs.cipher.head);
 749
 750                 if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {
 751                         aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;
 752                         aad_ccm_real_len = ctx->aad_len -
 753                                 ICP_QAT_HW_CCM_AAD_B0_LEN -
 754                                 ICP_QAT_HW_CCM_AAD_LEN_INFO;
 755                 } else {
 756                         aad_data = iv->va;
 757                         aad_iova = iv->iova;
 758                 }
 759
 760                 q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length;
 761                 aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(
 762                         aad_len_field_sz, ctx->digest_length, q);
 763                 if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {
 764                         memcpy(aad_data + ctx->cipher_iv.length +
 765                                 ICP_QAT_HW_CCM_NONCE_OFFSET + (q -
 766                                 ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),
 767                                 (uint8_t *)&msg_len_be,
 768                                 ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);
 769                 } else {
 770                         memcpy(aad_data + ctx->cipher_iv.length +
 771                                 ICP_QAT_HW_CCM_NONCE_OFFSET,
 772                                 (uint8_t *)&msg_len_be +
 773                                 (ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE
 774                                 - q), q);
 775                 }
 776
 777                 if (aad_len_field_sz > 0) {
 778                         *(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN] =
 779                                 rte_bswap16(aad_ccm_real_len);
 780
 781                         if ((aad_ccm_real_len + aad_len_field_sz)
 782                                 % ICP_QAT_HW_CCM_AAD_B0_LEN) {
 783                                 uint8_t pad_len = 0;
 784                                 uint8_t pad_idx = 0;
 785
 786                                 pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -
 787                                         ((aad_ccm_real_len +
 788                                         aad_len_field_sz) %
 789                                         ICP_QAT_HW_CCM_AAD_B0_LEN);
 790                                 pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +
 791                                         aad_ccm_real_len +
 792                                         aad_len_field_sz;
 793                                 memset(&aad_data[pad_idx], 0, pad_len);
 794                         }
 795                 }
 796
 797                 rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array)
 798                         + ICP_QAT_HW_CCM_NONCE_OFFSET,
 799                         (uint8_t *)iv->va +
 800                         ICP_QAT_HW_CCM_NONCE_OFFSET, ctx->cipher_iv.length);
 801                 *(uint8_t *)&cipher_param->u.cipher_IV_array[0] =
 802                         q - ICP_QAT_HW_CCM_NONCE_OFFSET;
 803
 804                 rte_memcpy((uint8_t *)aad->va +
 805                                 ICP_QAT_HW_CCM_NONCE_OFFSET,
 806                         (uint8_t *)iv->va + ICP_QAT_HW_CCM_NONCE_OFFSET,
 807                         ctx->cipher_iv.length);
 808                 break;
 809         default:
 810                 break;
 811         }
 812
 813         cipher_param->cipher_offset = ofs.ofs.cipher.head;
 814         cipher_param->cipher_length = data_len - ofs.ofs.cipher.head -
 815                         ofs.ofs.cipher.tail;
 816         auth_param->auth_off = ofs.ofs.cipher.head;
 817         auth_param->auth_len = cipher_param->cipher_length;
 818         auth_param->auth_res_addr = digest->iova;
 819         auth_param->u1.aad_adr = aad_iova;
 820 }
 821
 822 extern struct rte_cryptodev_ops qat_sym_crypto_ops_gen1;
 823 extern struct rte_cryptodev_ops qat_asym_crypto_ops_gen1;
 824
 825 /* -----------------GEN 1 sym crypto op data path APIs ---------------- */
 826 int
 827 qat_sym_build_op_cipher_gen1(void *in_op, struct qat_sym_session *ctx,
 828         uint8_t *out_msg, void *op_cookie);
 829
 830 int
 831 qat_sym_build_op_auth_gen1(void *in_op, struct qat_sym_session *ctx,
 832                 uint8_t *out_msg, void *op_cookie);
 833
 834 int
 835 qat_sym_build_op_aead_gen1(void *in_op, struct qat_sym_session *ctx,
 836                 uint8_t *out_msg, void *op_cookie);
 837
 838 int
 839 qat_sym_build_op_chain_gen1(void *in_op, struct qat_sym_session *ctx,
 840                 uint8_t *out_msg, void *op_cookie);
 841
 842 /* -----------------GEN 1 sym crypto raw data path APIs ---------------- */
 843 int
 844 qat_sym_dp_enqueue_single_cipher_gen1(void *qp_data, uint8_t *drv_ctx,
 845         struct rte_crypto_vec *data, uint16_t n_data_vecs,
 846         union rte_crypto_sym_ofs ofs,
 847         struct rte_crypto_va_iova_ptr *iv,
 848         struct rte_crypto_va_iova_ptr *digest __rte_unused,
 849         struct rte_crypto_va_iova_ptr *aad __rte_unused,
 850         void *user_data);
 851
 852 uint32_t
 853 qat_sym_dp_enqueue_cipher_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
 854         struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
 855         void *user_data[], int *status);
 856
 857 int
 858 qat_sym_dp_enqueue_single_auth_gen1(void *qp_data, uint8_t *drv_ctx,
 859         struct rte_crypto_vec *data, uint16_t n_data_vecs,
 860         union rte_crypto_sym_ofs ofs,
 861         struct rte_crypto_va_iova_ptr *iv __rte_unused,
 862         struct rte_crypto_va_iova_ptr *digest,
 863         struct rte_crypto_va_iova_ptr *auth_iv,
 864         void *user_data);
 865
 866 uint32_t
 867 qat_sym_dp_enqueue_auth_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
 868         struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
 869         void *user_data[], int *status);
 870
 871 int
 872 qat_sym_dp_enqueue_single_chain_gen1(void *qp_data, uint8_t *drv_ctx,
 873         struct rte_crypto_vec *data, uint16_t n_data_vecs,
 874         union rte_crypto_sym_ofs ofs,
 875         struct rte_crypto_va_iova_ptr *cipher_iv,
 876         struct rte_crypto_va_iova_ptr *digest,
 877         struct rte_crypto_va_iova_ptr *auth_iv,
 878         void *user_data);
 879
 880 uint32_t
 881 qat_sym_dp_enqueue_chain_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
 882         struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
 883         void *user_data[], int *status);
 884
 885 int
 886 qat_sym_dp_enqueue_single_aead_gen1(void *qp_data, uint8_t *drv_ctx,
 887         struct rte_crypto_vec *data, uint16_t n_data_vecs,
 888         union rte_crypto_sym_ofs ofs,
 889         struct rte_crypto_va_iova_ptr *iv,
 890         struct rte_crypto_va_iova_ptr *digest,
 891         struct rte_crypto_va_iova_ptr *aad,
 892         void *user_data);
 893
 894 uint32_t
 895 qat_sym_dp_enqueue_aead_jobs_gen1(void *qp_data, uint8_t *drv_ctx,
 896         struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
 897         void *user_data[], int *status);
 898
 899 void *
 900 qat_sym_dp_dequeue_single_gen1(void *qp_data, uint8_t *drv_ctx,
 901         int *dequeue_status, enum rte_crypto_op_status *op_status);
 902
 903 uint32_t
 904 qat_sym_dp_dequeue_burst_gen1(void *qp_data, uint8_t *drv_ctx,
 905         rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,
 906         uint32_t max_nb_to_dequeue,
 907         rte_cryptodev_raw_post_dequeue_t post_dequeue,
 908         void **out_user_data, uint8_t is_user_data_array,
 909         uint32_t *n_success_jobs, int *return_status);
 910
 911 int
 912 qat_sym_dp_enqueue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n);
 913
 914 int
 915 qat_sym_dp_dequeue_done_gen1(void *qp_data, uint8_t *drv_ctx, uint32_t n);
 916
 917 int
 918 qat_sym_configure_raw_dp_ctx_gen1(void *_raw_dp_ctx, void *_ctx);
 919
 920 /* -----------------GENx control path APIs ---------------- */
 921 uint64_t
 922 qat_sym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
 923
 924 int
 925 qat_sym_crypto_set_session_gen1(void *cryptodev, void *session);
 926
 927 void
 928 qat_sym_session_set_ext_hash_flags_gen2(struct qat_sym_session *session,
 929                 uint8_t hash_flag);
 930
 931 struct qat_capabilities_info
 932 qat_asym_crypto_cap_get_gen1(struct qat_pci_device *qat_dev);
 933
 934 uint64_t
 935 qat_asym_crypto_feature_flags_get_gen1(struct qat_pci_device *qat_dev);
 936
 937 int
 938 qat_asym_crypto_set_session_gen1(void *cryptodev, void *session);
 939
 940 #ifdef RTE_LIB_SECURITY
 941 extern struct rte_security_ops security_qat_ops_gen1;
 942
 943 void *
 944 qat_sym_create_security_gen1(void *cryptodev);
 945 #endif
 946
 947 #endif