drivers/common/cpt/cpt_ucode.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Cavium, Inc
   3  */
   4
   5 #ifndef _CPT_UCODE_H_
   6 #define _CPT_UCODE_H_
   7 #include <stdbool.h>
   8
   9 #include "cpt_common.h"
  10 #include "cpt_hw_types.h"
  11 #include "cpt_mcode_defines.h"
  12
  13 /*
  14  * This file defines functions that are interfaces to microcode spec.
  15  *
  16  */
  17
  18 static uint8_t zuc_d[32] = {
  19         0x44, 0xD7, 0x26, 0xBC, 0x62, 0x6B, 0x13, 0x5E,
  20         0x57, 0x89, 0x35, 0xE2, 0x71, 0x35, 0x09, 0xAF,
  21         0x4D, 0x78, 0x2F, 0x13, 0x6B, 0xC4, 0x1A, 0xF1,
  22         0x5E, 0x26, 0x3C, 0x4D, 0x78, 0x9A, 0x47, 0xAC
  23 };
  24
  25 static __rte_always_inline void
  26 gen_key_snow3g(const uint8_t *ck, uint32_t *keyx)
  27 {
  28         int i, base;
  29
  30         for (i = 0; i < 4; i++) {
  31                 base = 4 * i;
  32                 keyx[3 - i] = (ck[base] << 24) | (ck[base + 1] << 16) |
  33                         (ck[base + 2] << 8) | (ck[base + 3]);
  34                 keyx[3 - i] = rte_cpu_to_be_32(keyx[3 - i]);
  35         }
  36 }
  37
  38 static __rte_always_inline int
  39 cpt_mac_len_verify(struct rte_crypto_auth_xform *auth)
  40 {
  41         uint16_t mac_len = auth->digest_length;
  42         int ret;
  43
  44         switch (auth->algo) {
  45         case RTE_CRYPTO_AUTH_MD5:
  46         case RTE_CRYPTO_AUTH_MD5_HMAC:
  47                 ret = (mac_len == 16) ? 0 : -1;
  48                 break;
  49         case RTE_CRYPTO_AUTH_SHA1:
  50         case RTE_CRYPTO_AUTH_SHA1_HMAC:
  51                 ret = (mac_len == 20) ? 0 : -1;
  52                 break;
  53         case RTE_CRYPTO_AUTH_SHA224:
  54         case RTE_CRYPTO_AUTH_SHA224_HMAC:
  55                 ret = (mac_len == 28) ? 0 : -1;
  56                 break;
  57         case RTE_CRYPTO_AUTH_SHA256:
  58         case RTE_CRYPTO_AUTH_SHA256_HMAC:
  59                 ret = (mac_len == 32) ? 0 : -1;
  60                 break;
  61         case RTE_CRYPTO_AUTH_SHA384:
  62         case RTE_CRYPTO_AUTH_SHA384_HMAC:
  63                 ret = (mac_len == 48) ? 0 : -1;
  64                 break;
  65         case RTE_CRYPTO_AUTH_SHA512:
  66         case RTE_CRYPTO_AUTH_SHA512_HMAC:
  67                 ret = (mac_len == 64) ? 0 : -1;
  68                 break;
  69         case RTE_CRYPTO_AUTH_NULL:
  70                 ret = 0;
  71                 break;
  72         default:
  73                 ret = -1;
  74         }
  75
  76         return ret;
  77 }
  78
  79 static __rte_always_inline void
  80 cpt_fc_salt_update(struct cpt_ctx *cpt_ctx,
  81                    uint8_t *salt)
  82 {
  83         mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx;
  84         memcpy(fctx->enc.encr_iv, salt, 4);
  85 }
  86
  87 static __rte_always_inline int
  88 cpt_fc_ciph_validate_key_aes(uint16_t key_len)
  89 {
  90         switch (key_len) {
  91         case 16:
  92         case 24:
  93         case 32:
  94                 return 0;
  95         default:
  96                 return -1;
  97         }
  98 }
  99
 100 static __rte_always_inline int
 101 cpt_fc_ciph_set_type(cipher_type_t type, struct cpt_ctx *ctx, uint16_t key_len)
 102 {
 103         int fc_type = 0;
 104         switch (type) {
 105         case PASSTHROUGH:
 106                 fc_type = FC_GEN;
 107                 break;
 108         case DES3_CBC:
 109         case DES3_ECB:
 110                 fc_type = FC_GEN;
 111                 break;
 112         case AES_CBC:
 113         case AES_ECB:
 114         case AES_CFB:
 115         case AES_CTR:
 116         case AES_GCM:
 117                 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0))
 118                         return -1;
 119                 fc_type = FC_GEN;
 120                 break;
 121         case CHACHA20:
 122                 fc_type = FC_GEN;
 123                 break;
 124         case AES_XTS:
 125                 key_len = key_len / 2;
 126                 if (unlikely(key_len == 24)) {
 127                         CPT_LOG_DP_ERR("Invalid AES key len for XTS");
 128                         return -1;
 129                 }
 130                 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0))
 131                         return -1;
 132                 fc_type = FC_GEN;
 133                 break;
 134         case ZUC_EEA3:
 135         case SNOW3G_UEA2:
 136                 if (unlikely(key_len != 16))
 137                         return -1;
 138                 /* No support for AEAD yet */
 139                 if (unlikely(ctx->hash_type))
 140                         return -1;
 141                 fc_type = ZUC_SNOW3G;
 142                 break;
 143         case KASUMI_F8_CBC:
 144         case KASUMI_F8_ECB:
 145                 if (unlikely(key_len != 16))
 146                         return -1;
 147                 /* No support for AEAD yet */
 148                 if (unlikely(ctx->hash_type))
 149                         return -1;
 150                 fc_type = KASUMI;
 151                 break;
 152         default:
 153                 return -1;
 154         }
 155
 156         ctx->fc_type = fc_type;
 157         return 0;
 158 }
 159
 160 static __rte_always_inline void
 161 cpt_fc_ciph_set_key_passthrough(struct cpt_ctx *cpt_ctx, mc_fc_context_t *fctx)
 162 {
 163         cpt_ctx->enc_cipher = 0;
 164         fctx->enc.enc_cipher = 0;
 165 }
 166
 167 static __rte_always_inline void
 168 cpt_fc_ciph_set_key_set_aes_key_type(mc_fc_context_t *fctx, uint16_t key_len)
 169 {
 170         mc_aes_type_t aes_key_type = 0;
 171         switch (key_len) {
 172         case 16:
 173                 aes_key_type = AES_128_BIT;
 174                 break;
 175         case 24:
 176                 aes_key_type = AES_192_BIT;
 177                 break;
 178         case 32:
 179                 aes_key_type = AES_256_BIT;
 180                 break;
 181         default:
 182                 /* This should not happen */
 183                 CPT_LOG_DP_ERR("Invalid AES key len");
 184                 return;
 185         }
 186         fctx->enc.aes_key = aes_key_type;
 187 }
 188
 189 static __rte_always_inline void
 190 cpt_fc_ciph_set_key_snow3g_uea2(struct cpt_ctx *cpt_ctx, const uint8_t *key,
 191                 uint16_t key_len)
 192 {
 193         mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx;
 194         uint32_t keyx[4];
 195
 196         cpt_ctx->snow3g = 1;
 197         gen_key_snow3g(key, keyx);
 198         memcpy(zs_ctx->ci_key, keyx, key_len);
 199         cpt_ctx->zsk_flags = 0;
 200 }
 201
 202 static __rte_always_inline void
 203 cpt_fc_ciph_set_key_zuc_eea3(struct cpt_ctx *cpt_ctx, const uint8_t *key,
 204                 uint16_t key_len)
 205 {
 206         mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx;
 207
 208         cpt_ctx->snow3g = 0;
 209         memcpy(zs_ctx->ci_key, key, key_len);
 210         memcpy(zs_ctx->zuc_const, zuc_d, 32);
 211         cpt_ctx->zsk_flags = 0;
 212 }
 213
 214 static __rte_always_inline void
 215 cpt_fc_ciph_set_key_kasumi_f8_ecb(struct cpt_ctx *cpt_ctx, const uint8_t *key,
 216                 uint16_t key_len)
 217 {
 218         mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx;
 219
 220         cpt_ctx->k_ecb = 1;
 221         memcpy(k_ctx->ci_key, key, key_len);
 222         cpt_ctx->zsk_flags = 0;
 223 }
 224
 225 static __rte_always_inline void
 226 cpt_fc_ciph_set_key_kasumi_f8_cbc(struct cpt_ctx *cpt_ctx, const uint8_t *key,
 227                 uint16_t key_len)
 228 {
 229         mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx;
 230
 231         memcpy(k_ctx->ci_key, key, key_len);
 232         cpt_ctx->zsk_flags = 0;
 233 }
 234
 235 static __rte_always_inline int
 236 cpt_fc_ciph_set_key(struct cpt_ctx *cpt_ctx, cipher_type_t type,
 237                     const uint8_t *key, uint16_t key_len, uint8_t *salt)
 238 {
 239         mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx;
 240         int ret;
 241
 242         ret = cpt_fc_ciph_set_type(type, cpt_ctx, key_len);
 243         if (unlikely(ret))
 244                 return -1;
 245
 246         if (cpt_ctx->fc_type == FC_GEN) {
 247                 /*
 248                  * We need to always say IV is from DPTR as user can
 249                  * sometimes iverride IV per operation.
 250                  */
 251                 fctx->enc.iv_source = CPT_FROM_DPTR;
 252
 253                 if (cpt_ctx->auth_key_len > 64)
 254                         return -1;
 255         }
 256
 257         switch (type) {
 258         case PASSTHROUGH:
 259                 cpt_fc_ciph_set_key_passthrough(cpt_ctx, fctx);
 260                 goto success;
 261         case DES3_CBC:
 262                 /* CPT performs DES using 3DES with the 8B DES-key
 263                  * replicated 2 more times to match the 24B 3DES-key.
 264                  * Eg. If org. key is "0x0a 0x0b", then new key is
 265                  * "0x0a 0x0b 0x0a 0x0b 0x0a 0x0b"
 266                  */
 267                 if (key_len == 8) {
 268                         /* Skipping the first 8B as it will be copied
 269                          * in the regular code flow
 270                          */
 271                         memcpy(fctx->enc.encr_key+key_len, key, key_len);
 272                         memcpy(fctx->enc.encr_key+2*key_len, key, key_len);
 273                 }
 274                 break;
 275         case DES3_ECB:
 276                 /* For DES3_ECB IV need to be from CTX. */
 277                 fctx->enc.iv_source = CPT_FROM_CTX;
 278                 break;
 279         case AES_CBC:
 280         case AES_ECB:
 281         case AES_CFB:
 282         case AES_CTR:
 283         case CHACHA20:
 284                 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len);
 285                 break;
 286         case AES_GCM:
 287                 /* Even though iv source is from dptr,
 288                  * aes_gcm salt is taken from ctx
 289                  */
 290                 if (salt) {
 291                         memcpy(fctx->enc.encr_iv, salt, 4);
 292                         /* Assuming it was just salt update
 293                          * and nothing else
 294                          */
 295                         if (!key)
 296                                 goto success;
 297                 }
 298                 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len);
 299                 break;
 300         case AES_XTS:
 301                 key_len = key_len / 2;
 302                 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len);
 303
 304                 /* Copy key2 for XTS into ipad */
 305                 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad));
 306                 memcpy(fctx->hmac.ipad, &key[key_len], key_len);
 307                 break;
 308         case SNOW3G_UEA2:
 309                 cpt_fc_ciph_set_key_snow3g_uea2(cpt_ctx, key, key_len);
 310                 goto success;
 311         case ZUC_EEA3:
 312                 cpt_fc_ciph_set_key_zuc_eea3(cpt_ctx, key, key_len);
 313                 goto success;
 314         case KASUMI_F8_ECB:
 315                 cpt_fc_ciph_set_key_kasumi_f8_ecb(cpt_ctx, key, key_len);
 316                 goto success;
 317         case KASUMI_F8_CBC:
 318                 cpt_fc_ciph_set_key_kasumi_f8_cbc(cpt_ctx, key, key_len);
 319                 goto success;
 320         default:
 321                 return -1;
 322         }
 323
 324         /* Only for FC_GEN case */
 325
 326         /* For GMAC auth, cipher must be NULL */
 327         if (cpt_ctx->hash_type != GMAC_TYPE)
 328                 fctx->enc.enc_cipher = type;
 329
 330         memcpy(fctx->enc.encr_key, key, key_len);
 331
 332 success:
 333         cpt_ctx->enc_cipher = type;
 334
 335         return 0;
 336 }
 337
 338 static __rte_always_inline uint32_t
 339 fill_sg_comp(sg_comp_t *list,
 340              uint32_t i,
 341              phys_addr_t dma_addr,
 342              uint32_t size)
 343 {
 344         sg_comp_t *to = &list[i>>2];
 345
 346         to->u.s.len[i%4] = rte_cpu_to_be_16(size);
 347         to->ptr[i%4] = rte_cpu_to_be_64(dma_addr);
 348         i++;
 349         return i;
 350 }
 351
 352 static __rte_always_inline uint32_t
 353 fill_sg_comp_from_buf(sg_comp_t *list,
 354                       uint32_t i,
 355                       buf_ptr_t *from)
 356 {
 357         sg_comp_t *to = &list[i>>2];
 358
 359         to->u.s.len[i%4] = rte_cpu_to_be_16(from->size);
 360         to->ptr[i%4] = rte_cpu_to_be_64(from->dma_addr);
 361         i++;
 362         return i;
 363 }
 364
 365 static __rte_always_inline uint32_t
 366 fill_sg_comp_from_buf_min(sg_comp_t *list,
 367                           uint32_t i,
 368                           buf_ptr_t *from,
 369                           uint32_t *psize)
 370 {
 371         sg_comp_t *to = &list[i >> 2];
 372         uint32_t size = *psize;
 373         uint32_t e_len;
 374
 375         e_len = (size > from->size) ? from->size : size;
 376         to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
 377         to->ptr[i % 4] = rte_cpu_to_be_64(from->dma_addr);
 378         *psize -= e_len;
 379         i++;
 380         return i;
 381 }
 382
 383 /*
 384  * This fills the MC expected SGIO list
 385  * from IOV given by user.
 386  */
 387 static __rte_always_inline uint32_t
 388 fill_sg_comp_from_iov(sg_comp_t *list,
 389                       uint32_t i,
 390                       iov_ptr_t *from, uint32_t from_offset,
 391                       uint32_t *psize, buf_ptr_t *extra_buf,
 392                       uint32_t extra_offset)
 393 {
 394         int32_t j;
 395         uint32_t extra_len = extra_buf ? extra_buf->size : 0;
 396         uint32_t size = *psize;
 397         buf_ptr_t *bufs;
 398
 399         bufs = from->bufs;
 400         for (j = 0; (j < from->buf_cnt) && size; j++) {
 401                 phys_addr_t e_dma_addr;
 402                 uint32_t e_len;
 403                 sg_comp_t *to = &list[i >> 2];
 404
 405                 if (unlikely(from_offset)) {
 406                         if (from_offset >= bufs[j].size) {
 407                                 from_offset -= bufs[j].size;
 408                                 continue;
 409                         }
 410                         e_dma_addr = bufs[j].dma_addr + from_offset;
 411                         e_len = (size > (bufs[j].size - from_offset)) ?
 412                                 (bufs[j].size - from_offset) : size;
 413                         from_offset = 0;
 414                 } else {
 415                         e_dma_addr = bufs[j].dma_addr;
 416                         e_len = (size > bufs[j].size) ?
 417                                 bufs[j].size : size;
 418                 }
 419
 420                 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
 421                 to->ptr[i % 4] = rte_cpu_to_be_64(e_dma_addr);
 422
 423                 if (extra_len && (e_len >= extra_offset)) {
 424                         /* Break the data at given offset */
 425                         uint32_t next_len = e_len - extra_offset;
 426                         phys_addr_t next_dma = e_dma_addr + extra_offset;
 427
 428                         if (!extra_offset) {
 429                                 i--;
 430                         } else {
 431                                 e_len = extra_offset;
 432                                 size -= e_len;
 433                                 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
 434                         }
 435
 436                         extra_len = RTE_MIN(extra_len, size);
 437                         /* Insert extra data ptr */
 438                         if (extra_len) {
 439                                 i++;
 440                                 to = &list[i >> 2];
 441                                 to->u.s.len[i % 4] =
 442                                         rte_cpu_to_be_16(extra_len);
 443                                 to->ptr[i % 4] =
 444                                         rte_cpu_to_be_64(extra_buf->dma_addr);
 445                                 size -= extra_len;
 446                         }
 447
 448                         next_len = RTE_MIN(next_len, size);
 449                         /* insert the rest of the data */
 450                         if (next_len) {
 451                                 i++;
 452                                 to = &list[i >> 2];
 453                                 to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len);
 454                                 to->ptr[i % 4] = rte_cpu_to_be_64(next_dma);
 455                                 size -= next_len;
 456                         }
 457                         extra_len = 0;
 458
 459                 } else {
 460                         size -= e_len;
 461                 }
 462                 if (extra_offset)
 463                         extra_offset -= size;
 464                 i++;
 465         }
 466
 467         *psize = size;
 468         return (uint32_t)i;
 469 }
 470
 471 static __rte_always_inline void
 472 cpt_digest_gen_prep(uint32_t flags,
 473                     uint64_t d_lens,
 474                     digest_params_t *params,
 475                     void *op,
 476                     void **prep_req)
 477 {
 478         struct cpt_request_info *req;
 479         uint32_t size, i;
 480         uint16_t data_len, mac_len, key_len;
 481         auth_type_t hash_type;
 482         buf_ptr_t *meta_p;
 483         struct cpt_ctx *ctx;
 484         sg_comp_t *gather_comp;
 485         sg_comp_t *scatter_comp;
 486         uint8_t *in_buffer;
 487         uint32_t g_size_bytes, s_size_bytes;
 488         uint64_t dptr_dma, rptr_dma;
 489         vq_cmd_word0_t vq_cmd_w0;
 490         void *c_vaddr, *m_vaddr;
 491         uint64_t c_dma, m_dma;
 492
 493         ctx = params->ctx_buf.vaddr;
 494         meta_p = &params->meta_buf;
 495
 496         m_vaddr = meta_p->vaddr;
 497         m_dma = meta_p->dma_addr;
 498
 499         /*
 500          * Save initial space that followed app data for completion code &
 501          * alternate completion code to fall in same cache line as app data
 502          */
 503         m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
 504         m_dma += COMPLETION_CODE_SIZE;
 505         size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
 506                 (uint8_t *)m_vaddr;
 507         c_vaddr = (uint8_t *)m_vaddr + size;
 508         c_dma = m_dma + size;
 509         size += sizeof(cpt_res_s_t);
 510
 511         m_vaddr = (uint8_t *)m_vaddr + size;
 512         m_dma += size;
 513
 514         req = m_vaddr;
 515
 516         size = sizeof(struct cpt_request_info);
 517         m_vaddr = (uint8_t *)m_vaddr + size;
 518         m_dma += size;
 519
 520         hash_type = ctx->hash_type;
 521         mac_len = ctx->mac_len;
 522         key_len = ctx->auth_key_len;
 523         data_len = AUTH_DLEN(d_lens);
 524
 525         /*GP op header */
 526         vq_cmd_w0.s.opcode.minor = 0;
 527         vq_cmd_w0.s.param2 = ((uint16_t)hash_type << 8);
 528         if (ctx->hmac) {
 529                 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE;
 530                 vq_cmd_w0.s.param1 = key_len;
 531                 vq_cmd_w0.s.dlen = data_len + RTE_ALIGN_CEIL(key_len, 8);
 532         } else {
 533                 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE;
 534                 vq_cmd_w0.s.param1 = 0;
 535                 vq_cmd_w0.s.dlen = data_len;
 536         }
 537
 538         /* Null auth only case enters the if */
 539         if (unlikely(!hash_type && !ctx->enc_cipher)) {
 540                 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_MISC;
 541                 /* Minor op is passthrough */
 542                 vq_cmd_w0.s.opcode.minor = 0x03;
 543                 /* Send out completion code only */
 544                 vq_cmd_w0.s.param2 = 0x1;
 545         }
 546
 547         /* DPTR has SG list */
 548         in_buffer = m_vaddr;
 549         dptr_dma = m_dma;
 550
 551         ((uint16_t *)in_buffer)[0] = 0;
 552         ((uint16_t *)in_buffer)[1] = 0;
 553
 554         /* TODO Add error check if space will be sufficient */
 555         gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
 556
 557         /*
 558          * Input gather list
 559          */
 560
 561         i = 0;
 562
 563         if (ctx->hmac) {
 564                 uint64_t k_dma = params->ctx_buf.dma_addr +
 565                         offsetof(struct cpt_ctx, auth_key);
 566                 /* Key */
 567                 i = fill_sg_comp(gather_comp, i, k_dma,
 568                                  RTE_ALIGN_CEIL(key_len, 8));
 569         }
 570
 571         /* input data */
 572         size = data_len;
 573         if (size) {
 574                 i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov,
 575                                           0, &size, NULL, 0);
 576                 if (unlikely(size)) {
 577                         CPT_LOG_DP_DEBUG("Insufficient dst IOV size, short"
 578                                          " by %dB", size);
 579                         return;
 580                 }
 581         } else {
 582                 /*
 583                  * Looks like we need to support zero data
 584                  * gather ptr in case of hash & hmac
 585                  */
 586                 i++;
 587         }
 588         ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
 589         g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
 590
 591         /*
 592          * Output Gather list
 593          */
 594
 595         i = 0;
 596         scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
 597
 598         if (flags & VALID_MAC_BUF) {
 599                 if (unlikely(params->mac_buf.size < mac_len)) {
 600                         CPT_LOG_DP_ERR("Insufficient MAC size");
 601                         return;
 602                 }
 603
 604                 size = mac_len;
 605                 i = fill_sg_comp_from_buf_min(scatter_comp, i,
 606                                               &params->mac_buf, &size);
 607         } else {
 608                 size = mac_len;
 609                 i = fill_sg_comp_from_iov(scatter_comp, i,
 610                                           params->src_iov, data_len,
 611                                           &size, NULL, 0);
 612                 if (unlikely(size)) {
 613                         CPT_LOG_DP_ERR("Insufficient dst IOV size, short by"
 614                                        " %dB", size);
 615                         return;
 616                 }
 617         }
 618
 619         ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
 620         s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
 621
 622         size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
 623
 624         /* This is DPTR len incase of SG mode */
 625         vq_cmd_w0.s.dlen = size;
 626
 627         m_vaddr = (uint8_t *)m_vaddr + size;
 628         m_dma += size;
 629
 630         /* cpt alternate completion address saved earlier */
 631         req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
 632         *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
 633         rptr_dma = c_dma - 8;
 634
 635         req->ist.ei1 = dptr_dma;
 636         req->ist.ei2 = rptr_dma;
 637
 638         /* 16 byte aligned cpt res address */
 639         req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
 640         *req->completion_addr = COMPLETION_CODE_INIT;
 641         req->comp_baddr  = c_dma;
 642
 643         /* Fill microcode part of instruction */
 644         req->ist.ei0 = vq_cmd_w0.u64;
 645
 646         req->op = op;
 647
 648         *prep_req = req;
 649         return;
 650 }
 651
 652 static __rte_always_inline void
 653 cpt_enc_hmac_prep(uint32_t flags,
 654                   uint64_t d_offs,
 655                   uint64_t d_lens,
 656                   fc_params_t *fc_params,
 657                   void *op,
 658                   void **prep_req)
 659 {
 660         uint32_t iv_offset = 0;
 661         int32_t inputlen, outputlen, enc_dlen, auth_dlen;
 662         struct cpt_ctx *cpt_ctx;
 663         uint32_t cipher_type, hash_type;
 664         uint32_t mac_len, size;
 665         uint8_t iv_len = 16;
 666         struct cpt_request_info *req;
 667         buf_ptr_t *meta_p, *aad_buf = NULL;
 668         uint32_t encr_offset, auth_offset;
 669         uint32_t encr_data_len, auth_data_len, aad_len = 0;
 670         uint32_t passthrough_len = 0;
 671         void *m_vaddr, *offset_vaddr;
 672         uint64_t m_dma, offset_dma;
 673         vq_cmd_word0_t vq_cmd_w0;
 674         void *c_vaddr;
 675         uint64_t c_dma;
 676
 677         meta_p = &fc_params->meta_buf;
 678         m_vaddr = meta_p->vaddr;
 679         m_dma = meta_p->dma_addr;
 680
 681         encr_offset = ENCR_OFFSET(d_offs);
 682         auth_offset = AUTH_OFFSET(d_offs);
 683         encr_data_len = ENCR_DLEN(d_lens);
 684         auth_data_len = AUTH_DLEN(d_lens);
 685         if (unlikely(flags & VALID_AAD_BUF)) {
 686                 /*
 687                  * We dont support both aad
 688                  * and auth data separately
 689                  */
 690                 auth_data_len = 0;
 691                 auth_offset = 0;
 692                 aad_len = fc_params->aad_buf.size;
 693                 aad_buf = &fc_params->aad_buf;
 694         }
 695         cpt_ctx = fc_params->ctx_buf.vaddr;
 696         cipher_type = cpt_ctx->enc_cipher;
 697         hash_type = cpt_ctx->hash_type;
 698         mac_len = cpt_ctx->mac_len;
 699
 700         /*
 701          * Save initial space that followed app data for completion code &
 702          * alternate completion code to fall in same cache line as app data
 703          */
 704         m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
 705         m_dma += COMPLETION_CODE_SIZE;
 706         size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
 707                 (uint8_t *)m_vaddr;
 708
 709         c_vaddr = (uint8_t *)m_vaddr + size;
 710         c_dma = m_dma + size;
 711         size += sizeof(cpt_res_s_t);
 712
 713         m_vaddr = (uint8_t *)m_vaddr + size;
 714         m_dma += size;
 715
 716         /* start cpt request info struct at 8 byte boundary */
 717         size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) -
 718                 (uint8_t *)m_vaddr;
 719
 720         req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size);
 721
 722         size += sizeof(struct cpt_request_info);
 723         m_vaddr = (uint8_t *)m_vaddr + size;
 724         m_dma += size;
 725
 726         if (unlikely(!(flags & VALID_IV_BUF))) {
 727                 iv_len = 0;
 728                 iv_offset = ENCR_IV_OFFSET(d_offs);
 729         }
 730
 731         if (unlikely(flags & VALID_AAD_BUF)) {
 732                 /*
 733                  * When AAD is given, data above encr_offset is pass through
 734                  * Since AAD is given as separate pointer and not as offset,
 735                  * this is a special case as we need to fragment input data
 736                  * into passthrough + encr_data and then insert AAD in between.
 737                  */
 738                 if (hash_type != GMAC_TYPE) {
 739                         passthrough_len = encr_offset;
 740                         auth_offset = passthrough_len + iv_len;
 741                         encr_offset = passthrough_len + aad_len + iv_len;
 742                         auth_data_len = aad_len + encr_data_len;
 743                 } else {
 744                         passthrough_len = 16 + aad_len;
 745                         auth_offset = passthrough_len + iv_len;
 746                         auth_data_len = aad_len;
 747                 }
 748         } else {
 749                 encr_offset += iv_len;
 750                 auth_offset += iv_len;
 751         }
 752
 753         /* Encryption */
 754         vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_FC;
 755         vq_cmd_w0.s.opcode.minor = CPT_FC_MINOR_OP_ENCRYPT;
 756         vq_cmd_w0.s.opcode.minor |= (cpt_ctx->auth_enc <<
 757                                         CPT_HMAC_FIRST_BIT_POS);
 758
 759         if (hash_type == GMAC_TYPE) {
 760                 encr_offset = 0;
 761                 encr_data_len = 0;
 762         }
 763
 764         auth_dlen = auth_offset + auth_data_len;
 765         enc_dlen = encr_data_len + encr_offset;
 766         if (unlikely(encr_data_len & 0xf)) {
 767                 if ((cipher_type == DES3_CBC) || (cipher_type == DES3_ECB))
 768                         enc_dlen = RTE_ALIGN_CEIL(encr_data_len, 8) +
 769                                         encr_offset;
 770                 else if (likely((cipher_type == AES_CBC) ||
 771                                 (cipher_type == AES_ECB)))
 772                         enc_dlen = RTE_ALIGN_CEIL(encr_data_len, 8) +
 773                                         encr_offset;
 774         }
 775
 776         if (unlikely(auth_dlen > enc_dlen)) {
 777                 inputlen = auth_dlen;
 778                 outputlen = auth_dlen + mac_len;
 779         } else {
 780                 inputlen = enc_dlen;
 781                 outputlen = enc_dlen + mac_len;
 782         }
 783
 784         if (cpt_ctx->auth_enc != 0)
 785                 outputlen = enc_dlen;
 786
 787         /* GP op header */
 788         vq_cmd_w0.s.param1 = encr_data_len;
 789         vq_cmd_w0.s.param2 = auth_data_len;
 790         /*
 791          * In 83XX since we have a limitation of
 792          * IV & Offset control word not part of instruction
 793          * and need to be part of Data Buffer, we check if
 794          * head room is there and then only do the Direct mode processing
 795          */
 796         if (likely((flags & SINGLE_BUF_INPLACE) &&
 797                    (flags & SINGLE_BUF_HEADTAILROOM))) {
 798                 void *dm_vaddr = fc_params->bufs[0].vaddr;
 799                 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr;
 800                 /*
 801                  * This flag indicates that there is 24 bytes head room and
 802                  * 8 bytes tail room available, so that we get to do
 803                  * DIRECT MODE with limitation
 804                  */
 805
 806                 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len;
 807                 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len;
 808
 809                 /* DPTR */
 810                 req->ist.ei1 = offset_dma;
 811                 /* RPTR should just exclude offset control word */
 812                 req->ist.ei2 = dm_dma_addr - iv_len;
 813                 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr
 814                                                     + outputlen - iv_len);
 815
 816                 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN;
 817
 818                 if (likely(iv_len)) {
 819                         uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr
 820                                                       + OFF_CTRL_LEN);
 821                         uint64_t *src = fc_params->iv_buf;
 822                         dest[0] = src[0];
 823                         dest[1] = src[1];
 824                 }
 825
 826                 *(uint64_t *)offset_vaddr =
 827                         rte_cpu_to_be_64(((uint64_t)encr_offset << 16) |
 828                                 ((uint64_t)iv_offset << 8) |
 829                                 ((uint64_t)auth_offset));
 830
 831         } else {
 832                 uint32_t i, g_size_bytes, s_size_bytes;
 833                 uint64_t dptr_dma, rptr_dma;
 834                 sg_comp_t *gather_comp;
 835                 sg_comp_t *scatter_comp;
 836                 uint8_t *in_buffer;
 837
 838                 /* This falls under strict SG mode */
 839                 offset_vaddr = m_vaddr;
 840                 offset_dma = m_dma;
 841                 size = OFF_CTRL_LEN + iv_len;
 842
 843                 m_vaddr = (uint8_t *)m_vaddr + size;
 844                 m_dma += size;
 845
 846                 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE;
 847
 848                 if (likely(iv_len)) {
 849                         uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr
 850                                                       + OFF_CTRL_LEN);
 851                         uint64_t *src = fc_params->iv_buf;
 852                         dest[0] = src[0];
 853                         dest[1] = src[1];
 854                 }
 855
 856                 *(uint64_t *)offset_vaddr =
 857                         rte_cpu_to_be_64(((uint64_t)encr_offset << 16) |
 858                                 ((uint64_t)iv_offset << 8) |
 859                                 ((uint64_t)auth_offset));
 860
 861                 /* DPTR has SG list */
 862                 in_buffer = m_vaddr;
 863                 dptr_dma = m_dma;
 864
 865                 ((uint16_t *)in_buffer)[0] = 0;
 866                 ((uint16_t *)in_buffer)[1] = 0;
 867
 868                 /* TODO Add error check if space will be sufficient */
 869                 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
 870
 871                 /*
 872                  * Input Gather List
 873                  */
 874
 875                 i = 0;
 876
 877                 /* Offset control word that includes iv */
 878                 i = fill_sg_comp(gather_comp, i, offset_dma,
 879                                  OFF_CTRL_LEN + iv_len);
 880
 881                 /* Add input data */
 882                 size = inputlen - iv_len;
 883                 if (likely(size)) {
 884                         uint32_t aad_offset = aad_len ? passthrough_len : 0;
 885
 886                         if (unlikely(flags & SINGLE_BUF_INPLACE)) {
 887                                 i = fill_sg_comp_from_buf_min(gather_comp, i,
 888                                                               fc_params->bufs,
 889                                                               &size);
 890                         } else {
 891                                 i = fill_sg_comp_from_iov(gather_comp, i,
 892                                                           fc_params->src_iov,
 893                                                           0, &size,
 894                                                           aad_buf, aad_offset);
 895                         }
 896
 897                         if (unlikely(size)) {
 898                                 CPT_LOG_DP_ERR("Insufficient buffer space,"
 899                                                " size %d needed", size);
 900                                 return;
 901                         }
 902                 }
 903                 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
 904                 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
 905
 906                 /*
 907                  * Output Scatter list
 908                  */
 909                 i = 0;
 910                 scatter_comp =
 911                         (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
 912
 913                 /* Add IV */
 914                 if (likely(iv_len)) {
 915                         i = fill_sg_comp(scatter_comp, i,
 916                                          offset_dma + OFF_CTRL_LEN,
 917                                          iv_len);
 918                 }
 919
 920                 /* output data or output data + digest*/
 921                 if (unlikely(flags & VALID_MAC_BUF)) {
 922                         size = outputlen - iv_len - mac_len;
 923                         if (size) {
 924                                 uint32_t aad_offset =
 925                                         aad_len ? passthrough_len : 0;
 926
 927                                 if (unlikely(flags & SINGLE_BUF_INPLACE)) {
 928                                         i = fill_sg_comp_from_buf_min(
 929                                                         scatter_comp,
 930                                                         i,
 931                                                         fc_params->bufs,
 932                                                         &size);
 933                                 } else {
 934                                         i = fill_sg_comp_from_iov(scatter_comp,
 935                                                         i,
 936                                                         fc_params->dst_iov,
 937                                                         0,
 938                                                         &size,
 939                                                         aad_buf,
 940                                                         aad_offset);
 941                                 }
 942                                 if (unlikely(size)) {
 943                                         CPT_LOG_DP_ERR("Insufficient buffer"
 944                                                        " space, size %d needed",
 945                                                        size);
 946                                         return;
 947                                 }
 948                         }
 949                         /* mac_data */
 950                         if (mac_len) {
 951                                 i = fill_sg_comp_from_buf(scatter_comp, i,
 952                                                           &fc_params->mac_buf);
 953                         }
 954                 } else {
 955                         /* Output including mac */
 956                         size = outputlen - iv_len;
 957                         if (likely(size)) {
 958                                 uint32_t aad_offset =
 959                                         aad_len ? passthrough_len : 0;
 960
 961                                 if (unlikely(flags & SINGLE_BUF_INPLACE)) {
 962                                         i = fill_sg_comp_from_buf_min(
 963                                                         scatter_comp,
 964                                                         i,
 965                                                         fc_params->bufs,
 966                                                         &size);
 967                                 } else {
 968                                         i = fill_sg_comp_from_iov(scatter_comp,
 969                                                         i,
 970                                                         fc_params->dst_iov,
 971                                                         0,
 972                                                         &size,
 973                                                         aad_buf,
 974                                                         aad_offset);
 975                                 }
 976                                 if (unlikely(size)) {
 977                                         CPT_LOG_DP_ERR("Insufficient buffer"
 978                                                        " space, size %d needed",
 979                                                        size);
 980                                         return;
 981                                 }
 982                         }
 983                 }
 984                 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
 985                 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
 986
 987                 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
 988
 989                 /* This is DPTR len incase of SG mode */
 990                 vq_cmd_w0.s.dlen = size;
 991
 992                 m_vaddr = (uint8_t *)m_vaddr + size;
 993                 m_dma += size;
 994
 995                 /* cpt alternate completion address saved earlier */
 996                 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
 997                 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
 998                 rptr_dma = c_dma - 8;
 999
1000                 req->ist.ei1 = dptr_dma;
1001                 req->ist.ei2 = rptr_dma;
1002         }
1003
1004         /* 16 byte aligned cpt res address */
1005         req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
1006         *req->completion_addr = COMPLETION_CODE_INIT;
1007         req->comp_baddr  = c_dma;
1008
1009         /* Fill microcode part of instruction */
1010         req->ist.ei0 = vq_cmd_w0.u64;
1011
1012         req->op  = op;
1013
1014         *prep_req = req;
1015         return;
1016 }
1017
1018 static __rte_always_inline void
1019 cpt_dec_hmac_prep(uint32_t flags,
1020                   uint64_t d_offs,
1021                   uint64_t d_lens,
1022                   fc_params_t *fc_params,
1023                   void *op,
1024                   void **prep_req)
1025 {
1026         uint32_t iv_offset = 0, size;
1027         int32_t inputlen, outputlen, enc_dlen, auth_dlen;
1028         struct cpt_ctx *cpt_ctx;
1029         int32_t hash_type, mac_len;
1030         uint8_t iv_len = 16;
1031         struct cpt_request_info *req;
1032         buf_ptr_t *meta_p, *aad_buf = NULL;
1033         uint32_t encr_offset, auth_offset;
1034         uint32_t encr_data_len, auth_data_len, aad_len = 0;
1035         uint32_t passthrough_len = 0;
1036         void *m_vaddr, *offset_vaddr;
1037         uint64_t m_dma, offset_dma;
1038         vq_cmd_word0_t vq_cmd_w0;
1039         void *c_vaddr;
1040         uint64_t c_dma;
1041
1042         meta_p = &fc_params->meta_buf;
1043         m_vaddr = meta_p->vaddr;
1044         m_dma = meta_p->dma_addr;
1045
1046         encr_offset = ENCR_OFFSET(d_offs);
1047         auth_offset = AUTH_OFFSET(d_offs);
1048         encr_data_len = ENCR_DLEN(d_lens);
1049         auth_data_len = AUTH_DLEN(d_lens);
1050
1051         if (unlikely(flags & VALID_AAD_BUF)) {
1052                 /*
1053                  * We dont support both aad
1054                  * and auth data separately
1055                  */
1056                 auth_data_len = 0;
1057                 auth_offset = 0;
1058                 aad_len = fc_params->aad_buf.size;
1059                 aad_buf = &fc_params->aad_buf;
1060         }
1061
1062         cpt_ctx = fc_params->ctx_buf.vaddr;
1063         hash_type = cpt_ctx->hash_type;
1064         mac_len = cpt_ctx->mac_len;
1065
1066         if (unlikely(!(flags & VALID_IV_BUF))) {
1067                 iv_len = 0;
1068                 iv_offset = ENCR_IV_OFFSET(d_offs);
1069         }
1070
1071         if (unlikely(flags & VALID_AAD_BUF)) {
1072                 /*
1073                  * When AAD is given, data above encr_offset is pass through
1074                  * Since AAD is given as separate pointer and not as offset,
1075                  * this is a special case as we need to fragment input data
1076                  * into passthrough + encr_data and then insert AAD in between.
1077                  */
1078                 if (hash_type != GMAC_TYPE) {
1079                         passthrough_len = encr_offset;
1080                         auth_offset = passthrough_len + iv_len;
1081                         encr_offset = passthrough_len + aad_len + iv_len;
1082                         auth_data_len = aad_len + encr_data_len;
1083                 } else {
1084                         passthrough_len = 16 + aad_len;
1085                         auth_offset = passthrough_len + iv_len;
1086                         auth_data_len = aad_len;
1087                 }
1088         } else {
1089                 encr_offset += iv_len;
1090                 auth_offset += iv_len;
1091         }
1092
1093         /*
1094          * Save initial space that followed app data for completion code &
1095          * alternate completion code to fall in same cache line as app data
1096          */
1097         m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
1098         m_dma += COMPLETION_CODE_SIZE;
1099         size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
1100                (uint8_t *)m_vaddr;
1101         c_vaddr = (uint8_t *)m_vaddr + size;
1102         c_dma = m_dma + size;
1103         size += sizeof(cpt_res_s_t);
1104
1105         m_vaddr = (uint8_t *)m_vaddr + size;
1106         m_dma += size;
1107
1108         /* start cpt request info structure at 8 byte alignment */
1109         size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) -
1110                 (uint8_t *)m_vaddr;
1111
1112         req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size);
1113
1114         size += sizeof(struct cpt_request_info);
1115         m_vaddr = (uint8_t *)m_vaddr + size;
1116         m_dma += size;
1117
1118         /* Decryption */
1119         vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_FC;
1120         vq_cmd_w0.s.opcode.minor = CPT_FC_MINOR_OP_DECRYPT;
1121         vq_cmd_w0.s.opcode.minor |= (cpt_ctx->dec_auth <<
1122                                         CPT_HMAC_FIRST_BIT_POS);
1123
1124         if (hash_type == GMAC_TYPE) {
1125                 encr_offset = 0;
1126                 encr_data_len = 0;
1127         }
1128
1129         enc_dlen = encr_offset + encr_data_len;
1130         auth_dlen = auth_offset + auth_data_len;
1131
1132         if (auth_dlen > enc_dlen) {
1133                 inputlen = auth_dlen + mac_len;
1134                 outputlen = auth_dlen;
1135         } else {
1136                 inputlen = enc_dlen + mac_len;
1137                 outputlen = enc_dlen;
1138         }
1139
1140         if (cpt_ctx->dec_auth != 0)
1141                 outputlen = inputlen = enc_dlen;
1142
1143         vq_cmd_w0.s.param1 = encr_data_len;
1144         vq_cmd_w0.s.param2 = auth_data_len;
1145
1146         /*
1147          * In 83XX since we have a limitation of
1148          * IV & Offset control word not part of instruction
1149          * and need to be part of Data Buffer, we check if
1150          * head room is there and then only do the Direct mode processing
1151          */
1152         if (likely((flags & SINGLE_BUF_INPLACE) &&
1153                    (flags & SINGLE_BUF_HEADTAILROOM))) {
1154                 void *dm_vaddr = fc_params->bufs[0].vaddr;
1155                 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr;
1156                 /*
1157                  * This flag indicates that there is 24 bytes head room and
1158                  * 8 bytes tail room available, so that we get to do
1159                  * DIRECT MODE with limitation
1160                  */
1161
1162                 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len;
1163                 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len;
1164                 req->ist.ei1 = offset_dma;
1165
1166                 /* RPTR should just exclude offset control word */
1167                 req->ist.ei2 = dm_dma_addr - iv_len;
1168
1169                 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr +
1170                                         outputlen - iv_len);
1171                 /* since this is decryption,
1172                  * don't touch the content of
1173                  * alternate ccode space as it contains
1174                  * hmac.
1175                  */
1176
1177                 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN;
1178
1179                 if (likely(iv_len)) {
1180                         uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr +
1181                                                       OFF_CTRL_LEN);
1182                         uint64_t *src = fc_params->iv_buf;
1183                         dest[0] = src[0];
1184                         dest[1] = src[1];
1185                 }
1186
1187                 *(uint64_t *)offset_vaddr =
1188                         rte_cpu_to_be_64(((uint64_t)encr_offset << 16) |
1189                                 ((uint64_t)iv_offset << 8) |
1190                                 ((uint64_t)auth_offset));
1191
1192         } else {
1193                 uint64_t dptr_dma, rptr_dma;
1194                 uint32_t g_size_bytes, s_size_bytes;
1195                 sg_comp_t *gather_comp;
1196                 sg_comp_t *scatter_comp;
1197                 uint8_t *in_buffer;
1198                 uint8_t i = 0;
1199
1200                 /* This falls under strict SG mode */
1201                 offset_vaddr = m_vaddr;
1202                 offset_dma = m_dma;
1203                 size = OFF_CTRL_LEN + iv_len;
1204
1205                 m_vaddr = (uint8_t *)m_vaddr + size;
1206                 m_dma += size;
1207
1208                 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE;
1209
1210                 if (likely(iv_len)) {
1211                         uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr +
1212                                                       OFF_CTRL_LEN);
1213                         uint64_t *src = fc_params->iv_buf;
1214                         dest[0] = src[0];
1215                         dest[1] = src[1];
1216                 }
1217
1218                 *(uint64_t *)offset_vaddr =
1219                         rte_cpu_to_be_64(((uint64_t)encr_offset << 16) |
1220                                 ((uint64_t)iv_offset << 8) |
1221                                 ((uint64_t)auth_offset));
1222
1223                 /* DPTR has SG list */
1224                 in_buffer = m_vaddr;
1225                 dptr_dma = m_dma;
1226
1227                 ((uint16_t *)in_buffer)[0] = 0;
1228                 ((uint16_t *)in_buffer)[1] = 0;
1229
1230                 /* TODO Add error check if space will be sufficient */
1231                 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
1232
1233                 /*
1234                  * Input Gather List
1235                  */
1236                 i = 0;
1237
1238                 /* Offset control word that includes iv */
1239                 i = fill_sg_comp(gather_comp, i, offset_dma,
1240                                  OFF_CTRL_LEN + iv_len);
1241
1242                 /* Add input data */
1243                 if (flags & VALID_MAC_BUF) {
1244                         size = inputlen - iv_len - mac_len;
1245                         if (size) {
1246                                 /* input data only */
1247                                 if (unlikely(flags & SINGLE_BUF_INPLACE)) {
1248                                         i = fill_sg_comp_from_buf_min(
1249                                                         gather_comp, i,
1250                                                         fc_params->bufs,
1251                                                         &size);
1252                                 } else {
1253                                         uint32_t aad_offset = aad_len ?
1254                                                 passthrough_len : 0;
1255
1256                                         i = fill_sg_comp_from_iov(gather_comp,
1257                                                         i,
1258                                                         fc_params->src_iov,
1259                                                         0, &size,
1260                                                         aad_buf,
1261                                                         aad_offset);
1262                                 }
1263                                 if (unlikely(size)) {
1264                                         CPT_LOG_DP_ERR("Insufficient buffer"
1265                                                        " space, size %d needed",
1266                                                        size);
1267                                         return;
1268                                 }
1269                         }
1270
1271                         /* mac data */
1272                         if (mac_len) {
1273                                 i = fill_sg_comp_from_buf(gather_comp, i,
1274                                                           &fc_params->mac_buf);
1275                         }
1276                 } else {
1277                         /* input data + mac */
1278                         size = inputlen - iv_len;
1279                         if (size) {
1280                                 if (unlikely(flags & SINGLE_BUF_INPLACE)) {
1281                                         i = fill_sg_comp_from_buf_min(
1282                                                         gather_comp, i,
1283                                                         fc_params->bufs,
1284                                                         &size);
1285                                 } else {
1286                                         uint32_t aad_offset = aad_len ?
1287                                                 passthrough_len : 0;
1288
1289                                         if (unlikely(!fc_params->src_iov)) {
1290                                                 CPT_LOG_DP_ERR("Bad input args");
1291                                                 return;
1292                                         }
1293
1294                                         i = fill_sg_comp_from_iov(
1295                                                         gather_comp, i,
1296                                                         fc_params->src_iov,
1297                                                         0, &size,
1298                                                         aad_buf,
1299                                                         aad_offset);
1300                                 }
1301
1302                                 if (unlikely(size)) {
1303                                         CPT_LOG_DP_ERR("Insufficient buffer"
1304                                                        " space, size %d needed",
1305                                                        size);
1306                                         return;
1307                                 }
1308                         }
1309                 }
1310                 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
1311                 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
1312
1313                 /*
1314                  * Output Scatter List
1315                  */
1316
1317                 i = 0;
1318                 scatter_comp =
1319                         (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
1320
1321                 /* Add iv */
1322                 if (iv_len) {
1323                         i = fill_sg_comp(scatter_comp, i,
1324                                          offset_dma + OFF_CTRL_LEN,
1325                                          iv_len);
1326                 }
1327
1328                 /* Add output data */
1329                 size = outputlen - iv_len;
1330                 if (size) {
1331                         if (unlikely(flags & SINGLE_BUF_INPLACE)) {
1332                                 /* handle single buffer here */
1333                                 i = fill_sg_comp_from_buf_min(scatter_comp, i,
1334                                                               fc_params->bufs,
1335                                                               &size);
1336                         } else {
1337                                 uint32_t aad_offset = aad_len ?
1338                                         passthrough_len : 0;
1339
1340                                 if (unlikely(!fc_params->dst_iov)) {
1341                                         CPT_LOG_DP_ERR("Bad input args");
1342                                         return;
1343                                 }
1344
1345                                 i = fill_sg_comp_from_iov(scatter_comp, i,
1346                                                           fc_params->dst_iov, 0,
1347                                                           &size, aad_buf,
1348                                                           aad_offset);
1349                         }
1350
1351                         if (unlikely(size)) {
1352                                 CPT_LOG_DP_ERR("Insufficient buffer space,"
1353                                                " size %d needed", size);
1354                                 return;
1355                         }
1356                 }
1357
1358                 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
1359                 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
1360
1361                 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
1362
1363                 /* This is DPTR len incase of SG mode */
1364                 vq_cmd_w0.s.dlen = size;
1365
1366                 m_vaddr = (uint8_t *)m_vaddr + size;
1367                 m_dma += size;
1368
1369                 /* cpt alternate completion address saved earlier */
1370                 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
1371                 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
1372                 rptr_dma = c_dma - 8;
1373                 size += COMPLETION_CODE_SIZE;
1374
1375                 req->ist.ei1 = dptr_dma;
1376                 req->ist.ei2 = rptr_dma;
1377         }
1378
1379         /* 16 byte aligned cpt res address */
1380         req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
1381         *req->completion_addr = COMPLETION_CODE_INIT;
1382         req->comp_baddr  = c_dma;
1383
1384         /* Fill microcode part of instruction */
1385         req->ist.ei0 = vq_cmd_w0.u64;
1386
1387         req->op = op;
1388
1389         *prep_req = req;
1390         return;
1391 }
1392
1393 static __rte_always_inline void
1394 cpt_zuc_snow3g_enc_prep(uint32_t req_flags,
1395                         uint64_t d_offs,
1396                         uint64_t d_lens,
1397                         fc_params_t *params,
1398                         void *op,
1399                         void **prep_req)
1400 {
1401         uint32_t size;
1402         int32_t inputlen, outputlen;
1403         struct cpt_ctx *cpt_ctx;
1404         uint32_t mac_len = 0;
1405         uint8_t snow3g, j;
1406         struct cpt_request_info *req;
1407         buf_ptr_t *buf_p;
1408         uint32_t encr_offset = 0, auth_offset = 0;
1409         uint32_t encr_data_len = 0, auth_data_len = 0;
1410         int flags, iv_len = 16;
1411         void *m_vaddr, *c_vaddr;
1412         uint64_t m_dma, c_dma, offset_ctrl;
1413         uint64_t *offset_vaddr, offset_dma;
1414         uint32_t *iv_s, iv[4];
1415         vq_cmd_word0_t vq_cmd_w0;
1416
1417         buf_p = &params->meta_buf;
1418         m_vaddr = buf_p->vaddr;
1419         m_dma = buf_p->dma_addr;
1420
1421         cpt_ctx = params->ctx_buf.vaddr;
1422         flags = cpt_ctx->zsk_flags;
1423         mac_len = cpt_ctx->mac_len;
1424         snow3g = cpt_ctx->snow3g;
1425
1426         /*
1427          * Save initial space that followed app data for completion code &
1428          * alternate completion code to fall in same cache line as app data
1429          */
1430         m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
1431         m_dma += COMPLETION_CODE_SIZE;
1432         size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
1433                 (uint8_t *)m_vaddr;
1434
1435         c_vaddr = (uint8_t *)m_vaddr + size;
1436         c_dma = m_dma + size;
1437         size += sizeof(cpt_res_s_t);
1438
1439         m_vaddr = (uint8_t *)m_vaddr + size;
1440         m_dma += size;
1441
1442         /* Reserve memory for cpt request info */
1443         req = m_vaddr;
1444
1445         size = sizeof(struct cpt_request_info);
1446         m_vaddr = (uint8_t *)m_vaddr + size;
1447         m_dma += size;
1448
1449         vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_ZUC_SNOW3G;
1450
1451         /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */
1452
1453         vq_cmd_w0.s.opcode.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) |
1454                           (0 << 3) | (flags & 0x7));
1455
1456         if (flags == 0x1) {
1457                 /*
1458                  * Microcode expects offsets in bytes
1459                  * TODO: Rounding off
1460                  */
1461                 auth_data_len = AUTH_DLEN(d_lens);
1462
1463                 /* EIA3 or UIA2 */
1464                 auth_offset = AUTH_OFFSET(d_offs);
1465                 auth_offset = auth_offset / 8;
1466
1467                 /* consider iv len */
1468                 auth_offset += iv_len;
1469
1470                 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8);
1471                 outputlen = mac_len;
1472
1473                 offset_ctrl = rte_cpu_to_be_64((uint64_t)auth_offset);
1474
1475         } else {
1476                 /* EEA3 or UEA2 */
1477                 /*
1478                  * Microcode expects offsets in bytes
1479                  * TODO: Rounding off
1480                  */
1481                 encr_data_len = ENCR_DLEN(d_lens);
1482
1483                 encr_offset = ENCR_OFFSET(d_offs);
1484                 encr_offset = encr_offset / 8;
1485                 /* consider iv len */
1486                 encr_offset += iv_len;
1487
1488                 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8);
1489                 outputlen = inputlen;
1490
1491                 /* iv offset is 0 */
1492                 offset_ctrl = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
1493         }
1494
1495         /* IV */
1496         iv_s = (flags == 0x1) ? params->auth_iv_buf :
1497                 params->iv_buf;
1498
1499         if (snow3g) {
1500                 /*
1501                  * DPDK seems to provide it in form of IV3 IV2 IV1 IV0
1502                  * and BigEndian, MC needs it as IV0 IV1 IV2 IV3
1503                  */
1504
1505                 for (j = 0; j < 4; j++)
1506                         iv[j] = iv_s[3 - j];
1507         } else {
1508                 /* ZUC doesn't need a swap */
1509                 for (j = 0; j < 4; j++)
1510                         iv[j] = iv_s[j];
1511         }
1512
1513         /*
1514          * GP op header, lengths are expected in bits.
1515          */
1516         vq_cmd_w0.s.param1 = encr_data_len;
1517         vq_cmd_w0.s.param2 = auth_data_len;
1518
1519         /*
1520          * In 83XX since we have a limitation of
1521          * IV & Offset control word not part of instruction
1522          * and need to be part of Data Buffer, we check if
1523          * head room is there and then only do the Direct mode processing
1524          */
1525         if (likely((req_flags & SINGLE_BUF_INPLACE) &&
1526                    (req_flags & SINGLE_BUF_HEADTAILROOM))) {
1527                 void *dm_vaddr = params->bufs[0].vaddr;
1528                 uint64_t dm_dma_addr = params->bufs[0].dma_addr;
1529                 /*
1530                  * This flag indicates that there is 24 bytes head room and
1531                  * 8 bytes tail room available, so that we get to do
1532                  * DIRECT MODE with limitation
1533                  */
1534
1535                 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr -
1536                                             OFF_CTRL_LEN - iv_len);
1537                 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len;
1538
1539                 /* DPTR */
1540                 req->ist.ei1 = offset_dma;
1541                 /* RPTR should just exclude offset control word */
1542                 req->ist.ei2 = dm_dma_addr - iv_len;
1543                 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr
1544                                                     + outputlen - iv_len);
1545
1546                 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN;
1547
1548                 if (likely(iv_len)) {
1549                         uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr
1550                                                       + OFF_CTRL_LEN);
1551                         memcpy(iv_d, iv, 16);
1552                 }
1553
1554                 *offset_vaddr = offset_ctrl;
1555         } else {
1556                 uint32_t i, g_size_bytes, s_size_bytes;
1557                 uint64_t dptr_dma, rptr_dma;
1558                 sg_comp_t *gather_comp;
1559                 sg_comp_t *scatter_comp;
1560                 uint8_t *in_buffer;
1561                 uint32_t *iv_d;
1562
1563                 /* save space for iv */
1564                 offset_vaddr = m_vaddr;
1565                 offset_dma = m_dma;
1566
1567                 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
1568                 m_dma += OFF_CTRL_LEN + iv_len;
1569
1570                 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE;
1571
1572                 /* DPTR has SG list */
1573                 in_buffer = m_vaddr;
1574                 dptr_dma = m_dma;
1575
1576                 ((uint16_t *)in_buffer)[0] = 0;
1577                 ((uint16_t *)in_buffer)[1] = 0;
1578
1579                 /* TODO Add error check if space will be sufficient */
1580                 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
1581
1582                 /*
1583                  * Input Gather List
1584                  */
1585                 i = 0;
1586
1587                 /* Offset control word followed by iv */
1588
1589                 i = fill_sg_comp(gather_comp, i, offset_dma,
1590                                  OFF_CTRL_LEN + iv_len);
1591
1592                 /* iv offset is 0 */
1593                 *offset_vaddr = offset_ctrl;
1594
1595                 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN);
1596                 memcpy(iv_d, iv, 16);
1597
1598                 /* input data */
1599                 size = inputlen - iv_len;
1600                 if (size) {
1601                         i = fill_sg_comp_from_iov(gather_comp, i,
1602                                                   params->src_iov,
1603                                                   0, &size, NULL, 0);
1604                         if (unlikely(size)) {
1605                                 CPT_LOG_DP_ERR("Insufficient buffer space,"
1606                                                " size %d needed", size);
1607                                 return;
1608                         }
1609                 }
1610                 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
1611                 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
1612
1613                 /*
1614                  * Output Scatter List
1615                  */
1616
1617                 i = 0;
1618                 scatter_comp =
1619                         (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
1620
1621                 if (flags == 0x1) {
1622                         /* IV in SLIST only for EEA3 & UEA2 */
1623                         iv_len = 0;
1624                 }
1625
1626                 if (iv_len) {
1627                         i = fill_sg_comp(scatter_comp, i,
1628                                          offset_dma + OFF_CTRL_LEN, iv_len);
1629                 }
1630
1631                 /* Add output data */
1632                 if (req_flags & VALID_MAC_BUF) {
1633                         size = outputlen - iv_len - mac_len;
1634                         if (size) {
1635                                 i = fill_sg_comp_from_iov(scatter_comp, i,
1636                                                           params->dst_iov, 0,
1637                                                           &size, NULL, 0);
1638
1639                                 if (unlikely(size)) {
1640                                         CPT_LOG_DP_ERR("Insufficient buffer space,"
1641                                                        " size %d needed", size);
1642                                         return;
1643                                 }
1644                         }
1645
1646                         /* mac data */
1647                         if (mac_len) {
1648                                 i = fill_sg_comp_from_buf(scatter_comp, i,
1649                                                           &params->mac_buf);
1650                         }
1651                 } else {
1652                         /* Output including mac */
1653                         size = outputlen - iv_len;
1654                         if (size) {
1655                                 i = fill_sg_comp_from_iov(scatter_comp, i,
1656                                                           params->dst_iov, 0,
1657                                                           &size, NULL, 0);
1658
1659                                 if (unlikely(size)) {
1660                                         CPT_LOG_DP_ERR("Insufficient buffer space,"
1661                                                        " size %d needed", size);
1662                                         return;
1663                                 }
1664                         }
1665                 }
1666                 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
1667                 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
1668
1669                 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
1670
1671                 /* This is DPTR len incase of SG mode */
1672                 vq_cmd_w0.s.dlen = size;
1673
1674                 m_vaddr = (uint8_t *)m_vaddr + size;
1675                 m_dma += size;
1676
1677                 /* cpt alternate completion address saved earlier */
1678                 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
1679                 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
1680                 rptr_dma = c_dma - 8;
1681
1682                 req->ist.ei1 = dptr_dma;
1683                 req->ist.ei2 = rptr_dma;
1684         }
1685
1686         /* 16 byte aligned cpt res address */
1687         req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
1688         *req->completion_addr = COMPLETION_CODE_INIT;
1689         req->comp_baddr  = c_dma;
1690
1691         /* Fill microcode part of instruction */
1692         req->ist.ei0 = vq_cmd_w0.u64;
1693
1694         req->op = op;
1695
1696         *prep_req = req;
1697         return;
1698 }
1699
1700 static __rte_always_inline void
1701 cpt_zuc_snow3g_dec_prep(uint32_t req_flags,
1702                         uint64_t d_offs,
1703                         uint64_t d_lens,
1704                         fc_params_t *params,
1705                         void *op,
1706                         void **prep_req)
1707 {
1708         uint32_t size;
1709         int32_t inputlen = 0, outputlen;
1710         struct cpt_ctx *cpt_ctx;
1711         uint8_t snow3g, iv_len = 16;
1712         struct cpt_request_info *req;
1713         buf_ptr_t *buf_p;
1714         uint32_t encr_offset;
1715         uint32_t encr_data_len;
1716         int flags;
1717         void *m_vaddr, *c_vaddr;
1718         uint64_t m_dma, c_dma;
1719         uint64_t *offset_vaddr, offset_dma;
1720         uint32_t *iv_s, iv[4], j;
1721         vq_cmd_word0_t vq_cmd_w0;
1722
1723         buf_p = &params->meta_buf;
1724         m_vaddr = buf_p->vaddr;
1725         m_dma = buf_p->dma_addr;
1726
1727         /*
1728          * Microcode expects offsets in bytes
1729          * TODO: Rounding off
1730          */
1731         encr_offset = ENCR_OFFSET(d_offs) / 8;
1732         encr_data_len = ENCR_DLEN(d_lens);
1733
1734         cpt_ctx = params->ctx_buf.vaddr;
1735         flags = cpt_ctx->zsk_flags;
1736         snow3g = cpt_ctx->snow3g;
1737         /*
1738          * Save initial space that followed app data for completion code &
1739          * alternate completion code to fall in same cache line as app data
1740          */
1741         m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
1742         m_dma += COMPLETION_CODE_SIZE;
1743         size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
1744                 (uint8_t *)m_vaddr;
1745
1746         c_vaddr = (uint8_t *)m_vaddr + size;
1747         c_dma = m_dma + size;
1748         size += sizeof(cpt_res_s_t);
1749
1750         m_vaddr = (uint8_t *)m_vaddr + size;
1751         m_dma += size;
1752
1753         /* Reserve memory for cpt request info */
1754         req = m_vaddr;
1755
1756         size = sizeof(struct cpt_request_info);
1757         m_vaddr = (uint8_t *)m_vaddr + size;
1758         m_dma += size;
1759
1760         vq_cmd_w0.u64 = 0;
1761         vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_ZUC_SNOW3G;
1762
1763         /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */
1764
1765         vq_cmd_w0.s.opcode.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) |
1766                           (0 << 3) | (flags & 0x7));
1767
1768         /* consider iv len */
1769         encr_offset += iv_len;
1770
1771         inputlen = encr_offset +
1772                 (RTE_ALIGN(encr_data_len, 8) / 8);
1773         outputlen = inputlen;
1774
1775         /* IV */
1776         iv_s = params->iv_buf;
1777         if (snow3g) {
1778                 /*
1779                  * DPDK seems to provide it in form of IV3 IV2 IV1 IV0
1780                  * and BigEndian, MC needs it as IV0 IV1 IV2 IV3
1781                  */
1782
1783                 for (j = 0; j < 4; j++)
1784                         iv[j] = iv_s[3 - j];
1785         } else {
1786                 /* ZUC doesn't need a swap */
1787                 for (j = 0; j < 4; j++)
1788                         iv[j] = iv_s[j];
1789         }
1790
1791         /*
1792          * GP op header, lengths are expected in bits.
1793          */
1794         vq_cmd_w0.s.param1 = encr_data_len;
1795
1796         /*
1797          * In 83XX since we have a limitation of
1798          * IV & Offset control word not part of instruction
1799          * and need to be part of Data Buffer, we check if
1800          * head room is there and then only do the Direct mode processing
1801          */
1802         if (likely((req_flags & SINGLE_BUF_INPLACE) &&
1803                    (req_flags & SINGLE_BUF_HEADTAILROOM))) {
1804                 void *dm_vaddr = params->bufs[0].vaddr;
1805                 uint64_t dm_dma_addr = params->bufs[0].dma_addr;
1806                 /*
1807                  * This flag indicates that there is 24 bytes head room and
1808                  * 8 bytes tail room available, so that we get to do
1809                  * DIRECT MODE with limitation
1810                  */
1811
1812                 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr -
1813                                             OFF_CTRL_LEN - iv_len);
1814                 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len;
1815
1816                 /* DPTR */
1817                 req->ist.ei1 = offset_dma;
1818                 /* RPTR should just exclude offset control word */
1819                 req->ist.ei2 = dm_dma_addr - iv_len;
1820                 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr
1821                                                     + outputlen - iv_len);
1822
1823                 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN;
1824
1825                 if (likely(iv_len)) {
1826                         uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr
1827                                                       + OFF_CTRL_LEN);
1828                         memcpy(iv_d, iv, 16);
1829                 }
1830
1831                 /* iv offset is 0 */
1832                 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
1833         } else {
1834                 uint32_t i, g_size_bytes, s_size_bytes;
1835                 uint64_t dptr_dma, rptr_dma;
1836                 sg_comp_t *gather_comp;
1837                 sg_comp_t *scatter_comp;
1838                 uint8_t *in_buffer;
1839                 uint32_t *iv_d;
1840
1841                 /* save space for offset and iv... */
1842                 offset_vaddr = m_vaddr;
1843                 offset_dma = m_dma;
1844
1845                 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
1846                 m_dma += OFF_CTRL_LEN + iv_len;
1847
1848                 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE;
1849
1850                 /* DPTR has SG list */
1851                 in_buffer = m_vaddr;
1852                 dptr_dma = m_dma;
1853
1854                 ((uint16_t *)in_buffer)[0] = 0;
1855                 ((uint16_t *)in_buffer)[1] = 0;
1856
1857                 /* TODO Add error check if space will be sufficient */
1858                 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
1859
1860                 /*
1861                  * Input Gather List
1862                  */
1863                 i = 0;
1864
1865                 /* Offset control word */
1866
1867                 /* iv offset is 0 */
1868                 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
1869
1870                 i = fill_sg_comp(gather_comp, i, offset_dma,
1871                                  OFF_CTRL_LEN + iv_len);
1872
1873                 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN);
1874                 memcpy(iv_d, iv, 16);
1875
1876                 /* Add input data */
1877                 size = inputlen - iv_len;
1878                 if (size) {
1879                         i = fill_sg_comp_from_iov(gather_comp, i,
1880                                                   params->src_iov,
1881                                                   0, &size, NULL, 0);
1882                         if (unlikely(size)) {
1883                                 CPT_LOG_DP_ERR("Insufficient buffer space,"
1884                                                " size %d needed", size);
1885                                 return;
1886                         }
1887                 }
1888                 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
1889                 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
1890
1891                 /*
1892                  * Output Scatter List
1893                  */
1894
1895                 i = 0;
1896                 scatter_comp =
1897                         (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
1898
1899                 /* IV */
1900                 i = fill_sg_comp(scatter_comp, i,
1901                                  offset_dma + OFF_CTRL_LEN,
1902                                  iv_len);
1903
1904                 /* Add output data */
1905                 size = outputlen - iv_len;
1906                 if (size) {
1907                         i = fill_sg_comp_from_iov(scatter_comp, i,
1908                                                   params->dst_iov, 0,
1909                                                   &size, NULL, 0);
1910
1911                         if (unlikely(size)) {
1912                                 CPT_LOG_DP_ERR("Insufficient buffer space,"
1913                                                " size %d needed", size);
1914                                 return;
1915                         }
1916                 }
1917                 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
1918                 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
1919
1920                 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
1921
1922                 /* This is DPTR len incase of SG mode */
1923                 vq_cmd_w0.s.dlen = size;
1924
1925                 m_vaddr = (uint8_t *)m_vaddr + size;
1926                 m_dma += size;
1927
1928                 /* cpt alternate completion address saved earlier */
1929                 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
1930                 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
1931                 rptr_dma = c_dma - 8;
1932
1933                 req->ist.ei1 = dptr_dma;
1934                 req->ist.ei2 = rptr_dma;
1935         }
1936
1937         /* 16 byte aligned cpt res address */
1938         req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
1939         *req->completion_addr = COMPLETION_CODE_INIT;
1940         req->comp_baddr  = c_dma;
1941
1942         /* Fill microcode part of instruction */
1943         req->ist.ei0 = vq_cmd_w0.u64;
1944
1945         req->op = op;
1946
1947         *prep_req = req;
1948         return;
1949 }
1950
1951 static __rte_always_inline void
1952 cpt_kasumi_enc_prep(uint32_t req_flags,
1953                     uint64_t d_offs,
1954                     uint64_t d_lens,
1955                     fc_params_t *params,
1956                     void *op,
1957                     void **prep_req)
1958 {
1959         uint32_t size;
1960         int32_t inputlen = 0, outputlen = 0;
1961         struct cpt_ctx *cpt_ctx;
1962         uint32_t mac_len = 0;
1963         uint8_t i = 0;
1964         struct cpt_request_info *req;
1965         buf_ptr_t *buf_p;
1966         uint32_t encr_offset, auth_offset;
1967         uint32_t encr_data_len, auth_data_len;
1968         int flags;
1969         uint8_t *iv_s, *iv_d, iv_len = 8;
1970         uint8_t dir = 0;
1971         void *m_vaddr, *c_vaddr;
1972         uint64_t m_dma, c_dma;
1973         uint64_t *offset_vaddr, offset_dma;
1974         vq_cmd_word0_t vq_cmd_w0;
1975         uint8_t *in_buffer;
1976         uint32_t g_size_bytes, s_size_bytes;
1977         uint64_t dptr_dma, rptr_dma;
1978         sg_comp_t *gather_comp;
1979         sg_comp_t *scatter_comp;
1980
1981         buf_p = &params->meta_buf;
1982         m_vaddr = buf_p->vaddr;
1983         m_dma = buf_p->dma_addr;
1984
1985         encr_offset = ENCR_OFFSET(d_offs) / 8;
1986         auth_offset = AUTH_OFFSET(d_offs) / 8;
1987         encr_data_len = ENCR_DLEN(d_lens);
1988         auth_data_len = AUTH_DLEN(d_lens);
1989
1990         cpt_ctx = params->ctx_buf.vaddr;
1991         flags = cpt_ctx->zsk_flags;
1992         mac_len = cpt_ctx->mac_len;
1993
1994         if (flags == 0x0)
1995                 iv_s = params->iv_buf;
1996         else
1997                 iv_s = params->auth_iv_buf;
1998
1999         dir = iv_s[8] & 0x1;
2000
2001         /*
2002          * Save initial space that followed app data for completion code &
2003          * alternate completion code to fall in same cache line as app data
2004          */
2005         m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
2006         m_dma += COMPLETION_CODE_SIZE;
2007         size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
2008                 (uint8_t *)m_vaddr;
2009
2010         c_vaddr = (uint8_t *)m_vaddr + size;
2011         c_dma = m_dma + size;
2012         size += sizeof(cpt_res_s_t);
2013
2014         m_vaddr = (uint8_t *)m_vaddr + size;
2015         m_dma += size;
2016
2017         /* Reserve memory for cpt request info */
2018         req = m_vaddr;
2019
2020         size = sizeof(struct cpt_request_info);
2021         m_vaddr = (uint8_t *)m_vaddr + size;
2022         m_dma += size;
2023
2024         vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE;
2025
2026         /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */
2027         vq_cmd_w0.s.opcode.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) |
2028                           (dir << 4) | (0 << 3) | (flags & 0x7));
2029
2030         /*
2031          * GP op header, lengths are expected in bits.
2032          */
2033         vq_cmd_w0.s.param1 = encr_data_len;
2034         vq_cmd_w0.s.param2 = auth_data_len;
2035
2036         /* consider iv len */
2037         if (flags == 0x0) {
2038                 encr_offset += iv_len;
2039                 auth_offset += iv_len;
2040         }
2041
2042         /* save space for offset ctrl and iv */
2043         offset_vaddr = m_vaddr;
2044         offset_dma = m_dma;
2045
2046         m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
2047         m_dma += OFF_CTRL_LEN + iv_len;
2048
2049         /* DPTR has SG list */
2050         in_buffer = m_vaddr;
2051         dptr_dma = m_dma;
2052
2053         ((uint16_t *)in_buffer)[0] = 0;
2054         ((uint16_t *)in_buffer)[1] = 0;
2055
2056         /* TODO Add error check if space will be sufficient */
2057         gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
2058
2059         /*
2060          * Input Gather List
2061          */
2062         i = 0;
2063
2064         /* Offset control word followed by iv */
2065
2066         if (flags == 0x0) {
2067                 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8);
2068                 outputlen = inputlen;
2069                 /* iv offset is 0 */
2070                 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
2071         } else {
2072                 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8);
2073                 outputlen = mac_len;
2074                 /* iv offset is 0 */
2075                 *offset_vaddr = rte_cpu_to_be_64((uint64_t)auth_offset);
2076         }
2077
2078         i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len);
2079
2080         /* IV */
2081         iv_d = (uint8_t *)offset_vaddr + OFF_CTRL_LEN;
2082         memcpy(iv_d, iv_s, iv_len);
2083
2084         /* input data */
2085         size = inputlen - iv_len;
2086         if (size) {
2087                 i = fill_sg_comp_from_iov(gather_comp, i,
2088                                           params->src_iov, 0,
2089                                           &size, NULL, 0);
2090
2091                 if (unlikely(size)) {
2092                         CPT_LOG_DP_ERR("Insufficient buffer space,"
2093                                        " size %d needed", size);
2094                         return;
2095                 }
2096         }
2097         ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
2098         g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
2099
2100         /*
2101          * Output Scatter List
2102          */
2103
2104         i = 0;
2105         scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
2106
2107         if (flags == 0x1) {
2108                 /* IV in SLIST only for F8 */
2109                 iv_len = 0;
2110         }
2111
2112         /* IV */
2113         if (iv_len) {
2114                 i = fill_sg_comp(scatter_comp, i,
2115                                  offset_dma + OFF_CTRL_LEN,
2116                                  iv_len);
2117         }
2118
2119         /* Add output data */
2120         if (req_flags & VALID_MAC_BUF) {
2121                 size = outputlen - iv_len - mac_len;
2122                 if (size) {
2123                         i = fill_sg_comp_from_iov(scatter_comp, i,
2124                                                   params->dst_iov, 0,
2125                                                   &size, NULL, 0);
2126
2127                         if (unlikely(size)) {
2128                                 CPT_LOG_DP_ERR("Insufficient buffer space,"
2129                                                " size %d needed", size);
2130                                 return;
2131                         }
2132                 }
2133
2134                 /* mac data */
2135                 if (mac_len) {
2136                         i = fill_sg_comp_from_buf(scatter_comp, i,
2137                                                   &params->mac_buf);
2138                 }
2139         } else {
2140                 /* Output including mac */
2141                 size = outputlen - iv_len;
2142                 if (size) {
2143                         i = fill_sg_comp_from_iov(scatter_comp, i,
2144                                                   params->dst_iov, 0,
2145                                                   &size, NULL, 0);
2146
2147                         if (unlikely(size)) {
2148                                 CPT_LOG_DP_ERR("Insufficient buffer space,"
2149                                                " size %d needed", size);
2150                                 return;
2151                         }
2152                 }
2153         }
2154         ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
2155         s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
2156
2157         size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
2158
2159         /* This is DPTR len incase of SG mode */
2160         vq_cmd_w0.s.dlen = size;
2161
2162         m_vaddr = (uint8_t *)m_vaddr + size;
2163         m_dma += size;
2164
2165         /* cpt alternate completion address saved earlier */
2166         req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
2167         *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
2168         rptr_dma = c_dma - 8;
2169
2170         req->ist.ei1 = dptr_dma;
2171         req->ist.ei2 = rptr_dma;
2172
2173         /* 16 byte aligned cpt res address */
2174         req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
2175         *req->completion_addr = COMPLETION_CODE_INIT;
2176         req->comp_baddr  = c_dma;
2177
2178         /* Fill microcode part of instruction */
2179         req->ist.ei0 = vq_cmd_w0.u64;
2180
2181         req->op = op;
2182
2183         *prep_req = req;
2184         return;
2185 }
2186
2187 static __rte_always_inline void
2188 cpt_kasumi_dec_prep(uint64_t d_offs,
2189                     uint64_t d_lens,
2190                     fc_params_t *params,
2191                     void *op,
2192                     void **prep_req)
2193 {
2194         uint32_t size;
2195         int32_t inputlen = 0, outputlen;
2196         struct cpt_ctx *cpt_ctx;
2197         uint8_t i = 0, iv_len = 8;
2198         struct cpt_request_info *req;
2199         buf_ptr_t *buf_p;
2200         uint32_t encr_offset;
2201         uint32_t encr_data_len;
2202         int flags;
2203         uint8_t dir = 0;
2204         void *m_vaddr, *c_vaddr;
2205         uint64_t m_dma, c_dma;
2206         uint64_t *offset_vaddr, offset_dma;
2207         vq_cmd_word0_t vq_cmd_w0;
2208         uint8_t *in_buffer;
2209         uint32_t g_size_bytes, s_size_bytes;
2210         uint64_t dptr_dma, rptr_dma;
2211         sg_comp_t *gather_comp;
2212         sg_comp_t *scatter_comp;
2213
2214         buf_p = &params->meta_buf;
2215         m_vaddr = buf_p->vaddr;
2216         m_dma = buf_p->dma_addr;
2217
2218         encr_offset = ENCR_OFFSET(d_offs) / 8;
2219         encr_data_len = ENCR_DLEN(d_lens);
2220
2221         cpt_ctx = params->ctx_buf.vaddr;
2222         flags = cpt_ctx->zsk_flags;
2223         /*
2224          * Save initial space that followed app data for completion code &
2225          * alternate completion code to fall in same cache line as app data
2226          */
2227         m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
2228         m_dma += COMPLETION_CODE_SIZE;
2229         size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
2230                 (uint8_t *)m_vaddr;
2231
2232         c_vaddr = (uint8_t *)m_vaddr + size;
2233         c_dma = m_dma + size;
2234         size += sizeof(cpt_res_s_t);
2235
2236         m_vaddr = (uint8_t *)m_vaddr + size;
2237         m_dma += size;
2238
2239         /* Reserve memory for cpt request info */
2240         req = m_vaddr;
2241
2242         size = sizeof(struct cpt_request_info);
2243         m_vaddr = (uint8_t *)m_vaddr + size;
2244         m_dma += size;
2245
2246         vq_cmd_w0.u64 = 0;
2247         vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE;
2248
2249         /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */
2250         vq_cmd_w0.s.opcode.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) |
2251                           (dir << 4) | (0 << 3) | (flags & 0x7));
2252
2253         /*
2254          * GP op header, lengths are expected in bits.
2255          */
2256         vq_cmd_w0.s.param1 = encr_data_len;
2257
2258         /* consider iv len */
2259         encr_offset += iv_len;
2260
2261         inputlen = iv_len + (RTE_ALIGN(encr_data_len, 8) / 8);
2262         outputlen = inputlen;
2263
2264         /* save space for offset ctrl & iv */
2265         offset_vaddr = m_vaddr;
2266         offset_dma = m_dma;
2267
2268         m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
2269         m_dma += OFF_CTRL_LEN + iv_len;
2270
2271         /* DPTR has SG list */
2272         in_buffer = m_vaddr;
2273         dptr_dma = m_dma;
2274
2275         ((uint16_t *)in_buffer)[0] = 0;
2276         ((uint16_t *)in_buffer)[1] = 0;
2277
2278         /* TODO Add error check if space will be sufficient */
2279         gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
2280
2281         /*
2282          * Input Gather List
2283          */
2284         i = 0;
2285
2286         /* Offset control word followed by iv */
2287         *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
2288
2289         i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len);
2290
2291         /* IV */
2292         memcpy((uint8_t *)offset_vaddr + OFF_CTRL_LEN,
2293                params->iv_buf, iv_len);
2294
2295         /* Add input data */
2296         size = inputlen - iv_len;
2297         if (size) {
2298                 i = fill_sg_comp_from_iov(gather_comp, i,
2299                                           params->src_iov,
2300                                           0, &size, NULL, 0);
2301                 if (unlikely(size)) {
2302                         CPT_LOG_DP_ERR("Insufficient buffer space,"
2303                                        " size %d needed", size);
2304                         return;
2305                 }
2306         }
2307         ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
2308         g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
2309
2310         /*
2311          * Output Scatter List
2312          */
2313
2314         i = 0;
2315         scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
2316
2317         /* IV */
2318         i = fill_sg_comp(scatter_comp, i,
2319                          offset_dma + OFF_CTRL_LEN,
2320                          iv_len);
2321
2322         /* Add output data */
2323         size = outputlen - iv_len;
2324         if (size) {
2325                 i = fill_sg_comp_from_iov(scatter_comp, i,
2326                                           params->dst_iov, 0,
2327                                           &size, NULL, 0);
2328                 if (unlikely(size)) {
2329                         CPT_LOG_DP_ERR("Insufficient buffer space,"
2330                                        " size %d needed", size);
2331                         return;
2332                 }
2333         }
2334         ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
2335         s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
2336
2337         size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
2338
2339         /* This is DPTR len incase of SG mode */
2340         vq_cmd_w0.s.dlen = size;
2341
2342         m_vaddr = (uint8_t *)m_vaddr + size;
2343         m_dma += size;
2344
2345         /* cpt alternate completion address saved earlier */
2346         req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
2347         *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
2348         rptr_dma = c_dma - 8;
2349
2350         req->ist.ei1 = dptr_dma;
2351         req->ist.ei2 = rptr_dma;
2352
2353         /* 16 byte aligned cpt res address */
2354         req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
2355         *req->completion_addr = COMPLETION_CODE_INIT;
2356         req->comp_baddr  = c_dma;
2357
2358         /* Fill microcode part of instruction */
2359         req->ist.ei0 = vq_cmd_w0.u64;
2360
2361         req->op = op;
2362
2363         *prep_req = req;
2364         return;
2365 }
2366
2367 static __rte_always_inline void *
2368 cpt_fc_dec_hmac_prep(uint32_t flags,
2369                      uint64_t d_offs,
2370                      uint64_t d_lens,
2371                      fc_params_t *fc_params,
2372                      void *op)
2373 {
2374         struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr;
2375         uint8_t fc_type;
2376         void *prep_req = NULL;
2377
2378         fc_type = ctx->fc_type;
2379
2380         if (likely(fc_type == FC_GEN)) {
2381                 cpt_dec_hmac_prep(flags, d_offs, d_lens, fc_params, op,
2382                                   &prep_req);
2383         } else if (fc_type == ZUC_SNOW3G) {
2384                 cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens, fc_params, op,
2385                                         &prep_req);
2386         } else if (fc_type == KASUMI) {
2387                 cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, op, &prep_req);
2388         }
2389
2390         /*
2391          * For AUTH_ONLY case,
2392          * MC only supports digest generation and verification
2393          * should be done in software by memcmp()
2394          */
2395
2396         return prep_req;
2397 }
2398
2399 static __rte_always_inline void *__rte_hot
2400 cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
2401                      fc_params_t *fc_params, void *op)
2402 {
2403         struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr;
2404         uint8_t fc_type;
2405         void *prep_req = NULL;
2406
2407         fc_type = ctx->fc_type;
2408
2409         /* Common api for rest of the ops */
2410         if (likely(fc_type == FC_GEN)) {
2411                 cpt_enc_hmac_prep(flags, d_offs, d_lens, fc_params, op,
2412                                   &prep_req);
2413         } else if (fc_type == ZUC_SNOW3G) {
2414                 cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens, fc_params, op,
2415                                         &prep_req);
2416         } else if (fc_type == KASUMI) {
2417                 cpt_kasumi_enc_prep(flags, d_offs, d_lens, fc_params, op,
2418                                     &prep_req);
2419         } else if (fc_type == HASH_HMAC) {
2420                 cpt_digest_gen_prep(flags, d_lens, fc_params, op, &prep_req);
2421         }
2422
2423         return prep_req;
2424 }
2425
2426 static __rte_always_inline int
2427 cpt_fc_auth_set_key(struct cpt_ctx *cpt_ctx, auth_type_t type,
2428                     const uint8_t *key, uint16_t key_len, uint16_t mac_len)
2429 {
2430         mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx;
2431         mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx;
2432         mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx;
2433
2434         if ((type >= ZUC_EIA3) && (type <= KASUMI_F9_ECB)) {
2435                 uint32_t keyx[4];
2436
2437                 if (key_len != 16)
2438                         return -1;
2439                 /* No support for AEAD yet */
2440                 if (cpt_ctx->enc_cipher)
2441                         return -1;
2442                 /* For ZUC/SNOW3G/Kasumi */
2443                 switch (type) {
2444                 case SNOW3G_UIA2:
2445                         cpt_ctx->snow3g = 1;
2446                         gen_key_snow3g(key, keyx);
2447                         memcpy(zs_ctx->ci_key, keyx, key_len);
2448                         cpt_ctx->fc_type = ZUC_SNOW3G;
2449                         cpt_ctx->zsk_flags = 0x1;
2450                         break;
2451                 case ZUC_EIA3:
2452                         cpt_ctx->snow3g = 0;
2453                         memcpy(zs_ctx->ci_key, key, key_len);
2454                         memcpy(zs_ctx->zuc_const, zuc_d, 32);
2455                         cpt_ctx->fc_type = ZUC_SNOW3G;
2456                         cpt_ctx->zsk_flags = 0x1;
2457                         break;
2458                 case KASUMI_F9_ECB:
2459                         /* Kasumi ECB mode */
2460                         cpt_ctx->k_ecb = 1;
2461                         memcpy(k_ctx->ci_key, key, key_len);
2462                         cpt_ctx->fc_type = KASUMI;
2463                         cpt_ctx->zsk_flags = 0x1;
2464                         break;
2465                 case KASUMI_F9_CBC:
2466                         memcpy(k_ctx->ci_key, key, key_len);
2467                         cpt_ctx->fc_type = KASUMI;
2468                         cpt_ctx->zsk_flags = 0x1;
2469                         break;
2470                 default:
2471                         return -1;
2472                 }
2473                 cpt_ctx->mac_len = 4;
2474                 cpt_ctx->hash_type = type;
2475                 return 0;
2476         }
2477
2478         if (!(cpt_ctx->fc_type == FC_GEN && !type)) {
2479                 if (!cpt_ctx->fc_type || !cpt_ctx->enc_cipher)
2480                         cpt_ctx->fc_type = HASH_HMAC;
2481         }
2482
2483         if (cpt_ctx->fc_type == FC_GEN && key_len > 64)
2484                 return -1;
2485
2486         /* For GMAC auth, cipher must be NULL */
2487         if (type == GMAC_TYPE)
2488                 fctx->enc.enc_cipher = 0;
2489
2490         fctx->enc.hash_type = cpt_ctx->hash_type = type;
2491         fctx->enc.mac_len = cpt_ctx->mac_len = mac_len;
2492
2493         if (key_len) {
2494                 cpt_ctx->hmac = 1;
2495                 memset(cpt_ctx->auth_key, 0, sizeof(cpt_ctx->auth_key));
2496                 memcpy(cpt_ctx->auth_key, key, key_len);
2497                 cpt_ctx->auth_key_len = key_len;
2498                 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad));
2499                 memset(fctx->hmac.opad, 0, sizeof(fctx->hmac.opad));
2500
2501                 if (key_len <= 64)
2502                         memcpy(fctx->hmac.opad, key, key_len);
2503                 fctx->enc.auth_input_type = 1;
2504         }
2505         return 0;
2506 }
2507
2508 static __rte_always_inline int
2509 fill_sess_aead(struct rte_crypto_sym_xform *xform,
2510                  struct cpt_sess_misc *sess)
2511 {
2512         struct rte_crypto_aead_xform *aead_form;
2513         cipher_type_t enc_type = 0; /* NULL Cipher type */
2514         auth_type_t auth_type = 0; /* NULL Auth type */
2515         uint32_t cipher_key_len = 0;
2516         uint8_t aes_gcm = 0;
2517         aead_form = &xform->aead;
2518         void *ctx = SESS_PRIV(sess);
2519
2520         if (aead_form->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {
2521                 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT;
2522                 sess->cpt_op |= CPT_OP_AUTH_GENERATE;
2523         } else if (aead_form->op == RTE_CRYPTO_AEAD_OP_DECRYPT) {
2524                 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT;
2525                 sess->cpt_op |= CPT_OP_AUTH_VERIFY;
2526         } else {
2527                 CPT_LOG_DP_ERR("Unknown aead operation\n");
2528                 return -1;
2529         }
2530         switch (aead_form->algo) {
2531         case RTE_CRYPTO_AEAD_AES_GCM:
2532                 enc_type = AES_GCM;
2533                 cipher_key_len = 16;
2534                 aes_gcm = 1;
2535                 break;
2536         case RTE_CRYPTO_AEAD_AES_CCM:
2537                 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u",
2538                                aead_form->algo);
2539                 return -1;
2540         case RTE_CRYPTO_AEAD_CHACHA20_POLY1305:
2541                 enc_type = CHACHA20;
2542                 auth_type = POLY1305;
2543                 cipher_key_len = 32;
2544                 sess->chacha_poly = 1;
2545                 break;
2546         default:
2547                 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified",
2548                                aead_form->algo);
2549                 return -1;
2550         }
2551         if (aead_form->key.length < cipher_key_len) {
2552                 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu",
2553                                (unsigned int long)aead_form->key.length);
2554                 return -1;
2555         }
2556         sess->zsk_flag = 0;
2557         sess->aes_gcm = aes_gcm;
2558         sess->mac_len = aead_form->digest_length;
2559         sess->iv_offset = aead_form->iv.offset;
2560         sess->iv_length = aead_form->iv.length;
2561         sess->aad_length = aead_form->aad_length;
2562
2563         if (unlikely(cpt_fc_ciph_set_key(ctx, enc_type, aead_form->key.data,
2564                         aead_form->key.length, NULL)))
2565                 return -1;
2566
2567         if (unlikely(cpt_fc_auth_set_key(ctx, auth_type, NULL, 0,
2568                         aead_form->digest_length)))
2569                 return -1;
2570
2571         return 0;
2572 }
2573
2574 static __rte_always_inline int
2575 fill_sess_cipher(struct rte_crypto_sym_xform *xform,
2576                  struct cpt_sess_misc *sess)
2577 {
2578         struct rte_crypto_cipher_xform *c_form;
2579         struct cpt_ctx *ctx = SESS_PRIV(sess);
2580         cipher_type_t enc_type = 0; /* NULL Cipher type */
2581         uint32_t cipher_key_len = 0;
2582         uint8_t zsk_flag = 0, aes_ctr = 0, is_null = 0;
2583
2584         c_form = &xform->cipher;
2585
2586         if (c_form->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
2587                 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT;
2588         else if (c_form->op == RTE_CRYPTO_CIPHER_OP_DECRYPT) {
2589                 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT;
2590                 if (xform->next != NULL &&
2591                     xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
2592                         /* Perform decryption followed by auth verify */
2593                         ctx->dec_auth = 1;
2594                 }
2595         } else {
2596                 CPT_LOG_DP_ERR("Unknown cipher operation\n");
2597                 return -1;
2598         }
2599
2600         switch (c_form->algo) {
2601         case RTE_CRYPTO_CIPHER_AES_CBC:
2602                 enc_type = AES_CBC;
2603                 cipher_key_len = 16;
2604                 break;
2605         case RTE_CRYPTO_CIPHER_3DES_CBC:
2606                 enc_type = DES3_CBC;
2607                 cipher_key_len = 24;
2608                 break;
2609         case RTE_CRYPTO_CIPHER_DES_CBC:
2610                 /* DES is implemented using 3DES in hardware */
2611                 enc_type = DES3_CBC;
2612                 cipher_key_len = 8;
2613                 break;
2614         case RTE_CRYPTO_CIPHER_AES_CTR:
2615                 enc_type = AES_CTR;
2616                 cipher_key_len = 16;
2617                 aes_ctr = 1;
2618                 break;
2619         case RTE_CRYPTO_CIPHER_NULL:
2620                 enc_type = 0;
2621                 is_null = 1;
2622                 break;
2623         case RTE_CRYPTO_CIPHER_KASUMI_F8:
2624                 enc_type = KASUMI_F8_ECB;
2625                 cipher_key_len = 16;
2626                 zsk_flag = K_F8;
2627                 break;
2628         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
2629                 enc_type = SNOW3G_UEA2;
2630                 cipher_key_len = 16;
2631                 zsk_flag = ZS_EA;
2632                 break;
2633         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
2634                 enc_type = ZUC_EEA3;
2635                 cipher_key_len = 16;
2636                 zsk_flag = ZS_EA;
2637                 break;
2638         case RTE_CRYPTO_CIPHER_AES_XTS:
2639                 enc_type = AES_XTS;
2640                 cipher_key_len = 16;
2641                 break;
2642         case RTE_CRYPTO_CIPHER_3DES_ECB:
2643                 enc_type = DES3_ECB;
2644                 cipher_key_len = 24;
2645                 break;
2646         case RTE_CRYPTO_CIPHER_AES_ECB:
2647                 enc_type = AES_ECB;
2648                 cipher_key_len = 16;
2649                 break;
2650         case RTE_CRYPTO_CIPHER_3DES_CTR:
2651         case RTE_CRYPTO_CIPHER_AES_F8:
2652         case RTE_CRYPTO_CIPHER_ARC4:
2653                 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u",
2654                                c_form->algo);
2655                 return -1;
2656         default:
2657                 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified",
2658                                c_form->algo);
2659                 return -1;
2660         }
2661
2662         if (c_form->key.length < cipher_key_len) {
2663                 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu",
2664                                (unsigned long) c_form->key.length);
2665                 return -1;
2666         }
2667
2668         sess->zsk_flag = zsk_flag;
2669         sess->aes_gcm = 0;
2670         sess->aes_ctr = aes_ctr;
2671         sess->iv_offset = c_form->iv.offset;
2672         sess->iv_length = c_form->iv.length;
2673         sess->is_null = is_null;
2674
2675         if (unlikely(cpt_fc_ciph_set_key(SESS_PRIV(sess), enc_type,
2676                         c_form->key.data, c_form->key.length, NULL)))
2677                 return -1;
2678
2679         return 0;
2680 }
2681
2682 static __rte_always_inline int
2683 fill_sess_auth(struct rte_crypto_sym_xform *xform,
2684                struct cpt_sess_misc *sess)
2685 {
2686         struct cpt_ctx *ctx = SESS_PRIV(sess);
2687         struct rte_crypto_auth_xform *a_form;
2688         auth_type_t auth_type = 0; /* NULL Auth type */
2689         uint8_t zsk_flag = 0, aes_gcm = 0, is_null = 0;
2690
2691         if (xform->next != NULL &&
2692             xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
2693             xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
2694                 /* Perform auth followed by encryption */
2695                 ctx->auth_enc = 1;
2696         }
2697
2698         a_form = &xform->auth;
2699
2700         if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY)
2701                 sess->cpt_op |= CPT_OP_AUTH_VERIFY;
2702         else if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE)
2703                 sess->cpt_op |= CPT_OP_AUTH_GENERATE;
2704         else {
2705                 CPT_LOG_DP_ERR("Unknown auth operation");
2706                 return -1;
2707         }
2708
2709         switch (a_form->algo) {
2710         case RTE_CRYPTO_AUTH_SHA1_HMAC:
2711                 /* Fall through */
2712         case RTE_CRYPTO_AUTH_SHA1:
2713                 auth_type = SHA1_TYPE;
2714                 break;
2715         case RTE_CRYPTO_AUTH_SHA256_HMAC:
2716         case RTE_CRYPTO_AUTH_SHA256:
2717                 auth_type = SHA2_SHA256;
2718                 break;
2719         case RTE_CRYPTO_AUTH_SHA512_HMAC:
2720         case RTE_CRYPTO_AUTH_SHA512:
2721                 auth_type = SHA2_SHA512;
2722                 break;
2723         case RTE_CRYPTO_AUTH_AES_GMAC:
2724                 auth_type = GMAC_TYPE;
2725                 aes_gcm = 1;
2726                 break;
2727         case RTE_CRYPTO_AUTH_SHA224_HMAC:
2728         case RTE_CRYPTO_AUTH_SHA224:
2729                 auth_type = SHA2_SHA224;
2730                 break;
2731         case RTE_CRYPTO_AUTH_SHA384_HMAC:
2732         case RTE_CRYPTO_AUTH_SHA384:
2733                 auth_type = SHA2_SHA384;
2734                 break;
2735         case RTE_CRYPTO_AUTH_MD5_HMAC:
2736         case RTE_CRYPTO_AUTH_MD5:
2737                 auth_type = MD5_TYPE;
2738                 break;
2739         case RTE_CRYPTO_AUTH_KASUMI_F9:
2740                 auth_type = KASUMI_F9_ECB;
2741                 /*
2742                  * Indicate that direction needs to be taken out
2743                  * from end of src
2744                  */
2745                 zsk_flag = K_F9;
2746                 break;
2747         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2748                 auth_type = SNOW3G_UIA2;
2749                 zsk_flag = ZS_IA;
2750                 break;
2751         case RTE_CRYPTO_AUTH_ZUC_EIA3:
2752                 auth_type = ZUC_EIA3;
2753                 zsk_flag = ZS_IA;
2754                 break;
2755         case RTE_CRYPTO_AUTH_NULL:
2756                 auth_type = 0;
2757                 is_null = 1;
2758                 break;
2759         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2760         case RTE_CRYPTO_AUTH_AES_CMAC:
2761         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2762                 CPT_LOG_DP_ERR("Crypto: Unsupported hash algo %u",
2763                                a_form->algo);
2764                 return -1;
2765         default:
2766                 CPT_LOG_DP_ERR("Crypto: Undefined Hash algo %u specified",
2767                                a_form->algo);
2768                 return -1;
2769         }
2770
2771         sess->zsk_flag = zsk_flag;
2772         sess->aes_gcm = aes_gcm;
2773         sess->mac_len = a_form->digest_length;
2774         sess->is_null = is_null;
2775         if (zsk_flag) {
2776                 sess->auth_iv_offset = a_form->iv.offset;
2777                 sess->auth_iv_length = a_form->iv.length;
2778         }
2779         if (unlikely(cpt_fc_auth_set_key(SESS_PRIV(sess), auth_type,
2780                         a_form->key.data, a_form->key.length,
2781                         a_form->digest_length)))
2782                 return -1;
2783
2784         return 0;
2785 }
2786
2787 static __rte_always_inline int
2788 fill_sess_gmac(struct rte_crypto_sym_xform *xform,
2789                  struct cpt_sess_misc *sess)
2790 {
2791         struct rte_crypto_auth_xform *a_form;
2792         cipher_type_t enc_type = 0; /* NULL Cipher type */
2793         auth_type_t auth_type = 0; /* NULL Auth type */
2794         void *ctx = SESS_PRIV(sess);
2795
2796         a_form = &xform->auth;
2797
2798         if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE)
2799                 sess->cpt_op |= CPT_OP_ENCODE;
2800         else if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY)
2801                 sess->cpt_op |= CPT_OP_DECODE;
2802         else {
2803                 CPT_LOG_DP_ERR("Unknown auth operation");
2804                 return -1;
2805         }
2806
2807         switch (a_form->algo) {
2808         case RTE_CRYPTO_AUTH_AES_GMAC:
2809                 enc_type = AES_GCM;
2810                 auth_type = GMAC_TYPE;
2811                 break;
2812         default:
2813                 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified",
2814                                a_form->algo);
2815                 return -1;
2816         }
2817
2818         sess->zsk_flag = 0;
2819         sess->aes_gcm = 0;
2820         sess->is_gmac = 1;
2821         sess->iv_offset = a_form->iv.offset;
2822         sess->iv_length = a_form->iv.length;
2823         sess->mac_len = a_form->digest_length;
2824
2825         if (unlikely(cpt_fc_ciph_set_key(ctx, enc_type, a_form->key.data,
2826                         a_form->key.length, NULL)))
2827                 return -1;
2828
2829         if (unlikely(cpt_fc_auth_set_key(ctx, auth_type, NULL, 0,
2830                         a_form->digest_length)))
2831                 return -1;
2832
2833         return 0;
2834 }
2835
2836 static __rte_always_inline void *
2837 alloc_op_meta(struct rte_mbuf *m_src,
2838               buf_ptr_t *buf,
2839               int32_t len,
2840               struct rte_mempool *cpt_meta_pool)
2841 {
2842         uint8_t *mdata;
2843
2844 #ifndef CPT_ALWAYS_USE_SEPARATE_BUF
2845         if (likely(m_src && (m_src->nb_segs == 1))) {
2846                 int32_t tailroom;
2847                 phys_addr_t mphys;
2848
2849                 /* Check if tailroom is sufficient to hold meta data */
2850                 tailroom = rte_pktmbuf_tailroom(m_src);
2851                 if (likely(tailroom > len + 8)) {
2852                         mdata = (uint8_t *)m_src->buf_addr + m_src->buf_len;
2853                         mphys = m_src->buf_iova + m_src->buf_len;
2854                         mdata -= len;
2855                         mphys -= len;
2856                         buf->vaddr = mdata;
2857                         buf->dma_addr = mphys;
2858                         buf->size = len;
2859                         /* Indicate that this is a mbuf allocated mdata */
2860                         mdata = (uint8_t *)((uint64_t)mdata | 1ull);
2861                         return mdata;
2862                 }
2863         }
2864 #else
2865         RTE_SET_USED(m_src);
2866 #endif
2867
2868         if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0))
2869                 return NULL;
2870
2871         buf->vaddr = mdata;
2872         buf->dma_addr = rte_mempool_virt2iova(mdata);
2873         buf->size = len;
2874
2875         return mdata;
2876 }
2877
2878 /**
2879  * cpt_free_metabuf - free metabuf to mempool.
2880  * @param instance: pointer to instance.
2881  * @param objp: pointer to the metabuf.
2882  */
2883 static __rte_always_inline void
2884 free_op_meta(void *mdata, struct rte_mempool *cpt_meta_pool)
2885 {
2886         bool nofree = ((uintptr_t)mdata & 1ull);
2887
2888         if (likely(nofree))
2889                 return;
2890         rte_mempool_put(cpt_meta_pool, mdata);
2891 }
2892
2893 static __rte_always_inline uint32_t
2894 prepare_iov_from_pkt(struct rte_mbuf *pkt,
2895                      iov_ptr_t *iovec, uint32_t start_offset)
2896 {
2897         uint16_t index = 0;
2898         void *seg_data = NULL;
2899         phys_addr_t seg_phys;
2900         int32_t seg_size = 0;
2901
2902         if (!pkt) {
2903                 iovec->buf_cnt = 0;
2904                 return 0;
2905         }
2906
2907         if (!start_offset) {
2908                 seg_data = rte_pktmbuf_mtod(pkt, void *);
2909                 seg_phys = rte_pktmbuf_iova(pkt);
2910                 seg_size = pkt->data_len;
2911         } else {
2912                 while (start_offset >= pkt->data_len) {
2913                         start_offset -= pkt->data_len;
2914                         pkt = pkt->next;
2915                 }
2916
2917                 seg_data = rte_pktmbuf_mtod_offset(pkt, void *, start_offset);
2918                 seg_phys = rte_pktmbuf_iova_offset(pkt, start_offset);
2919                 seg_size = pkt->data_len - start_offset;
2920                 if (!seg_size)
2921                         return 1;
2922         }
2923
2924         /* first seg */
2925         iovec->bufs[index].vaddr = seg_data;
2926         iovec->bufs[index].dma_addr = seg_phys;
2927         iovec->bufs[index].size = seg_size;
2928         index++;
2929         pkt = pkt->next;
2930
2931         while (unlikely(pkt != NULL)) {
2932                 seg_data = rte_pktmbuf_mtod(pkt, void *);
2933                 seg_phys = rte_pktmbuf_iova(pkt);
2934                 seg_size = pkt->data_len;
2935                 if (!seg_size)
2936                         break;
2937
2938                 iovec->bufs[index].vaddr = seg_data;
2939                 iovec->bufs[index].dma_addr = seg_phys;
2940                 iovec->bufs[index].size = seg_size;
2941
2942                 index++;
2943
2944                 pkt = pkt->next;
2945         }
2946
2947         iovec->buf_cnt = index;
2948         return 0;
2949 }
2950
2951 static __rte_always_inline uint32_t
2952 prepare_iov_from_pkt_inplace(struct rte_mbuf *pkt,
2953                              fc_params_t *param,
2954                              uint32_t *flags)
2955 {
2956         uint16_t index = 0;
2957         void *seg_data = NULL;
2958         phys_addr_t seg_phys;
2959         uint32_t seg_size = 0;
2960         iov_ptr_t *iovec;
2961
2962         seg_data = rte_pktmbuf_mtod(pkt, void *);
2963         seg_phys = rte_pktmbuf_iova(pkt);
2964         seg_size = pkt->data_len;
2965
2966         /* first seg */
2967         if (likely(!pkt->next)) {
2968                 uint32_t headroom, tailroom;
2969
2970                 *flags |= SINGLE_BUF_INPLACE;
2971                 headroom = rte_pktmbuf_headroom(pkt);
2972                 tailroom = rte_pktmbuf_tailroom(pkt);
2973                 if (likely((headroom >= 24) &&
2974                     (tailroom >= 8))) {
2975                         /* In 83XX this is prerequivisit for Direct mode */
2976                         *flags |= SINGLE_BUF_HEADTAILROOM;
2977                 }
2978                 param->bufs[0].vaddr = seg_data;
2979                 param->bufs[0].dma_addr = seg_phys;
2980                 param->bufs[0].size = seg_size;
2981                 return 0;
2982         }
2983         iovec = param->src_iov;
2984         iovec->bufs[index].vaddr = seg_data;
2985         iovec->bufs[index].dma_addr = seg_phys;
2986         iovec->bufs[index].size = seg_size;
2987         index++;
2988         pkt = pkt->next;
2989
2990         while (unlikely(pkt != NULL)) {
2991                 seg_data = rte_pktmbuf_mtod(pkt, void *);
2992                 seg_phys = rte_pktmbuf_iova(pkt);
2993                 seg_size = pkt->data_len;
2994
2995                 if (!seg_size)
2996                         break;
2997
2998                 iovec->bufs[index].vaddr = seg_data;
2999                 iovec->bufs[index].dma_addr = seg_phys;
3000                 iovec->bufs[index].size = seg_size;
3001
3002                 index++;
3003
3004                 pkt = pkt->next;
3005         }
3006
3007         iovec->buf_cnt = index;
3008         return 0;
3009 }
3010
3011 static __rte_always_inline int
3012 fill_fc_params(struct rte_crypto_op *cop,
3013                struct cpt_sess_misc *sess_misc,
3014                struct cpt_qp_meta_info *m_info,
3015                void **mdata_ptr,
3016                void **prep_req)
3017 {
3018         uint32_t space = 0;
3019         struct rte_crypto_sym_op *sym_op = cop->sym;
3020         struct cpt_ctx *ctx = SESS_PRIV(sess_misc);
3021         void *mdata = NULL;
3022         uintptr_t *op;
3023         uint32_t mc_hash_off;
3024         uint32_t flags = 0;
3025         uint64_t d_offs, d_lens;
3026         struct rte_mbuf *m_src, *m_dst;
3027         uint8_t cpt_op = sess_misc->cpt_op;
3028 #ifdef CPT_ALWAYS_USE_SG_MODE
3029         uint8_t inplace = 0;
3030 #else
3031         uint8_t inplace = 1;
3032 #endif
3033         fc_params_t fc_params;
3034         char src[SRC_IOV_SIZE];
3035         char dst[SRC_IOV_SIZE];
3036         uint32_t iv_buf[4];
3037         int ret;
3038
3039         if (likely(sess_misc->iv_length)) {
3040                 flags |= VALID_IV_BUF;
3041                 fc_params.iv_buf = rte_crypto_op_ctod_offset(cop,
3042                                    uint8_t *, sess_misc->iv_offset);
3043                 if (sess_misc->aes_ctr &&
3044                     unlikely(sess_misc->iv_length != 16)) {
3045                         memcpy((uint8_t *)iv_buf,
3046                                 rte_crypto_op_ctod_offset(cop,
3047                                 uint8_t *, sess_misc->iv_offset), 12);
3048                         iv_buf[3] = rte_cpu_to_be_32(0x1);
3049                         fc_params.iv_buf = iv_buf;
3050                 }
3051         }
3052
3053         if (sess_misc->zsk_flag) {
3054                 fc_params.auth_iv_buf = rte_crypto_op_ctod_offset(cop,
3055                                         uint8_t *,
3056                                         sess_misc->auth_iv_offset);
3057                 if (sess_misc->zsk_flag != ZS_EA)
3058                         inplace = 0;
3059         }
3060         m_src = sym_op->m_src;
3061         m_dst = sym_op->m_dst;
3062
3063         if (sess_misc->aes_gcm || sess_misc->chacha_poly) {
3064                 uint8_t *salt;
3065                 uint8_t *aad_data;
3066                 uint16_t aad_len;
3067
3068                 d_offs = sym_op->aead.data.offset;
3069                 d_lens = sym_op->aead.data.length;
3070                 mc_hash_off = sym_op->aead.data.offset +
3071                               sym_op->aead.data.length;
3072
3073                 aad_data = sym_op->aead.aad.data;
3074                 aad_len = sess_misc->aad_length;
3075                 if (likely((aad_data + aad_len) ==
3076                            rte_pktmbuf_mtod_offset(m_src,
3077                                 uint8_t *,
3078                                 sym_op->aead.data.offset))) {
3079                         d_offs = (d_offs - aad_len) | (d_offs << 16);
3080                         d_lens = (d_lens + aad_len) | (d_lens << 32);
3081                 } else {
3082                         fc_params.aad_buf.vaddr = sym_op->aead.aad.data;
3083                         fc_params.aad_buf.dma_addr = sym_op->aead.aad.phys_addr;
3084                         fc_params.aad_buf.size = aad_len;
3085                         flags |= VALID_AAD_BUF;
3086                         inplace = 0;
3087                         d_offs = d_offs << 16;
3088                         d_lens = d_lens << 32;
3089                 }
3090
3091                 salt = fc_params.iv_buf;
3092                 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) {
3093                         cpt_fc_salt_update(SESS_PRIV(sess_misc), salt);
3094                         sess_misc->salt = *(uint32_t *)salt;
3095                 }
3096                 fc_params.iv_buf = salt + 4;
3097                 if (likely(sess_misc->mac_len)) {
3098                         struct rte_mbuf *m = (cpt_op & CPT_OP_ENCODE) ? m_dst :
3099                                              m_src;
3100
3101                         if (!m)
3102                                 m = m_src;
3103
3104                         /* hmac immediately following data is best case */
3105                         if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) +
3106                             mc_hash_off !=
3107                             (uint8_t *)sym_op->aead.digest.data)) {
3108                                 flags |= VALID_MAC_BUF;
3109                                 fc_params.mac_buf.size = sess_misc->mac_len;
3110                                 fc_params.mac_buf.vaddr =
3111                                   sym_op->aead.digest.data;
3112                                 fc_params.mac_buf.dma_addr =
3113                                  sym_op->aead.digest.phys_addr;
3114                                 inplace = 0;
3115                         }
3116                 }
3117         } else {
3118                 d_offs = sym_op->cipher.data.offset;
3119                 d_lens = sym_op->cipher.data.length;
3120                 mc_hash_off = sym_op->cipher.data.offset +
3121                               sym_op->cipher.data.length;
3122                 d_offs = (d_offs << 16) | sym_op->auth.data.offset;
3123                 d_lens = (d_lens << 32) | sym_op->auth.data.length;
3124
3125                 if (mc_hash_off < (sym_op->auth.data.offset +
3126                                    sym_op->auth.data.length)){
3127                         mc_hash_off = (sym_op->auth.data.offset +
3128                                        sym_op->auth.data.length);
3129                 }
3130                 /* for gmac, salt should be updated like in gcm */
3131                 if (unlikely(sess_misc->is_gmac)) {
3132                         uint8_t *salt;
3133                         salt = fc_params.iv_buf;
3134                         if (unlikely(*(uint32_t *)salt != sess_misc->salt)) {
3135                                 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt);
3136                                 sess_misc->salt = *(uint32_t *)salt;
3137                         }
3138                         fc_params.iv_buf = salt + 4;
3139                 }
3140                 if (likely(sess_misc->mac_len)) {
3141                         struct rte_mbuf *m;
3142
3143                         m = (cpt_op & CPT_OP_ENCODE) ? m_dst : m_src;
3144                         if (!m)
3145                                 m = m_src;
3146
3147                         /* hmac immediately following data is best case */
3148                         if (!ctx->dec_auth && !ctx->auth_enc &&
3149                                  (unlikely(rte_pktmbuf_mtod(m, uint8_t *) +
3150                             mc_hash_off !=
3151                              (uint8_t *)sym_op->auth.digest.data))) {
3152                                 flags |= VALID_MAC_BUF;
3153                                 fc_params.mac_buf.size =
3154                                         sess_misc->mac_len;
3155                                 fc_params.mac_buf.vaddr =
3156                                         sym_op->auth.digest.data;
3157                                 fc_params.mac_buf.dma_addr =
3158                                 sym_op->auth.digest.phys_addr;
3159                                 inplace = 0;
3160                         }
3161                 }
3162         }
3163         fc_params.ctx_buf.vaddr = SESS_PRIV(sess_misc);
3164         fc_params.ctx_buf.dma_addr = sess_misc->ctx_dma_addr;
3165
3166         if (!ctx->dec_auth &&
3167                   unlikely(sess_misc->is_null ||
3168                   sess_misc->cpt_op == CPT_OP_DECODE))
3169                 inplace = 0;
3170
3171         if (likely(!m_dst && inplace)) {
3172                 /* Case of single buffer without AAD buf or
3173                  * separate mac buf in place and
3174                  * not air crypto
3175                  */
3176                 fc_params.dst_iov = fc_params.src_iov = (void *)src;
3177
3178                 if (unlikely(prepare_iov_from_pkt_inplace(m_src,
3179                                                           &fc_params,
3180                                                           &flags))) {
3181                         CPT_LOG_DP_ERR("Prepare inplace src iov failed");
3182                         ret = -EINVAL;
3183                         goto err_exit;
3184                 }
3185
3186         } else {
3187                 /* Out of place processing */
3188                 fc_params.src_iov = (void *)src;
3189                 fc_params.dst_iov = (void *)dst;
3190
3191                 /* Store SG I/O in the api for reuse */
3192                 if (prepare_iov_from_pkt(m_src, fc_params.src_iov, 0)) {
3193                         CPT_LOG_DP_ERR("Prepare src iov failed");
3194                         ret = -EINVAL;
3195                         goto err_exit;
3196                 }
3197
3198                 if (unlikely(m_dst != NULL)) {
3199                         uint32_t pkt_len;
3200
3201                         /* Try to make room as much as src has */
3202                         pkt_len = rte_pktmbuf_pkt_len(m_dst);
3203
3204                         if (unlikely(pkt_len < rte_pktmbuf_pkt_len(m_src))) {
3205                                 pkt_len = rte_pktmbuf_pkt_len(m_src) - pkt_len;
3206                                 if (!rte_pktmbuf_append(m_dst, pkt_len)) {
3207                                         CPT_LOG_DP_ERR("Not enough space in "
3208                                                        "m_dst %p, need %u"
3209                                                        " more",
3210                                                        m_dst, pkt_len);
3211                                         ret = -EINVAL;
3212                                         goto err_exit;
3213                                 }
3214                         }
3215
3216                         if (prepare_iov_from_pkt(m_dst, fc_params.dst_iov, 0)) {
3217                                 CPT_LOG_DP_ERR("Prepare dst iov failed for "
3218                                                "m_dst %p", m_dst);
3219                                 ret = -EINVAL;
3220                                 goto err_exit;
3221                         }
3222                 } else {
3223                         fc_params.dst_iov = (void *)src;
3224                 }
3225         }
3226
3227         if (likely(flags & SINGLE_BUF_HEADTAILROOM))
3228                 mdata = alloc_op_meta(m_src, &fc_params.meta_buf,
3229                                       m_info->lb_mlen, m_info->pool);
3230         else
3231                 mdata = alloc_op_meta(NULL, &fc_params.meta_buf,
3232                                       m_info->sg_mlen, m_info->pool);
3233
3234         if (unlikely(mdata == NULL)) {
3235                 CPT_LOG_DP_ERR("Error allocating meta buffer for request");
3236                 ret = -ENOMEM;
3237                 goto err_exit;
3238         }
3239
3240         op = (uintptr_t *)((uintptr_t)mdata & (uintptr_t)~1ull);
3241         op[0] = (uintptr_t)mdata;
3242         op[1] = (uintptr_t)cop;
3243         op[2] = op[3] = 0; /* Used to indicate auth verify */
3244         space += 4 * sizeof(uint64_t);
3245
3246         fc_params.meta_buf.vaddr = (uint8_t *)op + space;
3247         fc_params.meta_buf.dma_addr += space;
3248         fc_params.meta_buf.size -= space;
3249
3250         /* Finally prepare the instruction */
3251         if (cpt_op & CPT_OP_ENCODE)
3252                 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens,
3253                                                  &fc_params, op);
3254         else
3255                 *prep_req = cpt_fc_dec_hmac_prep(flags, d_offs, d_lens,
3256                                                  &fc_params, op);
3257
3258         if (unlikely(*prep_req == NULL)) {
3259                 CPT_LOG_DP_ERR("Preparing request failed due to bad input arg");
3260                 ret = -EINVAL;
3261                 goto free_mdata_and_exit;
3262         }
3263
3264         *mdata_ptr = mdata;
3265
3266         return 0;
3267
3268 free_mdata_and_exit:
3269         free_op_meta(mdata, m_info->pool);
3270 err_exit:
3271         return ret;
3272 }
3273
3274 static __rte_always_inline void
3275 compl_auth_verify(struct rte_crypto_op *op,
3276                       uint8_t *gen_mac,
3277                       uint64_t mac_len)
3278 {
3279         uint8_t *mac;
3280         struct rte_crypto_sym_op *sym_op = op->sym;
3281
3282         if (sym_op->auth.digest.data)
3283                 mac = sym_op->auth.digest.data;
3284         else
3285                 mac = rte_pktmbuf_mtod_offset(sym_op->m_src,
3286                                               uint8_t *,
3287                                               sym_op->auth.data.length +
3288                                               sym_op->auth.data.offset);
3289         if (!mac) {
3290                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
3291                 return;
3292         }
3293
3294         if (memcmp(mac, gen_mac, mac_len))
3295                 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
3296         else
3297                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
3298 }
3299
3300 static __rte_always_inline void
3301 find_kasumif9_direction_and_length(uint8_t *src,
3302                                    uint32_t counter_num_bytes,
3303                                    uint32_t *addr_length_in_bits,
3304                                    uint8_t *addr_direction)
3305 {
3306         uint8_t found = 0;
3307         uint32_t pos;
3308         uint8_t last_byte;
3309         while (!found && counter_num_bytes > 0) {
3310                 counter_num_bytes--;
3311                 if (src[counter_num_bytes] == 0x00)
3312                         continue;
3313                 pos = rte_bsf32(src[counter_num_bytes]);
3314                 if (pos == 7) {
3315                         if (likely(counter_num_bytes > 0)) {
3316                                 last_byte = src[counter_num_bytes - 1];
3317                                 *addr_direction  =  last_byte & 0x1;
3318                                 *addr_length_in_bits = counter_num_bytes * 8
3319                                                         - 1;
3320                         }
3321                 } else {
3322                         last_byte = src[counter_num_bytes];
3323                         *addr_direction = (last_byte >> (pos + 1)) & 0x1;
3324                         *addr_length_in_bits = counter_num_bytes * 8
3325                                                 + (8 - (pos + 2));
3326                 }
3327                 found = 1;
3328         }
3329 }
3330
3331 /*
3332  * This handles all auth only except AES_GMAC
3333  */
3334 static __rte_always_inline int
3335 fill_digest_params(struct rte_crypto_op *cop,
3336                    struct cpt_sess_misc *sess,
3337                    struct cpt_qp_meta_info *m_info,
3338                    void **mdata_ptr,
3339                    void **prep_req)
3340 {
3341         uint32_t space = 0;
3342         struct rte_crypto_sym_op *sym_op = cop->sym;
3343         void *mdata;
3344         phys_addr_t mphys;
3345         uint64_t *op;
3346         uint32_t auth_range_off;
3347         uint32_t flags = 0;
3348         uint64_t d_offs = 0, d_lens;
3349         struct rte_mbuf *m_src, *m_dst;
3350         uint16_t auth_op = sess->cpt_op & CPT_OP_AUTH_MASK;
3351         uint16_t mac_len = sess->mac_len;
3352         fc_params_t params;
3353         char src[SRC_IOV_SIZE];
3354         uint8_t iv_buf[16];
3355         int ret;
3356
3357         memset(&params, 0, sizeof(fc_params_t));
3358
3359         m_src = sym_op->m_src;
3360
3361         /* For just digest lets force mempool alloc */
3362         mdata = alloc_op_meta(NULL, &params.meta_buf, m_info->sg_mlen,
3363                               m_info->pool);
3364         if (mdata == NULL) {
3365                 ret = -ENOMEM;
3366                 goto err_exit;
3367         }
3368
3369         mphys = params.meta_buf.dma_addr;
3370
3371         op = mdata;
3372         op[0] = (uintptr_t)mdata;
3373         op[1] = (uintptr_t)cop;
3374         op[2] = op[3] = 0; /* Used to indicate auth verify */
3375         space += 4 * sizeof(uint64_t);
3376
3377         auth_range_off = sym_op->auth.data.offset;
3378
3379         flags = VALID_MAC_BUF;
3380         params.src_iov = (void *)src;
3381         if (unlikely(sess->zsk_flag)) {
3382                 /*
3383                  * Since for Zuc, Kasumi, Snow3g offsets are in bits
3384                  * we will send pass through even for auth only case,
3385                  * let MC handle it
3386                  */
3387                 d_offs = auth_range_off;
3388                 auth_range_off = 0;
3389                 params.auth_iv_buf = rte_crypto_op_ctod_offset(cop,
3390                                         uint8_t *, sess->auth_iv_offset);
3391                 if (sess->zsk_flag == K_F9) {
3392                         uint32_t length_in_bits, num_bytes;
3393                         uint8_t *src, direction = 0;
3394
3395                         memcpy(iv_buf, rte_pktmbuf_mtod(cop->sym->m_src,
3396                                                         uint8_t *), 8);
3397                         /*
3398                          * This is kasumi f9, take direction from
3399                          * source buffer
3400                          */
3401                         length_in_bits = cop->sym->auth.data.length;
3402                         num_bytes = (length_in_bits >> 3);
3403                         src = rte_pktmbuf_mtod(cop->sym->m_src, uint8_t *);
3404                         find_kasumif9_direction_and_length(src,
3405                                                 num_bytes,
3406                                                 &length_in_bits,
3407                                                 &direction);
3408                         length_in_bits -= 64;
3409                         cop->sym->auth.data.offset += 64;
3410                         d_offs = cop->sym->auth.data.offset;
3411                         auth_range_off = d_offs / 8;
3412                         cop->sym->auth.data.length = length_in_bits;
3413
3414                         /* Store it at end of auth iv */
3415                         iv_buf[8] = direction;
3416                         params.auth_iv_buf = iv_buf;
3417                 }
3418         }
3419
3420         d_lens = sym_op->auth.data.length;
3421
3422         params.ctx_buf.vaddr = SESS_PRIV(sess);
3423         params.ctx_buf.dma_addr = sess->ctx_dma_addr;
3424
3425         if (auth_op == CPT_OP_AUTH_GENERATE) {
3426                 if (sym_op->auth.digest.data) {
3427                         /*
3428                          * Digest to be generated
3429                          * in separate buffer
3430                          */
3431                         params.mac_buf.size =
3432                                 sess->mac_len;
3433                         params.mac_buf.vaddr =
3434                                 sym_op->auth.digest.data;
3435                         params.mac_buf.dma_addr =
3436                                 sym_op->auth.digest.phys_addr;
3437                 } else {
3438                         uint32_t off = sym_op->auth.data.offset +
3439                                 sym_op->auth.data.length;
3440                         int32_t dlen, space;
3441
3442                         m_dst = sym_op->m_dst ?
3443                                 sym_op->m_dst : sym_op->m_src;
3444                         dlen = rte_pktmbuf_pkt_len(m_dst);
3445
3446                         space = off + mac_len - dlen;
3447                         if (space > 0)
3448                                 if (!rte_pktmbuf_append(m_dst, space)) {
3449                                         CPT_LOG_DP_ERR("Failed to extend "
3450                                                        "mbuf by %uB", space);
3451                                         ret = -EINVAL;
3452                                         goto free_mdata_and_exit;
3453                                 }
3454
3455                         params.mac_buf.vaddr =
3456                                 rte_pktmbuf_mtod_offset(m_dst, void *, off);
3457                         params.mac_buf.dma_addr =
3458                                 rte_pktmbuf_iova_offset(m_dst, off);
3459                         params.mac_buf.size = mac_len;
3460                 }
3461         } else {
3462                 /* Need space for storing generated mac */
3463                 params.mac_buf.vaddr = (uint8_t *)mdata + space;
3464                 params.mac_buf.dma_addr = mphys + space;
3465                 params.mac_buf.size = mac_len;
3466                 space += RTE_ALIGN_CEIL(mac_len, 8);
3467                 op[2] = (uintptr_t)params.mac_buf.vaddr;
3468                 op[3] = mac_len;
3469         }
3470
3471         params.meta_buf.vaddr = (uint8_t *)mdata + space;
3472         params.meta_buf.dma_addr = mphys + space;
3473         params.meta_buf.size -= space;
3474
3475         /* Out of place processing */
3476         params.src_iov = (void *)src;
3477
3478         /*Store SG I/O in the api for reuse */
3479         if (prepare_iov_from_pkt(m_src, params.src_iov, auth_range_off)) {
3480                 CPT_LOG_DP_ERR("Prepare src iov failed");
3481                 ret = -EINVAL;
3482                 goto free_mdata_and_exit;
3483         }
3484
3485         *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, &params, op);
3486         if (unlikely(*prep_req == NULL)) {
3487                 ret = -EINVAL;
3488                 goto free_mdata_and_exit;
3489         }
3490
3491         *mdata_ptr = mdata;
3492
3493         return 0;
3494
3495 free_mdata_and_exit:
3496         free_op_meta(mdata, m_info->pool);
3497 err_exit:
3498         return ret;
3499 }
3500
3501 #endif /*_CPT_UCODE_H_ */