drivers/common/cpt/cpt_ucode.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Cavium, Inc
   3  */
   4
   5 #ifndef _CPT_UCODE_H_
   6 #define _CPT_UCODE_H_
   7 #include <stdbool.h>
   8
   9 #include "cpt_common.h"
  10 #include "cpt_hw_types.h"
  11 #include "cpt_mcode_defines.h"
  12
  13 /*
  14  * This file defines functions that are interfaces to microcode spec.
  15  *
  16  */
  17
  18 static uint8_t zuc_d[32] = {
  19         0x44, 0xD7, 0x26, 0xBC, 0x62, 0x6B, 0x13, 0x5E,
  20         0x57, 0x89, 0x35, 0xE2, 0x71, 0x35, 0x09, 0xAF,
  21         0x4D, 0x78, 0x2F, 0x13, 0x6B, 0xC4, 0x1A, 0xF1,
  22         0x5E, 0x26, 0x3C, 0x4D, 0x78, 0x9A, 0x47, 0xAC
  23 };
  24
  25 static __rte_always_inline void
  26 gen_key_snow3g(const uint8_t *ck, uint32_t *keyx)
  27 {
  28         int i, base;
  29
  30         for (i = 0; i < 4; i++) {
  31                 base = 4 * i;
  32                 keyx[3 - i] = (ck[base] << 24) | (ck[base + 1] << 16) |
  33                         (ck[base + 2] << 8) | (ck[base + 3]);
  34                 keyx[3 - i] = rte_cpu_to_be_32(keyx[3 - i]);
  35         }
  36 }
  37
  38 static __rte_always_inline int
  39 cpt_mac_len_verify(struct rte_crypto_auth_xform *auth)
  40 {
  41         uint16_t mac_len = auth->digest_length;
  42         int ret;
  43
  44         switch (auth->algo) {
  45         case RTE_CRYPTO_AUTH_MD5:
  46         case RTE_CRYPTO_AUTH_MD5_HMAC:
  47                 ret = (mac_len == 16) ? 0 : -1;
  48                 break;
  49         case RTE_CRYPTO_AUTH_SHA1:
  50         case RTE_CRYPTO_AUTH_SHA1_HMAC:
  51                 ret = (mac_len == 20) ? 0 : -1;
  52                 break;
  53         case RTE_CRYPTO_AUTH_SHA224:
  54         case RTE_CRYPTO_AUTH_SHA224_HMAC:
  55                 ret = (mac_len == 28) ? 0 : -1;
  56                 break;
  57         case RTE_CRYPTO_AUTH_SHA256:
  58         case RTE_CRYPTO_AUTH_SHA256_HMAC:
  59                 ret = (mac_len == 32) ? 0 : -1;
  60                 break;
  61         case RTE_CRYPTO_AUTH_SHA384:
  62         case RTE_CRYPTO_AUTH_SHA384_HMAC:
  63                 ret = (mac_len == 48) ? 0 : -1;
  64                 break;
  65         case RTE_CRYPTO_AUTH_SHA512:
  66         case RTE_CRYPTO_AUTH_SHA512_HMAC:
  67                 ret = (mac_len == 64) ? 0 : -1;
  68                 break;
  69         case RTE_CRYPTO_AUTH_NULL:
  70                 ret = 0;
  71                 break;
  72         default:
  73                 ret = -1;
  74         }
  75
  76         return ret;
  77 }
  78
  79 static __rte_always_inline void
  80 cpt_fc_salt_update(struct cpt_ctx *cpt_ctx,
  81                    uint8_t *salt)
  82 {
  83         mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx;
  84         memcpy(fctx->enc.encr_iv, salt, 4);
  85 }
  86
  87 static __rte_always_inline int
  88 cpt_fc_ciph_validate_key_aes(uint16_t key_len)
  89 {
  90         switch (key_len) {
  91         case 16:
  92         case 24:
  93         case 32:
  94                 return 0;
  95         default:
  96                 return -1;
  97         }
  98 }
  99
 100 static __rte_always_inline int
 101 cpt_fc_ciph_set_type(cipher_type_t type, struct cpt_ctx *ctx, uint16_t key_len)
 102 {
 103         int fc_type = 0;
 104         switch (type) {
 105         case PASSTHROUGH:
 106                 fc_type = FC_GEN;
 107                 break;
 108         case DES3_CBC:
 109         case DES3_ECB:
 110                 fc_type = FC_GEN;
 111                 break;
 112         case AES_CBC:
 113         case AES_ECB:
 114         case AES_CFB:
 115         case AES_CTR:
 116         case AES_GCM:
 117                 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0))
 118                         return -1;
 119                 fc_type = FC_GEN;
 120                 break;
 121         case CHACHA20:
 122                 fc_type = FC_GEN;
 123                 break;
 124         case AES_XTS:
 125                 key_len = key_len / 2;
 126                 if (unlikely(key_len == 24)) {
 127                         CPT_LOG_DP_ERR("Invalid AES key len for XTS");
 128                         return -1;
 129                 }
 130                 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0))
 131                         return -1;
 132                 fc_type = FC_GEN;
 133                 break;
 134         case ZUC_EEA3:
 135         case SNOW3G_UEA2:
 136                 if (unlikely(key_len != 16))
 137                         return -1;
 138                 /* No support for AEAD yet */
 139                 if (unlikely(ctx->hash_type))
 140                         return -1;
 141                 fc_type = ZUC_SNOW3G;
 142                 break;
 143         case KASUMI_F8_CBC:
 144         case KASUMI_F8_ECB:
 145                 if (unlikely(key_len != 16))
 146                         return -1;
 147                 /* No support for AEAD yet */
 148                 if (unlikely(ctx->hash_type))
 149                         return -1;
 150                 fc_type = KASUMI;
 151                 break;
 152         default:
 153                 return -1;
 154         }
 155
 156         ctx->fc_type = fc_type;
 157         return 0;
 158 }
 159
 160 static __rte_always_inline void
 161 cpt_fc_ciph_set_key_passthrough(struct cpt_ctx *cpt_ctx, mc_fc_context_t *fctx)
 162 {
 163         cpt_ctx->enc_cipher = 0;
 164         fctx->enc.enc_cipher = 0;
 165 }
 166
 167 static __rte_always_inline void
 168 cpt_fc_ciph_set_key_set_aes_key_type(mc_fc_context_t *fctx, uint16_t key_len)
 169 {
 170         mc_aes_type_t aes_key_type = 0;
 171         switch (key_len) {
 172         case 16:
 173                 aes_key_type = AES_128_BIT;
 174                 break;
 175         case 24:
 176                 aes_key_type = AES_192_BIT;
 177                 break;
 178         case 32:
 179                 aes_key_type = AES_256_BIT;
 180                 break;
 181         default:
 182                 /* This should not happen */
 183                 CPT_LOG_DP_ERR("Invalid AES key len");
 184                 return;
 185         }
 186         fctx->enc.aes_key = aes_key_type;
 187 }
 188
 189 static __rte_always_inline void
 190 cpt_fc_ciph_set_key_snow3g_uea2(struct cpt_ctx *cpt_ctx, const uint8_t *key,
 191                 uint16_t key_len)
 192 {
 193         mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx;
 194         uint32_t keyx[4];
 195
 196         cpt_ctx->snow3g = 1;
 197         gen_key_snow3g(key, keyx);
 198         memcpy(zs_ctx->ci_key, keyx, key_len);
 199         cpt_ctx->zsk_flags = 0;
 200 }
 201
 202 static __rte_always_inline void
 203 cpt_fc_ciph_set_key_zuc_eea3(struct cpt_ctx *cpt_ctx, const uint8_t *key,
 204                 uint16_t key_len)
 205 {
 206         mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx;
 207
 208         cpt_ctx->snow3g = 0;
 209         memcpy(zs_ctx->ci_key, key, key_len);
 210         memcpy(zs_ctx->zuc_const, zuc_d, 32);
 211         cpt_ctx->zsk_flags = 0;
 212 }
 213
 214 static __rte_always_inline void
 215 cpt_fc_ciph_set_key_kasumi_f8_ecb(struct cpt_ctx *cpt_ctx, const uint8_t *key,
 216                 uint16_t key_len)
 217 {
 218         mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx;
 219
 220         cpt_ctx->k_ecb = 1;
 221         memcpy(k_ctx->ci_key, key, key_len);
 222         cpt_ctx->zsk_flags = 0;
 223 }
 224
 225 static __rte_always_inline void
 226 cpt_fc_ciph_set_key_kasumi_f8_cbc(struct cpt_ctx *cpt_ctx, const uint8_t *key,
 227                 uint16_t key_len)
 228 {
 229         mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx;
 230
 231         memcpy(k_ctx->ci_key, key, key_len);
 232         cpt_ctx->zsk_flags = 0;
 233 }
 234
 235 static __rte_always_inline int
 236 cpt_fc_ciph_set_key(struct cpt_ctx *cpt_ctx, cipher_type_t type,
 237                     const uint8_t *key, uint16_t key_len, uint8_t *salt)
 238 {
 239         mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx;
 240         int ret;
 241
 242         ret = cpt_fc_ciph_set_type(type, cpt_ctx, key_len);
 243         if (unlikely(ret))
 244                 return -1;
 245
 246         if (cpt_ctx->fc_type == FC_GEN) {
 247                 /*
 248                  * We need to always say IV is from DPTR as user can
 249                  * sometimes iverride IV per operation.
 250                  */
 251                 fctx->enc.iv_source = CPT_FROM_DPTR;
 252
 253                 if (cpt_ctx->auth_key_len > 64)
 254                         return -1;
 255         }
 256
 257         switch (type) {
 258         case PASSTHROUGH:
 259                 cpt_fc_ciph_set_key_passthrough(cpt_ctx, fctx);
 260                 goto success;
 261         case DES3_CBC:
 262                 /* CPT performs DES using 3DES with the 8B DES-key
 263                  * replicated 2 more times to match the 24B 3DES-key.
 264                  * Eg. If org. key is "0x0a 0x0b", then new key is
 265                  * "0x0a 0x0b 0x0a 0x0b 0x0a 0x0b"
 266                  */
 267                 if (key_len == 8) {
 268                         /* Skipping the first 8B as it will be copied
 269                          * in the regular code flow
 270                          */
 271                         memcpy(fctx->enc.encr_key+key_len, key, key_len);
 272                         memcpy(fctx->enc.encr_key+2*key_len, key, key_len);
 273                 }
 274                 break;
 275         case DES3_ECB:
 276                 /* For DES3_ECB IV need to be from CTX. */
 277                 fctx->enc.iv_source = CPT_FROM_CTX;
 278                 break;
 279         case AES_CBC:
 280         case AES_ECB:
 281         case AES_CFB:
 282         case AES_CTR:
 283         case CHACHA20:
 284                 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len);
 285                 break;
 286         case AES_GCM:
 287                 /* Even though iv source is from dptr,
 288                  * aes_gcm salt is taken from ctx
 289                  */
 290                 if (salt) {
 291                         memcpy(fctx->enc.encr_iv, salt, 4);
 292                         /* Assuming it was just salt update
 293                          * and nothing else
 294                          */
 295                         if (!key)
 296                                 goto success;
 297                 }
 298                 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len);
 299                 break;
 300         case AES_XTS:
 301                 key_len = key_len / 2;
 302                 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len);
 303
 304                 /* Copy key2 for XTS into ipad */
 305                 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad));
 306                 memcpy(fctx->hmac.ipad, &key[key_len], key_len);
 307                 break;
 308         case SNOW3G_UEA2:
 309                 cpt_fc_ciph_set_key_snow3g_uea2(cpt_ctx, key, key_len);
 310                 goto success;
 311         case ZUC_EEA3:
 312                 cpt_fc_ciph_set_key_zuc_eea3(cpt_ctx, key, key_len);
 313                 goto success;
 314         case KASUMI_F8_ECB:
 315                 cpt_fc_ciph_set_key_kasumi_f8_ecb(cpt_ctx, key, key_len);
 316                 goto success;
 317         case KASUMI_F8_CBC:
 318                 cpt_fc_ciph_set_key_kasumi_f8_cbc(cpt_ctx, key, key_len);
 319                 goto success;
 320         default:
 321                 return -1;
 322         }
 323
 324         /* Only for FC_GEN case */
 325
 326         /* For GMAC auth, cipher must be NULL */
 327         if (cpt_ctx->hash_type != GMAC_TYPE)
 328                 fctx->enc.enc_cipher = type;
 329
 330         memcpy(fctx->enc.encr_key, key, key_len);
 331
 332 success:
 333         cpt_ctx->enc_cipher = type;
 334
 335         return 0;
 336 }
 337
 338 static __rte_always_inline uint32_t
 339 fill_sg_comp(sg_comp_t *list,
 340              uint32_t i,
 341              phys_addr_t dma_addr,
 342              uint32_t size)
 343 {
 344         sg_comp_t *to = &list[i>>2];
 345
 346         to->u.s.len[i%4] = rte_cpu_to_be_16(size);
 347         to->ptr[i%4] = rte_cpu_to_be_64(dma_addr);
 348         i++;
 349         return i;
 350 }
 351
 352 static __rte_always_inline uint32_t
 353 fill_sg_comp_from_buf(sg_comp_t *list,
 354                       uint32_t i,
 355                       buf_ptr_t *from)
 356 {
 357         sg_comp_t *to = &list[i>>2];
 358
 359         to->u.s.len[i%4] = rte_cpu_to_be_16(from->size);
 360         to->ptr[i%4] = rte_cpu_to_be_64(from->dma_addr);
 361         i++;
 362         return i;
 363 }
 364
 365 static __rte_always_inline uint32_t
 366 fill_sg_comp_from_buf_min(sg_comp_t *list,
 367                           uint32_t i,
 368                           buf_ptr_t *from,
 369                           uint32_t *psize)
 370 {
 371         sg_comp_t *to = &list[i >> 2];
 372         uint32_t size = *psize;
 373         uint32_t e_len;
 374
 375         e_len = (size > from->size) ? from->size : size;
 376         to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
 377         to->ptr[i % 4] = rte_cpu_to_be_64(from->dma_addr);
 378         *psize -= e_len;
 379         i++;
 380         return i;
 381 }
 382
 383 /*
 384  * This fills the MC expected SGIO list
 385  * from IOV given by user.
 386  */
 387 static __rte_always_inline uint32_t
 388 fill_sg_comp_from_iov(sg_comp_t *list,
 389                       uint32_t i,
 390                       iov_ptr_t *from, uint32_t from_offset,
 391                       uint32_t *psize, buf_ptr_t *extra_buf,
 392                       uint32_t extra_offset)
 393 {
 394         int32_t j;
 395         uint32_t extra_len = extra_buf ? extra_buf->size : 0;
 396         uint32_t size = *psize;
 397         buf_ptr_t *bufs;
 398
 399         bufs = from->bufs;
 400         for (j = 0; (j < from->buf_cnt) && size; j++) {
 401                 phys_addr_t e_dma_addr;
 402                 uint32_t e_len;
 403                 sg_comp_t *to = &list[i >> 2];
 404
 405                 if (unlikely(from_offset)) {
 406                         if (from_offset >= bufs[j].size) {
 407                                 from_offset -= bufs[j].size;
 408                                 continue;
 409                         }
 410                         e_dma_addr = bufs[j].dma_addr + from_offset;
 411                         e_len = (size > (bufs[j].size - from_offset)) ?
 412                                 (bufs[j].size - from_offset) : size;
 413                         from_offset = 0;
 414                 } else {
 415                         e_dma_addr = bufs[j].dma_addr;
 416                         e_len = (size > bufs[j].size) ?
 417                                 bufs[j].size : size;
 418                 }
 419
 420                 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
 421                 to->ptr[i % 4] = rte_cpu_to_be_64(e_dma_addr);
 422
 423                 if (extra_len && (e_len >= extra_offset)) {
 424                         /* Break the data at given offset */
 425                         uint32_t next_len = e_len - extra_offset;
 426                         phys_addr_t next_dma = e_dma_addr + extra_offset;
 427
 428                         if (!extra_offset) {
 429                                 i--;
 430                         } else {
 431                                 e_len = extra_offset;
 432                                 size -= e_len;
 433                                 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
 434                         }
 435
 436                         extra_len = RTE_MIN(extra_len, size);
 437                         /* Insert extra data ptr */
 438                         if (extra_len) {
 439                                 i++;
 440                                 to = &list[i >> 2];
 441                                 to->u.s.len[i % 4] =
 442                                         rte_cpu_to_be_16(extra_len);
 443                                 to->ptr[i % 4] =
 444                                         rte_cpu_to_be_64(extra_buf->dma_addr);
 445                                 size -= extra_len;
 446                         }
 447
 448                         next_len = RTE_MIN(next_len, size);
 449                         /* insert the rest of the data */
 450                         if (next_len) {
 451                                 i++;
 452                                 to = &list[i >> 2];
 453                                 to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len);
 454                                 to->ptr[i % 4] = rte_cpu_to_be_64(next_dma);
 455                                 size -= next_len;
 456                         }
 457                         extra_len = 0;
 458
 459                 } else {
 460                         size -= e_len;
 461                 }
 462                 if (extra_offset)
 463                         extra_offset -= size;
 464                 i++;
 465         }
 466
 467         *psize = size;
 468         return (uint32_t)i;
 469 }
 470
 471 static __rte_always_inline void
 472 cpt_digest_gen_prep(uint32_t flags,
 473                     uint64_t d_lens,
 474                     digest_params_t *params,
 475                     void *op,
 476                     void **prep_req)
 477 {
 478         struct cpt_request_info *req;
 479         uint32_t size, i;
 480         uint16_t data_len, mac_len, key_len;
 481         auth_type_t hash_type;
 482         buf_ptr_t *meta_p;
 483         struct cpt_ctx *ctx;
 484         sg_comp_t *gather_comp;
 485         sg_comp_t *scatter_comp;
 486         uint8_t *in_buffer;
 487         uint32_t g_size_bytes, s_size_bytes;
 488         uint64_t dptr_dma, rptr_dma;
 489         vq_cmd_word0_t vq_cmd_w0;
 490         void *c_vaddr, *m_vaddr;
 491         uint64_t c_dma, m_dma;
 492
 493         ctx = params->ctx_buf.vaddr;
 494         meta_p = &params->meta_buf;
 495
 496         m_vaddr = meta_p->vaddr;
 497         m_dma = meta_p->dma_addr;
 498
 499         /*
 500          * Save initial space that followed app data for completion code &
 501          * alternate completion code to fall in same cache line as app data
 502          */
 503         m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
 504         m_dma += COMPLETION_CODE_SIZE;
 505         size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
 506                 (uint8_t *)m_vaddr;
 507         c_vaddr = (uint8_t *)m_vaddr + size;
 508         c_dma = m_dma + size;
 509         size += sizeof(cpt_res_s_t);
 510
 511         m_vaddr = (uint8_t *)m_vaddr + size;
 512         m_dma += size;
 513
 514         req = m_vaddr;
 515
 516         size = sizeof(struct cpt_request_info);
 517         m_vaddr = (uint8_t *)m_vaddr + size;
 518         m_dma += size;
 519
 520         hash_type = ctx->hash_type;
 521         mac_len = ctx->mac_len;
 522         key_len = ctx->auth_key_len;
 523         data_len = AUTH_DLEN(d_lens);
 524
 525         /*GP op header */
 526         vq_cmd_w0.s.opcode.minor = 0;
 527         vq_cmd_w0.s.param2 = ((uint16_t)hash_type << 8);
 528         if (ctx->hmac) {
 529                 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE;
 530                 vq_cmd_w0.s.param1 = key_len;
 531                 vq_cmd_w0.s.dlen = data_len + RTE_ALIGN_CEIL(key_len, 8);
 532         } else {
 533                 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE;
 534                 vq_cmd_w0.s.param1 = 0;
 535                 vq_cmd_w0.s.dlen = data_len;
 536         }
 537
 538         /* Null auth only case enters the if */
 539         if (unlikely(!hash_type && !ctx->enc_cipher)) {
 540                 vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_MISC;
 541                 /* Minor op is passthrough */
 542                 vq_cmd_w0.s.opcode.minor = 0x03;
 543                 /* Send out completion code only */
 544                 vq_cmd_w0.s.param2 = 0x1;
 545         }
 546
 547         /* DPTR has SG list */
 548         in_buffer = m_vaddr;
 549         dptr_dma = m_dma;
 550
 551         ((uint16_t *)in_buffer)[0] = 0;
 552         ((uint16_t *)in_buffer)[1] = 0;
 553
 554         /* TODO Add error check if space will be sufficient */
 555         gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
 556
 557         /*
 558          * Input gather list
 559          */
 560
 561         i = 0;
 562
 563         if (ctx->hmac) {
 564                 uint64_t k_dma = params->ctx_buf.dma_addr +
 565                         offsetof(struct cpt_ctx, auth_key);
 566                 /* Key */
 567                 i = fill_sg_comp(gather_comp, i, k_dma,
 568                                  RTE_ALIGN_CEIL(key_len, 8));
 569         }
 570
 571         /* input data */
 572         size = data_len;
 573         if (size) {
 574                 i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov,
 575                                           0, &size, NULL, 0);
 576                 if (unlikely(size)) {
 577                         CPT_LOG_DP_DEBUG("Insufficient dst IOV size, short"
 578                                          " by %dB", size);
 579                         return;
 580                 }
 581         } else {
 582                 /*
 583                  * Looks like we need to support zero data
 584                  * gather ptr in case of hash & hmac
 585                  */
 586                 i++;
 587         }
 588         ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
 589         g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
 590
 591         /*
 592          * Output Gather list
 593          */
 594
 595         i = 0;
 596         scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
 597
 598         if (flags & VALID_MAC_BUF) {
 599                 if (unlikely(params->mac_buf.size < mac_len)) {
 600                         CPT_LOG_DP_ERR("Insufficient MAC size");
 601                         return;
 602                 }
 603
 604                 size = mac_len;
 605                 i = fill_sg_comp_from_buf_min(scatter_comp, i,
 606                                               &params->mac_buf, &size);
 607         } else {
 608                 size = mac_len;
 609                 i = fill_sg_comp_from_iov(scatter_comp, i,
 610                                           params->src_iov, data_len,
 611                                           &size, NULL, 0);
 612                 if (unlikely(size)) {
 613                         CPT_LOG_DP_ERR("Insufficient dst IOV size, short by"
 614                                        " %dB", size);
 615                         return;
 616                 }
 617         }
 618
 619         ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
 620         s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
 621
 622         size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
 623
 624         /* This is DPTR len incase of SG mode */
 625         vq_cmd_w0.s.dlen = size;
 626
 627         m_vaddr = (uint8_t *)m_vaddr + size;
 628         m_dma += size;
 629
 630         /* cpt alternate completion address saved earlier */
 631         req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
 632         *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
 633         rptr_dma = c_dma - 8;
 634
 635         req->ist.ei1 = dptr_dma;
 636         req->ist.ei2 = rptr_dma;
 637
 638         /* 16 byte aligned cpt res address */
 639         req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
 640         *req->completion_addr = COMPLETION_CODE_INIT;
 641         req->comp_baddr  = c_dma;
 642
 643         /* Fill microcode part of instruction */
 644         req->ist.ei0 = vq_cmd_w0.u64;
 645
 646         req->op = op;
 647
 648         *prep_req = req;
 649         return;
 650 }
 651
 652 static __rte_always_inline void
 653 cpt_enc_hmac_prep(uint32_t flags,
 654                   uint64_t d_offs,
 655                   uint64_t d_lens,
 656                   fc_params_t *fc_params,
 657                   void *op,
 658                   void **prep_req)
 659 {
 660         uint32_t iv_offset = 0;
 661         int32_t inputlen, outputlen, enc_dlen, auth_dlen;
 662         struct cpt_ctx *cpt_ctx;
 663         uint32_t cipher_type, hash_type;
 664         uint32_t mac_len, size;
 665         uint8_t iv_len = 16;
 666         struct cpt_request_info *req;
 667         buf_ptr_t *meta_p, *aad_buf = NULL;
 668         uint32_t encr_offset, auth_offset;
 669         uint32_t encr_data_len, auth_data_len, aad_len = 0;
 670         uint32_t passthrough_len = 0;
 671         void *m_vaddr, *offset_vaddr;
 672         uint64_t m_dma, offset_dma;
 673         vq_cmd_word0_t vq_cmd_w0;
 674         void *c_vaddr;
 675         uint64_t c_dma;
 676
 677         meta_p = &fc_params->meta_buf;
 678         m_vaddr = meta_p->vaddr;
 679         m_dma = meta_p->dma_addr;
 680
 681         encr_offset = ENCR_OFFSET(d_offs);
 682         auth_offset = AUTH_OFFSET(d_offs);
 683         encr_data_len = ENCR_DLEN(d_lens);
 684         auth_data_len = AUTH_DLEN(d_lens);
 685         if (unlikely(flags & VALID_AAD_BUF)) {
 686                 /*
 687                  * We dont support both aad
 688                  * and auth data separately
 689                  */
 690                 auth_data_len = 0;
 691                 auth_offset = 0;
 692                 aad_len = fc_params->aad_buf.size;
 693                 aad_buf = &fc_params->aad_buf;
 694         }
 695         cpt_ctx = fc_params->ctx_buf.vaddr;
 696         cipher_type = cpt_ctx->enc_cipher;
 697         hash_type = cpt_ctx->hash_type;
 698         mac_len = cpt_ctx->mac_len;
 699
 700         /*
 701          * Save initial space that followed app data for completion code &
 702          * alternate completion code to fall in same cache line as app data
 703          */
 704         m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
 705         m_dma += COMPLETION_CODE_SIZE;
 706         size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
 707                 (uint8_t *)m_vaddr;
 708
 709         c_vaddr = (uint8_t *)m_vaddr + size;
 710         c_dma = m_dma + size;
 711         size += sizeof(cpt_res_s_t);
 712
 713         m_vaddr = (uint8_t *)m_vaddr + size;
 714         m_dma += size;
 715
 716         /* start cpt request info struct at 8 byte boundary */
 717         size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) -
 718                 (uint8_t *)m_vaddr;
 719
 720         req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size);
 721
 722         size += sizeof(struct cpt_request_info);
 723         m_vaddr = (uint8_t *)m_vaddr + size;
 724         m_dma += size;
 725
 726         if (unlikely(!(flags & VALID_IV_BUF))) {
 727                 iv_len = 0;
 728                 iv_offset = ENCR_IV_OFFSET(d_offs);
 729         }
 730
 731         if (unlikely(flags & VALID_AAD_BUF)) {
 732                 /*
 733                  * When AAD is given, data above encr_offset is pass through
 734                  * Since AAD is given as separate pointer and not as offset,
 735                  * this is a special case as we need to fragment input data
 736                  * into passthrough + encr_data and then insert AAD in between.
 737                  */
 738                 if (hash_type != GMAC_TYPE) {
 739                         passthrough_len = encr_offset;
 740                         auth_offset = passthrough_len + iv_len;
 741                         encr_offset = passthrough_len + aad_len + iv_len;
 742                         auth_data_len = aad_len + encr_data_len;
 743                 } else {
 744                         passthrough_len = 16 + aad_len;
 745                         auth_offset = passthrough_len + iv_len;
 746                         auth_data_len = aad_len;
 747                 }
 748         } else {
 749                 encr_offset += iv_len;
 750                 auth_offset += iv_len;
 751         }
 752
 753         /* Encryption */
 754         vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_FC;
 755         vq_cmd_w0.s.opcode.minor = 0;
 756
 757         if (hash_type == GMAC_TYPE) {
 758                 encr_offset = 0;
 759                 encr_data_len = 0;
 760         }
 761
 762         auth_dlen = auth_offset + auth_data_len;
 763         enc_dlen = encr_data_len + encr_offset;
 764         if (unlikely(encr_data_len & 0xf)) {
 765                 if ((cipher_type == DES3_CBC) || (cipher_type == DES3_ECB))
 766                         enc_dlen = RTE_ALIGN_CEIL(encr_data_len, 8) +
 767                                         encr_offset;
 768                 else if (likely((cipher_type == AES_CBC) ||
 769                                 (cipher_type == AES_ECB)))
 770                         enc_dlen = RTE_ALIGN_CEIL(encr_data_len, 8) +
 771                                         encr_offset;
 772         }
 773
 774         if (unlikely(auth_dlen > enc_dlen)) {
 775                 inputlen = auth_dlen;
 776                 outputlen = auth_dlen + mac_len;
 777         } else {
 778                 inputlen = enc_dlen;
 779                 outputlen = enc_dlen + mac_len;
 780         }
 781
 782         /* GP op header */
 783         vq_cmd_w0.s.param1 = encr_data_len;
 784         vq_cmd_w0.s.param2 = auth_data_len;
 785         /*
 786          * In 83XX since we have a limitation of
 787          * IV & Offset control word not part of instruction
 788          * and need to be part of Data Buffer, we check if
 789          * head room is there and then only do the Direct mode processing
 790          */
 791         if (likely((flags & SINGLE_BUF_INPLACE) &&
 792                    (flags & SINGLE_BUF_HEADTAILROOM))) {
 793                 void *dm_vaddr = fc_params->bufs[0].vaddr;
 794                 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr;
 795                 /*
 796                  * This flag indicates that there is 24 bytes head room and
 797                  * 8 bytes tail room available, so that we get to do
 798                  * DIRECT MODE with limitation
 799                  */
 800
 801                 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len;
 802                 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len;
 803
 804                 /* DPTR */
 805                 req->ist.ei1 = offset_dma;
 806                 /* RPTR should just exclude offset control word */
 807                 req->ist.ei2 = dm_dma_addr - iv_len;
 808                 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr
 809                                                     + outputlen - iv_len);
 810
 811                 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN;
 812
 813                 if (likely(iv_len)) {
 814                         uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr
 815                                                       + OFF_CTRL_LEN);
 816                         uint64_t *src = fc_params->iv_buf;
 817                         dest[0] = src[0];
 818                         dest[1] = src[1];
 819                 }
 820
 821                 *(uint64_t *)offset_vaddr =
 822                         rte_cpu_to_be_64(((uint64_t)encr_offset << 16) |
 823                                 ((uint64_t)iv_offset << 8) |
 824                                 ((uint64_t)auth_offset));
 825
 826         } else {
 827                 uint32_t i, g_size_bytes, s_size_bytes;
 828                 uint64_t dptr_dma, rptr_dma;
 829                 sg_comp_t *gather_comp;
 830                 sg_comp_t *scatter_comp;
 831                 uint8_t *in_buffer;
 832
 833                 /* This falls under strict SG mode */
 834                 offset_vaddr = m_vaddr;
 835                 offset_dma = m_dma;
 836                 size = OFF_CTRL_LEN + iv_len;
 837
 838                 m_vaddr = (uint8_t *)m_vaddr + size;
 839                 m_dma += size;
 840
 841                 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE;
 842
 843                 if (likely(iv_len)) {
 844                         uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr
 845                                                       + OFF_CTRL_LEN);
 846                         uint64_t *src = fc_params->iv_buf;
 847                         dest[0] = src[0];
 848                         dest[1] = src[1];
 849                 }
 850
 851                 *(uint64_t *)offset_vaddr =
 852                         rte_cpu_to_be_64(((uint64_t)encr_offset << 16) |
 853                                 ((uint64_t)iv_offset << 8) |
 854                                 ((uint64_t)auth_offset));
 855
 856                 /* DPTR has SG list */
 857                 in_buffer = m_vaddr;
 858                 dptr_dma = m_dma;
 859
 860                 ((uint16_t *)in_buffer)[0] = 0;
 861                 ((uint16_t *)in_buffer)[1] = 0;
 862
 863                 /* TODO Add error check if space will be sufficient */
 864                 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
 865
 866                 /*
 867                  * Input Gather List
 868                  */
 869
 870                 i = 0;
 871
 872                 /* Offset control word that includes iv */
 873                 i = fill_sg_comp(gather_comp, i, offset_dma,
 874                                  OFF_CTRL_LEN + iv_len);
 875
 876                 /* Add input data */
 877                 size = inputlen - iv_len;
 878                 if (likely(size)) {
 879                         uint32_t aad_offset = aad_len ? passthrough_len : 0;
 880
 881                         if (unlikely(flags & SINGLE_BUF_INPLACE)) {
 882                                 i = fill_sg_comp_from_buf_min(gather_comp, i,
 883                                                               fc_params->bufs,
 884                                                               &size);
 885                         } else {
 886                                 i = fill_sg_comp_from_iov(gather_comp, i,
 887                                                           fc_params->src_iov,
 888                                                           0, &size,
 889                                                           aad_buf, aad_offset);
 890                         }
 891
 892                         if (unlikely(size)) {
 893                                 CPT_LOG_DP_ERR("Insufficient buffer space,"
 894                                                " size %d needed", size);
 895                                 return;
 896                         }
 897                 }
 898                 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
 899                 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
 900
 901                 /*
 902                  * Output Scatter list
 903                  */
 904                 i = 0;
 905                 scatter_comp =
 906                         (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
 907
 908                 /* Add IV */
 909                 if (likely(iv_len)) {
 910                         i = fill_sg_comp(scatter_comp, i,
 911                                          offset_dma + OFF_CTRL_LEN,
 912                                          iv_len);
 913                 }
 914
 915                 /* output data or output data + digest*/
 916                 if (unlikely(flags & VALID_MAC_BUF)) {
 917                         size = outputlen - iv_len - mac_len;
 918                         if (size) {
 919                                 uint32_t aad_offset =
 920                                         aad_len ? passthrough_len : 0;
 921
 922                                 if (unlikely(flags & SINGLE_BUF_INPLACE)) {
 923                                         i = fill_sg_comp_from_buf_min(
 924                                                         scatter_comp,
 925                                                         i,
 926                                                         fc_params->bufs,
 927                                                         &size);
 928                                 } else {
 929                                         i = fill_sg_comp_from_iov(scatter_comp,
 930                                                         i,
 931                                                         fc_params->dst_iov,
 932                                                         0,
 933                                                         &size,
 934                                                         aad_buf,
 935                                                         aad_offset);
 936                                 }
 937                                 if (unlikely(size)) {
 938                                         CPT_LOG_DP_ERR("Insufficient buffer"
 939                                                        " space, size %d needed",
 940                                                        size);
 941                                         return;
 942                                 }
 943                         }
 944                         /* mac_data */
 945                         if (mac_len) {
 946                                 i = fill_sg_comp_from_buf(scatter_comp, i,
 947                                                           &fc_params->mac_buf);
 948                         }
 949                 } else {
 950                         /* Output including mac */
 951                         size = outputlen - iv_len;
 952                         if (likely(size)) {
 953                                 uint32_t aad_offset =
 954                                         aad_len ? passthrough_len : 0;
 955
 956                                 if (unlikely(flags & SINGLE_BUF_INPLACE)) {
 957                                         i = fill_sg_comp_from_buf_min(
 958                                                         scatter_comp,
 959                                                         i,
 960                                                         fc_params->bufs,
 961                                                         &size);
 962                                 } else {
 963                                         i = fill_sg_comp_from_iov(scatter_comp,
 964                                                         i,
 965                                                         fc_params->dst_iov,
 966                                                         0,
 967                                                         &size,
 968                                                         aad_buf,
 969                                                         aad_offset);
 970                                 }
 971                                 if (unlikely(size)) {
 972                                         CPT_LOG_DP_ERR("Insufficient buffer"
 973                                                        " space, size %d needed",
 974                                                        size);
 975                                         return;
 976                                 }
 977                         }
 978                 }
 979                 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
 980                 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
 981
 982                 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
 983
 984                 /* This is DPTR len incase of SG mode */
 985                 vq_cmd_w0.s.dlen = size;
 986
 987                 m_vaddr = (uint8_t *)m_vaddr + size;
 988                 m_dma += size;
 989
 990                 /* cpt alternate completion address saved earlier */
 991                 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
 992                 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
 993                 rptr_dma = c_dma - 8;
 994
 995                 req->ist.ei1 = dptr_dma;
 996                 req->ist.ei2 = rptr_dma;
 997         }
 998
 999         /* 16 byte aligned cpt res address */
1000         req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
1001         *req->completion_addr = COMPLETION_CODE_INIT;
1002         req->comp_baddr  = c_dma;
1003
1004         /* Fill microcode part of instruction */
1005         req->ist.ei0 = vq_cmd_w0.u64;
1006
1007         req->op  = op;
1008
1009         *prep_req = req;
1010         return;
1011 }
1012
1013 static __rte_always_inline void
1014 cpt_dec_hmac_prep(uint32_t flags,
1015                   uint64_t d_offs,
1016                   uint64_t d_lens,
1017                   fc_params_t *fc_params,
1018                   void *op,
1019                   void **prep_req)
1020 {
1021         uint32_t iv_offset = 0, size;
1022         int32_t inputlen, outputlen, enc_dlen, auth_dlen;
1023         struct cpt_ctx *cpt_ctx;
1024         int32_t hash_type, mac_len;
1025         uint8_t iv_len = 16;
1026         struct cpt_request_info *req;
1027         buf_ptr_t *meta_p, *aad_buf = NULL;
1028         uint32_t encr_offset, auth_offset;
1029         uint32_t encr_data_len, auth_data_len, aad_len = 0;
1030         uint32_t passthrough_len = 0;
1031         void *m_vaddr, *offset_vaddr;
1032         uint64_t m_dma, offset_dma;
1033         vq_cmd_word0_t vq_cmd_w0;
1034         void *c_vaddr;
1035         uint64_t c_dma;
1036
1037         meta_p = &fc_params->meta_buf;
1038         m_vaddr = meta_p->vaddr;
1039         m_dma = meta_p->dma_addr;
1040
1041         encr_offset = ENCR_OFFSET(d_offs);
1042         auth_offset = AUTH_OFFSET(d_offs);
1043         encr_data_len = ENCR_DLEN(d_lens);
1044         auth_data_len = AUTH_DLEN(d_lens);
1045
1046         if (unlikely(flags & VALID_AAD_BUF)) {
1047                 /*
1048                  * We dont support both aad
1049                  * and auth data separately
1050                  */
1051                 auth_data_len = 0;
1052                 auth_offset = 0;
1053                 aad_len = fc_params->aad_buf.size;
1054                 aad_buf = &fc_params->aad_buf;
1055         }
1056
1057         cpt_ctx = fc_params->ctx_buf.vaddr;
1058         hash_type = cpt_ctx->hash_type;
1059         mac_len = cpt_ctx->mac_len;
1060
1061         if (unlikely(!(flags & VALID_IV_BUF))) {
1062                 iv_len = 0;
1063                 iv_offset = ENCR_IV_OFFSET(d_offs);
1064         }
1065
1066         if (unlikely(flags & VALID_AAD_BUF)) {
1067                 /*
1068                  * When AAD is given, data above encr_offset is pass through
1069                  * Since AAD is given as separate pointer and not as offset,
1070                  * this is a special case as we need to fragment input data
1071                  * into passthrough + encr_data and then insert AAD in between.
1072                  */
1073                 if (hash_type != GMAC_TYPE) {
1074                         passthrough_len = encr_offset;
1075                         auth_offset = passthrough_len + iv_len;
1076                         encr_offset = passthrough_len + aad_len + iv_len;
1077                         auth_data_len = aad_len + encr_data_len;
1078                 } else {
1079                         passthrough_len = 16 + aad_len;
1080                         auth_offset = passthrough_len + iv_len;
1081                         auth_data_len = aad_len;
1082                 }
1083         } else {
1084                 encr_offset += iv_len;
1085                 auth_offset += iv_len;
1086         }
1087
1088         /*
1089          * Save initial space that followed app data for completion code &
1090          * alternate completion code to fall in same cache line as app data
1091          */
1092         m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
1093         m_dma += COMPLETION_CODE_SIZE;
1094         size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
1095                (uint8_t *)m_vaddr;
1096         c_vaddr = (uint8_t *)m_vaddr + size;
1097         c_dma = m_dma + size;
1098         size += sizeof(cpt_res_s_t);
1099
1100         m_vaddr = (uint8_t *)m_vaddr + size;
1101         m_dma += size;
1102
1103         /* start cpt request info structure at 8 byte alignment */
1104         size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) -
1105                 (uint8_t *)m_vaddr;
1106
1107         req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size);
1108
1109         size += sizeof(struct cpt_request_info);
1110         m_vaddr = (uint8_t *)m_vaddr + size;
1111         m_dma += size;
1112
1113         /* Decryption */
1114         vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_FC;
1115         vq_cmd_w0.s.opcode.minor = 1;
1116
1117         if (hash_type == GMAC_TYPE) {
1118                 encr_offset = 0;
1119                 encr_data_len = 0;
1120         }
1121
1122         enc_dlen = encr_offset + encr_data_len;
1123         auth_dlen = auth_offset + auth_data_len;
1124
1125         if (auth_dlen > enc_dlen) {
1126                 inputlen = auth_dlen + mac_len;
1127                 outputlen = auth_dlen;
1128         } else {
1129                 inputlen = enc_dlen + mac_len;
1130                 outputlen = enc_dlen;
1131         }
1132
1133         vq_cmd_w0.s.param1 = encr_data_len;
1134         vq_cmd_w0.s.param2 = auth_data_len;
1135
1136         /*
1137          * In 83XX since we have a limitation of
1138          * IV & Offset control word not part of instruction
1139          * and need to be part of Data Buffer, we check if
1140          * head room is there and then only do the Direct mode processing
1141          */
1142         if (likely((flags & SINGLE_BUF_INPLACE) &&
1143                    (flags & SINGLE_BUF_HEADTAILROOM))) {
1144                 void *dm_vaddr = fc_params->bufs[0].vaddr;
1145                 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr;
1146                 /*
1147                  * This flag indicates that there is 24 bytes head room and
1148                  * 8 bytes tail room available, so that we get to do
1149                  * DIRECT MODE with limitation
1150                  */
1151
1152                 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len;
1153                 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len;
1154                 req->ist.ei1 = offset_dma;
1155
1156                 /* RPTR should just exclude offset control word */
1157                 req->ist.ei2 = dm_dma_addr - iv_len;
1158
1159                 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr +
1160                                         outputlen - iv_len);
1161                 /* since this is decryption,
1162                  * don't touch the content of
1163                  * alternate ccode space as it contains
1164                  * hmac.
1165                  */
1166
1167                 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN;
1168
1169                 if (likely(iv_len)) {
1170                         uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr +
1171                                                       OFF_CTRL_LEN);
1172                         uint64_t *src = fc_params->iv_buf;
1173                         dest[0] = src[0];
1174                         dest[1] = src[1];
1175                 }
1176
1177                 *(uint64_t *)offset_vaddr =
1178                         rte_cpu_to_be_64(((uint64_t)encr_offset << 16) |
1179                                 ((uint64_t)iv_offset << 8) |
1180                                 ((uint64_t)auth_offset));
1181
1182         } else {
1183                 uint64_t dptr_dma, rptr_dma;
1184                 uint32_t g_size_bytes, s_size_bytes;
1185                 sg_comp_t *gather_comp;
1186                 sg_comp_t *scatter_comp;
1187                 uint8_t *in_buffer;
1188                 uint8_t i = 0;
1189
1190                 /* This falls under strict SG mode */
1191                 offset_vaddr = m_vaddr;
1192                 offset_dma = m_dma;
1193                 size = OFF_CTRL_LEN + iv_len;
1194
1195                 m_vaddr = (uint8_t *)m_vaddr + size;
1196                 m_dma += size;
1197
1198                 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE;
1199
1200                 if (likely(iv_len)) {
1201                         uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr +
1202                                                       OFF_CTRL_LEN);
1203                         uint64_t *src = fc_params->iv_buf;
1204                         dest[0] = src[0];
1205                         dest[1] = src[1];
1206                 }
1207
1208                 *(uint64_t *)offset_vaddr =
1209                         rte_cpu_to_be_64(((uint64_t)encr_offset << 16) |
1210                                 ((uint64_t)iv_offset << 8) |
1211                                 ((uint64_t)auth_offset));
1212
1213                 /* DPTR has SG list */
1214                 in_buffer = m_vaddr;
1215                 dptr_dma = m_dma;
1216
1217                 ((uint16_t *)in_buffer)[0] = 0;
1218                 ((uint16_t *)in_buffer)[1] = 0;
1219
1220                 /* TODO Add error check if space will be sufficient */
1221                 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
1222
1223                 /*
1224                  * Input Gather List
1225                  */
1226                 i = 0;
1227
1228                 /* Offset control word that includes iv */
1229                 i = fill_sg_comp(gather_comp, i, offset_dma,
1230                                  OFF_CTRL_LEN + iv_len);
1231
1232                 /* Add input data */
1233                 if (flags & VALID_MAC_BUF) {
1234                         size = inputlen - iv_len - mac_len;
1235                         if (size) {
1236                                 /* input data only */
1237                                 if (unlikely(flags & SINGLE_BUF_INPLACE)) {
1238                                         i = fill_sg_comp_from_buf_min(
1239                                                         gather_comp, i,
1240                                                         fc_params->bufs,
1241                                                         &size);
1242                                 } else {
1243                                         uint32_t aad_offset = aad_len ?
1244                                                 passthrough_len : 0;
1245
1246                                         i = fill_sg_comp_from_iov(gather_comp,
1247                                                         i,
1248                                                         fc_params->src_iov,
1249                                                         0, &size,
1250                                                         aad_buf,
1251                                                         aad_offset);
1252                                 }
1253                                 if (unlikely(size)) {
1254                                         CPT_LOG_DP_ERR("Insufficient buffer"
1255                                                        " space, size %d needed",
1256                                                        size);
1257                                         return;
1258                                 }
1259                         }
1260
1261                         /* mac data */
1262                         if (mac_len) {
1263                                 i = fill_sg_comp_from_buf(gather_comp, i,
1264                                                           &fc_params->mac_buf);
1265                         }
1266                 } else {
1267                         /* input data + mac */
1268                         size = inputlen - iv_len;
1269                         if (size) {
1270                                 if (unlikely(flags & SINGLE_BUF_INPLACE)) {
1271                                         i = fill_sg_comp_from_buf_min(
1272                                                         gather_comp, i,
1273                                                         fc_params->bufs,
1274                                                         &size);
1275                                 } else {
1276                                         uint32_t aad_offset = aad_len ?
1277                                                 passthrough_len : 0;
1278
1279                                         if (unlikely(!fc_params->src_iov)) {
1280                                                 CPT_LOG_DP_ERR("Bad input args");
1281                                                 return;
1282                                         }
1283
1284                                         i = fill_sg_comp_from_iov(
1285                                                         gather_comp, i,
1286                                                         fc_params->src_iov,
1287                                                         0, &size,
1288                                                         aad_buf,
1289                                                         aad_offset);
1290                                 }
1291
1292                                 if (unlikely(size)) {
1293                                         CPT_LOG_DP_ERR("Insufficient buffer"
1294                                                        " space, size %d needed",
1295                                                        size);
1296                                         return;
1297                                 }
1298                         }
1299                 }
1300                 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
1301                 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
1302
1303                 /*
1304                  * Output Scatter List
1305                  */
1306
1307                 i = 0;
1308                 scatter_comp =
1309                         (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
1310
1311                 /* Add iv */
1312                 if (iv_len) {
1313                         i = fill_sg_comp(scatter_comp, i,
1314                                          offset_dma + OFF_CTRL_LEN,
1315                                          iv_len);
1316                 }
1317
1318                 /* Add output data */
1319                 size = outputlen - iv_len;
1320                 if (size) {
1321                         if (unlikely(flags & SINGLE_BUF_INPLACE)) {
1322                                 /* handle single buffer here */
1323                                 i = fill_sg_comp_from_buf_min(scatter_comp, i,
1324                                                               fc_params->bufs,
1325                                                               &size);
1326                         } else {
1327                                 uint32_t aad_offset = aad_len ?
1328                                         passthrough_len : 0;
1329
1330                                 if (unlikely(!fc_params->dst_iov)) {
1331                                         CPT_LOG_DP_ERR("Bad input args");
1332                                         return;
1333                                 }
1334
1335                                 i = fill_sg_comp_from_iov(scatter_comp, i,
1336                                                           fc_params->dst_iov, 0,
1337                                                           &size, aad_buf,
1338                                                           aad_offset);
1339                         }
1340
1341                         if (unlikely(size)) {
1342                                 CPT_LOG_DP_ERR("Insufficient buffer space,"
1343                                                " size %d needed", size);
1344                                 return;
1345                         }
1346                 }
1347
1348                 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
1349                 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
1350
1351                 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
1352
1353                 /* This is DPTR len incase of SG mode */
1354                 vq_cmd_w0.s.dlen = size;
1355
1356                 m_vaddr = (uint8_t *)m_vaddr + size;
1357                 m_dma += size;
1358
1359                 /* cpt alternate completion address saved earlier */
1360                 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
1361                 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
1362                 rptr_dma = c_dma - 8;
1363                 size += COMPLETION_CODE_SIZE;
1364
1365                 req->ist.ei1 = dptr_dma;
1366                 req->ist.ei2 = rptr_dma;
1367         }
1368
1369         /* 16 byte aligned cpt res address */
1370         req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
1371         *req->completion_addr = COMPLETION_CODE_INIT;
1372         req->comp_baddr  = c_dma;
1373
1374         /* Fill microcode part of instruction */
1375         req->ist.ei0 = vq_cmd_w0.u64;
1376
1377         req->op = op;
1378
1379         *prep_req = req;
1380         return;
1381 }
1382
1383 static __rte_always_inline void
1384 cpt_zuc_snow3g_enc_prep(uint32_t req_flags,
1385                         uint64_t d_offs,
1386                         uint64_t d_lens,
1387                         fc_params_t *params,
1388                         void *op,
1389                         void **prep_req)
1390 {
1391         uint32_t size;
1392         int32_t inputlen, outputlen;
1393         struct cpt_ctx *cpt_ctx;
1394         uint32_t mac_len = 0;
1395         uint8_t snow3g, j;
1396         struct cpt_request_info *req;
1397         buf_ptr_t *buf_p;
1398         uint32_t encr_offset = 0, auth_offset = 0;
1399         uint32_t encr_data_len = 0, auth_data_len = 0;
1400         int flags, iv_len = 16;
1401         void *m_vaddr, *c_vaddr;
1402         uint64_t m_dma, c_dma, offset_ctrl;
1403         uint64_t *offset_vaddr, offset_dma;
1404         uint32_t *iv_s, iv[4];
1405         vq_cmd_word0_t vq_cmd_w0;
1406
1407         buf_p = &params->meta_buf;
1408         m_vaddr = buf_p->vaddr;
1409         m_dma = buf_p->dma_addr;
1410
1411         cpt_ctx = params->ctx_buf.vaddr;
1412         flags = cpt_ctx->zsk_flags;
1413         mac_len = cpt_ctx->mac_len;
1414         snow3g = cpt_ctx->snow3g;
1415
1416         /*
1417          * Save initial space that followed app data for completion code &
1418          * alternate completion code to fall in same cache line as app data
1419          */
1420         m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
1421         m_dma += COMPLETION_CODE_SIZE;
1422         size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
1423                 (uint8_t *)m_vaddr;
1424
1425         c_vaddr = (uint8_t *)m_vaddr + size;
1426         c_dma = m_dma + size;
1427         size += sizeof(cpt_res_s_t);
1428
1429         m_vaddr = (uint8_t *)m_vaddr + size;
1430         m_dma += size;
1431
1432         /* Reserve memory for cpt request info */
1433         req = m_vaddr;
1434
1435         size = sizeof(struct cpt_request_info);
1436         m_vaddr = (uint8_t *)m_vaddr + size;
1437         m_dma += size;
1438
1439         vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_ZUC_SNOW3G;
1440
1441         /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */
1442
1443         vq_cmd_w0.s.opcode.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) |
1444                           (0 << 3) | (flags & 0x7));
1445
1446         if (flags == 0x1) {
1447                 /*
1448                  * Microcode expects offsets in bytes
1449                  * TODO: Rounding off
1450                  */
1451                 auth_data_len = AUTH_DLEN(d_lens);
1452
1453                 /* EIA3 or UIA2 */
1454                 auth_offset = AUTH_OFFSET(d_offs);
1455                 auth_offset = auth_offset / 8;
1456
1457                 /* consider iv len */
1458                 auth_offset += iv_len;
1459
1460                 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8);
1461                 outputlen = mac_len;
1462
1463                 offset_ctrl = rte_cpu_to_be_64((uint64_t)auth_offset);
1464
1465         } else {
1466                 /* EEA3 or UEA2 */
1467                 /*
1468                  * Microcode expects offsets in bytes
1469                  * TODO: Rounding off
1470                  */
1471                 encr_data_len = ENCR_DLEN(d_lens);
1472
1473                 encr_offset = ENCR_OFFSET(d_offs);
1474                 encr_offset = encr_offset / 8;
1475                 /* consider iv len */
1476                 encr_offset += iv_len;
1477
1478                 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8);
1479                 outputlen = inputlen;
1480
1481                 /* iv offset is 0 */
1482                 offset_ctrl = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
1483         }
1484
1485         /* IV */
1486         iv_s = (flags == 0x1) ? params->auth_iv_buf :
1487                 params->iv_buf;
1488
1489         if (snow3g) {
1490                 /*
1491                  * DPDK seems to provide it in form of IV3 IV2 IV1 IV0
1492                  * and BigEndian, MC needs it as IV0 IV1 IV2 IV3
1493                  */
1494
1495                 for (j = 0; j < 4; j++)
1496                         iv[j] = iv_s[3 - j];
1497         } else {
1498                 /* ZUC doesn't need a swap */
1499                 for (j = 0; j < 4; j++)
1500                         iv[j] = iv_s[j];
1501         }
1502
1503         /*
1504          * GP op header, lengths are expected in bits.
1505          */
1506         vq_cmd_w0.s.param1 = encr_data_len;
1507         vq_cmd_w0.s.param2 = auth_data_len;
1508
1509         /*
1510          * In 83XX since we have a limitation of
1511          * IV & Offset control word not part of instruction
1512          * and need to be part of Data Buffer, we check if
1513          * head room is there and then only do the Direct mode processing
1514          */
1515         if (likely((req_flags & SINGLE_BUF_INPLACE) &&
1516                    (req_flags & SINGLE_BUF_HEADTAILROOM))) {
1517                 void *dm_vaddr = params->bufs[0].vaddr;
1518                 uint64_t dm_dma_addr = params->bufs[0].dma_addr;
1519                 /*
1520                  * This flag indicates that there is 24 bytes head room and
1521                  * 8 bytes tail room available, so that we get to do
1522                  * DIRECT MODE with limitation
1523                  */
1524
1525                 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr -
1526                                             OFF_CTRL_LEN - iv_len);
1527                 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len;
1528
1529                 /* DPTR */
1530                 req->ist.ei1 = offset_dma;
1531                 /* RPTR should just exclude offset control word */
1532                 req->ist.ei2 = dm_dma_addr - iv_len;
1533                 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr
1534                                                     + outputlen - iv_len);
1535
1536                 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN;
1537
1538                 if (likely(iv_len)) {
1539                         uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr
1540                                                       + OFF_CTRL_LEN);
1541                         memcpy(iv_d, iv, 16);
1542                 }
1543
1544                 *offset_vaddr = offset_ctrl;
1545         } else {
1546                 uint32_t i, g_size_bytes, s_size_bytes;
1547                 uint64_t dptr_dma, rptr_dma;
1548                 sg_comp_t *gather_comp;
1549                 sg_comp_t *scatter_comp;
1550                 uint8_t *in_buffer;
1551                 uint32_t *iv_d;
1552
1553                 /* save space for iv */
1554                 offset_vaddr = m_vaddr;
1555                 offset_dma = m_dma;
1556
1557                 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
1558                 m_dma += OFF_CTRL_LEN + iv_len;
1559
1560                 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE;
1561
1562                 /* DPTR has SG list */
1563                 in_buffer = m_vaddr;
1564                 dptr_dma = m_dma;
1565
1566                 ((uint16_t *)in_buffer)[0] = 0;
1567                 ((uint16_t *)in_buffer)[1] = 0;
1568
1569                 /* TODO Add error check if space will be sufficient */
1570                 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
1571
1572                 /*
1573                  * Input Gather List
1574                  */
1575                 i = 0;
1576
1577                 /* Offset control word followed by iv */
1578
1579                 i = fill_sg_comp(gather_comp, i, offset_dma,
1580                                  OFF_CTRL_LEN + iv_len);
1581
1582                 /* iv offset is 0 */
1583                 *offset_vaddr = offset_ctrl;
1584
1585                 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN);
1586                 memcpy(iv_d, iv, 16);
1587
1588                 /* input data */
1589                 size = inputlen - iv_len;
1590                 if (size) {
1591                         i = fill_sg_comp_from_iov(gather_comp, i,
1592                                                   params->src_iov,
1593                                                   0, &size, NULL, 0);
1594                         if (unlikely(size)) {
1595                                 CPT_LOG_DP_ERR("Insufficient buffer space,"
1596                                                " size %d needed", size);
1597                                 return;
1598                         }
1599                 }
1600                 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
1601                 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
1602
1603                 /*
1604                  * Output Scatter List
1605                  */
1606
1607                 i = 0;
1608                 scatter_comp =
1609                         (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
1610
1611                 if (flags == 0x1) {
1612                         /* IV in SLIST only for EEA3 & UEA2 */
1613                         iv_len = 0;
1614                 }
1615
1616                 if (iv_len) {
1617                         i = fill_sg_comp(scatter_comp, i,
1618                                          offset_dma + OFF_CTRL_LEN, iv_len);
1619                 }
1620
1621                 /* Add output data */
1622                 if (req_flags & VALID_MAC_BUF) {
1623                         size = outputlen - iv_len - mac_len;
1624                         if (size) {
1625                                 i = fill_sg_comp_from_iov(scatter_comp, i,
1626                                                           params->dst_iov, 0,
1627                                                           &size, NULL, 0);
1628
1629                                 if (unlikely(size)) {
1630                                         CPT_LOG_DP_ERR("Insufficient buffer space,"
1631                                                        " size %d needed", size);
1632                                         return;
1633                                 }
1634                         }
1635
1636                         /* mac data */
1637                         if (mac_len) {
1638                                 i = fill_sg_comp_from_buf(scatter_comp, i,
1639                                                           &params->mac_buf);
1640                         }
1641                 } else {
1642                         /* Output including mac */
1643                         size = outputlen - iv_len;
1644                         if (size) {
1645                                 i = fill_sg_comp_from_iov(scatter_comp, i,
1646                                                           params->dst_iov, 0,
1647                                                           &size, NULL, 0);
1648
1649                                 if (unlikely(size)) {
1650                                         CPT_LOG_DP_ERR("Insufficient buffer space,"
1651                                                        " size %d needed", size);
1652                                         return;
1653                                 }
1654                         }
1655                 }
1656                 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
1657                 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
1658
1659                 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
1660
1661                 /* This is DPTR len incase of SG mode */
1662                 vq_cmd_w0.s.dlen = size;
1663
1664                 m_vaddr = (uint8_t *)m_vaddr + size;
1665                 m_dma += size;
1666
1667                 /* cpt alternate completion address saved earlier */
1668                 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
1669                 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
1670                 rptr_dma = c_dma - 8;
1671
1672                 req->ist.ei1 = dptr_dma;
1673                 req->ist.ei2 = rptr_dma;
1674         }
1675
1676         /* 16 byte aligned cpt res address */
1677         req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
1678         *req->completion_addr = COMPLETION_CODE_INIT;
1679         req->comp_baddr  = c_dma;
1680
1681         /* Fill microcode part of instruction */
1682         req->ist.ei0 = vq_cmd_w0.u64;
1683
1684         req->op = op;
1685
1686         *prep_req = req;
1687         return;
1688 }
1689
1690 static __rte_always_inline void
1691 cpt_zuc_snow3g_dec_prep(uint32_t req_flags,
1692                         uint64_t d_offs,
1693                         uint64_t d_lens,
1694                         fc_params_t *params,
1695                         void *op,
1696                         void **prep_req)
1697 {
1698         uint32_t size;
1699         int32_t inputlen = 0, outputlen;
1700         struct cpt_ctx *cpt_ctx;
1701         uint8_t snow3g, iv_len = 16;
1702         struct cpt_request_info *req;
1703         buf_ptr_t *buf_p;
1704         uint32_t encr_offset;
1705         uint32_t encr_data_len;
1706         int flags;
1707         void *m_vaddr, *c_vaddr;
1708         uint64_t m_dma, c_dma;
1709         uint64_t *offset_vaddr, offset_dma;
1710         uint32_t *iv_s, iv[4], j;
1711         vq_cmd_word0_t vq_cmd_w0;
1712
1713         buf_p = &params->meta_buf;
1714         m_vaddr = buf_p->vaddr;
1715         m_dma = buf_p->dma_addr;
1716
1717         /*
1718          * Microcode expects offsets in bytes
1719          * TODO: Rounding off
1720          */
1721         encr_offset = ENCR_OFFSET(d_offs) / 8;
1722         encr_data_len = ENCR_DLEN(d_lens);
1723
1724         cpt_ctx = params->ctx_buf.vaddr;
1725         flags = cpt_ctx->zsk_flags;
1726         snow3g = cpt_ctx->snow3g;
1727         /*
1728          * Save initial space that followed app data for completion code &
1729          * alternate completion code to fall in same cache line as app data
1730          */
1731         m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
1732         m_dma += COMPLETION_CODE_SIZE;
1733         size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
1734                 (uint8_t *)m_vaddr;
1735
1736         c_vaddr = (uint8_t *)m_vaddr + size;
1737         c_dma = m_dma + size;
1738         size += sizeof(cpt_res_s_t);
1739
1740         m_vaddr = (uint8_t *)m_vaddr + size;
1741         m_dma += size;
1742
1743         /* Reserve memory for cpt request info */
1744         req = m_vaddr;
1745
1746         size = sizeof(struct cpt_request_info);
1747         m_vaddr = (uint8_t *)m_vaddr + size;
1748         m_dma += size;
1749
1750         vq_cmd_w0.u64 = 0;
1751         vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_ZUC_SNOW3G;
1752
1753         /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */
1754
1755         vq_cmd_w0.s.opcode.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) |
1756                           (0 << 3) | (flags & 0x7));
1757
1758         /* consider iv len */
1759         encr_offset += iv_len;
1760
1761         inputlen = encr_offset +
1762                 (RTE_ALIGN(encr_data_len, 8) / 8);
1763         outputlen = inputlen;
1764
1765         /* IV */
1766         iv_s = params->iv_buf;
1767         if (snow3g) {
1768                 /*
1769                  * DPDK seems to provide it in form of IV3 IV2 IV1 IV0
1770                  * and BigEndian, MC needs it as IV0 IV1 IV2 IV3
1771                  */
1772
1773                 for (j = 0; j < 4; j++)
1774                         iv[j] = iv_s[3 - j];
1775         } else {
1776                 /* ZUC doesn't need a swap */
1777                 for (j = 0; j < 4; j++)
1778                         iv[j] = iv_s[j];
1779         }
1780
1781         /*
1782          * GP op header, lengths are expected in bits.
1783          */
1784         vq_cmd_w0.s.param1 = encr_data_len;
1785
1786         /*
1787          * In 83XX since we have a limitation of
1788          * IV & Offset control word not part of instruction
1789          * and need to be part of Data Buffer, we check if
1790          * head room is there and then only do the Direct mode processing
1791          */
1792         if (likely((req_flags & SINGLE_BUF_INPLACE) &&
1793                    (req_flags & SINGLE_BUF_HEADTAILROOM))) {
1794                 void *dm_vaddr = params->bufs[0].vaddr;
1795                 uint64_t dm_dma_addr = params->bufs[0].dma_addr;
1796                 /*
1797                  * This flag indicates that there is 24 bytes head room and
1798                  * 8 bytes tail room available, so that we get to do
1799                  * DIRECT MODE with limitation
1800                  */
1801
1802                 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr -
1803                                             OFF_CTRL_LEN - iv_len);
1804                 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len;
1805
1806                 /* DPTR */
1807                 req->ist.ei1 = offset_dma;
1808                 /* RPTR should just exclude offset control word */
1809                 req->ist.ei2 = dm_dma_addr - iv_len;
1810                 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr
1811                                                     + outputlen - iv_len);
1812
1813                 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN;
1814
1815                 if (likely(iv_len)) {
1816                         uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr
1817                                                       + OFF_CTRL_LEN);
1818                         memcpy(iv_d, iv, 16);
1819                 }
1820
1821                 /* iv offset is 0 */
1822                 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
1823         } else {
1824                 uint32_t i, g_size_bytes, s_size_bytes;
1825                 uint64_t dptr_dma, rptr_dma;
1826                 sg_comp_t *gather_comp;
1827                 sg_comp_t *scatter_comp;
1828                 uint8_t *in_buffer;
1829                 uint32_t *iv_d;
1830
1831                 /* save space for offset and iv... */
1832                 offset_vaddr = m_vaddr;
1833                 offset_dma = m_dma;
1834
1835                 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
1836                 m_dma += OFF_CTRL_LEN + iv_len;
1837
1838                 vq_cmd_w0.s.opcode.major |= CPT_DMA_MODE;
1839
1840                 /* DPTR has SG list */
1841                 in_buffer = m_vaddr;
1842                 dptr_dma = m_dma;
1843
1844                 ((uint16_t *)in_buffer)[0] = 0;
1845                 ((uint16_t *)in_buffer)[1] = 0;
1846
1847                 /* TODO Add error check if space will be sufficient */
1848                 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
1849
1850                 /*
1851                  * Input Gather List
1852                  */
1853                 i = 0;
1854
1855                 /* Offset control word */
1856
1857                 /* iv offset is 0 */
1858                 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
1859
1860                 i = fill_sg_comp(gather_comp, i, offset_dma,
1861                                  OFF_CTRL_LEN + iv_len);
1862
1863                 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN);
1864                 memcpy(iv_d, iv, 16);
1865
1866                 /* Add input data */
1867                 size = inputlen - iv_len;
1868                 if (size) {
1869                         i = fill_sg_comp_from_iov(gather_comp, i,
1870                                                   params->src_iov,
1871                                                   0, &size, NULL, 0);
1872                         if (unlikely(size)) {
1873                                 CPT_LOG_DP_ERR("Insufficient buffer space,"
1874                                                " size %d needed", size);
1875                                 return;
1876                         }
1877                 }
1878                 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
1879                 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
1880
1881                 /*
1882                  * Output Scatter List
1883                  */
1884
1885                 i = 0;
1886                 scatter_comp =
1887                         (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
1888
1889                 /* IV */
1890                 i = fill_sg_comp(scatter_comp, i,
1891                                  offset_dma + OFF_CTRL_LEN,
1892                                  iv_len);
1893
1894                 /* Add output data */
1895                 size = outputlen - iv_len;
1896                 if (size) {
1897                         i = fill_sg_comp_from_iov(scatter_comp, i,
1898                                                   params->dst_iov, 0,
1899                                                   &size, NULL, 0);
1900
1901                         if (unlikely(size)) {
1902                                 CPT_LOG_DP_ERR("Insufficient buffer space,"
1903                                                " size %d needed", size);
1904                                 return;
1905                         }
1906                 }
1907                 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
1908                 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
1909
1910                 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
1911
1912                 /* This is DPTR len incase of SG mode */
1913                 vq_cmd_w0.s.dlen = size;
1914
1915                 m_vaddr = (uint8_t *)m_vaddr + size;
1916                 m_dma += size;
1917
1918                 /* cpt alternate completion address saved earlier */
1919                 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
1920                 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
1921                 rptr_dma = c_dma - 8;
1922
1923                 req->ist.ei1 = dptr_dma;
1924                 req->ist.ei2 = rptr_dma;
1925         }
1926
1927         /* 16 byte aligned cpt res address */
1928         req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
1929         *req->completion_addr = COMPLETION_CODE_INIT;
1930         req->comp_baddr  = c_dma;
1931
1932         /* Fill microcode part of instruction */
1933         req->ist.ei0 = vq_cmd_w0.u64;
1934
1935         req->op = op;
1936
1937         *prep_req = req;
1938         return;
1939 }
1940
1941 static __rte_always_inline void
1942 cpt_kasumi_enc_prep(uint32_t req_flags,
1943                     uint64_t d_offs,
1944                     uint64_t d_lens,
1945                     fc_params_t *params,
1946                     void *op,
1947                     void **prep_req)
1948 {
1949         uint32_t size;
1950         int32_t inputlen = 0, outputlen = 0;
1951         struct cpt_ctx *cpt_ctx;
1952         uint32_t mac_len = 0;
1953         uint8_t i = 0;
1954         struct cpt_request_info *req;
1955         buf_ptr_t *buf_p;
1956         uint32_t encr_offset, auth_offset;
1957         uint32_t encr_data_len, auth_data_len;
1958         int flags;
1959         uint8_t *iv_s, *iv_d, iv_len = 8;
1960         uint8_t dir = 0;
1961         void *m_vaddr, *c_vaddr;
1962         uint64_t m_dma, c_dma;
1963         uint64_t *offset_vaddr, offset_dma;
1964         vq_cmd_word0_t vq_cmd_w0;
1965         uint8_t *in_buffer;
1966         uint32_t g_size_bytes, s_size_bytes;
1967         uint64_t dptr_dma, rptr_dma;
1968         sg_comp_t *gather_comp;
1969         sg_comp_t *scatter_comp;
1970
1971         buf_p = &params->meta_buf;
1972         m_vaddr = buf_p->vaddr;
1973         m_dma = buf_p->dma_addr;
1974
1975         encr_offset = ENCR_OFFSET(d_offs) / 8;
1976         auth_offset = AUTH_OFFSET(d_offs) / 8;
1977         encr_data_len = ENCR_DLEN(d_lens);
1978         auth_data_len = AUTH_DLEN(d_lens);
1979
1980         cpt_ctx = params->ctx_buf.vaddr;
1981         flags = cpt_ctx->zsk_flags;
1982         mac_len = cpt_ctx->mac_len;
1983
1984         if (flags == 0x0)
1985                 iv_s = params->iv_buf;
1986         else
1987                 iv_s = params->auth_iv_buf;
1988
1989         dir = iv_s[8] & 0x1;
1990
1991         /*
1992          * Save initial space that followed app data for completion code &
1993          * alternate completion code to fall in same cache line as app data
1994          */
1995         m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
1996         m_dma += COMPLETION_CODE_SIZE;
1997         size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
1998                 (uint8_t *)m_vaddr;
1999
2000         c_vaddr = (uint8_t *)m_vaddr + size;
2001         c_dma = m_dma + size;
2002         size += sizeof(cpt_res_s_t);
2003
2004         m_vaddr = (uint8_t *)m_vaddr + size;
2005         m_dma += size;
2006
2007         /* Reserve memory for cpt request info */
2008         req = m_vaddr;
2009
2010         size = sizeof(struct cpt_request_info);
2011         m_vaddr = (uint8_t *)m_vaddr + size;
2012         m_dma += size;
2013
2014         vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE;
2015
2016         /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */
2017         vq_cmd_w0.s.opcode.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) |
2018                           (dir << 4) | (0 << 3) | (flags & 0x7));
2019
2020         /*
2021          * GP op header, lengths are expected in bits.
2022          */
2023         vq_cmd_w0.s.param1 = encr_data_len;
2024         vq_cmd_w0.s.param2 = auth_data_len;
2025
2026         /* consider iv len */
2027         if (flags == 0x0) {
2028                 encr_offset += iv_len;
2029                 auth_offset += iv_len;
2030         }
2031
2032         /* save space for offset ctrl and iv */
2033         offset_vaddr = m_vaddr;
2034         offset_dma = m_dma;
2035
2036         m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
2037         m_dma += OFF_CTRL_LEN + iv_len;
2038
2039         /* DPTR has SG list */
2040         in_buffer = m_vaddr;
2041         dptr_dma = m_dma;
2042
2043         ((uint16_t *)in_buffer)[0] = 0;
2044         ((uint16_t *)in_buffer)[1] = 0;
2045
2046         /* TODO Add error check if space will be sufficient */
2047         gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
2048
2049         /*
2050          * Input Gather List
2051          */
2052         i = 0;
2053
2054         /* Offset control word followed by iv */
2055
2056         if (flags == 0x0) {
2057                 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8);
2058                 outputlen = inputlen;
2059                 /* iv offset is 0 */
2060                 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
2061         } else {
2062                 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8);
2063                 outputlen = mac_len;
2064                 /* iv offset is 0 */
2065                 *offset_vaddr = rte_cpu_to_be_64((uint64_t)auth_offset);
2066         }
2067
2068         i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len);
2069
2070         /* IV */
2071         iv_d = (uint8_t *)offset_vaddr + OFF_CTRL_LEN;
2072         memcpy(iv_d, iv_s, iv_len);
2073
2074         /* input data */
2075         size = inputlen - iv_len;
2076         if (size) {
2077                 i = fill_sg_comp_from_iov(gather_comp, i,
2078                                           params->src_iov, 0,
2079                                           &size, NULL, 0);
2080
2081                 if (unlikely(size)) {
2082                         CPT_LOG_DP_ERR("Insufficient buffer space,"
2083                                        " size %d needed", size);
2084                         return;
2085                 }
2086         }
2087         ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
2088         g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
2089
2090         /*
2091          * Output Scatter List
2092          */
2093
2094         i = 0;
2095         scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
2096
2097         if (flags == 0x1) {
2098                 /* IV in SLIST only for F8 */
2099                 iv_len = 0;
2100         }
2101
2102         /* IV */
2103         if (iv_len) {
2104                 i = fill_sg_comp(scatter_comp, i,
2105                                  offset_dma + OFF_CTRL_LEN,
2106                                  iv_len);
2107         }
2108
2109         /* Add output data */
2110         if (req_flags & VALID_MAC_BUF) {
2111                 size = outputlen - iv_len - mac_len;
2112                 if (size) {
2113                         i = fill_sg_comp_from_iov(scatter_comp, i,
2114                                                   params->dst_iov, 0,
2115                                                   &size, NULL, 0);
2116
2117                         if (unlikely(size)) {
2118                                 CPT_LOG_DP_ERR("Insufficient buffer space,"
2119                                                " size %d needed", size);
2120                                 return;
2121                         }
2122                 }
2123
2124                 /* mac data */
2125                 if (mac_len) {
2126                         i = fill_sg_comp_from_buf(scatter_comp, i,
2127                                                   &params->mac_buf);
2128                 }
2129         } else {
2130                 /* Output including mac */
2131                 size = outputlen - iv_len;
2132                 if (size) {
2133                         i = fill_sg_comp_from_iov(scatter_comp, i,
2134                                                   params->dst_iov, 0,
2135                                                   &size, NULL, 0);
2136
2137                         if (unlikely(size)) {
2138                                 CPT_LOG_DP_ERR("Insufficient buffer space,"
2139                                                " size %d needed", size);
2140                                 return;
2141                         }
2142                 }
2143         }
2144         ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
2145         s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
2146
2147         size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
2148
2149         /* This is DPTR len incase of SG mode */
2150         vq_cmd_w0.s.dlen = size;
2151
2152         m_vaddr = (uint8_t *)m_vaddr + size;
2153         m_dma += size;
2154
2155         /* cpt alternate completion address saved earlier */
2156         req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
2157         *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
2158         rptr_dma = c_dma - 8;
2159
2160         req->ist.ei1 = dptr_dma;
2161         req->ist.ei2 = rptr_dma;
2162
2163         /* 16 byte aligned cpt res address */
2164         req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
2165         *req->completion_addr = COMPLETION_CODE_INIT;
2166         req->comp_baddr  = c_dma;
2167
2168         /* Fill microcode part of instruction */
2169         req->ist.ei0 = vq_cmd_w0.u64;
2170
2171         req->op = op;
2172
2173         *prep_req = req;
2174         return;
2175 }
2176
2177 static __rte_always_inline void
2178 cpt_kasumi_dec_prep(uint64_t d_offs,
2179                     uint64_t d_lens,
2180                     fc_params_t *params,
2181                     void *op,
2182                     void **prep_req)
2183 {
2184         uint32_t size;
2185         int32_t inputlen = 0, outputlen;
2186         struct cpt_ctx *cpt_ctx;
2187         uint8_t i = 0, iv_len = 8;
2188         struct cpt_request_info *req;
2189         buf_ptr_t *buf_p;
2190         uint32_t encr_offset;
2191         uint32_t encr_data_len;
2192         int flags;
2193         uint8_t dir = 0;
2194         void *m_vaddr, *c_vaddr;
2195         uint64_t m_dma, c_dma;
2196         uint64_t *offset_vaddr, offset_dma;
2197         vq_cmd_word0_t vq_cmd_w0;
2198         uint8_t *in_buffer;
2199         uint32_t g_size_bytes, s_size_bytes;
2200         uint64_t dptr_dma, rptr_dma;
2201         sg_comp_t *gather_comp;
2202         sg_comp_t *scatter_comp;
2203
2204         buf_p = &params->meta_buf;
2205         m_vaddr = buf_p->vaddr;
2206         m_dma = buf_p->dma_addr;
2207
2208         encr_offset = ENCR_OFFSET(d_offs) / 8;
2209         encr_data_len = ENCR_DLEN(d_lens);
2210
2211         cpt_ctx = params->ctx_buf.vaddr;
2212         flags = cpt_ctx->zsk_flags;
2213         /*
2214          * Save initial space that followed app data for completion code &
2215          * alternate completion code to fall in same cache line as app data
2216          */
2217         m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
2218         m_dma += COMPLETION_CODE_SIZE;
2219         size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
2220                 (uint8_t *)m_vaddr;
2221
2222         c_vaddr = (uint8_t *)m_vaddr + size;
2223         c_dma = m_dma + size;
2224         size += sizeof(cpt_res_s_t);
2225
2226         m_vaddr = (uint8_t *)m_vaddr + size;
2227         m_dma += size;
2228
2229         /* Reserve memory for cpt request info */
2230         req = m_vaddr;
2231
2232         size = sizeof(struct cpt_request_info);
2233         m_vaddr = (uint8_t *)m_vaddr + size;
2234         m_dma += size;
2235
2236         vq_cmd_w0.u64 = 0;
2237         vq_cmd_w0.s.opcode.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE;
2238
2239         /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */
2240         vq_cmd_w0.s.opcode.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) |
2241                           (dir << 4) | (0 << 3) | (flags & 0x7));
2242
2243         /*
2244          * GP op header, lengths are expected in bits.
2245          */
2246         vq_cmd_w0.s.param1 = encr_data_len;
2247
2248         /* consider iv len */
2249         encr_offset += iv_len;
2250
2251         inputlen = iv_len + (RTE_ALIGN(encr_data_len, 8) / 8);
2252         outputlen = inputlen;
2253
2254         /* save space for offset ctrl & iv */
2255         offset_vaddr = m_vaddr;
2256         offset_dma = m_dma;
2257
2258         m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
2259         m_dma += OFF_CTRL_LEN + iv_len;
2260
2261         /* DPTR has SG list */
2262         in_buffer = m_vaddr;
2263         dptr_dma = m_dma;
2264
2265         ((uint16_t *)in_buffer)[0] = 0;
2266         ((uint16_t *)in_buffer)[1] = 0;
2267
2268         /* TODO Add error check if space will be sufficient */
2269         gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
2270
2271         /*
2272          * Input Gather List
2273          */
2274         i = 0;
2275
2276         /* Offset control word followed by iv */
2277         *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
2278
2279         i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len);
2280
2281         /* IV */
2282         memcpy((uint8_t *)offset_vaddr + OFF_CTRL_LEN,
2283                params->iv_buf, iv_len);
2284
2285         /* Add input data */
2286         size = inputlen - iv_len;
2287         if (size) {
2288                 i = fill_sg_comp_from_iov(gather_comp, i,
2289                                           params->src_iov,
2290                                           0, &size, NULL, 0);
2291                 if (unlikely(size)) {
2292                         CPT_LOG_DP_ERR("Insufficient buffer space,"
2293                                        " size %d needed", size);
2294                         return;
2295                 }
2296         }
2297         ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
2298         g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
2299
2300         /*
2301          * Output Scatter List
2302          */
2303
2304         i = 0;
2305         scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
2306
2307         /* IV */
2308         i = fill_sg_comp(scatter_comp, i,
2309                          offset_dma + OFF_CTRL_LEN,
2310                          iv_len);
2311
2312         /* Add output data */
2313         size = outputlen - iv_len;
2314         if (size) {
2315                 i = fill_sg_comp_from_iov(scatter_comp, i,
2316                                           params->dst_iov, 0,
2317                                           &size, NULL, 0);
2318                 if (unlikely(size)) {
2319                         CPT_LOG_DP_ERR("Insufficient buffer space,"
2320                                        " size %d needed", size);
2321                         return;
2322                 }
2323         }
2324         ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
2325         s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
2326
2327         size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
2328
2329         /* This is DPTR len incase of SG mode */
2330         vq_cmd_w0.s.dlen = size;
2331
2332         m_vaddr = (uint8_t *)m_vaddr + size;
2333         m_dma += size;
2334
2335         /* cpt alternate completion address saved earlier */
2336         req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
2337         *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
2338         rptr_dma = c_dma - 8;
2339
2340         req->ist.ei1 = dptr_dma;
2341         req->ist.ei2 = rptr_dma;
2342
2343         /* 16 byte aligned cpt res address */
2344         req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
2345         *req->completion_addr = COMPLETION_CODE_INIT;
2346         req->comp_baddr  = c_dma;
2347
2348         /* Fill microcode part of instruction */
2349         req->ist.ei0 = vq_cmd_w0.u64;
2350
2351         req->op = op;
2352
2353         *prep_req = req;
2354         return;
2355 }
2356
2357 static __rte_always_inline void *
2358 cpt_fc_dec_hmac_prep(uint32_t flags,
2359                      uint64_t d_offs,
2360                      uint64_t d_lens,
2361                      fc_params_t *fc_params,
2362                      void *op)
2363 {
2364         struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr;
2365         uint8_t fc_type;
2366         void *prep_req = NULL;
2367
2368         fc_type = ctx->fc_type;
2369
2370         if (likely(fc_type == FC_GEN)) {
2371                 cpt_dec_hmac_prep(flags, d_offs, d_lens, fc_params, op,
2372                                   &prep_req);
2373         } else if (fc_type == ZUC_SNOW3G) {
2374                 cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens, fc_params, op,
2375                                         &prep_req);
2376         } else if (fc_type == KASUMI) {
2377                 cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, op, &prep_req);
2378         }
2379
2380         /*
2381          * For AUTH_ONLY case,
2382          * MC only supports digest generation and verification
2383          * should be done in software by memcmp()
2384          */
2385
2386         return prep_req;
2387 }
2388
2389 static __rte_always_inline void *__rte_hot
2390 cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
2391                      fc_params_t *fc_params, void *op)
2392 {
2393         struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr;
2394         uint8_t fc_type;
2395         void *prep_req = NULL;
2396
2397         fc_type = ctx->fc_type;
2398
2399         /* Common api for rest of the ops */
2400         if (likely(fc_type == FC_GEN)) {
2401                 cpt_enc_hmac_prep(flags, d_offs, d_lens, fc_params, op,
2402                                   &prep_req);
2403         } else if (fc_type == ZUC_SNOW3G) {
2404                 cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens, fc_params, op,
2405                                         &prep_req);
2406         } else if (fc_type == KASUMI) {
2407                 cpt_kasumi_enc_prep(flags, d_offs, d_lens, fc_params, op,
2408                                     &prep_req);
2409         } else if (fc_type == HASH_HMAC) {
2410                 cpt_digest_gen_prep(flags, d_lens, fc_params, op, &prep_req);
2411         }
2412
2413         return prep_req;
2414 }
2415
2416 static __rte_always_inline int
2417 cpt_fc_auth_set_key(struct cpt_ctx *cpt_ctx, auth_type_t type,
2418                     const uint8_t *key, uint16_t key_len, uint16_t mac_len)
2419 {
2420         mc_fc_context_t *fctx = &cpt_ctx->mc_ctx.fctx;
2421         mc_zuc_snow3g_ctx_t *zs_ctx = &cpt_ctx->mc_ctx.zs_ctx;
2422         mc_kasumi_ctx_t *k_ctx = &cpt_ctx->mc_ctx.k_ctx;
2423
2424         if ((type >= ZUC_EIA3) && (type <= KASUMI_F9_ECB)) {
2425                 uint32_t keyx[4];
2426
2427                 if (key_len != 16)
2428                         return -1;
2429                 /* No support for AEAD yet */
2430                 if (cpt_ctx->enc_cipher)
2431                         return -1;
2432                 /* For ZUC/SNOW3G/Kasumi */
2433                 switch (type) {
2434                 case SNOW3G_UIA2:
2435                         cpt_ctx->snow3g = 1;
2436                         gen_key_snow3g(key, keyx);
2437                         memcpy(zs_ctx->ci_key, keyx, key_len);
2438                         cpt_ctx->fc_type = ZUC_SNOW3G;
2439                         cpt_ctx->zsk_flags = 0x1;
2440                         break;
2441                 case ZUC_EIA3:
2442                         cpt_ctx->snow3g = 0;
2443                         memcpy(zs_ctx->ci_key, key, key_len);
2444                         memcpy(zs_ctx->zuc_const, zuc_d, 32);
2445                         cpt_ctx->fc_type = ZUC_SNOW3G;
2446                         cpt_ctx->zsk_flags = 0x1;
2447                         break;
2448                 case KASUMI_F9_ECB:
2449                         /* Kasumi ECB mode */
2450                         cpt_ctx->k_ecb = 1;
2451                         memcpy(k_ctx->ci_key, key, key_len);
2452                         cpt_ctx->fc_type = KASUMI;
2453                         cpt_ctx->zsk_flags = 0x1;
2454                         break;
2455                 case KASUMI_F9_CBC:
2456                         memcpy(k_ctx->ci_key, key, key_len);
2457                         cpt_ctx->fc_type = KASUMI;
2458                         cpt_ctx->zsk_flags = 0x1;
2459                         break;
2460                 default:
2461                         return -1;
2462                 }
2463                 cpt_ctx->mac_len = 4;
2464                 cpt_ctx->hash_type = type;
2465                 return 0;
2466         }
2467
2468         if (!(cpt_ctx->fc_type == FC_GEN && !type)) {
2469                 if (!cpt_ctx->fc_type || !cpt_ctx->enc_cipher)
2470                         cpt_ctx->fc_type = HASH_HMAC;
2471         }
2472
2473         if (cpt_ctx->fc_type == FC_GEN && key_len > 64)
2474                 return -1;
2475
2476         /* For GMAC auth, cipher must be NULL */
2477         if (type == GMAC_TYPE)
2478                 fctx->enc.enc_cipher = 0;
2479
2480         fctx->enc.hash_type = cpt_ctx->hash_type = type;
2481         fctx->enc.mac_len = cpt_ctx->mac_len = mac_len;
2482
2483         if (key_len) {
2484                 cpt_ctx->hmac = 1;
2485                 memset(cpt_ctx->auth_key, 0, sizeof(cpt_ctx->auth_key));
2486                 memcpy(cpt_ctx->auth_key, key, key_len);
2487                 cpt_ctx->auth_key_len = key_len;
2488                 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad));
2489                 memset(fctx->hmac.opad, 0, sizeof(fctx->hmac.opad));
2490
2491                 if (key_len <= 64)
2492                         memcpy(fctx->hmac.opad, key, key_len);
2493                 fctx->enc.auth_input_type = 1;
2494         }
2495         return 0;
2496 }
2497
2498 static __rte_always_inline int
2499 fill_sess_aead(struct rte_crypto_sym_xform *xform,
2500                  struct cpt_sess_misc *sess)
2501 {
2502         struct rte_crypto_aead_xform *aead_form;
2503         cipher_type_t enc_type = 0; /* NULL Cipher type */
2504         auth_type_t auth_type = 0; /* NULL Auth type */
2505         uint32_t cipher_key_len = 0;
2506         uint8_t aes_gcm = 0;
2507         aead_form = &xform->aead;
2508         void *ctx = SESS_PRIV(sess);
2509
2510         if (aead_form->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {
2511                 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT;
2512                 sess->cpt_op |= CPT_OP_AUTH_GENERATE;
2513         } else if (aead_form->op == RTE_CRYPTO_AEAD_OP_DECRYPT) {
2514                 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT;
2515                 sess->cpt_op |= CPT_OP_AUTH_VERIFY;
2516         } else {
2517                 CPT_LOG_DP_ERR("Unknown aead operation\n");
2518                 return -1;
2519         }
2520         switch (aead_form->algo) {
2521         case RTE_CRYPTO_AEAD_AES_GCM:
2522                 enc_type = AES_GCM;
2523                 cipher_key_len = 16;
2524                 aes_gcm = 1;
2525                 break;
2526         case RTE_CRYPTO_AEAD_AES_CCM:
2527                 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u",
2528                                aead_form->algo);
2529                 return -1;
2530         case RTE_CRYPTO_AEAD_CHACHA20_POLY1305:
2531                 enc_type = CHACHA20;
2532                 auth_type = POLY1305;
2533                 cipher_key_len = 32;
2534                 sess->chacha_poly = 1;
2535                 break;
2536         default:
2537                 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified",
2538                                aead_form->algo);
2539                 return -1;
2540         }
2541         if (aead_form->key.length < cipher_key_len) {
2542                 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu",
2543                                (unsigned int long)aead_form->key.length);
2544                 return -1;
2545         }
2546         sess->zsk_flag = 0;
2547         sess->aes_gcm = aes_gcm;
2548         sess->mac_len = aead_form->digest_length;
2549         sess->iv_offset = aead_form->iv.offset;
2550         sess->iv_length = aead_form->iv.length;
2551         sess->aad_length = aead_form->aad_length;
2552
2553         if (unlikely(cpt_fc_ciph_set_key(ctx, enc_type, aead_form->key.data,
2554                         aead_form->key.length, NULL)))
2555                 return -1;
2556
2557         if (unlikely(cpt_fc_auth_set_key(ctx, auth_type, NULL, 0,
2558                         aead_form->digest_length)))
2559                 return -1;
2560
2561         return 0;
2562 }
2563
2564 static __rte_always_inline int
2565 fill_sess_cipher(struct rte_crypto_sym_xform *xform,
2566                  struct cpt_sess_misc *sess)
2567 {
2568         struct rte_crypto_cipher_xform *c_form;
2569         cipher_type_t enc_type = 0; /* NULL Cipher type */
2570         uint32_t cipher_key_len = 0;
2571         uint8_t zsk_flag = 0, aes_ctr = 0, is_null = 0;
2572
2573         c_form = &xform->cipher;
2574
2575         if (c_form->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
2576                 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT;
2577         else if (c_form->op == RTE_CRYPTO_CIPHER_OP_DECRYPT)
2578                 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT;
2579         else {
2580                 CPT_LOG_DP_ERR("Unknown cipher operation\n");
2581                 return -1;
2582         }
2583
2584         switch (c_form->algo) {
2585         case RTE_CRYPTO_CIPHER_AES_CBC:
2586                 enc_type = AES_CBC;
2587                 cipher_key_len = 16;
2588                 break;
2589         case RTE_CRYPTO_CIPHER_3DES_CBC:
2590                 enc_type = DES3_CBC;
2591                 cipher_key_len = 24;
2592                 break;
2593         case RTE_CRYPTO_CIPHER_DES_CBC:
2594                 /* DES is implemented using 3DES in hardware */
2595                 enc_type = DES3_CBC;
2596                 cipher_key_len = 8;
2597                 break;
2598         case RTE_CRYPTO_CIPHER_AES_CTR:
2599                 enc_type = AES_CTR;
2600                 cipher_key_len = 16;
2601                 aes_ctr = 1;
2602                 break;
2603         case RTE_CRYPTO_CIPHER_NULL:
2604                 enc_type = 0;
2605                 is_null = 1;
2606                 break;
2607         case RTE_CRYPTO_CIPHER_KASUMI_F8:
2608                 enc_type = KASUMI_F8_ECB;
2609                 cipher_key_len = 16;
2610                 zsk_flag = K_F8;
2611                 break;
2612         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
2613                 enc_type = SNOW3G_UEA2;
2614                 cipher_key_len = 16;
2615                 zsk_flag = ZS_EA;
2616                 break;
2617         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
2618                 enc_type = ZUC_EEA3;
2619                 cipher_key_len = 16;
2620                 zsk_flag = ZS_EA;
2621                 break;
2622         case RTE_CRYPTO_CIPHER_AES_XTS:
2623                 enc_type = AES_XTS;
2624                 cipher_key_len = 16;
2625                 break;
2626         case RTE_CRYPTO_CIPHER_3DES_ECB:
2627                 enc_type = DES3_ECB;
2628                 cipher_key_len = 24;
2629                 break;
2630         case RTE_CRYPTO_CIPHER_AES_ECB:
2631                 enc_type = AES_ECB;
2632                 cipher_key_len = 16;
2633                 break;
2634         case RTE_CRYPTO_CIPHER_3DES_CTR:
2635         case RTE_CRYPTO_CIPHER_AES_F8:
2636         case RTE_CRYPTO_CIPHER_ARC4:
2637                 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u",
2638                                c_form->algo);
2639                 return -1;
2640         default:
2641                 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified",
2642                                c_form->algo);
2643                 return -1;
2644         }
2645
2646         if (c_form->key.length < cipher_key_len) {
2647                 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu",
2648                                (unsigned long) c_form->key.length);
2649                 return -1;
2650         }
2651
2652         sess->zsk_flag = zsk_flag;
2653         sess->aes_gcm = 0;
2654         sess->aes_ctr = aes_ctr;
2655         sess->iv_offset = c_form->iv.offset;
2656         sess->iv_length = c_form->iv.length;
2657         sess->is_null = is_null;
2658
2659         if (unlikely(cpt_fc_ciph_set_key(SESS_PRIV(sess), enc_type,
2660                         c_form->key.data, c_form->key.length, NULL)))
2661                 return -1;
2662
2663         return 0;
2664 }
2665
2666 static __rte_always_inline int
2667 fill_sess_auth(struct rte_crypto_sym_xform *xform,
2668                struct cpt_sess_misc *sess)
2669 {
2670         struct rte_crypto_auth_xform *a_form;
2671         auth_type_t auth_type = 0; /* NULL Auth type */
2672         uint8_t zsk_flag = 0, aes_gcm = 0, is_null = 0;
2673
2674         a_form = &xform->auth;
2675
2676         if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY)
2677                 sess->cpt_op |= CPT_OP_AUTH_VERIFY;
2678         else if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE)
2679                 sess->cpt_op |= CPT_OP_AUTH_GENERATE;
2680         else {
2681                 CPT_LOG_DP_ERR("Unknown auth operation");
2682                 return -1;
2683         }
2684
2685         switch (a_form->algo) {
2686         case RTE_CRYPTO_AUTH_SHA1_HMAC:
2687                 /* Fall through */
2688         case RTE_CRYPTO_AUTH_SHA1:
2689                 auth_type = SHA1_TYPE;
2690                 break;
2691         case RTE_CRYPTO_AUTH_SHA256_HMAC:
2692         case RTE_CRYPTO_AUTH_SHA256:
2693                 auth_type = SHA2_SHA256;
2694                 break;
2695         case RTE_CRYPTO_AUTH_SHA512_HMAC:
2696         case RTE_CRYPTO_AUTH_SHA512:
2697                 auth_type = SHA2_SHA512;
2698                 break;
2699         case RTE_CRYPTO_AUTH_AES_GMAC:
2700                 auth_type = GMAC_TYPE;
2701                 aes_gcm = 1;
2702                 break;
2703         case RTE_CRYPTO_AUTH_SHA224_HMAC:
2704         case RTE_CRYPTO_AUTH_SHA224:
2705                 auth_type = SHA2_SHA224;
2706                 break;
2707         case RTE_CRYPTO_AUTH_SHA384_HMAC:
2708         case RTE_CRYPTO_AUTH_SHA384:
2709                 auth_type = SHA2_SHA384;
2710                 break;
2711         case RTE_CRYPTO_AUTH_MD5_HMAC:
2712         case RTE_CRYPTO_AUTH_MD5:
2713                 auth_type = MD5_TYPE;
2714                 break;
2715         case RTE_CRYPTO_AUTH_KASUMI_F9:
2716                 auth_type = KASUMI_F9_ECB;
2717                 /*
2718                  * Indicate that direction needs to be taken out
2719                  * from end of src
2720                  */
2721                 zsk_flag = K_F9;
2722                 break;
2723         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2724                 auth_type = SNOW3G_UIA2;
2725                 zsk_flag = ZS_IA;
2726                 break;
2727         case RTE_CRYPTO_AUTH_ZUC_EIA3:
2728                 auth_type = ZUC_EIA3;
2729                 zsk_flag = ZS_IA;
2730                 break;
2731         case RTE_CRYPTO_AUTH_NULL:
2732                 auth_type = 0;
2733                 is_null = 1;
2734                 break;
2735         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2736         case RTE_CRYPTO_AUTH_AES_CMAC:
2737         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2738                 CPT_LOG_DP_ERR("Crypto: Unsupported hash algo %u",
2739                                a_form->algo);
2740                 return -1;
2741         default:
2742                 CPT_LOG_DP_ERR("Crypto: Undefined Hash algo %u specified",
2743                                a_form->algo);
2744                 return -1;
2745         }
2746
2747         sess->zsk_flag = zsk_flag;
2748         sess->aes_gcm = aes_gcm;
2749         sess->mac_len = a_form->digest_length;
2750         sess->is_null = is_null;
2751         if (zsk_flag) {
2752                 sess->auth_iv_offset = a_form->iv.offset;
2753                 sess->auth_iv_length = a_form->iv.length;
2754         }
2755         if (unlikely(cpt_fc_auth_set_key(SESS_PRIV(sess), auth_type,
2756                         a_form->key.data, a_form->key.length,
2757                         a_form->digest_length)))
2758                 return -1;
2759
2760         return 0;
2761 }
2762
2763 static __rte_always_inline int
2764 fill_sess_gmac(struct rte_crypto_sym_xform *xform,
2765                  struct cpt_sess_misc *sess)
2766 {
2767         struct rte_crypto_auth_xform *a_form;
2768         cipher_type_t enc_type = 0; /* NULL Cipher type */
2769         auth_type_t auth_type = 0; /* NULL Auth type */
2770         void *ctx = SESS_PRIV(sess);
2771
2772         a_form = &xform->auth;
2773
2774         if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE)
2775                 sess->cpt_op |= CPT_OP_ENCODE;
2776         else if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY)
2777                 sess->cpt_op |= CPT_OP_DECODE;
2778         else {
2779                 CPT_LOG_DP_ERR("Unknown auth operation");
2780                 return -1;
2781         }
2782
2783         switch (a_form->algo) {
2784         case RTE_CRYPTO_AUTH_AES_GMAC:
2785                 enc_type = AES_GCM;
2786                 auth_type = GMAC_TYPE;
2787                 break;
2788         default:
2789                 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified",
2790                                a_form->algo);
2791                 return -1;
2792         }
2793
2794         sess->zsk_flag = 0;
2795         sess->aes_gcm = 0;
2796         sess->is_gmac = 1;
2797         sess->iv_offset = a_form->iv.offset;
2798         sess->iv_length = a_form->iv.length;
2799         sess->mac_len = a_form->digest_length;
2800
2801         if (unlikely(cpt_fc_ciph_set_key(ctx, enc_type, a_form->key.data,
2802                         a_form->key.length, NULL)))
2803                 return -1;
2804
2805         if (unlikely(cpt_fc_auth_set_key(ctx, auth_type, NULL, 0,
2806                         a_form->digest_length)))
2807                 return -1;
2808
2809         return 0;
2810 }
2811
2812 static __rte_always_inline void *
2813 alloc_op_meta(struct rte_mbuf *m_src,
2814               buf_ptr_t *buf,
2815               int32_t len,
2816               struct rte_mempool *cpt_meta_pool)
2817 {
2818         uint8_t *mdata;
2819
2820 #ifndef CPT_ALWAYS_USE_SEPARATE_BUF
2821         if (likely(m_src && (m_src->nb_segs == 1))) {
2822                 int32_t tailroom;
2823                 phys_addr_t mphys;
2824
2825                 /* Check if tailroom is sufficient to hold meta data */
2826                 tailroom = rte_pktmbuf_tailroom(m_src);
2827                 if (likely(tailroom > len + 8)) {
2828                         mdata = (uint8_t *)m_src->buf_addr + m_src->buf_len;
2829                         mphys = m_src->buf_iova + m_src->buf_len;
2830                         mdata -= len;
2831                         mphys -= len;
2832                         buf->vaddr = mdata;
2833                         buf->dma_addr = mphys;
2834                         buf->size = len;
2835                         /* Indicate that this is a mbuf allocated mdata */
2836                         mdata = (uint8_t *)((uint64_t)mdata | 1ull);
2837                         return mdata;
2838                 }
2839         }
2840 #else
2841         RTE_SET_USED(m_src);
2842 #endif
2843
2844         if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0))
2845                 return NULL;
2846
2847         buf->vaddr = mdata;
2848         buf->dma_addr = rte_mempool_virt2iova(mdata);
2849         buf->size = len;
2850
2851         return mdata;
2852 }
2853
2854 /**
2855  * cpt_free_metabuf - free metabuf to mempool.
2856  * @param instance: pointer to instance.
2857  * @param objp: pointer to the metabuf.
2858  */
2859 static __rte_always_inline void
2860 free_op_meta(void *mdata, struct rte_mempool *cpt_meta_pool)
2861 {
2862         bool nofree = ((uintptr_t)mdata & 1ull);
2863
2864         if (likely(nofree))
2865                 return;
2866         rte_mempool_put(cpt_meta_pool, mdata);
2867 }
2868
2869 static __rte_always_inline uint32_t
2870 prepare_iov_from_pkt(struct rte_mbuf *pkt,
2871                      iov_ptr_t *iovec, uint32_t start_offset)
2872 {
2873         uint16_t index = 0;
2874         void *seg_data = NULL;
2875         phys_addr_t seg_phys;
2876         int32_t seg_size = 0;
2877
2878         if (!pkt) {
2879                 iovec->buf_cnt = 0;
2880                 return 0;
2881         }
2882
2883         if (!start_offset) {
2884                 seg_data = rte_pktmbuf_mtod(pkt, void *);
2885                 seg_phys = rte_pktmbuf_iova(pkt);
2886                 seg_size = pkt->data_len;
2887         } else {
2888                 while (start_offset >= pkt->data_len) {
2889                         start_offset -= pkt->data_len;
2890                         pkt = pkt->next;
2891                 }
2892
2893                 seg_data = rte_pktmbuf_mtod_offset(pkt, void *, start_offset);
2894                 seg_phys = rte_pktmbuf_iova_offset(pkt, start_offset);
2895                 seg_size = pkt->data_len - start_offset;
2896                 if (!seg_size)
2897                         return 1;
2898         }
2899
2900         /* first seg */
2901         iovec->bufs[index].vaddr = seg_data;
2902         iovec->bufs[index].dma_addr = seg_phys;
2903         iovec->bufs[index].size = seg_size;
2904         index++;
2905         pkt = pkt->next;
2906
2907         while (unlikely(pkt != NULL)) {
2908                 seg_data = rte_pktmbuf_mtod(pkt, void *);
2909                 seg_phys = rte_pktmbuf_iova(pkt);
2910                 seg_size = pkt->data_len;
2911                 if (!seg_size)
2912                         break;
2913
2914                 iovec->bufs[index].vaddr = seg_data;
2915                 iovec->bufs[index].dma_addr = seg_phys;
2916                 iovec->bufs[index].size = seg_size;
2917
2918                 index++;
2919
2920                 pkt = pkt->next;
2921         }
2922
2923         iovec->buf_cnt = index;
2924         return 0;
2925 }
2926
2927 static __rte_always_inline uint32_t
2928 prepare_iov_from_pkt_inplace(struct rte_mbuf *pkt,
2929                              fc_params_t *param,
2930                              uint32_t *flags)
2931 {
2932         uint16_t index = 0;
2933         void *seg_data = NULL;
2934         phys_addr_t seg_phys;
2935         uint32_t seg_size = 0;
2936         iov_ptr_t *iovec;
2937
2938         seg_data = rte_pktmbuf_mtod(pkt, void *);
2939         seg_phys = rte_pktmbuf_iova(pkt);
2940         seg_size = pkt->data_len;
2941
2942         /* first seg */
2943         if (likely(!pkt->next)) {
2944                 uint32_t headroom, tailroom;
2945
2946                 *flags |= SINGLE_BUF_INPLACE;
2947                 headroom = rte_pktmbuf_headroom(pkt);
2948                 tailroom = rte_pktmbuf_tailroom(pkt);
2949                 if (likely((headroom >= 24) &&
2950                     (tailroom >= 8))) {
2951                         /* In 83XX this is prerequivisit for Direct mode */
2952                         *flags |= SINGLE_BUF_HEADTAILROOM;
2953                 }
2954                 param->bufs[0].vaddr = seg_data;
2955                 param->bufs[0].dma_addr = seg_phys;
2956                 param->bufs[0].size = seg_size;
2957                 return 0;
2958         }
2959         iovec = param->src_iov;
2960         iovec->bufs[index].vaddr = seg_data;
2961         iovec->bufs[index].dma_addr = seg_phys;
2962         iovec->bufs[index].size = seg_size;
2963         index++;
2964         pkt = pkt->next;
2965
2966         while (unlikely(pkt != NULL)) {
2967                 seg_data = rte_pktmbuf_mtod(pkt, void *);
2968                 seg_phys = rte_pktmbuf_iova(pkt);
2969                 seg_size = pkt->data_len;
2970
2971                 if (!seg_size)
2972                         break;
2973
2974                 iovec->bufs[index].vaddr = seg_data;
2975                 iovec->bufs[index].dma_addr = seg_phys;
2976                 iovec->bufs[index].size = seg_size;
2977
2978                 index++;
2979
2980                 pkt = pkt->next;
2981         }
2982
2983         iovec->buf_cnt = index;
2984         return 0;
2985 }
2986
2987 static __rte_always_inline int
2988 fill_fc_params(struct rte_crypto_op *cop,
2989                struct cpt_sess_misc *sess_misc,
2990                struct cpt_qp_meta_info *m_info,
2991                void **mdata_ptr,
2992                void **prep_req)
2993 {
2994         uint32_t space = 0;
2995         struct rte_crypto_sym_op *sym_op = cop->sym;
2996         void *mdata = NULL;
2997         uintptr_t *op;
2998         uint32_t mc_hash_off;
2999         uint32_t flags = 0;
3000         uint64_t d_offs, d_lens;
3001         struct rte_mbuf *m_src, *m_dst;
3002         uint8_t cpt_op = sess_misc->cpt_op;
3003 #ifdef CPT_ALWAYS_USE_SG_MODE
3004         uint8_t inplace = 0;
3005 #else
3006         uint8_t inplace = 1;
3007 #endif
3008         fc_params_t fc_params;
3009         char src[SRC_IOV_SIZE];
3010         char dst[SRC_IOV_SIZE];
3011         uint32_t iv_buf[4];
3012         int ret;
3013
3014         if (likely(sess_misc->iv_length)) {
3015                 flags |= VALID_IV_BUF;
3016                 fc_params.iv_buf = rte_crypto_op_ctod_offset(cop,
3017                                    uint8_t *, sess_misc->iv_offset);
3018                 if (sess_misc->aes_ctr &&
3019                     unlikely(sess_misc->iv_length != 16)) {
3020                         memcpy((uint8_t *)iv_buf,
3021                                 rte_crypto_op_ctod_offset(cop,
3022                                 uint8_t *, sess_misc->iv_offset), 12);
3023                         iv_buf[3] = rte_cpu_to_be_32(0x1);
3024                         fc_params.iv_buf = iv_buf;
3025                 }
3026         }
3027
3028         if (sess_misc->zsk_flag) {
3029                 fc_params.auth_iv_buf = rte_crypto_op_ctod_offset(cop,
3030                                         uint8_t *,
3031                                         sess_misc->auth_iv_offset);
3032                 if (sess_misc->zsk_flag != ZS_EA)
3033                         inplace = 0;
3034         }
3035         m_src = sym_op->m_src;
3036         m_dst = sym_op->m_dst;
3037
3038         if (sess_misc->aes_gcm || sess_misc->chacha_poly) {
3039                 uint8_t *salt;
3040                 uint8_t *aad_data;
3041                 uint16_t aad_len;
3042
3043                 d_offs = sym_op->aead.data.offset;
3044                 d_lens = sym_op->aead.data.length;
3045                 mc_hash_off = sym_op->aead.data.offset +
3046                               sym_op->aead.data.length;
3047
3048                 aad_data = sym_op->aead.aad.data;
3049                 aad_len = sess_misc->aad_length;
3050                 if (likely((aad_data + aad_len) ==
3051                            rte_pktmbuf_mtod_offset(m_src,
3052                                 uint8_t *,
3053                                 sym_op->aead.data.offset))) {
3054                         d_offs = (d_offs - aad_len) | (d_offs << 16);
3055                         d_lens = (d_lens + aad_len) | (d_lens << 32);
3056                 } else {
3057                         fc_params.aad_buf.vaddr = sym_op->aead.aad.data;
3058                         fc_params.aad_buf.dma_addr = sym_op->aead.aad.phys_addr;
3059                         fc_params.aad_buf.size = aad_len;
3060                         flags |= VALID_AAD_BUF;
3061                         inplace = 0;
3062                         d_offs = d_offs << 16;
3063                         d_lens = d_lens << 32;
3064                 }
3065
3066                 salt = fc_params.iv_buf;
3067                 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) {
3068                         cpt_fc_salt_update(SESS_PRIV(sess_misc), salt);
3069                         sess_misc->salt = *(uint32_t *)salt;
3070                 }
3071                 fc_params.iv_buf = salt + 4;
3072                 if (likely(sess_misc->mac_len)) {
3073                         struct rte_mbuf *m = (cpt_op & CPT_OP_ENCODE) ? m_dst :
3074                                              m_src;
3075
3076                         if (!m)
3077                                 m = m_src;
3078
3079                         /* hmac immediately following data is best case */
3080                         if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) +
3081                             mc_hash_off !=
3082                             (uint8_t *)sym_op->aead.digest.data)) {
3083                                 flags |= VALID_MAC_BUF;
3084                                 fc_params.mac_buf.size = sess_misc->mac_len;
3085                                 fc_params.mac_buf.vaddr =
3086                                   sym_op->aead.digest.data;
3087                                 fc_params.mac_buf.dma_addr =
3088                                  sym_op->aead.digest.phys_addr;
3089                                 inplace = 0;
3090                         }
3091                 }
3092         } else {
3093                 d_offs = sym_op->cipher.data.offset;
3094                 d_lens = sym_op->cipher.data.length;
3095                 mc_hash_off = sym_op->cipher.data.offset +
3096                               sym_op->cipher.data.length;
3097                 d_offs = (d_offs << 16) | sym_op->auth.data.offset;
3098                 d_lens = (d_lens << 32) | sym_op->auth.data.length;
3099
3100                 if (mc_hash_off < (sym_op->auth.data.offset +
3101                                    sym_op->auth.data.length)){
3102                         mc_hash_off = (sym_op->auth.data.offset +
3103                                        sym_op->auth.data.length);
3104                 }
3105                 /* for gmac, salt should be updated like in gcm */
3106                 if (unlikely(sess_misc->is_gmac)) {
3107                         uint8_t *salt;
3108                         salt = fc_params.iv_buf;
3109                         if (unlikely(*(uint32_t *)salt != sess_misc->salt)) {
3110                                 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt);
3111                                 sess_misc->salt = *(uint32_t *)salt;
3112                         }
3113                         fc_params.iv_buf = salt + 4;
3114                 }
3115                 if (likely(sess_misc->mac_len)) {
3116                         struct rte_mbuf *m;
3117
3118                         m = (cpt_op & CPT_OP_ENCODE) ? m_dst : m_src;
3119                         if (!m)
3120                                 m = m_src;
3121
3122                         /* hmac immediately following data is best case */
3123                         if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) +
3124                             mc_hash_off !=
3125                              (uint8_t *)sym_op->auth.digest.data)) {
3126                                 flags |= VALID_MAC_BUF;
3127                                 fc_params.mac_buf.size =
3128                                         sess_misc->mac_len;
3129                                 fc_params.mac_buf.vaddr =
3130                                         sym_op->auth.digest.data;
3131                                 fc_params.mac_buf.dma_addr =
3132                                 sym_op->auth.digest.phys_addr;
3133                                 inplace = 0;
3134                         }
3135                 }
3136         }
3137         fc_params.ctx_buf.vaddr = SESS_PRIV(sess_misc);
3138         fc_params.ctx_buf.dma_addr = sess_misc->ctx_dma_addr;
3139
3140         if (unlikely(sess_misc->is_null || sess_misc->cpt_op == CPT_OP_DECODE))
3141                 inplace = 0;
3142
3143         if (likely(!m_dst && inplace)) {
3144                 /* Case of single buffer without AAD buf or
3145                  * separate mac buf in place and
3146                  * not air crypto
3147                  */
3148                 fc_params.dst_iov = fc_params.src_iov = (void *)src;
3149
3150                 if (unlikely(prepare_iov_from_pkt_inplace(m_src,
3151                                                           &fc_params,
3152                                                           &flags))) {
3153                         CPT_LOG_DP_ERR("Prepare inplace src iov failed");
3154                         ret = -EINVAL;
3155                         goto err_exit;
3156                 }
3157
3158         } else {
3159                 /* Out of place processing */
3160                 fc_params.src_iov = (void *)src;
3161                 fc_params.dst_iov = (void *)dst;
3162
3163                 /* Store SG I/O in the api for reuse */
3164                 if (prepare_iov_from_pkt(m_src, fc_params.src_iov, 0)) {
3165                         CPT_LOG_DP_ERR("Prepare src iov failed");
3166                         ret = -EINVAL;
3167                         goto err_exit;
3168                 }
3169
3170                 if (unlikely(m_dst != NULL)) {
3171                         uint32_t pkt_len;
3172
3173                         /* Try to make room as much as src has */
3174                         pkt_len = rte_pktmbuf_pkt_len(m_dst);
3175
3176                         if (unlikely(pkt_len < rte_pktmbuf_pkt_len(m_src))) {
3177                                 pkt_len = rte_pktmbuf_pkt_len(m_src) - pkt_len;
3178                                 if (!rte_pktmbuf_append(m_dst, pkt_len)) {
3179                                         CPT_LOG_DP_ERR("Not enough space in "
3180                                                        "m_dst %p, need %u"
3181                                                        " more",
3182                                                        m_dst, pkt_len);
3183                                         ret = -EINVAL;
3184                                         goto err_exit;
3185                                 }
3186                         }
3187
3188                         if (prepare_iov_from_pkt(m_dst, fc_params.dst_iov, 0)) {
3189                                 CPT_LOG_DP_ERR("Prepare dst iov failed for "
3190                                                "m_dst %p", m_dst);
3191                                 ret = -EINVAL;
3192                                 goto err_exit;
3193                         }
3194                 } else {
3195                         fc_params.dst_iov = (void *)src;
3196                 }
3197         }
3198
3199         if (likely(flags & SINGLE_BUF_HEADTAILROOM))
3200                 mdata = alloc_op_meta(m_src, &fc_params.meta_buf,
3201                                       m_info->lb_mlen, m_info->pool);
3202         else
3203                 mdata = alloc_op_meta(NULL, &fc_params.meta_buf,
3204                                       m_info->sg_mlen, m_info->pool);
3205
3206         if (unlikely(mdata == NULL)) {
3207                 CPT_LOG_DP_ERR("Error allocating meta buffer for request");
3208                 ret = -ENOMEM;
3209                 goto err_exit;
3210         }
3211
3212         op = (uintptr_t *)((uintptr_t)mdata & (uintptr_t)~1ull);
3213         op[0] = (uintptr_t)mdata;
3214         op[1] = (uintptr_t)cop;
3215         op[2] = op[3] = 0; /* Used to indicate auth verify */
3216         space += 4 * sizeof(uint64_t);
3217
3218         fc_params.meta_buf.vaddr = (uint8_t *)op + space;
3219         fc_params.meta_buf.dma_addr += space;
3220         fc_params.meta_buf.size -= space;
3221
3222         /* Finally prepare the instruction */
3223         if (cpt_op & CPT_OP_ENCODE)
3224                 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens,
3225                                                  &fc_params, op);
3226         else
3227                 *prep_req = cpt_fc_dec_hmac_prep(flags, d_offs, d_lens,
3228                                                  &fc_params, op);
3229
3230         if (unlikely(*prep_req == NULL)) {
3231                 CPT_LOG_DP_ERR("Preparing request failed due to bad input arg");
3232                 ret = -EINVAL;
3233                 goto free_mdata_and_exit;
3234         }
3235
3236         *mdata_ptr = mdata;
3237
3238         return 0;
3239
3240 free_mdata_and_exit:
3241         free_op_meta(mdata, m_info->pool);
3242 err_exit:
3243         return ret;
3244 }
3245
3246 static __rte_always_inline void
3247 compl_auth_verify(struct rte_crypto_op *op,
3248                       uint8_t *gen_mac,
3249                       uint64_t mac_len)
3250 {
3251         uint8_t *mac;
3252         struct rte_crypto_sym_op *sym_op = op->sym;
3253
3254         if (sym_op->auth.digest.data)
3255                 mac = sym_op->auth.digest.data;
3256         else
3257                 mac = rte_pktmbuf_mtod_offset(sym_op->m_src,
3258                                               uint8_t *,
3259                                               sym_op->auth.data.length +
3260                                               sym_op->auth.data.offset);
3261         if (!mac) {
3262                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
3263                 return;
3264         }
3265
3266         if (memcmp(mac, gen_mac, mac_len))
3267                 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
3268         else
3269                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
3270 }
3271
3272 static __rte_always_inline void
3273 find_kasumif9_direction_and_length(uint8_t *src,
3274                                    uint32_t counter_num_bytes,
3275                                    uint32_t *addr_length_in_bits,
3276                                    uint8_t *addr_direction)
3277 {
3278         uint8_t found = 0;
3279         uint32_t pos;
3280         uint8_t last_byte;
3281         while (!found && counter_num_bytes > 0) {
3282                 counter_num_bytes--;
3283                 if (src[counter_num_bytes] == 0x00)
3284                         continue;
3285                 pos = rte_bsf32(src[counter_num_bytes]);
3286                 if (pos == 7) {
3287                         if (likely(counter_num_bytes > 0)) {
3288                                 last_byte = src[counter_num_bytes - 1];
3289                                 *addr_direction  =  last_byte & 0x1;
3290                                 *addr_length_in_bits = counter_num_bytes * 8
3291                                                         - 1;
3292                         }
3293                 } else {
3294                         last_byte = src[counter_num_bytes];
3295                         *addr_direction = (last_byte >> (pos + 1)) & 0x1;
3296                         *addr_length_in_bits = counter_num_bytes * 8
3297                                                 + (8 - (pos + 2));
3298                 }
3299                 found = 1;
3300         }
3301 }
3302
3303 /*
3304  * This handles all auth only except AES_GMAC
3305  */
3306 static __rte_always_inline int
3307 fill_digest_params(struct rte_crypto_op *cop,
3308                    struct cpt_sess_misc *sess,
3309                    struct cpt_qp_meta_info *m_info,
3310                    void **mdata_ptr,
3311                    void **prep_req)
3312 {
3313         uint32_t space = 0;
3314         struct rte_crypto_sym_op *sym_op = cop->sym;
3315         void *mdata;
3316         phys_addr_t mphys;
3317         uint64_t *op;
3318         uint32_t auth_range_off;
3319         uint32_t flags = 0;
3320         uint64_t d_offs = 0, d_lens;
3321         struct rte_mbuf *m_src, *m_dst;
3322         uint16_t auth_op = sess->cpt_op & CPT_OP_AUTH_MASK;
3323         uint16_t mac_len = sess->mac_len;
3324         fc_params_t params;
3325         char src[SRC_IOV_SIZE];
3326         uint8_t iv_buf[16];
3327         int ret;
3328
3329         memset(&params, 0, sizeof(fc_params_t));
3330
3331         m_src = sym_op->m_src;
3332
3333         /* For just digest lets force mempool alloc */
3334         mdata = alloc_op_meta(NULL, &params.meta_buf, m_info->sg_mlen,
3335                               m_info->pool);
3336         if (mdata == NULL) {
3337                 ret = -ENOMEM;
3338                 goto err_exit;
3339         }
3340
3341         mphys = params.meta_buf.dma_addr;
3342
3343         op = mdata;
3344         op[0] = (uintptr_t)mdata;
3345         op[1] = (uintptr_t)cop;
3346         op[2] = op[3] = 0; /* Used to indicate auth verify */
3347         space += 4 * sizeof(uint64_t);
3348
3349         auth_range_off = sym_op->auth.data.offset;
3350
3351         flags = VALID_MAC_BUF;
3352         params.src_iov = (void *)src;
3353         if (unlikely(sess->zsk_flag)) {
3354                 /*
3355                  * Since for Zuc, Kasumi, Snow3g offsets are in bits
3356                  * we will send pass through even for auth only case,
3357                  * let MC handle it
3358                  */
3359                 d_offs = auth_range_off;
3360                 auth_range_off = 0;
3361                 params.auth_iv_buf = rte_crypto_op_ctod_offset(cop,
3362                                         uint8_t *, sess->auth_iv_offset);
3363                 if (sess->zsk_flag == K_F9) {
3364                         uint32_t length_in_bits, num_bytes;
3365                         uint8_t *src, direction = 0;
3366
3367                         memcpy(iv_buf, rte_pktmbuf_mtod(cop->sym->m_src,
3368                                                         uint8_t *), 8);
3369                         /*
3370                          * This is kasumi f9, take direction from
3371                          * source buffer
3372                          */
3373                         length_in_bits = cop->sym->auth.data.length;
3374                         num_bytes = (length_in_bits >> 3);
3375                         src = rte_pktmbuf_mtod(cop->sym->m_src, uint8_t *);
3376                         find_kasumif9_direction_and_length(src,
3377                                                 num_bytes,
3378                                                 &length_in_bits,
3379                                                 &direction);
3380                         length_in_bits -= 64;
3381                         cop->sym->auth.data.offset += 64;
3382                         d_offs = cop->sym->auth.data.offset;
3383                         auth_range_off = d_offs / 8;
3384                         cop->sym->auth.data.length = length_in_bits;
3385
3386                         /* Store it at end of auth iv */
3387                         iv_buf[8] = direction;
3388                         params.auth_iv_buf = iv_buf;
3389                 }
3390         }
3391
3392         d_lens = sym_op->auth.data.length;
3393
3394         params.ctx_buf.vaddr = SESS_PRIV(sess);
3395         params.ctx_buf.dma_addr = sess->ctx_dma_addr;
3396
3397         if (auth_op == CPT_OP_AUTH_GENERATE) {
3398                 if (sym_op->auth.digest.data) {
3399                         /*
3400                          * Digest to be generated
3401                          * in separate buffer
3402                          */
3403                         params.mac_buf.size =
3404                                 sess->mac_len;
3405                         params.mac_buf.vaddr =
3406                                 sym_op->auth.digest.data;
3407                         params.mac_buf.dma_addr =
3408                                 sym_op->auth.digest.phys_addr;
3409                 } else {
3410                         uint32_t off = sym_op->auth.data.offset +
3411                                 sym_op->auth.data.length;
3412                         int32_t dlen, space;
3413
3414                         m_dst = sym_op->m_dst ?
3415                                 sym_op->m_dst : sym_op->m_src;
3416                         dlen = rte_pktmbuf_pkt_len(m_dst);
3417
3418                         space = off + mac_len - dlen;
3419                         if (space > 0)
3420                                 if (!rte_pktmbuf_append(m_dst, space)) {
3421                                         CPT_LOG_DP_ERR("Failed to extend "
3422                                                        "mbuf by %uB", space);
3423                                         ret = -EINVAL;
3424                                         goto free_mdata_and_exit;
3425                                 }
3426
3427                         params.mac_buf.vaddr =
3428                                 rte_pktmbuf_mtod_offset(m_dst, void *, off);
3429                         params.mac_buf.dma_addr =
3430                                 rte_pktmbuf_iova_offset(m_dst, off);
3431                         params.mac_buf.size = mac_len;
3432                 }
3433         } else {
3434                 /* Need space for storing generated mac */
3435                 params.mac_buf.vaddr = (uint8_t *)mdata + space;
3436                 params.mac_buf.dma_addr = mphys + space;
3437                 params.mac_buf.size = mac_len;
3438                 space += RTE_ALIGN_CEIL(mac_len, 8);
3439                 op[2] = (uintptr_t)params.mac_buf.vaddr;
3440                 op[3] = mac_len;
3441         }
3442
3443         params.meta_buf.vaddr = (uint8_t *)mdata + space;
3444         params.meta_buf.dma_addr = mphys + space;
3445         params.meta_buf.size -= space;
3446
3447         /* Out of place processing */
3448         params.src_iov = (void *)src;
3449
3450         /*Store SG I/O in the api for reuse */
3451         if (prepare_iov_from_pkt(m_src, params.src_iov, auth_range_off)) {
3452                 CPT_LOG_DP_ERR("Prepare src iov failed");
3453                 ret = -EINVAL;
3454                 goto free_mdata_and_exit;
3455         }
3456
3457         *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, &params, op);
3458         if (unlikely(*prep_req == NULL)) {
3459                 ret = -EINVAL;
3460                 goto free_mdata_and_exit;
3461         }
3462
3463         *mdata_ptr = mdata;
3464
3465         return 0;
3466
3467 free_mdata_and_exit:
3468         free_op_meta(mdata, m_info->pool);
3469 err_exit:
3470         return ret;
3471 }
3472
3473 #endif /*_CPT_UCODE_H_ */