drivers/common/cpt/cpt_ucode.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Cavium, Inc
   3  */
   4
   5 #ifndef _CPT_UCODE_H_
   6 #define _CPT_UCODE_H_
   7 #include <stdbool.h>
   8
   9 #include "cpt_common.h"
  10 #include "cpt_hw_types.h"
  11 #include "cpt_mcode_defines.h"
  12
  13 /*
  14  * This file defines functions that are interfaces to microcode spec.
  15  *
  16  */
  17
  18 static uint8_t zuc_d[32] = {
  19         0x44, 0xD7, 0x26, 0xBC, 0x62, 0x6B, 0x13, 0x5E,
  20         0x57, 0x89, 0x35, 0xE2, 0x71, 0x35, 0x09, 0xAF,
  21         0x4D, 0x78, 0x2F, 0x13, 0x6B, 0xC4, 0x1A, 0xF1,
  22         0x5E, 0x26, 0x3C, 0x4D, 0x78, 0x9A, 0x47, 0xAC
  23 };
  24
  25 static __rte_always_inline int
  26 cpt_is_algo_supported(struct rte_crypto_sym_xform *xform)
  27 {
  28         /*
  29          * Microcode only supports the following combination.
  30          * Encryption followed by authentication
  31          * Authentication followed by decryption
  32          */
  33         if (xform->next) {
  34                 if ((xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) &&
  35                     (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
  36                     (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)) {
  37                         /* Unsupported as of now by microcode */
  38                         CPT_LOG_DP_ERR("Unsupported combination");
  39                         return -1;
  40                 }
  41                 if ((xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) &&
  42                     (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) &&
  43                     (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT)) {
  44                         /* For GMAC auth there is no cipher operation */
  45                         if (xform->aead.algo != RTE_CRYPTO_AEAD_AES_GCM ||
  46                             xform->next->auth.algo !=
  47                             RTE_CRYPTO_AUTH_AES_GMAC) {
  48                                 /* Unsupported as of now by microcode */
  49                                 CPT_LOG_DP_ERR("Unsupported combination");
  50                                 return -1;
  51                         }
  52                 }
  53         }
  54         return 0;
  55 }
  56
  57 static __rte_always_inline void
  58 gen_key_snow3g(const uint8_t *ck, uint32_t *keyx)
  59 {
  60         int i, base;
  61
  62         for (i = 0; i < 4; i++) {
  63                 base = 4 * i;
  64                 keyx[3 - i] = (ck[base] << 24) | (ck[base + 1] << 16) |
  65                         (ck[base + 2] << 8) | (ck[base + 3]);
  66                 keyx[3 - i] = rte_cpu_to_be_32(keyx[3 - i]);
  67         }
  68 }
  69
  70 static __rte_always_inline void
  71 cpt_fc_salt_update(void *ctx,
  72                    uint8_t *salt)
  73 {
  74         struct cpt_ctx *cpt_ctx = ctx;
  75         memcpy(&cpt_ctx->fctx.enc.encr_iv, salt, 4);
  76 }
  77
  78 static __rte_always_inline int
  79 cpt_fc_ciph_validate_key_aes(uint16_t key_len)
  80 {
  81         switch (key_len) {
  82         case CPT_BYTE_16:
  83         case CPT_BYTE_24:
  84         case CPT_BYTE_32:
  85                 return 0;
  86         default:
  87                 return -1;
  88         }
  89 }
  90
  91 static __rte_always_inline int
  92 cpt_fc_ciph_set_type(cipher_type_t type, struct cpt_ctx *ctx, uint16_t key_len)
  93 {
  94         int fc_type = 0;
  95         switch (type) {
  96         case PASSTHROUGH:
  97                 fc_type = FC_GEN;
  98                 break;
  99         case DES3_CBC:
 100         case DES3_ECB:
 101                 fc_type = FC_GEN;
 102                 break;
 103         case AES_CBC:
 104         case AES_ECB:
 105         case AES_CFB:
 106         case AES_CTR:
 107         case AES_GCM:
 108                 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0))
 109                         return -1;
 110                 fc_type = FC_GEN;
 111                 break;
 112         case AES_XTS:
 113                 key_len = key_len / 2;
 114                 if (unlikely(key_len == CPT_BYTE_24)) {
 115                         CPT_LOG_DP_ERR("Invalid AES key len for XTS");
 116                         return -1;
 117                 }
 118                 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0))
 119                         return -1;
 120                 fc_type = FC_GEN;
 121                 break;
 122         case ZUC_EEA3:
 123         case SNOW3G_UEA2:
 124                 if (unlikely(key_len != 16))
 125                         return -1;
 126                 /* No support for AEAD yet */
 127                 if (unlikely(ctx->hash_type))
 128                         return -1;
 129                 fc_type = ZUC_SNOW3G;
 130                 break;
 131         case KASUMI_F8_CBC:
 132         case KASUMI_F8_ECB:
 133                 if (unlikely(key_len != 16))
 134                         return -1;
 135                 /* No support for AEAD yet */
 136                 if (unlikely(ctx->hash_type))
 137                         return -1;
 138                 fc_type = KASUMI;
 139                 break;
 140         default:
 141                 return -1;
 142         }
 143
 144         ctx->fc_type = fc_type;
 145         return 0;
 146 }
 147
 148 static __rte_always_inline void
 149 cpt_fc_ciph_set_key_passthrough(struct cpt_ctx *cpt_ctx, mc_fc_context_t *fctx)
 150 {
 151         cpt_ctx->enc_cipher = 0;
 152         fctx->enc.enc_cipher = 0;
 153 }
 154
 155 static __rte_always_inline void
 156 cpt_fc_ciph_set_key_set_aes_key_type(mc_fc_context_t *fctx, uint16_t key_len)
 157 {
 158         mc_aes_type_t aes_key_type = 0;
 159         switch (key_len) {
 160         case CPT_BYTE_16:
 161                 aes_key_type = AES_128_BIT;
 162                 break;
 163         case CPT_BYTE_24:
 164                 aes_key_type = AES_192_BIT;
 165                 break;
 166         case CPT_BYTE_32:
 167                 aes_key_type = AES_256_BIT;
 168                 break;
 169         default:
 170                 /* This should not happen */
 171                 CPT_LOG_DP_ERR("Invalid AES key len");
 172                 return;
 173         }
 174         fctx->enc.aes_key = aes_key_type;
 175 }
 176
 177 static __rte_always_inline void
 178 cpt_fc_ciph_set_key_snow3g_uea2(struct cpt_ctx *cpt_ctx, const uint8_t *key,
 179                 uint16_t key_len)
 180 {
 181         uint32_t keyx[4];
 182         cpt_ctx->snow3g = 1;
 183         gen_key_snow3g(key, keyx);
 184         memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len);
 185         cpt_ctx->zsk_flags = 0;
 186 }
 187
 188 static __rte_always_inline void
 189 cpt_fc_ciph_set_key_zuc_eea3(struct cpt_ctx *cpt_ctx, const uint8_t *key,
 190                 uint16_t key_len)
 191 {
 192         cpt_ctx->snow3g = 0;
 193         memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len);
 194         memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32);
 195         cpt_ctx->zsk_flags = 0;
 196 }
 197
 198 static __rte_always_inline void
 199 cpt_fc_ciph_set_key_kasumi_f8_ecb(struct cpt_ctx *cpt_ctx, const uint8_t *key,
 200                 uint16_t key_len)
 201 {
 202         cpt_ctx->k_ecb = 1;
 203         memcpy(cpt_ctx->k_ctx.ci_key, key, key_len);
 204         cpt_ctx->zsk_flags = 0;
 205 }
 206
 207 static __rte_always_inline void
 208 cpt_fc_ciph_set_key_kasumi_f8_cbc(struct cpt_ctx *cpt_ctx, const uint8_t *key,
 209                 uint16_t key_len)
 210 {
 211         memcpy(cpt_ctx->k_ctx.ci_key, key, key_len);
 212         cpt_ctx->zsk_flags = 0;
 213 }
 214
 215 static __rte_always_inline int
 216 cpt_fc_ciph_set_key(void *ctx, cipher_type_t type, const uint8_t *key,
 217                     uint16_t key_len, uint8_t *salt)
 218 {
 219         struct cpt_ctx *cpt_ctx = ctx;
 220         mc_fc_context_t *fctx = &cpt_ctx->fctx;
 221         int ret;
 222
 223         ret = cpt_fc_ciph_set_type(type, cpt_ctx, key_len);
 224         if (unlikely(ret))
 225                 return -1;
 226
 227         if (cpt_ctx->fc_type == FC_GEN) {
 228                 /*
 229                  * We need to always say IV is from DPTR as user can
 230                  * sometimes iverride IV per operation.
 231                  */
 232                 fctx->enc.iv_source = CPT_FROM_DPTR;
 233
 234                 if (cpt_ctx->auth_key_len > 64)
 235                         return -1;
 236         }
 237
 238         switch (type) {
 239         case PASSTHROUGH:
 240                 cpt_fc_ciph_set_key_passthrough(cpt_ctx, fctx);
 241                 goto success;
 242         case DES3_CBC:
 243                 /* CPT performs DES using 3DES with the 8B DES-key
 244                  * replicated 2 more times to match the 24B 3DES-key.
 245                  * Eg. If org. key is "0x0a 0x0b", then new key is
 246                  * "0x0a 0x0b 0x0a 0x0b 0x0a 0x0b"
 247                  */
 248                 if (key_len == 8) {
 249                         /* Skipping the first 8B as it will be copied
 250                          * in the regular code flow
 251                          */
 252                         memcpy(fctx->enc.encr_key+key_len, key, key_len);
 253                         memcpy(fctx->enc.encr_key+2*key_len, key, key_len);
 254                 }
 255                 break;
 256         case DES3_ECB:
 257                 /* For DES3_ECB IV need to be from CTX. */
 258                 fctx->enc.iv_source = CPT_FROM_CTX;
 259                 break;
 260         case AES_CBC:
 261         case AES_ECB:
 262         case AES_CFB:
 263         case AES_CTR:
 264                 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len);
 265                 break;
 266         case AES_GCM:
 267                 /* Even though iv source is from dptr,
 268                  * aes_gcm salt is taken from ctx
 269                  */
 270                 if (salt) {
 271                         memcpy(fctx->enc.encr_iv, salt, 4);
 272                         /* Assuming it was just salt update
 273                          * and nothing else
 274                          */
 275                         if (!key)
 276                                 goto success;
 277                 }
 278                 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len);
 279                 break;
 280         case AES_XTS:
 281                 key_len = key_len / 2;
 282                 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len);
 283
 284                 /* Copy key2 for XTS into ipad */
 285                 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad));
 286                 memcpy(fctx->hmac.ipad, &key[key_len], key_len);
 287                 break;
 288         case SNOW3G_UEA2:
 289                 cpt_fc_ciph_set_key_snow3g_uea2(cpt_ctx, key, key_len);
 290                 goto success;
 291         case ZUC_EEA3:
 292                 cpt_fc_ciph_set_key_zuc_eea3(cpt_ctx, key, key_len);
 293                 goto success;
 294         case KASUMI_F8_ECB:
 295                 cpt_fc_ciph_set_key_kasumi_f8_ecb(cpt_ctx, key, key_len);
 296                 goto success;
 297         case KASUMI_F8_CBC:
 298                 cpt_fc_ciph_set_key_kasumi_f8_cbc(cpt_ctx, key, key_len);
 299                 goto success;
 300         default:
 301                 break;
 302         }
 303
 304         /* Only for FC_GEN case */
 305
 306         /* For GMAC auth, cipher must be NULL */
 307         if (cpt_ctx->hash_type != GMAC_TYPE)
 308                 fctx->enc.enc_cipher = type;
 309
 310         memcpy(fctx->enc.encr_key, key, key_len);
 311
 312 success:
 313         cpt_ctx->enc_cipher = type;
 314
 315         return 0;
 316 }
 317
 318 static __rte_always_inline uint32_t
 319 fill_sg_comp(sg_comp_t *list,
 320              uint32_t i,
 321              phys_addr_t dma_addr,
 322              uint32_t size)
 323 {
 324         sg_comp_t *to = &list[i>>2];
 325
 326         to->u.s.len[i%4] = rte_cpu_to_be_16(size);
 327         to->ptr[i%4] = rte_cpu_to_be_64(dma_addr);
 328         i++;
 329         return i;
 330 }
 331
 332 static __rte_always_inline uint32_t
 333 fill_sg_comp_from_buf(sg_comp_t *list,
 334                       uint32_t i,
 335                       buf_ptr_t *from)
 336 {
 337         sg_comp_t *to = &list[i>>2];
 338
 339         to->u.s.len[i%4] = rte_cpu_to_be_16(from->size);
 340         to->ptr[i%4] = rte_cpu_to_be_64(from->dma_addr);
 341         i++;
 342         return i;
 343 }
 344
 345 static __rte_always_inline uint32_t
 346 fill_sg_comp_from_buf_min(sg_comp_t *list,
 347                           uint32_t i,
 348                           buf_ptr_t *from,
 349                           uint32_t *psize)
 350 {
 351         sg_comp_t *to = &list[i >> 2];
 352         uint32_t size = *psize;
 353         uint32_t e_len;
 354
 355         e_len = (size > from->size) ? from->size : size;
 356         to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
 357         to->ptr[i % 4] = rte_cpu_to_be_64(from->dma_addr);
 358         *psize -= e_len;
 359         i++;
 360         return i;
 361 }
 362
 363 /*
 364  * This fills the MC expected SGIO list
 365  * from IOV given by user.
 366  */
 367 static __rte_always_inline uint32_t
 368 fill_sg_comp_from_iov(sg_comp_t *list,
 369                       uint32_t i,
 370                       iov_ptr_t *from, uint32_t from_offset,
 371                       uint32_t *psize, buf_ptr_t *extra_buf,
 372                       uint32_t extra_offset)
 373 {
 374         int32_t j;
 375         uint32_t extra_len = extra_buf ? extra_buf->size : 0;
 376         uint32_t size = *psize - extra_len;
 377         buf_ptr_t *bufs;
 378
 379         bufs = from->bufs;
 380         for (j = 0; (j < from->buf_cnt) && size; j++) {
 381                 phys_addr_t e_dma_addr;
 382                 uint32_t e_len;
 383                 sg_comp_t *to = &list[i >> 2];
 384
 385                 if (!bufs[j].size)
 386                         continue;
 387
 388                 if (unlikely(from_offset)) {
 389                         if (from_offset >= bufs[j].size) {
 390                                 from_offset -= bufs[j].size;
 391                                 continue;
 392                         }
 393                         e_dma_addr = bufs[j].dma_addr + from_offset;
 394                         e_len = (size > (bufs[j].size - from_offset)) ?
 395                                 (bufs[j].size - from_offset) : size;
 396                         from_offset = 0;
 397                 } else {
 398                         e_dma_addr = bufs[j].dma_addr;
 399                         e_len = (size > bufs[j].size) ?
 400                                 bufs[j].size : size;
 401                 }
 402
 403                 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
 404                 to->ptr[i % 4] = rte_cpu_to_be_64(e_dma_addr);
 405
 406                 if (extra_len && (e_len >= extra_offset)) {
 407                         /* Break the data at given offset */
 408                         uint32_t next_len = e_len - extra_offset;
 409                         phys_addr_t next_dma = e_dma_addr + extra_offset;
 410
 411                         if (!extra_offset) {
 412                                 i--;
 413                         } else {
 414                                 e_len = extra_offset;
 415                                 size -= e_len;
 416                                 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len);
 417                         }
 418
 419                         /* Insert extra data ptr */
 420                         if (extra_len) {
 421                                 i++;
 422                                 to = &list[i >> 2];
 423                                 to->u.s.len[i % 4] =
 424                                         rte_cpu_to_be_16(extra_buf->size);
 425                                 to->ptr[i % 4] =
 426                                         rte_cpu_to_be_64(extra_buf->dma_addr);
 427
 428                                 /* size already decremented by extra len */
 429                         }
 430
 431                         /* insert the rest of the data */
 432                         if (next_len) {
 433                                 i++;
 434                                 to = &list[i >> 2];
 435                                 to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len);
 436                                 to->ptr[i % 4] = rte_cpu_to_be_64(next_dma);
 437                                 size -= next_len;
 438                         }
 439                         extra_len = 0;
 440
 441                 } else {
 442                         size -= e_len;
 443                 }
 444                 if (extra_offset)
 445                         extra_offset -= size;
 446                 i++;
 447         }
 448
 449         *psize = size;
 450         return (uint32_t)i;
 451 }
 452
 453 static __rte_always_inline void
 454 cpt_digest_gen_prep(uint32_t flags,
 455                     uint64_t d_lens,
 456                     digest_params_t *params,
 457                     void *op,
 458                     void **prep_req)
 459 {
 460         struct cpt_request_info *req;
 461         uint32_t size, i;
 462         uint16_t data_len, mac_len, key_len;
 463         auth_type_t hash_type;
 464         buf_ptr_t *meta_p;
 465         struct cpt_ctx *ctx;
 466         sg_comp_t *gather_comp;
 467         sg_comp_t *scatter_comp;
 468         uint8_t *in_buffer;
 469         uint32_t g_size_bytes, s_size_bytes;
 470         uint64_t dptr_dma, rptr_dma;
 471         vq_cmd_word0_t vq_cmd_w0;
 472         vq_cmd_word3_t vq_cmd_w3;
 473         void *c_vaddr, *m_vaddr;
 474         uint64_t c_dma, m_dma;
 475         opcode_info_t opcode;
 476
 477         ctx = params->ctx_buf.vaddr;
 478         meta_p = &params->meta_buf;
 479
 480         m_vaddr = meta_p->vaddr;
 481         m_dma = meta_p->dma_addr;
 482
 483         /*
 484          * Save initial space that followed app data for completion code &
 485          * alternate completion code to fall in same cache line as app data
 486          */
 487         m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
 488         m_dma += COMPLETION_CODE_SIZE;
 489         size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
 490                 (uint8_t *)m_vaddr;
 491         c_vaddr = (uint8_t *)m_vaddr + size;
 492         c_dma = m_dma + size;
 493         size += sizeof(cpt_res_s_t);
 494
 495         m_vaddr = (uint8_t *)m_vaddr + size;
 496         m_dma += size;
 497
 498         req = m_vaddr;
 499
 500         size = sizeof(struct cpt_request_info);
 501         m_vaddr = (uint8_t *)m_vaddr + size;
 502         m_dma += size;
 503
 504         hash_type = ctx->hash_type;
 505         mac_len = ctx->mac_len;
 506         key_len = ctx->auth_key_len;
 507         data_len = AUTH_DLEN(d_lens);
 508
 509         /*GP op header */
 510         vq_cmd_w0.u64 = 0;
 511         vq_cmd_w0.s.param2 = ((uint16_t)hash_type << 8);
 512         if (ctx->hmac) {
 513                 opcode.s.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE;
 514                 vq_cmd_w0.s.param1 = key_len;
 515                 vq_cmd_w0.s.dlen = data_len + ROUNDUP8(key_len);
 516         } else {
 517                 opcode.s.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE;
 518                 vq_cmd_w0.s.param1 = 0;
 519                 vq_cmd_w0.s.dlen = data_len;
 520         }
 521
 522         opcode.s.minor = 0;
 523
 524         /* Null auth only case enters the if */
 525         if (unlikely(!hash_type && !ctx->enc_cipher)) {
 526                 opcode.s.major = CPT_MAJOR_OP_MISC;
 527                 /* Minor op is passthrough */
 528                 opcode.s.minor = 0x03;
 529                 /* Send out completion code only */
 530                 vq_cmd_w0.s.param2 = 0x1;
 531         }
 532
 533         vq_cmd_w0.s.opcode = opcode.flags;
 534
 535         /* DPTR has SG list */
 536         in_buffer = m_vaddr;
 537         dptr_dma = m_dma;
 538
 539         ((uint16_t *)in_buffer)[0] = 0;
 540         ((uint16_t *)in_buffer)[1] = 0;
 541
 542         /* TODO Add error check if space will be sufficient */
 543         gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
 544
 545         /*
 546          * Input gather list
 547          */
 548
 549         i = 0;
 550
 551         if (ctx->hmac) {
 552                 uint64_t k_dma = params->ctx_buf.dma_addr +
 553                         offsetof(struct cpt_ctx, auth_key);
 554                 /* Key */
 555                 i = fill_sg_comp(gather_comp, i, k_dma, ROUNDUP8(key_len));
 556         }
 557
 558         /* input data */
 559         size = data_len;
 560         if (size) {
 561                 i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov,
 562                                           0, &size, NULL, 0);
 563                 if (unlikely(size)) {
 564                         CPT_LOG_DP_DEBUG("Insufficient dst IOV size, short"
 565                                          " by %dB", size);
 566                         return;
 567                 }
 568         } else {
 569                 /*
 570                  * Looks like we need to support zero data
 571                  * gather ptr in case of hash & hmac
 572                  */
 573                 i++;
 574         }
 575         ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
 576         g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
 577
 578         /*
 579          * Output Gather list
 580          */
 581
 582         i = 0;
 583         scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
 584
 585         if (flags & VALID_MAC_BUF) {
 586                 if (unlikely(params->mac_buf.size < mac_len)) {
 587                         CPT_LOG_DP_ERR("Insufficient MAC size");
 588                         return;
 589                 }
 590
 591                 size = mac_len;
 592                 i = fill_sg_comp_from_buf_min(scatter_comp, i,
 593                                               &params->mac_buf, &size);
 594         } else {
 595                 size = mac_len;
 596                 i = fill_sg_comp_from_iov(scatter_comp, i,
 597                                           params->src_iov, data_len,
 598                                           &size, NULL, 0);
 599                 if (unlikely(size)) {
 600                         CPT_LOG_DP_ERR("Insufficient dst IOV size, short by"
 601                                        " %dB", size);
 602                         return;
 603                 }
 604         }
 605
 606         ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
 607         s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
 608
 609         size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
 610
 611         /* This is DPTR len incase of SG mode */
 612         vq_cmd_w0.s.dlen = size;
 613
 614         m_vaddr = (uint8_t *)m_vaddr + size;
 615         m_dma += size;
 616
 617         /* cpt alternate completion address saved earlier */
 618         req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
 619         *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
 620         rptr_dma = c_dma - 8;
 621
 622         req->ist.ei1 = dptr_dma;
 623         req->ist.ei2 = rptr_dma;
 624
 625         /* vq command w3 */
 626         vq_cmd_w3.u64 = 0;
 627
 628         /* 16 byte aligned cpt res address */
 629         req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
 630         *req->completion_addr = COMPLETION_CODE_INIT;
 631         req->comp_baddr  = c_dma;
 632
 633         /* Fill microcode part of instruction */
 634         req->ist.ei0 = vq_cmd_w0.u64;
 635         req->ist.ei3 = vq_cmd_w3.u64;
 636
 637         req->op = op;
 638
 639         *prep_req = req;
 640         return;
 641 }
 642
 643 static __rte_always_inline void
 644 cpt_enc_hmac_prep(uint32_t flags,
 645                   uint64_t d_offs,
 646                   uint64_t d_lens,
 647                   fc_params_t *fc_params,
 648                   void *op,
 649                   void **prep_req)
 650 {
 651         uint32_t iv_offset = 0;
 652         int32_t inputlen, outputlen, enc_dlen, auth_dlen;
 653         struct cpt_ctx *cpt_ctx;
 654         uint32_t cipher_type, hash_type;
 655         uint32_t mac_len, size;
 656         uint8_t iv_len = 16;
 657         struct cpt_request_info *req;
 658         buf_ptr_t *meta_p, *aad_buf = NULL;
 659         uint32_t encr_offset, auth_offset;
 660         uint32_t encr_data_len, auth_data_len, aad_len = 0;
 661         uint32_t passthrough_len = 0;
 662         void *m_vaddr, *offset_vaddr;
 663         uint64_t m_dma, offset_dma, ctx_dma;
 664         vq_cmd_word0_t vq_cmd_w0;
 665         vq_cmd_word3_t vq_cmd_w3;
 666         void *c_vaddr;
 667         uint64_t c_dma;
 668         opcode_info_t opcode;
 669
 670         meta_p = &fc_params->meta_buf;
 671         m_vaddr = meta_p->vaddr;
 672         m_dma = meta_p->dma_addr;
 673
 674         encr_offset = ENCR_OFFSET(d_offs);
 675         auth_offset = AUTH_OFFSET(d_offs);
 676         encr_data_len = ENCR_DLEN(d_lens);
 677         auth_data_len = AUTH_DLEN(d_lens);
 678         if (unlikely(flags & VALID_AAD_BUF)) {
 679                 /*
 680                  * We dont support both aad
 681                  * and auth data separately
 682                  */
 683                 auth_data_len = 0;
 684                 auth_offset = 0;
 685                 aad_len = fc_params->aad_buf.size;
 686                 aad_buf = &fc_params->aad_buf;
 687         }
 688         cpt_ctx = fc_params->ctx_buf.vaddr;
 689         cipher_type = cpt_ctx->enc_cipher;
 690         hash_type = cpt_ctx->hash_type;
 691         mac_len = cpt_ctx->mac_len;
 692
 693         /*
 694          * Save initial space that followed app data for completion code &
 695          * alternate completion code to fall in same cache line as app data
 696          */
 697         m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
 698         m_dma += COMPLETION_CODE_SIZE;
 699         size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
 700                 (uint8_t *)m_vaddr;
 701
 702         c_vaddr = (uint8_t *)m_vaddr + size;
 703         c_dma = m_dma + size;
 704         size += sizeof(cpt_res_s_t);
 705
 706         m_vaddr = (uint8_t *)m_vaddr + size;
 707         m_dma += size;
 708
 709         /* start cpt request info struct at 8 byte boundary */
 710         size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) -
 711                 (uint8_t *)m_vaddr;
 712
 713         req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size);
 714
 715         size += sizeof(struct cpt_request_info);
 716         m_vaddr = (uint8_t *)m_vaddr + size;
 717         m_dma += size;
 718
 719         if (hash_type == GMAC_TYPE)
 720                 encr_data_len = 0;
 721
 722         if (unlikely(!(flags & VALID_IV_BUF))) {
 723                 iv_len = 0;
 724                 iv_offset = ENCR_IV_OFFSET(d_offs);
 725         }
 726
 727         if (unlikely(flags & VALID_AAD_BUF)) {
 728                 /*
 729                  * When AAD is given, data above encr_offset is pass through
 730                  * Since AAD is given as separate pointer and not as offset,
 731                  * this is a special case as we need to fragment input data
 732                  * into passthrough + encr_data and then insert AAD in between.
 733                  */
 734                 if (hash_type != GMAC_TYPE) {
 735                         passthrough_len = encr_offset;
 736                         auth_offset = passthrough_len + iv_len;
 737                         encr_offset = passthrough_len + aad_len + iv_len;
 738                         auth_data_len = aad_len + encr_data_len;
 739                 } else {
 740                         passthrough_len = 16 + aad_len;
 741                         auth_offset = passthrough_len + iv_len;
 742                         auth_data_len = aad_len;
 743                 }
 744         } else {
 745                 encr_offset += iv_len;
 746                 auth_offset += iv_len;
 747         }
 748
 749         /* Encryption */
 750         opcode.s.major = CPT_MAJOR_OP_FC;
 751         opcode.s.minor = 0;
 752
 753         auth_dlen = auth_offset + auth_data_len;
 754         enc_dlen = encr_data_len + encr_offset;
 755         if (unlikely(encr_data_len & 0xf)) {
 756                 if ((cipher_type == DES3_CBC) || (cipher_type == DES3_ECB))
 757                         enc_dlen = ROUNDUP8(encr_data_len) + encr_offset;
 758                 else if (likely((cipher_type == AES_CBC) ||
 759                                 (cipher_type == AES_ECB)))
 760                         enc_dlen = ROUNDUP16(encr_data_len) + encr_offset;
 761         }
 762
 763         if (unlikely(hash_type == GMAC_TYPE)) {
 764                 encr_offset = auth_dlen;
 765                 enc_dlen = 0;
 766         }
 767
 768         if (unlikely(auth_dlen > enc_dlen)) {
 769                 inputlen = auth_dlen;
 770                 outputlen = auth_dlen + mac_len;
 771         } else {
 772                 inputlen = enc_dlen;
 773                 outputlen = enc_dlen + mac_len;
 774         }
 775
 776         /* GP op header */
 777         vq_cmd_w0.u64 = 0;
 778         vq_cmd_w0.s.param1 = encr_data_len;
 779         vq_cmd_w0.s.param2 = auth_data_len;
 780         /*
 781          * In 83XX since we have a limitation of
 782          * IV & Offset control word not part of instruction
 783          * and need to be part of Data Buffer, we check if
 784          * head room is there and then only do the Direct mode processing
 785          */
 786         if (likely((flags & SINGLE_BUF_INPLACE) &&
 787                    (flags & SINGLE_BUF_HEADTAILROOM))) {
 788                 void *dm_vaddr = fc_params->bufs[0].vaddr;
 789                 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr;
 790                 /*
 791                  * This flag indicates that there is 24 bytes head room and
 792                  * 8 bytes tail room available, so that we get to do
 793                  * DIRECT MODE with limitation
 794                  */
 795
 796                 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len;
 797                 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len;
 798
 799                 /* DPTR */
 800                 req->ist.ei1 = offset_dma;
 801                 /* RPTR should just exclude offset control word */
 802                 req->ist.ei2 = dm_dma_addr - iv_len;
 803                 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr
 804                                                     + outputlen - iv_len);
 805
 806                 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN;
 807
 808                 vq_cmd_w0.s.opcode = opcode.flags;
 809
 810                 if (likely(iv_len)) {
 811                         uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr
 812                                                       + OFF_CTRL_LEN);
 813                         uint64_t *src = fc_params->iv_buf;
 814                         dest[0] = src[0];
 815                         dest[1] = src[1];
 816                 }
 817
 818                 *(uint64_t *)offset_vaddr =
 819                         rte_cpu_to_be_64(((uint64_t)encr_offset << 16) |
 820                                 ((uint64_t)iv_offset << 8) |
 821                                 ((uint64_t)auth_offset));
 822
 823         } else {
 824                 uint32_t i, g_size_bytes, s_size_bytes;
 825                 uint64_t dptr_dma, rptr_dma;
 826                 sg_comp_t *gather_comp;
 827                 sg_comp_t *scatter_comp;
 828                 uint8_t *in_buffer;
 829
 830                 /* This falls under strict SG mode */
 831                 offset_vaddr = m_vaddr;
 832                 offset_dma = m_dma;
 833                 size = OFF_CTRL_LEN + iv_len;
 834
 835                 m_vaddr = (uint8_t *)m_vaddr + size;
 836                 m_dma += size;
 837
 838                 opcode.s.major |= CPT_DMA_MODE;
 839
 840                 vq_cmd_w0.s.opcode = opcode.flags;
 841
 842                 if (likely(iv_len)) {
 843                         uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr
 844                                                       + OFF_CTRL_LEN);
 845                         uint64_t *src = fc_params->iv_buf;
 846                         dest[0] = src[0];
 847                         dest[1] = src[1];
 848                 }
 849
 850                 *(uint64_t *)offset_vaddr =
 851                         rte_cpu_to_be_64(((uint64_t)encr_offset << 16) |
 852                                 ((uint64_t)iv_offset << 8) |
 853                                 ((uint64_t)auth_offset));
 854
 855                 /* DPTR has SG list */
 856                 in_buffer = m_vaddr;
 857                 dptr_dma = m_dma;
 858
 859                 ((uint16_t *)in_buffer)[0] = 0;
 860                 ((uint16_t *)in_buffer)[1] = 0;
 861
 862                 /* TODO Add error check if space will be sufficient */
 863                 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
 864
 865                 /*
 866                  * Input Gather List
 867                  */
 868
 869                 i = 0;
 870
 871                 /* Offset control word that includes iv */
 872                 i = fill_sg_comp(gather_comp, i, offset_dma,
 873                                  OFF_CTRL_LEN + iv_len);
 874
 875                 /* Add input data */
 876                 size = inputlen - iv_len;
 877                 if (likely(size)) {
 878                         uint32_t aad_offset = aad_len ? passthrough_len : 0;
 879
 880                         if (unlikely(flags & SINGLE_BUF_INPLACE)) {
 881                                 i = fill_sg_comp_from_buf_min(gather_comp, i,
 882                                                               fc_params->bufs,
 883                                                               &size);
 884                         } else {
 885                                 i = fill_sg_comp_from_iov(gather_comp, i,
 886                                                           fc_params->src_iov,
 887                                                           0, &size,
 888                                                           aad_buf, aad_offset);
 889                         }
 890
 891                         if (unlikely(size)) {
 892                                 CPT_LOG_DP_ERR("Insufficient buffer space,"
 893                                                " size %d needed", size);
 894                                 return;
 895                         }
 896                 }
 897                 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
 898                 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
 899
 900                 /*
 901                  * Output Scatter list
 902                  */
 903                 i = 0;
 904                 scatter_comp =
 905                         (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
 906
 907                 /* Add IV */
 908                 if (likely(iv_len)) {
 909                         i = fill_sg_comp(scatter_comp, i,
 910                                          offset_dma + OFF_CTRL_LEN,
 911                                          iv_len);
 912                 }
 913
 914                 /* output data or output data + digest*/
 915                 if (unlikely(flags & VALID_MAC_BUF)) {
 916                         size = outputlen - iv_len - mac_len;
 917                         if (size) {
 918                                 uint32_t aad_offset =
 919                                         aad_len ? passthrough_len : 0;
 920
 921                                 if (unlikely(flags & SINGLE_BUF_INPLACE)) {
 922                                         i = fill_sg_comp_from_buf_min(
 923                                                         scatter_comp,
 924                                                         i,
 925                                                         fc_params->bufs,
 926                                                         &size);
 927                                 } else {
 928                                         i = fill_sg_comp_from_iov(scatter_comp,
 929                                                         i,
 930                                                         fc_params->dst_iov,
 931                                                         0,
 932                                                         &size,
 933                                                         aad_buf,
 934                                                         aad_offset);
 935                                 }
 936                                 if (unlikely(size)) {
 937                                         CPT_LOG_DP_ERR("Insufficient buffer"
 938                                                        " space, size %d needed",
 939                                                        size);
 940                                         return;
 941                                 }
 942                         }
 943                         /* mac_data */
 944                         if (mac_len) {
 945                                 i = fill_sg_comp_from_buf(scatter_comp, i,
 946                                                           &fc_params->mac_buf);
 947                         }
 948                 } else {
 949                         /* Output including mac */
 950                         size = outputlen - iv_len;
 951                         if (likely(size)) {
 952                                 uint32_t aad_offset =
 953                                         aad_len ? passthrough_len : 0;
 954
 955                                 if (unlikely(flags & SINGLE_BUF_INPLACE)) {
 956                                         i = fill_sg_comp_from_buf_min(
 957                                                         scatter_comp,
 958                                                         i,
 959                                                         fc_params->bufs,
 960                                                         &size);
 961                                 } else {
 962                                         i = fill_sg_comp_from_iov(scatter_comp,
 963                                                         i,
 964                                                         fc_params->dst_iov,
 965                                                         0,
 966                                                         &size,
 967                                                         aad_buf,
 968                                                         aad_offset);
 969                                 }
 970                                 if (unlikely(size)) {
 971                                         CPT_LOG_DP_ERR("Insufficient buffer"
 972                                                        " space, size %d needed",
 973                                                        size);
 974                                         return;
 975                                 }
 976                         }
 977                 }
 978                 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
 979                 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
 980
 981                 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
 982
 983                 /* This is DPTR len incase of SG mode */
 984                 vq_cmd_w0.s.dlen = size;
 985
 986                 m_vaddr = (uint8_t *)m_vaddr + size;
 987                 m_dma += size;
 988
 989                 /* cpt alternate completion address saved earlier */
 990                 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
 991                 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
 992                 rptr_dma = c_dma - 8;
 993
 994                 req->ist.ei1 = dptr_dma;
 995                 req->ist.ei2 = rptr_dma;
 996         }
 997
 998         ctx_dma = fc_params->ctx_buf.dma_addr +
 999                 offsetof(struct cpt_ctx, fctx);
1000         /* vq command w3 */
1001         vq_cmd_w3.u64 = 0;
1002         vq_cmd_w3.s.grp = 0;
1003         vq_cmd_w3.s.cptr = ctx_dma;
1004
1005         /* 16 byte aligned cpt res address */
1006         req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
1007         *req->completion_addr = COMPLETION_CODE_INIT;
1008         req->comp_baddr  = c_dma;
1009
1010         /* Fill microcode part of instruction */
1011         req->ist.ei0 = vq_cmd_w0.u64;
1012         req->ist.ei3 = vq_cmd_w3.u64;
1013
1014         req->op  = op;
1015
1016         *prep_req = req;
1017         return;
1018 }
1019
1020 static __rte_always_inline void
1021 cpt_dec_hmac_prep(uint32_t flags,
1022                   uint64_t d_offs,
1023                   uint64_t d_lens,
1024                   fc_params_t *fc_params,
1025                   void *op,
1026                   void **prep_req)
1027 {
1028         uint32_t iv_offset = 0, size;
1029         int32_t inputlen, outputlen, enc_dlen, auth_dlen;
1030         struct cpt_ctx *cpt_ctx;
1031         int32_t hash_type, mac_len;
1032         uint8_t iv_len = 16;
1033         struct cpt_request_info *req;
1034         buf_ptr_t *meta_p, *aad_buf = NULL;
1035         uint32_t encr_offset, auth_offset;
1036         uint32_t encr_data_len, auth_data_len, aad_len = 0;
1037         uint32_t passthrough_len = 0;
1038         void *m_vaddr, *offset_vaddr;
1039         uint64_t m_dma, offset_dma, ctx_dma;
1040         opcode_info_t opcode;
1041         vq_cmd_word0_t vq_cmd_w0;
1042         vq_cmd_word3_t vq_cmd_w3;
1043         void *c_vaddr;
1044         uint64_t c_dma;
1045
1046         meta_p = &fc_params->meta_buf;
1047         m_vaddr = meta_p->vaddr;
1048         m_dma = meta_p->dma_addr;
1049
1050         encr_offset = ENCR_OFFSET(d_offs);
1051         auth_offset = AUTH_OFFSET(d_offs);
1052         encr_data_len = ENCR_DLEN(d_lens);
1053         auth_data_len = AUTH_DLEN(d_lens);
1054
1055         if (unlikely(flags & VALID_AAD_BUF)) {
1056                 /*
1057                  * We dont support both aad
1058                  * and auth data separately
1059                  */
1060                 auth_data_len = 0;
1061                 auth_offset = 0;
1062                 aad_len = fc_params->aad_buf.size;
1063                 aad_buf = &fc_params->aad_buf;
1064         }
1065
1066         cpt_ctx = fc_params->ctx_buf.vaddr;
1067         hash_type = cpt_ctx->hash_type;
1068         mac_len = cpt_ctx->mac_len;
1069
1070         if (hash_type == GMAC_TYPE)
1071                 encr_data_len = 0;
1072
1073         if (unlikely(!(flags & VALID_IV_BUF))) {
1074                 iv_len = 0;
1075                 iv_offset = ENCR_IV_OFFSET(d_offs);
1076         }
1077
1078         if (unlikely(flags & VALID_AAD_BUF)) {
1079                 /*
1080                  * When AAD is given, data above encr_offset is pass through
1081                  * Since AAD is given as separate pointer and not as offset,
1082                  * this is a special case as we need to fragment input data
1083                  * into passthrough + encr_data and then insert AAD in between.
1084                  */
1085                 if (hash_type != GMAC_TYPE) {
1086                         passthrough_len = encr_offset;
1087                         auth_offset = passthrough_len + iv_len;
1088                         encr_offset = passthrough_len + aad_len + iv_len;
1089                         auth_data_len = aad_len + encr_data_len;
1090                 } else {
1091                         passthrough_len = 16 + aad_len;
1092                         auth_offset = passthrough_len + iv_len;
1093                         auth_data_len = aad_len;
1094                 }
1095         } else {
1096                 encr_offset += iv_len;
1097                 auth_offset += iv_len;
1098         }
1099
1100         /*
1101          * Save initial space that followed app data for completion code &
1102          * alternate completion code to fall in same cache line as app data
1103          */
1104         m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
1105         m_dma += COMPLETION_CODE_SIZE;
1106         size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
1107                (uint8_t *)m_vaddr;
1108         c_vaddr = (uint8_t *)m_vaddr + size;
1109         c_dma = m_dma + size;
1110         size += sizeof(cpt_res_s_t);
1111
1112         m_vaddr = (uint8_t *)m_vaddr + size;
1113         m_dma += size;
1114
1115         /* start cpt request info structure at 8 byte alignment */
1116         size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) -
1117                 (uint8_t *)m_vaddr;
1118
1119         req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size);
1120
1121         size += sizeof(struct cpt_request_info);
1122         m_vaddr = (uint8_t *)m_vaddr + size;
1123         m_dma += size;
1124
1125         /* Decryption */
1126         opcode.s.major = CPT_MAJOR_OP_FC;
1127         opcode.s.minor = 1;
1128
1129         enc_dlen = encr_offset + encr_data_len;
1130         auth_dlen = auth_offset + auth_data_len;
1131
1132         if (auth_dlen > enc_dlen) {
1133                 inputlen = auth_dlen + mac_len;
1134                 outputlen = auth_dlen;
1135         } else {
1136                 inputlen = enc_dlen + mac_len;
1137                 outputlen = enc_dlen;
1138         }
1139
1140         if (hash_type == GMAC_TYPE)
1141                 encr_offset = inputlen;
1142
1143         vq_cmd_w0.u64 = 0;
1144         vq_cmd_w0.s.param1 = encr_data_len;
1145         vq_cmd_w0.s.param2 = auth_data_len;
1146
1147         /*
1148          * In 83XX since we have a limitation of
1149          * IV & Offset control word not part of instruction
1150          * and need to be part of Data Buffer, we check if
1151          * head room is there and then only do the Direct mode processing
1152          */
1153         if (likely((flags & SINGLE_BUF_INPLACE) &&
1154                    (flags & SINGLE_BUF_HEADTAILROOM))) {
1155                 void *dm_vaddr = fc_params->bufs[0].vaddr;
1156                 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr;
1157                 /*
1158                  * This flag indicates that there is 24 bytes head room and
1159                  * 8 bytes tail room available, so that we get to do
1160                  * DIRECT MODE with limitation
1161                  */
1162
1163                 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len;
1164                 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len;
1165                 req->ist.ei1 = offset_dma;
1166
1167                 /* RPTR should just exclude offset control word */
1168                 req->ist.ei2 = dm_dma_addr - iv_len;
1169
1170                 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr +
1171                                         outputlen - iv_len);
1172                 /* since this is decryption,
1173                  * don't touch the content of
1174                  * alternate ccode space as it contains
1175                  * hmac.
1176                  */
1177
1178                 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN;
1179
1180                 vq_cmd_w0.s.opcode = opcode.flags;
1181
1182                 if (likely(iv_len)) {
1183                         uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr +
1184                                                       OFF_CTRL_LEN);
1185                         uint64_t *src = fc_params->iv_buf;
1186                         dest[0] = src[0];
1187                         dest[1] = src[1];
1188                 }
1189
1190                 *(uint64_t *)offset_vaddr =
1191                         rte_cpu_to_be_64(((uint64_t)encr_offset << 16) |
1192                                 ((uint64_t)iv_offset << 8) |
1193                                 ((uint64_t)auth_offset));
1194
1195         } else {
1196                 uint64_t dptr_dma, rptr_dma;
1197                 uint32_t g_size_bytes, s_size_bytes;
1198                 sg_comp_t *gather_comp;
1199                 sg_comp_t *scatter_comp;
1200                 uint8_t *in_buffer;
1201                 uint8_t i = 0;
1202
1203                 /* This falls under strict SG mode */
1204                 offset_vaddr = m_vaddr;
1205                 offset_dma = m_dma;
1206                 size = OFF_CTRL_LEN + iv_len;
1207
1208                 m_vaddr = (uint8_t *)m_vaddr + size;
1209                 m_dma += size;
1210
1211                 opcode.s.major |= CPT_DMA_MODE;
1212
1213                 vq_cmd_w0.s.opcode = opcode.flags;
1214
1215                 if (likely(iv_len)) {
1216                         uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr +
1217                                                       OFF_CTRL_LEN);
1218                         uint64_t *src = fc_params->iv_buf;
1219                         dest[0] = src[0];
1220                         dest[1] = src[1];
1221                 }
1222
1223                 *(uint64_t *)offset_vaddr =
1224                         rte_cpu_to_be_64(((uint64_t)encr_offset << 16) |
1225                                 ((uint64_t)iv_offset << 8) |
1226                                 ((uint64_t)auth_offset));
1227
1228                 /* DPTR has SG list */
1229                 in_buffer = m_vaddr;
1230                 dptr_dma = m_dma;
1231
1232                 ((uint16_t *)in_buffer)[0] = 0;
1233                 ((uint16_t *)in_buffer)[1] = 0;
1234
1235                 /* TODO Add error check if space will be sufficient */
1236                 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
1237
1238                 /*
1239                  * Input Gather List
1240                  */
1241                 i = 0;
1242
1243                 /* Offset control word that includes iv */
1244                 i = fill_sg_comp(gather_comp, i, offset_dma,
1245                                  OFF_CTRL_LEN + iv_len);
1246
1247                 /* Add input data */
1248                 if (flags & VALID_MAC_BUF) {
1249                         size = inputlen - iv_len - mac_len;
1250                         if (size) {
1251                                 /* input data only */
1252                                 if (unlikely(flags & SINGLE_BUF_INPLACE)) {
1253                                         i = fill_sg_comp_from_buf_min(
1254                                                         gather_comp, i,
1255                                                         fc_params->bufs,
1256                                                         &size);
1257                                 } else {
1258                                         uint32_t aad_offset = aad_len ?
1259                                                 passthrough_len : 0;
1260
1261                                         i = fill_sg_comp_from_iov(gather_comp,
1262                                                         i,
1263                                                         fc_params->src_iov,
1264                                                         0, &size,
1265                                                         aad_buf,
1266                                                         aad_offset);
1267                                 }
1268                                 if (unlikely(size)) {
1269                                         CPT_LOG_DP_ERR("Insufficient buffer"
1270                                                        " space, size %d needed",
1271                                                        size);
1272                                         return;
1273                                 }
1274                         }
1275
1276                         /* mac data */
1277                         if (mac_len) {
1278                                 i = fill_sg_comp_from_buf(gather_comp, i,
1279                                                           &fc_params->mac_buf);
1280                         }
1281                 } else {
1282                         /* input data + mac */
1283                         size = inputlen - iv_len;
1284                         if (size) {
1285                                 if (unlikely(flags & SINGLE_BUF_INPLACE)) {
1286                                         i = fill_sg_comp_from_buf_min(
1287                                                         gather_comp, i,
1288                                                         fc_params->bufs,
1289                                                         &size);
1290                                 } else {
1291                                         uint32_t aad_offset = aad_len ?
1292                                                 passthrough_len : 0;
1293
1294                                         if (unlikely(!fc_params->src_iov)) {
1295                                                 CPT_LOG_DP_ERR("Bad input args");
1296                                                 return;
1297                                         }
1298
1299                                         i = fill_sg_comp_from_iov(
1300                                                         gather_comp, i,
1301                                                         fc_params->src_iov,
1302                                                         0, &size,
1303                                                         aad_buf,
1304                                                         aad_offset);
1305                                 }
1306
1307                                 if (unlikely(size)) {
1308                                         CPT_LOG_DP_ERR("Insufficient buffer"
1309                                                        " space, size %d needed",
1310                                                        size);
1311                                         return;
1312                                 }
1313                         }
1314                 }
1315                 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
1316                 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
1317
1318                 /*
1319                  * Output Scatter List
1320                  */
1321
1322                 i = 0;
1323                 scatter_comp =
1324                         (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
1325
1326                 /* Add iv */
1327                 if (iv_len) {
1328                         i = fill_sg_comp(scatter_comp, i,
1329                                          offset_dma + OFF_CTRL_LEN,
1330                                          iv_len);
1331                 }
1332
1333                 /* Add output data */
1334                 size = outputlen - iv_len;
1335                 if (size) {
1336                         if (unlikely(flags & SINGLE_BUF_INPLACE)) {
1337                                 /* handle single buffer here */
1338                                 i = fill_sg_comp_from_buf_min(scatter_comp, i,
1339                                                               fc_params->bufs,
1340                                                               &size);
1341                         } else {
1342                                 uint32_t aad_offset = aad_len ?
1343                                         passthrough_len : 0;
1344
1345                                 if (unlikely(!fc_params->dst_iov)) {
1346                                         CPT_LOG_DP_ERR("Bad input args");
1347                                         return;
1348                                 }
1349
1350                                 i = fill_sg_comp_from_iov(scatter_comp, i,
1351                                                           fc_params->dst_iov, 0,
1352                                                           &size, aad_buf,
1353                                                           aad_offset);
1354                         }
1355
1356                         if (unlikely(size)) {
1357                                 CPT_LOG_DP_ERR("Insufficient buffer space,"
1358                                                " size %d needed", size);
1359                                 return;
1360                         }
1361                 }
1362
1363                 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
1364                 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
1365
1366                 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
1367
1368                 /* This is DPTR len incase of SG mode */
1369                 vq_cmd_w0.s.dlen = size;
1370
1371                 m_vaddr = (uint8_t *)m_vaddr + size;
1372                 m_dma += size;
1373
1374                 /* cpt alternate completion address saved earlier */
1375                 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
1376                 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
1377                 rptr_dma = c_dma - 8;
1378                 size += COMPLETION_CODE_SIZE;
1379
1380                 req->ist.ei1 = dptr_dma;
1381                 req->ist.ei2 = rptr_dma;
1382         }
1383
1384         ctx_dma = fc_params->ctx_buf.dma_addr +
1385                 offsetof(struct cpt_ctx, fctx);
1386         /* vq command w3 */
1387         vq_cmd_w3.u64 = 0;
1388         vq_cmd_w3.s.grp = 0;
1389         vq_cmd_w3.s.cptr = ctx_dma;
1390
1391         /* 16 byte aligned cpt res address */
1392         req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
1393         *req->completion_addr = COMPLETION_CODE_INIT;
1394         req->comp_baddr  = c_dma;
1395
1396         /* Fill microcode part of instruction */
1397         req->ist.ei0 = vq_cmd_w0.u64;
1398         req->ist.ei3 = vq_cmd_w3.u64;
1399
1400         req->op = op;
1401
1402         *prep_req = req;
1403         return;
1404 }
1405
1406 static __rte_always_inline void
1407 cpt_zuc_snow3g_enc_prep(uint32_t req_flags,
1408                         uint64_t d_offs,
1409                         uint64_t d_lens,
1410                         fc_params_t *params,
1411                         void *op,
1412                         void **prep_req)
1413 {
1414         uint32_t size;
1415         int32_t inputlen, outputlen;
1416         struct cpt_ctx *cpt_ctx;
1417         uint32_t mac_len = 0;
1418         uint8_t snow3g, j;
1419         struct cpt_request_info *req;
1420         buf_ptr_t *buf_p;
1421         uint32_t encr_offset = 0, auth_offset = 0;
1422         uint32_t encr_data_len = 0, auth_data_len = 0;
1423         int flags, iv_len = 16;
1424         void *m_vaddr, *c_vaddr;
1425         uint64_t m_dma, c_dma, offset_ctrl;
1426         uint64_t *offset_vaddr, offset_dma;
1427         uint32_t *iv_s, iv[4];
1428         vq_cmd_word0_t vq_cmd_w0;
1429         vq_cmd_word3_t vq_cmd_w3;
1430         opcode_info_t opcode;
1431
1432         buf_p = &params->meta_buf;
1433         m_vaddr = buf_p->vaddr;
1434         m_dma = buf_p->dma_addr;
1435
1436         cpt_ctx = params->ctx_buf.vaddr;
1437         flags = cpt_ctx->zsk_flags;
1438         mac_len = cpt_ctx->mac_len;
1439         snow3g = cpt_ctx->snow3g;
1440
1441         /*
1442          * Save initial space that followed app data for completion code &
1443          * alternate completion code to fall in same cache line as app data
1444          */
1445         m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
1446         m_dma += COMPLETION_CODE_SIZE;
1447         size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
1448                 (uint8_t *)m_vaddr;
1449
1450         c_vaddr = (uint8_t *)m_vaddr + size;
1451         c_dma = m_dma + size;
1452         size += sizeof(cpt_res_s_t);
1453
1454         m_vaddr = (uint8_t *)m_vaddr + size;
1455         m_dma += size;
1456
1457         /* Reserve memory for cpt request info */
1458         req = m_vaddr;
1459
1460         size = sizeof(struct cpt_request_info);
1461         m_vaddr = (uint8_t *)m_vaddr + size;
1462         m_dma += size;
1463
1464         opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G;
1465
1466         /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */
1467
1468         opcode.s.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) |
1469                           (0 << 3) | (flags & 0x7));
1470
1471         if (flags == 0x1) {
1472                 /*
1473                  * Microcode expects offsets in bytes
1474                  * TODO: Rounding off
1475                  */
1476                 auth_data_len = AUTH_DLEN(d_lens);
1477
1478                 /* EIA3 or UIA2 */
1479                 auth_offset = AUTH_OFFSET(d_offs);
1480                 auth_offset = auth_offset / 8;
1481
1482                 /* consider iv len */
1483                 auth_offset += iv_len;
1484
1485                 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8);
1486                 outputlen = mac_len;
1487
1488                 offset_ctrl = rte_cpu_to_be_64((uint64_t)auth_offset);
1489
1490         } else {
1491                 /* EEA3 or UEA2 */
1492                 /*
1493                  * Microcode expects offsets in bytes
1494                  * TODO: Rounding off
1495                  */
1496                 encr_data_len = ENCR_DLEN(d_lens);
1497
1498                 encr_offset = ENCR_OFFSET(d_offs);
1499                 encr_offset = encr_offset / 8;
1500                 /* consider iv len */
1501                 encr_offset += iv_len;
1502
1503                 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8);
1504                 outputlen = inputlen;
1505
1506                 /* iv offset is 0 */
1507                 offset_ctrl = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
1508         }
1509
1510         /* IV */
1511         iv_s = (flags == 0x1) ? params->auth_iv_buf :
1512                 params->iv_buf;
1513
1514         if (snow3g) {
1515                 /*
1516                  * DPDK seems to provide it in form of IV3 IV2 IV1 IV0
1517                  * and BigEndian, MC needs it as IV0 IV1 IV2 IV3
1518                  */
1519
1520                 for (j = 0; j < 4; j++)
1521                         iv[j] = iv_s[3 - j];
1522         } else {
1523                 /* ZUC doesn't need a swap */
1524                 for (j = 0; j < 4; j++)
1525                         iv[j] = iv_s[j];
1526         }
1527
1528         /*
1529          * GP op header, lengths are expected in bits.
1530          */
1531         vq_cmd_w0.u64 = 0;
1532         vq_cmd_w0.s.param1 = encr_data_len;
1533         vq_cmd_w0.s.param2 = auth_data_len;
1534
1535         /*
1536          * In 83XX since we have a limitation of
1537          * IV & Offset control word not part of instruction
1538          * and need to be part of Data Buffer, we check if
1539          * head room is there and then only do the Direct mode processing
1540          */
1541         if (likely((req_flags & SINGLE_BUF_INPLACE) &&
1542                    (req_flags & SINGLE_BUF_HEADTAILROOM))) {
1543                 void *dm_vaddr = params->bufs[0].vaddr;
1544                 uint64_t dm_dma_addr = params->bufs[0].dma_addr;
1545                 /*
1546                  * This flag indicates that there is 24 bytes head room and
1547                  * 8 bytes tail room available, so that we get to do
1548                  * DIRECT MODE with limitation
1549                  */
1550
1551                 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr -
1552                                             OFF_CTRL_LEN - iv_len);
1553                 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len;
1554
1555                 /* DPTR */
1556                 req->ist.ei1 = offset_dma;
1557                 /* RPTR should just exclude offset control word */
1558                 req->ist.ei2 = dm_dma_addr - iv_len;
1559                 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr
1560                                                     + outputlen - iv_len);
1561
1562                 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN;
1563
1564                 vq_cmd_w0.s.opcode = opcode.flags;
1565
1566                 if (likely(iv_len)) {
1567                         uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr
1568                                                       + OFF_CTRL_LEN);
1569                         memcpy(iv_d, iv, 16);
1570                 }
1571
1572                 *offset_vaddr = offset_ctrl;
1573         } else {
1574                 uint32_t i, g_size_bytes, s_size_bytes;
1575                 uint64_t dptr_dma, rptr_dma;
1576                 sg_comp_t *gather_comp;
1577                 sg_comp_t *scatter_comp;
1578                 uint8_t *in_buffer;
1579                 uint32_t *iv_d;
1580
1581                 /* save space for iv */
1582                 offset_vaddr = m_vaddr;
1583                 offset_dma = m_dma;
1584
1585                 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
1586                 m_dma += OFF_CTRL_LEN + iv_len;
1587
1588                 opcode.s.major |= CPT_DMA_MODE;
1589
1590                 vq_cmd_w0.s.opcode = opcode.flags;
1591
1592                 /* DPTR has SG list */
1593                 in_buffer = m_vaddr;
1594                 dptr_dma = m_dma;
1595
1596                 ((uint16_t *)in_buffer)[0] = 0;
1597                 ((uint16_t *)in_buffer)[1] = 0;
1598
1599                 /* TODO Add error check if space will be sufficient */
1600                 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
1601
1602                 /*
1603                  * Input Gather List
1604                  */
1605                 i = 0;
1606
1607                 /* Offset control word followed by iv */
1608
1609                 i = fill_sg_comp(gather_comp, i, offset_dma,
1610                                  OFF_CTRL_LEN + iv_len);
1611
1612                 /* iv offset is 0 */
1613                 *offset_vaddr = offset_ctrl;
1614
1615                 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN);
1616                 memcpy(iv_d, iv, 16);
1617
1618                 /* input data */
1619                 size = inputlen - iv_len;
1620                 if (size) {
1621                         i = fill_sg_comp_from_iov(gather_comp, i,
1622                                                   params->src_iov,
1623                                                   0, &size, NULL, 0);
1624                         if (unlikely(size)) {
1625                                 CPT_LOG_DP_ERR("Insufficient buffer space,"
1626                                                " size %d needed", size);
1627                                 return;
1628                         }
1629                 }
1630                 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
1631                 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
1632
1633                 /*
1634                  * Output Scatter List
1635                  */
1636
1637                 i = 0;
1638                 scatter_comp =
1639                         (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
1640
1641                 if (flags == 0x1) {
1642                         /* IV in SLIST only for EEA3 & UEA2 */
1643                         iv_len = 0;
1644                 }
1645
1646                 if (iv_len) {
1647                         i = fill_sg_comp(scatter_comp, i,
1648                                          offset_dma + OFF_CTRL_LEN, iv_len);
1649                 }
1650
1651                 /* Add output data */
1652                 if (req_flags & VALID_MAC_BUF) {
1653                         size = outputlen - iv_len - mac_len;
1654                         if (size) {
1655                                 i = fill_sg_comp_from_iov(scatter_comp, i,
1656                                                           params->dst_iov, 0,
1657                                                           &size, NULL, 0);
1658
1659                                 if (unlikely(size)) {
1660                                         CPT_LOG_DP_ERR("Insufficient buffer space,"
1661                                                        " size %d needed", size);
1662                                         return;
1663                                 }
1664                         }
1665
1666                         /* mac data */
1667                         if (mac_len) {
1668                                 i = fill_sg_comp_from_buf(scatter_comp, i,
1669                                                           &params->mac_buf);
1670                         }
1671                 } else {
1672                         /* Output including mac */
1673                         size = outputlen - iv_len;
1674                         if (size) {
1675                                 i = fill_sg_comp_from_iov(scatter_comp, i,
1676                                                           params->dst_iov, 0,
1677                                                           &size, NULL, 0);
1678
1679                                 if (unlikely(size)) {
1680                                         CPT_LOG_DP_ERR("Insufficient buffer space,"
1681                                                        " size %d needed", size);
1682                                         return;
1683                                 }
1684                         }
1685                 }
1686                 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
1687                 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
1688
1689                 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
1690
1691                 /* This is DPTR len incase of SG mode */
1692                 vq_cmd_w0.s.dlen = size;
1693
1694                 m_vaddr = (uint8_t *)m_vaddr + size;
1695                 m_dma += size;
1696
1697                 /* cpt alternate completion address saved earlier */
1698                 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
1699                 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
1700                 rptr_dma = c_dma - 8;
1701
1702                 req->ist.ei1 = dptr_dma;
1703                 req->ist.ei2 = rptr_dma;
1704         }
1705
1706         /* vq command w3 */
1707         vq_cmd_w3.u64 = 0;
1708         vq_cmd_w3.s.grp = 0;
1709         vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr +
1710                 offsetof(struct cpt_ctx, zs_ctx);
1711
1712         /* 16 byte aligned cpt res address */
1713         req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
1714         *req->completion_addr = COMPLETION_CODE_INIT;
1715         req->comp_baddr  = c_dma;
1716
1717         /* Fill microcode part of instruction */
1718         req->ist.ei0 = vq_cmd_w0.u64;
1719         req->ist.ei3 = vq_cmd_w3.u64;
1720
1721         req->op = op;
1722
1723         *prep_req = req;
1724         return;
1725 }
1726
1727 static __rte_always_inline void
1728 cpt_zuc_snow3g_dec_prep(uint32_t req_flags,
1729                         uint64_t d_offs,
1730                         uint64_t d_lens,
1731                         fc_params_t *params,
1732                         void *op,
1733                         void **prep_req)
1734 {
1735         uint32_t size;
1736         int32_t inputlen = 0, outputlen;
1737         struct cpt_ctx *cpt_ctx;
1738         uint8_t snow3g, iv_len = 16;
1739         struct cpt_request_info *req;
1740         buf_ptr_t *buf_p;
1741         uint32_t encr_offset;
1742         uint32_t encr_data_len;
1743         int flags;
1744         void *m_vaddr, *c_vaddr;
1745         uint64_t m_dma, c_dma;
1746         uint64_t *offset_vaddr, offset_dma;
1747         uint32_t *iv_s, iv[4], j;
1748         vq_cmd_word0_t vq_cmd_w0;
1749         vq_cmd_word3_t vq_cmd_w3;
1750         opcode_info_t opcode;
1751
1752         buf_p = &params->meta_buf;
1753         m_vaddr = buf_p->vaddr;
1754         m_dma = buf_p->dma_addr;
1755
1756         /*
1757          * Microcode expects offsets in bytes
1758          * TODO: Rounding off
1759          */
1760         encr_offset = ENCR_OFFSET(d_offs) / 8;
1761         encr_data_len = ENCR_DLEN(d_lens);
1762
1763         cpt_ctx = params->ctx_buf.vaddr;
1764         flags = cpt_ctx->zsk_flags;
1765         snow3g = cpt_ctx->snow3g;
1766         /*
1767          * Save initial space that followed app data for completion code &
1768          * alternate completion code to fall in same cache line as app data
1769          */
1770         m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
1771         m_dma += COMPLETION_CODE_SIZE;
1772         size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
1773                 (uint8_t *)m_vaddr;
1774
1775         c_vaddr = (uint8_t *)m_vaddr + size;
1776         c_dma = m_dma + size;
1777         size += sizeof(cpt_res_s_t);
1778
1779         m_vaddr = (uint8_t *)m_vaddr + size;
1780         m_dma += size;
1781
1782         /* Reserve memory for cpt request info */
1783         req = m_vaddr;
1784
1785         size = sizeof(struct cpt_request_info);
1786         m_vaddr = (uint8_t *)m_vaddr + size;
1787         m_dma += size;
1788
1789         opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G;
1790
1791         /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */
1792
1793         opcode.s.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) |
1794                           (0 << 3) | (flags & 0x7));
1795
1796         /* consider iv len */
1797         encr_offset += iv_len;
1798
1799         inputlen = encr_offset +
1800                 (RTE_ALIGN(encr_data_len, 8) / 8);
1801         outputlen = inputlen;
1802
1803         /* IV */
1804         iv_s = params->iv_buf;
1805         if (snow3g) {
1806                 /*
1807                  * DPDK seems to provide it in form of IV3 IV2 IV1 IV0
1808                  * and BigEndian, MC needs it as IV0 IV1 IV2 IV3
1809                  */
1810
1811                 for (j = 0; j < 4; j++)
1812                         iv[j] = iv_s[3 - j];
1813         } else {
1814                 /* ZUC doesn't need a swap */
1815                 for (j = 0; j < 4; j++)
1816                         iv[j] = iv_s[j];
1817         }
1818
1819         /*
1820          * GP op header, lengths are expected in bits.
1821          */
1822         vq_cmd_w0.u64 = 0;
1823         vq_cmd_w0.s.param1 = encr_data_len;
1824
1825         /*
1826          * In 83XX since we have a limitation of
1827          * IV & Offset control word not part of instruction
1828          * and need to be part of Data Buffer, we check if
1829          * head room is there and then only do the Direct mode processing
1830          */
1831         if (likely((req_flags & SINGLE_BUF_INPLACE) &&
1832                    (req_flags & SINGLE_BUF_HEADTAILROOM))) {
1833                 void *dm_vaddr = params->bufs[0].vaddr;
1834                 uint64_t dm_dma_addr = params->bufs[0].dma_addr;
1835                 /*
1836                  * This flag indicates that there is 24 bytes head room and
1837                  * 8 bytes tail room available, so that we get to do
1838                  * DIRECT MODE with limitation
1839                  */
1840
1841                 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr -
1842                                             OFF_CTRL_LEN - iv_len);
1843                 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len;
1844
1845                 /* DPTR */
1846                 req->ist.ei1 = offset_dma;
1847                 /* RPTR should just exclude offset control word */
1848                 req->ist.ei2 = dm_dma_addr - iv_len;
1849                 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr
1850                                                     + outputlen - iv_len);
1851
1852                 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN;
1853
1854                 vq_cmd_w0.s.opcode = opcode.flags;
1855
1856                 if (likely(iv_len)) {
1857                         uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr
1858                                                       + OFF_CTRL_LEN);
1859                         memcpy(iv_d, iv, 16);
1860                 }
1861
1862                 /* iv offset is 0 */
1863                 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
1864         } else {
1865                 uint32_t i, g_size_bytes, s_size_bytes;
1866                 uint64_t dptr_dma, rptr_dma;
1867                 sg_comp_t *gather_comp;
1868                 sg_comp_t *scatter_comp;
1869                 uint8_t *in_buffer;
1870                 uint32_t *iv_d;
1871
1872                 /* save space for offset and iv... */
1873                 offset_vaddr = m_vaddr;
1874                 offset_dma = m_dma;
1875
1876                 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
1877                 m_dma += OFF_CTRL_LEN + iv_len;
1878
1879                 opcode.s.major |= CPT_DMA_MODE;
1880
1881                 vq_cmd_w0.s.opcode = opcode.flags;
1882
1883                 /* DPTR has SG list */
1884                 in_buffer = m_vaddr;
1885                 dptr_dma = m_dma;
1886
1887                 ((uint16_t *)in_buffer)[0] = 0;
1888                 ((uint16_t *)in_buffer)[1] = 0;
1889
1890                 /* TODO Add error check if space will be sufficient */
1891                 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
1892
1893                 /*
1894                  * Input Gather List
1895                  */
1896                 i = 0;
1897
1898                 /* Offset control word */
1899
1900                 /* iv offset is 0 */
1901                 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
1902
1903                 i = fill_sg_comp(gather_comp, i, offset_dma,
1904                                  OFF_CTRL_LEN + iv_len);
1905
1906                 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN);
1907                 memcpy(iv_d, iv, 16);
1908
1909                 /* Add input data */
1910                 size = inputlen - iv_len;
1911                 if (size) {
1912                         i = fill_sg_comp_from_iov(gather_comp, i,
1913                                                   params->src_iov,
1914                                                   0, &size, NULL, 0);
1915                         if (unlikely(size)) {
1916                                 CPT_LOG_DP_ERR("Insufficient buffer space,"
1917                                                " size %d needed", size);
1918                                 return;
1919                         }
1920                 }
1921                 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
1922                 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
1923
1924                 /*
1925                  * Output Scatter List
1926                  */
1927
1928                 i = 0;
1929                 scatter_comp =
1930                         (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
1931
1932                 /* IV */
1933                 i = fill_sg_comp(scatter_comp, i,
1934                                  offset_dma + OFF_CTRL_LEN,
1935                                  iv_len);
1936
1937                 /* Add output data */
1938                 size = outputlen - iv_len;
1939                 if (size) {
1940                         i = fill_sg_comp_from_iov(scatter_comp, i,
1941                                                   params->dst_iov, 0,
1942                                                   &size, NULL, 0);
1943
1944                         if (unlikely(size)) {
1945                                 CPT_LOG_DP_ERR("Insufficient buffer space,"
1946                                                " size %d needed", size);
1947                                 return;
1948                         }
1949                 }
1950                 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
1951                 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
1952
1953                 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
1954
1955                 /* This is DPTR len incase of SG mode */
1956                 vq_cmd_w0.s.dlen = size;
1957
1958                 m_vaddr = (uint8_t *)m_vaddr + size;
1959                 m_dma += size;
1960
1961                 /* cpt alternate completion address saved earlier */
1962                 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
1963                 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
1964                 rptr_dma = c_dma - 8;
1965
1966                 req->ist.ei1 = dptr_dma;
1967                 req->ist.ei2 = rptr_dma;
1968         }
1969
1970         /* vq command w3 */
1971         vq_cmd_w3.u64 = 0;
1972         vq_cmd_w3.s.grp = 0;
1973         vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr +
1974                 offsetof(struct cpt_ctx, zs_ctx);
1975
1976         /* 16 byte aligned cpt res address */
1977         req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
1978         *req->completion_addr = COMPLETION_CODE_INIT;
1979         req->comp_baddr  = c_dma;
1980
1981         /* Fill microcode part of instruction */
1982         req->ist.ei0 = vq_cmd_w0.u64;
1983         req->ist.ei3 = vq_cmd_w3.u64;
1984
1985         req->op = op;
1986
1987         *prep_req = req;
1988         return;
1989 }
1990
1991 static __rte_always_inline void
1992 cpt_kasumi_enc_prep(uint32_t req_flags,
1993                     uint64_t d_offs,
1994                     uint64_t d_lens,
1995                     fc_params_t *params,
1996                     void *op,
1997                     void **prep_req)
1998 {
1999         uint32_t size;
2000         int32_t inputlen = 0, outputlen = 0;
2001         struct cpt_ctx *cpt_ctx;
2002         uint32_t mac_len = 0;
2003         uint8_t i = 0;
2004         struct cpt_request_info *req;
2005         buf_ptr_t *buf_p;
2006         uint32_t encr_offset, auth_offset;
2007         uint32_t encr_data_len, auth_data_len;
2008         int flags;
2009         uint8_t *iv_s, *iv_d, iv_len = 8;
2010         uint8_t dir = 0;
2011         void *m_vaddr, *c_vaddr;
2012         uint64_t m_dma, c_dma;
2013         uint64_t *offset_vaddr, offset_dma;
2014         vq_cmd_word0_t vq_cmd_w0;
2015         vq_cmd_word3_t vq_cmd_w3;
2016         opcode_info_t opcode;
2017         uint8_t *in_buffer;
2018         uint32_t g_size_bytes, s_size_bytes;
2019         uint64_t dptr_dma, rptr_dma;
2020         sg_comp_t *gather_comp;
2021         sg_comp_t *scatter_comp;
2022
2023         buf_p = &params->meta_buf;
2024         m_vaddr = buf_p->vaddr;
2025         m_dma = buf_p->dma_addr;
2026
2027         encr_offset = ENCR_OFFSET(d_offs) / 8;
2028         auth_offset = AUTH_OFFSET(d_offs) / 8;
2029         encr_data_len = ENCR_DLEN(d_lens);
2030         auth_data_len = AUTH_DLEN(d_lens);
2031
2032         cpt_ctx = params->ctx_buf.vaddr;
2033         flags = cpt_ctx->zsk_flags;
2034         mac_len = cpt_ctx->mac_len;
2035
2036         if (flags == 0x0)
2037                 iv_s = params->iv_buf;
2038         else
2039                 iv_s = params->auth_iv_buf;
2040
2041         dir = iv_s[8] & 0x1;
2042
2043         /*
2044          * Save initial space that followed app data for completion code &
2045          * alternate completion code to fall in same cache line as app data
2046          */
2047         m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
2048         m_dma += COMPLETION_CODE_SIZE;
2049         size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
2050                 (uint8_t *)m_vaddr;
2051
2052         c_vaddr = (uint8_t *)m_vaddr + size;
2053         c_dma = m_dma + size;
2054         size += sizeof(cpt_res_s_t);
2055
2056         m_vaddr = (uint8_t *)m_vaddr + size;
2057         m_dma += size;
2058
2059         /* Reserve memory for cpt request info */
2060         req = m_vaddr;
2061
2062         size = sizeof(struct cpt_request_info);
2063         m_vaddr = (uint8_t *)m_vaddr + size;
2064         m_dma += size;
2065
2066         opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE;
2067
2068         /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */
2069         opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) |
2070                           (dir << 4) | (0 << 3) | (flags & 0x7));
2071
2072         /*
2073          * GP op header, lengths are expected in bits.
2074          */
2075         vq_cmd_w0.u64 = 0;
2076         vq_cmd_w0.s.param1 = encr_data_len;
2077         vq_cmd_w0.s.param2 = auth_data_len;
2078         vq_cmd_w0.s.opcode = opcode.flags;
2079
2080         /* consider iv len */
2081         if (flags == 0x0) {
2082                 encr_offset += iv_len;
2083                 auth_offset += iv_len;
2084         }
2085
2086         /* save space for offset ctrl and iv */
2087         offset_vaddr = m_vaddr;
2088         offset_dma = m_dma;
2089
2090         m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
2091         m_dma += OFF_CTRL_LEN + iv_len;
2092
2093         /* DPTR has SG list */
2094         in_buffer = m_vaddr;
2095         dptr_dma = m_dma;
2096
2097         ((uint16_t *)in_buffer)[0] = 0;
2098         ((uint16_t *)in_buffer)[1] = 0;
2099
2100         /* TODO Add error check if space will be sufficient */
2101         gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
2102
2103         /*
2104          * Input Gather List
2105          */
2106         i = 0;
2107
2108         /* Offset control word followed by iv */
2109
2110         if (flags == 0x0) {
2111                 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8);
2112                 outputlen = inputlen;
2113                 /* iv offset is 0 */
2114                 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
2115         } else {
2116                 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8);
2117                 outputlen = mac_len;
2118                 /* iv offset is 0 */
2119                 *offset_vaddr = rte_cpu_to_be_64((uint64_t)auth_offset);
2120         }
2121
2122         i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len);
2123
2124         /* IV */
2125         iv_d = (uint8_t *)offset_vaddr + OFF_CTRL_LEN;
2126         memcpy(iv_d, iv_s, iv_len);
2127
2128         /* input data */
2129         size = inputlen - iv_len;
2130         if (size) {
2131                 i = fill_sg_comp_from_iov(gather_comp, i,
2132                                           params->src_iov, 0,
2133                                           &size, NULL, 0);
2134
2135                 if (unlikely(size)) {
2136                         CPT_LOG_DP_ERR("Insufficient buffer space,"
2137                                        " size %d needed", size);
2138                         return;
2139                 }
2140         }
2141         ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
2142         g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
2143
2144         /*
2145          * Output Scatter List
2146          */
2147
2148         i = 0;
2149         scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
2150
2151         if (flags == 0x1) {
2152                 /* IV in SLIST only for F8 */
2153                 iv_len = 0;
2154         }
2155
2156         /* IV */
2157         if (iv_len) {
2158                 i = fill_sg_comp(scatter_comp, i,
2159                                  offset_dma + OFF_CTRL_LEN,
2160                                  iv_len);
2161         }
2162
2163         /* Add output data */
2164         if (req_flags & VALID_MAC_BUF) {
2165                 size = outputlen - iv_len - mac_len;
2166                 if (size) {
2167                         i = fill_sg_comp_from_iov(scatter_comp, i,
2168                                                   params->dst_iov, 0,
2169                                                   &size, NULL, 0);
2170
2171                         if (unlikely(size)) {
2172                                 CPT_LOG_DP_ERR("Insufficient buffer space,"
2173                                                " size %d needed", size);
2174                                 return;
2175                         }
2176                 }
2177
2178                 /* mac data */
2179                 if (mac_len) {
2180                         i = fill_sg_comp_from_buf(scatter_comp, i,
2181                                                   &params->mac_buf);
2182                 }
2183         } else {
2184                 /* Output including mac */
2185                 size = outputlen - iv_len;
2186                 if (size) {
2187                         i = fill_sg_comp_from_iov(scatter_comp, i,
2188                                                   params->dst_iov, 0,
2189                                                   &size, NULL, 0);
2190
2191                         if (unlikely(size)) {
2192                                 CPT_LOG_DP_ERR("Insufficient buffer space,"
2193                                                " size %d needed", size);
2194                                 return;
2195                         }
2196                 }
2197         }
2198         ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
2199         s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
2200
2201         size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
2202
2203         /* This is DPTR len incase of SG mode */
2204         vq_cmd_w0.s.dlen = size;
2205
2206         m_vaddr = (uint8_t *)m_vaddr + size;
2207         m_dma += size;
2208
2209         /* cpt alternate completion address saved earlier */
2210         req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
2211         *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
2212         rptr_dma = c_dma - 8;
2213
2214         req->ist.ei1 = dptr_dma;
2215         req->ist.ei2 = rptr_dma;
2216
2217         /* vq command w3 */
2218         vq_cmd_w3.u64 = 0;
2219         vq_cmd_w3.s.grp = 0;
2220         vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr +
2221                 offsetof(struct cpt_ctx, k_ctx);
2222
2223         /* 16 byte aligned cpt res address */
2224         req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
2225         *req->completion_addr = COMPLETION_CODE_INIT;
2226         req->comp_baddr  = c_dma;
2227
2228         /* Fill microcode part of instruction */
2229         req->ist.ei0 = vq_cmd_w0.u64;
2230         req->ist.ei3 = vq_cmd_w3.u64;
2231
2232         req->op = op;
2233
2234         *prep_req = req;
2235         return;
2236 }
2237
2238 static __rte_always_inline void
2239 cpt_kasumi_dec_prep(uint64_t d_offs,
2240                     uint64_t d_lens,
2241                     fc_params_t *params,
2242                     void *op,
2243                     void **prep_req)
2244 {
2245         uint32_t size;
2246         int32_t inputlen = 0, outputlen;
2247         struct cpt_ctx *cpt_ctx;
2248         uint8_t i = 0, iv_len = 8;
2249         struct cpt_request_info *req;
2250         buf_ptr_t *buf_p;
2251         uint32_t encr_offset;
2252         uint32_t encr_data_len;
2253         int flags;
2254         uint8_t dir = 0;
2255         void *m_vaddr, *c_vaddr;
2256         uint64_t m_dma, c_dma;
2257         uint64_t *offset_vaddr, offset_dma;
2258         vq_cmd_word0_t vq_cmd_w0;
2259         vq_cmd_word3_t vq_cmd_w3;
2260         opcode_info_t opcode;
2261         uint8_t *in_buffer;
2262         uint32_t g_size_bytes, s_size_bytes;
2263         uint64_t dptr_dma, rptr_dma;
2264         sg_comp_t *gather_comp;
2265         sg_comp_t *scatter_comp;
2266
2267         buf_p = &params->meta_buf;
2268         m_vaddr = buf_p->vaddr;
2269         m_dma = buf_p->dma_addr;
2270
2271         encr_offset = ENCR_OFFSET(d_offs) / 8;
2272         encr_data_len = ENCR_DLEN(d_lens);
2273
2274         cpt_ctx = params->ctx_buf.vaddr;
2275         flags = cpt_ctx->zsk_flags;
2276         /*
2277          * Save initial space that followed app data for completion code &
2278          * alternate completion code to fall in same cache line as app data
2279          */
2280         m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE;
2281         m_dma += COMPLETION_CODE_SIZE;
2282         size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) -
2283                 (uint8_t *)m_vaddr;
2284
2285         c_vaddr = (uint8_t *)m_vaddr + size;
2286         c_dma = m_dma + size;
2287         size += sizeof(cpt_res_s_t);
2288
2289         m_vaddr = (uint8_t *)m_vaddr + size;
2290         m_dma += size;
2291
2292         /* Reserve memory for cpt request info */
2293         req = m_vaddr;
2294
2295         size = sizeof(struct cpt_request_info);
2296         m_vaddr = (uint8_t *)m_vaddr + size;
2297         m_dma += size;
2298
2299         opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE;
2300
2301         /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */
2302         opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) |
2303                           (dir << 4) | (0 << 3) | (flags & 0x7));
2304
2305         /*
2306          * GP op header, lengths are expected in bits.
2307          */
2308         vq_cmd_w0.u64 = 0;
2309         vq_cmd_w0.s.param1 = encr_data_len;
2310         vq_cmd_w0.s.opcode = opcode.flags;
2311
2312         /* consider iv len */
2313         encr_offset += iv_len;
2314
2315         inputlen = iv_len + (RTE_ALIGN(encr_data_len, 8) / 8);
2316         outputlen = inputlen;
2317
2318         /* save space for offset ctrl & iv */
2319         offset_vaddr = m_vaddr;
2320         offset_dma = m_dma;
2321
2322         m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len;
2323         m_dma += OFF_CTRL_LEN + iv_len;
2324
2325         /* DPTR has SG list */
2326         in_buffer = m_vaddr;
2327         dptr_dma = m_dma;
2328
2329         ((uint16_t *)in_buffer)[0] = 0;
2330         ((uint16_t *)in_buffer)[1] = 0;
2331
2332         /* TODO Add error check if space will be sufficient */
2333         gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8);
2334
2335         /*
2336          * Input Gather List
2337          */
2338         i = 0;
2339
2340         /* Offset control word followed by iv */
2341         *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16);
2342
2343         i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len);
2344
2345         /* IV */
2346         memcpy((uint8_t *)offset_vaddr + OFF_CTRL_LEN,
2347                params->iv_buf, iv_len);
2348
2349         /* Add input data */
2350         size = inputlen - iv_len;
2351         if (size) {
2352                 i = fill_sg_comp_from_iov(gather_comp, i,
2353                                           params->src_iov,
2354                                           0, &size, NULL, 0);
2355                 if (unlikely(size)) {
2356                         CPT_LOG_DP_ERR("Insufficient buffer space,"
2357                                        " size %d needed", size);
2358                         return;
2359                 }
2360         }
2361         ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i);
2362         g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
2363
2364         /*
2365          * Output Scatter List
2366          */
2367
2368         i = 0;
2369         scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes);
2370
2371         /* IV */
2372         i = fill_sg_comp(scatter_comp, i,
2373                          offset_dma + OFF_CTRL_LEN,
2374                          iv_len);
2375
2376         /* Add output data */
2377         size = outputlen - iv_len;
2378         if (size) {
2379                 i = fill_sg_comp_from_iov(scatter_comp, i,
2380                                           params->dst_iov, 0,
2381                                           &size, NULL, 0);
2382                 if (unlikely(size)) {
2383                         CPT_LOG_DP_ERR("Insufficient buffer space,"
2384                                        " size %d needed", size);
2385                         return;
2386                 }
2387         }
2388         ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i);
2389         s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t);
2390
2391         size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE;
2392
2393         /* This is DPTR len incase of SG mode */
2394         vq_cmd_w0.s.dlen = size;
2395
2396         m_vaddr = (uint8_t *)m_vaddr + size;
2397         m_dma += size;
2398
2399         /* cpt alternate completion address saved earlier */
2400         req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8);
2401         *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT);
2402         rptr_dma = c_dma - 8;
2403
2404         req->ist.ei1 = dptr_dma;
2405         req->ist.ei2 = rptr_dma;
2406
2407         /* vq command w3 */
2408         vq_cmd_w3.u64 = 0;
2409         vq_cmd_w3.s.grp = 0;
2410         vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr +
2411                 offsetof(struct cpt_ctx, k_ctx);
2412
2413         /* 16 byte aligned cpt res address */
2414         req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr);
2415         *req->completion_addr = COMPLETION_CODE_INIT;
2416         req->comp_baddr  = c_dma;
2417
2418         /* Fill microcode part of instruction */
2419         req->ist.ei0 = vq_cmd_w0.u64;
2420         req->ist.ei3 = vq_cmd_w3.u64;
2421
2422         req->op = op;
2423
2424         *prep_req = req;
2425         return;
2426 }
2427
2428 static __rte_always_inline void *
2429 cpt_fc_dec_hmac_prep(uint32_t flags,
2430                      uint64_t d_offs,
2431                      uint64_t d_lens,
2432                      fc_params_t *fc_params,
2433                      void *op)
2434 {
2435         struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr;
2436         uint8_t fc_type;
2437         void *prep_req = NULL;
2438
2439         fc_type = ctx->fc_type;
2440
2441         if (likely(fc_type == FC_GEN)) {
2442                 cpt_dec_hmac_prep(flags, d_offs, d_lens, fc_params, op,
2443                                   &prep_req);
2444         } else if (fc_type == ZUC_SNOW3G) {
2445                 cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens, fc_params, op,
2446                                         &prep_req);
2447         } else if (fc_type == KASUMI) {
2448                 cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, op, &prep_req);
2449         }
2450
2451         /*
2452          * For AUTH_ONLY case,
2453          * MC only supports digest generation and verification
2454          * should be done in software by memcmp()
2455          */
2456
2457         return prep_req;
2458 }
2459
2460 static __rte_always_inline void *__hot
2461 cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens,
2462                      fc_params_t *fc_params, void *op)
2463 {
2464         struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr;
2465         uint8_t fc_type;
2466         void *prep_req = NULL;
2467
2468         fc_type = ctx->fc_type;
2469
2470         /* Common api for rest of the ops */
2471         if (likely(fc_type == FC_GEN)) {
2472                 cpt_enc_hmac_prep(flags, d_offs, d_lens, fc_params, op,
2473                                   &prep_req);
2474         } else if (fc_type == ZUC_SNOW3G) {
2475                 cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens, fc_params, op,
2476                                         &prep_req);
2477         } else if (fc_type == KASUMI) {
2478                 cpt_kasumi_enc_prep(flags, d_offs, d_lens, fc_params, op,
2479                                     &prep_req);
2480         } else if (fc_type == HASH_HMAC) {
2481                 cpt_digest_gen_prep(flags, d_lens, fc_params, op, &prep_req);
2482         }
2483
2484         return prep_req;
2485 }
2486
2487 static __rte_always_inline int
2488 cpt_fc_auth_set_key(void *ctx, auth_type_t type, const uint8_t *key,
2489                     uint16_t key_len, uint16_t mac_len)
2490 {
2491         struct cpt_ctx *cpt_ctx = ctx;
2492         mc_fc_context_t *fctx = &cpt_ctx->fctx;
2493
2494         if ((type >= ZUC_EIA3) && (type <= KASUMI_F9_ECB)) {
2495                 uint32_t keyx[4];
2496
2497                 if (key_len != 16)
2498                         return -1;
2499                 /* No support for AEAD yet */
2500                 if (cpt_ctx->enc_cipher)
2501                         return -1;
2502                 /* For ZUC/SNOW3G/Kasumi */
2503                 switch (type) {
2504                 case SNOW3G_UIA2:
2505                         cpt_ctx->snow3g = 1;
2506                         gen_key_snow3g(key, keyx);
2507                         memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len);
2508                         cpt_ctx->fc_type = ZUC_SNOW3G;
2509                         cpt_ctx->zsk_flags = 0x1;
2510                         break;
2511                 case ZUC_EIA3:
2512                         cpt_ctx->snow3g = 0;
2513                         memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len);
2514                         memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32);
2515                         cpt_ctx->fc_type = ZUC_SNOW3G;
2516                         cpt_ctx->zsk_flags = 0x1;
2517                         break;
2518                 case KASUMI_F9_ECB:
2519                         /* Kasumi ECB mode */
2520                         cpt_ctx->k_ecb = 1;
2521                         memcpy(cpt_ctx->k_ctx.ci_key, key, key_len);
2522                         cpt_ctx->fc_type = KASUMI;
2523                         cpt_ctx->zsk_flags = 0x1;
2524                         break;
2525                 case KASUMI_F9_CBC:
2526                         memcpy(cpt_ctx->k_ctx.ci_key, key, key_len);
2527                         cpt_ctx->fc_type = KASUMI;
2528                         cpt_ctx->zsk_flags = 0x1;
2529                         break;
2530                 default:
2531                         return -1;
2532                 }
2533                 cpt_ctx->mac_len = 4;
2534                 cpt_ctx->hash_type = type;
2535                 return 0;
2536         }
2537
2538         if (!(cpt_ctx->fc_type == FC_GEN && !type)) {
2539                 if (!cpt_ctx->fc_type || !cpt_ctx->enc_cipher)
2540                         cpt_ctx->fc_type = HASH_HMAC;
2541         }
2542
2543         if (cpt_ctx->fc_type == FC_GEN && key_len > 64)
2544                 return -1;
2545
2546         /* For GMAC auth, cipher must be NULL */
2547         if (type == GMAC_TYPE)
2548                 fctx->enc.enc_cipher = 0;
2549
2550         fctx->enc.hash_type = cpt_ctx->hash_type = type;
2551         fctx->enc.mac_len = cpt_ctx->mac_len = mac_len;
2552
2553         if (key_len) {
2554                 cpt_ctx->hmac = 1;
2555                 memset(cpt_ctx->auth_key, 0, sizeof(cpt_ctx->auth_key));
2556                 memcpy(cpt_ctx->auth_key, key, key_len);
2557                 cpt_ctx->auth_key_len = key_len;
2558                 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad));
2559                 memset(fctx->hmac.opad, 0, sizeof(fctx->hmac.opad));
2560
2561                 if (key_len <= 64)
2562                         memcpy(fctx->hmac.opad, key, key_len);
2563                 fctx->enc.auth_input_type = 1;
2564         }
2565         return 0;
2566 }
2567
2568 static __rte_always_inline int
2569 fill_sess_aead(struct rte_crypto_sym_xform *xform,
2570                  struct cpt_sess_misc *sess)
2571 {
2572         struct rte_crypto_aead_xform *aead_form;
2573         cipher_type_t enc_type = 0; /* NULL Cipher type */
2574         auth_type_t auth_type = 0; /* NULL Auth type */
2575         uint32_t cipher_key_len = 0;
2576         uint8_t aes_gcm = 0;
2577         aead_form = &xform->aead;
2578         void *ctx = SESS_PRIV(sess);
2579
2580         if (aead_form->op == RTE_CRYPTO_AEAD_OP_ENCRYPT &&
2581            aead_form->algo == RTE_CRYPTO_AEAD_AES_GCM) {
2582                 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT;
2583                 sess->cpt_op |= CPT_OP_AUTH_GENERATE;
2584         } else if (aead_form->op == RTE_CRYPTO_AEAD_OP_DECRYPT &&
2585                 aead_form->algo == RTE_CRYPTO_AEAD_AES_GCM) {
2586                 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT;
2587                 sess->cpt_op |= CPT_OP_AUTH_VERIFY;
2588         } else {
2589                 CPT_LOG_DP_ERR("Unknown cipher operation\n");
2590                 return -1;
2591         }
2592         switch (aead_form->algo) {
2593         case RTE_CRYPTO_AEAD_AES_GCM:
2594                 enc_type = AES_GCM;
2595                 cipher_key_len = 16;
2596                 aes_gcm = 1;
2597                 break;
2598         case RTE_CRYPTO_AEAD_AES_CCM:
2599                 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u",
2600                                aead_form->algo);
2601                 return -1;
2602         default:
2603                 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified",
2604                                aead_form->algo);
2605                 return -1;
2606         }
2607         if (aead_form->key.length < cipher_key_len) {
2608                 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu",
2609                                (unsigned int long)aead_form->key.length);
2610                 return -1;
2611         }
2612         sess->zsk_flag = 0;
2613         sess->aes_gcm = aes_gcm;
2614         sess->mac_len = aead_form->digest_length;
2615         sess->iv_offset = aead_form->iv.offset;
2616         sess->iv_length = aead_form->iv.length;
2617         sess->aad_length = aead_form->aad_length;
2618
2619         cpt_fc_ciph_set_key(ctx, enc_type, aead_form->key.data,
2620                         aead_form->key.length, NULL);
2621
2622         cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, aead_form->digest_length);
2623
2624         return 0;
2625 }
2626
2627 static __rte_always_inline int
2628 fill_sess_cipher(struct rte_crypto_sym_xform *xform,
2629                  struct cpt_sess_misc *sess)
2630 {
2631         struct rte_crypto_cipher_xform *c_form;
2632         cipher_type_t enc_type = 0; /* NULL Cipher type */
2633         uint32_t cipher_key_len = 0;
2634         uint8_t zsk_flag = 0, aes_ctr = 0, is_null = 0;
2635
2636         c_form = &xform->cipher;
2637
2638         if (c_form->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
2639                 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT;
2640         else if (c_form->op == RTE_CRYPTO_CIPHER_OP_DECRYPT)
2641                 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT;
2642         else {
2643                 CPT_LOG_DP_ERR("Unknown cipher operation\n");
2644                 return -1;
2645         }
2646
2647         switch (c_form->algo) {
2648         case RTE_CRYPTO_CIPHER_AES_CBC:
2649                 enc_type = AES_CBC;
2650                 cipher_key_len = 16;
2651                 break;
2652         case RTE_CRYPTO_CIPHER_3DES_CBC:
2653                 enc_type = DES3_CBC;
2654                 cipher_key_len = 24;
2655                 break;
2656         case RTE_CRYPTO_CIPHER_DES_CBC:
2657                 /* DES is implemented using 3DES in hardware */
2658                 enc_type = DES3_CBC;
2659                 cipher_key_len = 8;
2660                 break;
2661         case RTE_CRYPTO_CIPHER_AES_CTR:
2662                 enc_type = AES_CTR;
2663                 cipher_key_len = 16;
2664                 aes_ctr = 1;
2665                 break;
2666         case RTE_CRYPTO_CIPHER_NULL:
2667                 enc_type = 0;
2668                 is_null = 1;
2669                 break;
2670         case RTE_CRYPTO_CIPHER_KASUMI_F8:
2671                 enc_type = KASUMI_F8_ECB;
2672                 cipher_key_len = 16;
2673                 zsk_flag = K_F8;
2674                 break;
2675         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
2676                 enc_type = SNOW3G_UEA2;
2677                 cipher_key_len = 16;
2678                 zsk_flag = ZS_EA;
2679                 break;
2680         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
2681                 enc_type = ZUC_EEA3;
2682                 cipher_key_len = 16;
2683                 zsk_flag = ZS_EA;
2684                 break;
2685         case RTE_CRYPTO_CIPHER_AES_XTS:
2686                 enc_type = AES_XTS;
2687                 cipher_key_len = 16;
2688                 break;
2689         case RTE_CRYPTO_CIPHER_3DES_ECB:
2690                 enc_type = DES3_ECB;
2691                 cipher_key_len = 24;
2692                 break;
2693         case RTE_CRYPTO_CIPHER_AES_ECB:
2694                 enc_type = AES_ECB;
2695                 cipher_key_len = 16;
2696                 break;
2697         case RTE_CRYPTO_CIPHER_3DES_CTR:
2698         case RTE_CRYPTO_CIPHER_AES_F8:
2699         case RTE_CRYPTO_CIPHER_ARC4:
2700                 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u",
2701                                c_form->algo);
2702                 return -1;
2703         default:
2704                 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified",
2705                                c_form->algo);
2706                 return -1;
2707         }
2708
2709         if (c_form->key.length < cipher_key_len) {
2710                 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu",
2711                                (unsigned long) c_form->key.length);
2712                 return -1;
2713         }
2714
2715         sess->zsk_flag = zsk_flag;
2716         sess->aes_gcm = 0;
2717         sess->aes_ctr = aes_ctr;
2718         sess->iv_offset = c_form->iv.offset;
2719         sess->iv_length = c_form->iv.length;
2720         sess->is_null = is_null;
2721
2722         cpt_fc_ciph_set_key(SESS_PRIV(sess), enc_type, c_form->key.data,
2723                             c_form->key.length, NULL);
2724
2725         return 0;
2726 }
2727
2728 static __rte_always_inline int
2729 fill_sess_auth(struct rte_crypto_sym_xform *xform,
2730                struct cpt_sess_misc *sess)
2731 {
2732         struct rte_crypto_auth_xform *a_form;
2733         auth_type_t auth_type = 0; /* NULL Auth type */
2734         uint8_t zsk_flag = 0, aes_gcm = 0, is_null = 0;
2735
2736         a_form = &xform->auth;
2737
2738         if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY)
2739                 sess->cpt_op |= CPT_OP_AUTH_VERIFY;
2740         else if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE)
2741                 sess->cpt_op |= CPT_OP_AUTH_GENERATE;
2742         else {
2743                 CPT_LOG_DP_ERR("Unknown auth operation");
2744                 return -1;
2745         }
2746
2747         switch (a_form->algo) {
2748         case RTE_CRYPTO_AUTH_SHA1_HMAC:
2749                 /* Fall through */
2750         case RTE_CRYPTO_AUTH_SHA1:
2751                 auth_type = SHA1_TYPE;
2752                 break;
2753         case RTE_CRYPTO_AUTH_SHA256_HMAC:
2754         case RTE_CRYPTO_AUTH_SHA256:
2755                 auth_type = SHA2_SHA256;
2756                 break;
2757         case RTE_CRYPTO_AUTH_SHA512_HMAC:
2758         case RTE_CRYPTO_AUTH_SHA512:
2759                 auth_type = SHA2_SHA512;
2760                 break;
2761         case RTE_CRYPTO_AUTH_AES_GMAC:
2762                 auth_type = GMAC_TYPE;
2763                 aes_gcm = 1;
2764                 break;
2765         case RTE_CRYPTO_AUTH_SHA224_HMAC:
2766         case RTE_CRYPTO_AUTH_SHA224:
2767                 auth_type = SHA2_SHA224;
2768                 break;
2769         case RTE_CRYPTO_AUTH_SHA384_HMAC:
2770         case RTE_CRYPTO_AUTH_SHA384:
2771                 auth_type = SHA2_SHA384;
2772                 break;
2773         case RTE_CRYPTO_AUTH_MD5_HMAC:
2774         case RTE_CRYPTO_AUTH_MD5:
2775                 auth_type = MD5_TYPE;
2776                 break;
2777         case RTE_CRYPTO_AUTH_KASUMI_F9:
2778                 auth_type = KASUMI_F9_ECB;
2779                 /*
2780                  * Indicate that direction needs to be taken out
2781                  * from end of src
2782                  */
2783                 zsk_flag = K_F9;
2784                 break;
2785         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
2786                 auth_type = SNOW3G_UIA2;
2787                 zsk_flag = ZS_IA;
2788                 break;
2789         case RTE_CRYPTO_AUTH_ZUC_EIA3:
2790                 auth_type = ZUC_EIA3;
2791                 zsk_flag = ZS_IA;
2792                 break;
2793         case RTE_CRYPTO_AUTH_NULL:
2794                 auth_type = 0;
2795                 is_null = 1;
2796                 break;
2797         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
2798         case RTE_CRYPTO_AUTH_AES_CMAC:
2799         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
2800                 CPT_LOG_DP_ERR("Crypto: Unsupported hash algo %u",
2801                                a_form->algo);
2802                 return -1;
2803         default:
2804                 CPT_LOG_DP_ERR("Crypto: Undefined Hash algo %u specified",
2805                                a_form->algo);
2806                 return -1;
2807         }
2808
2809         sess->zsk_flag = zsk_flag;
2810         sess->aes_gcm = aes_gcm;
2811         sess->mac_len = a_form->digest_length;
2812         sess->is_null = is_null;
2813         if (zsk_flag) {
2814                 sess->auth_iv_offset = a_form->iv.offset;
2815                 sess->auth_iv_length = a_form->iv.length;
2816         }
2817         cpt_fc_auth_set_key(SESS_PRIV(sess), auth_type, a_form->key.data,
2818                             a_form->key.length, a_form->digest_length);
2819
2820         return 0;
2821 }
2822
2823 static __rte_always_inline int
2824 fill_sess_gmac(struct rte_crypto_sym_xform *xform,
2825                  struct cpt_sess_misc *sess)
2826 {
2827         struct rte_crypto_auth_xform *a_form;
2828         cipher_type_t enc_type = 0; /* NULL Cipher type */
2829         auth_type_t auth_type = 0; /* NULL Auth type */
2830         void *ctx = SESS_PRIV(sess);
2831
2832         a_form = &xform->auth;
2833
2834         if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE)
2835                 sess->cpt_op |= CPT_OP_ENCODE;
2836         else if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY)
2837                 sess->cpt_op |= CPT_OP_DECODE;
2838         else {
2839                 CPT_LOG_DP_ERR("Unknown auth operation");
2840                 return -1;
2841         }
2842
2843         switch (a_form->algo) {
2844         case RTE_CRYPTO_AUTH_AES_GMAC:
2845                 enc_type = AES_GCM;
2846                 auth_type = GMAC_TYPE;
2847                 break;
2848         default:
2849                 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified",
2850                                a_form->algo);
2851                 return -1;
2852         }
2853
2854         sess->zsk_flag = 0;
2855         sess->aes_gcm = 0;
2856         sess->is_gmac = 1;
2857         sess->iv_offset = a_form->iv.offset;
2858         sess->iv_length = a_form->iv.length;
2859         sess->mac_len = a_form->digest_length;
2860
2861         cpt_fc_ciph_set_key(ctx, enc_type, a_form->key.data,
2862                         a_form->key.length, NULL);
2863         cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, a_form->digest_length);
2864
2865         return 0;
2866 }
2867
2868 static __rte_always_inline void *
2869 alloc_op_meta(struct rte_mbuf *m_src,
2870               buf_ptr_t *buf,
2871               int32_t len,
2872               struct rte_mempool *cpt_meta_pool)
2873 {
2874         uint8_t *mdata;
2875
2876 #ifndef CPT_ALWAYS_USE_SEPARATE_BUF
2877         if (likely(m_src && (m_src->nb_segs == 1))) {
2878                 int32_t tailroom;
2879                 phys_addr_t mphys;
2880
2881                 /* Check if tailroom is sufficient to hold meta data */
2882                 tailroom = rte_pktmbuf_tailroom(m_src);
2883                 if (likely(tailroom > len + 8)) {
2884                         mdata = (uint8_t *)m_src->buf_addr + m_src->buf_len;
2885                         mphys = m_src->buf_physaddr + m_src->buf_len;
2886                         mdata -= len;
2887                         mphys -= len;
2888                         buf->vaddr = mdata;
2889                         buf->dma_addr = mphys;
2890                         buf->size = len;
2891                         /* Indicate that this is a mbuf allocated mdata */
2892                         mdata = (uint8_t *)((uint64_t)mdata | 1ull);
2893                         return mdata;
2894                 }
2895         }
2896 #else
2897         RTE_SET_USED(m_src);
2898 #endif
2899
2900         if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0))
2901                 return NULL;
2902
2903         buf->vaddr = mdata;
2904         buf->dma_addr = rte_mempool_virt2iova(mdata);
2905         buf->size = len;
2906
2907         return mdata;
2908 }
2909
2910 /**
2911  * cpt_free_metabuf - free metabuf to mempool.
2912  * @param instance: pointer to instance.
2913  * @param objp: pointer to the metabuf.
2914  */
2915 static __rte_always_inline void
2916 free_op_meta(void *mdata, struct rte_mempool *cpt_meta_pool)
2917 {
2918         bool nofree = ((uintptr_t)mdata & 1ull);
2919
2920         if (likely(nofree))
2921                 return;
2922         rte_mempool_put(cpt_meta_pool, mdata);
2923 }
2924
2925 static __rte_always_inline uint32_t
2926 prepare_iov_from_pkt(struct rte_mbuf *pkt,
2927                      iov_ptr_t *iovec, uint32_t start_offset)
2928 {
2929         uint16_t index = 0;
2930         void *seg_data = NULL;
2931         phys_addr_t seg_phys;
2932         int32_t seg_size = 0;
2933
2934         if (!pkt) {
2935                 iovec->buf_cnt = 0;
2936                 return 0;
2937         }
2938
2939         if (!start_offset) {
2940                 seg_data = rte_pktmbuf_mtod(pkt, void *);
2941                 seg_phys = rte_pktmbuf_mtophys(pkt);
2942                 seg_size = pkt->data_len;
2943         } else {
2944                 while (start_offset >= pkt->data_len) {
2945                         start_offset -= pkt->data_len;
2946                         pkt = pkt->next;
2947                 }
2948
2949                 seg_data = rte_pktmbuf_mtod_offset(pkt, void *, start_offset);
2950                 seg_phys = rte_pktmbuf_mtophys_offset(pkt, start_offset);
2951                 seg_size = pkt->data_len - start_offset;
2952                 if (!seg_size)
2953                         return 1;
2954         }
2955
2956         /* first seg */
2957         iovec->bufs[index].vaddr = seg_data;
2958         iovec->bufs[index].dma_addr = seg_phys;
2959         iovec->bufs[index].size = seg_size;
2960         index++;
2961         pkt = pkt->next;
2962
2963         while (unlikely(pkt != NULL)) {
2964                 seg_data = rte_pktmbuf_mtod(pkt, void *);
2965                 seg_phys = rte_pktmbuf_mtophys(pkt);
2966                 seg_size = pkt->data_len;
2967                 if (!seg_size)
2968                         break;
2969
2970                 iovec->bufs[index].vaddr = seg_data;
2971                 iovec->bufs[index].dma_addr = seg_phys;
2972                 iovec->bufs[index].size = seg_size;
2973
2974                 index++;
2975
2976                 pkt = pkt->next;
2977         }
2978
2979         iovec->buf_cnt = index;
2980         return 0;
2981 }
2982
2983 static __rte_always_inline uint32_t
2984 prepare_iov_from_pkt_inplace(struct rte_mbuf *pkt,
2985                              fc_params_t *param,
2986                              uint32_t *flags)
2987 {
2988         uint16_t index = 0;
2989         void *seg_data = NULL;
2990         phys_addr_t seg_phys;
2991         uint32_t seg_size = 0;
2992         iov_ptr_t *iovec;
2993
2994         seg_data = rte_pktmbuf_mtod(pkt, void *);
2995         seg_phys = rte_pktmbuf_mtophys(pkt);
2996         seg_size = pkt->data_len;
2997
2998         /* first seg */
2999         if (likely(!pkt->next)) {
3000                 uint32_t headroom, tailroom;
3001
3002                 *flags |= SINGLE_BUF_INPLACE;
3003                 headroom = rte_pktmbuf_headroom(pkt);
3004                 tailroom = rte_pktmbuf_tailroom(pkt);
3005                 if (likely((headroom >= 24) &&
3006                     (tailroom >= 8))) {
3007                         /* In 83XX this is prerequivisit for Direct mode */
3008                         *flags |= SINGLE_BUF_HEADTAILROOM;
3009                 }
3010                 param->bufs[0].vaddr = seg_data;
3011                 param->bufs[0].dma_addr = seg_phys;
3012                 param->bufs[0].size = seg_size;
3013                 return 0;
3014         }
3015         iovec = param->src_iov;
3016         iovec->bufs[index].vaddr = seg_data;
3017         iovec->bufs[index].dma_addr = seg_phys;
3018         iovec->bufs[index].size = seg_size;
3019         index++;
3020         pkt = pkt->next;
3021
3022         while (unlikely(pkt != NULL)) {
3023                 seg_data = rte_pktmbuf_mtod(pkt, void *);
3024                 seg_phys = rte_pktmbuf_mtophys(pkt);
3025                 seg_size = pkt->data_len;
3026
3027                 if (!seg_size)
3028                         break;
3029
3030                 iovec->bufs[index].vaddr = seg_data;
3031                 iovec->bufs[index].dma_addr = seg_phys;
3032                 iovec->bufs[index].size = seg_size;
3033
3034                 index++;
3035
3036                 pkt = pkt->next;
3037         }
3038
3039         iovec->buf_cnt = index;
3040         return 0;
3041 }
3042
3043 static __rte_always_inline int
3044 fill_fc_params(struct rte_crypto_op *cop,
3045                struct cpt_sess_misc *sess_misc,
3046                struct cpt_qp_meta_info *m_info,
3047                void **mdata_ptr,
3048                void **prep_req)
3049 {
3050         uint32_t space = 0;
3051         struct rte_crypto_sym_op *sym_op = cop->sym;
3052         void *mdata = NULL;
3053         uintptr_t *op;
3054         uint32_t mc_hash_off;
3055         uint32_t flags = 0;
3056         uint64_t d_offs, d_lens;
3057         struct rte_mbuf *m_src, *m_dst;
3058         uint8_t cpt_op = sess_misc->cpt_op;
3059 #ifdef CPT_ALWAYS_USE_SG_MODE
3060         uint8_t inplace = 0;
3061 #else
3062         uint8_t inplace = 1;
3063 #endif
3064         fc_params_t fc_params;
3065         char src[SRC_IOV_SIZE];
3066         char dst[SRC_IOV_SIZE];
3067         uint32_t iv_buf[4];
3068         int ret;
3069
3070         if (likely(sess_misc->iv_length)) {
3071                 flags |= VALID_IV_BUF;
3072                 fc_params.iv_buf = rte_crypto_op_ctod_offset(cop,
3073                                    uint8_t *, sess_misc->iv_offset);
3074                 if (sess_misc->aes_ctr &&
3075                     unlikely(sess_misc->iv_length != 16)) {
3076                         memcpy((uint8_t *)iv_buf,
3077                                 rte_crypto_op_ctod_offset(cop,
3078                                 uint8_t *, sess_misc->iv_offset), 12);
3079                         iv_buf[3] = rte_cpu_to_be_32(0x1);
3080                         fc_params.iv_buf = iv_buf;
3081                 }
3082         }
3083
3084         if (sess_misc->zsk_flag) {
3085                 fc_params.auth_iv_buf = rte_crypto_op_ctod_offset(cop,
3086                                         uint8_t *,
3087                                         sess_misc->auth_iv_offset);
3088                 if (sess_misc->zsk_flag != ZS_EA)
3089                         inplace = 0;
3090         }
3091         m_src = sym_op->m_src;
3092         m_dst = sym_op->m_dst;
3093
3094         if (sess_misc->aes_gcm) {
3095                 uint8_t *salt;
3096                 uint8_t *aad_data;
3097                 uint16_t aad_len;
3098
3099                 d_offs = sym_op->aead.data.offset;
3100                 d_lens = sym_op->aead.data.length;
3101                 mc_hash_off = sym_op->aead.data.offset +
3102                               sym_op->aead.data.length;
3103
3104                 aad_data = sym_op->aead.aad.data;
3105                 aad_len = sess_misc->aad_length;
3106                 if (likely((aad_data + aad_len) ==
3107                            rte_pktmbuf_mtod_offset(m_src,
3108                                 uint8_t *,
3109                                 sym_op->aead.data.offset))) {
3110                         d_offs = (d_offs - aad_len) | (d_offs << 16);
3111                         d_lens = (d_lens + aad_len) | (d_lens << 32);
3112                 } else {
3113                         fc_params.aad_buf.vaddr = sym_op->aead.aad.data;
3114                         fc_params.aad_buf.dma_addr = sym_op->aead.aad.phys_addr;
3115                         fc_params.aad_buf.size = aad_len;
3116                         flags |= VALID_AAD_BUF;
3117                         inplace = 0;
3118                         d_offs = d_offs << 16;
3119                         d_lens = d_lens << 32;
3120                 }
3121
3122                 salt = fc_params.iv_buf;
3123                 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) {
3124                         cpt_fc_salt_update(SESS_PRIV(sess_misc), salt);
3125                         sess_misc->salt = *(uint32_t *)salt;
3126                 }
3127                 fc_params.iv_buf = salt + 4;
3128                 if (likely(sess_misc->mac_len)) {
3129                         struct rte_mbuf *m = (cpt_op & CPT_OP_ENCODE) ? m_dst :
3130                                              m_src;
3131
3132                         if (!m)
3133                                 m = m_src;
3134
3135                         /* hmac immediately following data is best case */
3136                         if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) +
3137                             mc_hash_off !=
3138                             (uint8_t *)sym_op->aead.digest.data)) {
3139                                 flags |= VALID_MAC_BUF;
3140                                 fc_params.mac_buf.size = sess_misc->mac_len;
3141                                 fc_params.mac_buf.vaddr =
3142                                   sym_op->aead.digest.data;
3143                                 fc_params.mac_buf.dma_addr =
3144                                  sym_op->aead.digest.phys_addr;
3145                                 inplace = 0;
3146                         }
3147                 }
3148         } else {
3149                 d_offs = sym_op->cipher.data.offset;
3150                 d_lens = sym_op->cipher.data.length;
3151                 mc_hash_off = sym_op->cipher.data.offset +
3152                               sym_op->cipher.data.length;
3153                 d_offs = (d_offs << 16) | sym_op->auth.data.offset;
3154                 d_lens = (d_lens << 32) | sym_op->auth.data.length;
3155
3156                 if (mc_hash_off < (sym_op->auth.data.offset +
3157                                    sym_op->auth.data.length)){
3158                         mc_hash_off = (sym_op->auth.data.offset +
3159                                        sym_op->auth.data.length);
3160                 }
3161                 /* for gmac, salt should be updated like in gcm */
3162                 if (unlikely(sess_misc->is_gmac)) {
3163                         uint8_t *salt;
3164                         salt = fc_params.iv_buf;
3165                         if (unlikely(*(uint32_t *)salt != sess_misc->salt)) {
3166                                 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt);
3167                                 sess_misc->salt = *(uint32_t *)salt;
3168                         }
3169                         fc_params.iv_buf = salt + 4;
3170                 }
3171                 if (likely(sess_misc->mac_len)) {
3172                         struct rte_mbuf *m;
3173
3174                         m = (cpt_op & CPT_OP_ENCODE) ? m_dst : m_src;
3175                         if (!m)
3176                                 m = m_src;
3177
3178                         /* hmac immediately following data is best case */
3179                         if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) +
3180                             mc_hash_off !=
3181                              (uint8_t *)sym_op->auth.digest.data)) {
3182                                 flags |= VALID_MAC_BUF;
3183                                 fc_params.mac_buf.size =
3184                                         sess_misc->mac_len;
3185                                 fc_params.mac_buf.vaddr =
3186                                         sym_op->auth.digest.data;
3187                                 fc_params.mac_buf.dma_addr =
3188                                 sym_op->auth.digest.phys_addr;
3189                                 inplace = 0;
3190                         }
3191                 }
3192         }
3193         fc_params.ctx_buf.vaddr = SESS_PRIV(sess_misc);
3194         fc_params.ctx_buf.dma_addr = sess_misc->ctx_dma_addr;
3195
3196         if (unlikely(sess_misc->is_null || sess_misc->cpt_op == CPT_OP_DECODE))
3197                 inplace = 0;
3198
3199         if (likely(!m_dst && inplace)) {
3200                 /* Case of single buffer without AAD buf or
3201                  * separate mac buf in place and
3202                  * not air crypto
3203                  */
3204                 fc_params.dst_iov = fc_params.src_iov = (void *)src;
3205
3206                 if (unlikely(prepare_iov_from_pkt_inplace(m_src,
3207                                                           &fc_params,
3208                                                           &flags))) {
3209                         CPT_LOG_DP_ERR("Prepare inplace src iov failed");
3210                         ret = -EINVAL;
3211                         goto err_exit;
3212                 }
3213
3214         } else {
3215                 /* Out of place processing */
3216                 fc_params.src_iov = (void *)src;
3217                 fc_params.dst_iov = (void *)dst;
3218
3219                 /* Store SG I/O in the api for reuse */
3220                 if (prepare_iov_from_pkt(m_src, fc_params.src_iov, 0)) {
3221                         CPT_LOG_DP_ERR("Prepare src iov failed");
3222                         ret = -EINVAL;
3223                         goto err_exit;
3224                 }
3225
3226                 if (unlikely(m_dst != NULL)) {
3227                         uint32_t pkt_len;
3228
3229                         /* Try to make room as much as src has */
3230                         pkt_len = rte_pktmbuf_pkt_len(m_dst);
3231
3232                         if (unlikely(pkt_len < rte_pktmbuf_pkt_len(m_src))) {
3233                                 pkt_len = rte_pktmbuf_pkt_len(m_src) - pkt_len;
3234                                 if (!rte_pktmbuf_append(m_dst, pkt_len)) {
3235                                         CPT_LOG_DP_ERR("Not enough space in "
3236                                                        "m_dst %p, need %u"
3237                                                        " more",
3238                                                        m_dst, pkt_len);
3239                                         ret = -EINVAL;
3240                                         goto err_exit;
3241                                 }
3242                         }
3243
3244                         if (prepare_iov_from_pkt(m_dst, fc_params.dst_iov, 0)) {
3245                                 CPT_LOG_DP_ERR("Prepare dst iov failed for "
3246                                                "m_dst %p", m_dst);
3247                                 ret = -EINVAL;
3248                                 goto err_exit;
3249                         }
3250                 } else {
3251                         fc_params.dst_iov = (void *)src;
3252                 }
3253         }
3254
3255         if (likely(flags & SINGLE_BUF_HEADTAILROOM))
3256                 mdata = alloc_op_meta(m_src, &fc_params.meta_buf,
3257                                       m_info->lb_mlen, m_info->pool);
3258         else
3259                 mdata = alloc_op_meta(NULL, &fc_params.meta_buf,
3260                                       m_info->sg_mlen, m_info->pool);
3261
3262         if (unlikely(mdata == NULL)) {
3263                 CPT_LOG_DP_ERR("Error allocating meta buffer for request");
3264                 ret = -ENOMEM;
3265                 goto err_exit;
3266         }
3267
3268         op = (uintptr_t *)((uintptr_t)mdata & (uintptr_t)~1ull);
3269         op[0] = (uintptr_t)mdata;
3270         op[1] = (uintptr_t)cop;
3271         op[2] = op[3] = 0; /* Used to indicate auth verify */
3272         space += 4 * sizeof(uint64_t);
3273
3274         fc_params.meta_buf.vaddr = (uint8_t *)op + space;
3275         fc_params.meta_buf.dma_addr += space;
3276         fc_params.meta_buf.size -= space;
3277
3278         /* Finally prepare the instruction */
3279         if (cpt_op & CPT_OP_ENCODE)
3280                 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens,
3281                                                  &fc_params, op);
3282         else
3283                 *prep_req = cpt_fc_dec_hmac_prep(flags, d_offs, d_lens,
3284                                                  &fc_params, op);
3285
3286         if (unlikely(*prep_req == NULL)) {
3287                 CPT_LOG_DP_ERR("Preparing request failed due to bad input arg");
3288                 ret = -EINVAL;
3289                 goto free_mdata_and_exit;
3290         }
3291
3292         *mdata_ptr = mdata;
3293
3294         return 0;
3295
3296 free_mdata_and_exit:
3297         free_op_meta(mdata, m_info->pool);
3298 err_exit:
3299         return ret;
3300 }
3301
3302 static __rte_always_inline void
3303 compl_auth_verify(struct rte_crypto_op *op,
3304                       uint8_t *gen_mac,
3305                       uint64_t mac_len)
3306 {
3307         uint8_t *mac;
3308         struct rte_crypto_sym_op *sym_op = op->sym;
3309
3310         if (sym_op->auth.digest.data)
3311                 mac = sym_op->auth.digest.data;
3312         else
3313                 mac = rte_pktmbuf_mtod_offset(sym_op->m_src,
3314                                               uint8_t *,
3315                                               sym_op->auth.data.length +
3316                                               sym_op->auth.data.offset);
3317         if (!mac) {
3318                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
3319                 return;
3320         }
3321
3322         if (memcmp(mac, gen_mac, mac_len))
3323                 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
3324         else
3325                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
3326 }
3327
3328 static __rte_always_inline int
3329 instance_session_cfg(struct rte_crypto_sym_xform *xform, void *sess)
3330 {
3331         struct rte_crypto_sym_xform *chain;
3332
3333         CPT_PMD_INIT_FUNC_TRACE();
3334
3335         if (cpt_is_algo_supported(xform))
3336                 goto err;
3337
3338         chain = xform;
3339         while (chain) {
3340                 switch (chain->type) {
3341                 case RTE_CRYPTO_SYM_XFORM_AEAD:
3342                         if (fill_sess_aead(chain, sess))
3343                                 goto err;
3344                         break;
3345                 case RTE_CRYPTO_SYM_XFORM_CIPHER:
3346                         if (fill_sess_cipher(chain, sess))
3347                                 goto err;
3348                         break;
3349                 case RTE_CRYPTO_SYM_XFORM_AUTH:
3350                         if (chain->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
3351                                 if (fill_sess_gmac(chain, sess))
3352                                         goto err;
3353                         } else {
3354                                 if (fill_sess_auth(chain, sess))
3355                                         goto err;
3356                         }
3357                         break;
3358                 default:
3359                         CPT_LOG_DP_ERR("Invalid crypto xform type");
3360                         break;
3361                 }
3362                 chain = chain->next;
3363         }
3364
3365         return 0;
3366
3367 err:
3368         return -1;
3369 }
3370
3371 static __rte_always_inline void
3372 find_kasumif9_direction_and_length(uint8_t *src,
3373                                    uint32_t counter_num_bytes,
3374                                    uint32_t *addr_length_in_bits,
3375                                    uint8_t *addr_direction)
3376 {
3377         uint8_t found = 0;
3378         uint32_t pos;
3379         uint8_t last_byte;
3380         while (!found && counter_num_bytes > 0) {
3381                 counter_num_bytes--;
3382                 if (src[counter_num_bytes] == 0x00)
3383                         continue;
3384                 pos = rte_bsf32(src[counter_num_bytes]);
3385                 if (pos == 7) {
3386                         if (likely(counter_num_bytes > 0)) {
3387                                 last_byte = src[counter_num_bytes - 1];
3388                                 *addr_direction  =  last_byte & 0x1;
3389                                 *addr_length_in_bits = counter_num_bytes * 8
3390                                                         - 1;
3391                         }
3392                 } else {
3393                         last_byte = src[counter_num_bytes];
3394                         *addr_direction = (last_byte >> (pos + 1)) & 0x1;
3395                         *addr_length_in_bits = counter_num_bytes * 8
3396                                                 + (8 - (pos + 2));
3397                 }
3398                 found = 1;
3399         }
3400 }
3401
3402 /*
3403  * This handles all auth only except AES_GMAC
3404  */
3405 static __rte_always_inline int
3406 fill_digest_params(struct rte_crypto_op *cop,
3407                    struct cpt_sess_misc *sess,
3408                    struct cpt_qp_meta_info *m_info,
3409                    void **mdata_ptr,
3410                    void **prep_req)
3411 {
3412         uint32_t space = 0;
3413         struct rte_crypto_sym_op *sym_op = cop->sym;
3414         void *mdata;
3415         phys_addr_t mphys;
3416         uint64_t *op;
3417         uint32_t auth_range_off;
3418         uint32_t flags = 0;
3419         uint64_t d_offs = 0, d_lens;
3420         struct rte_mbuf *m_src, *m_dst;
3421         uint16_t auth_op = sess->cpt_op & CPT_OP_AUTH_MASK;
3422         uint16_t mac_len = sess->mac_len;
3423         fc_params_t params;
3424         char src[SRC_IOV_SIZE];
3425         uint8_t iv_buf[16];
3426         int ret;
3427
3428         memset(&params, 0, sizeof(fc_params_t));
3429
3430         m_src = sym_op->m_src;
3431
3432         /* For just digest lets force mempool alloc */
3433         mdata = alloc_op_meta(NULL, &params.meta_buf, m_info->sg_mlen,
3434                               m_info->pool);
3435         if (mdata == NULL) {
3436                 ret = -ENOMEM;
3437                 goto err_exit;
3438         }
3439
3440         mphys = params.meta_buf.dma_addr;
3441
3442         op = mdata;
3443         op[0] = (uintptr_t)mdata;
3444         op[1] = (uintptr_t)cop;
3445         op[2] = op[3] = 0; /* Used to indicate auth verify */
3446         space += 4 * sizeof(uint64_t);
3447
3448         auth_range_off = sym_op->auth.data.offset;
3449
3450         flags = VALID_MAC_BUF;
3451         params.src_iov = (void *)src;
3452         if (unlikely(sess->zsk_flag)) {
3453                 /*
3454                  * Since for Zuc, Kasumi, Snow3g offsets are in bits
3455                  * we will send pass through even for auth only case,
3456                  * let MC handle it
3457                  */
3458                 d_offs = auth_range_off;
3459                 auth_range_off = 0;
3460                 params.auth_iv_buf = rte_crypto_op_ctod_offset(cop,
3461                                         uint8_t *, sess->auth_iv_offset);
3462                 if (sess->zsk_flag == K_F9) {
3463                         uint32_t length_in_bits, num_bytes;
3464                         uint8_t *src, direction = 0;
3465
3466                         memcpy(iv_buf, rte_pktmbuf_mtod(cop->sym->m_src,
3467                                                         uint8_t *), 8);
3468                         /*
3469                          * This is kasumi f9, take direction from
3470                          * source buffer
3471                          */
3472                         length_in_bits = cop->sym->auth.data.length;
3473                         num_bytes = (length_in_bits >> 3);
3474                         src = rte_pktmbuf_mtod(cop->sym->m_src, uint8_t *);
3475                         find_kasumif9_direction_and_length(src,
3476                                                 num_bytes,
3477                                                 &length_in_bits,
3478                                                 &direction);
3479                         length_in_bits -= 64;
3480                         cop->sym->auth.data.offset += 64;
3481                         d_offs = cop->sym->auth.data.offset;
3482                         auth_range_off = d_offs / 8;
3483                         cop->sym->auth.data.length = length_in_bits;
3484
3485                         /* Store it at end of auth iv */
3486                         iv_buf[8] = direction;
3487                         params.auth_iv_buf = iv_buf;
3488                 }
3489         }
3490
3491         d_lens = sym_op->auth.data.length;
3492
3493         params.ctx_buf.vaddr = SESS_PRIV(sess);
3494         params.ctx_buf.dma_addr = sess->ctx_dma_addr;
3495
3496         if (auth_op == CPT_OP_AUTH_GENERATE) {
3497                 if (sym_op->auth.digest.data) {
3498                         /*
3499                          * Digest to be generated
3500                          * in separate buffer
3501                          */
3502                         params.mac_buf.size =
3503                                 sess->mac_len;
3504                         params.mac_buf.vaddr =
3505                                 sym_op->auth.digest.data;
3506                         params.mac_buf.dma_addr =
3507                                 sym_op->auth.digest.phys_addr;
3508                 } else {
3509                         uint32_t off = sym_op->auth.data.offset +
3510                                 sym_op->auth.data.length;
3511                         int32_t dlen, space;
3512
3513                         m_dst = sym_op->m_dst ?
3514                                 sym_op->m_dst : sym_op->m_src;
3515                         dlen = rte_pktmbuf_pkt_len(m_dst);
3516
3517                         space = off + mac_len - dlen;
3518                         if (space > 0)
3519                                 if (!rte_pktmbuf_append(m_dst, space)) {
3520                                         CPT_LOG_DP_ERR("Failed to extend "
3521                                                        "mbuf by %uB", space);
3522                                         ret = -EINVAL;
3523                                         goto free_mdata_and_exit;
3524                                 }
3525
3526                         params.mac_buf.vaddr =
3527                                 rte_pktmbuf_mtod_offset(m_dst, void *, off);
3528                         params.mac_buf.dma_addr =
3529                                 rte_pktmbuf_mtophys_offset(m_dst, off);
3530                         params.mac_buf.size = mac_len;
3531                 }
3532         } else {
3533                 /* Need space for storing generated mac */
3534                 params.mac_buf.vaddr = (uint8_t *)mdata + space;
3535                 params.mac_buf.dma_addr = mphys + space;
3536                 params.mac_buf.size = mac_len;
3537                 space += RTE_ALIGN_CEIL(mac_len, 8);
3538                 op[2] = (uintptr_t)params.mac_buf.vaddr;
3539                 op[3] = mac_len;
3540         }
3541
3542         params.meta_buf.vaddr = (uint8_t *)mdata + space;
3543         params.meta_buf.dma_addr = mphys + space;
3544         params.meta_buf.size -= space;
3545
3546         /* Out of place processing */
3547         params.src_iov = (void *)src;
3548
3549         /*Store SG I/O in the api for reuse */
3550         if (prepare_iov_from_pkt(m_src, params.src_iov, auth_range_off)) {
3551                 CPT_LOG_DP_ERR("Prepare src iov failed");
3552                 ret = -EINVAL;
3553                 goto free_mdata_and_exit;
3554         }
3555
3556         *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, &params, op);
3557         if (unlikely(*prep_req == NULL)) {
3558                 ret = -EINVAL;
3559                 goto free_mdata_and_exit;
3560         }
3561
3562         *mdata_ptr = mdata;
3563
3564         return 0;
3565
3566 free_mdata_and_exit:
3567         free_op_meta(mdata, m_info->pool);
3568 err_exit:
3569         return ret;
3570 }
3571
3572 #endif /*_CPT_UCODE_H_ */