drivers/crypto/dpaa_sec/dpaa_sec_raw_dp.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2021 NXP
   3  */
   4
   5 #include <rte_byteorder.h>
   6 #include <rte_common.h>
   7 #include <cryptodev_pmd.h>
   8 #include <rte_crypto.h>
   9 #include <rte_cryptodev.h>
  10 #ifdef RTE_LIB_SECURITY
  11 #include <rte_security_driver.h>
  12 #endif
  13
  14 /* RTA header files */
  15 #include <desc/algo.h>
  16 #include <desc/ipsec.h>
  17
  18 #include <rte_dpaa_bus.h>
  19 #include <dpaa_sec.h>
  20 #include <dpaa_sec_log.h>
  21
  22 struct dpaa_sec_raw_dp_ctx {
  23         dpaa_sec_session *session;
  24         uint32_t tail;
  25         uint32_t head;
  26         uint16_t cached_enqueue;
  27         uint16_t cached_dequeue;
  28 };
  29
  30 static inline int
  31 is_encode(dpaa_sec_session *ses)
  32 {
  33         return ses->dir == DIR_ENC;
  34 }
  35
  36 static inline int is_decode(dpaa_sec_session *ses)
  37 {
  38         return ses->dir == DIR_DEC;
  39 }
  40
  41 static __rte_always_inline int
  42 dpaa_sec_raw_enqueue_done(void *qp_data, uint8_t *drv_ctx, uint32_t n)
  43 {
  44         RTE_SET_USED(qp_data);
  45         RTE_SET_USED(drv_ctx);
  46         RTE_SET_USED(n);
  47
  48         return 0;
  49 }
  50
  51 static __rte_always_inline int
  52 dpaa_sec_raw_dequeue_done(void *qp_data, uint8_t *drv_ctx, uint32_t n)
  53 {
  54         RTE_SET_USED(qp_data);
  55         RTE_SET_USED(drv_ctx);
  56         RTE_SET_USED(n);
  57
  58         return 0;
  59 }
  60
  61 static inline struct dpaa_sec_op_ctx *
  62 dpaa_sec_alloc_raw_ctx(dpaa_sec_session *ses, int sg_count)
  63 {
  64         struct dpaa_sec_op_ctx *ctx;
  65         int i, retval;
  66
  67         retval = rte_mempool_get(
  68                         ses->qp[rte_lcore_id() % MAX_DPAA_CORES]->ctx_pool,
  69                         (void **)(&ctx));
  70         if (!ctx || retval) {
  71                 DPAA_SEC_DP_WARN("Alloc sec descriptor failed!");
  72                 return NULL;
  73         }
  74         /*
  75          * Clear SG memory. There are 16 SG entries of 16 Bytes each.
  76          * one call to dcbz_64() clear 64 bytes, hence calling it 4 times
  77          * to clear all the SG entries. dpaa_sec_alloc_ctx() is called for
  78          * each packet, memset is costlier than dcbz_64().
  79          */
  80         for (i = 0; i < sg_count && i < MAX_JOB_SG_ENTRIES; i += 4)
  81                 dcbz_64(&ctx->job.sg[i]);
  82
  83         ctx->ctx_pool = ses->qp[rte_lcore_id() % MAX_DPAA_CORES]->ctx_pool;
  84         ctx->vtop_offset = (size_t) ctx - rte_mempool_virt2iova(ctx);
  85
  86         return ctx;
  87 }
  88
  89 static struct dpaa_sec_job *
  90 build_dpaa_raw_dp_auth_fd(uint8_t *drv_ctx,
  91                         struct rte_crypto_sgl *sgl,
  92                         struct rte_crypto_sgl *dest_sgl,
  93                         struct rte_crypto_va_iova_ptr *iv,
  94                         struct rte_crypto_va_iova_ptr *digest,
  95                         struct rte_crypto_va_iova_ptr *auth_iv,
  96                         union rte_crypto_sym_ofs ofs,
  97                         void *userdata,
  98                         struct qm_fd *fd)
  99 {
 100         RTE_SET_USED(dest_sgl);
 101         RTE_SET_USED(iv);
 102         RTE_SET_USED(auth_iv);
 103         RTE_SET_USED(fd);
 104
 105         dpaa_sec_session *ses =
 106                 ((struct dpaa_sec_raw_dp_ctx *)drv_ctx)->session;
 107         struct dpaa_sec_job *cf;
 108         struct dpaa_sec_op_ctx *ctx;
 109         struct qm_sg_entry *sg, *out_sg, *in_sg;
 110         phys_addr_t start_addr;
 111         uint8_t *old_digest, extra_segs;
 112         int data_len, data_offset, total_len = 0;
 113         unsigned int i;
 114
 115         for (i = 0; i < sgl->num; i++)
 116                 total_len += sgl->vec[i].len;
 117
 118         data_len = total_len - ofs.ofs.auth.head - ofs.ofs.auth.tail;
 119         data_offset =  ofs.ofs.auth.head;
 120
 121         /* Support only length in bits for SNOW3G and ZUC */
 122
 123         if (is_decode(ses))
 124                 extra_segs = 3;
 125         else
 126                 extra_segs = 2;
 127
 128         if (sgl->num > MAX_SG_ENTRIES) {
 129                 DPAA_SEC_DP_ERR("Auth: Max sec segs supported is %d",
 130                                 MAX_SG_ENTRIES);
 131                 return NULL;
 132         }
 133         ctx = dpaa_sec_alloc_raw_ctx(ses, sgl->num * 2 + extra_segs);
 134         if (!ctx)
 135                 return NULL;
 136
 137         cf = &ctx->job;
 138         ctx->userdata = (void *)userdata;
 139         old_digest = ctx->digest;
 140
 141         /* output */
 142         out_sg = &cf->sg[0];
 143         qm_sg_entry_set64(out_sg, digest->iova);
 144         out_sg->length = ses->digest_length;
 145         cpu_to_hw_sg(out_sg);
 146
 147         /* input */
 148         in_sg = &cf->sg[1];
 149         /* need to extend the input to a compound frame */
 150         in_sg->extension = 1;
 151         in_sg->final = 1;
 152         in_sg->length = data_len;
 153         qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(&cf->sg[2]));
 154
 155         /* 1st seg */
 156         sg = in_sg + 1;
 157
 158         if (ses->iv.length) {
 159                 uint8_t *iv_ptr;
 160
 161                 iv_ptr = rte_crypto_op_ctod_offset(userdata, uint8_t *,
 162                                                    ses->iv.offset);
 163
 164                 if (ses->auth_alg == RTE_CRYPTO_AUTH_SNOW3G_UIA2) {
 165                         iv_ptr = conv_to_snow_f9_iv(iv_ptr);
 166                         sg->length = 12;
 167                 } else if (ses->auth_alg == RTE_CRYPTO_AUTH_ZUC_EIA3) {
 168                         iv_ptr = conv_to_zuc_eia_iv(iv_ptr);
 169                         sg->length = 8;
 170                 } else {
 171                         sg->length = ses->iv.length;
 172                 }
 173                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(iv_ptr));
 174                 in_sg->length += sg->length;
 175                 cpu_to_hw_sg(sg);
 176                 sg++;
 177         }
 178
 179         qm_sg_entry_set64(sg, sgl->vec[0].iova);
 180         sg->offset = data_offset;
 181
 182         if (data_len <= (int)(sgl->vec[0].len - data_offset)) {
 183                 sg->length = data_len;
 184         } else {
 185                 sg->length = sgl->vec[0].len - data_offset;
 186
 187                 /* remaining i/p segs */
 188                 for (i = 1; i < sgl->num; i++) {
 189                         cpu_to_hw_sg(sg);
 190                         sg++;
 191                         qm_sg_entry_set64(sg, sgl->vec[i].iova);
 192                         if (data_len > (int)sgl->vec[i].len)
 193                                 sg->length = sgl->vec[0].len;
 194                         else
 195                                 sg->length = data_len;
 196
 197                         data_len = data_len - sg->length;
 198                         if (data_len < 1)
 199                                 break;
 200                 }
 201         }
 202
 203         if (is_decode(ses)) {
 204                 /* Digest verification case */
 205                 cpu_to_hw_sg(sg);
 206                 sg++;
 207                 rte_memcpy(old_digest, digest->va,
 208                                 ses->digest_length);
 209                 start_addr = rte_dpaa_mem_vtop(old_digest);
 210                 qm_sg_entry_set64(sg, start_addr);
 211                 sg->length = ses->digest_length;
 212                 in_sg->length += ses->digest_length;
 213         }
 214         sg->final = 1;
 215         cpu_to_hw_sg(sg);
 216         cpu_to_hw_sg(in_sg);
 217
 218         return cf;
 219 }
 220
 221 static inline struct dpaa_sec_job *
 222 build_raw_cipher_auth_gcm_sg(uint8_t *drv_ctx,
 223                         struct rte_crypto_sgl *sgl,
 224                         struct rte_crypto_sgl *dest_sgl,
 225                         struct rte_crypto_va_iova_ptr *iv,
 226                         struct rte_crypto_va_iova_ptr *digest,
 227                         struct rte_crypto_va_iova_ptr *auth_iv,
 228                         union rte_crypto_sym_ofs ofs,
 229                         void *userdata,
 230                         struct qm_fd *fd)
 231 {
 232         dpaa_sec_session *ses =
 233                 ((struct dpaa_sec_raw_dp_ctx *)drv_ctx)->session;
 234         struct dpaa_sec_job *cf;
 235         struct dpaa_sec_op_ctx *ctx;
 236         struct qm_sg_entry *sg, *out_sg, *in_sg;
 237         uint8_t extra_req_segs;
 238         uint8_t *IV_ptr = iv->va;
 239         int data_len = 0, aead_len = 0;
 240         unsigned int i;
 241
 242         for (i = 0; i < sgl->num; i++)
 243                 data_len += sgl->vec[i].len;
 244
 245         extra_req_segs = 4;
 246         aead_len = data_len - ofs.ofs.cipher.head - ofs.ofs.cipher.tail;
 247
 248         if (ses->auth_only_len)
 249                 extra_req_segs++;
 250
 251         if (sgl->num > MAX_SG_ENTRIES) {
 252                 DPAA_SEC_DP_ERR("AEAD: Max sec segs supported is %d",
 253                                 MAX_SG_ENTRIES);
 254                 return NULL;
 255         }
 256
 257         ctx = dpaa_sec_alloc_raw_ctx(ses,  sgl->num * 2 + extra_req_segs);
 258         if (!ctx)
 259                 return NULL;
 260
 261         cf = &ctx->job;
 262         ctx->userdata = (void *)userdata;
 263
 264         rte_prefetch0(cf->sg);
 265
 266         /* output */
 267         out_sg = &cf->sg[0];
 268         out_sg->extension = 1;
 269         if (is_encode(ses))
 270                 out_sg->length = aead_len + ses->digest_length;
 271         else
 272                 out_sg->length = aead_len;
 273
 274         /* output sg entries */
 275         sg = &cf->sg[2];
 276         qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(sg));
 277         cpu_to_hw_sg(out_sg);
 278
 279         if (dest_sgl) {
 280                 /* 1st seg */
 281                 qm_sg_entry_set64(sg, dest_sgl->vec[0].iova);
 282                 sg->length = dest_sgl->vec[0].len - ofs.ofs.cipher.head;
 283                 sg->offset = ofs.ofs.cipher.head;
 284
 285                 /* Successive segs */
 286                 for (i = 1; i < dest_sgl->num; i++) {
 287                         cpu_to_hw_sg(sg);
 288                         sg++;
 289                         qm_sg_entry_set64(sg, dest_sgl->vec[i].iova);
 290                         sg->length = dest_sgl->vec[i].len;
 291                 }
 292         } else {
 293                 /* 1st seg */
 294                 qm_sg_entry_set64(sg, sgl->vec[0].iova);
 295                 sg->length = sgl->vec[0].len - ofs.ofs.cipher.head;
 296                 sg->offset = ofs.ofs.cipher.head;
 297
 298                 /* Successive segs */
 299                 for (i = 1; i < sgl->num; i++) {
 300                         cpu_to_hw_sg(sg);
 301                         sg++;
 302                         qm_sg_entry_set64(sg, sgl->vec[i].iova);
 303                         sg->length = sgl->vec[i].len;
 304                 }
 305
 306         }
 307
 308         if (is_encode(ses)) {
 309                 cpu_to_hw_sg(sg);
 310                 /* set auth output */
 311                 sg++;
 312                 qm_sg_entry_set64(sg, digest->iova);
 313                 sg->length = ses->digest_length;
 314         }
 315         sg->final = 1;
 316         cpu_to_hw_sg(sg);
 317
 318         /* input */
 319         in_sg = &cf->sg[1];
 320         in_sg->extension = 1;
 321         in_sg->final = 1;
 322         if (is_encode(ses))
 323                 in_sg->length = ses->iv.length + aead_len
 324                                                 + ses->auth_only_len;
 325         else
 326                 in_sg->length = ses->iv.length + aead_len
 327                                 + ses->auth_only_len + ses->digest_length;
 328
 329         /* input sg entries */
 330         sg++;
 331         qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg));
 332         cpu_to_hw_sg(in_sg);
 333
 334         /* 1st seg IV */
 335         qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
 336         sg->length = ses->iv.length;
 337         cpu_to_hw_sg(sg);
 338
 339         /* 2 seg auth only */
 340         if (ses->auth_only_len) {
 341                 sg++;
 342                 qm_sg_entry_set64(sg, auth_iv->iova);
 343                 sg->length = ses->auth_only_len;
 344                 cpu_to_hw_sg(sg);
 345         }
 346
 347         /* 3rd seg */
 348         sg++;
 349         qm_sg_entry_set64(sg, sgl->vec[0].iova);
 350         sg->length = sgl->vec[0].len - ofs.ofs.cipher.head;
 351         sg->offset = ofs.ofs.cipher.head;
 352
 353         /* Successive segs */
 354         for (i = 1; i < sgl->num; i++) {
 355                 cpu_to_hw_sg(sg);
 356                 sg++;
 357                 qm_sg_entry_set64(sg, sgl->vec[i].iova);
 358                 sg->length =  sgl->vec[i].len;
 359         }
 360
 361         if (is_decode(ses)) {
 362                 cpu_to_hw_sg(sg);
 363                 sg++;
 364                 memcpy(ctx->digest, digest->va,
 365                         ses->digest_length);
 366                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest));
 367                 sg->length = ses->digest_length;
 368         }
 369         sg->final = 1;
 370         cpu_to_hw_sg(sg);
 371
 372         if (ses->auth_only_len)
 373                 fd->cmd = 0x80000000 | ses->auth_only_len;
 374
 375         return cf;
 376 }
 377
 378 static inline struct dpaa_sec_job *
 379 build_dpaa_raw_dp_chain_fd(uint8_t *drv_ctx,
 380                         struct rte_crypto_sgl *sgl,
 381                         struct rte_crypto_sgl *dest_sgl,
 382                         struct rte_crypto_va_iova_ptr *iv,
 383                         struct rte_crypto_va_iova_ptr *digest,
 384                         struct rte_crypto_va_iova_ptr *auth_iv,
 385                         union rte_crypto_sym_ofs ofs,
 386                         void *userdata,
 387                         struct qm_fd *fd)
 388 {
 389         RTE_SET_USED(auth_iv);
 390
 391         dpaa_sec_session *ses =
 392                 ((struct dpaa_sec_raw_dp_ctx *)drv_ctx)->session;
 393         struct dpaa_sec_job *cf;
 394         struct dpaa_sec_op_ctx *ctx;
 395         struct qm_sg_entry *sg, *out_sg, *in_sg;
 396         uint8_t *IV_ptr = iv->va;
 397         unsigned int i;
 398         uint16_t auth_hdr_len = ofs.ofs.cipher.head -
 399                                 ofs.ofs.auth.head;
 400         uint16_t auth_tail_len = ofs.ofs.auth.tail;
 401         uint32_t auth_only_len = (auth_tail_len << 16) | auth_hdr_len;
 402         int data_len = 0, auth_len = 0, cipher_len = 0;
 403
 404         for (i = 0; i < sgl->num; i++)
 405                 data_len += sgl->vec[i].len;
 406
 407         cipher_len = data_len - ofs.ofs.cipher.head - ofs.ofs.cipher.tail;
 408         auth_len = data_len - ofs.ofs.auth.head - ofs.ofs.auth.tail;
 409
 410         if (sgl->num > MAX_SG_ENTRIES) {
 411                 DPAA_SEC_DP_ERR("Cipher-Auth: Max sec segs supported is %d",
 412                                 MAX_SG_ENTRIES);
 413                 return NULL;
 414         }
 415
 416         ctx = dpaa_sec_alloc_raw_ctx(ses, sgl->num * 2 + 4);
 417         if (!ctx)
 418                 return NULL;
 419
 420         cf = &ctx->job;
 421         ctx->userdata = (void *)userdata;
 422
 423         rte_prefetch0(cf->sg);
 424
 425         /* output */
 426         out_sg = &cf->sg[0];
 427         out_sg->extension = 1;
 428         if (is_encode(ses))
 429                 out_sg->length = cipher_len + ses->digest_length;
 430         else
 431                 out_sg->length = cipher_len;
 432
 433         /* output sg entries */
 434         sg = &cf->sg[2];
 435         qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(sg));
 436         cpu_to_hw_sg(out_sg);
 437
 438         /* 1st seg */
 439         if (dest_sgl) {
 440                 qm_sg_entry_set64(sg, dest_sgl->vec[0].iova);
 441                 sg->length = dest_sgl->vec[0].len - ofs.ofs.cipher.head;
 442                 sg->offset = ofs.ofs.cipher.head;
 443
 444                 /* Successive segs */
 445                 for (i = 1; i < dest_sgl->num; i++) {
 446                         cpu_to_hw_sg(sg);
 447                         sg++;
 448                         qm_sg_entry_set64(sg, dest_sgl->vec[i].iova);
 449                         sg->length = dest_sgl->vec[i].len;
 450                 }
 451         } else {
 452                 qm_sg_entry_set64(sg, sgl->vec[0].iova);
 453                 sg->length = sgl->vec[0].len - ofs.ofs.cipher.head;
 454                 sg->offset = ofs.ofs.cipher.head;
 455
 456                 /* Successive segs */
 457                 for (i = 1; i < sgl->num; i++) {
 458                         cpu_to_hw_sg(sg);
 459                         sg++;
 460                         qm_sg_entry_set64(sg, sgl->vec[i].iova);
 461                         sg->length = sgl->vec[i].len;
 462                 }
 463         }
 464
 465         if (is_encode(ses)) {
 466                 cpu_to_hw_sg(sg);
 467                 /* set auth output */
 468                 sg++;
 469                 qm_sg_entry_set64(sg, digest->iova);
 470                 sg->length = ses->digest_length;
 471         }
 472         sg->final = 1;
 473         cpu_to_hw_sg(sg);
 474
 475         /* input */
 476         in_sg = &cf->sg[1];
 477         in_sg->extension = 1;
 478         in_sg->final = 1;
 479         if (is_encode(ses))
 480                 in_sg->length = ses->iv.length + auth_len;
 481         else
 482                 in_sg->length = ses->iv.length + auth_len
 483                                                 + ses->digest_length;
 484
 485         /* input sg entries */
 486         sg++;
 487         qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg));
 488         cpu_to_hw_sg(in_sg);
 489
 490         /* 1st seg IV */
 491         qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
 492         sg->length = ses->iv.length;
 493         cpu_to_hw_sg(sg);
 494
 495         /* 2 seg */
 496         sg++;
 497         qm_sg_entry_set64(sg, sgl->vec[0].iova);
 498         sg->length = sgl->vec[0].len - ofs.ofs.auth.head;
 499         sg->offset = ofs.ofs.auth.head;
 500
 501         /* Successive segs */
 502         for (i = 1; i < sgl->num; i++) {
 503                 cpu_to_hw_sg(sg);
 504                 sg++;
 505                 qm_sg_entry_set64(sg, sgl->vec[i].iova);
 506                 sg->length = sgl->vec[i].len;
 507         }
 508
 509         if (is_decode(ses)) {
 510                 cpu_to_hw_sg(sg);
 511                 sg++;
 512                 memcpy(ctx->digest, digest->va,
 513                         ses->digest_length);
 514                 qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(ctx->digest));
 515                 sg->length = ses->digest_length;
 516         }
 517         sg->final = 1;
 518         cpu_to_hw_sg(sg);
 519
 520         if (auth_only_len)
 521                 fd->cmd = 0x80000000 | auth_only_len;
 522
 523         return cf;
 524 }
 525
 526 static struct dpaa_sec_job *
 527 build_dpaa_raw_dp_cipher_fd(uint8_t *drv_ctx,
 528                         struct rte_crypto_sgl *sgl,
 529                         struct rte_crypto_sgl *dest_sgl,
 530                         struct rte_crypto_va_iova_ptr *iv,
 531                         struct rte_crypto_va_iova_ptr *digest,
 532                         struct rte_crypto_va_iova_ptr *auth_iv,
 533                         union rte_crypto_sym_ofs ofs,
 534                         void *userdata,
 535                         struct qm_fd *fd)
 536 {
 537         RTE_SET_USED(digest);
 538         RTE_SET_USED(auth_iv);
 539         RTE_SET_USED(fd);
 540
 541         dpaa_sec_session *ses =
 542                 ((struct dpaa_sec_raw_dp_ctx *)drv_ctx)->session;
 543         struct dpaa_sec_job *cf;
 544         struct dpaa_sec_op_ctx *ctx;
 545         struct qm_sg_entry *sg, *out_sg, *in_sg;
 546         unsigned int i;
 547         uint8_t *IV_ptr = iv->va;
 548         int data_len, total_len = 0, data_offset;
 549
 550         for (i = 0; i < sgl->num; i++)
 551                 total_len += sgl->vec[i].len;
 552
 553         data_len = total_len - ofs.ofs.cipher.head - ofs.ofs.cipher.tail;
 554         data_offset = ofs.ofs.cipher.head;
 555
 556         /* Support lengths in bits only for SNOW3G and ZUC */
 557         if (sgl->num > MAX_SG_ENTRIES) {
 558                 DPAA_SEC_DP_ERR("Cipher: Max sec segs supported is %d",
 559                                 MAX_SG_ENTRIES);
 560                 return NULL;
 561         }
 562
 563         ctx = dpaa_sec_alloc_raw_ctx(ses, sgl->num * 2 + 3);
 564         if (!ctx)
 565                 return NULL;
 566
 567         cf = &ctx->job;
 568         ctx->userdata = (void *)userdata;
 569
 570         /* output */
 571         out_sg = &cf->sg[0];
 572         out_sg->extension = 1;
 573         out_sg->length = data_len;
 574         qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(&cf->sg[2]));
 575         cpu_to_hw_sg(out_sg);
 576
 577         if (dest_sgl) {
 578                 /* 1st seg */
 579                 sg = &cf->sg[2];
 580                 qm_sg_entry_set64(sg, dest_sgl->vec[0].iova);
 581                 sg->length = dest_sgl->vec[0].len - data_offset;
 582                 sg->offset = data_offset;
 583
 584                 /* Successive segs */
 585                 for (i = 1; i < dest_sgl->num; i++) {
 586                         cpu_to_hw_sg(sg);
 587                         sg++;
 588                         qm_sg_entry_set64(sg, dest_sgl->vec[i].iova);
 589                         sg->length = dest_sgl->vec[i].len;
 590                 }
 591         } else {
 592                 /* 1st seg */
 593                 sg = &cf->sg[2];
 594                 qm_sg_entry_set64(sg, sgl->vec[0].iova);
 595                 sg->length = sgl->vec[0].len - data_offset;
 596                 sg->offset = data_offset;
 597
 598                 /* Successive segs */
 599                 for (i = 1; i < sgl->num; i++) {
 600                         cpu_to_hw_sg(sg);
 601                         sg++;
 602                         qm_sg_entry_set64(sg, sgl->vec[i].iova);
 603                         sg->length = sgl->vec[i].len;
 604                 }
 605
 606         }
 607         sg->final = 1;
 608         cpu_to_hw_sg(sg);
 609
 610         /* input */
 611         in_sg = &cf->sg[1];
 612         in_sg->extension = 1;
 613         in_sg->final = 1;
 614         in_sg->length = data_len + ses->iv.length;
 615
 616         sg++;
 617         qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg));
 618         cpu_to_hw_sg(in_sg);
 619
 620         /* IV */
 621         qm_sg_entry_set64(sg, rte_dpaa_mem_vtop(IV_ptr));
 622         sg->length = ses->iv.length;
 623         cpu_to_hw_sg(sg);
 624
 625         /* 1st seg */
 626         sg++;
 627         qm_sg_entry_set64(sg, sgl->vec[0].iova);
 628         sg->length = sgl->vec[0].len - data_offset;
 629         sg->offset = data_offset;
 630
 631         /* Successive segs */
 632         for (i = 1; i < sgl->num; i++) {
 633                 cpu_to_hw_sg(sg);
 634                 sg++;
 635                 qm_sg_entry_set64(sg, sgl->vec[i].iova);
 636                 sg->length = sgl->vec[i].len;
 637         }
 638         sg->final = 1;
 639         cpu_to_hw_sg(sg);
 640
 641         return cf;
 642 }
 643
 644 #ifdef RTE_LIBRTE_SECURITY
 645 static inline struct dpaa_sec_job *
 646 build_dpaa_raw_proto_sg(uint8_t *drv_ctx,
 647                         struct rte_crypto_sgl *sgl,
 648                         struct rte_crypto_sgl *dest_sgl,
 649                         struct rte_crypto_va_iova_ptr *iv,
 650                         struct rte_crypto_va_iova_ptr *digest,
 651                         struct rte_crypto_va_iova_ptr *auth_iv,
 652                         union rte_crypto_sym_ofs ofs,
 653                         void *userdata,
 654                         struct qm_fd *fd)
 655 {
 656         RTE_SET_USED(iv);
 657         RTE_SET_USED(digest);
 658         RTE_SET_USED(auth_iv);
 659         RTE_SET_USED(ofs);
 660
 661         dpaa_sec_session *ses =
 662                 ((struct dpaa_sec_raw_dp_ctx *)drv_ctx)->session;
 663         struct dpaa_sec_job *cf;
 664         struct dpaa_sec_op_ctx *ctx;
 665         struct qm_sg_entry *sg, *out_sg, *in_sg;
 666         uint32_t in_len = 0, out_len = 0;
 667         unsigned int i;
 668
 669         if (sgl->num > MAX_SG_ENTRIES) {
 670                 DPAA_SEC_DP_ERR("Proto: Max sec segs supported is %d",
 671                                 MAX_SG_ENTRIES);
 672                 return NULL;
 673         }
 674
 675         ctx = dpaa_sec_alloc_raw_ctx(ses, sgl->num * 2 + 4);
 676         if (!ctx)
 677                 return NULL;
 678         cf = &ctx->job;
 679         ctx->userdata = (void *)userdata;
 680         /* output */
 681         out_sg = &cf->sg[0];
 682         out_sg->extension = 1;
 683         qm_sg_entry_set64(out_sg, rte_dpaa_mem_vtop(&cf->sg[2]));
 684
 685         if (dest_sgl) {
 686                 /* 1st seg */
 687                 sg = &cf->sg[2];
 688                 qm_sg_entry_set64(sg, dest_sgl->vec[0].iova);
 689                 sg->offset = 0;
 690                 sg->length = dest_sgl->vec[0].len;
 691                 out_len += sg->length;
 692
 693                 for (i = 1; i < dest_sgl->num; i++) {
 694                 /* Successive segs */
 695                         cpu_to_hw_sg(sg);
 696                         sg++;
 697                         qm_sg_entry_set64(sg, dest_sgl->vec[i].iova);
 698                         sg->offset = 0;
 699                         sg->length = dest_sgl->vec[i].len;
 700                         out_len += sg->length;
 701                 }
 702                 sg->length = dest_sgl->vec[i - 1].tot_len;
 703         } else {
 704                 /* 1st seg */
 705                 sg = &cf->sg[2];
 706                 qm_sg_entry_set64(sg, sgl->vec[0].iova);
 707                 sg->offset = 0;
 708                 sg->length = sgl->vec[0].len;
 709                 out_len += sg->length;
 710
 711                 for (i = 1; i < sgl->num; i++) {
 712                 /* Successive segs */
 713                         cpu_to_hw_sg(sg);
 714                         sg++;
 715                         qm_sg_entry_set64(sg, sgl->vec[i].iova);
 716                         sg->offset = 0;
 717                         sg->length = sgl->vec[i].len;
 718                         out_len += sg->length;
 719                 }
 720                 sg->length = sgl->vec[i - 1].tot_len;
 721
 722         }
 723         out_len += sg->length;
 724         sg->final = 1;
 725         cpu_to_hw_sg(sg);
 726
 727         out_sg->length = out_len;
 728         cpu_to_hw_sg(out_sg);
 729
 730         /* input */
 731         in_sg = &cf->sg[1];
 732         in_sg->extension = 1;
 733         in_sg->final = 1;
 734         in_len = sgl->vec[0].len;
 735
 736         sg++;
 737         qm_sg_entry_set64(in_sg, rte_dpaa_mem_vtop(sg));
 738
 739         /* 1st seg */
 740         qm_sg_entry_set64(sg, sgl->vec[0].iova);
 741         sg->length = sgl->vec[0].len;
 742         sg->offset = 0;
 743
 744         /* Successive segs */
 745         for (i = 1; i < sgl->num; i++) {
 746                 cpu_to_hw_sg(sg);
 747                 sg++;
 748                 qm_sg_entry_set64(sg, sgl->vec[i].iova);
 749                 sg->length = sgl->vec[i].len;
 750                 sg->offset = 0;
 751                 in_len += sg->length;
 752         }
 753         sg->final = 1;
 754         cpu_to_hw_sg(sg);
 755
 756         in_sg->length = in_len;
 757         cpu_to_hw_sg(in_sg);
 758
 759         if ((ses->ctxt == DPAA_SEC_PDCP) && ses->pdcp.hfn_ovd) {
 760                 fd->cmd = 0x80000000 |
 761                         *((uint32_t *)((uint8_t *)userdata +
 762                         ses->pdcp.hfn_ovd_offset));
 763                 DPAA_SEC_DP_DEBUG("Per packet HFN: %x, ovd:%u\n",
 764                         *((uint32_t *)((uint8_t *)userdata +
 765                         ses->pdcp.hfn_ovd_offset)),
 766                         ses->pdcp.hfn_ovd);
 767         }
 768
 769         return cf;
 770 }
 771 #endif
 772
 773 static uint32_t
 774 dpaa_sec_raw_enqueue_burst(void *qp_data, uint8_t *drv_ctx,
 775         struct rte_crypto_sym_vec *vec, union rte_crypto_sym_ofs ofs,
 776         void *user_data[], int *status)
 777 {
 778         /* Function to transmit the frames to given device and queuepair */
 779         uint32_t loop;
 780         struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp_data;
 781         uint16_t num_tx = 0;
 782         struct qm_fd fds[DPAA_SEC_BURST], *fd;
 783         uint32_t frames_to_send;
 784         struct dpaa_sec_job *cf;
 785         dpaa_sec_session *ses =
 786                         ((struct dpaa_sec_raw_dp_ctx *)drv_ctx)->session;
 787         uint32_t flags[DPAA_SEC_BURST] = {0};
 788         struct qman_fq *inq[DPAA_SEC_BURST];
 789
 790         if (unlikely(!DPAA_PER_LCORE_PORTAL)) {
 791                 if (rte_dpaa_portal_init((void *)0)) {
 792                         DPAA_SEC_ERR("Failure in affining portal");
 793                         return 0;
 794                 }
 795         }
 796
 797         while (vec->num) {
 798                 frames_to_send = (vec->num > DPAA_SEC_BURST) ?
 799                                 DPAA_SEC_BURST : vec->num;
 800                 for (loop = 0; loop < frames_to_send; loop++) {
 801                         if (unlikely(!ses->qp[rte_lcore_id() % MAX_DPAA_CORES])) {
 802                                 if (dpaa_sec_attach_sess_q(dpaa_qp, ses)) {
 803                                         frames_to_send = loop;
 804                                         goto send_pkts;
 805                                 }
 806                         } else if (unlikely(ses->qp[rte_lcore_id() %
 807                                                 MAX_DPAA_CORES] != dpaa_qp)) {
 808                                 DPAA_SEC_DP_ERR("Old:sess->qp = %p"
 809                                         " New qp = %p\n",
 810                                         ses->qp[rte_lcore_id() %
 811                                         MAX_DPAA_CORES], dpaa_qp);
 812                                 frames_to_send = loop;
 813                                 goto send_pkts;
 814                         }
 815
 816                         /*Clear the unused FD fields before sending*/
 817                         fd = &fds[loop];
 818                         memset(fd, 0, sizeof(struct qm_fd));
 819                         cf = ses->build_raw_dp_fd(drv_ctx,
 820                                                 &vec->src_sgl[loop],
 821                                                 &vec->dest_sgl[loop],
 822                                                 &vec->iv[loop],
 823                                                 &vec->digest[loop],
 824                                                 &vec->auth_iv[loop],
 825                                                 ofs,
 826                                                 user_data[loop],
 827                                                 fd);
 828                         if (!cf) {
 829                                 DPAA_SEC_ERR("error: Improper packet contents"
 830                                         " for crypto operation");
 831                                 goto skip_tx;
 832                         }
 833                         inq[loop] = ses->inq[rte_lcore_id() % MAX_DPAA_CORES];
 834                         qm_fd_addr_set64(fd, rte_dpaa_mem_vtop(cf->sg));
 835                         fd->_format1 = qm_fd_compound;
 836                         fd->length29 = 2 * sizeof(struct qm_sg_entry);
 837
 838                         status[loop] = 1;
 839                 }
 840 send_pkts:
 841                 loop = 0;
 842                 while (loop < frames_to_send) {
 843                         loop += qman_enqueue_multi_fq(&inq[loop], &fds[loop],
 844                                         &flags[loop], frames_to_send - loop);
 845                 }
 846                 vec->num -= frames_to_send;
 847                 num_tx += frames_to_send;
 848         }
 849
 850 skip_tx:
 851         dpaa_qp->tx_pkts += num_tx;
 852         dpaa_qp->tx_errs += vec->num - num_tx;
 853
 854         return num_tx;
 855 }
 856
 857 static int
 858 dpaa_sec_deq_raw(struct dpaa_sec_qp *qp, void **out_user_data,
 859                 uint8_t is_user_data_array,
 860                 rte_cryptodev_raw_post_dequeue_t post_dequeue,
 861                 int nb_ops)
 862 {
 863         struct qman_fq *fq;
 864         unsigned int pkts = 0;
 865         int num_rx_bufs, ret;
 866         struct qm_dqrr_entry *dq;
 867         uint32_t vdqcr_flags = 0;
 868         uint8_t is_success = 0;
 869
 870         fq = &qp->outq;
 871         /*
 872          * Until request for four buffers, we provide exact number of buffers.
 873          * Otherwise we do not set the QM_VDQCR_EXACT flag.
 874          * Not setting QM_VDQCR_EXACT flag can provide two more buffers than
 875          * requested, so we request two less in this case.
 876          */
 877         if (nb_ops < 4) {
 878                 vdqcr_flags = QM_VDQCR_EXACT;
 879                 num_rx_bufs = nb_ops;
 880         } else {
 881                 num_rx_bufs = nb_ops > DPAA_MAX_DEQUEUE_NUM_FRAMES ?
 882                         (DPAA_MAX_DEQUEUE_NUM_FRAMES - 2) : (nb_ops - 2);
 883         }
 884         ret = qman_set_vdq(fq, num_rx_bufs, vdqcr_flags);
 885         if (ret)
 886                 return 0;
 887
 888         do {
 889                 const struct qm_fd *fd;
 890                 struct dpaa_sec_job *job;
 891                 struct dpaa_sec_op_ctx *ctx;
 892
 893                 dq = qman_dequeue(fq);
 894                 if (!dq)
 895                         continue;
 896
 897                 fd = &dq->fd;
 898                 /* sg is embedded in an op ctx,
 899                  * sg[0] is for output
 900                  * sg[1] for input
 901                  */
 902                 job = rte_dpaa_mem_ptov(qm_fd_addr_get64(fd));
 903
 904                 ctx = container_of(job, struct dpaa_sec_op_ctx, job);
 905                 ctx->fd_status = fd->status;
 906                 if (is_user_data_array)
 907                         out_user_data[pkts] = ctx->userdata;
 908                 else
 909                         out_user_data[0] = ctx->userdata;
 910
 911                 if (!ctx->fd_status) {
 912                         is_success = true;
 913                 } else {
 914                         is_success = false;
 915                         DPAA_SEC_DP_WARN("SEC return err:0x%x", ctx->fd_status);
 916                 }
 917                 post_dequeue(ctx->op, pkts, is_success);
 918                 pkts++;
 919
 920                 /* report op status to sym->op and then free the ctx memory */
 921                 rte_mempool_put(ctx->ctx_pool, (void *)ctx);
 922
 923                 qman_dqrr_consume(fq, dq);
 924         } while (fq->flags & QMAN_FQ_STATE_VDQCR);
 925
 926         return pkts;
 927 }
 928
 929
 930 static __rte_always_inline uint32_t
 931 dpaa_sec_raw_dequeue_burst(void *qp_data, uint8_t *drv_ctx,
 932         rte_cryptodev_raw_get_dequeue_count_t get_dequeue_count,
 933         uint32_t max_nb_to_dequeue,
 934         rte_cryptodev_raw_post_dequeue_t post_dequeue,
 935         void **out_user_data, uint8_t is_user_data_array,
 936         uint32_t *n_success, int *dequeue_status)
 937 {
 938         RTE_SET_USED(drv_ctx);
 939         RTE_SET_USED(get_dequeue_count);
 940         uint16_t num_rx;
 941         struct dpaa_sec_qp *dpaa_qp = (struct dpaa_sec_qp *)qp_data;
 942         uint32_t nb_ops = max_nb_to_dequeue;
 943
 944         if (unlikely(!DPAA_PER_LCORE_PORTAL)) {
 945                 if (rte_dpaa_portal_init((void *)0)) {
 946                         DPAA_SEC_ERR("Failure in affining portal");
 947                         return 0;
 948                 }
 949         }
 950
 951         num_rx = dpaa_sec_deq_raw(dpaa_qp, out_user_data,
 952                         is_user_data_array, post_dequeue, nb_ops);
 953
 954         dpaa_qp->rx_pkts += num_rx;
 955         *dequeue_status = 1;
 956         *n_success = num_rx;
 957
 958         DPAA_SEC_DP_DEBUG("SEC Received %d Packets\n", num_rx);
 959
 960         return num_rx;
 961 }
 962
 963 static __rte_always_inline int
 964 dpaa_sec_raw_enqueue(void *qp_data, uint8_t *drv_ctx,
 965         struct rte_crypto_vec *data_vec,
 966         uint16_t n_data_vecs, union rte_crypto_sym_ofs ofs,
 967         struct rte_crypto_va_iova_ptr *iv,
 968         struct rte_crypto_va_iova_ptr *digest,
 969         struct rte_crypto_va_iova_ptr *aad_or_auth_iv,
 970         void *user_data)
 971 {
 972         RTE_SET_USED(qp_data);
 973         RTE_SET_USED(drv_ctx);
 974         RTE_SET_USED(data_vec);
 975         RTE_SET_USED(n_data_vecs);
 976         RTE_SET_USED(ofs);
 977         RTE_SET_USED(iv);
 978         RTE_SET_USED(digest);
 979         RTE_SET_USED(aad_or_auth_iv);
 980         RTE_SET_USED(user_data);
 981
 982         return 0;
 983 }
 984
 985 static __rte_always_inline void *
 986 dpaa_sec_raw_dequeue(void *qp_data, uint8_t *drv_ctx, int *dequeue_status,
 987         enum rte_crypto_op_status *op_status)
 988 {
 989         RTE_SET_USED(qp_data);
 990         RTE_SET_USED(drv_ctx);
 991         RTE_SET_USED(dequeue_status);
 992         RTE_SET_USED(op_status);
 993
 994         return NULL;
 995 }
 996
 997 int
 998 dpaa_sec_configure_raw_dp_ctx(struct rte_cryptodev *dev, uint16_t qp_id,
 999         struct rte_crypto_raw_dp_ctx *raw_dp_ctx,
1000         enum rte_crypto_op_sess_type sess_type,
1001         union rte_cryptodev_session_ctx session_ctx, uint8_t is_update)
1002 {
1003         dpaa_sec_session *sess;
1004         struct dpaa_sec_raw_dp_ctx *dp_ctx;
1005         RTE_SET_USED(qp_id);
1006
1007         if (!is_update) {
1008                 memset(raw_dp_ctx, 0, sizeof(*raw_dp_ctx));
1009                 raw_dp_ctx->qp_data = dev->data->queue_pairs[qp_id];
1010         }
1011
1012         if (sess_type == RTE_CRYPTO_OP_SECURITY_SESSION)
1013                 sess = (dpaa_sec_session *)get_sec_session_private_data(
1014                                 session_ctx.sec_sess);
1015         else if (sess_type == RTE_CRYPTO_OP_WITH_SESSION)
1016                 sess = (dpaa_sec_session *)get_sym_session_private_data(
1017                         session_ctx.crypto_sess, dpaa_cryptodev_driver_id);
1018         else
1019                 return -ENOTSUP;
1020         raw_dp_ctx->dequeue_burst = dpaa_sec_raw_dequeue_burst;
1021         raw_dp_ctx->dequeue = dpaa_sec_raw_dequeue;
1022         raw_dp_ctx->dequeue_done = dpaa_sec_raw_dequeue_done;
1023         raw_dp_ctx->enqueue_burst = dpaa_sec_raw_enqueue_burst;
1024         raw_dp_ctx->enqueue = dpaa_sec_raw_enqueue;
1025         raw_dp_ctx->enqueue_done = dpaa_sec_raw_enqueue_done;
1026
1027         if (sess->ctxt == DPAA_SEC_CIPHER)
1028                 sess->build_raw_dp_fd = build_dpaa_raw_dp_cipher_fd;
1029         else if (sess->ctxt == DPAA_SEC_AUTH)
1030                 sess->build_raw_dp_fd = build_dpaa_raw_dp_auth_fd;
1031         else if (sess->ctxt == DPAA_SEC_CIPHER_HASH)
1032                 sess->build_raw_dp_fd = build_dpaa_raw_dp_chain_fd;
1033         else if (sess->ctxt == DPAA_SEC_AEAD)
1034                 sess->build_raw_dp_fd = build_raw_cipher_auth_gcm_sg;
1035 #ifdef RTE_LIBRTE_SECURITY
1036         else if (sess->ctxt == DPAA_SEC_IPSEC ||
1037                         sess->ctxt == DPAA_SEC_PDCP)
1038                 sess->build_raw_dp_fd = build_dpaa_raw_proto_sg;
1039 #endif
1040         else
1041                 return -ENOTSUP;
1042         dp_ctx = (struct dpaa_sec_raw_dp_ctx *)raw_dp_ctx->drv_ctx_data;
1043         dp_ctx->session = sess;
1044
1045         return 0;
1046 }
1047
1048 int
1049 dpaa_sec_get_dp_ctx_size(__rte_unused struct rte_cryptodev *dev)
1050 {
1051         return sizeof(struct dpaa_sec_raw_dp_ctx);
1052 }