drivers/crypto/ccp/ccp_crypto.c

   1 /*   SPDX-License-Identifier: BSD-3-Clause
   2  *   Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved.
   3  */
   4
   5 #include <dirent.h>
   6 #include <fcntl.h>
   7 #include <stdio.h>
   8 #include <string.h>
   9 #include <sys/mman.h>
  10 #include <sys/queue.h>
  11 #include <sys/types.h>
  12 #include <unistd.h>
  13
  14 #include <rte_hexdump.h>
  15 #include <rte_memzone.h>
  16 #include <rte_malloc.h>
  17 #include <rte_memory.h>
  18 #include <rte_spinlock.h>
  19 #include <rte_string_fns.h>
  20 #include <rte_cryptodev_pmd.h>
  21
  22 #include "ccp_dev.h"
  23 #include "ccp_crypto.h"
  24 #include "ccp_pci.h"
  25 #include "ccp_pmd_private.h"
  26
  27 static enum ccp_cmd_order
  28 ccp_get_cmd_id(const struct rte_crypto_sym_xform *xform)
  29 {
  30         enum ccp_cmd_order res = CCP_CMD_NOT_SUPPORTED;
  31
  32         if (xform == NULL)
  33                 return res;
  34         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
  35                 if (xform->next == NULL)
  36                         return CCP_CMD_AUTH;
  37                 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
  38                         return CCP_CMD_HASH_CIPHER;
  39         }
  40         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
  41                 if (xform->next == NULL)
  42                         return CCP_CMD_CIPHER;
  43                 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
  44                         return CCP_CMD_CIPHER_HASH;
  45         }
  46         if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD)
  47                 return CCP_CMD_COMBINED;
  48         return res;
  49 }
  50
  51 /* configure session */
  52 static int
  53 ccp_configure_session_cipher(struct ccp_session *sess,
  54                              const struct rte_crypto_sym_xform *xform)
  55 {
  56         const struct rte_crypto_cipher_xform *cipher_xform = NULL;
  57         size_t i;
  58
  59         cipher_xform = &xform->cipher;
  60
  61         /* set cipher direction */
  62         if (cipher_xform->op ==  RTE_CRYPTO_CIPHER_OP_ENCRYPT)
  63                 sess->cipher.dir = CCP_CIPHER_DIR_ENCRYPT;
  64         else
  65                 sess->cipher.dir = CCP_CIPHER_DIR_DECRYPT;
  66
  67         /* set cipher key */
  68         sess->cipher.key_length = cipher_xform->key.length;
  69         rte_memcpy(sess->cipher.key, cipher_xform->key.data,
  70                    cipher_xform->key.length);
  71
  72         /* set iv parameters */
  73         sess->iv.offset = cipher_xform->iv.offset;
  74         sess->iv.length = cipher_xform->iv.length;
  75
  76         switch (cipher_xform->algo) {
  77         case RTE_CRYPTO_CIPHER_AES_CTR:
  78                 sess->cipher.algo = CCP_CIPHER_ALGO_AES_CTR;
  79                 sess->cipher.um.aes_mode = CCP_AES_MODE_CTR;
  80                 sess->cipher.engine = CCP_ENGINE_AES;
  81                 break;
  82         case RTE_CRYPTO_CIPHER_AES_ECB:
  83                 sess->cipher.algo = CCP_CIPHER_ALGO_AES_CBC;
  84                 sess->cipher.um.aes_mode = CCP_AES_MODE_ECB;
  85                 sess->cipher.engine = CCP_ENGINE_AES;
  86                 break;
  87         case RTE_CRYPTO_CIPHER_AES_CBC:
  88                 sess->cipher.algo = CCP_CIPHER_ALGO_AES_CBC;
  89                 sess->cipher.um.aes_mode = CCP_AES_MODE_CBC;
  90                 sess->cipher.engine = CCP_ENGINE_AES;
  91                 break;
  92         default:
  93                 CCP_LOG_ERR("Unsupported cipher algo");
  94                 return -1;
  95         }
  96
  97
  98         switch (sess->cipher.engine) {
  99         case CCP_ENGINE_AES:
 100                 if (sess->cipher.key_length == 16)
 101                         sess->cipher.ut.aes_type = CCP_AES_TYPE_128;
 102                 else if (sess->cipher.key_length == 24)
 103                         sess->cipher.ut.aes_type = CCP_AES_TYPE_192;
 104                 else if (sess->cipher.key_length == 32)
 105                         sess->cipher.ut.aes_type = CCP_AES_TYPE_256;
 106                 else {
 107                         CCP_LOG_ERR("Invalid cipher key length");
 108                         return -1;
 109                 }
 110                 for (i = 0; i < sess->cipher.key_length ; i++)
 111                         sess->cipher.key_ccp[sess->cipher.key_length - i - 1] =
 112                                 sess->cipher.key[i];
 113                 break;
 114         default:
 115                 CCP_LOG_ERR("Invalid CCP Engine");
 116                 return -ENOTSUP;
 117         }
 118         sess->cipher.nonce_phys = rte_mem_virt2phy(sess->cipher.nonce);
 119         sess->cipher.key_phys = rte_mem_virt2phy(sess->cipher.key_ccp);
 120         return 0;
 121 }
 122
 123 static int
 124 ccp_configure_session_auth(struct ccp_session *sess,
 125                            const struct rte_crypto_sym_xform *xform)
 126 {
 127         const struct rte_crypto_auth_xform *auth_xform = NULL;
 128
 129         auth_xform = &xform->auth;
 130
 131         sess->auth.digest_length = auth_xform->digest_length;
 132         if (auth_xform->op ==  RTE_CRYPTO_AUTH_OP_GENERATE)
 133                 sess->auth.op = CCP_AUTH_OP_GENERATE;
 134         else
 135                 sess->auth.op = CCP_AUTH_OP_VERIFY;
 136         switch (auth_xform->algo) {
 137         default:
 138                 CCP_LOG_ERR("Unsupported hash algo");
 139                 return -ENOTSUP;
 140         }
 141         return 0;
 142 }
 143
 144 static int
 145 ccp_configure_session_aead(struct ccp_session *sess,
 146                            const struct rte_crypto_sym_xform *xform)
 147 {
 148         const struct rte_crypto_aead_xform *aead_xform = NULL;
 149
 150         aead_xform = &xform->aead;
 151
 152         sess->cipher.key_length = aead_xform->key.length;
 153         rte_memcpy(sess->cipher.key, aead_xform->key.data,
 154                    aead_xform->key.length);
 155
 156         if (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) {
 157                 sess->cipher.dir = CCP_CIPHER_DIR_ENCRYPT;
 158                 sess->auth.op = CCP_AUTH_OP_GENERATE;
 159         } else {
 160                 sess->cipher.dir = CCP_CIPHER_DIR_DECRYPT;
 161                 sess->auth.op = CCP_AUTH_OP_VERIFY;
 162         }
 163         sess->auth.aad_length = aead_xform->aad_length;
 164         sess->auth.digest_length = aead_xform->digest_length;
 165
 166         /* set iv parameters */
 167         sess->iv.offset = aead_xform->iv.offset;
 168         sess->iv.length = aead_xform->iv.length;
 169
 170         switch (aead_xform->algo) {
 171         default:
 172                 CCP_LOG_ERR("Unsupported aead algo");
 173                 return -ENOTSUP;
 174         }
 175         return 0;
 176 }
 177
 178 int
 179 ccp_set_session_parameters(struct ccp_session *sess,
 180                            const struct rte_crypto_sym_xform *xform)
 181 {
 182         const struct rte_crypto_sym_xform *cipher_xform = NULL;
 183         const struct rte_crypto_sym_xform *auth_xform = NULL;
 184         const struct rte_crypto_sym_xform *aead_xform = NULL;
 185         int ret = 0;
 186
 187         sess->cmd_id = ccp_get_cmd_id(xform);
 188
 189         switch (sess->cmd_id) {
 190         case CCP_CMD_CIPHER:
 191                 cipher_xform = xform;
 192                 break;
 193         case CCP_CMD_AUTH:
 194                 auth_xform = xform;
 195                 break;
 196         case CCP_CMD_CIPHER_HASH:
 197                 cipher_xform = xform;
 198                 auth_xform = xform->next;
 199                 break;
 200         case CCP_CMD_HASH_CIPHER:
 201                 auth_xform = xform;
 202                 cipher_xform = xform->next;
 203                 break;
 204         case CCP_CMD_COMBINED:
 205                 aead_xform = xform;
 206                 break;
 207         default:
 208                 CCP_LOG_ERR("Unsupported cmd_id");
 209                 return -1;
 210         }
 211
 212         /* Default IV length = 0 */
 213         sess->iv.length = 0;
 214         if (cipher_xform) {
 215                 ret = ccp_configure_session_cipher(sess, cipher_xform);
 216                 if (ret != 0) {
 217                         CCP_LOG_ERR("Invalid/unsupported cipher parameters");
 218                         return ret;
 219                 }
 220         }
 221         if (auth_xform) {
 222                 ret = ccp_configure_session_auth(sess, auth_xform);
 223                 if (ret != 0) {
 224                         CCP_LOG_ERR("Invalid/unsupported auth parameters");
 225                         return ret;
 226                 }
 227         }
 228         if (aead_xform) {
 229                 ret = ccp_configure_session_aead(sess, aead_xform);
 230                 if (ret != 0) {
 231                         CCP_LOG_ERR("Invalid/unsupported aead parameters");
 232                         return ret;
 233                 }
 234         }
 235         return ret;
 236 }
 237
 238 /* calculate CCP descriptors requirement */
 239 static inline int
 240 ccp_cipher_slot(struct ccp_session *session)
 241 {
 242         int count = 0;
 243
 244         switch (session->cipher.algo) {
 245         case CCP_CIPHER_ALGO_AES_CBC:
 246                 count = 2;
 247                 /**< op + passthrough for iv */
 248                 break;
 249         case CCP_CIPHER_ALGO_AES_ECB:
 250                 count = 1;
 251                 /**<only op*/
 252                 break;
 253         case CCP_CIPHER_ALGO_AES_CTR:
 254                 count = 2;
 255                 /**< op + passthrough for iv */
 256                 break;
 257         default:
 258                 CCP_LOG_ERR("Unsupported cipher algo %d",
 259                             session->cipher.algo);
 260         }
 261         return count;
 262 }
 263
 264 static inline int
 265 ccp_auth_slot(struct ccp_session *session)
 266 {
 267         int count = 0;
 268
 269         switch (session->auth.algo) {
 270         default:
 271                 CCP_LOG_ERR("Unsupported auth algo %d",
 272                             session->auth.algo);
 273         }
 274
 275         return count;
 276 }
 277
 278 static int
 279 ccp_aead_slot(struct ccp_session *session)
 280 {
 281         int count = 0;
 282
 283         switch (session->aead_algo) {
 284         default:
 285                 CCP_LOG_ERR("Unsupported aead algo %d",
 286                             session->aead_algo);
 287         }
 288         return count;
 289 }
 290
 291 int
 292 ccp_compute_slot_count(struct ccp_session *session)
 293 {
 294         int count = 0;
 295
 296         switch (session->cmd_id) {
 297         case CCP_CMD_CIPHER:
 298                 count = ccp_cipher_slot(session);
 299                 break;
 300         case CCP_CMD_AUTH:
 301                 count = ccp_auth_slot(session);
 302                 break;
 303         case CCP_CMD_CIPHER_HASH:
 304         case CCP_CMD_HASH_CIPHER:
 305                 count = ccp_cipher_slot(session);
 306                 count += ccp_auth_slot(session);
 307                 break;
 308         case CCP_CMD_COMBINED:
 309                 count = ccp_aead_slot(session);
 310                 break;
 311         default:
 312                 CCP_LOG_ERR("Unsupported cmd_id");
 313
 314         }
 315
 316         return count;
 317 }
 318
 319 static void
 320 ccp_perform_passthru(struct ccp_passthru *pst,
 321                      struct ccp_queue *cmd_q)
 322 {
 323         struct ccp_desc *desc;
 324         union ccp_function function;
 325
 326         desc = &cmd_q->qbase_desc[cmd_q->qidx];
 327
 328         CCP_CMD_ENGINE(desc) = CCP_ENGINE_PASSTHRU;
 329
 330         CCP_CMD_SOC(desc) = 0;
 331         CCP_CMD_IOC(desc) = 0;
 332         CCP_CMD_INIT(desc) = 0;
 333         CCP_CMD_EOM(desc) = 0;
 334         CCP_CMD_PROT(desc) = 0;
 335
 336         function.raw = 0;
 337         CCP_PT_BYTESWAP(&function) = pst->byte_swap;
 338         CCP_PT_BITWISE(&function) = pst->bit_mod;
 339         CCP_CMD_FUNCTION(desc) = function.raw;
 340
 341         CCP_CMD_LEN(desc) = pst->len;
 342
 343         if (pst->dir) {
 344                 CCP_CMD_SRC_LO(desc) = (uint32_t)(pst->src_addr);
 345                 CCP_CMD_SRC_HI(desc) = high32_value(pst->src_addr);
 346                 CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
 347
 348                 CCP_CMD_DST_LO(desc) = (uint32_t)(pst->dest_addr);
 349                 CCP_CMD_DST_HI(desc) = 0;
 350                 CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SB;
 351
 352                 if (pst->bit_mod != CCP_PASSTHRU_BITWISE_NOOP)
 353                         CCP_CMD_LSB_ID(desc) = cmd_q->sb_key;
 354         } else {
 355
 356                 CCP_CMD_SRC_LO(desc) = (uint32_t)(pst->src_addr);
 357                 CCP_CMD_SRC_HI(desc) = 0;
 358                 CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SB;
 359
 360                 CCP_CMD_DST_LO(desc) = (uint32_t)(pst->dest_addr);
 361                 CCP_CMD_DST_HI(desc) = high32_value(pst->dest_addr);
 362                 CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SYSTEM;
 363         }
 364
 365         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
 366 }
 367
 368 static int
 369 ccp_perform_aes(struct rte_crypto_op *op,
 370                 struct ccp_queue *cmd_q,
 371                 struct ccp_batch_info *b_info)
 372 {
 373         struct ccp_session *session;
 374         union ccp_function function;
 375         uint8_t *lsb_buf;
 376         struct ccp_passthru pst = {0};
 377         struct ccp_desc *desc;
 378         phys_addr_t src_addr, dest_addr, key_addr;
 379         uint8_t *iv;
 380
 381         session = (struct ccp_session *)get_session_private_data(
 382                                          op->sym->session,
 383                                         ccp_cryptodev_driver_id);
 384         function.raw = 0;
 385
 386         iv = rte_crypto_op_ctod_offset(op, uint8_t *, session->iv.offset);
 387         if (session->cipher.um.aes_mode != CCP_AES_MODE_ECB) {
 388                 if (session->cipher.um.aes_mode == CCP_AES_MODE_CTR) {
 389                         rte_memcpy(session->cipher.nonce + AES_BLOCK_SIZE,
 390                                    iv, session->iv.length);
 391                         pst.src_addr = (phys_addr_t)session->cipher.nonce_phys;
 392                         CCP_AES_SIZE(&function) = 0x1F;
 393                 } else {
 394                         lsb_buf =
 395                         &(b_info->lsb_buf[b_info->lsb_buf_idx*CCP_SB_BYTES]);
 396                         rte_memcpy(lsb_buf +
 397                                    (CCP_SB_BYTES - session->iv.length),
 398                                    iv, session->iv.length);
 399                         pst.src_addr = b_info->lsb_buf_phys +
 400                                 (b_info->lsb_buf_idx * CCP_SB_BYTES);
 401                         b_info->lsb_buf_idx++;
 402                 }
 403
 404                 pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES);
 405                 pst.len = CCP_SB_BYTES;
 406                 pst.dir = 1;
 407                 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP;
 408                 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT;
 409                 ccp_perform_passthru(&pst, cmd_q);
 410         }
 411
 412         desc = &cmd_q->qbase_desc[cmd_q->qidx];
 413
 414         src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src,
 415                                               op->sym->cipher.data.offset);
 416         if (likely(op->sym->m_dst != NULL))
 417                 dest_addr = rte_pktmbuf_mtophys_offset(op->sym->m_dst,
 418                                                 op->sym->cipher.data.offset);
 419         else
 420                 dest_addr = src_addr;
 421         key_addr = session->cipher.key_phys;
 422
 423         /* prepare desc for aes command */
 424         CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES;
 425         CCP_CMD_INIT(desc) = 1;
 426         CCP_CMD_EOM(desc) = 1;
 427
 428         CCP_AES_ENCRYPT(&function) = session->cipher.dir;
 429         CCP_AES_MODE(&function) = session->cipher.um.aes_mode;
 430         CCP_AES_TYPE(&function) = session->cipher.ut.aes_type;
 431         CCP_CMD_FUNCTION(desc) = function.raw;
 432
 433         CCP_CMD_LEN(desc) = op->sym->cipher.data.length;
 434
 435         CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr);
 436         CCP_CMD_SRC_HI(desc) = high32_value(src_addr);
 437         CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM;
 438
 439         CCP_CMD_DST_LO(desc) = ((uint32_t)dest_addr);
 440         CCP_CMD_DST_HI(desc) = high32_value(dest_addr);
 441         CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SYSTEM;
 442
 443         CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr);
 444         CCP_CMD_KEY_HI(desc) = high32_value(key_addr);
 445         CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM;
 446
 447         if (session->cipher.um.aes_mode != CCP_AES_MODE_ECB)
 448                 CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv;
 449
 450         cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE;
 451         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 452         return 0;
 453 }
 454
 455 static inline int
 456 ccp_crypto_cipher(struct rte_crypto_op *op,
 457                   struct ccp_queue *cmd_q,
 458                   struct ccp_batch_info *b_info)
 459 {
 460         int result = 0;
 461         struct ccp_session *session;
 462
 463         session = (struct ccp_session *)get_session_private_data(
 464                                          op->sym->session,
 465                                          ccp_cryptodev_driver_id);
 466
 467         switch (session->cipher.algo) {
 468         case CCP_CIPHER_ALGO_AES_CBC:
 469                 result = ccp_perform_aes(op, cmd_q, b_info);
 470                 b_info->desccnt += 2;
 471                 break;
 472         case CCP_CIPHER_ALGO_AES_CTR:
 473                 result = ccp_perform_aes(op, cmd_q, b_info);
 474                 b_info->desccnt += 2;
 475                 break;
 476         case CCP_CIPHER_ALGO_AES_ECB:
 477                 result = ccp_perform_aes(op, cmd_q, b_info);
 478                 b_info->desccnt += 1;
 479                 break;
 480         default:
 481                 CCP_LOG_ERR("Unsupported cipher algo %d",
 482                             session->cipher.algo);
 483                 return -ENOTSUP;
 484         }
 485         return result;
 486 }
 487
 488 static inline int
 489 ccp_crypto_auth(struct rte_crypto_op *op,
 490                 struct ccp_queue *cmd_q __rte_unused,
 491                 struct ccp_batch_info *b_info __rte_unused)
 492 {
 493
 494         int result = 0;
 495         struct ccp_session *session;
 496
 497         session = (struct ccp_session *)get_session_private_data(
 498                                          op->sym->session,
 499                                         ccp_cryptodev_driver_id);
 500
 501         switch (session->auth.algo) {
 502         default:
 503                 CCP_LOG_ERR("Unsupported auth algo %d",
 504                             session->auth.algo);
 505                 return -ENOTSUP;
 506         }
 507
 508         return result;
 509 }
 510
 511 static inline int
 512 ccp_crypto_aead(struct rte_crypto_op *op,
 513                 struct ccp_queue *cmd_q __rte_unused,
 514                 struct ccp_batch_info *b_info __rte_unused)
 515 {
 516         int result = 0;
 517         struct ccp_session *session;
 518
 519         session = (struct ccp_session *)get_session_private_data(
 520                                          op->sym->session,
 521                                         ccp_cryptodev_driver_id);
 522
 523         switch (session->aead_algo) {
 524         default:
 525                 CCP_LOG_ERR("Unsupported aead algo %d",
 526                             session->aead_algo);
 527                 return -ENOTSUP;
 528         }
 529         return result;
 530 }
 531
 532 int
 533 process_ops_to_enqueue(const struct ccp_qp *qp,
 534                        struct rte_crypto_op **op,
 535                        struct ccp_queue *cmd_q,
 536                        uint16_t nb_ops,
 537                        int slots_req)
 538 {
 539         int i, result = 0;
 540         struct ccp_batch_info *b_info;
 541         struct ccp_session *session;
 542
 543         if (rte_mempool_get(qp->batch_mp, (void **)&b_info)) {
 544                 CCP_LOG_ERR("batch info allocation failed");
 545                 return 0;
 546         }
 547         /* populate batch info necessary for dequeue */
 548         b_info->op_idx = 0;
 549         b_info->lsb_buf_idx = 0;
 550         b_info->desccnt = 0;
 551         b_info->cmd_q = cmd_q;
 552         b_info->lsb_buf_phys =
 553                 (phys_addr_t)rte_mem_virt2phy((void *)b_info->lsb_buf);
 554         rte_atomic64_sub(&b_info->cmd_q->free_slots, slots_req);
 555
 556         b_info->head_offset = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx *
 557                                          Q_DESC_SIZE);
 558         for (i = 0; i < nb_ops; i++) {
 559                 session = (struct ccp_session *)get_session_private_data(
 560                                                  op[i]->sym->session,
 561                                                  ccp_cryptodev_driver_id);
 562                 switch (session->cmd_id) {
 563                 case CCP_CMD_CIPHER:
 564                         result = ccp_crypto_cipher(op[i], cmd_q, b_info);
 565                         break;
 566                 case CCP_CMD_AUTH:
 567                         result = ccp_crypto_auth(op[i], cmd_q, b_info);
 568                         break;
 569                 case CCP_CMD_CIPHER_HASH:
 570                         result = ccp_crypto_cipher(op[i], cmd_q, b_info);
 571                         if (result)
 572                                 break;
 573                         result = ccp_crypto_auth(op[i], cmd_q, b_info);
 574                         break;
 575                 case CCP_CMD_HASH_CIPHER:
 576                         result = ccp_crypto_auth(op[i], cmd_q, b_info);
 577                         if (result)
 578                                 break;
 579                         result = ccp_crypto_cipher(op[i], cmd_q, b_info);
 580                         break;
 581                 case CCP_CMD_COMBINED:
 582                         result = ccp_crypto_aead(op[i], cmd_q, b_info);
 583                         break;
 584                 default:
 585                         CCP_LOG_ERR("Unsupported cmd_id");
 586                         result = -1;
 587                 }
 588                 if (unlikely(result < 0)) {
 589                         rte_atomic64_add(&b_info->cmd_q->free_slots,
 590                                          (slots_req - b_info->desccnt));
 591                         break;
 592                 }
 593                 b_info->op[i] = op[i];
 594         }
 595
 596         b_info->opcnt = i;
 597         b_info->tail_offset = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx *
 598                                          Q_DESC_SIZE);
 599
 600         rte_wmb();
 601         /* Write the new tail address back to the queue register */
 602         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE,
 603                               b_info->tail_offset);
 604         /* Turn the queue back on using our cached control register */
 605         CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE,
 606                               cmd_q->qcontrol | CMD_Q_RUN);
 607
 608         rte_ring_enqueue(qp->processed_pkts, (void *)b_info);
 609
 610         return i;
 611 }
 612
 613 static inline void ccp_auth_dq_prepare(struct rte_crypto_op *op)
 614 {
 615         struct ccp_session *session;
 616         uint8_t *digest_data, *addr;
 617         struct rte_mbuf *m_last;
 618         int offset, digest_offset;
 619         uint8_t digest_le[64];
 620
 621         session = (struct ccp_session *)get_session_private_data(
 622                                          op->sym->session,
 623                                         ccp_cryptodev_driver_id);
 624
 625         if (session->cmd_id == CCP_CMD_COMBINED) {
 626                 digest_data = op->sym->aead.digest.data;
 627                 digest_offset = op->sym->aead.data.offset +
 628                                         op->sym->aead.data.length;
 629         } else {
 630                 digest_data = op->sym->auth.digest.data;
 631                 digest_offset = op->sym->auth.data.offset +
 632                                         op->sym->auth.data.length;
 633         }
 634         m_last = rte_pktmbuf_lastseg(op->sym->m_src);
 635         addr = (uint8_t *)((char *)m_last->buf_addr + m_last->data_off +
 636                            m_last->data_len - session->auth.ctx_len);
 637
 638         rte_mb();
 639         offset = session->auth.offset;
 640
 641         if (session->auth.engine == CCP_ENGINE_SHA)
 642                 if ((session->auth.ut.sha_type != CCP_SHA_TYPE_1) &&
 643                     (session->auth.ut.sha_type != CCP_SHA_TYPE_224) &&
 644                     (session->auth.ut.sha_type != CCP_SHA_TYPE_256)) {
 645                         /* All other algorithms require byte
 646                          * swap done by host
 647                          */
 648                         unsigned int i;
 649
 650                         offset = session->auth.ctx_len -
 651                                 session->auth.offset - 1;
 652                         for (i = 0; i < session->auth.digest_length; i++)
 653                                 digest_le[i] = addr[offset - i];
 654                         offset = 0;
 655                         addr = digest_le;
 656                 }
 657
 658         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 659         if (session->auth.op == CCP_AUTH_OP_VERIFY) {
 660                 if (memcmp(addr + offset, digest_data,
 661                            session->auth.digest_length) != 0)
 662                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 663
 664         } else {
 665                 if (unlikely(digest_data == 0))
 666                         digest_data = rte_pktmbuf_mtod_offset(
 667                                         op->sym->m_dst, uint8_t *,
 668                                         digest_offset);
 669                 rte_memcpy(digest_data, addr + offset,
 670                            session->auth.digest_length);
 671         }
 672         /* Trim area used for digest from mbuf. */
 673         rte_pktmbuf_trim(op->sym->m_src,
 674                          session->auth.ctx_len);
 675 }
 676
 677 static int
 678 ccp_prepare_ops(struct rte_crypto_op **op_d,
 679                 struct ccp_batch_info *b_info,
 680                 uint16_t nb_ops)
 681 {
 682         int i, min_ops;
 683         struct ccp_session *session;
 684
 685         min_ops = RTE_MIN(nb_ops, b_info->opcnt);
 686
 687         for (i = 0; i < min_ops; i++) {
 688                 op_d[i] = b_info->op[b_info->op_idx++];
 689                 session = (struct ccp_session *)get_session_private_data(
 690                                                  op_d[i]->sym->session,
 691                                                 ccp_cryptodev_driver_id);
 692                 switch (session->cmd_id) {
 693                 case CCP_CMD_CIPHER:
 694                         op_d[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 695                         break;
 696                 case CCP_CMD_AUTH:
 697                 case CCP_CMD_CIPHER_HASH:
 698                 case CCP_CMD_HASH_CIPHER:
 699                 case CCP_CMD_COMBINED:
 700                         ccp_auth_dq_prepare(op_d[i]);
 701                         break;
 702                 default:
 703                         CCP_LOG_ERR("Unsupported cmd_id");
 704                 }
 705         }
 706
 707         b_info->opcnt -= min_ops;
 708         return min_ops;
 709 }
 710
 711 int
 712 process_ops_to_dequeue(struct ccp_qp *qp,
 713                        struct rte_crypto_op **op,
 714                        uint16_t nb_ops)
 715 {
 716         struct ccp_batch_info *b_info;
 717         uint32_t cur_head_offset;
 718
 719         if (qp->b_info != NULL) {
 720                 b_info = qp->b_info;
 721                 if (unlikely(b_info->op_idx > 0))
 722                         goto success;
 723         } else if (rte_ring_dequeue(qp->processed_pkts,
 724                                     (void **)&b_info))
 725                 return 0;
 726         cur_head_offset = CCP_READ_REG(b_info->cmd_q->reg_base,
 727                                        CMD_Q_HEAD_LO_BASE);
 728
 729         if (b_info->head_offset < b_info->tail_offset) {
 730                 if ((cur_head_offset >= b_info->head_offset) &&
 731                     (cur_head_offset < b_info->tail_offset)) {
 732                         qp->b_info = b_info;
 733                         return 0;
 734                 }
 735         } else {
 736                 if ((cur_head_offset >= b_info->head_offset) ||
 737                     (cur_head_offset < b_info->tail_offset)) {
 738                         qp->b_info = b_info;
 739                         return 0;
 740                 }
 741         }
 742
 743
 744 success:
 745         nb_ops = ccp_prepare_ops(op, b_info, nb_ops);
 746         rte_atomic64_add(&b_info->cmd_q->free_slots, b_info->desccnt);
 747         b_info->desccnt = 0;
 748         if (b_info->opcnt > 0) {
 749                 qp->b_info = b_info;
 750         } else {
 751                 rte_mempool_put(qp->batch_mp, (void *)b_info);
 752                 qp->b_info = NULL;
 753         }
 754
 755         return nb_ops;
 756 }