drivers/crypto/ipsec_mb/pmd_zuc.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016-2021 Intel Corporation
   3  */
   4
   5 #include "pmd_zuc_priv.h"
   6
   7 /** Parse crypto xform chain and set private session parameters. */
   8 static int
   9 zuc_session_configure(__rte_unused IMB_MGR * mgr, void *zuc_sess,
  10                 const struct rte_crypto_sym_xform *xform)
  11 {
  12         struct zuc_session *sess = (struct zuc_session *) zuc_sess;
  13         const struct rte_crypto_sym_xform *auth_xform = NULL;
  14         const struct rte_crypto_sym_xform *cipher_xform = NULL;
  15         enum ipsec_mb_operation mode;
  16         /* Select Crypto operation - hash then cipher / cipher then hash */
  17         int ret = ipsec_mb_parse_xform(xform, &mode, &auth_xform,
  18                                 &cipher_xform, NULL);
  19
  20         if (ret)
  21                 return ret;
  22
  23         if (cipher_xform) {
  24                 /* Only ZUC EEA3 supported */
  25                 if (cipher_xform->cipher.algo != RTE_CRYPTO_CIPHER_ZUC_EEA3)
  26                         return -ENOTSUP;
  27
  28                 if (cipher_xform->cipher.iv.length != ZUC_IV_KEY_LENGTH) {
  29                         IPSEC_MB_LOG(ERR, "Wrong IV length");
  30                         return -EINVAL;
  31                 }
  32                 sess->cipher_iv_offset = cipher_xform->cipher.iv.offset;
  33
  34                 /* Copy the key */
  35                 memcpy(sess->pKey_cipher, cipher_xform->cipher.key.data,
  36                                 ZUC_IV_KEY_LENGTH);
  37         }
  38
  39         if (auth_xform) {
  40                 /* Only ZUC EIA3 supported */
  41                 if (auth_xform->auth.algo != RTE_CRYPTO_AUTH_ZUC_EIA3)
  42                         return -ENOTSUP;
  43
  44                 if (auth_xform->auth.digest_length != ZUC_DIGEST_LENGTH) {
  45                         IPSEC_MB_LOG(ERR, "Wrong digest length");
  46                         return -EINVAL;
  47                 }
  48
  49                 sess->auth_op = auth_xform->auth.op;
  50
  51                 if (auth_xform->auth.iv.length != ZUC_IV_KEY_LENGTH) {
  52                         IPSEC_MB_LOG(ERR, "Wrong IV length");
  53                         return -EINVAL;
  54                 }
  55                 sess->auth_iv_offset = auth_xform->auth.iv.offset;
  56
  57                 /* Copy the key */
  58                 memcpy(sess->pKey_hash, auth_xform->auth.key.data,
  59                                 ZUC_IV_KEY_LENGTH);
  60         }
  61
  62         sess->op = mode;
  63         return 0;
  64 }
  65
  66 /** Encrypt/decrypt mbufs. */
  67 static uint8_t
  68 process_zuc_cipher_op(struct ipsec_mb_qp *qp, struct rte_crypto_op **ops,
  69                 struct zuc_session **sessions,
  70                 uint8_t num_ops)
  71 {
  72         unsigned int i;
  73         uint8_t processed_ops = 0;
  74         const void *src[ZUC_MAX_BURST];
  75         void *dst[ZUC_MAX_BURST];
  76         const void *iv[ZUC_MAX_BURST];
  77         uint32_t num_bytes[ZUC_MAX_BURST];
  78         const void *cipher_keys[ZUC_MAX_BURST];
  79         struct zuc_session *sess;
  80
  81         for (i = 0; i < num_ops; i++) {
  82                 if (((ops[i]->sym->cipher.data.length % BYTE_LEN) != 0)
  83                                 || ((ops[i]->sym->cipher.data.offset
  84                                         % BYTE_LEN) != 0)) {
  85                         ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
  86                         IPSEC_MB_LOG(ERR, "Data Length or offset");
  87                         break;
  88                 }
  89
  90                 sess = sessions[i];
  91
  92 #ifdef RTE_LIBRTE_PMD_ZUC_DEBUG
  93                 if (!rte_pktmbuf_is_contiguous(ops[i]->sym->m_src) ||
  94                                 (ops[i]->sym->m_dst != NULL &&
  95                                 !rte_pktmbuf_is_contiguous(
  96                                                 ops[i]->sym->m_dst))) {
  97                         IPSEC_MB_LOG(ERR, "PMD supports only "
  98                                 " contiguous mbufs, op (%p) "
  99                                 "provides noncontiguous mbuf "
 100                                 "as source/destination buffer.\n",
 101                                 "PMD supports only contiguous mbufs, "
 102                                 "op (%p) provides noncontiguous mbuf "
 103                                 "as source/destination buffer.\n",
 104                                 ops[i]);
 105                         ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 106                         break;
 107                 }
 108 #endif
 109
 110                 src[i] = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) +
 111                                 (ops[i]->sym->cipher.data.offset >> 3);
 112                 dst[i] = ops[i]->sym->m_dst ?
 113                         rte_pktmbuf_mtod(ops[i]->sym->m_dst, uint8_t *) +
 114                                 (ops[i]->sym->cipher.data.offset >> 3) :
 115                         rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) +
 116                                 (ops[i]->sym->cipher.data.offset >> 3);
 117                 iv[i] = rte_crypto_op_ctod_offset(ops[i], uint8_t *,
 118                                 sess->cipher_iv_offset);
 119                 num_bytes[i] = ops[i]->sym->cipher.data.length >> 3;
 120
 121                 cipher_keys[i] = sess->pKey_cipher;
 122
 123                 processed_ops++;
 124         }
 125
 126         IMB_ZUC_EEA3_N_BUFFER(qp->mb_mgr, (const void **)cipher_keys,
 127                         (const void **)iv, (const void **)src, (void **)dst,
 128                         num_bytes, processed_ops);
 129
 130         return processed_ops;
 131 }
 132
 133 /** Generate/verify hash from mbufs. */
 134 static int
 135 process_zuc_hash_op(struct ipsec_mb_qp *qp, struct rte_crypto_op **ops,
 136                 struct zuc_session **sessions,
 137                 uint8_t num_ops)
 138 {
 139         unsigned int i;
 140         uint8_t processed_ops = 0;
 141         uint8_t *src[ZUC_MAX_BURST] = { 0 };
 142         uint32_t *dst[ZUC_MAX_BURST];
 143         uint32_t length_in_bits[ZUC_MAX_BURST] = { 0 };
 144         uint8_t *iv[ZUC_MAX_BURST] = { 0 };
 145         const void *hash_keys[ZUC_MAX_BURST] = { 0 };
 146         struct zuc_session *sess;
 147         struct zuc_qp_data *qp_data = ipsec_mb_get_qp_private_data(qp);
 148
 149
 150         for (i = 0; i < num_ops; i++) {
 151                 /* Data must be byte aligned */
 152                 if ((ops[i]->sym->auth.data.offset % BYTE_LEN) != 0) {
 153                         ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 154                         IPSEC_MB_LOG(ERR, "Offset");
 155                         break;
 156                 }
 157
 158                 sess = sessions[i];
 159
 160                 length_in_bits[i] = ops[i]->sym->auth.data.length;
 161
 162                 src[i] = rte_pktmbuf_mtod(ops[i]->sym->m_src, uint8_t *) +
 163                                 (ops[i]->sym->auth.data.offset >> 3);
 164                 iv[i] = rte_crypto_op_ctod_offset(ops[i], uint8_t *,
 165                                 sess->auth_iv_offset);
 166
 167                 hash_keys[i] = sess->pKey_hash;
 168                 if (sess->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY)
 169                         dst[i] = (uint32_t *)qp_data->temp_digest[i];
 170                 else
 171                         dst[i] = (uint32_t *)ops[i]->sym->auth.digest.data;
 172
 173                 processed_ops++;
 174         }
 175
 176         IMB_ZUC_EIA3_N_BUFFER(qp->mb_mgr, (const void **)hash_keys,
 177                         (const void * const *)iv, (const void * const *)src,
 178                         length_in_bits, dst, processed_ops);
 179
 180         /*
 181          * If tag needs to be verified, compare generated tag
 182          * with attached tag
 183          */
 184         for (i = 0; i < processed_ops; i++)
 185                 if (sessions[i]->auth_op == RTE_CRYPTO_AUTH_OP_VERIFY)
 186                         if (memcmp(dst[i], ops[i]->sym->auth.digest.data,
 187                                         ZUC_DIGEST_LENGTH) != 0)
 188                                 ops[i]->status =
 189                                         RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 190
 191         return processed_ops;
 192 }
 193
 194 /** Process a batch of crypto ops which shares the same operation type. */
 195 static int
 196 process_ops(struct rte_crypto_op **ops, enum ipsec_mb_operation op_type,
 197                 struct zuc_session **sessions,
 198                 struct ipsec_mb_qp *qp, uint8_t num_ops)
 199 {
 200         unsigned int i;
 201         unsigned int processed_ops = 0;
 202
 203         switch (op_type) {
 204         case IPSEC_MB_OP_ENCRYPT_ONLY:
 205         case IPSEC_MB_OP_DECRYPT_ONLY:
 206                 processed_ops = process_zuc_cipher_op(qp, ops,
 207                                 sessions, num_ops);
 208                 break;
 209         case IPSEC_MB_OP_HASH_GEN_ONLY:
 210         case IPSEC_MB_OP_HASH_VERIFY_ONLY:
 211                 processed_ops = process_zuc_hash_op(qp, ops, sessions,
 212                                 num_ops);
 213                 break;
 214         case IPSEC_MB_OP_ENCRYPT_THEN_HASH_GEN:
 215         case IPSEC_MB_OP_DECRYPT_THEN_HASH_VERIFY:
 216                 processed_ops = process_zuc_cipher_op(qp, ops, sessions,
 217                                 num_ops);
 218                 process_zuc_hash_op(qp, ops, sessions, processed_ops);
 219                 break;
 220         case IPSEC_MB_OP_HASH_VERIFY_THEN_DECRYPT:
 221         case IPSEC_MB_OP_HASH_GEN_THEN_ENCRYPT:
 222                 processed_ops = process_zuc_hash_op(qp, ops, sessions,
 223                                 num_ops);
 224                 process_zuc_cipher_op(qp, ops, sessions, processed_ops);
 225                 break;
 226         default:
 227                 /* Operation not supported. */
 228                 for (i = 0; i < num_ops; i++)
 229                         ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 230         }
 231
 232         for (i = 0; i < num_ops; i++) {
 233                 /*
 234                  * If there was no error/authentication failure,
 235                  * change status to successful.
 236                  */
 237                 if (ops[i]->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 238                         ops[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 239                 /* Free session if a session-less crypto op. */
 240                 if (ops[i]->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
 241                         memset(sessions[i], 0, sizeof(struct zuc_session));
 242                         memset(ops[i]->sym->session, 0,
 243                         rte_cryptodev_sym_get_existing_header_session_size(
 244                                         ops[i]->sym->session));
 245                         rte_mempool_put(qp->sess_mp_priv, sessions[i]);
 246                         rte_mempool_put(qp->sess_mp, ops[i]->sym->session);
 247                         ops[i]->sym->session = NULL;
 248                 }
 249         }
 250         return processed_ops;
 251 }
 252
 253 static uint16_t
 254 zuc_pmd_dequeue_burst(void *queue_pair,
 255                 struct rte_crypto_op **c_ops, uint16_t nb_ops)
 256 {
 257
 258         struct rte_crypto_op *curr_c_op;
 259
 260         struct zuc_session *curr_sess;
 261         struct zuc_session *sessions[ZUC_MAX_BURST];
 262         struct rte_crypto_op *int_c_ops[ZUC_MAX_BURST];
 263         enum ipsec_mb_operation prev_zuc_op = IPSEC_MB_OP_NOT_SUPPORTED;
 264         enum ipsec_mb_operation curr_zuc_op;
 265         struct ipsec_mb_qp *qp = queue_pair;
 266         unsigned int nb_dequeued;
 267         unsigned int i;
 268         uint8_t burst_size = 0;
 269         uint8_t processed_ops;
 270
 271         nb_dequeued = rte_ring_dequeue_burst(qp->ingress_queue,
 272                         (void **)c_ops, nb_ops, NULL);
 273
 274
 275         for (i = 0; i < nb_dequeued; i++) {
 276                 curr_c_op = c_ops[i];
 277
 278                 curr_sess = (struct zuc_session *)
 279                         ipsec_mb_get_session_private(qp, curr_c_op);
 280                 if (unlikely(curr_sess == NULL)) {
 281                         curr_c_op->status =
 282                                         RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 283                         break;
 284                 }
 285
 286                 curr_zuc_op = curr_sess->op;
 287
 288                 /*
 289                  * Batch ops that share the same operation type
 290                  * (cipher only, auth only...).
 291                  */
 292                 if (burst_size == 0) {
 293                         prev_zuc_op = curr_zuc_op;
 294                         int_c_ops[0] = curr_c_op;
 295                         sessions[0] = curr_sess;
 296                         burst_size++;
 297                 } else if (curr_zuc_op == prev_zuc_op) {
 298                         int_c_ops[burst_size] = curr_c_op;
 299                         sessions[burst_size] = curr_sess;
 300                         burst_size++;
 301                         /*
 302                          * When there are enough ops to process in a batch,
 303                          * process them, and start a new batch.
 304                          */
 305                         if (burst_size == ZUC_MAX_BURST) {
 306                                 processed_ops = process_ops(int_c_ops, curr_zuc_op,
 307                                                 sessions, qp, burst_size);
 308                                 if (processed_ops < burst_size) {
 309                                         burst_size = 0;
 310                                         break;
 311                                 }
 312
 313                                 burst_size = 0;
 314                         }
 315                 } else {
 316                         /*
 317                          * Different operation type, process the ops
 318                          * of the previous type.
 319                          */
 320                         processed_ops = process_ops(int_c_ops, prev_zuc_op,
 321                                         sessions, qp, burst_size);
 322                         if (processed_ops < burst_size) {
 323                                 burst_size = 0;
 324                                 break;
 325                         }
 326
 327                         burst_size = 0;
 328                         prev_zuc_op = curr_zuc_op;
 329
 330                         int_c_ops[0] = curr_c_op;
 331                         sessions[0] = curr_sess;
 332                         burst_size++;
 333                 }
 334         }
 335
 336         if (burst_size != 0) {
 337                 /* Process the crypto ops of the last operation type. */
 338                 processed_ops = process_ops(int_c_ops, prev_zuc_op,
 339                                 sessions, qp, burst_size);
 340         }
 341
 342         qp->stats.dequeued_count += i;
 343         return i;
 344 }
 345
 346 struct rte_cryptodev_ops zuc_pmd_ops = {
 347         .dev_configure = ipsec_mb_config,
 348         .dev_start = ipsec_mb_start,
 349         .dev_stop = ipsec_mb_stop,
 350         .dev_close = ipsec_mb_close,
 351
 352         .stats_get = ipsec_mb_stats_get,
 353         .stats_reset = ipsec_mb_stats_reset,
 354
 355         .dev_infos_get = ipsec_mb_info_get,
 356
 357         .queue_pair_setup = ipsec_mb_qp_setup,
 358         .queue_pair_release = ipsec_mb_qp_release,
 359
 360         .sym_session_get_size = ipsec_mb_sym_session_get_size,
 361         .sym_session_configure = ipsec_mb_sym_session_configure,
 362         .sym_session_clear = ipsec_mb_sym_session_clear
 363 };
 364
 365 struct rte_cryptodev_ops *rte_zuc_pmd_ops = &zuc_pmd_ops;
 366
 367 static int
 368 zuc_probe(struct rte_vdev_device *vdev)
 369 {
 370         return ipsec_mb_create(vdev, IPSEC_MB_PMD_TYPE_ZUC);
 371 }
 372
 373 static struct rte_vdev_driver cryptodev_zuc_pmd_drv = {
 374         .probe = zuc_probe,
 375         .remove = ipsec_mb_remove
 376
 377 };
 378
 379 static struct cryptodev_driver zuc_crypto_drv;
 380
 381 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_ZUC_PMD, cryptodev_zuc_pmd_drv);
 382 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_ZUC_PMD, cryptodev_zuc_pmd);
 383 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ZUC_PMD,
 384         "max_nb_queue_pairs=<int> socket_id=<int>");
 385 RTE_PMD_REGISTER_CRYPTO_DRIVER(zuc_crypto_drv, cryptodev_zuc_pmd_drv.driver,
 386                 pmd_driver_id_zuc);
 387
 388 /* Constructor function to register zuc PMD */
 389 RTE_INIT(ipsec_mb_register_zuc)
 390 {
 391         struct ipsec_mb_internals *zuc_data
 392             = &ipsec_mb_pmds[IPSEC_MB_PMD_TYPE_ZUC];
 393
 394         zuc_data->caps = zuc_capabilities;
 395         zuc_data->dequeue_burst = zuc_pmd_dequeue_burst;
 396         zuc_data->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO
 397                         | RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING
 398                         | RTE_CRYPTODEV_FF_NON_BYTE_ALIGNED_DATA
 399                         | RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT
 400                         | RTE_CRYPTODEV_FF_SYM_SESSIONLESS
 401                         | RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT;
 402         zuc_data->internals_priv_size = 0;
 403         zuc_data->ops = &zuc_pmd_ops;
 404         zuc_data->qp_priv_size = sizeof(struct zuc_qp_data);
 405         zuc_data->session_configure = zuc_session_configure;
 406         zuc_data->session_priv_size = sizeof(struct zuc_session);
 407 }