drivers/crypto/ipsec_mb/pmd_chacha_poly.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2021 Intel Corporation
   3  */
   4
   5 #include "pmd_chacha_poly_priv.h"
   6
   7 /** Parse crypto xform chain and set private session parameters. */
   8 static int
   9 chacha20_poly1305_session_configure(IMB_MGR * mb_mgr __rte_unused,
  10                 void *priv_sess, const struct rte_crypto_sym_xform *xform)
  11 {
  12         struct chacha20_poly1305_session *sess = priv_sess;
  13         const struct rte_crypto_sym_xform *auth_xform;
  14         const struct rte_crypto_sym_xform *cipher_xform;
  15         const struct rte_crypto_sym_xform *aead_xform;
  16
  17         uint8_t key_length;
  18         const uint8_t *key;
  19         enum ipsec_mb_operation mode;
  20         int ret = 0;
  21
  22         ret = ipsec_mb_parse_xform(xform, &mode, &auth_xform,
  23                                 &cipher_xform, &aead_xform);
  24         if (ret)
  25                 return ret;
  26
  27         sess->op = mode;
  28
  29         switch (sess->op) {
  30         case IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT:
  31         case IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT:
  32                 if (aead_xform->aead.algo !=
  33                                 RTE_CRYPTO_AEAD_CHACHA20_POLY1305) {
  34                         IPSEC_MB_LOG(ERR,
  35                         "The only combined operation supported is CHACHA20 POLY1305");
  36                         ret = -ENOTSUP;
  37                         goto error_exit;
  38                 }
  39                 /* Set IV parameters */
  40                 sess->iv.offset = aead_xform->aead.iv.offset;
  41                 sess->iv.length = aead_xform->aead.iv.length;
  42                 key_length = aead_xform->aead.key.length;
  43                 key = aead_xform->aead.key.data;
  44                 sess->aad_length = aead_xform->aead.aad_length;
  45                 sess->req_digest_length = aead_xform->aead.digest_length;
  46                 break;
  47         default:
  48                 IPSEC_MB_LOG(
  49                     ERR, "Wrong xform type, has to be AEAD or authentication");
  50                 ret = -ENOTSUP;
  51                 goto error_exit;
  52         }
  53
  54         /* IV check */
  55         if (sess->iv.length != CHACHA20_POLY1305_IV_LENGTH &&
  56                 sess->iv.length != 0) {
  57                 IPSEC_MB_LOG(ERR, "Wrong IV length");
  58                 ret = -EINVAL;
  59                 goto error_exit;
  60         }
  61
  62         /* Check key length */
  63         if (key_length != CHACHA20_POLY1305_KEY_SIZE) {
  64                 IPSEC_MB_LOG(ERR, "Invalid key length");
  65                 ret = -EINVAL;
  66                 goto error_exit;
  67         } else {
  68                 memcpy(sess->key, key, CHACHA20_POLY1305_KEY_SIZE);
  69         }
  70
  71         /* Digest check */
  72         if (sess->req_digest_length !=  CHACHA20_POLY1305_DIGEST_LENGTH) {
  73                 IPSEC_MB_LOG(ERR, "Invalid digest length");
  74                 ret = -EINVAL;
  75                 goto error_exit;
  76         } else {
  77                 sess->gen_digest_length = CHACHA20_POLY1305_DIGEST_LENGTH;
  78         }
  79
  80 error_exit:
  81         return ret;
  82 }
  83
  84 /**
  85  * Process a crypto operation, calling
  86  * the direct chacha poly API from the multi buffer library.
  87  *
  88  * @param       qp              queue pair
  89  * @param       op              symmetric crypto operation
  90  * @param       session         chacha poly session
  91  *
  92  * @return
  93  * - Return 0 if success
  94  */
  95 static int
  96 chacha20_poly1305_crypto_op(struct ipsec_mb_qp *qp, struct rte_crypto_op *op,
  97                 struct chacha20_poly1305_session *session)
  98 {
  99         struct chacha20_poly1305_qp_data *qp_data =
 100                                         ipsec_mb_get_qp_private_data(qp);
 101         uint8_t *src, *dst;
 102         uint8_t *iv_ptr;
 103         struct rte_crypto_sym_op *sym_op = op->sym;
 104         struct rte_mbuf *m_src = sym_op->m_src;
 105         uint32_t offset, data_offset, data_length;
 106         uint32_t part_len, data_len;
 107         int total_len;
 108         uint8_t *tag;
 109         unsigned int oop = 0;
 110
 111         offset = sym_op->aead.data.offset;
 112         data_offset = offset;
 113         data_length = sym_op->aead.data.length;
 114         RTE_ASSERT(m_src != NULL);
 115
 116         while (offset >= m_src->data_len && data_length != 0) {
 117                 offset -= m_src->data_len;
 118                 m_src = m_src->next;
 119
 120                 RTE_ASSERT(m_src != NULL);
 121         }
 122
 123         src = rte_pktmbuf_mtod_offset(m_src, uint8_t *, offset);
 124
 125         data_len = m_src->data_len - offset;
 126         part_len = (data_len < data_length) ? data_len :
 127                         data_length;
 128
 129         /* In-place */
 130         if (sym_op->m_dst == NULL || (sym_op->m_dst == sym_op->m_src))
 131                 dst = src;
 132         /* Out-of-place */
 133         else {
 134                 oop = 1;
 135                 /* Segmented destination buffer is not supported
 136                  * if operation is Out-of-place
 137                  */
 138                 RTE_ASSERT(rte_pktmbuf_is_contiguous(sym_op->m_dst));
 139                 dst = rte_pktmbuf_mtod_offset(sym_op->m_dst, uint8_t *,
 140                                         data_offset);
 141         }
 142
 143         iv_ptr = rte_crypto_op_ctod_offset(op, uint8_t *,
 144                                 session->iv.offset);
 145
 146         IMB_CHACHA20_POLY1305_INIT(qp->mb_mgr, session->key,
 147                                 &qp_data->chacha20_poly1305_ctx_data,
 148                                 iv_ptr, sym_op->aead.aad.data,
 149                                 (uint64_t)session->aad_length);
 150
 151         if (session->op == IPSEC_MB_OP_AEAD_AUTHENTICATED_ENCRYPT) {
 152                 IMB_CHACHA20_POLY1305_ENC_UPDATE(qp->mb_mgr,
 153                                 session->key,
 154                                 &qp_data->chacha20_poly1305_ctx_data,
 155                                 dst, src, (uint64_t)part_len);
 156                 total_len = data_length - part_len;
 157
 158                 while (total_len) {
 159                         m_src = m_src->next;
 160                         RTE_ASSERT(m_src != NULL);
 161
 162                         src = rte_pktmbuf_mtod(m_src, uint8_t *);
 163                         if (oop)
 164                                 dst += part_len;
 165                         else
 166                                 dst = src;
 167                         part_len = (m_src->data_len < total_len) ?
 168                                         m_src->data_len : total_len;
 169
 170                         if (dst == NULL || src == NULL) {
 171                                 IPSEC_MB_LOG(ERR, "Invalid src or dst input");
 172                                 return -EINVAL;
 173                         }
 174                         IMB_CHACHA20_POLY1305_ENC_UPDATE(qp->mb_mgr,
 175                                         session->key,
 176                                         &qp_data->chacha20_poly1305_ctx_data,
 177                                         dst, src, (uint64_t)part_len);
 178                         total_len -= part_len;
 179                         if (total_len < 0) {
 180                                 IPSEC_MB_LOG(ERR, "Invalid part len");
 181                                 return -EINVAL;
 182                         }
 183                 }
 184
 185                 tag = sym_op->aead.digest.data;
 186                 IMB_CHACHA20_POLY1305_ENC_FINALIZE(qp->mb_mgr,
 187                                         &qp_data->chacha20_poly1305_ctx_data,
 188                                         tag, session->gen_digest_length);
 189
 190         } else {
 191                 IMB_CHACHA20_POLY1305_DEC_UPDATE(qp->mb_mgr,
 192                                         session->key,
 193                                         &qp_data->chacha20_poly1305_ctx_data,
 194                                         dst, src, (uint64_t)part_len);
 195
 196                 total_len = data_length - part_len;
 197
 198                 while (total_len) {
 199                         m_src = m_src->next;
 200
 201                         RTE_ASSERT(m_src != NULL);
 202
 203                         src = rte_pktmbuf_mtod(m_src, uint8_t *);
 204                         if (oop)
 205                                 dst += part_len;
 206                         else
 207                                 dst = src;
 208                         part_len = (m_src->data_len < total_len) ?
 209                                         m_src->data_len : total_len;
 210
 211                         if (dst == NULL || src == NULL) {
 212                                 IPSEC_MB_LOG(ERR, "Invalid src or dst input");
 213                                 return -EINVAL;
 214                         }
 215                         IMB_CHACHA20_POLY1305_DEC_UPDATE(qp->mb_mgr,
 216                                         session->key,
 217                                         &qp_data->chacha20_poly1305_ctx_data,
 218                                         dst, src, (uint64_t)part_len);
 219                         total_len -= part_len;
 220                         if (total_len < 0) {
 221                                 IPSEC_MB_LOG(ERR, "Invalid part len");
 222                                 return -EINVAL;
 223                         }
 224                 }
 225
 226                 tag = qp_data->temp_digest;
 227                 IMB_CHACHA20_POLY1305_DEC_FINALIZE(qp->mb_mgr,
 228                                         &qp_data->chacha20_poly1305_ctx_data,
 229                                         tag, session->gen_digest_length);
 230         }
 231
 232         return 0;
 233 }
 234
 235 /**
 236  * Process a completed chacha poly op
 237  *
 238  * @param qp            Queue Pair to process
 239  * @param op            Crypto operation
 240  * @param sess          Crypto session
 241  *
 242  * @return
 243  * - void
 244  */
 245 static void
 246 post_process_chacha20_poly1305_crypto_op(struct ipsec_mb_qp *qp,
 247                 struct rte_crypto_op *op,
 248                 struct chacha20_poly1305_session *session)
 249 {
 250         struct chacha20_poly1305_qp_data *qp_data =
 251                                         ipsec_mb_get_qp_private_data(qp);
 252
 253         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 254         /* Verify digest if required */
 255         if (session->op == IPSEC_MB_OP_AEAD_AUTHENTICATED_DECRYPT ||
 256                         session->op == IPSEC_MB_OP_HASH_VERIFY_ONLY) {
 257                 uint8_t *digest = op->sym->aead.digest.data;
 258                 uint8_t *tag = qp_data->temp_digest;
 259
 260 #ifdef RTE_LIBRTE_PMD_CHACHA20_POLY1305_DEBUG
 261                 rte_hexdump(stdout, "auth tag (orig):",
 262                                 digest, session->req_digest_length);
 263                 rte_hexdump(stdout, "auth tag (calc):",
 264                                 tag, session->req_digest_length);
 265 #endif
 266                 if (memcmp(tag, digest, session->req_digest_length) != 0)
 267                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 268
 269         }
 270
 271 }
 272
 273 /**
 274  * Process a completed Chacha20_poly1305 request
 275  *
 276  * @param qp            Queue Pair to process
 277  * @param op            Crypto operation
 278  * @param sess          Crypto session
 279  *
 280  * @return
 281  * - void
 282  */
 283 static void
 284 handle_completed_chacha20_poly1305_crypto_op(struct ipsec_mb_qp *qp,
 285                 struct rte_crypto_op *op,
 286                 struct chacha20_poly1305_session *sess)
 287 {
 288         post_process_chacha20_poly1305_crypto_op(qp, op, sess);
 289
 290         /* Free session if a session-less crypto op */
 291         if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
 292                 memset(sess, 0, sizeof(struct chacha20_poly1305_session));
 293                 memset(op->sym->session, 0,
 294                         rte_cryptodev_sym_get_existing_header_session_size(
 295                                 op->sym->session));
 296                 rte_mempool_put(qp->sess_mp_priv, sess);
 297                 rte_mempool_put(qp->sess_mp, op->sym->session);
 298                 op->sym->session = NULL;
 299         }
 300 }
 301
 302 static uint16_t
 303 chacha20_poly1305_pmd_dequeue_burst(void *queue_pair,
 304                 struct rte_crypto_op **ops, uint16_t nb_ops)
 305 {
 306         struct chacha20_poly1305_session *sess;
 307         struct ipsec_mb_qp *qp = queue_pair;
 308
 309         int retval = 0;
 310         unsigned int i = 0, nb_dequeued;
 311
 312         nb_dequeued = rte_ring_dequeue_burst(qp->ingress_queue,
 313                         (void **)ops, nb_ops, NULL);
 314
 315         for (i = 0; i < nb_dequeued; i++) {
 316
 317                 sess = ipsec_mb_get_session_private(qp, ops[i]);
 318                 if (unlikely(sess == NULL)) {
 319                         ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 320                         qp->stats.dequeue_err_count++;
 321                         break;
 322                 }
 323
 324                 retval = chacha20_poly1305_crypto_op(qp, ops[i], sess);
 325                 if (retval < 0) {
 326                         ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 327                         qp->stats.dequeue_err_count++;
 328                         break;
 329                 }
 330
 331                 handle_completed_chacha20_poly1305_crypto_op(qp, ops[i], sess);
 332         }
 333
 334         qp->stats.dequeued_count += i;
 335
 336         return i;
 337 }
 338
 339 struct rte_cryptodev_ops chacha20_poly1305_pmd_ops = {
 340         .dev_configure = ipsec_mb_config,
 341         .dev_start = ipsec_mb_start,
 342         .dev_stop = ipsec_mb_stop,
 343         .dev_close = ipsec_mb_close,
 344
 345         .stats_get = ipsec_mb_stats_get,
 346         .stats_reset = ipsec_mb_stats_reset,
 347
 348         .dev_infos_get = ipsec_mb_info_get,
 349
 350         .queue_pair_setup = ipsec_mb_qp_setup,
 351         .queue_pair_release = ipsec_mb_qp_release,
 352
 353         .sym_session_get_size = ipsec_mb_sym_session_get_size,
 354         .sym_session_configure = ipsec_mb_sym_session_configure,
 355         .sym_session_clear = ipsec_mb_sym_session_clear
 356 };
 357
 358 struct rte_cryptodev_ops *rte_chacha20_poly1305_pmd_ops =
 359                                                 &chacha20_poly1305_pmd_ops;
 360
 361 static int
 362 chacha20_poly1305_probe(struct rte_vdev_device *vdev)
 363 {
 364         return ipsec_mb_create(vdev, IPSEC_MB_PMD_TYPE_CHACHA20_POLY1305);
 365 }
 366
 367 static struct rte_vdev_driver cryptodev_chacha20_poly1305_pmd_drv = {
 368         .probe = chacha20_poly1305_probe,
 369         .remove = ipsec_mb_remove
 370 };
 371
 372 static struct cryptodev_driver chacha20_poly1305_crypto_drv;
 373
 374 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_CHACHA20_POLY1305_PMD,
 375                                         cryptodev_chacha20_poly1305_pmd_drv);
 376 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_CHACHA20_POLY1305_PMD,
 377                                         cryptodev_chacha20_poly1305_pmd);
 378 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_CHACHA20_POLY1305_PMD,
 379                                "max_nb_queue_pairs=<int> socket_id=<int>");
 380 RTE_PMD_REGISTER_CRYPTO_DRIVER(chacha20_poly1305_crypto_drv,
 381                                 cryptodev_chacha20_poly1305_pmd_drv.driver,
 382                                 pmd_driver_id_chacha20_poly1305);
 383
 384 /* Constructor function to register chacha20_poly1305 PMD */
 385 RTE_INIT(ipsec_mb_register_chacha20_poly1305)
 386 {
 387         struct ipsec_mb_internals *chacha_poly_data
 388                 = &ipsec_mb_pmds[IPSEC_MB_PMD_TYPE_CHACHA20_POLY1305];
 389
 390         chacha_poly_data->caps = chacha20_poly1305_capabilities;
 391         chacha_poly_data->dequeue_burst = chacha20_poly1305_pmd_dequeue_burst;
 392         chacha_poly_data->feature_flags =
 393                 RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
 394                 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
 395                 RTE_CRYPTODEV_FF_IN_PLACE_SGL |
 396                 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT |
 397                 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT |
 398                 RTE_CRYPTODEV_FF_SYM_CPU_CRYPTO |
 399                 RTE_CRYPTODEV_FF_SYM_SESSIONLESS;
 400         chacha_poly_data->internals_priv_size = 0;
 401         chacha_poly_data->ops = &chacha20_poly1305_pmd_ops;
 402         chacha_poly_data->qp_priv_size =
 403                         sizeof(struct chacha20_poly1305_qp_data);
 404         chacha_poly_data->session_configure =
 405                         chacha20_poly1305_session_configure;
 406         chacha_poly_data->session_priv_size =
 407                         sizeof(struct chacha20_poly1305_session);
 408 }