drivers/crypto/cnxk/cn10k_ipsec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 Marvell.
   3  */
   4
   5 #include <cryptodev_pmd.h>
   6 #include <rte_esp.h>
   7 #include <rte_ip.h>
   8 #include <rte_malloc.h>
   9 #include <rte_security.h>
  10 #include <rte_security_driver.h>
  11 #include <rte_udp.h>
  12
  13 #include "cn10k_ipsec.h"
  14 #include "cnxk_cryptodev.h"
  15 #include "cnxk_cryptodev_ops.h"
  16 #include "cnxk_ipsec.h"
  17 #include "cnxk_security.h"
  18
  19 #include "roc_api.h"
  20
  21 static uint64_t
  22 ipsec_cpt_inst_w7_get(struct roc_cpt *roc_cpt, void *sa)
  23 {
  24         union cpt_inst_w7 w7;
  25
  26         w7.u64 = 0;
  27         w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_IE];
  28         w7.s.ctx_val = 1;
  29         w7.s.cptr = (uint64_t)sa;
  30         rte_mb();
  31
  32         return w7.u64;
  33 }
  34
  35 static int
  36 cn10k_ipsec_outb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
  37                            struct rte_security_ipsec_xform *ipsec_xfrm,
  38                            struct rte_crypto_sym_xform *crypto_xfrm,
  39                            struct rte_security_session *sec_sess)
  40 {
  41         union roc_ot_ipsec_outb_param1 param1;
  42         struct roc_ot_ipsec_outb_sa *sa_dptr;
  43         struct cnxk_ipsec_outb_rlens rlens;
  44         struct cn10k_sec_session *sess;
  45         struct cn10k_ipsec_sa *sa;
  46         union cpt_inst_w4 inst_w4;
  47         void *out_sa;
  48         int ret = 0;
  49
  50         sess = get_sec_session_private_data(sec_sess);
  51         sa = &sess->sa;
  52         out_sa = &sa->out_sa;
  53
  54         /* Allocate memory to be used as dptr for CPT ucode WRITE_SA op */
  55         sa_dptr = plt_zmalloc(sizeof(struct roc_ot_ipsec_outb_sa), 8);
  56         if (sa_dptr == NULL) {
  57                 plt_err("Couldn't allocate memory for SA dptr");
  58                 return -ENOMEM;
  59         }
  60
  61         /* Translate security parameters to SA */
  62         ret = cnxk_ot_ipsec_outb_sa_fill(sa_dptr, ipsec_xfrm, crypto_xfrm);
  63         if (ret) {
  64                 plt_err("Could not fill outbound session parameters");
  65                 goto sa_dptr_free;
  66         }
  67
  68         sa->inst.w7 = ipsec_cpt_inst_w7_get(roc_cpt, out_sa);
  69
  70 #ifdef LA_IPSEC_DEBUG
  71         /* Use IV from application in debug mode */
  72         if (ipsec_xfrm->options.iv_gen_disable == 1) {
  73                 sa_dptr->w2.s.iv_src = ROC_IE_OT_SA_IV_SRC_FROM_SA;
  74                 if (crypto_xfrm->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
  75                         sa->iv_offset = crypto_xfrm->aead.iv.offset;
  76                         sa->iv_length = crypto_xfrm->aead.iv.length;
  77                 } else if (crypto_xfrm->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
  78                         sa->iv_offset = crypto_xfrm->cipher.iv.offset;
  79                         sa->iv_length = crypto_xfrm->cipher.iv.length;
  80                 } else {
  81                         sa->iv_offset = crypto_xfrm->auth.iv.offset;
  82                         sa->iv_length = crypto_xfrm->auth.iv.length;
  83                 }
  84         }
  85 #else
  86         if (ipsec_xfrm->options.iv_gen_disable != 0) {
  87                 plt_err("Application provided IV not supported");
  88                 ret = -ENOTSUP;
  89                 goto sa_dptr_free;
  90         }
  91 #endif
  92
  93         sa->is_outbound = true;
  94
  95         /* Get Rlen calculation data */
  96         ret = cnxk_ipsec_outb_rlens_get(&rlens, ipsec_xfrm, crypto_xfrm);
  97         if (ret)
  98                 goto sa_dptr_free;
  99
 100         sa->max_extended_len = rlens.max_extended_len;
 101
 102         /* pre-populate CPT INST word 4 */
 103         inst_w4.u64 = 0;
 104         inst_w4.s.opcode_major = ROC_IE_OT_MAJOR_OP_PROCESS_OUTBOUND_IPSEC;
 105
 106         param1.u16 = 0;
 107
 108         param1.s.ttl_or_hop_limit = ipsec_xfrm->options.dec_ttl;
 109
 110         /* Disable IP checksum computation by default */
 111         param1.s.ip_csum_disable = ROC_IE_OT_SA_INNER_PKT_IP_CSUM_DISABLE;
 112
 113         if (ipsec_xfrm->options.ip_csum_enable) {
 114                 param1.s.ip_csum_disable =
 115                         ROC_IE_OT_SA_INNER_PKT_IP_CSUM_ENABLE;
 116         }
 117
 118         /* Disable L4 checksum computation by default */
 119         param1.s.l4_csum_disable = ROC_IE_OT_SA_INNER_PKT_L4_CSUM_DISABLE;
 120
 121         if (ipsec_xfrm->options.l4_csum_enable) {
 122                 param1.s.l4_csum_disable =
 123                         ROC_IE_OT_SA_INNER_PKT_L4_CSUM_ENABLE;
 124         }
 125
 126         inst_w4.s.param1 = param1.u16;
 127
 128         sa->inst.w4 = inst_w4.u64;
 129
 130         if (ipsec_xfrm->options.stats == 1) {
 131                 /* Enable mib counters */
 132                 sa_dptr->w0.s.count_mib_bytes = 1;
 133                 sa_dptr->w0.s.count_mib_pkts = 1;
 134         }
 135
 136         memset(out_sa, 0, sizeof(struct roc_ot_ipsec_outb_sa));
 137
 138         /* Copy word0 from sa_dptr to populate ctx_push_sz ctx_size fields */
 139         memcpy(out_sa, sa_dptr, 8);
 140
 141         plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
 142
 143         /* Write session using microcode opcode */
 144         ret = roc_cpt_ctx_write(lf, sa_dptr, out_sa,
 145                                 sizeof(struct roc_ot_ipsec_outb_sa));
 146         if (ret) {
 147                 plt_err("Could not write outbound session to hardware");
 148                 goto sa_dptr_free;
 149         }
 150
 151         /* Trigger CTX flush so that data is written back to DRAM */
 152         roc_cpt_lf_ctx_flush(lf, out_sa, false);
 153
 154         plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
 155
 156 sa_dptr_free:
 157         plt_free(sa_dptr);
 158
 159         return ret;
 160 }
 161
 162 static int
 163 cn10k_ipsec_inb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
 164                           struct rte_security_ipsec_xform *ipsec_xfrm,
 165                           struct rte_crypto_sym_xform *crypto_xfrm,
 166                           struct rte_security_session *sec_sess)
 167 {
 168         union roc_ot_ipsec_inb_param1 param1;
 169         struct roc_ot_ipsec_inb_sa *sa_dptr;
 170         struct cn10k_sec_session *sess;
 171         struct cn10k_ipsec_sa *sa;
 172         union cpt_inst_w4 inst_w4;
 173         void *in_sa;
 174         int ret = 0;
 175
 176         sess = get_sec_session_private_data(sec_sess);
 177         sa = &sess->sa;
 178         in_sa = &sa->in_sa;
 179
 180         /* Allocate memory to be used as dptr for CPT ucode WRITE_SA op */
 181         sa_dptr = plt_zmalloc(sizeof(struct roc_ot_ipsec_inb_sa), 8);
 182         if (sa_dptr == NULL) {
 183                 plt_err("Couldn't allocate memory for SA dptr");
 184                 return -ENOMEM;
 185         }
 186
 187         /* Translate security parameters to SA */
 188         ret = cnxk_ot_ipsec_inb_sa_fill(sa_dptr, ipsec_xfrm, crypto_xfrm,
 189                                         false);
 190         if (ret) {
 191                 plt_err("Could not fill inbound session parameters");
 192                 goto sa_dptr_free;
 193         }
 194
 195         sa->is_outbound = false;
 196         sa->inst.w7 = ipsec_cpt_inst_w7_get(roc_cpt, in_sa);
 197
 198         /* pre-populate CPT INST word 4 */
 199         inst_w4.u64 = 0;
 200         inst_w4.s.opcode_major = ROC_IE_OT_MAJOR_OP_PROCESS_INBOUND_IPSEC;
 201
 202         param1.u16 = 0;
 203
 204         /* Disable IP checksum verification by default */
 205         param1.s.ip_csum_disable = ROC_IE_OT_SA_INNER_PKT_IP_CSUM_DISABLE;
 206
 207         if (ipsec_xfrm->options.ip_csum_enable) {
 208                 param1.s.ip_csum_disable =
 209                         ROC_IE_OT_SA_INNER_PKT_IP_CSUM_ENABLE;
 210         }
 211
 212         /* Disable L4 checksum verification by default */
 213         param1.s.l4_csum_disable = ROC_IE_OT_SA_INNER_PKT_L4_CSUM_DISABLE;
 214
 215         if (ipsec_xfrm->options.l4_csum_enable) {
 216                 param1.s.l4_csum_disable =
 217                         ROC_IE_OT_SA_INNER_PKT_L4_CSUM_ENABLE;
 218         }
 219
 220         param1.s.esp_trailer_disable = 1;
 221
 222         inst_w4.s.param1 = param1.u16;
 223
 224         sa->inst.w4 = inst_w4.u64;
 225
 226         if (ipsec_xfrm->options.stats == 1) {
 227                 /* Enable mib counters */
 228                 sa_dptr->w0.s.count_mib_bytes = 1;
 229                 sa_dptr->w0.s.count_mib_pkts = 1;
 230         }
 231
 232         memset(in_sa, 0, sizeof(struct roc_ot_ipsec_inb_sa));
 233
 234         /* Copy word0 from sa_dptr to populate ctx_push_sz ctx_size fields */
 235         memcpy(in_sa, sa_dptr, 8);
 236
 237         plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
 238
 239         /* Write session using microcode opcode */
 240         ret = roc_cpt_ctx_write(lf, sa_dptr, in_sa,
 241                                 sizeof(struct roc_ot_ipsec_inb_sa));
 242         if (ret) {
 243                 plt_err("Could not write inbound session to hardware");
 244                 goto sa_dptr_free;
 245         }
 246
 247         /* Trigger CTX flush so that data is written back to DRAM */
 248         roc_cpt_lf_ctx_flush(lf, in_sa, true);
 249
 250         plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
 251
 252 sa_dptr_free:
 253         plt_free(sa_dptr);
 254
 255         return ret;
 256 }
 257
 258 static int
 259 cn10k_ipsec_session_create(void *dev,
 260                            struct rte_security_ipsec_xform *ipsec_xfrm,
 261                            struct rte_crypto_sym_xform *crypto_xfrm,
 262                            struct rte_security_session *sess)
 263 {
 264         struct rte_cryptodev *crypto_dev = dev;
 265         struct roc_cpt *roc_cpt;
 266         struct cnxk_cpt_vf *vf;
 267         struct cnxk_cpt_qp *qp;
 268         int ret;
 269
 270         qp = crypto_dev->data->queue_pairs[0];
 271         if (qp == NULL) {
 272                 plt_err("Setup cpt queue pair before creating security session");
 273                 return -EPERM;
 274         }
 275
 276         ret = cnxk_ipsec_xform_verify(ipsec_xfrm, crypto_xfrm);
 277         if (ret)
 278                 return ret;
 279
 280         vf = crypto_dev->data->dev_private;
 281         roc_cpt = &vf->cpt;
 282
 283         if (ipsec_xfrm->direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS)
 284                 return cn10k_ipsec_inb_sa_create(roc_cpt, &qp->lf, ipsec_xfrm,
 285                                                  crypto_xfrm, sess);
 286         else
 287                 return cn10k_ipsec_outb_sa_create(roc_cpt, &qp->lf, ipsec_xfrm,
 288                                                   crypto_xfrm, sess);
 289 }
 290
 291 static int
 292 cn10k_sec_session_create(void *device, struct rte_security_session_conf *conf,
 293                          struct rte_security_session *sess,
 294                          struct rte_mempool *mempool)
 295 {
 296         struct cn10k_sec_session *priv;
 297         int ret;
 298
 299         if (conf->action_type != RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
 300                 return -EINVAL;
 301
 302         if (rte_mempool_get(mempool, (void **)&priv)) {
 303                 plt_err("Could not allocate security session private data");
 304                 return -ENOMEM;
 305         }
 306
 307         set_sec_session_private_data(sess, priv);
 308
 309         if (conf->protocol != RTE_SECURITY_PROTOCOL_IPSEC) {
 310                 ret = -ENOTSUP;
 311                 goto mempool_put;
 312         }
 313         ret = cn10k_ipsec_session_create(device, &conf->ipsec,
 314                                          conf->crypto_xform, sess);
 315         if (ret)
 316                 goto mempool_put;
 317
 318         return 0;
 319
 320 mempool_put:
 321         rte_mempool_put(mempool, priv);
 322         set_sec_session_private_data(sess, NULL);
 323         return ret;
 324 }
 325
 326 static int
 327 cn10k_sec_session_destroy(void *dev, struct rte_security_session *sec_sess)
 328 {
 329         struct rte_cryptodev *crypto_dev = dev;
 330         union roc_ot_ipsec_sa_word2 *w2;
 331         struct cn10k_sec_session *sess;
 332         struct rte_mempool *sess_mp;
 333         struct cn10k_ipsec_sa *sa;
 334         struct cnxk_cpt_qp *qp;
 335         struct roc_cpt_lf *lf;
 336         void *sa_dptr = NULL;
 337         int ret;
 338
 339         sess = get_sec_session_private_data(sec_sess);
 340         if (sess == NULL)
 341                 return 0;
 342
 343         qp = crypto_dev->data->queue_pairs[0];
 344         if (qp == NULL)
 345                 return 0;
 346
 347         lf = &qp->lf;
 348
 349         sa = &sess->sa;
 350
 351         /* Trigger CTX flush to write dirty data back to DRAM */
 352         roc_cpt_lf_ctx_flush(lf, &sa->in_sa, false);
 353
 354         ret = -1;
 355
 356         if (sa->is_outbound) {
 357                 sa_dptr = plt_zmalloc(sizeof(struct roc_ot_ipsec_outb_sa), 8);
 358                 if (sa_dptr != NULL) {
 359                         roc_ot_ipsec_outb_sa_init(sa_dptr);
 360
 361                         ret = roc_cpt_ctx_write(
 362                                 lf, sa_dptr, &sa->out_sa,
 363                                 sizeof(struct roc_ot_ipsec_outb_sa));
 364                 }
 365         } else {
 366                 sa_dptr = plt_zmalloc(sizeof(struct roc_ot_ipsec_inb_sa), 8);
 367                 if (sa_dptr != NULL) {
 368                         roc_ot_ipsec_inb_sa_init(sa_dptr, false);
 369
 370                         ret = roc_cpt_ctx_write(
 371                                 lf, sa_dptr, &sa->in_sa,
 372                                 sizeof(struct roc_ot_ipsec_inb_sa));
 373                 }
 374         }
 375
 376         plt_free(sa_dptr);
 377
 378         if (ret) {
 379                 /* MC write_ctx failed. Attempt reload of CTX */
 380
 381                 /* Wait for 1 ms so that flush is complete */
 382                 rte_delay_ms(1);
 383
 384                 w2 = (union roc_ot_ipsec_sa_word2 *)&sa->in_sa.w2;
 385                 w2->s.valid = 0;
 386
 387                 plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
 388
 389                 /* Trigger CTX reload to fetch new data from DRAM */
 390                 roc_cpt_lf_ctx_reload(lf, &sa->in_sa);
 391         }
 392
 393         sess_mp = rte_mempool_from_obj(sess);
 394
 395         set_sec_session_private_data(sec_sess, NULL);
 396         rte_mempool_put(sess_mp, sess);
 397
 398         return 0;
 399 }
 400
 401 static unsigned int
 402 cn10k_sec_session_get_size(void *device __rte_unused)
 403 {
 404         return sizeof(struct cn10k_sec_session);
 405 }
 406
 407 static int
 408 cn10k_sec_session_stats_get(void *device, struct rte_security_session *sess,
 409                             struct rte_security_stats *stats)
 410 {
 411         struct rte_cryptodev *crypto_dev = device;
 412         struct roc_ot_ipsec_outb_sa *out_sa;
 413         struct roc_ot_ipsec_inb_sa *in_sa;
 414         union roc_ot_ipsec_sa_word2 *w2;
 415         struct cn10k_sec_session *priv;
 416         struct cn10k_ipsec_sa *sa;
 417         struct cnxk_cpt_qp *qp;
 418
 419         priv = get_sec_session_private_data(sess);
 420         if (priv == NULL)
 421                 return -EINVAL;
 422
 423         qp = crypto_dev->data->queue_pairs[0];
 424         if (qp == NULL)
 425                 return -EINVAL;
 426
 427         sa = &priv->sa;
 428         w2 = (union roc_ot_ipsec_sa_word2 *)&sa->in_sa.w2;
 429
 430         stats->protocol = RTE_SECURITY_PROTOCOL_IPSEC;
 431
 432         if (w2->s.dir == ROC_IE_SA_DIR_OUTBOUND) {
 433                 out_sa = &sa->out_sa;
 434                 roc_cpt_lf_ctx_flush(&qp->lf, out_sa, false);
 435                 rte_delay_ms(1);
 436                 stats->ipsec.opackets = out_sa->ctx.mib_pkts;
 437                 stats->ipsec.obytes = out_sa->ctx.mib_octs;
 438         } else {
 439                 in_sa = &sa->in_sa;
 440                 roc_cpt_lf_ctx_flush(&qp->lf, in_sa, false);
 441                 rte_delay_ms(1);
 442                 stats->ipsec.ipackets = in_sa->ctx.mib_pkts;
 443                 stats->ipsec.ibytes = in_sa->ctx.mib_octs;
 444         }
 445
 446         return 0;
 447 }
 448
 449 static int
 450 cn10k_sec_session_update(void *device, struct rte_security_session *sess,
 451                          struct rte_security_session_conf *conf)
 452 {
 453         struct rte_cryptodev *crypto_dev = device;
 454         struct cn10k_sec_session *priv;
 455         struct roc_cpt *roc_cpt;
 456         struct cnxk_cpt_qp *qp;
 457         struct cnxk_cpt_vf *vf;
 458         int ret;
 459
 460         priv = get_sec_session_private_data(sess);
 461         if (priv == NULL)
 462                 return -EINVAL;
 463
 464         qp = crypto_dev->data->queue_pairs[0];
 465         if (qp == NULL)
 466                 return -EINVAL;
 467
 468         if (conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS)
 469                 return -ENOTSUP;
 470
 471         ret = cnxk_ipsec_xform_verify(&conf->ipsec, conf->crypto_xform);
 472         if (ret)
 473                 return ret;
 474
 475         vf = crypto_dev->data->dev_private;
 476         roc_cpt = &vf->cpt;
 477
 478         return cn10k_ipsec_outb_sa_create(roc_cpt, &qp->lf, &conf->ipsec,
 479                                           conf->crypto_xform, sess);
 480 }
 481
 482 /* Update platform specific security ops */
 483 void
 484 cn10k_sec_ops_override(void)
 485 {
 486         /* Update platform specific ops */
 487         cnxk_sec_ops.session_create = cn10k_sec_session_create;
 488         cnxk_sec_ops.session_destroy = cn10k_sec_session_destroy;
 489         cnxk_sec_ops.session_get_size = cn10k_sec_session_get_size;
 490         cnxk_sec_ops.session_stats_get = cn10k_sec_session_stats_get;
 491         cnxk_sec_ops.session_update = cn10k_sec_session_update;
 492 }