drivers/crypto/cnxk/cn10k_ipsec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 Marvell.
   3  */
   4
   5 #include <cryptodev_pmd.h>
   6 #include <rte_esp.h>
   7 #include <rte_ip.h>
   8 #include <rte_malloc.h>
   9 #include <rte_security.h>
  10 #include <rte_security_driver.h>
  11 #include <rte_udp.h>
  12
  13 #include "cn10k_ipsec.h"
  14 #include "cnxk_cryptodev.h"
  15 #include "cnxk_cryptodev_ops.h"
  16 #include "cnxk_ipsec.h"
  17 #include "cnxk_security.h"
  18
  19 #include "roc_api.h"
  20
  21 static uint64_t
  22 ipsec_cpt_inst_w7_get(struct roc_cpt *roc_cpt, void *sa)
  23 {
  24         union cpt_inst_w7 w7;
  25
  26         w7.u64 = 0;
  27         w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_IE];
  28         w7.s.ctx_val = 1;
  29         w7.s.cptr = (uint64_t)sa;
  30         rte_mb();
  31
  32         return w7.u64;
  33 }
  34
  35 static int
  36 cn10k_ipsec_outb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
  37                            struct rte_security_ipsec_xform *ipsec_xfrm,
  38                            struct rte_crypto_sym_xform *crypto_xfrm,
  39                            struct rte_security_session *sec_sess)
  40 {
  41         union roc_ot_ipsec_outb_param1 param1;
  42         struct roc_ot_ipsec_outb_sa *sa_dptr;
  43         struct cnxk_ipsec_outb_rlens rlens;
  44         struct cn10k_sec_session *sess;
  45         struct cn10k_ipsec_sa *sa;
  46         union cpt_inst_w4 inst_w4;
  47         void *out_sa;
  48         int ret = 0;
  49
  50         sess = get_sec_session_private_data(sec_sess);
  51         sa = &sess->sa;
  52         out_sa = &sa->out_sa;
  53
  54         /* Allocate memory to be used as dptr for CPT ucode WRITE_SA op */
  55         sa_dptr = plt_zmalloc(sizeof(struct roc_ot_ipsec_outb_sa), 8);
  56         if (sa_dptr == NULL) {
  57                 plt_err("Couldn't allocate memory for SA dptr");
  58                 return -ENOMEM;
  59         }
  60
  61         /* Translate security parameters to SA */
  62         ret = cnxk_ot_ipsec_outb_sa_fill(sa_dptr, ipsec_xfrm, crypto_xfrm);
  63         if (ret) {
  64                 plt_err("Could not fill outbound session parameters");
  65                 goto sa_dptr_free;
  66         }
  67
  68         sa->inst.w7 = ipsec_cpt_inst_w7_get(roc_cpt, out_sa);
  69
  70 #ifdef LA_IPSEC_DEBUG
  71         /* Use IV from application in debug mode */
  72         if (ipsec_xfrm->options.iv_gen_disable == 1) {
  73                 sa_dptr->w2.s.iv_src = ROC_IE_OT_SA_IV_SRC_FROM_SA;
  74                 if (crypto_xfrm->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
  75                         sa->iv_offset = crypto_xfrm->aead.iv.offset;
  76                         sa->iv_length = crypto_xfrm->aead.iv.length;
  77                 } else {
  78                         sa->iv_offset = crypto_xfrm->cipher.iv.offset;
  79                         sa->iv_length = crypto_xfrm->cipher.iv.length;
  80                 }
  81         }
  82 #else
  83         if (ipsec_xfrm->options.iv_gen_disable != 0) {
  84                 plt_err("Application provided IV not supported");
  85                 ret = -ENOTSUP;
  86                 goto sa_dptr_free;
  87         }
  88 #endif
  89
  90         sa->is_outbound = true;
  91
  92         /* Get Rlen calculation data */
  93         ret = cnxk_ipsec_outb_rlens_get(&rlens, ipsec_xfrm, crypto_xfrm);
  94         if (ret)
  95                 goto sa_dptr_free;
  96
  97         sa->max_extended_len = rlens.max_extended_len;
  98
  99         /* pre-populate CPT INST word 4 */
 100         inst_w4.u64 = 0;
 101         inst_w4.s.opcode_major = ROC_IE_OT_MAJOR_OP_PROCESS_OUTBOUND_IPSEC;
 102
 103         param1.u16 = 0;
 104
 105         param1.s.ttl_or_hop_limit = ipsec_xfrm->options.dec_ttl;
 106
 107         /* Disable IP checksum computation by default */
 108         param1.s.ip_csum_disable = ROC_IE_OT_SA_INNER_PKT_IP_CSUM_DISABLE;
 109
 110         if (ipsec_xfrm->options.ip_csum_enable) {
 111                 param1.s.ip_csum_disable =
 112                         ROC_IE_OT_SA_INNER_PKT_IP_CSUM_ENABLE;
 113         }
 114
 115         /* Disable L4 checksum computation by default */
 116         param1.s.l4_csum_disable = ROC_IE_OT_SA_INNER_PKT_L4_CSUM_DISABLE;
 117
 118         if (ipsec_xfrm->options.l4_csum_enable) {
 119                 param1.s.l4_csum_disable =
 120                         ROC_IE_OT_SA_INNER_PKT_L4_CSUM_ENABLE;
 121         }
 122
 123         inst_w4.s.param1 = param1.u16;
 124
 125         sa->inst.w4 = inst_w4.u64;
 126
 127         if (ipsec_xfrm->options.stats == 1) {
 128                 /* Enable mib counters */
 129                 sa_dptr->w0.s.count_mib_bytes = 1;
 130                 sa_dptr->w0.s.count_mib_pkts = 1;
 131         }
 132
 133         memset(out_sa, 0, sizeof(struct roc_ot_ipsec_outb_sa));
 134
 135         /* Copy word0 from sa_dptr to populate ctx_push_sz ctx_size fields */
 136         memcpy(out_sa, sa_dptr, 8);
 137
 138         plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
 139
 140         /* Write session using microcode opcode */
 141         ret = roc_cpt_ctx_write(lf, sa_dptr, out_sa,
 142                                 sizeof(struct roc_ot_ipsec_outb_sa));
 143         if (ret) {
 144                 plt_err("Could not write outbound session to hardware");
 145                 goto sa_dptr_free;
 146         }
 147
 148         /* Trigger CTX flush so that data is written back to DRAM */
 149         roc_cpt_lf_ctx_flush(lf, out_sa, false);
 150
 151         plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
 152
 153 sa_dptr_free:
 154         plt_free(sa_dptr);
 155
 156         return ret;
 157 }
 158
 159 static int
 160 cn10k_ipsec_inb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
 161                           struct rte_security_ipsec_xform *ipsec_xfrm,
 162                           struct rte_crypto_sym_xform *crypto_xfrm,
 163                           struct rte_security_session *sec_sess)
 164 {
 165         union roc_ot_ipsec_inb_param1 param1;
 166         struct roc_ot_ipsec_inb_sa *sa_dptr;
 167         struct cn10k_sec_session *sess;
 168         struct cn10k_ipsec_sa *sa;
 169         union cpt_inst_w4 inst_w4;
 170         void *in_sa;
 171         int ret = 0;
 172
 173         sess = get_sec_session_private_data(sec_sess);
 174         sa = &sess->sa;
 175         in_sa = &sa->in_sa;
 176
 177         /* Allocate memory to be used as dptr for CPT ucode WRITE_SA op */
 178         sa_dptr = plt_zmalloc(sizeof(struct roc_ot_ipsec_inb_sa), 8);
 179         if (sa_dptr == NULL) {
 180                 plt_err("Couldn't allocate memory for SA dptr");
 181                 return -ENOMEM;
 182         }
 183
 184         /* Translate security parameters to SA */
 185         ret = cnxk_ot_ipsec_inb_sa_fill(sa_dptr, ipsec_xfrm, crypto_xfrm);
 186         if (ret) {
 187                 plt_err("Could not fill inbound session parameters");
 188                 goto sa_dptr_free;
 189         }
 190
 191         sa->is_outbound = false;
 192         sa->inst.w7 = ipsec_cpt_inst_w7_get(roc_cpt, in_sa);
 193
 194         /* pre-populate CPT INST word 4 */
 195         inst_w4.u64 = 0;
 196         inst_w4.s.opcode_major = ROC_IE_OT_MAJOR_OP_PROCESS_INBOUND_IPSEC;
 197
 198         param1.u16 = 0;
 199
 200         /* Disable IP checksum verification by default */
 201         param1.s.ip_csum_disable = ROC_IE_OT_SA_INNER_PKT_IP_CSUM_DISABLE;
 202
 203         if (ipsec_xfrm->options.ip_csum_enable) {
 204                 param1.s.ip_csum_disable =
 205                         ROC_IE_OT_SA_INNER_PKT_IP_CSUM_ENABLE;
 206         }
 207
 208         /* Disable L4 checksum verification by default */
 209         param1.s.l4_csum_disable = ROC_IE_OT_SA_INNER_PKT_L4_CSUM_DISABLE;
 210
 211         if (ipsec_xfrm->options.l4_csum_enable) {
 212                 param1.s.l4_csum_disable =
 213                         ROC_IE_OT_SA_INNER_PKT_L4_CSUM_ENABLE;
 214         }
 215
 216         param1.s.esp_trailer_disable = 1;
 217
 218         inst_w4.s.param1 = param1.u16;
 219
 220         sa->inst.w4 = inst_w4.u64;
 221
 222         if (ipsec_xfrm->options.stats == 1) {
 223                 /* Enable mib counters */
 224                 sa_dptr->w0.s.count_mib_bytes = 1;
 225                 sa_dptr->w0.s.count_mib_pkts = 1;
 226         }
 227
 228         memset(in_sa, 0, sizeof(struct roc_ot_ipsec_inb_sa));
 229
 230         /* Copy word0 from sa_dptr to populate ctx_push_sz ctx_size fields */
 231         memcpy(in_sa, sa_dptr, 8);
 232
 233         plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
 234
 235         /* Write session using microcode opcode */
 236         ret = roc_cpt_ctx_write(lf, sa_dptr, in_sa,
 237                                 sizeof(struct roc_ot_ipsec_inb_sa));
 238         if (ret) {
 239                 plt_err("Could not write inbound session to hardware");
 240                 goto sa_dptr_free;
 241         }
 242
 243         /* Trigger CTX flush so that data is written back to DRAM */
 244         roc_cpt_lf_ctx_flush(lf, in_sa, true);
 245
 246         plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
 247
 248 sa_dptr_free:
 249         plt_free(sa_dptr);
 250
 251         return ret;
 252 }
 253
 254 static int
 255 cn10k_ipsec_session_create(void *dev,
 256                            struct rte_security_ipsec_xform *ipsec_xfrm,
 257                            struct rte_crypto_sym_xform *crypto_xfrm,
 258                            struct rte_security_session *sess)
 259 {
 260         struct rte_cryptodev *crypto_dev = dev;
 261         struct roc_cpt *roc_cpt;
 262         struct cnxk_cpt_vf *vf;
 263         struct cnxk_cpt_qp *qp;
 264         int ret;
 265
 266         qp = crypto_dev->data->queue_pairs[0];
 267         if (qp == NULL) {
 268                 plt_err("Setup cpt queue pair before creating security session");
 269                 return -EPERM;
 270         }
 271
 272         ret = cnxk_ipsec_xform_verify(ipsec_xfrm, crypto_xfrm);
 273         if (ret)
 274                 return ret;
 275
 276         vf = crypto_dev->data->dev_private;
 277         roc_cpt = &vf->cpt;
 278
 279         if (ipsec_xfrm->direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS)
 280                 return cn10k_ipsec_inb_sa_create(roc_cpt, &qp->lf, ipsec_xfrm,
 281                                                  crypto_xfrm, sess);
 282         else
 283                 return cn10k_ipsec_outb_sa_create(roc_cpt, &qp->lf, ipsec_xfrm,
 284                                                   crypto_xfrm, sess);
 285 }
 286
 287 static int
 288 cn10k_sec_session_create(void *device, struct rte_security_session_conf *conf,
 289                          struct rte_security_session *sess,
 290                          struct rte_mempool *mempool)
 291 {
 292         struct cn10k_sec_session *priv;
 293         int ret;
 294
 295         if (conf->action_type != RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
 296                 return -EINVAL;
 297
 298         if (rte_mempool_get(mempool, (void **)&priv)) {
 299                 plt_err("Could not allocate security session private data");
 300                 return -ENOMEM;
 301         }
 302
 303         set_sec_session_private_data(sess, priv);
 304
 305         if (conf->protocol != RTE_SECURITY_PROTOCOL_IPSEC) {
 306                 ret = -ENOTSUP;
 307                 goto mempool_put;
 308         }
 309         ret = cn10k_ipsec_session_create(device, &conf->ipsec,
 310                                          conf->crypto_xform, sess);
 311         if (ret)
 312                 goto mempool_put;
 313
 314         return 0;
 315
 316 mempool_put:
 317         rte_mempool_put(mempool, priv);
 318         set_sec_session_private_data(sess, NULL);
 319         return ret;
 320 }
 321
 322 static int
 323 cn10k_sec_session_destroy(void *dev, struct rte_security_session *sec_sess)
 324 {
 325         struct rte_cryptodev *crypto_dev = dev;
 326         union roc_ot_ipsec_sa_word2 *w2;
 327         struct cn10k_sec_session *sess;
 328         struct rte_mempool *sess_mp;
 329         struct cn10k_ipsec_sa *sa;
 330         struct cnxk_cpt_qp *qp;
 331         struct roc_cpt_lf *lf;
 332
 333         sess = get_sec_session_private_data(sec_sess);
 334         if (sess == NULL)
 335                 return 0;
 336
 337         qp = crypto_dev->data->queue_pairs[0];
 338         if (qp == NULL)
 339                 return 0;
 340
 341         lf = &qp->lf;
 342
 343         sa = &sess->sa;
 344
 345         /* Trigger CTX flush to write dirty data back to DRAM */
 346         roc_cpt_lf_ctx_flush(lf, &sa->in_sa, false);
 347
 348         /* Wait for 1 ms so that flush is complete */
 349         rte_delay_ms(1);
 350
 351         w2 = (union roc_ot_ipsec_sa_word2 *)&sa->in_sa.w2;
 352         w2->s.valid = 0;
 353
 354         plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
 355
 356         /* Trigger CTX reload to fetch new data from DRAM */
 357         roc_cpt_lf_ctx_reload(lf, &sa->in_sa);
 358
 359         sess_mp = rte_mempool_from_obj(sess);
 360
 361         set_sec_session_private_data(sec_sess, NULL);
 362         rte_mempool_put(sess_mp, sess);
 363
 364         return 0;
 365 }
 366
 367 static unsigned int
 368 cn10k_sec_session_get_size(void *device __rte_unused)
 369 {
 370         return sizeof(struct cn10k_sec_session);
 371 }
 372
 373 static int
 374 cn10k_sec_session_stats_get(void *device, struct rte_security_session *sess,
 375                             struct rte_security_stats *stats)
 376 {
 377         struct rte_cryptodev *crypto_dev = device;
 378         struct roc_ot_ipsec_outb_sa *out_sa;
 379         struct roc_ot_ipsec_inb_sa *in_sa;
 380         union roc_ot_ipsec_sa_word2 *w2;
 381         struct cn10k_sec_session *priv;
 382         struct cn10k_ipsec_sa *sa;
 383         struct cnxk_cpt_qp *qp;
 384
 385         priv = get_sec_session_private_data(sess);
 386         if (priv == NULL)
 387                 return -EINVAL;
 388
 389         qp = crypto_dev->data->queue_pairs[0];
 390         if (qp == NULL)
 391                 return -EINVAL;
 392
 393         sa = &priv->sa;
 394         w2 = (union roc_ot_ipsec_sa_word2 *)&sa->in_sa.w2;
 395
 396         stats->protocol = RTE_SECURITY_PROTOCOL_IPSEC;
 397
 398         if (w2->s.dir == ROC_IE_SA_DIR_OUTBOUND) {
 399                 out_sa = &sa->out_sa;
 400                 roc_cpt_lf_ctx_flush(&qp->lf, out_sa, false);
 401                 rte_delay_ms(1);
 402                 stats->ipsec.opackets = out_sa->ctx.mib_pkts;
 403                 stats->ipsec.obytes = out_sa->ctx.mib_octs;
 404         } else {
 405                 in_sa = &sa->in_sa;
 406                 roc_cpt_lf_ctx_flush(&qp->lf, in_sa, false);
 407                 rte_delay_ms(1);
 408                 stats->ipsec.ipackets = in_sa->ctx.mib_pkts;
 409                 stats->ipsec.ibytes = in_sa->ctx.mib_octs;
 410         }
 411
 412         return 0;
 413 }
 414
 415 static int
 416 cn10k_sec_session_update(void *device, struct rte_security_session *sess,
 417                          struct rte_security_session_conf *conf)
 418 {
 419         struct rte_cryptodev *crypto_dev = device;
 420         struct cn10k_sec_session *priv;
 421         struct roc_cpt *roc_cpt;
 422         struct cnxk_cpt_qp *qp;
 423         struct cnxk_cpt_vf *vf;
 424         int ret;
 425
 426         priv = get_sec_session_private_data(sess);
 427         if (priv == NULL)
 428                 return -EINVAL;
 429
 430         qp = crypto_dev->data->queue_pairs[0];
 431         if (qp == NULL)
 432                 return -EINVAL;
 433
 434         if (conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS)
 435                 return -ENOTSUP;
 436
 437         ret = cnxk_ipsec_xform_verify(&conf->ipsec, conf->crypto_xform);
 438         if (ret)
 439                 return ret;
 440
 441         vf = crypto_dev->data->dev_private;
 442         roc_cpt = &vf->cpt;
 443
 444         return cn10k_ipsec_outb_sa_create(roc_cpt, &qp->lf, &conf->ipsec,
 445                                           conf->crypto_xform, sess);
 446 }
 447
 448 /* Update platform specific security ops */
 449 void
 450 cn10k_sec_ops_override(void)
 451 {
 452         /* Update platform specific ops */
 453         cnxk_sec_ops.session_create = cn10k_sec_session_create;
 454         cnxk_sec_ops.session_destroy = cn10k_sec_session_destroy;
 455         cnxk_sec_ops.session_get_size = cn10k_sec_session_get_size;
 456         cnxk_sec_ops.session_stats_get = cn10k_sec_session_stats_get;
 457         cnxk_sec_ops.session_update = cn10k_sec_session_update;
 458 }