drivers/crypto/cnxk/cn10k_ipsec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2021 Marvell.
   3  */
   4
   5 #include <cryptodev_pmd.h>
   6 #include <rte_esp.h>
   7 #include <rte_ip.h>
   8 #include <rte_malloc.h>
   9 #include <rte_security.h>
  10 #include <rte_security_driver.h>
  11 #include <rte_udp.h>
  12
  13 #include "cn10k_ipsec.h"
  14 #include "cnxk_cryptodev.h"
  15 #include "cnxk_cryptodev_ops.h"
  16 #include "cnxk_ipsec.h"
  17 #include "cnxk_security.h"
  18
  19 #include "roc_api.h"
  20
  21 static uint64_t
  22 ipsec_cpt_inst_w7_get(struct roc_cpt *roc_cpt, void *sa)
  23 {
  24         union cpt_inst_w7 w7;
  25
  26         w7.u64 = 0;
  27         w7.s.egrp = roc_cpt->eng_grp[CPT_ENG_TYPE_IE];
  28         w7.s.ctx_val = 1;
  29         w7.s.cptr = (uint64_t)sa;
  30         rte_mb();
  31
  32         return w7.u64;
  33 }
  34
  35 static int
  36 cn10k_ipsec_outb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
  37                            struct rte_security_ipsec_xform *ipsec_xfrm,
  38                            struct rte_crypto_sym_xform *crypto_xfrm,
  39                            struct rte_security_session *sec_sess)
  40 {
  41         union roc_ot_ipsec_outb_param1 param1;
  42         struct roc_ot_ipsec_outb_sa *sa_dptr;
  43         struct cnxk_ipsec_outb_rlens rlens;
  44         struct cn10k_sec_session *sess;
  45         struct cn10k_ipsec_sa *sa;
  46         union cpt_inst_w4 inst_w4;
  47         void *out_sa;
  48         int ret = 0;
  49
  50         sess = get_sec_session_private_data(sec_sess);
  51         sa = &sess->sa;
  52         out_sa = &sa->out_sa;
  53
  54         /* Allocate memory to be used as dptr for CPT ucode WRITE_SA op */
  55         sa_dptr = plt_zmalloc(ROC_NIX_INL_OT_IPSEC_OUTB_HW_SZ, 0);
  56         if (sa_dptr == NULL) {
  57                 plt_err("Couldn't allocate memory for SA dptr");
  58                 return -ENOMEM;
  59         }
  60
  61         memset(sa_dptr, 0, sizeof(struct roc_ot_ipsec_outb_sa));
  62
  63         /* Translate security parameters to SA */
  64         ret = cnxk_ot_ipsec_outb_sa_fill(sa_dptr, ipsec_xfrm, crypto_xfrm);
  65         if (ret) {
  66                 plt_err("Could not fill outbound session parameters");
  67                 goto sa_dptr_free;
  68         }
  69
  70         sa->inst.w7 = ipsec_cpt_inst_w7_get(roc_cpt, out_sa);
  71
  72 #ifdef LA_IPSEC_DEBUG
  73         /* Use IV from application in debug mode */
  74         if (ipsec_xfrm->options.iv_gen_disable == 1) {
  75                 sa_dptr->w2.s.iv_src = ROC_IE_OT_SA_IV_SRC_FROM_SA;
  76                 if (crypto_xfrm->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
  77                         sa->iv_offset = crypto_xfrm->aead.iv.offset;
  78                         sa->iv_length = crypto_xfrm->aead.iv.length;
  79                 } else {
  80                         sa->iv_offset = crypto_xfrm->cipher.iv.offset;
  81                         sa->iv_length = crypto_xfrm->cipher.iv.length;
  82                 }
  83         }
  84 #else
  85         if (ipsec_xfrm->options.iv_gen_disable != 0) {
  86                 plt_err("Application provided IV not supported");
  87                 ret = -ENOTSUP;
  88                 goto sa_dptr_free;
  89         }
  90 #endif
  91
  92         sa->is_outbound = true;
  93
  94         /* Get Rlen calculation data */
  95         ret = cnxk_ipsec_outb_rlens_get(&rlens, ipsec_xfrm, crypto_xfrm);
  96         if (ret)
  97                 goto sa_dptr_free;
  98
  99         sa->max_extended_len = rlens.max_extended_len;
 100
 101         /* pre-populate CPT INST word 4 */
 102         inst_w4.u64 = 0;
 103         inst_w4.s.opcode_major = ROC_IE_OT_MAJOR_OP_PROCESS_OUTBOUND_IPSEC;
 104
 105         param1.u16 = 0;
 106
 107         /* Disable IP checksum computation by default */
 108         param1.s.ip_csum_disable = ROC_IE_OT_SA_INNER_PKT_IP_CSUM_DISABLE;
 109
 110         if (ipsec_xfrm->options.ip_csum_enable) {
 111                 param1.s.ip_csum_disable =
 112                         ROC_IE_OT_SA_INNER_PKT_IP_CSUM_ENABLE;
 113         }
 114
 115         /* Disable L4 checksum computation by default */
 116         param1.s.l4_csum_disable = ROC_IE_OT_SA_INNER_PKT_L4_CSUM_DISABLE;
 117
 118         if (ipsec_xfrm->options.l4_csum_enable) {
 119                 param1.s.l4_csum_disable =
 120                         ROC_IE_OT_SA_INNER_PKT_L4_CSUM_ENABLE;
 121         }
 122
 123         inst_w4.s.param1 = param1.u16;
 124
 125         sa->inst.w4 = inst_w4.u64;
 126
 127         memset(out_sa, 0, sizeof(struct roc_ot_ipsec_outb_sa));
 128
 129         /* Copy word0 from sa_dptr to populate ctx_push_sz ctx_size fields */
 130         memcpy(out_sa, sa_dptr, 8);
 131
 132         plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
 133
 134         /* Write session using microcode opcode */
 135         ret = roc_cpt_ctx_write(lf, sa_dptr, out_sa,
 136                                 ROC_NIX_INL_OT_IPSEC_OUTB_HW_SZ);
 137         if (ret) {
 138                 plt_err("Could not write outbound session to hardware");
 139                 goto sa_dptr_free;
 140         }
 141
 142         /* Trigger CTX flush so that data is written back to DRAM */
 143         roc_cpt_lf_ctx_flush(lf, out_sa, false);
 144
 145         plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
 146
 147 sa_dptr_free:
 148         plt_free(sa_dptr);
 149
 150         return ret;
 151 }
 152
 153 static int
 154 cn10k_ipsec_inb_sa_create(struct roc_cpt *roc_cpt, struct roc_cpt_lf *lf,
 155                           struct rte_security_ipsec_xform *ipsec_xfrm,
 156                           struct rte_crypto_sym_xform *crypto_xfrm,
 157                           struct rte_security_session *sec_sess)
 158 {
 159         union roc_ot_ipsec_inb_param1 param1;
 160         struct roc_ot_ipsec_inb_sa *sa_dptr;
 161         struct cn10k_sec_session *sess;
 162         struct cn10k_ipsec_sa *sa;
 163         union cpt_inst_w4 inst_w4;
 164         void *in_sa;
 165         int ret = 0;
 166
 167         sess = get_sec_session_private_data(sec_sess);
 168         sa = &sess->sa;
 169         in_sa = &sa->in_sa;
 170
 171         /* Allocate memory to be used as dptr for CPT ucode WRITE_SA op */
 172         sa_dptr = plt_zmalloc(ROC_NIX_INL_OT_IPSEC_INB_HW_SZ, 0);
 173         if (sa_dptr == NULL) {
 174                 plt_err("Couldn't allocate memory for SA dptr");
 175                 return -ENOMEM;
 176         }
 177
 178         memset(sa_dptr, 0, sizeof(struct roc_ot_ipsec_inb_sa));
 179
 180         /* Translate security parameters to SA */
 181         ret = cnxk_ot_ipsec_inb_sa_fill(sa_dptr, ipsec_xfrm, crypto_xfrm);
 182         if (ret) {
 183                 plt_err("Could not fill inbound session parameters");
 184                 goto sa_dptr_free;
 185         }
 186
 187         sa->is_outbound = false;
 188         sa->inst.w7 = ipsec_cpt_inst_w7_get(roc_cpt, in_sa);
 189
 190         /* pre-populate CPT INST word 4 */
 191         inst_w4.u64 = 0;
 192         inst_w4.s.opcode_major = ROC_IE_OT_MAJOR_OP_PROCESS_INBOUND_IPSEC;
 193
 194         param1.u16 = 0;
 195
 196         /* Disable IP checksum verification by default */
 197         param1.s.ip_csum_disable = ROC_IE_OT_SA_INNER_PKT_IP_CSUM_DISABLE;
 198
 199         if (ipsec_xfrm->options.ip_csum_enable) {
 200                 param1.s.ip_csum_disable =
 201                         ROC_IE_OT_SA_INNER_PKT_IP_CSUM_ENABLE;
 202                 sa->ip_csum_enable = true;
 203         }
 204
 205         /* Disable L4 checksum verification by default */
 206         param1.s.l4_csum_disable = ROC_IE_OT_SA_INNER_PKT_L4_CSUM_DISABLE;
 207
 208         if (ipsec_xfrm->options.l4_csum_enable) {
 209                 param1.s.l4_csum_disable =
 210                         ROC_IE_OT_SA_INNER_PKT_L4_CSUM_ENABLE;
 211         }
 212
 213         param1.s.esp_trailer_disable = 1;
 214
 215         inst_w4.s.param1 = param1.u16;
 216
 217         sa->inst.w4 = inst_w4.u64;
 218
 219         memset(in_sa, 0, sizeof(struct roc_ot_ipsec_inb_sa));
 220
 221         /* Copy word0 from sa_dptr to populate ctx_push_sz ctx_size fields */
 222         memcpy(in_sa, sa_dptr, 8);
 223
 224         plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
 225
 226         /* Write session using microcode opcode */
 227         ret = roc_cpt_ctx_write(lf, sa_dptr, in_sa,
 228                                 ROC_NIX_INL_OT_IPSEC_INB_HW_SZ);
 229         if (ret) {
 230                 plt_err("Could not write inbound session to hardware");
 231                 goto sa_dptr_free;
 232         }
 233
 234         /* Trigger CTX flush so that data is written back to DRAM */
 235         roc_cpt_lf_ctx_flush(lf, in_sa, false);
 236
 237         plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
 238
 239 sa_dptr_free:
 240         plt_free(sa_dptr);
 241
 242         return ret;
 243 }
 244
 245 static int
 246 cn10k_ipsec_session_create(void *dev,
 247                            struct rte_security_ipsec_xform *ipsec_xfrm,
 248                            struct rte_crypto_sym_xform *crypto_xfrm,
 249                            struct rte_security_session *sess)
 250 {
 251         struct rte_cryptodev *crypto_dev = dev;
 252         struct roc_cpt *roc_cpt;
 253         struct cnxk_cpt_vf *vf;
 254         struct cnxk_cpt_qp *qp;
 255         int ret;
 256
 257         qp = crypto_dev->data->queue_pairs[0];
 258         if (qp == NULL) {
 259                 plt_err("Setup cpt queue pair before creating security session");
 260                 return -EPERM;
 261         }
 262
 263         ret = cnxk_ipsec_xform_verify(ipsec_xfrm, crypto_xfrm);
 264         if (ret)
 265                 return ret;
 266
 267         vf = crypto_dev->data->dev_private;
 268         roc_cpt = &vf->cpt;
 269
 270         if (ipsec_xfrm->direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS)
 271                 return cn10k_ipsec_inb_sa_create(roc_cpt, &qp->lf, ipsec_xfrm,
 272                                                  crypto_xfrm, sess);
 273         else
 274                 return cn10k_ipsec_outb_sa_create(roc_cpt, &qp->lf, ipsec_xfrm,
 275                                                   crypto_xfrm, sess);
 276 }
 277
 278 static int
 279 cn10k_sec_session_create(void *device, struct rte_security_session_conf *conf,
 280                          struct rte_security_session *sess,
 281                          struct rte_mempool *mempool)
 282 {
 283         struct cn10k_sec_session *priv;
 284         int ret;
 285
 286         if (conf->action_type != RTE_SECURITY_ACTION_TYPE_LOOKASIDE_PROTOCOL)
 287                 return -EINVAL;
 288
 289         if (rte_mempool_get(mempool, (void **)&priv)) {
 290                 plt_err("Could not allocate security session private data");
 291                 return -ENOMEM;
 292         }
 293
 294         set_sec_session_private_data(sess, priv);
 295
 296         if (conf->protocol != RTE_SECURITY_PROTOCOL_IPSEC) {
 297                 ret = -ENOTSUP;
 298                 goto mempool_put;
 299         }
 300         ret = cn10k_ipsec_session_create(device, &conf->ipsec,
 301                                          conf->crypto_xform, sess);
 302         if (ret)
 303                 goto mempool_put;
 304
 305         return 0;
 306
 307 mempool_put:
 308         rte_mempool_put(mempool, priv);
 309         set_sec_session_private_data(sess, NULL);
 310         return ret;
 311 }
 312
 313 static int
 314 cn10k_sec_session_destroy(void *dev, struct rte_security_session *sec_sess)
 315 {
 316         struct rte_cryptodev *crypto_dev = dev;
 317         union roc_ot_ipsec_sa_word2 *w2;
 318         struct cn10k_sec_session *sess;
 319         struct rte_mempool *sess_mp;
 320         struct cn10k_ipsec_sa *sa;
 321         struct cnxk_cpt_qp *qp;
 322         struct roc_cpt_lf *lf;
 323
 324         sess = get_sec_session_private_data(sec_sess);
 325         if (sess == NULL)
 326                 return 0;
 327
 328         qp = crypto_dev->data->queue_pairs[0];
 329         if (qp == NULL)
 330                 return 0;
 331
 332         lf = &qp->lf;
 333
 334         sa = &sess->sa;
 335
 336         /* Trigger CTX flush to write dirty data back to DRAM */
 337         roc_cpt_lf_ctx_flush(lf, &sa->in_sa, false);
 338
 339         /* Wait for 1 ms so that flush is complete */
 340         rte_delay_ms(1);
 341
 342         w2 = (union roc_ot_ipsec_sa_word2 *)&sa->in_sa.w2;
 343         w2->s.valid = 0;
 344
 345         plt_atomic_thread_fence(__ATOMIC_SEQ_CST);
 346
 347         /* Trigger CTX reload to fetch new data from DRAM */
 348         roc_cpt_lf_ctx_reload(lf, &sa->in_sa);
 349
 350         sess_mp = rte_mempool_from_obj(sess);
 351
 352         set_sec_session_private_data(sec_sess, NULL);
 353         rte_mempool_put(sess_mp, sess);
 354
 355         return 0;
 356 }
 357
 358 static unsigned int
 359 cn10k_sec_session_get_size(void *device __rte_unused)
 360 {
 361         return sizeof(struct cn10k_sec_session);
 362 }
 363
 364 /* Update platform specific security ops */
 365 void
 366 cn10k_sec_ops_override(void)
 367 {
 368         /* Update platform specific ops */
 369         cnxk_sec_ops.session_create = cn10k_sec_session_create;
 370         cnxk_sec_ops.session_destroy = cn10k_sec_session_destroy;
 371         cnxk_sec_ops.session_get_size = cn10k_sec_session_get_size;
 372 }