drivers/net/octeontx2/otx2_ethdev_sec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (C) 2020 Marvell International Ltd.
   3  */
   4
   5 #include <rte_cryptodev.h>
   6 #include <rte_esp.h>
   7 #include <rte_ethdev.h>
   8 #include <rte_eventdev.h>
   9 #include <rte_ip.h>
  10 #include <rte_malloc.h>
  11 #include <rte_memzone.h>
  12 #include <rte_security.h>
  13 #include <rte_security_driver.h>
  14 #include <rte_udp.h>
  15
  16 #include "otx2_common.h"
  17 #include "otx2_cryptodev_qp.h"
  18 #include "otx2_ethdev.h"
  19 #include "otx2_ethdev_sec.h"
  20 #include "otx2_ipsec_fp.h"
  21 #include "otx2_sec_idev.h"
  22 #include "otx2_security.h"
  23
  24 #define ERR_STR_SZ 256
  25
  26 struct eth_sec_tag_const {
  27         RTE_STD_C11
  28         union {
  29                 struct {
  30                         uint32_t rsvd_11_0  : 12;
  31                         uint32_t port       : 8;
  32                         uint32_t event_type : 4;
  33                         uint32_t rsvd_31_24 : 8;
  34                 };
  35                 uint32_t u32;
  36         };
  37 };
  38
  39 static struct rte_cryptodev_capabilities otx2_eth_sec_crypto_caps[] = {
  40         {       /* AES GCM */
  41                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  42                 {.sym = {
  43                         .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
  44                         {.aead = {
  45                                 .algo = RTE_CRYPTO_AEAD_AES_GCM,
  46                                 .block_size = 16,
  47                                 .key_size = {
  48                                         .min = 16,
  49                                         .max = 32,
  50                                         .increment = 8
  51                                 },
  52                                 .digest_size = {
  53                                         .min = 16,
  54                                         .max = 16,
  55                                         .increment = 0
  56                                 },
  57                                 .aad_size = {
  58                                         .min = 8,
  59                                         .max = 12,
  60                                         .increment = 4
  61                                 },
  62                                 .iv_size = {
  63                                         .min = 12,
  64                                         .max = 12,
  65                                         .increment = 0
  66                                 }
  67                         }, }
  68                 }, }
  69         },
  70         {       /* AES CBC */
  71                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  72                 {.sym = {
  73                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
  74                         {.cipher = {
  75                                 .algo = RTE_CRYPTO_CIPHER_AES_CBC,
  76                                 .block_size = 16,
  77                                 .key_size = {
  78                                         .min = 16,
  79                                         .max = 32,
  80                                         .increment = 8
  81                                 },
  82                                 .iv_size = {
  83                                         .min = 16,
  84                                         .max = 16,
  85                                         .increment = 0
  86                                 }
  87                         }, }
  88                 }, }
  89         },
  90         {       /* SHA1 HMAC */
  91                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  92                 {.sym = {
  93                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  94                         {.auth = {
  95                                 .algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
  96                                 .block_size = 64,
  97                                 .key_size = {
  98                                         .min = 20,
  99                                         .max = 64,
 100                                         .increment = 1
 101                                 },
 102                                 .digest_size = {
 103                                         .min = 12,
 104                                         .max = 12,
 105                                         .increment = 0
 106                                 },
 107                         }, }
 108                 }, }
 109         },
 110         RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 111 };
 112
 113 static const struct rte_security_capability otx2_eth_sec_capabilities[] = {
 114         {       /* IPsec Inline Protocol ESP Tunnel Ingress */
 115                 .action = RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL,
 116                 .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 117                 .ipsec = {
 118                         .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 119                         .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
 120                         .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
 121                         .options = { 0 }
 122                 },
 123                 .crypto_capabilities = otx2_eth_sec_crypto_caps,
 124                 .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
 125         },
 126         {       /* IPsec Inline Protocol ESP Tunnel Egress */
 127                 .action = RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL,
 128                 .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 129                 .ipsec = {
 130                         .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 131                         .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
 132                         .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
 133                         .options = { 0 }
 134                 },
 135                 .crypto_capabilities = otx2_eth_sec_crypto_caps,
 136                 .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
 137         },
 138         {
 139                 .action = RTE_SECURITY_ACTION_TYPE_NONE
 140         }
 141 };
 142
 143 static void
 144 lookup_mem_sa_tbl_clear(struct rte_eth_dev *eth_dev)
 145 {
 146         static const char name[] = OTX2_NIX_FASTPATH_LOOKUP_MEM;
 147         uint16_t port = eth_dev->data->port_id;
 148         const struct rte_memzone *mz;
 149         uint64_t **sa_tbl;
 150         uint8_t *mem;
 151
 152         mz = rte_memzone_lookup(name);
 153         if (mz == NULL)
 154                 return;
 155
 156         mem = mz->addr;
 157
 158         sa_tbl  = (uint64_t **)RTE_PTR_ADD(mem, OTX2_NIX_SA_TBL_START);
 159         if (sa_tbl[port] == NULL)
 160                 return;
 161
 162         rte_free(sa_tbl[port]);
 163         sa_tbl[port] = NULL;
 164 }
 165
 166 static int
 167 lookup_mem_sa_index_update(struct rte_eth_dev *eth_dev, int spi, void *sa,
 168                            char *err_str)
 169 {
 170         static const char name[] = OTX2_NIX_FASTPATH_LOOKUP_MEM;
 171         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
 172         uint16_t port = eth_dev->data->port_id;
 173         const struct rte_memzone *mz;
 174         uint64_t **sa_tbl;
 175         uint8_t *mem;
 176
 177         mz = rte_memzone_lookup(name);
 178         if (mz == NULL) {
 179                 snprintf(err_str, ERR_STR_SZ,
 180                          "Could not find fastpath lookup table");
 181                 return -EINVAL;
 182         }
 183
 184         mem = mz->addr;
 185
 186         sa_tbl = (uint64_t **)RTE_PTR_ADD(mem, OTX2_NIX_SA_TBL_START);
 187
 188         if (sa_tbl[port] == NULL) {
 189                 sa_tbl[port] = rte_malloc(NULL, dev->ipsec_in_max_spi *
 190                                           sizeof(uint64_t), 0);
 191         }
 192
 193         sa_tbl[port][spi] = (uint64_t)sa;
 194
 195         return 0;
 196 }
 197
 198 static inline void
 199 in_sa_mz_name_get(char *name, int size, uint16_t port)
 200 {
 201         snprintf(name, size, "otx2_ipsec_in_sadb_%u", port);
 202 }
 203
 204 static struct otx2_ipsec_fp_in_sa *
 205 in_sa_get(uint16_t port, int sa_index)
 206 {
 207         char name[RTE_MEMZONE_NAMESIZE];
 208         struct otx2_ipsec_fp_in_sa *sa;
 209         const struct rte_memzone *mz;
 210
 211         in_sa_mz_name_get(name, RTE_MEMZONE_NAMESIZE, port);
 212         mz = rte_memzone_lookup(name);
 213         if (mz == NULL) {
 214                 otx2_err("Could not get the memzone reserved for IN SA DB");
 215                 return NULL;
 216         }
 217
 218         sa = mz->addr;
 219
 220         return sa + sa_index;
 221 }
 222
 223 static int
 224 ipsec_sa_const_set(struct rte_security_ipsec_xform *ipsec,
 225                    struct rte_crypto_sym_xform *xform,
 226                    struct otx2_sec_session_ipsec_ip *sess)
 227 {
 228         struct rte_crypto_sym_xform *cipher_xform, *auth_xform;
 229
 230         sess->partial_len = sizeof(struct rte_ipv4_hdr);
 231
 232         if (ipsec->proto == RTE_SECURITY_IPSEC_SA_PROTO_ESP) {
 233                 sess->partial_len += sizeof(struct rte_esp_hdr);
 234                 sess->roundup_len = sizeof(struct rte_esp_tail);
 235         } else if (ipsec->proto == RTE_SECURITY_IPSEC_SA_PROTO_AH) {
 236                 sess->partial_len += OTX2_SEC_AH_HDR_LEN;
 237         } else {
 238                 return -EINVAL;
 239         }
 240
 241         if (ipsec->options.udp_encap)
 242                 sess->partial_len += sizeof(struct rte_udp_hdr);
 243
 244         if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 245                 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) {
 246                         sess->partial_len += OTX2_SEC_AES_GCM_IV_LEN;
 247                         sess->partial_len += OTX2_SEC_AES_GCM_MAC_LEN;
 248                         sess->roundup_byte = OTX2_SEC_AES_GCM_ROUNDUP_BYTE_LEN;
 249                 }
 250                 return 0;
 251         }
 252
 253         if (ipsec->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
 254                 cipher_xform = xform;
 255                 auth_xform = xform->next;
 256         } else if (ipsec->direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
 257                 auth_xform = xform;
 258                 cipher_xform = xform->next;
 259         } else {
 260                 return -EINVAL;
 261         }
 262         if (cipher_xform->cipher.algo == RTE_CRYPTO_CIPHER_AES_CBC) {
 263                 sess->partial_len += OTX2_SEC_AES_CBC_IV_LEN;
 264                 sess->roundup_byte = OTX2_SEC_AES_CBC_ROUNDUP_BYTE_LEN;
 265         } else {
 266                 return -EINVAL;
 267         }
 268
 269         if (auth_xform->auth.algo == RTE_CRYPTO_AUTH_SHA1_HMAC)
 270                 sess->partial_len += OTX2_SEC_SHA1_HMAC_LEN;
 271         else
 272                 return -EINVAL;
 273
 274         return 0;
 275 }
 276
 277 static int
 278 hmac_init(struct otx2_ipsec_fp_sa_ctl *ctl, struct otx2_cpt_qp *qp,
 279           const uint8_t *auth_key, int len, uint8_t *hmac_key)
 280 {
 281         struct inst_data {
 282                 struct otx2_cpt_res cpt_res;
 283                 uint8_t buffer[64];
 284         } *md;
 285
 286         volatile struct otx2_cpt_res *res;
 287         uint64_t timeout, lmt_status;
 288         struct otx2_cpt_inst_s inst;
 289         rte_iova_t md_iova;
 290         int ret;
 291
 292         memset(&inst, 0, sizeof(struct otx2_cpt_inst_s));
 293
 294         md = rte_zmalloc(NULL, sizeof(struct inst_data), OTX2_CPT_RES_ALIGN);
 295         if (md == NULL)
 296                 return -ENOMEM;
 297
 298         memcpy(md->buffer, auth_key, len);
 299
 300         md_iova = rte_malloc_virt2iova(md);
 301         if (md_iova == RTE_BAD_IOVA) {
 302                 ret = -EINVAL;
 303                 goto free_md;
 304         }
 305
 306         inst.res_addr = md_iova + offsetof(struct inst_data, cpt_res);
 307         inst.opcode = OTX2_CPT_OP_WRITE_HMAC_IPAD_OPAD;
 308         inst.param2 = ctl->auth_type;
 309         inst.dlen = len;
 310         inst.dptr = md_iova + offsetof(struct inst_data, buffer);
 311         inst.rptr = inst.dptr;
 312         inst.egrp = OTX2_CPT_EGRP_INLINE_IPSEC;
 313
 314         md->cpt_res.compcode = 0;
 315         md->cpt_res.uc_compcode = 0xff;
 316
 317         timeout = rte_get_timer_cycles() + 5 * rte_get_timer_hz();
 318
 319         rte_io_wmb();
 320
 321         do {
 322                 otx2_lmt_mov(qp->lmtline, &inst, 2);
 323                 lmt_status = otx2_lmt_submit(qp->lf_nq_reg);
 324         } while (lmt_status == 0);
 325
 326         res = (volatile struct otx2_cpt_res *)&md->cpt_res;
 327
 328         /* Wait until instruction completes or times out */
 329         while (res->uc_compcode == 0xff) {
 330                 if (rte_get_timer_cycles() > timeout)
 331                         break;
 332         }
 333
 334         if (res->u16[0] != OTX2_SEC_COMP_GOOD) {
 335                 ret = -EIO;
 336                 goto free_md;
 337         }
 338
 339         /* Retrieve the ipad and opad from rptr */
 340         memcpy(hmac_key, md->buffer, 48);
 341
 342         ret = 0;
 343
 344 free_md:
 345         rte_free(md);
 346         return ret;
 347 }
 348
 349 static int
 350 eth_sec_ipsec_out_sess_create(struct rte_eth_dev *eth_dev,
 351                               struct rte_security_ipsec_xform *ipsec,
 352                               struct rte_crypto_sym_xform *crypto_xform,
 353                               struct rte_security_session *sec_sess)
 354 {
 355         struct rte_crypto_sym_xform *auth_xform, *cipher_xform;
 356         struct otx2_sec_session_ipsec_ip *sess;
 357         uint16_t port = eth_dev->data->port_id;
 358         int cipher_key_len, auth_key_len, ret;
 359         const uint8_t *cipher_key, *auth_key;
 360         struct otx2_ipsec_fp_sa_ctl *ctl;
 361         struct otx2_ipsec_fp_out_sa *sa;
 362         struct otx2_sec_session *priv;
 363         struct otx2_cpt_inst_s inst;
 364         struct otx2_cpt_qp *qp;
 365
 366         priv = get_sec_session_private_data(sec_sess);
 367         priv->ipsec.dir = RTE_SECURITY_IPSEC_SA_DIR_EGRESS;
 368         sess = &priv->ipsec.ip;
 369
 370         sa = &sess->out_sa;
 371         ctl = &sa->ctl;
 372         if (ctl->valid) {
 373                 otx2_err("SA already registered");
 374                 return -EINVAL;
 375         }
 376
 377         memset(sess, 0, sizeof(struct otx2_sec_session_ipsec_ip));
 378
 379         sess->seq = 1;
 380
 381         ret = ipsec_sa_const_set(ipsec, crypto_xform, sess);
 382         if (ret < 0)
 383                 return ret;
 384
 385         if (crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD)
 386                 memcpy(sa->nonce, &ipsec->salt, 4);
 387
 388         if (ipsec->options.udp_encap == 1) {
 389                 sa->udp_src = 4500;
 390                 sa->udp_dst = 4500;
 391         }
 392
 393         if (ipsec->mode == RTE_SECURITY_IPSEC_SA_MODE_TUNNEL) {
 394                 /* Start ip id from 1 */
 395                 sess->ip_id = 1;
 396
 397                 if (ipsec->tunnel.type == RTE_SECURITY_IPSEC_TUNNEL_IPV4) {
 398                         memcpy(&sa->ip_src, &ipsec->tunnel.ipv4.src_ip,
 399                                sizeof(struct in_addr));
 400                         memcpy(&sa->ip_dst, &ipsec->tunnel.ipv4.dst_ip,
 401                                sizeof(struct in_addr));
 402                 } else {
 403                         return -EINVAL;
 404                 }
 405         } else {
 406                 return -EINVAL;
 407         }
 408
 409         cipher_xform = crypto_xform;
 410         auth_xform = crypto_xform->next;
 411
 412         cipher_key_len = 0;
 413         auth_key_len = 0;
 414         auth_key = NULL;
 415
 416         if (crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 417                 cipher_key = crypto_xform->aead.key.data;
 418                 cipher_key_len = crypto_xform->aead.key.length;
 419         } else {
 420                 cipher_key = cipher_xform->cipher.key.data;
 421                 cipher_key_len = cipher_xform->cipher.key.length;
 422                 auth_key = auth_xform->auth.key.data;
 423                 auth_key_len = auth_xform->auth.key.length;
 424         }
 425
 426         if (cipher_key_len != 0)
 427                 memcpy(sa->cipher_key, cipher_key, cipher_key_len);
 428         else
 429                 return -EINVAL;
 430
 431         /* Determine word 7 of CPT instruction */
 432         inst.u64[7] = 0;
 433         inst.egrp = OTX2_CPT_EGRP_INLINE_IPSEC;
 434         inst.cptr = rte_mempool_virt2iova(sa);
 435         sess->inst_w7 = inst.u64[7];
 436
 437         /* Get CPT QP to be used for this SA */
 438         ret = otx2_sec_idev_tx_cpt_qp_get(port, &qp);
 439         if (ret)
 440                 return ret;
 441
 442         sess->qp = qp;
 443
 444         sess->cpt_lmtline = qp->lmtline;
 445         sess->cpt_nq_reg = qp->lf_nq_reg;
 446
 447         /* Populate control word */
 448         ret = ipsec_fp_sa_ctl_set(ipsec, crypto_xform, ctl);
 449         if (ret)
 450                 goto cpt_put;
 451
 452         if (auth_key_len && auth_key) {
 453                 ret = hmac_init(ctl, qp, auth_key, auth_key_len, sa->hmac_key);
 454                 if (ret)
 455                         goto cpt_put;
 456         }
 457
 458         return 0;
 459 cpt_put:
 460         otx2_sec_idev_tx_cpt_qp_put(sess->qp);
 461         return ret;
 462 }
 463
 464 static int
 465 eth_sec_ipsec_in_sess_create(struct rte_eth_dev *eth_dev,
 466                              struct rte_security_ipsec_xform *ipsec,
 467                              struct rte_crypto_sym_xform *crypto_xform,
 468                              struct rte_security_session *sec_sess)
 469 {
 470         struct rte_crypto_sym_xform *auth_xform, *cipher_xform;
 471         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
 472         struct otx2_sec_session_ipsec_ip *sess;
 473         uint16_t port = eth_dev->data->port_id;
 474         int cipher_key_len, auth_key_len, ret;
 475         const uint8_t *cipher_key, *auth_key;
 476         struct otx2_ipsec_fp_sa_ctl *ctl;
 477         struct otx2_ipsec_fp_in_sa *sa;
 478         struct otx2_sec_session *priv;
 479         char err_str[ERR_STR_SZ];
 480         struct otx2_cpt_qp *qp;
 481
 482         memset(err_str, 0, ERR_STR_SZ);
 483
 484         if (ipsec->spi >= dev->ipsec_in_max_spi) {
 485                 otx2_err("SPI exceeds max supported");
 486                 return -EINVAL;
 487         }
 488
 489         sa = in_sa_get(port, ipsec->spi);
 490         if (sa == NULL)
 491                 return -ENOMEM;
 492
 493         ctl = &sa->ctl;
 494
 495         priv = get_sec_session_private_data(sec_sess);
 496         priv->ipsec.dir = RTE_SECURITY_IPSEC_SA_DIR_INGRESS;
 497         sess = &priv->ipsec.ip;
 498
 499         rte_spinlock_lock(&dev->ipsec_tbl_lock);
 500
 501         if (ctl->valid) {
 502                 snprintf(err_str, ERR_STR_SZ, "SA already registered");
 503                 ret = -EEXIST;
 504                 goto tbl_unlock;
 505         }
 506
 507         memset(sa, 0, sizeof(struct otx2_ipsec_fp_in_sa));
 508
 509         auth_xform = crypto_xform;
 510         cipher_xform = crypto_xform->next;
 511
 512         cipher_key_len = 0;
 513         auth_key_len = 0;
 514         auth_key = NULL;
 515
 516         if (crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 517                 if (crypto_xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM)
 518                         memcpy(sa->nonce, &ipsec->salt, 4);
 519                 cipher_key = crypto_xform->aead.key.data;
 520                 cipher_key_len = crypto_xform->aead.key.length;
 521         } else {
 522                 cipher_key = cipher_xform->cipher.key.data;
 523                 cipher_key_len = cipher_xform->cipher.key.length;
 524                 auth_key = auth_xform->auth.key.data;
 525                 auth_key_len = auth_xform->auth.key.length;
 526         }
 527
 528         if (cipher_key_len != 0) {
 529                 memcpy(sa->cipher_key, cipher_key, cipher_key_len);
 530         } else {
 531                 snprintf(err_str, ERR_STR_SZ, "Invalid cipher key len");
 532                 ret = -EINVAL;
 533                 goto sa_clear;
 534         }
 535
 536         sess->in_sa = sa;
 537
 538         sa->userdata = priv->userdata;
 539
 540         sa->replay_win_sz = ipsec->replay_win_sz;
 541
 542         if (lookup_mem_sa_index_update(eth_dev, ipsec->spi, sa, err_str)) {
 543                 ret = -EINVAL;
 544                 goto sa_clear;
 545         }
 546
 547         ret = ipsec_fp_sa_ctl_set(ipsec, crypto_xform, ctl);
 548         if (ret) {
 549                 snprintf(err_str, ERR_STR_SZ,
 550                         "Could not set SA CTL word (err: %d)", ret);
 551                 goto sa_clear;
 552         }
 553
 554         if (auth_key_len && auth_key) {
 555                 /* Get a queue pair for HMAC init */
 556                 ret = otx2_sec_idev_tx_cpt_qp_get(port, &qp);
 557                 if (ret) {
 558                         snprintf(err_str, ERR_STR_SZ, "Could not get CPT QP");
 559                         goto sa_clear;
 560                 }
 561
 562                 ret = hmac_init(ctl, qp, auth_key, auth_key_len, sa->hmac_key);
 563                 otx2_sec_idev_tx_cpt_qp_put(qp);
 564                 if (ret) {
 565                         snprintf(err_str, ERR_STR_SZ, "Could not put CPT QP");
 566                         goto sa_clear;
 567                 }
 568         }
 569
 570         if (sa->replay_win_sz) {
 571                 if (sa->replay_win_sz > OTX2_IPSEC_MAX_REPLAY_WIN_SZ) {
 572                         snprintf(err_str, ERR_STR_SZ,
 573                                  "Replay window size is not supported");
 574                         ret = -ENOTSUP;
 575                         goto sa_clear;
 576                 }
 577                 sa->replay = rte_zmalloc(NULL, sizeof(struct otx2_ipsec_replay),
 578                                 0);
 579                 if (sa->replay == NULL) {
 580                         snprintf(err_str, ERR_STR_SZ,
 581                                 "Could not allocate memory");
 582                         ret = -ENOMEM;
 583                         goto sa_clear;
 584                 }
 585
 586                 rte_spinlock_init(&sa->replay->lock);
 587                 /*
 588                  * Set window bottom to 1, base and top to size of
 589                  * window
 590                  */
 591                 sa->replay->winb = 1;
 592                 sa->replay->wint = sa->replay_win_sz;
 593                 sa->replay->base = sa->replay_win_sz;
 594                 sa->esn_low = 0;
 595                 sa->esn_hi = 0;
 596         }
 597
 598         rte_spinlock_unlock(&dev->ipsec_tbl_lock);
 599         return 0;
 600
 601 sa_clear:
 602         memset(sa, 0, sizeof(struct otx2_ipsec_fp_in_sa));
 603
 604 tbl_unlock:
 605         rte_spinlock_unlock(&dev->ipsec_tbl_lock);
 606
 607         otx2_err("%s", err_str);
 608
 609         return ret;
 610 }
 611
 612 static int
 613 eth_sec_ipsec_sess_create(struct rte_eth_dev *eth_dev,
 614                           struct rte_security_ipsec_xform *ipsec,
 615                           struct rte_crypto_sym_xform *crypto_xform,
 616                           struct rte_security_session *sess)
 617 {
 618         int ret;
 619
 620         ret = ipsec_fp_xform_verify(ipsec, crypto_xform);
 621         if (ret)
 622                 return ret;
 623
 624         if (ipsec->direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS)
 625                 return eth_sec_ipsec_in_sess_create(eth_dev, ipsec,
 626                                                     crypto_xform, sess);
 627         else
 628                 return eth_sec_ipsec_out_sess_create(eth_dev, ipsec,
 629                                                      crypto_xform, sess);
 630 }
 631
 632 static int
 633 otx2_eth_sec_session_create(void *device,
 634                             struct rte_security_session_conf *conf,
 635                             struct rte_security_session *sess,
 636                             struct rte_mempool *mempool)
 637 {
 638         struct otx2_sec_session *priv;
 639         int ret;
 640
 641         if (conf->action_type != RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
 642                 return -ENOTSUP;
 643
 644         if (rte_mempool_get(mempool, (void **)&priv)) {
 645                 otx2_err("Could not allocate security session private data");
 646                 return -ENOMEM;
 647         }
 648
 649         set_sec_session_private_data(sess, priv);
 650
 651         /*
 652          * Save userdata provided by the application. For ingress packets, this
 653          * could be used to identify the SA.
 654          */
 655         priv->userdata = conf->userdata;
 656
 657         if (conf->protocol == RTE_SECURITY_PROTOCOL_IPSEC)
 658                 ret = eth_sec_ipsec_sess_create(device, &conf->ipsec,
 659                                                 conf->crypto_xform,
 660                                                 sess);
 661         else
 662                 ret = -ENOTSUP;
 663
 664         if (ret)
 665                 goto mempool_put;
 666
 667         return 0;
 668
 669 mempool_put:
 670         rte_mempool_put(mempool, priv);
 671         set_sec_session_private_data(sess, NULL);
 672         return ret;
 673 }
 674
 675 static void
 676 otx2_eth_sec_free_anti_replay(struct otx2_ipsec_fp_in_sa *sa)
 677 {
 678         if (sa != NULL) {
 679                 if (sa->replay_win_sz && sa->replay)
 680                         rte_free(sa->replay);
 681         }
 682 }
 683
 684 static int
 685 otx2_eth_sec_session_destroy(void *device __rte_unused,
 686                              struct rte_security_session *sess)
 687 {
 688         struct otx2_sec_session_ipsec_ip *sess_ip;
 689         struct otx2_sec_session *priv;
 690         struct rte_mempool *sess_mp;
 691         int ret;
 692
 693         priv = get_sec_session_private_data(sess);
 694         if (priv == NULL)
 695                 return -EINVAL;
 696
 697         sess_ip = &priv->ipsec.ip;
 698
 699         /* Release the anti replay window */
 700         if (priv->ipsec.dir == RTE_SECURITY_IPSEC_SA_DIR_INGRESS)
 701                 otx2_eth_sec_free_anti_replay(sess_ip->in_sa);
 702
 703         /* Release CPT LF used for this session */
 704         if (sess_ip->qp != NULL) {
 705                 ret = otx2_sec_idev_tx_cpt_qp_put(sess_ip->qp);
 706                 if (ret)
 707                         return ret;
 708         }
 709
 710         sess_mp = rte_mempool_from_obj(priv);
 711
 712         set_sec_session_private_data(sess, NULL);
 713         rte_mempool_put(sess_mp, priv);
 714
 715         return 0;
 716 }
 717
 718 static unsigned int
 719 otx2_eth_sec_session_get_size(void *device __rte_unused)
 720 {
 721         return sizeof(struct otx2_sec_session);
 722 }
 723
 724 static int
 725 otx2_eth_sec_set_pkt_mdata(void *device __rte_unused,
 726                             struct rte_security_session *session,
 727                             struct rte_mbuf *m, void *params __rte_unused)
 728 {
 729         /* Set security session as the pkt metadata */
 730         *rte_security_dynfield(m) = (rte_security_dynfield_t)session;
 731
 732         return 0;
 733 }
 734
 735 static int
 736 otx2_eth_sec_get_userdata(void *device __rte_unused, uint64_t md,
 737                            void **userdata)
 738 {
 739         /* Retrieve userdata  */
 740         *userdata = (void *)md;
 741
 742         return 0;
 743 }
 744
 745 static const struct rte_security_capability *
 746 otx2_eth_sec_capabilities_get(void *device __rte_unused)
 747 {
 748         return otx2_eth_sec_capabilities;
 749 }
 750
 751 static struct rte_security_ops otx2_eth_sec_ops = {
 752         .session_create         = otx2_eth_sec_session_create,
 753         .session_destroy        = otx2_eth_sec_session_destroy,
 754         .session_get_size       = otx2_eth_sec_session_get_size,
 755         .set_pkt_metadata       = otx2_eth_sec_set_pkt_mdata,
 756         .get_userdata           = otx2_eth_sec_get_userdata,
 757         .capabilities_get       = otx2_eth_sec_capabilities_get
 758 };
 759
 760 int
 761 otx2_eth_sec_ctx_create(struct rte_eth_dev *eth_dev)
 762 {
 763         struct rte_security_ctx *ctx;
 764         int ret;
 765
 766         ctx = rte_malloc("otx2_eth_sec_ctx",
 767                          sizeof(struct rte_security_ctx), 0);
 768         if (ctx == NULL)
 769                 return -ENOMEM;
 770
 771         ret = otx2_sec_idev_cfg_init(eth_dev->data->port_id);
 772         if (ret) {
 773                 rte_free(ctx);
 774                 return ret;
 775         }
 776
 777         /* Populate ctx */
 778
 779         ctx->device = eth_dev;
 780         ctx->ops = &otx2_eth_sec_ops;
 781         ctx->sess_cnt = 0;
 782
 783         eth_dev->security_ctx = ctx;
 784
 785         return 0;
 786 }
 787
 788 void
 789 otx2_eth_sec_ctx_destroy(struct rte_eth_dev *eth_dev)
 790 {
 791         rte_free(eth_dev->security_ctx);
 792 }
 793
 794 static int
 795 eth_sec_ipsec_cfg(struct rte_eth_dev *eth_dev, uint8_t tt)
 796 {
 797         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
 798         uint16_t port = eth_dev->data->port_id;
 799         struct nix_inline_ipsec_lf_cfg *req;
 800         struct otx2_mbox *mbox = dev->mbox;
 801         struct eth_sec_tag_const tag_const;
 802         char name[RTE_MEMZONE_NAMESIZE];
 803         const struct rte_memzone *mz;
 804
 805         in_sa_mz_name_get(name, RTE_MEMZONE_NAMESIZE, port);
 806         mz = rte_memzone_lookup(name);
 807         if (mz == NULL)
 808                 return -EINVAL;
 809
 810         req = otx2_mbox_alloc_msg_nix_inline_ipsec_lf_cfg(mbox);
 811         req->enable = 1;
 812         req->sa_base_addr = mz->iova;
 813
 814         req->ipsec_cfg0.tt = tt;
 815
 816         tag_const.u32 = 0;
 817         tag_const.event_type = RTE_EVENT_TYPE_ETHDEV;
 818         tag_const.port = port;
 819         req->ipsec_cfg0.tag_const = tag_const.u32;
 820
 821         req->ipsec_cfg0.sa_pow2_size =
 822                         rte_log2_u32(sizeof(struct otx2_ipsec_fp_in_sa));
 823         req->ipsec_cfg0.lenm1_max = NIX_MAX_FRS - 1;
 824
 825         req->ipsec_cfg1.sa_idx_w = rte_log2_u32(dev->ipsec_in_max_spi);
 826         req->ipsec_cfg1.sa_idx_max = dev->ipsec_in_max_spi - 1;
 827
 828         return otx2_mbox_process(mbox);
 829 }
 830
 831 int
 832 otx2_eth_sec_update_tag_type(struct rte_eth_dev *eth_dev)
 833 {
 834         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
 835         struct otx2_mbox *mbox = dev->mbox;
 836         struct nix_aq_enq_rsp *rsp;
 837         struct nix_aq_enq_req *aq;
 838         int ret;
 839
 840         aq = otx2_mbox_alloc_msg_nix_aq_enq(mbox);
 841         aq->qidx = 0; /* Read RQ:0 context */
 842         aq->ctype = NIX_AQ_CTYPE_RQ;
 843         aq->op = NIX_AQ_INSTOP_READ;
 844
 845         ret = otx2_mbox_process_msg(mbox, (void *)&rsp);
 846         if (ret < 0) {
 847                 otx2_err("Could not read RQ context");
 848                 return ret;
 849         }
 850
 851         /* Update tag type */
 852         ret = eth_sec_ipsec_cfg(eth_dev, rsp->rq.sso_tt);
 853         if (ret < 0)
 854                 otx2_err("Could not update sec eth tag type");
 855
 856         return ret;
 857 }
 858
 859 int
 860 otx2_eth_sec_init(struct rte_eth_dev *eth_dev)
 861 {
 862         const size_t sa_width = sizeof(struct otx2_ipsec_fp_in_sa);
 863         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
 864         uint16_t port = eth_dev->data->port_id;
 865         char name[RTE_MEMZONE_NAMESIZE];
 866         const struct rte_memzone *mz;
 867         int mz_sz, ret;
 868         uint16_t nb_sa;
 869
 870         RTE_BUILD_BUG_ON(sa_width < 32 || sa_width > 512 ||
 871                          !RTE_IS_POWER_OF_2(sa_width));
 872
 873         if (!(dev->tx_offloads & DEV_TX_OFFLOAD_SECURITY) &&
 874             !(dev->rx_offloads & DEV_RX_OFFLOAD_SECURITY))
 875                 return 0;
 876
 877         if (rte_security_dynfield_register() < 0)
 878                 return -rte_errno;
 879
 880         nb_sa = dev->ipsec_in_max_spi;
 881         mz_sz = nb_sa * sa_width;
 882         in_sa_mz_name_get(name, RTE_MEMZONE_NAMESIZE, port);
 883         mz = rte_memzone_reserve_aligned(name, mz_sz, rte_socket_id(),
 884                                          RTE_MEMZONE_IOVA_CONTIG, OTX2_ALIGN);
 885
 886         if (mz == NULL) {
 887                 otx2_err("Could not allocate inbound SA DB");
 888                 return -ENOMEM;
 889         }
 890
 891         memset(mz->addr, 0, mz_sz);
 892
 893         ret = eth_sec_ipsec_cfg(eth_dev, SSO_TT_ORDERED);
 894         if (ret < 0) {
 895                 otx2_err("Could not configure inline IPsec");
 896                 goto sec_fini;
 897         }
 898
 899         rte_spinlock_init(&dev->ipsec_tbl_lock);
 900
 901         return 0;
 902
 903 sec_fini:
 904         otx2_err("Could not configure device for security");
 905         otx2_eth_sec_fini(eth_dev);
 906         return ret;
 907 }
 908
 909 void
 910 otx2_eth_sec_fini(struct rte_eth_dev *eth_dev)
 911 {
 912         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
 913         uint16_t port = eth_dev->data->port_id;
 914         char name[RTE_MEMZONE_NAMESIZE];
 915
 916         if (!(dev->tx_offloads & DEV_TX_OFFLOAD_SECURITY) &&
 917             !(dev->rx_offloads & DEV_RX_OFFLOAD_SECURITY))
 918                 return;
 919
 920         lookup_mem_sa_tbl_clear(eth_dev);
 921
 922         in_sa_mz_name_get(name, RTE_MEMZONE_NAMESIZE, port);
 923         rte_memzone_free(rte_memzone_lookup(name));
 924 }