drivers/net/octeontx2/otx2_ethdev_sec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (C) 2020 Marvell International Ltd.
   3  */
   4
   5 #include <rte_cryptodev.h>
   6 #include <rte_esp.h>
   7 #include <rte_ethdev.h>
   8 #include <rte_eventdev.h>
   9 #include <rte_ip.h>
  10 #include <rte_malloc.h>
  11 #include <rte_memzone.h>
  12 #include <rte_security.h>
  13 #include <rte_security_driver.h>
  14 #include <rte_udp.h>
  15
  16 #include "otx2_common.h"
  17 #include "otx2_cryptodev_qp.h"
  18 #include "otx2_ethdev.h"
  19 #include "otx2_ethdev_sec.h"
  20 #include "otx2_ipsec_fp.h"
  21 #include "otx2_sec_idev.h"
  22 #include "otx2_security.h"
  23
  24 #define ERR_STR_SZ 256
  25
  26 struct eth_sec_tag_const {
  27         RTE_STD_C11
  28         union {
  29                 struct {
  30                         uint32_t rsvd_11_0  : 12;
  31                         uint32_t port       : 8;
  32                         uint32_t event_type : 4;
  33                         uint32_t rsvd_31_24 : 8;
  34                 };
  35                 uint32_t u32;
  36         };
  37 };
  38
  39 static struct rte_cryptodev_capabilities otx2_eth_sec_crypto_caps[] = {
  40         {       /* AES GCM */
  41                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  42                 {.sym = {
  43                         .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
  44                         {.aead = {
  45                                 .algo = RTE_CRYPTO_AEAD_AES_GCM,
  46                                 .block_size = 16,
  47                                 .key_size = {
  48                                         .min = 16,
  49                                         .max = 32,
  50                                         .increment = 8
  51                                 },
  52                                 .digest_size = {
  53                                         .min = 16,
  54                                         .max = 16,
  55                                         .increment = 0
  56                                 },
  57                                 .aad_size = {
  58                                         .min = 8,
  59                                         .max = 12,
  60                                         .increment = 4
  61                                 },
  62                                 .iv_size = {
  63                                         .min = 12,
  64                                         .max = 12,
  65                                         .increment = 0
  66                                 }
  67                         }, }
  68                 }, }
  69         },
  70         {       /* AES CBC */
  71                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  72                 {.sym = {
  73                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
  74                         {.cipher = {
  75                                 .algo = RTE_CRYPTO_CIPHER_AES_CBC,
  76                                 .block_size = 16,
  77                                 .key_size = {
  78                                         .min = 16,
  79                                         .max = 32,
  80                                         .increment = 8
  81                                 },
  82                                 .iv_size = {
  83                                         .min = 16,
  84                                         .max = 16,
  85                                         .increment = 0
  86                                 }
  87                         }, }
  88                 }, }
  89         },
  90         {       /* SHA1 HMAC */
  91                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  92                 {.sym = {
  93                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  94                         {.auth = {
  95                                 .algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
  96                                 .block_size = 64,
  97                                 .key_size = {
  98                                         .min = 20,
  99                                         .max = 64,
 100                                         .increment = 1
 101                                 },
 102                                 .digest_size = {
 103                                         .min = 12,
 104                                         .max = 12,
 105                                         .increment = 0
 106                                 },
 107                         }, }
 108                 }, }
 109         },
 110         RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 111 };
 112
 113 static const struct rte_security_capability otx2_eth_sec_capabilities[] = {
 114         {       /* IPsec Inline Protocol ESP Tunnel Ingress */
 115                 .action = RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL,
 116                 .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 117                 .ipsec = {
 118                         .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 119                         .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
 120                         .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
 121                         .options = { 0 }
 122                 },
 123                 .crypto_capabilities = otx2_eth_sec_crypto_caps,
 124                 .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
 125         },
 126         {       /* IPsec Inline Protocol ESP Tunnel Egress */
 127                 .action = RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL,
 128                 .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 129                 .ipsec = {
 130                         .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 131                         .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
 132                         .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
 133                         .options = { 0 }
 134                 },
 135                 .crypto_capabilities = otx2_eth_sec_crypto_caps,
 136                 .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
 137         },
 138         {
 139                 .action = RTE_SECURITY_ACTION_TYPE_NONE
 140         }
 141 };
 142
 143 static void
 144 lookup_mem_sa_tbl_clear(struct rte_eth_dev *eth_dev)
 145 {
 146         static const char name[] = OTX2_NIX_FASTPATH_LOOKUP_MEM;
 147         uint16_t port = eth_dev->data->port_id;
 148         const struct rte_memzone *mz;
 149         uint64_t **sa_tbl;
 150         uint8_t *mem;
 151
 152         mz = rte_memzone_lookup(name);
 153         if (mz == NULL)
 154                 return;
 155
 156         mem = mz->addr;
 157
 158         sa_tbl  = (uint64_t **)RTE_PTR_ADD(mem, OTX2_NIX_SA_TBL_START);
 159         if (sa_tbl[port] == NULL)
 160                 return;
 161
 162         rte_free(sa_tbl[port]);
 163         sa_tbl[port] = NULL;
 164 }
 165
 166 static int
 167 lookup_mem_sa_index_update(struct rte_eth_dev *eth_dev, int spi, void *sa,
 168                            char *err_str)
 169 {
 170         static const char name[] = OTX2_NIX_FASTPATH_LOOKUP_MEM;
 171         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
 172         uint16_t port = eth_dev->data->port_id;
 173         const struct rte_memzone *mz;
 174         uint64_t **sa_tbl;
 175         uint8_t *mem;
 176
 177         mz = rte_memzone_lookup(name);
 178         if (mz == NULL) {
 179                 snprintf(err_str, ERR_STR_SZ,
 180                          "Could not find fastpath lookup table");
 181                 return -EINVAL;
 182         }
 183
 184         mem = mz->addr;
 185
 186         sa_tbl = (uint64_t **)RTE_PTR_ADD(mem, OTX2_NIX_SA_TBL_START);
 187
 188         if (sa_tbl[port] == NULL) {
 189                 sa_tbl[port] = rte_malloc(NULL, dev->ipsec_in_max_spi *
 190                                           sizeof(uint64_t), 0);
 191         }
 192
 193         sa_tbl[port][spi] = (uint64_t)sa;
 194
 195         return 0;
 196 }
 197
 198 static inline void
 199 in_sa_mz_name_get(char *name, int size, uint16_t port)
 200 {
 201         snprintf(name, size, "otx2_ipsec_in_sadb_%u", port);
 202 }
 203
 204 static struct otx2_ipsec_fp_in_sa *
 205 in_sa_get(uint16_t port, int sa_index)
 206 {
 207         char name[RTE_MEMZONE_NAMESIZE];
 208         struct otx2_ipsec_fp_in_sa *sa;
 209         const struct rte_memzone *mz;
 210
 211         in_sa_mz_name_get(name, RTE_MEMZONE_NAMESIZE, port);
 212         mz = rte_memzone_lookup(name);
 213         if (mz == NULL) {
 214                 otx2_err("Could not get the memzone reserved for IN SA DB");
 215                 return NULL;
 216         }
 217
 218         sa = mz->addr;
 219
 220         return sa + sa_index;
 221 }
 222
 223 static int
 224 ipsec_sa_const_set(struct rte_security_ipsec_xform *ipsec,
 225                    struct rte_crypto_sym_xform *xform,
 226                    struct otx2_sec_session_ipsec_ip *sess)
 227 {
 228         struct rte_crypto_sym_xform *cipher_xform, *auth_xform;
 229
 230         sess->partial_len = sizeof(struct rte_ipv4_hdr);
 231
 232         if (ipsec->proto == RTE_SECURITY_IPSEC_SA_PROTO_ESP) {
 233                 sess->partial_len += sizeof(struct rte_esp_hdr);
 234                 sess->roundup_len = sizeof(struct rte_esp_tail);
 235         } else if (ipsec->proto == RTE_SECURITY_IPSEC_SA_PROTO_AH) {
 236                 sess->partial_len += OTX2_SEC_AH_HDR_LEN;
 237         } else {
 238                 return -EINVAL;
 239         }
 240
 241         if (ipsec->options.udp_encap)
 242                 sess->partial_len += sizeof(struct rte_udp_hdr);
 243
 244         if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 245                 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM) {
 246                         sess->partial_len += OTX2_SEC_AES_GCM_IV_LEN;
 247                         sess->partial_len += OTX2_SEC_AES_GCM_MAC_LEN;
 248                         sess->roundup_byte = OTX2_SEC_AES_GCM_ROUNDUP_BYTE_LEN;
 249                 }
 250                 return 0;
 251         }
 252
 253         if (ipsec->direction == RTE_SECURITY_IPSEC_SA_DIR_EGRESS) {
 254                 cipher_xform = xform;
 255                 auth_xform = xform->next;
 256         } else if (ipsec->direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
 257                 auth_xform = xform;
 258                 cipher_xform = xform->next;
 259         } else {
 260                 return -EINVAL;
 261         }
 262         if (cipher_xform->cipher.algo == RTE_CRYPTO_CIPHER_AES_CBC) {
 263                 sess->partial_len += OTX2_SEC_AES_CBC_IV_LEN;
 264                 sess->roundup_byte = OTX2_SEC_AES_CBC_ROUNDUP_BYTE_LEN;
 265         } else {
 266                 return -EINVAL;
 267         }
 268
 269         if (auth_xform->auth.algo == RTE_CRYPTO_AUTH_SHA1_HMAC)
 270                 sess->partial_len += OTX2_SEC_SHA1_HMAC_LEN;
 271         else
 272                 return -EINVAL;
 273
 274         return 0;
 275 }
 276
 277 static int
 278 hmac_init(struct otx2_ipsec_fp_sa_ctl *ctl, struct otx2_cpt_qp *qp,
 279           const uint8_t *auth_key, int len, uint8_t *hmac_key)
 280 {
 281         struct inst_data {
 282                 struct otx2_cpt_res cpt_res;
 283                 uint8_t buffer[64];
 284         } *md;
 285
 286         volatile struct otx2_cpt_res *res;
 287         uint64_t timeout, lmt_status;
 288         struct otx2_cpt_inst_s inst;
 289         rte_iova_t md_iova;
 290         int ret;
 291
 292         memset(&inst, 0, sizeof(struct otx2_cpt_inst_s));
 293
 294         md = rte_zmalloc(NULL, sizeof(struct inst_data), OTX2_CPT_RES_ALIGN);
 295         if (md == NULL)
 296                 return -ENOMEM;
 297
 298         memcpy(md->buffer, auth_key, len);
 299
 300         md_iova = rte_malloc_virt2iova(md);
 301         if (md_iova == RTE_BAD_IOVA) {
 302                 ret = -EINVAL;
 303                 goto free_md;
 304         }
 305
 306         inst.res_addr = md_iova + offsetof(struct inst_data, cpt_res);
 307         inst.opcode = OTX2_CPT_OP_WRITE_HMAC_IPAD_OPAD;
 308         inst.param2 = ctl->auth_type;
 309         inst.dlen = len;
 310         inst.dptr = md_iova + offsetof(struct inst_data, buffer);
 311         inst.rptr = inst.dptr;
 312         inst.egrp = OTX2_CPT_EGRP_INLINE_IPSEC;
 313
 314         md->cpt_res.compcode = 0;
 315         md->cpt_res.uc_compcode = 0xff;
 316
 317         timeout = rte_get_timer_cycles() + 5 * rte_get_timer_hz();
 318
 319         rte_io_wmb();
 320
 321         do {
 322                 otx2_lmt_mov(qp->lmtline, &inst, 2);
 323                 lmt_status = otx2_lmt_submit(qp->lf_nq_reg);
 324         } while (lmt_status == 0);
 325
 326         res = (volatile struct otx2_cpt_res *)&md->cpt_res;
 327
 328         /* Wait until instruction completes or times out */
 329         while (res->uc_compcode == 0xff) {
 330                 if (rte_get_timer_cycles() > timeout)
 331                         break;
 332         }
 333
 334         if (res->u16[0] != OTX2_SEC_COMP_GOOD) {
 335                 ret = -EIO;
 336                 goto free_md;
 337         }
 338
 339         /* Retrieve the ipad and opad from rptr */
 340         memcpy(hmac_key, md->buffer, 48);
 341
 342         ret = 0;
 343
 344 free_md:
 345         rte_free(md);
 346         return ret;
 347 }
 348
 349 static int
 350 eth_sec_ipsec_out_sess_create(struct rte_eth_dev *eth_dev,
 351                               struct rte_security_ipsec_xform *ipsec,
 352                               struct rte_crypto_sym_xform *crypto_xform,
 353                               struct rte_security_session *sec_sess)
 354 {
 355         struct rte_crypto_sym_xform *auth_xform, *cipher_xform;
 356         struct otx2_sec_session_ipsec_ip *sess;
 357         uint16_t port = eth_dev->data->port_id;
 358         int cipher_key_len, auth_key_len, ret;
 359         const uint8_t *cipher_key, *auth_key;
 360         struct otx2_ipsec_fp_sa_ctl *ctl;
 361         struct otx2_ipsec_fp_out_sa *sa;
 362         struct otx2_sec_session *priv;
 363         struct otx2_cpt_inst_s inst;
 364         struct otx2_cpt_qp *qp;
 365
 366         priv = get_sec_session_private_data(sec_sess);
 367         priv->ipsec.dir = RTE_SECURITY_IPSEC_SA_DIR_EGRESS;
 368         sess = &priv->ipsec.ip;
 369
 370         sa = &sess->out_sa;
 371         ctl = &sa->ctl;
 372         if (ctl->valid) {
 373                 otx2_err("SA already registered");
 374                 return -EINVAL;
 375         }
 376
 377         memset(sess, 0, sizeof(struct otx2_sec_session_ipsec_ip));
 378
 379         sess->seq = 1;
 380
 381         ret = ipsec_sa_const_set(ipsec, crypto_xform, sess);
 382         if (ret < 0)
 383                 return ret;
 384
 385         if (crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD)
 386                 memcpy(sa->nonce, &ipsec->salt, 4);
 387
 388         if (ipsec->options.udp_encap == 1) {
 389                 sa->udp_src = 4500;
 390                 sa->udp_dst = 4500;
 391         }
 392
 393         if (ipsec->mode == RTE_SECURITY_IPSEC_SA_MODE_TUNNEL) {
 394                 /* Start ip id from 1 */
 395                 sess->ip_id = 1;
 396
 397                 if (ipsec->tunnel.type == RTE_SECURITY_IPSEC_TUNNEL_IPV4) {
 398                         memcpy(&sa->ip_src, &ipsec->tunnel.ipv4.src_ip,
 399                                sizeof(struct in_addr));
 400                         memcpy(&sa->ip_dst, &ipsec->tunnel.ipv4.dst_ip,
 401                                sizeof(struct in_addr));
 402                 } else {
 403                         return -EINVAL;
 404                 }
 405         } else {
 406                 return -EINVAL;
 407         }
 408
 409         cipher_xform = crypto_xform;
 410         auth_xform = crypto_xform->next;
 411
 412         cipher_key_len = 0;
 413         auth_key_len = 0;
 414         auth_key = NULL;
 415
 416         if (crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 417                 cipher_key = crypto_xform->aead.key.data;
 418                 cipher_key_len = crypto_xform->aead.key.length;
 419         } else {
 420                 cipher_key = cipher_xform->cipher.key.data;
 421                 cipher_key_len = cipher_xform->cipher.key.length;
 422                 auth_key = auth_xform->auth.key.data;
 423                 auth_key_len = auth_xform->auth.key.length;
 424         }
 425
 426         if (cipher_key_len != 0)
 427                 memcpy(sa->cipher_key, cipher_key, cipher_key_len);
 428         else
 429                 return -EINVAL;
 430
 431         /* Determine word 7 of CPT instruction */
 432         inst.u64[7] = 0;
 433         inst.egrp = OTX2_CPT_EGRP_INLINE_IPSEC;
 434         inst.cptr = rte_mempool_virt2iova(sa);
 435         sess->inst_w7 = inst.u64[7];
 436
 437         /* Get CPT QP to be used for this SA */
 438         ret = otx2_sec_idev_tx_cpt_qp_get(port, &qp);
 439         if (ret)
 440                 return ret;
 441
 442         sess->qp = qp;
 443
 444         sess->cpt_lmtline = qp->lmtline;
 445         sess->cpt_nq_reg = qp->lf_nq_reg;
 446
 447         /* Populate control word */
 448         ret = ipsec_fp_sa_ctl_set(ipsec, crypto_xform, ctl);
 449         if (ret)
 450                 goto cpt_put;
 451
 452         if (auth_key_len && auth_key) {
 453                 ret = hmac_init(ctl, qp, auth_key, auth_key_len, sa->hmac_key);
 454                 if (ret)
 455                         goto cpt_put;
 456         }
 457
 458         rte_io_wmb();
 459         ctl->valid = 1;
 460
 461         return 0;
 462 cpt_put:
 463         otx2_sec_idev_tx_cpt_qp_put(sess->qp);
 464         return ret;
 465 }
 466
 467 static int
 468 eth_sec_ipsec_in_sess_create(struct rte_eth_dev *eth_dev,
 469                              struct rte_security_ipsec_xform *ipsec,
 470                              struct rte_crypto_sym_xform *crypto_xform,
 471                              struct rte_security_session *sec_sess)
 472 {
 473         struct rte_crypto_sym_xform *auth_xform, *cipher_xform;
 474         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
 475         struct otx2_sec_session_ipsec_ip *sess;
 476         uint16_t port = eth_dev->data->port_id;
 477         int cipher_key_len, auth_key_len, ret;
 478         const uint8_t *cipher_key, *auth_key;
 479         struct otx2_ipsec_fp_sa_ctl *ctl;
 480         struct otx2_ipsec_fp_in_sa *sa;
 481         struct otx2_sec_session *priv;
 482         char err_str[ERR_STR_SZ];
 483         struct otx2_cpt_qp *qp;
 484
 485         memset(err_str, 0, ERR_STR_SZ);
 486
 487         if (ipsec->spi >= dev->ipsec_in_max_spi) {
 488                 otx2_err("SPI exceeds max supported");
 489                 return -EINVAL;
 490         }
 491
 492         sa = in_sa_get(port, ipsec->spi);
 493         if (sa == NULL)
 494                 return -ENOMEM;
 495
 496         ctl = &sa->ctl;
 497
 498         priv = get_sec_session_private_data(sec_sess);
 499         priv->ipsec.dir = RTE_SECURITY_IPSEC_SA_DIR_INGRESS;
 500         sess = &priv->ipsec.ip;
 501
 502         rte_spinlock_lock(&dev->ipsec_tbl_lock);
 503
 504         if (ctl->valid) {
 505                 snprintf(err_str, ERR_STR_SZ, "SA already registered");
 506                 ret = -EEXIST;
 507                 goto tbl_unlock;
 508         }
 509
 510         memset(sa, 0, sizeof(struct otx2_ipsec_fp_in_sa));
 511
 512         auth_xform = crypto_xform;
 513         cipher_xform = crypto_xform->next;
 514
 515         cipher_key_len = 0;
 516         auth_key_len = 0;
 517         auth_key = NULL;
 518
 519         if (crypto_xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 520                 if (crypto_xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM)
 521                         memcpy(sa->nonce, &ipsec->salt, 4);
 522                 cipher_key = crypto_xform->aead.key.data;
 523                 cipher_key_len = crypto_xform->aead.key.length;
 524         } else {
 525                 cipher_key = cipher_xform->cipher.key.data;
 526                 cipher_key_len = cipher_xform->cipher.key.length;
 527                 auth_key = auth_xform->auth.key.data;
 528                 auth_key_len = auth_xform->auth.key.length;
 529         }
 530
 531         if (cipher_key_len != 0) {
 532                 memcpy(sa->cipher_key, cipher_key, cipher_key_len);
 533         } else {
 534                 snprintf(err_str, ERR_STR_SZ, "Invalid cipher key len");
 535                 ret = -EINVAL;
 536                 goto sa_clear;
 537         }
 538
 539         sess->in_sa = sa;
 540
 541         sa->userdata = priv->userdata;
 542
 543         sa->replay_win_sz = ipsec->replay_win_sz;
 544
 545         if (lookup_mem_sa_index_update(eth_dev, ipsec->spi, sa, err_str)) {
 546                 ret = -EINVAL;
 547                 goto sa_clear;
 548         }
 549
 550         ret = ipsec_fp_sa_ctl_set(ipsec, crypto_xform, ctl);
 551         if (ret) {
 552                 snprintf(err_str, ERR_STR_SZ,
 553                         "Could not set SA CTL word (err: %d)", ret);
 554                 goto sa_clear;
 555         }
 556
 557         if (auth_key_len && auth_key) {
 558                 /* Get a queue pair for HMAC init */
 559                 ret = otx2_sec_idev_tx_cpt_qp_get(port, &qp);
 560                 if (ret) {
 561                         snprintf(err_str, ERR_STR_SZ, "Could not get CPT QP");
 562                         goto sa_clear;
 563                 }
 564
 565                 ret = hmac_init(ctl, qp, auth_key, auth_key_len, sa->hmac_key);
 566                 otx2_sec_idev_tx_cpt_qp_put(qp);
 567                 if (ret) {
 568                         snprintf(err_str, ERR_STR_SZ, "Could not put CPT QP");
 569                         goto sa_clear;
 570                 }
 571         }
 572
 573         if (sa->replay_win_sz) {
 574                 if (sa->replay_win_sz > OTX2_IPSEC_MAX_REPLAY_WIN_SZ) {
 575                         snprintf(err_str, ERR_STR_SZ,
 576                                  "Replay window size is not supported");
 577                         ret = -ENOTSUP;
 578                         goto sa_clear;
 579                 }
 580                 sa->replay = rte_zmalloc(NULL, sizeof(struct otx2_ipsec_replay),
 581                                 0);
 582                 if (sa->replay == NULL) {
 583                         snprintf(err_str, ERR_STR_SZ,
 584                                 "Could not allocate memory");
 585                         ret = -ENOMEM;
 586                         goto sa_clear;
 587                 }
 588
 589                 rte_spinlock_init(&sa->replay->lock);
 590                 /*
 591                  * Set window bottom to 1, base and top to size of
 592                  * window
 593                  */
 594                 sa->replay->winb = 1;
 595                 sa->replay->wint = sa->replay_win_sz;
 596                 sa->replay->base = sa->replay_win_sz;
 597                 sa->esn_low = 0;
 598                 sa->esn_hi = 0;
 599         }
 600
 601         rte_io_wmb();
 602         ctl->valid = 1;
 603
 604         rte_spinlock_unlock(&dev->ipsec_tbl_lock);
 605         return 0;
 606
 607 sa_clear:
 608         memset(sa, 0, sizeof(struct otx2_ipsec_fp_in_sa));
 609
 610 tbl_unlock:
 611         rte_spinlock_unlock(&dev->ipsec_tbl_lock);
 612
 613         otx2_err("%s", err_str);
 614
 615         return ret;
 616 }
 617
 618 static int
 619 eth_sec_ipsec_sess_create(struct rte_eth_dev *eth_dev,
 620                           struct rte_security_ipsec_xform *ipsec,
 621                           struct rte_crypto_sym_xform *crypto_xform,
 622                           struct rte_security_session *sess)
 623 {
 624         int ret;
 625
 626         ret = ipsec_fp_xform_verify(ipsec, crypto_xform);
 627         if (ret)
 628                 return ret;
 629
 630         if (ipsec->direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS)
 631                 return eth_sec_ipsec_in_sess_create(eth_dev, ipsec,
 632                                                     crypto_xform, sess);
 633         else
 634                 return eth_sec_ipsec_out_sess_create(eth_dev, ipsec,
 635                                                      crypto_xform, sess);
 636 }
 637
 638 static int
 639 otx2_eth_sec_session_create(void *device,
 640                             struct rte_security_session_conf *conf,
 641                             struct rte_security_session *sess,
 642                             struct rte_mempool *mempool)
 643 {
 644         struct otx2_sec_session *priv;
 645         int ret;
 646
 647         if (conf->action_type != RTE_SECURITY_ACTION_TYPE_INLINE_PROTOCOL)
 648                 return -ENOTSUP;
 649
 650         if (rte_mempool_get(mempool, (void **)&priv)) {
 651                 otx2_err("Could not allocate security session private data");
 652                 return -ENOMEM;
 653         }
 654
 655         set_sec_session_private_data(sess, priv);
 656
 657         /*
 658          * Save userdata provided by the application. For ingress packets, this
 659          * could be used to identify the SA.
 660          */
 661         priv->userdata = conf->userdata;
 662
 663         if (conf->protocol == RTE_SECURITY_PROTOCOL_IPSEC)
 664                 ret = eth_sec_ipsec_sess_create(device, &conf->ipsec,
 665                                                 conf->crypto_xform,
 666                                                 sess);
 667         else
 668                 ret = -ENOTSUP;
 669
 670         if (ret)
 671                 goto mempool_put;
 672
 673         return 0;
 674
 675 mempool_put:
 676         rte_mempool_put(mempool, priv);
 677         set_sec_session_private_data(sess, NULL);
 678         return ret;
 679 }
 680
 681 static void
 682 otx2_eth_sec_free_anti_replay(struct otx2_ipsec_fp_in_sa *sa)
 683 {
 684         if (sa != NULL) {
 685                 if (sa->replay_win_sz && sa->replay)
 686                         rte_free(sa->replay);
 687         }
 688 }
 689
 690 static int
 691 otx2_eth_sec_session_destroy(void *device,
 692                              struct rte_security_session *sess)
 693 {
 694         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(device);
 695         struct otx2_sec_session_ipsec_ip *sess_ip;
 696         struct otx2_ipsec_fp_in_sa *sa;
 697         struct otx2_sec_session *priv;
 698         struct rte_mempool *sess_mp;
 699         int ret;
 700
 701         priv = get_sec_session_private_data(sess);
 702         if (priv == NULL)
 703                 return -EINVAL;
 704
 705         sess_ip = &priv->ipsec.ip;
 706
 707         if (priv->ipsec.dir == RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
 708                 rte_spinlock_lock(&dev->ipsec_tbl_lock);
 709                 sa = sess_ip->in_sa;
 710
 711                 /* Release the anti replay window */
 712                 otx2_eth_sec_free_anti_replay(sa);
 713
 714                 /* Clear SA table entry */
 715                 if (sa != NULL) {
 716                         sa->ctl.valid = 0;
 717                         rte_io_wmb();
 718                 }
 719
 720                 rte_spinlock_unlock(&dev->ipsec_tbl_lock);
 721         }
 722
 723         /* Release CPT LF used for this session */
 724         if (sess_ip->qp != NULL) {
 725                 ret = otx2_sec_idev_tx_cpt_qp_put(sess_ip->qp);
 726                 if (ret)
 727                         return ret;
 728         }
 729
 730         sess_mp = rte_mempool_from_obj(priv);
 731
 732         set_sec_session_private_data(sess, NULL);
 733         rte_mempool_put(sess_mp, priv);
 734
 735         return 0;
 736 }
 737
 738 static unsigned int
 739 otx2_eth_sec_session_get_size(void *device __rte_unused)
 740 {
 741         return sizeof(struct otx2_sec_session);
 742 }
 743
 744 static int
 745 otx2_eth_sec_set_pkt_mdata(void *device __rte_unused,
 746                             struct rte_security_session *session,
 747                             struct rte_mbuf *m, void *params __rte_unused)
 748 {
 749         /* Set security session as the pkt metadata */
 750         *rte_security_dynfield(m) = (rte_security_dynfield_t)session;
 751
 752         return 0;
 753 }
 754
 755 static int
 756 otx2_eth_sec_get_userdata(void *device __rte_unused, uint64_t md,
 757                            void **userdata)
 758 {
 759         /* Retrieve userdata  */
 760         *userdata = (void *)md;
 761
 762         return 0;
 763 }
 764
 765 static const struct rte_security_capability *
 766 otx2_eth_sec_capabilities_get(void *device __rte_unused)
 767 {
 768         return otx2_eth_sec_capabilities;
 769 }
 770
 771 static struct rte_security_ops otx2_eth_sec_ops = {
 772         .session_create         = otx2_eth_sec_session_create,
 773         .session_destroy        = otx2_eth_sec_session_destroy,
 774         .session_get_size       = otx2_eth_sec_session_get_size,
 775         .set_pkt_metadata       = otx2_eth_sec_set_pkt_mdata,
 776         .get_userdata           = otx2_eth_sec_get_userdata,
 777         .capabilities_get       = otx2_eth_sec_capabilities_get
 778 };
 779
 780 int
 781 otx2_eth_sec_ctx_create(struct rte_eth_dev *eth_dev)
 782 {
 783         struct rte_security_ctx *ctx;
 784         int ret;
 785
 786         ctx = rte_malloc("otx2_eth_sec_ctx",
 787                          sizeof(struct rte_security_ctx), 0);
 788         if (ctx == NULL)
 789                 return -ENOMEM;
 790
 791         ret = otx2_sec_idev_cfg_init(eth_dev->data->port_id);
 792         if (ret) {
 793                 rte_free(ctx);
 794                 return ret;
 795         }
 796
 797         /* Populate ctx */
 798
 799         ctx->device = eth_dev;
 800         ctx->ops = &otx2_eth_sec_ops;
 801         ctx->sess_cnt = 0;
 802
 803         eth_dev->security_ctx = ctx;
 804
 805         return 0;
 806 }
 807
 808 void
 809 otx2_eth_sec_ctx_destroy(struct rte_eth_dev *eth_dev)
 810 {
 811         rte_free(eth_dev->security_ctx);
 812 }
 813
 814 static int
 815 eth_sec_ipsec_cfg(struct rte_eth_dev *eth_dev, uint8_t tt)
 816 {
 817         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
 818         uint16_t port = eth_dev->data->port_id;
 819         struct nix_inline_ipsec_lf_cfg *req;
 820         struct otx2_mbox *mbox = dev->mbox;
 821         struct eth_sec_tag_const tag_const;
 822         char name[RTE_MEMZONE_NAMESIZE];
 823         const struct rte_memzone *mz;
 824
 825         in_sa_mz_name_get(name, RTE_MEMZONE_NAMESIZE, port);
 826         mz = rte_memzone_lookup(name);
 827         if (mz == NULL)
 828                 return -EINVAL;
 829
 830         req = otx2_mbox_alloc_msg_nix_inline_ipsec_lf_cfg(mbox);
 831         req->enable = 1;
 832         req->sa_base_addr = mz->iova;
 833
 834         req->ipsec_cfg0.tt = tt;
 835
 836         tag_const.u32 = 0;
 837         tag_const.event_type = RTE_EVENT_TYPE_ETHDEV;
 838         tag_const.port = port;
 839         req->ipsec_cfg0.tag_const = tag_const.u32;
 840
 841         req->ipsec_cfg0.sa_pow2_size =
 842                         rte_log2_u32(sizeof(struct otx2_ipsec_fp_in_sa));
 843         req->ipsec_cfg0.lenm1_max = NIX_MAX_FRS - 1;
 844
 845         req->ipsec_cfg1.sa_idx_w = rte_log2_u32(dev->ipsec_in_max_spi);
 846         req->ipsec_cfg1.sa_idx_max = dev->ipsec_in_max_spi - 1;
 847
 848         return otx2_mbox_process(mbox);
 849 }
 850
 851 int
 852 otx2_eth_sec_update_tag_type(struct rte_eth_dev *eth_dev)
 853 {
 854         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
 855         struct otx2_mbox *mbox = dev->mbox;
 856         struct nix_aq_enq_rsp *rsp;
 857         struct nix_aq_enq_req *aq;
 858         int ret;
 859
 860         aq = otx2_mbox_alloc_msg_nix_aq_enq(mbox);
 861         aq->qidx = 0; /* Read RQ:0 context */
 862         aq->ctype = NIX_AQ_CTYPE_RQ;
 863         aq->op = NIX_AQ_INSTOP_READ;
 864
 865         ret = otx2_mbox_process_msg(mbox, (void *)&rsp);
 866         if (ret < 0) {
 867                 otx2_err("Could not read RQ context");
 868                 return ret;
 869         }
 870
 871         /* Update tag type */
 872         ret = eth_sec_ipsec_cfg(eth_dev, rsp->rq.sso_tt);
 873         if (ret < 0)
 874                 otx2_err("Could not update sec eth tag type");
 875
 876         return ret;
 877 }
 878
 879 int
 880 otx2_eth_sec_init(struct rte_eth_dev *eth_dev)
 881 {
 882         const size_t sa_width = sizeof(struct otx2_ipsec_fp_in_sa);
 883         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
 884         uint16_t port = eth_dev->data->port_id;
 885         char name[RTE_MEMZONE_NAMESIZE];
 886         const struct rte_memzone *mz;
 887         int mz_sz, ret;
 888         uint16_t nb_sa;
 889
 890         RTE_BUILD_BUG_ON(sa_width < 32 || sa_width > 512 ||
 891                          !RTE_IS_POWER_OF_2(sa_width));
 892
 893         if (!(dev->tx_offloads & DEV_TX_OFFLOAD_SECURITY) &&
 894             !(dev->rx_offloads & DEV_RX_OFFLOAD_SECURITY))
 895                 return 0;
 896
 897         if (rte_security_dynfield_register() < 0)
 898                 return -rte_errno;
 899
 900         nb_sa = dev->ipsec_in_max_spi;
 901         mz_sz = nb_sa * sa_width;
 902         in_sa_mz_name_get(name, RTE_MEMZONE_NAMESIZE, port);
 903         mz = rte_memzone_reserve_aligned(name, mz_sz, rte_socket_id(),
 904                                          RTE_MEMZONE_IOVA_CONTIG, OTX2_ALIGN);
 905
 906         if (mz == NULL) {
 907                 otx2_err("Could not allocate inbound SA DB");
 908                 return -ENOMEM;
 909         }
 910
 911         memset(mz->addr, 0, mz_sz);
 912
 913         ret = eth_sec_ipsec_cfg(eth_dev, SSO_TT_ORDERED);
 914         if (ret < 0) {
 915                 otx2_err("Could not configure inline IPsec");
 916                 goto sec_fini;
 917         }
 918
 919         rte_spinlock_init(&dev->ipsec_tbl_lock);
 920
 921         return 0;
 922
 923 sec_fini:
 924         otx2_err("Could not configure device for security");
 925         otx2_eth_sec_fini(eth_dev);
 926         return ret;
 927 }
 928
 929 void
 930 otx2_eth_sec_fini(struct rte_eth_dev *eth_dev)
 931 {
 932         struct otx2_eth_dev *dev = otx2_eth_pmd_priv(eth_dev);
 933         uint16_t port = eth_dev->data->port_id;
 934         char name[RTE_MEMZONE_NAMESIZE];
 935
 936         if (!(dev->tx_offloads & DEV_TX_OFFLOAD_SECURITY) &&
 937             !(dev->rx_offloads & DEV_RX_OFFLOAD_SECURITY))
 938                 return;
 939
 940         lookup_mem_sa_tbl_clear(eth_dev);
 941
 942         in_sa_mz_name_get(name, RTE_MEMZONE_NAMESIZE, port);
 943         rte_memzone_free(rte_memzone_lookup(name));
 944 }