drivers/net/ixgbe/ixgbe_ipsec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <ethdev_driver.h>
   6 #include <ethdev_pci.h>
   7 #include <rte_ip.h>
   8 #include <rte_jhash.h>
   9 #include <rte_security_driver.h>
  10 #include <rte_cryptodev.h>
  11 #include <rte_flow.h>
  12
  13 #include "base/ixgbe_type.h"
  14 #include "base/ixgbe_api.h"
  15 #include "ixgbe_ethdev.h"
  16 #include "ixgbe_ipsec.h"
  17
  18 #define RTE_IXGBE_REGISTER_POLL_WAIT_5_MS  5
  19
  20 #define IXGBE_WAIT_RREAD \
  21         IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSRXIDX, reg_val, \
  22         IPSRXIDX_READ, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS)
  23 #define IXGBE_WAIT_RWRITE \
  24         IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSRXIDX, reg_val, \
  25         IPSRXIDX_WRITE, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS)
  26 #define IXGBE_WAIT_TREAD \
  27         IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSTXIDX, reg_val, \
  28         IPSRXIDX_READ, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS)
  29 #define IXGBE_WAIT_TWRITE \
  30         IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSTXIDX, reg_val, \
  31         IPSRXIDX_WRITE, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS)
  32
  33 #define CMP_IP(a, b) (\
  34         (a).ipv6[0] == (b).ipv6[0] && \
  35         (a).ipv6[1] == (b).ipv6[1] && \
  36         (a).ipv6[2] == (b).ipv6[2] && \
  37         (a).ipv6[3] == (b).ipv6[3])
  38
  39
  40 static void
  41 ixgbe_crypto_clear_ipsec_tables(struct rte_eth_dev *dev)
  42 {
  43         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
  44         struct ixgbe_ipsec *priv = IXGBE_DEV_PRIVATE_TO_IPSEC(
  45                                 dev->data->dev_private);
  46         int i = 0;
  47
  48         /* clear Rx IP table*/
  49         for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
  50                 uint16_t index = i << 3;
  51                 uint32_t reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_IP | index;
  52                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0), 0);
  53                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1), 0);
  54                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2), 0);
  55                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3), 0);
  56                 IXGBE_WAIT_RWRITE;
  57         }
  58
  59         /* clear Rx SPI and Rx/Tx SA tables*/
  60         for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
  61                 uint32_t index = i << 3;
  62                 uint32_t reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_SPI | index;
  63                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSPI, 0);
  64                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPIDX, 0);
  65                 IXGBE_WAIT_RWRITE;
  66                 reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_KEY | index;
  67                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(0), 0);
  68                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(1), 0);
  69                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(2), 0);
  70                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(3), 0);
  71                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSALT, 0);
  72                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXMOD, 0);
  73                 IXGBE_WAIT_RWRITE;
  74                 reg_val = IPSRXIDX_WRITE | index;
  75                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(0), 0);
  76                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(1), 0);
  77                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(2), 0);
  78                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(3), 0);
  79                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXSALT, 0);
  80                 IXGBE_WAIT_TWRITE;
  81         }
  82
  83         memset(priv->rx_ip_tbl, 0, sizeof(priv->rx_ip_tbl));
  84         memset(priv->rx_sa_tbl, 0, sizeof(priv->rx_sa_tbl));
  85         memset(priv->tx_sa_tbl, 0, sizeof(priv->tx_sa_tbl));
  86 }
  87
  88 static int
  89 ixgbe_crypto_add_sa(struct ixgbe_crypto_session *ic_session)
  90 {
  91         struct rte_eth_dev *dev = ic_session->dev;
  92         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
  93         struct ixgbe_ipsec *priv = IXGBE_DEV_PRIVATE_TO_IPSEC(
  94                         dev->data->dev_private);
  95         uint32_t reg_val;
  96         int sa_index = -1;
  97
  98         if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION) {
  99                 int i, ip_index = -1;
 100                 uint8_t *key;
 101
 102                 /* Find a match in the IP table*/
 103                 for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
 104                         if (CMP_IP(priv->rx_ip_tbl[i].ip,
 105                                    ic_session->dst_ip)) {
 106                                 ip_index = i;
 107                                 break;
 108                         }
 109                 }
 110                 /* If no match, find a free entry in the IP table*/
 111                 if (ip_index < 0) {
 112                         for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
 113                                 if (priv->rx_ip_tbl[i].ref_count == 0) {
 114                                         ip_index = i;
 115                                         break;
 116                                 }
 117                         }
 118                 }
 119
 120                 /* Fail if no match and no free entries*/
 121                 if (ip_index < 0) {
 122                         PMD_DRV_LOG(ERR,
 123                                     "No free entry left in the Rx IP table\n");
 124                         return -1;
 125                 }
 126
 127                 /* Find a free entry in the SA table*/
 128                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
 129                         if (priv->rx_sa_tbl[i].used == 0) {
 130                                 sa_index = i;
 131                                 break;
 132                         }
 133                 }
 134                 /* Fail if no free entries*/
 135                 if (sa_index < 0) {
 136                         PMD_DRV_LOG(ERR,
 137                                     "No free entry left in the Rx SA table\n");
 138                         return -1;
 139                 }
 140
 141                 priv->rx_ip_tbl[ip_index].ip.ipv6[0] =
 142                                 ic_session->dst_ip.ipv6[0];
 143                 priv->rx_ip_tbl[ip_index].ip.ipv6[1] =
 144                                 ic_session->dst_ip.ipv6[1];
 145                 priv->rx_ip_tbl[ip_index].ip.ipv6[2] =
 146                                 ic_session->dst_ip.ipv6[2];
 147                 priv->rx_ip_tbl[ip_index].ip.ipv6[3] =
 148                                 ic_session->dst_ip.ipv6[3];
 149                 priv->rx_ip_tbl[ip_index].ref_count++;
 150
 151                 priv->rx_sa_tbl[sa_index].spi =
 152                         rte_cpu_to_be_32(ic_session->spi);
 153                 priv->rx_sa_tbl[sa_index].ip_index = ip_index;
 154                 priv->rx_sa_tbl[sa_index].mode = IPSRXMOD_VALID;
 155                 if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION)
 156                         priv->rx_sa_tbl[sa_index].mode |=
 157                                         (IPSRXMOD_PROTO | IPSRXMOD_DECRYPT);
 158                 if (ic_session->dst_ip.type == IPv6) {
 159                         priv->rx_sa_tbl[sa_index].mode |= IPSRXMOD_IPV6;
 160                         priv->rx_ip_tbl[ip_index].ip.type = IPv6;
 161                 } else if (ic_session->dst_ip.type == IPv4)
 162                         priv->rx_ip_tbl[ip_index].ip.type = IPv4;
 163
 164                 priv->rx_sa_tbl[sa_index].used = 1;
 165
 166                 /* write IP table entry*/
 167                 reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE |
 168                                 IPSRXIDX_TABLE_IP | (ip_index << 3);
 169                 if (priv->rx_ip_tbl[ip_index].ip.type == IPv4) {
 170                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0), 0);
 171                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1), 0);
 172                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2), 0);
 173                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3),
 174                                         priv->rx_ip_tbl[ip_index].ip.ipv4);
 175                 } else {
 176                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0),
 177                                         priv->rx_ip_tbl[ip_index].ip.ipv6[0]);
 178                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1),
 179                                         priv->rx_ip_tbl[ip_index].ip.ipv6[1]);
 180                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2),
 181                                         priv->rx_ip_tbl[ip_index].ip.ipv6[2]);
 182                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3),
 183                                         priv->rx_ip_tbl[ip_index].ip.ipv6[3]);
 184                 }
 185                 IXGBE_WAIT_RWRITE;
 186
 187                 /* write SPI table entry*/
 188                 reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE |
 189                                 IPSRXIDX_TABLE_SPI | (sa_index << 3);
 190                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSPI,
 191                                 priv->rx_sa_tbl[sa_index].spi);
 192                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPIDX,
 193                                 priv->rx_sa_tbl[sa_index].ip_index);
 194                 IXGBE_WAIT_RWRITE;
 195
 196                 /* write Key table entry*/
 197                 key = malloc(ic_session->key_len);
 198                 if (!key)
 199                         return -ENOMEM;
 200
 201                 memcpy(key, ic_session->key, ic_session->key_len);
 202
 203                 reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE |
 204                                 IPSRXIDX_TABLE_KEY | (sa_index << 3);
 205                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(0),
 206                         rte_cpu_to_be_32(*(uint32_t *)&key[12]));
 207                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(1),
 208                         rte_cpu_to_be_32(*(uint32_t *)&key[8]));
 209                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(2),
 210                         rte_cpu_to_be_32(*(uint32_t *)&key[4]));
 211                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(3),
 212                         rte_cpu_to_be_32(*(uint32_t *)&key[0]));
 213                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSALT,
 214                                 rte_cpu_to_be_32(ic_session->salt));
 215                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXMOD,
 216                                 priv->rx_sa_tbl[sa_index].mode);
 217                 IXGBE_WAIT_RWRITE;
 218
 219                 free(key);
 220
 221         } else { /* sess->dir == RTE_CRYPTO_OUTBOUND */
 222                 uint8_t *key;
 223                 int i;
 224
 225                 /* Find a free entry in the SA table*/
 226                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
 227                         if (priv->tx_sa_tbl[i].used == 0) {
 228                                 sa_index = i;
 229                                 break;
 230                         }
 231                 }
 232                 /* Fail if no free entries*/
 233                 if (sa_index < 0) {
 234                         PMD_DRV_LOG(ERR,
 235                                     "No free entry left in the Tx SA table\n");
 236                         return -1;
 237                 }
 238
 239                 priv->tx_sa_tbl[sa_index].spi =
 240                         rte_cpu_to_be_32(ic_session->spi);
 241                 priv->tx_sa_tbl[i].used = 1;
 242                 ic_session->sa_index = sa_index;
 243
 244                 key = malloc(ic_session->key_len);
 245                 if (!key)
 246                         return -ENOMEM;
 247
 248                 memcpy(key, ic_session->key, ic_session->key_len);
 249
 250                 /* write Key table entry*/
 251                 reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE | (sa_index << 3);
 252                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(0),
 253                         rte_cpu_to_be_32(*(uint32_t *)&key[12]));
 254                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(1),
 255                         rte_cpu_to_be_32(*(uint32_t *)&key[8]));
 256                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(2),
 257                         rte_cpu_to_be_32(*(uint32_t *)&key[4]));
 258                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(3),
 259                         rte_cpu_to_be_32(*(uint32_t *)&key[0]));
 260                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXSALT,
 261                                 rte_cpu_to_be_32(ic_session->salt));
 262                 IXGBE_WAIT_TWRITE;
 263
 264                 free(key);
 265         }
 266
 267         return 0;
 268 }
 269
 270 static int
 271 ixgbe_crypto_remove_sa(struct rte_eth_dev *dev,
 272                        struct ixgbe_crypto_session *ic_session)
 273 {
 274         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 275         struct ixgbe_ipsec *priv =
 276                         IXGBE_DEV_PRIVATE_TO_IPSEC(dev->data->dev_private);
 277         uint32_t reg_val;
 278         int sa_index = -1;
 279
 280         if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION) {
 281                 int i, ip_index = -1;
 282
 283                 /* Find a match in the IP table*/
 284                 for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
 285                         if (CMP_IP(priv->rx_ip_tbl[i].ip, ic_session->dst_ip)) {
 286                                 ip_index = i;
 287                                 break;
 288                         }
 289                 }
 290
 291                 /* Fail if no match*/
 292                 if (ip_index < 0) {
 293                         PMD_DRV_LOG(ERR,
 294                                     "Entry not found in the Rx IP table\n");
 295                         return -1;
 296                 }
 297
 298                 /* Find a free entry in the SA table*/
 299                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
 300                         if (priv->rx_sa_tbl[i].spi ==
 301                                   rte_cpu_to_be_32(ic_session->spi)) {
 302                                 sa_index = i;
 303                                 break;
 304                         }
 305                 }
 306                 /* Fail if no match*/
 307                 if (sa_index < 0) {
 308                         PMD_DRV_LOG(ERR,
 309                                     "Entry not found in the Rx SA table\n");
 310                         return -1;
 311                 }
 312
 313                 /* Disable and clear Rx SPI and key table table entryes*/
 314                 reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_SPI | (sa_index << 3);
 315                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSPI, 0);
 316                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPIDX, 0);
 317                 IXGBE_WAIT_RWRITE;
 318                 reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_KEY | (sa_index << 3);
 319                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(0), 0);
 320                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(1), 0);
 321                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(2), 0);
 322                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(3), 0);
 323                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSALT, 0);
 324                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXMOD, 0);
 325                 IXGBE_WAIT_RWRITE;
 326                 priv->rx_sa_tbl[sa_index].used = 0;
 327
 328                 /* If last used then clear the IP table entry*/
 329                 priv->rx_ip_tbl[ip_index].ref_count--;
 330                 if (priv->rx_ip_tbl[ip_index].ref_count == 0) {
 331                         reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_IP |
 332                                         (ip_index << 3);
 333                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0), 0);
 334                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1), 0);
 335                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2), 0);
 336                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3), 0);
 337                 }
 338         } else { /* session->dir == RTE_CRYPTO_OUTBOUND */
 339                 int i;
 340
 341                 /* Find a match in the SA table*/
 342                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
 343                         if (priv->tx_sa_tbl[i].spi ==
 344                                     rte_cpu_to_be_32(ic_session->spi)) {
 345                                 sa_index = i;
 346                                 break;
 347                         }
 348                 }
 349                 /* Fail if no match entries*/
 350                 if (sa_index < 0) {
 351                         PMD_DRV_LOG(ERR,
 352                                     "Entry not found in the Tx SA table\n");
 353                         return -1;
 354                 }
 355                 reg_val = IPSRXIDX_WRITE | (sa_index << 3);
 356                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(0), 0);
 357                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(1), 0);
 358                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(2), 0);
 359                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(3), 0);
 360                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXSALT, 0);
 361                 IXGBE_WAIT_TWRITE;
 362
 363                 priv->tx_sa_tbl[sa_index].used = 0;
 364         }
 365
 366         return 0;
 367 }
 368
 369 static int
 370 ixgbe_crypto_create_session(void *device,
 371                 struct rte_security_session_conf *conf,
 372                 struct rte_security_session *session,
 373                 struct rte_mempool *mempool)
 374 {
 375         struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)device;
 376         struct ixgbe_crypto_session *ic_session = NULL;
 377         struct rte_crypto_aead_xform *aead_xform;
 378         struct rte_eth_conf *dev_conf = &eth_dev->data->dev_conf;
 379
 380         if (rte_mempool_get(mempool, (void **)&ic_session)) {
 381                 PMD_DRV_LOG(ERR, "Cannot get object from ic_session mempool");
 382                 return -ENOMEM;
 383         }
 384
 385         if (conf->crypto_xform->type != RTE_CRYPTO_SYM_XFORM_AEAD ||
 386                         conf->crypto_xform->aead.algo !=
 387                                         RTE_CRYPTO_AEAD_AES_GCM) {
 388                 PMD_DRV_LOG(ERR, "Unsupported crypto transformation mode\n");
 389                 rte_mempool_put(mempool, (void *)ic_session);
 390                 return -ENOTSUP;
 391         }
 392         aead_xform = &conf->crypto_xform->aead;
 393
 394         if (conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
 395                 if (dev_conf->rxmode.offloads & RTE_ETH_RX_OFFLOAD_SECURITY) {
 396                         ic_session->op = IXGBE_OP_AUTHENTICATED_DECRYPTION;
 397                 } else {
 398                         PMD_DRV_LOG(ERR, "IPsec decryption not enabled\n");
 399                         rte_mempool_put(mempool, (void *)ic_session);
 400                         return -ENOTSUP;
 401                 }
 402         } else {
 403                 if (dev_conf->txmode.offloads & RTE_ETH_TX_OFFLOAD_SECURITY) {
 404                         ic_session->op = IXGBE_OP_AUTHENTICATED_ENCRYPTION;
 405                 } else {
 406                         PMD_DRV_LOG(ERR, "IPsec encryption not enabled\n");
 407                         rte_mempool_put(mempool, (void *)ic_session);
 408                         return -ENOTSUP;
 409                 }
 410         }
 411
 412         ic_session->key = aead_xform->key.data;
 413         ic_session->key_len = aead_xform->key.length;
 414         memcpy(&ic_session->salt,
 415                &aead_xform->key.data[aead_xform->key.length], 4);
 416         ic_session->spi = conf->ipsec.spi;
 417         ic_session->dev = eth_dev;
 418
 419         set_sec_session_private_data(session, ic_session);
 420
 421         if (ic_session->op == IXGBE_OP_AUTHENTICATED_ENCRYPTION) {
 422                 if (ixgbe_crypto_add_sa(ic_session)) {
 423                         PMD_DRV_LOG(ERR, "Failed to add SA\n");
 424                         rte_mempool_put(mempool, (void *)ic_session);
 425                         return -EPERM;
 426                 }
 427         }
 428
 429         return 0;
 430 }
 431
 432 static unsigned int
 433 ixgbe_crypto_session_get_size(__rte_unused void *device)
 434 {
 435         return sizeof(struct ixgbe_crypto_session);
 436 }
 437
 438 static int
 439 ixgbe_crypto_remove_session(void *device,
 440                 struct rte_security_session *session)
 441 {
 442         struct rte_eth_dev *eth_dev = device;
 443         struct ixgbe_crypto_session *ic_session =
 444                 (struct ixgbe_crypto_session *)
 445                 get_sec_session_private_data(session);
 446         struct rte_mempool *mempool = rte_mempool_from_obj(ic_session);
 447
 448         if (eth_dev != ic_session->dev) {
 449                 PMD_DRV_LOG(ERR, "Session not bound to this device\n");
 450                 return -ENODEV;
 451         }
 452
 453         if (ixgbe_crypto_remove_sa(eth_dev, ic_session)) {
 454                 PMD_DRV_LOG(ERR, "Failed to remove session\n");
 455                 return -EFAULT;
 456         }
 457
 458         rte_mempool_put(mempool, (void *)ic_session);
 459
 460         return 0;
 461 }
 462
 463 static inline uint8_t
 464 ixgbe_crypto_compute_pad_len(struct rte_mbuf *m)
 465 {
 466         if (m->nb_segs == 1) {
 467                 /* 16 bytes ICV + 2 bytes ESP trailer + payload padding size
 468                  * payload padding size is stored at <pkt_len - 18>
 469                  */
 470                 uint8_t *esp_pad_len = rte_pktmbuf_mtod_offset(m, uint8_t *,
 471                                         rte_pktmbuf_pkt_len(m) -
 472                                         (ESP_TRAILER_SIZE + ESP_ICV_SIZE));
 473                 return *esp_pad_len + ESP_TRAILER_SIZE + ESP_ICV_SIZE;
 474         }
 475         return 0;
 476 }
 477
 478 static int
 479 ixgbe_crypto_update_mb(void *device __rte_unused,
 480                 struct rte_security_session *session,
 481                        struct rte_mbuf *m, void *params __rte_unused)
 482 {
 483         struct ixgbe_crypto_session *ic_session =
 484                         get_sec_session_private_data(session);
 485         if (ic_session->op == IXGBE_OP_AUTHENTICATED_ENCRYPTION) {
 486                 union ixgbe_crypto_tx_desc_md *mdata =
 487                         (union ixgbe_crypto_tx_desc_md *)
 488                                 rte_security_dynfield(m);
 489                 mdata->enc = 1;
 490                 mdata->sa_idx = ic_session->sa_index;
 491                 mdata->pad_len = ixgbe_crypto_compute_pad_len(m);
 492         }
 493         return 0;
 494 }
 495
 496
 497 static const struct rte_security_capability *
 498 ixgbe_crypto_capabilities_get(void *device __rte_unused)
 499 {
 500         static const struct rte_cryptodev_capabilities
 501         aes_gcm_gmac_crypto_capabilities[] = {
 502                 {       /* AES GMAC (128-bit) */
 503                         .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 504                         {.sym = {
 505                                 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 506                                 {.auth = {
 507                                         .algo = RTE_CRYPTO_AUTH_AES_GMAC,
 508                                         .block_size = 16,
 509                                         .key_size = {
 510                                                 .min = 16,
 511                                                 .max = 16,
 512                                                 .increment = 0
 513                                         },
 514                                         .digest_size = {
 515                                                 .min = 16,
 516                                                 .max = 16,
 517                                                 .increment = 0
 518                                         },
 519                                         .iv_size = {
 520                                                 .min = 12,
 521                                                 .max = 12,
 522                                                 .increment = 0
 523                                         }
 524                                 }, }
 525                         }, }
 526                 },
 527                 {       /* AES GCM (128-bit) */
 528                         .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 529                         {.sym = {
 530                                 .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
 531                                 {.aead = {
 532                                         .algo = RTE_CRYPTO_AEAD_AES_GCM,
 533                                         .block_size = 16,
 534                                         .key_size = {
 535                                                 .min = 16,
 536                                                 .max = 16,
 537                                                 .increment = 0
 538                                         },
 539                                         .digest_size = {
 540                                                 .min = 16,
 541                                                 .max = 16,
 542                                                 .increment = 0
 543                                         },
 544                                         .aad_size = {
 545                                                 .min = 0,
 546                                                 .max = 65535,
 547                                                 .increment = 1
 548                                         },
 549                                         .iv_size = {
 550                                                 .min = 12,
 551                                                 .max = 12,
 552                                                 .increment = 0
 553                                         }
 554                                 }, }
 555                         }, }
 556                 },
 557                 {
 558                         .op = RTE_CRYPTO_OP_TYPE_UNDEFINED,
 559                         {.sym = {
 560                                 .xform_type = RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED
 561                         }, }
 562                 },
 563         };
 564
 565         static const struct rte_security_capability
 566         ixgbe_security_capabilities[] = {
 567                 { /* IPsec Inline Crypto ESP Transport Egress */
 568                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
 569                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 570                         {.ipsec = {
 571                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 572                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
 573                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
 574                                 .options = { 0 }
 575                         } },
 576                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
 577                         .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
 578                 },
 579                 { /* IPsec Inline Crypto ESP Transport Ingress */
 580                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
 581                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 582                         {.ipsec = {
 583                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 584                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
 585                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
 586                                 .options = { 0 }
 587                         } },
 588                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
 589                         .ol_flags = 0
 590                 },
 591                 { /* IPsec Inline Crypto ESP Tunnel Egress */
 592                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
 593                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 594                         {.ipsec = {
 595                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 596                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
 597                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
 598                                 .options = { 0 }
 599                         } },
 600                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
 601                         .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
 602                 },
 603                 { /* IPsec Inline Crypto ESP Tunnel Ingress */
 604                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
 605                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 606                         {.ipsec = {
 607                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 608                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
 609                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
 610                                 .options = { 0 }
 611                         } },
 612                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
 613                         .ol_flags = 0
 614                 },
 615                 {
 616                         .action = RTE_SECURITY_ACTION_TYPE_NONE
 617                 }
 618         };
 619
 620         return ixgbe_security_capabilities;
 621 }
 622
 623
 624 int
 625 ixgbe_crypto_enable_ipsec(struct rte_eth_dev *dev)
 626 {
 627         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 628         uint32_t reg;
 629         uint64_t rx_offloads;
 630         uint64_t tx_offloads;
 631
 632         rx_offloads = dev->data->dev_conf.rxmode.offloads;
 633         tx_offloads = dev->data->dev_conf.txmode.offloads;
 634
 635         /* sanity checks */
 636         if (rx_offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO) {
 637                 PMD_DRV_LOG(ERR, "RSC and IPsec not supported");
 638                 return -1;
 639         }
 640         if (rx_offloads & RTE_ETH_RX_OFFLOAD_KEEP_CRC) {
 641                 PMD_DRV_LOG(ERR, "HW CRC strip needs to be enabled for IPsec");
 642                 return -1;
 643         }
 644
 645
 646         /* Set IXGBE_SECTXBUFFAF to 0x15 as required in the datasheet*/
 647         IXGBE_WRITE_REG(hw, IXGBE_SECTXBUFFAF, 0x15);
 648
 649         /* IFG needs to be set to 3 when we are using security. Otherwise a Tx
 650          * hang will occur with heavy traffic.
 651          */
 652         reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
 653         reg = (reg & 0xFFFFFFF0) | 0x3;
 654         IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg);
 655
 656         reg  = IXGBE_READ_REG(hw, IXGBE_HLREG0);
 657         reg |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_RXCRCSTRP;
 658         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg);
 659
 660         if (rx_offloads & RTE_ETH_RX_OFFLOAD_SECURITY) {
 661                 IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, 0);
 662                 reg = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL);
 663                 if (reg != 0) {
 664                         PMD_DRV_LOG(ERR, "Error enabling Rx Crypto");
 665                         return -1;
 666                 }
 667         }
 668         if (tx_offloads & RTE_ETH_TX_OFFLOAD_SECURITY) {
 669                 IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL,
 670                                 IXGBE_SECTXCTRL_STORE_FORWARD);
 671                 reg = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL);
 672                 if (reg != IXGBE_SECTXCTRL_STORE_FORWARD) {
 673                         PMD_DRV_LOG(ERR, "Error enabling Rx Crypto");
 674                         return -1;
 675                 }
 676         }
 677
 678         ixgbe_crypto_clear_ipsec_tables(dev);
 679
 680         return 0;
 681 }
 682
 683 int
 684 ixgbe_crypto_add_ingress_sa_from_flow(const void *sess,
 685                                       const void *ip_spec,
 686                                       uint8_t is_ipv6)
 687 {
 688         struct ixgbe_crypto_session *ic_session
 689                 = get_sec_session_private_data(sess);
 690
 691         if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION) {
 692                 if (is_ipv6) {
 693                         const struct rte_flow_item_ipv6 *ipv6 = ip_spec;
 694                         ic_session->src_ip.type = IPv6;
 695                         ic_session->dst_ip.type = IPv6;
 696                         rte_memcpy(ic_session->src_ip.ipv6,
 697                                    ipv6->hdr.src_addr, 16);
 698                         rte_memcpy(ic_session->dst_ip.ipv6,
 699                                    ipv6->hdr.dst_addr, 16);
 700                 } else {
 701                         const struct rte_flow_item_ipv4 *ipv4 = ip_spec;
 702                         ic_session->src_ip.type = IPv4;
 703                         ic_session->dst_ip.type = IPv4;
 704                         ic_session->src_ip.ipv4 = ipv4->hdr.src_addr;
 705                         ic_session->dst_ip.ipv4 = ipv4->hdr.dst_addr;
 706                 }
 707                 return ixgbe_crypto_add_sa(ic_session);
 708         }
 709
 710         return 0;
 711 }
 712
 713 static struct rte_security_ops ixgbe_security_ops = {
 714         .session_create = ixgbe_crypto_create_session,
 715         .session_update = NULL,
 716         .session_get_size = ixgbe_crypto_session_get_size,
 717         .session_stats_get = NULL,
 718         .session_destroy = ixgbe_crypto_remove_session,
 719         .set_pkt_metadata = ixgbe_crypto_update_mb,
 720         .capabilities_get = ixgbe_crypto_capabilities_get
 721 };
 722
 723 static int
 724 ixgbe_crypto_capable(struct rte_eth_dev *dev)
 725 {
 726         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 727         uint32_t reg_i, reg, capable = 1;
 728         /* test if rx crypto can be enabled and then write back initial value*/
 729         reg_i = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL);
 730         IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, 0);
 731         reg = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL);
 732         if (reg != 0)
 733                 capable = 0;
 734         IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, reg_i);
 735         return capable;
 736 }
 737
 738 int
 739 ixgbe_ipsec_ctx_create(struct rte_eth_dev *dev)
 740 {
 741         struct rte_security_ctx *ctx = NULL;
 742
 743         if (ixgbe_crypto_capable(dev)) {
 744                 ctx = rte_malloc("rte_security_instances_ops",
 745                                  sizeof(struct rte_security_ctx), 0);
 746                 if (ctx) {
 747                         ctx->device = (void *)dev;
 748                         ctx->ops = &ixgbe_security_ops;
 749                         ctx->sess_cnt = 0;
 750                         dev->security_ctx = ctx;
 751                 } else {
 752                         return -ENOMEM;
 753                 }
 754         }
 755         if (rte_security_dynfield_register() < 0)
 756                 return -rte_errno;
 757         return 0;
 758 }