drivers/net/ixgbe/ixgbe_ipsec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <rte_ethdev_driver.h>
   6 #include <rte_ethdev_pci.h>
   7 #include <rte_ip.h>
   8 #include <rte_jhash.h>
   9 #include <rte_security_driver.h>
  10 #include <rte_cryptodev.h>
  11 #include <rte_flow.h>
  12
  13 #include "base/ixgbe_type.h"
  14 #include "base/ixgbe_api.h"
  15 #include "ixgbe_ethdev.h"
  16 #include "ixgbe_ipsec.h"
  17
  18 #define RTE_IXGBE_REGISTER_POLL_WAIT_5_MS  5
  19
  20 #define IXGBE_WAIT_RREAD \
  21         IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSRXIDX, reg_val, \
  22         IPSRXIDX_READ, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS)
  23 #define IXGBE_WAIT_RWRITE \
  24         IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSRXIDX, reg_val, \
  25         IPSRXIDX_WRITE, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS)
  26 #define IXGBE_WAIT_TREAD \
  27         IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSTXIDX, reg_val, \
  28         IPSRXIDX_READ, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS)
  29 #define IXGBE_WAIT_TWRITE \
  30         IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSTXIDX, reg_val, \
  31         IPSRXIDX_WRITE, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS)
  32
  33 #define CMP_IP(a, b) (\
  34         (a).ipv6[0] == (b).ipv6[0] && \
  35         (a).ipv6[1] == (b).ipv6[1] && \
  36         (a).ipv6[2] == (b).ipv6[2] && \
  37         (a).ipv6[3] == (b).ipv6[3])
  38
  39
  40 static void
  41 ixgbe_crypto_clear_ipsec_tables(struct rte_eth_dev *dev)
  42 {
  43         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
  44         struct ixgbe_ipsec *priv = IXGBE_DEV_PRIVATE_TO_IPSEC(
  45                                 dev->data->dev_private);
  46         int i = 0;
  47
  48         /* clear Rx IP table*/
  49         for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
  50                 uint16_t index = i << 3;
  51                 uint32_t reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_IP | index;
  52                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0), 0);
  53                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1), 0);
  54                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2), 0);
  55                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3), 0);
  56                 IXGBE_WAIT_RWRITE;
  57         }
  58
  59         /* clear Rx SPI and Rx/Tx SA tables*/
  60         for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
  61                 uint32_t index = i << 3;
  62                 uint32_t reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_SPI | index;
  63                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSPI, 0);
  64                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPIDX, 0);
  65                 IXGBE_WAIT_RWRITE;
  66                 reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_KEY | index;
  67                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(0), 0);
  68                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(1), 0);
  69                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(2), 0);
  70                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(3), 0);
  71                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSALT, 0);
  72                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXMOD, 0);
  73                 IXGBE_WAIT_RWRITE;
  74                 reg_val = IPSRXIDX_WRITE | index;
  75                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(0), 0);
  76                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(1), 0);
  77                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(2), 0);
  78                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(3), 0);
  79                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXSALT, 0);
  80                 IXGBE_WAIT_TWRITE;
  81         }
  82
  83         memset(priv->rx_ip_tbl, 0, sizeof(priv->rx_ip_tbl));
  84         memset(priv->rx_sa_tbl, 0, sizeof(priv->rx_sa_tbl));
  85         memset(priv->tx_sa_tbl, 0, sizeof(priv->tx_sa_tbl));
  86 }
  87
  88 static int
  89 ixgbe_crypto_add_sa(struct ixgbe_crypto_session *ic_session)
  90 {
  91         struct rte_eth_dev *dev = ic_session->dev;
  92         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
  93         struct ixgbe_ipsec *priv = IXGBE_DEV_PRIVATE_TO_IPSEC(
  94                         dev->data->dev_private);
  95         uint32_t reg_val;
  96         int sa_index = -1;
  97
  98         if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION) {
  99                 int i, ip_index = -1;
 100                 uint8_t *key;
 101
 102                 /* Find a match in the IP table*/
 103                 for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
 104                         if (CMP_IP(priv->rx_ip_tbl[i].ip,
 105                                    ic_session->dst_ip)) {
 106                                 ip_index = i;
 107                                 break;
 108                         }
 109                 }
 110                 /* If no match, find a free entry in the IP table*/
 111                 if (ip_index < 0) {
 112                         for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
 113                                 if (priv->rx_ip_tbl[i].ref_count == 0) {
 114                                         ip_index = i;
 115                                         break;
 116                                 }
 117                         }
 118                 }
 119
 120                 /* Fail if no match and no free entries*/
 121                 if (ip_index < 0) {
 122                         PMD_DRV_LOG(ERR,
 123                                     "No free entry left in the Rx IP table\n");
 124                         return -1;
 125                 }
 126
 127                 /* Find a free entry in the SA table*/
 128                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
 129                         if (priv->rx_sa_tbl[i].used == 0) {
 130                                 sa_index = i;
 131                                 break;
 132                         }
 133                 }
 134                 /* Fail if no free entries*/
 135                 if (sa_index < 0) {
 136                         PMD_DRV_LOG(ERR,
 137                                     "No free entry left in the Rx SA table\n");
 138                         return -1;
 139                 }
 140
 141                 priv->rx_ip_tbl[ip_index].ip.ipv6[0] =
 142                                 ic_session->dst_ip.ipv6[0];
 143                 priv->rx_ip_tbl[ip_index].ip.ipv6[1] =
 144                                 ic_session->dst_ip.ipv6[1];
 145                 priv->rx_ip_tbl[ip_index].ip.ipv6[2] =
 146                                 ic_session->dst_ip.ipv6[2];
 147                 priv->rx_ip_tbl[ip_index].ip.ipv6[3] =
 148                                 ic_session->dst_ip.ipv6[3];
 149                 priv->rx_ip_tbl[ip_index].ref_count++;
 150
 151                 priv->rx_sa_tbl[sa_index].spi =
 152                         rte_cpu_to_be_32(ic_session->spi);
 153                 priv->rx_sa_tbl[sa_index].ip_index = ip_index;
 154                 priv->rx_sa_tbl[sa_index].mode = IPSRXMOD_VALID;
 155                 if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION)
 156                         priv->rx_sa_tbl[sa_index].mode |=
 157                                         (IPSRXMOD_PROTO | IPSRXMOD_DECRYPT);
 158                 if (ic_session->dst_ip.type == IPv6) {
 159                         priv->rx_sa_tbl[sa_index].mode |= IPSRXMOD_IPV6;
 160                         priv->rx_ip_tbl[ip_index].ip.type = IPv6;
 161                 } else if (ic_session->dst_ip.type == IPv4)
 162                         priv->rx_ip_tbl[ip_index].ip.type = IPv4;
 163
 164                 priv->rx_sa_tbl[sa_index].used = 1;
 165
 166                 /* write IP table entry*/
 167                 reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE |
 168                                 IPSRXIDX_TABLE_IP | (ip_index << 3);
 169                 if (priv->rx_ip_tbl[ip_index].ip.type == IPv4) {
 170                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0), 0);
 171                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1), 0);
 172                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2), 0);
 173                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3),
 174                                         priv->rx_ip_tbl[ip_index].ip.ipv4);
 175                 } else {
 176                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0),
 177                                         priv->rx_ip_tbl[ip_index].ip.ipv6[0]);
 178                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1),
 179                                         priv->rx_ip_tbl[ip_index].ip.ipv6[1]);
 180                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2),
 181                                         priv->rx_ip_tbl[ip_index].ip.ipv6[2]);
 182                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3),
 183                                         priv->rx_ip_tbl[ip_index].ip.ipv6[3]);
 184                 }
 185                 IXGBE_WAIT_RWRITE;
 186
 187                 /* write SPI table entry*/
 188                 reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE |
 189                                 IPSRXIDX_TABLE_SPI | (sa_index << 3);
 190                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSPI,
 191                                 priv->rx_sa_tbl[sa_index].spi);
 192                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPIDX,
 193                                 priv->rx_sa_tbl[sa_index].ip_index);
 194                 IXGBE_WAIT_RWRITE;
 195
 196                 /* write Key table entry*/
 197                 key = malloc(ic_session->key_len);
 198                 if (!key)
 199                         return -ENOMEM;
 200
 201                 memcpy(key, ic_session->key, ic_session->key_len);
 202
 203                 reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE |
 204                                 IPSRXIDX_TABLE_KEY | (sa_index << 3);
 205                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(0),
 206                         rte_cpu_to_be_32(*(uint32_t *)&key[12]));
 207                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(1),
 208                         rte_cpu_to_be_32(*(uint32_t *)&key[8]));
 209                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(2),
 210                         rte_cpu_to_be_32(*(uint32_t *)&key[4]));
 211                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(3),
 212                         rte_cpu_to_be_32(*(uint32_t *)&key[0]));
 213                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSALT,
 214                                 rte_cpu_to_be_32(ic_session->salt));
 215                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXMOD,
 216                                 priv->rx_sa_tbl[sa_index].mode);
 217                 IXGBE_WAIT_RWRITE;
 218
 219                 free(key);
 220
 221         } else { /* sess->dir == RTE_CRYPTO_OUTBOUND */
 222                 uint8_t *key;
 223                 int i;
 224
 225                 /* Find a free entry in the SA table*/
 226                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
 227                         if (priv->tx_sa_tbl[i].used == 0) {
 228                                 sa_index = i;
 229                                 break;
 230                         }
 231                 }
 232                 /* Fail if no free entries*/
 233                 if (sa_index < 0) {
 234                         PMD_DRV_LOG(ERR,
 235                                     "No free entry left in the Tx SA table\n");
 236                         return -1;
 237                 }
 238
 239                 priv->tx_sa_tbl[sa_index].spi =
 240                         rte_cpu_to_be_32(ic_session->spi);
 241                 priv->tx_sa_tbl[i].used = 1;
 242                 ic_session->sa_index = sa_index;
 243
 244                 key = malloc(ic_session->key_len);
 245                 if (!key)
 246                         return -ENOMEM;
 247
 248                 memcpy(key, ic_session->key, ic_session->key_len);
 249
 250                 /* write Key table entry*/
 251                 reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE | (sa_index << 3);
 252                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(0),
 253                         rte_cpu_to_be_32(*(uint32_t *)&key[12]));
 254                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(1),
 255                         rte_cpu_to_be_32(*(uint32_t *)&key[8]));
 256                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(2),
 257                         rte_cpu_to_be_32(*(uint32_t *)&key[4]));
 258                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(3),
 259                         rte_cpu_to_be_32(*(uint32_t *)&key[0]));
 260                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXSALT,
 261                                 rte_cpu_to_be_32(ic_session->salt));
 262                 IXGBE_WAIT_TWRITE;
 263
 264                 free(key);
 265         }
 266
 267         return 0;
 268 }
 269
 270 static int
 271 ixgbe_crypto_remove_sa(struct rte_eth_dev *dev,
 272                        struct ixgbe_crypto_session *ic_session)
 273 {
 274         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 275         struct ixgbe_ipsec *priv =
 276                         IXGBE_DEV_PRIVATE_TO_IPSEC(dev->data->dev_private);
 277         uint32_t reg_val;
 278         int sa_index = -1;
 279
 280         if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION) {
 281                 int i, ip_index = -1;
 282
 283                 /* Find a match in the IP table*/
 284                 for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
 285                         if (CMP_IP(priv->rx_ip_tbl[i].ip, ic_session->dst_ip)) {
 286                                 ip_index = i;
 287                                 break;
 288                         }
 289                 }
 290
 291                 /* Fail if no match*/
 292                 if (ip_index < 0) {
 293                         PMD_DRV_LOG(ERR,
 294                                     "Entry not found in the Rx IP table\n");
 295                         return -1;
 296                 }
 297
 298                 /* Find a free entry in the SA table*/
 299                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
 300                         if (priv->rx_sa_tbl[i].spi ==
 301                                   rte_cpu_to_be_32(ic_session->spi)) {
 302                                 sa_index = i;
 303                                 break;
 304                         }
 305                 }
 306                 /* Fail if no match*/
 307                 if (sa_index < 0) {
 308                         PMD_DRV_LOG(ERR,
 309                                     "Entry not found in the Rx SA table\n");
 310                         return -1;
 311                 }
 312
 313                 /* Disable and clear Rx SPI and key table table entryes*/
 314                 reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_SPI | (sa_index << 3);
 315                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSPI, 0);
 316                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPIDX, 0);
 317                 IXGBE_WAIT_RWRITE;
 318                 reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_KEY | (sa_index << 3);
 319                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(0), 0);
 320                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(1), 0);
 321                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(2), 0);
 322                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(3), 0);
 323                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSALT, 0);
 324                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXMOD, 0);
 325                 IXGBE_WAIT_RWRITE;
 326                 priv->rx_sa_tbl[sa_index].used = 0;
 327
 328                 /* If last used then clear the IP table entry*/
 329                 priv->rx_ip_tbl[ip_index].ref_count--;
 330                 if (priv->rx_ip_tbl[ip_index].ref_count == 0) {
 331                         reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_IP |
 332                                         (ip_index << 3);
 333                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0), 0);
 334                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1), 0);
 335                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2), 0);
 336                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3), 0);
 337                 }
 338         } else { /* session->dir == RTE_CRYPTO_OUTBOUND */
 339                 int i;
 340
 341                 /* Find a match in the SA table*/
 342                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
 343                         if (priv->tx_sa_tbl[i].spi ==
 344                                     rte_cpu_to_be_32(ic_session->spi)) {
 345                                 sa_index = i;
 346                                 break;
 347                         }
 348                 }
 349                 /* Fail if no match entries*/
 350                 if (sa_index < 0) {
 351                         PMD_DRV_LOG(ERR,
 352                                     "Entry not found in the Tx SA table\n");
 353                         return -1;
 354                 }
 355                 reg_val = IPSRXIDX_WRITE | (sa_index << 3);
 356                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(0), 0);
 357                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(1), 0);
 358                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(2), 0);
 359                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(3), 0);
 360                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXSALT, 0);
 361                 IXGBE_WAIT_TWRITE;
 362
 363                 priv->tx_sa_tbl[sa_index].used = 0;
 364         }
 365
 366         return 0;
 367 }
 368
 369 static int
 370 ixgbe_crypto_create_session(void *device,
 371                 struct rte_security_session_conf *conf,
 372                 struct rte_security_session *session,
 373                 struct rte_mempool *mempool)
 374 {
 375         struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)device;
 376         struct ixgbe_crypto_session *ic_session = NULL;
 377         struct rte_crypto_aead_xform *aead_xform;
 378         struct rte_eth_conf *dev_conf = &eth_dev->data->dev_conf;
 379
 380         if (rte_mempool_get(mempool, (void **)&ic_session)) {
 381                 PMD_DRV_LOG(ERR, "Cannot get object from ic_session mempool");
 382                 return -ENOMEM;
 383         }
 384
 385         if (conf->crypto_xform->type != RTE_CRYPTO_SYM_XFORM_AEAD ||
 386                         conf->crypto_xform->aead.algo !=
 387                                         RTE_CRYPTO_AEAD_AES_GCM) {
 388                 PMD_DRV_LOG(ERR, "Unsupported crypto transformation mode\n");
 389                 rte_mempool_put(mempool, (void *)ic_session);
 390                 return -ENOTSUP;
 391         }
 392         aead_xform = &conf->crypto_xform->aead;
 393
 394         if (conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
 395                 if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_SECURITY) {
 396                         ic_session->op = IXGBE_OP_AUTHENTICATED_DECRYPTION;
 397                 } else {
 398                         PMD_DRV_LOG(ERR, "IPsec decryption not enabled\n");
 399                         rte_mempool_put(mempool, (void *)ic_session);
 400                         return -ENOTSUP;
 401                 }
 402         } else {
 403                 if (dev_conf->txmode.offloads & DEV_TX_OFFLOAD_SECURITY) {
 404                         ic_session->op = IXGBE_OP_AUTHENTICATED_ENCRYPTION;
 405                 } else {
 406                         PMD_DRV_LOG(ERR, "IPsec encryption not enabled\n");
 407                         rte_mempool_put(mempool, (void *)ic_session);
 408                         return -ENOTSUP;
 409                 }
 410         }
 411
 412         ic_session->key = aead_xform->key.data;
 413         ic_session->key_len = aead_xform->key.length;
 414         memcpy(&ic_session->salt,
 415                &aead_xform->key.data[aead_xform->key.length], 4);
 416         ic_session->spi = conf->ipsec.spi;
 417         ic_session->dev = eth_dev;
 418
 419         set_sec_session_private_data(session, ic_session);
 420
 421         if (ic_session->op == IXGBE_OP_AUTHENTICATED_ENCRYPTION) {
 422                 if (ixgbe_crypto_add_sa(ic_session)) {
 423                         PMD_DRV_LOG(ERR, "Failed to add SA\n");
 424                         rte_mempool_put(mempool, (void *)ic_session);
 425                         return -EPERM;
 426                 }
 427         }
 428
 429         return 0;
 430 }
 431
 432 static unsigned int
 433 ixgbe_crypto_session_get_size(__rte_unused void *device)
 434 {
 435         return sizeof(struct ixgbe_crypto_session);
 436 }
 437
 438 static int
 439 ixgbe_crypto_remove_session(void *device,
 440                 struct rte_security_session *session)
 441 {
 442         struct rte_eth_dev *eth_dev = device;
 443         struct ixgbe_crypto_session *ic_session =
 444                 (struct ixgbe_crypto_session *)
 445                 get_sec_session_private_data(session);
 446         struct rte_mempool *mempool = rte_mempool_from_obj(ic_session);
 447
 448         if (eth_dev != ic_session->dev) {
 449                 PMD_DRV_LOG(ERR, "Session not bound to this device\n");
 450                 return -ENODEV;
 451         }
 452
 453         if (ixgbe_crypto_remove_sa(eth_dev, ic_session)) {
 454                 PMD_DRV_LOG(ERR, "Failed to remove session\n");
 455                 return -EFAULT;
 456         }
 457
 458         rte_mempool_put(mempool, (void *)ic_session);
 459
 460         return 0;
 461 }
 462
 463 static inline uint8_t
 464 ixgbe_crypto_compute_pad_len(struct rte_mbuf *m)
 465 {
 466         if (m->nb_segs == 1) {
 467                 /* 16 bytes ICV + 2 bytes ESP trailer + payload padding size
 468                  * payload padding size is stored at <pkt_len - 18>
 469                  */
 470                 uint8_t *esp_pad_len = rte_pktmbuf_mtod_offset(m, uint8_t *,
 471                                         rte_pktmbuf_pkt_len(m) -
 472                                         (ESP_TRAILER_SIZE + ESP_ICV_SIZE));
 473                 return *esp_pad_len + ESP_TRAILER_SIZE + ESP_ICV_SIZE;
 474         }
 475         return 0;
 476 }
 477
 478 static int
 479 ixgbe_crypto_update_mb(void *device __rte_unused,
 480                 struct rte_security_session *session,
 481                        struct rte_mbuf *m, void *params __rte_unused)
 482 {
 483         struct ixgbe_crypto_session *ic_session =
 484                         get_sec_session_private_data(session);
 485         if (ic_session->op == IXGBE_OP_AUTHENTICATED_ENCRYPTION) {
 486                 union ixgbe_crypto_tx_desc_md *mdata =
 487                         (union ixgbe_crypto_tx_desc_md *)&m->udata64;
 488                 mdata->enc = 1;
 489                 mdata->sa_idx = ic_session->sa_index;
 490                 mdata->pad_len = ixgbe_crypto_compute_pad_len(m);
 491         }
 492         return 0;
 493 }
 494
 495
 496 static const struct rte_security_capability *
 497 ixgbe_crypto_capabilities_get(void *device __rte_unused)
 498 {
 499         static const struct rte_cryptodev_capabilities
 500         aes_gcm_gmac_crypto_capabilities[] = {
 501                 {       /* AES GMAC (128-bit) */
 502                         .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 503                         {.sym = {
 504                                 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 505                                 {.auth = {
 506                                         .algo = RTE_CRYPTO_AUTH_AES_GMAC,
 507                                         .block_size = 16,
 508                                         .key_size = {
 509                                                 .min = 16,
 510                                                 .max = 16,
 511                                                 .increment = 0
 512                                         },
 513                                         .digest_size = {
 514                                                 .min = 16,
 515                                                 .max = 16,
 516                                                 .increment = 0
 517                                         },
 518                                         .iv_size = {
 519                                                 .min = 12,
 520                                                 .max = 12,
 521                                                 .increment = 0
 522                                         }
 523                                 }, }
 524                         }, }
 525                 },
 526                 {       /* AES GCM (128-bit) */
 527                         .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 528                         {.sym = {
 529                                 .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
 530                                 {.aead = {
 531                                         .algo = RTE_CRYPTO_AEAD_AES_GCM,
 532                                         .block_size = 16,
 533                                         .key_size = {
 534                                                 .min = 16,
 535                                                 .max = 16,
 536                                                 .increment = 0
 537                                         },
 538                                         .digest_size = {
 539                                                 .min = 16,
 540                                                 .max = 16,
 541                                                 .increment = 0
 542                                         },
 543                                         .aad_size = {
 544                                                 .min = 0,
 545                                                 .max = 65535,
 546                                                 .increment = 1
 547                                         },
 548                                         .iv_size = {
 549                                                 .min = 12,
 550                                                 .max = 12,
 551                                                 .increment = 0
 552                                         }
 553                                 }, }
 554                         }, }
 555                 },
 556                 {
 557                         .op = RTE_CRYPTO_OP_TYPE_UNDEFINED,
 558                         {.sym = {
 559                                 .xform_type = RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED
 560                         }, }
 561                 },
 562         };
 563
 564         static const struct rte_security_capability
 565         ixgbe_security_capabilities[] = {
 566                 { /* IPsec Inline Crypto ESP Transport Egress */
 567                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
 568                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 569                         {.ipsec = {
 570                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 571                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
 572                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
 573                                 .options = { 0 }
 574                         } },
 575                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
 576                         .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
 577                 },
 578                 { /* IPsec Inline Crypto ESP Transport Ingress */
 579                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
 580                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 581                         {.ipsec = {
 582                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 583                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
 584                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
 585                                 .options = { 0 }
 586                         } },
 587                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
 588                         .ol_flags = 0
 589                 },
 590                 { /* IPsec Inline Crypto ESP Tunnel Egress */
 591                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
 592                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 593                         {.ipsec = {
 594                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 595                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
 596                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
 597                                 .options = { 0 }
 598                         } },
 599                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
 600                         .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
 601                 },
 602                 { /* IPsec Inline Crypto ESP Tunnel Ingress */
 603                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
 604                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 605                         {.ipsec = {
 606                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 607                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
 608                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
 609                                 .options = { 0 }
 610                         } },
 611                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
 612                         .ol_flags = 0
 613                 },
 614                 {
 615                         .action = RTE_SECURITY_ACTION_TYPE_NONE
 616                 }
 617         };
 618
 619         return ixgbe_security_capabilities;
 620 }
 621
 622
 623 int
 624 ixgbe_crypto_enable_ipsec(struct rte_eth_dev *dev)
 625 {
 626         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 627         uint32_t reg;
 628         uint64_t rx_offloads;
 629         uint64_t tx_offloads;
 630
 631         rx_offloads = dev->data->dev_conf.rxmode.offloads;
 632         tx_offloads = dev->data->dev_conf.txmode.offloads;
 633
 634         /* sanity checks */
 635         if (rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) {
 636                 PMD_DRV_LOG(ERR, "RSC and IPsec not supported");
 637                 return -1;
 638         }
 639         if (rx_offloads & DEV_RX_OFFLOAD_KEEP_CRC) {
 640                 PMD_DRV_LOG(ERR, "HW CRC strip needs to be enabled for IPsec");
 641                 return -1;
 642         }
 643
 644
 645         /* Set IXGBE_SECTXBUFFAF to 0x15 as required in the datasheet*/
 646         IXGBE_WRITE_REG(hw, IXGBE_SECTXBUFFAF, 0x15);
 647
 648         /* IFG needs to be set to 3 when we are using security. Otherwise a Tx
 649          * hang will occur with heavy traffic.
 650          */
 651         reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
 652         reg = (reg & 0xFFFFFFF0) | 0x3;
 653         IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg);
 654
 655         reg  = IXGBE_READ_REG(hw, IXGBE_HLREG0);
 656         reg |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_RXCRCSTRP;
 657         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg);
 658
 659         if (rx_offloads & DEV_RX_OFFLOAD_SECURITY) {
 660                 IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, 0);
 661                 reg = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL);
 662                 if (reg != 0) {
 663                         PMD_DRV_LOG(ERR, "Error enabling Rx Crypto");
 664                         return -1;
 665                 }
 666         }
 667         if (tx_offloads & DEV_TX_OFFLOAD_SECURITY) {
 668                 IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL,
 669                                 IXGBE_SECTXCTRL_STORE_FORWARD);
 670                 reg = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL);
 671                 if (reg != IXGBE_SECTXCTRL_STORE_FORWARD) {
 672                         PMD_DRV_LOG(ERR, "Error enabling Rx Crypto");
 673                         return -1;
 674                 }
 675         }
 676
 677         ixgbe_crypto_clear_ipsec_tables(dev);
 678
 679         return 0;
 680 }
 681
 682 int
 683 ixgbe_crypto_add_ingress_sa_from_flow(const void *sess,
 684                                       const void *ip_spec,
 685                                       uint8_t is_ipv6)
 686 {
 687         struct ixgbe_crypto_session *ic_session
 688                 = get_sec_session_private_data(sess);
 689
 690         if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION) {
 691                 if (is_ipv6) {
 692                         const struct rte_flow_item_ipv6 *ipv6 = ip_spec;
 693                         ic_session->src_ip.type = IPv6;
 694                         ic_session->dst_ip.type = IPv6;
 695                         rte_memcpy(ic_session->src_ip.ipv6,
 696                                    ipv6->hdr.src_addr, 16);
 697                         rte_memcpy(ic_session->dst_ip.ipv6,
 698                                    ipv6->hdr.dst_addr, 16);
 699                 } else {
 700                         const struct rte_flow_item_ipv4 *ipv4 = ip_spec;
 701                         ic_session->src_ip.type = IPv4;
 702                         ic_session->dst_ip.type = IPv4;
 703                         ic_session->src_ip.ipv4 = ipv4->hdr.src_addr;
 704                         ic_session->dst_ip.ipv4 = ipv4->hdr.dst_addr;
 705                 }
 706                 return ixgbe_crypto_add_sa(ic_session);
 707         }
 708
 709         return 0;
 710 }
 711
 712 static struct rte_security_ops ixgbe_security_ops = {
 713         .session_create = ixgbe_crypto_create_session,
 714         .session_update = NULL,
 715         .session_get_size = ixgbe_crypto_session_get_size,
 716         .session_stats_get = NULL,
 717         .session_destroy = ixgbe_crypto_remove_session,
 718         .set_pkt_metadata = ixgbe_crypto_update_mb,
 719         .capabilities_get = ixgbe_crypto_capabilities_get
 720 };
 721
 722 static int
 723 ixgbe_crypto_capable(struct rte_eth_dev *dev)
 724 {
 725         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
 726         uint32_t reg_i, reg, capable = 1;
 727         /* test if rx crypto can be enabled and then write back initial value*/
 728         reg_i = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL);
 729         IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, 0);
 730         reg = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL);
 731         if (reg != 0)
 732                 capable = 0;
 733         IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, reg_i);
 734         return capable;
 735 }
 736
 737 int
 738 ixgbe_ipsec_ctx_create(struct rte_eth_dev *dev)
 739 {
 740         struct rte_security_ctx *ctx = NULL;
 741
 742         if (ixgbe_crypto_capable(dev)) {
 743                 ctx = rte_malloc("rte_security_instances_ops",
 744                                  sizeof(struct rte_security_ctx), 0);
 745                 if (ctx) {
 746                         ctx->device = (void *)dev;
 747                         ctx->ops = &ixgbe_security_ops;
 748                         ctx->sess_cnt = 0;
 749                         dev->security_ctx = ctx;
 750                 } else {
 751                         return -ENOMEM;
 752                 }
 753         }
 754         return 0;
 755 }