drivers/net/txgbe/txgbe_ipsec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2020 Beijing WangXun Technology Co., Ltd.
   3  * Copyright(c) 2010-2017 Intel Corporation
   4  */
   5
   6 #include <ethdev_pci.h>
   7 #include <rte_security_driver.h>
   8 #include <rte_cryptodev.h>
   9
  10 #include "base/txgbe.h"
  11 #include "txgbe_ethdev.h"
  12 #include "txgbe_ipsec.h"
  13
  14 #define CMP_IP(a, b) (\
  15         (a).ipv6[0] == (b).ipv6[0] && \
  16         (a).ipv6[1] == (b).ipv6[1] && \
  17         (a).ipv6[2] == (b).ipv6[2] && \
  18         (a).ipv6[3] == (b).ipv6[3])
  19
  20 static void
  21 txgbe_crypto_clear_ipsec_tables(struct rte_eth_dev *dev)
  22 {
  23         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
  24         struct txgbe_ipsec *priv = TXGBE_DEV_IPSEC(dev);
  25         int i = 0;
  26
  27         /* clear Rx IP table*/
  28         for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
  29                 uint16_t index = i << 3;
  30                 uint32_t reg_val = TXGBE_IPSRXIDX_WRITE |
  31                                 TXGBE_IPSRXIDX_TB_IP | index;
  32                 wr32(hw, TXGBE_IPSRXADDR(0), 0);
  33                 wr32(hw, TXGBE_IPSRXADDR(1), 0);
  34                 wr32(hw, TXGBE_IPSRXADDR(2), 0);
  35                 wr32(hw, TXGBE_IPSRXADDR(3), 0);
  36                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
  37         }
  38
  39         /* clear Rx SPI and Rx/Tx SA tables*/
  40         for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
  41                 uint32_t index = i << 3;
  42                 uint32_t reg_val = TXGBE_IPSRXIDX_WRITE |
  43                                 TXGBE_IPSRXIDX_TB_SPI | index;
  44                 wr32(hw, TXGBE_IPSRXSPI, 0);
  45                 wr32(hw, TXGBE_IPSRXADDRIDX, 0);
  46                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
  47                 reg_val = TXGBE_IPSRXIDX_WRITE | TXGBE_IPSRXIDX_TB_KEY | index;
  48                 wr32(hw, TXGBE_IPSRXKEY(0), 0);
  49                 wr32(hw, TXGBE_IPSRXKEY(1), 0);
  50                 wr32(hw, TXGBE_IPSRXKEY(2), 0);
  51                 wr32(hw, TXGBE_IPSRXKEY(3), 0);
  52                 wr32(hw, TXGBE_IPSRXSALT, 0);
  53                 wr32(hw, TXGBE_IPSRXMODE, 0);
  54                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
  55                 reg_val = TXGBE_IPSTXIDX_WRITE | index;
  56                 wr32(hw, TXGBE_IPSTXKEY(0), 0);
  57                 wr32(hw, TXGBE_IPSTXKEY(1), 0);
  58                 wr32(hw, TXGBE_IPSTXKEY(2), 0);
  59                 wr32(hw, TXGBE_IPSTXKEY(3), 0);
  60                 wr32(hw, TXGBE_IPSTXSALT, 0);
  61                 wr32w(hw, TXGBE_IPSTXIDX, reg_val, TXGBE_IPSTXIDX_WRITE, 1000);
  62         }
  63
  64         memset(priv->rx_ip_tbl, 0, sizeof(priv->rx_ip_tbl));
  65         memset(priv->rx_sa_tbl, 0, sizeof(priv->rx_sa_tbl));
  66         memset(priv->tx_sa_tbl, 0, sizeof(priv->tx_sa_tbl));
  67 }
  68
  69 static int
  70 txgbe_crypto_add_sa(struct txgbe_crypto_session *ic_session)
  71 {
  72         struct rte_eth_dev *dev = ic_session->dev;
  73         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
  74         struct txgbe_ipsec *priv = TXGBE_DEV_IPSEC(dev);
  75         uint32_t reg_val;
  76         int sa_index = -1;
  77
  78         if (ic_session->op == TXGBE_OP_AUTHENTICATED_DECRYPTION) {
  79                 int i, ip_index = -1;
  80                 uint8_t *key;
  81
  82                 /* Find a match in the IP table*/
  83                 for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
  84                         if (CMP_IP(priv->rx_ip_tbl[i].ip,
  85                                    ic_session->dst_ip)) {
  86                                 ip_index = i;
  87                                 break;
  88                         }
  89                 }
  90                 /* If no match, find a free entry in the IP table*/
  91                 if (ip_index < 0) {
  92                         for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
  93                                 if (priv->rx_ip_tbl[i].ref_count == 0) {
  94                                         ip_index = i;
  95                                         break;
  96                                 }
  97                         }
  98                 }
  99
 100                 /* Fail if no match and no free entries*/
 101                 if (ip_index < 0) {
 102                         PMD_DRV_LOG(ERR,
 103                                     "No free entry left in the Rx IP table\n");
 104                         return -1;
 105                 }
 106
 107                 /* Find a free entry in the SA table*/
 108                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
 109                         if (priv->rx_sa_tbl[i].used == 0) {
 110                                 sa_index = i;
 111                                 break;
 112                         }
 113                 }
 114                 /* Fail if no free entries*/
 115                 if (sa_index < 0) {
 116                         PMD_DRV_LOG(ERR,
 117                                     "No free entry left in the Rx SA table\n");
 118                         return -1;
 119                 }
 120
 121                 priv->rx_ip_tbl[ip_index].ip.ipv6[0] =
 122                                 ic_session->dst_ip.ipv6[0];
 123                 priv->rx_ip_tbl[ip_index].ip.ipv6[1] =
 124                                 ic_session->dst_ip.ipv6[1];
 125                 priv->rx_ip_tbl[ip_index].ip.ipv6[2] =
 126                                 ic_session->dst_ip.ipv6[2];
 127                 priv->rx_ip_tbl[ip_index].ip.ipv6[3] =
 128                                 ic_session->dst_ip.ipv6[3];
 129                 priv->rx_ip_tbl[ip_index].ref_count++;
 130
 131                 priv->rx_sa_tbl[sa_index].spi = ic_session->spi;
 132                 priv->rx_sa_tbl[sa_index].ip_index = ip_index;
 133                 priv->rx_sa_tbl[sa_index].mode = IPSRXMOD_VALID;
 134                 if (ic_session->op == TXGBE_OP_AUTHENTICATED_DECRYPTION)
 135                         priv->rx_sa_tbl[sa_index].mode |=
 136                                         (IPSRXMOD_PROTO | IPSRXMOD_DECRYPT);
 137                 if (ic_session->dst_ip.type == IPv6) {
 138                         priv->rx_sa_tbl[sa_index].mode |= IPSRXMOD_IPV6;
 139                         priv->rx_ip_tbl[ip_index].ip.type = IPv6;
 140                 } else if (ic_session->dst_ip.type == IPv4) {
 141                         priv->rx_ip_tbl[ip_index].ip.type = IPv4;
 142                 }
 143                 priv->rx_sa_tbl[sa_index].used = 1;
 144
 145                 /* write IP table entry*/
 146                 reg_val = TXGBE_IPSRXIDX_ENA | TXGBE_IPSRXIDX_WRITE |
 147                                 TXGBE_IPSRXIDX_TB_IP | (ip_index << 3);
 148                 if (priv->rx_ip_tbl[ip_index].ip.type == IPv4) {
 149                         uint32_t ipv4 = priv->rx_ip_tbl[ip_index].ip.ipv4;
 150                         wr32(hw, TXGBE_IPSRXADDR(0), rte_cpu_to_be_32(ipv4));
 151                         wr32(hw, TXGBE_IPSRXADDR(1), 0);
 152                         wr32(hw, TXGBE_IPSRXADDR(2), 0);
 153                         wr32(hw, TXGBE_IPSRXADDR(3), 0);
 154                 } else {
 155                         wr32(hw, TXGBE_IPSRXADDR(0),
 156                                         priv->rx_ip_tbl[ip_index].ip.ipv6[0]);
 157                         wr32(hw, TXGBE_IPSRXADDR(1),
 158                                         priv->rx_ip_tbl[ip_index].ip.ipv6[1]);
 159                         wr32(hw, TXGBE_IPSRXADDR(2),
 160                                         priv->rx_ip_tbl[ip_index].ip.ipv6[2]);
 161                         wr32(hw, TXGBE_IPSRXADDR(3),
 162                                         priv->rx_ip_tbl[ip_index].ip.ipv6[3]);
 163                 }
 164                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
 165
 166                 /* write SPI table entry*/
 167                 reg_val = TXGBE_IPSRXIDX_ENA | TXGBE_IPSRXIDX_WRITE |
 168                                 TXGBE_IPSRXIDX_TB_SPI | (sa_index << 3);
 169                 wr32(hw, TXGBE_IPSRXSPI,
 170                                 priv->rx_sa_tbl[sa_index].spi);
 171                 wr32(hw, TXGBE_IPSRXADDRIDX,
 172                                 priv->rx_sa_tbl[sa_index].ip_index);
 173                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
 174
 175                 /* write Key table entry*/
 176                 key = malloc(ic_session->key_len);
 177                 if (!key)
 178                         return -ENOMEM;
 179
 180                 memcpy(key, ic_session->key, ic_session->key_len);
 181
 182                 reg_val = TXGBE_IPSRXIDX_ENA | TXGBE_IPSRXIDX_WRITE |
 183                                 TXGBE_IPSRXIDX_TB_KEY | (sa_index << 3);
 184                 wr32(hw, TXGBE_IPSRXKEY(0),
 185                         rte_cpu_to_be_32(*(uint32_t *)&key[12]));
 186                 wr32(hw, TXGBE_IPSRXKEY(1),
 187                         rte_cpu_to_be_32(*(uint32_t *)&key[8]));
 188                 wr32(hw, TXGBE_IPSRXKEY(2),
 189                         rte_cpu_to_be_32(*(uint32_t *)&key[4]));
 190                 wr32(hw, TXGBE_IPSRXKEY(3),
 191                         rte_cpu_to_be_32(*(uint32_t *)&key[0]));
 192                 wr32(hw, TXGBE_IPSRXSALT,
 193                                 rte_cpu_to_be_32(ic_session->salt));
 194                 wr32(hw, TXGBE_IPSRXMODE,
 195                                 priv->rx_sa_tbl[sa_index].mode);
 196                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
 197
 198                 free(key);
 199         } else { /* sess->dir == RTE_CRYPTO_OUTBOUND */
 200                 uint8_t *key;
 201                 int i;
 202
 203                 /* Find a free entry in the SA table*/
 204                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
 205                         if (priv->tx_sa_tbl[i].used == 0) {
 206                                 sa_index = i;
 207                                 break;
 208                         }
 209                 }
 210                 /* Fail if no free entries*/
 211                 if (sa_index < 0) {
 212                         PMD_DRV_LOG(ERR,
 213                                     "No free entry left in the Tx SA table\n");
 214                         return -1;
 215                 }
 216
 217                 priv->tx_sa_tbl[sa_index].spi =
 218                         rte_cpu_to_be_32(ic_session->spi);
 219                 priv->tx_sa_tbl[i].used = 1;
 220                 ic_session->sa_index = sa_index;
 221
 222                 key = malloc(ic_session->key_len);
 223                 if (!key)
 224                         return -ENOMEM;
 225
 226                 memcpy(key, ic_session->key, ic_session->key_len);
 227
 228                 /* write Key table entry*/
 229                 reg_val = TXGBE_IPSRXIDX_ENA |
 230                         TXGBE_IPSRXIDX_WRITE | (sa_index << 3);
 231                 wr32(hw, TXGBE_IPSTXKEY(0),
 232                         rte_cpu_to_be_32(*(uint32_t *)&key[12]));
 233                 wr32(hw, TXGBE_IPSTXKEY(1),
 234                         rte_cpu_to_be_32(*(uint32_t *)&key[8]));
 235                 wr32(hw, TXGBE_IPSTXKEY(2),
 236                         rte_cpu_to_be_32(*(uint32_t *)&key[4]));
 237                 wr32(hw, TXGBE_IPSTXKEY(3),
 238                         rte_cpu_to_be_32(*(uint32_t *)&key[0]));
 239                 wr32(hw, TXGBE_IPSTXSALT,
 240                                 rte_cpu_to_be_32(ic_session->salt));
 241                 wr32w(hw, TXGBE_IPSTXIDX, reg_val, TXGBE_IPSTXIDX_WRITE, 1000);
 242
 243                 free(key);
 244         }
 245
 246         return 0;
 247 }
 248
 249 static int
 250 txgbe_crypto_remove_sa(struct rte_eth_dev *dev,
 251                        struct txgbe_crypto_session *ic_session)
 252 {
 253         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
 254         struct txgbe_ipsec *priv = TXGBE_DEV_IPSEC(dev);
 255         uint32_t reg_val;
 256         int sa_index = -1;
 257
 258         if (ic_session->op == TXGBE_OP_AUTHENTICATED_DECRYPTION) {
 259                 int i, ip_index = -1;
 260
 261                 /* Find a match in the IP table*/
 262                 for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
 263                         if (CMP_IP(priv->rx_ip_tbl[i].ip, ic_session->dst_ip)) {
 264                                 ip_index = i;
 265                                 break;
 266                         }
 267                 }
 268
 269                 /* Fail if no match*/
 270                 if (ip_index < 0) {
 271                         PMD_DRV_LOG(ERR,
 272                                     "Entry not found in the Rx IP table\n");
 273                         return -1;
 274                 }
 275
 276                 /* Find a free entry in the SA table*/
 277                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
 278                         if (priv->rx_sa_tbl[i].spi ==
 279                                   rte_cpu_to_be_32(ic_session->spi)) {
 280                                 sa_index = i;
 281                                 break;
 282                         }
 283                 }
 284                 /* Fail if no match*/
 285                 if (sa_index < 0) {
 286                         PMD_DRV_LOG(ERR,
 287                                     "Entry not found in the Rx SA table\n");
 288                         return -1;
 289                 }
 290
 291                 /* Disable and clear Rx SPI and key table entryes*/
 292                 reg_val = TXGBE_IPSRXIDX_WRITE |
 293                         TXGBE_IPSRXIDX_TB_SPI | (sa_index << 3);
 294                 wr32(hw, TXGBE_IPSRXSPI, 0);
 295                 wr32(hw, TXGBE_IPSRXADDRIDX, 0);
 296                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
 297                 reg_val = TXGBE_IPSRXIDX_WRITE |
 298                         TXGBE_IPSRXIDX_TB_KEY | (sa_index << 3);
 299                 wr32(hw, TXGBE_IPSRXKEY(0), 0);
 300                 wr32(hw, TXGBE_IPSRXKEY(1), 0);
 301                 wr32(hw, TXGBE_IPSRXKEY(2), 0);
 302                 wr32(hw, TXGBE_IPSRXKEY(3), 0);
 303                 wr32(hw, TXGBE_IPSRXSALT, 0);
 304                 wr32(hw, TXGBE_IPSRXMODE, 0);
 305                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
 306                 priv->rx_sa_tbl[sa_index].used = 0;
 307
 308                 /* If last used then clear the IP table entry*/
 309                 priv->rx_ip_tbl[ip_index].ref_count--;
 310                 if (priv->rx_ip_tbl[ip_index].ref_count == 0) {
 311                         reg_val = TXGBE_IPSRXIDX_WRITE | TXGBE_IPSRXIDX_TB_IP |
 312                                         (ip_index << 3);
 313                         wr32(hw, TXGBE_IPSRXADDR(0), 0);
 314                         wr32(hw, TXGBE_IPSRXADDR(1), 0);
 315                         wr32(hw, TXGBE_IPSRXADDR(2), 0);
 316                         wr32(hw, TXGBE_IPSRXADDR(3), 0);
 317                 }
 318         } else { /* session->dir == RTE_CRYPTO_OUTBOUND */
 319                 int i;
 320
 321                 /* Find a match in the SA table*/
 322                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
 323                         if (priv->tx_sa_tbl[i].spi ==
 324                                     rte_cpu_to_be_32(ic_session->spi)) {
 325                                 sa_index = i;
 326                                 break;
 327                         }
 328                 }
 329                 /* Fail if no match entries*/
 330                 if (sa_index < 0) {
 331                         PMD_DRV_LOG(ERR,
 332                                     "Entry not found in the Tx SA table\n");
 333                         return -1;
 334                 }
 335                 reg_val = TXGBE_IPSRXIDX_WRITE | (sa_index << 3);
 336                 wr32(hw, TXGBE_IPSTXKEY(0), 0);
 337                 wr32(hw, TXGBE_IPSTXKEY(1), 0);
 338                 wr32(hw, TXGBE_IPSTXKEY(2), 0);
 339                 wr32(hw, TXGBE_IPSTXKEY(3), 0);
 340                 wr32(hw, TXGBE_IPSTXSALT, 0);
 341                 wr32w(hw, TXGBE_IPSTXIDX, reg_val, TXGBE_IPSTXIDX_WRITE, 1000);
 342
 343                 priv->tx_sa_tbl[sa_index].used = 0;
 344         }
 345
 346         return 0;
 347 }
 348
 349 static int
 350 txgbe_crypto_create_session(void *device,
 351                 struct rte_security_session_conf *conf,
 352                 struct rte_security_session *session,
 353                 struct rte_mempool *mempool)
 354 {
 355         struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)device;
 356         struct txgbe_crypto_session *ic_session = NULL;
 357         struct rte_crypto_aead_xform *aead_xform;
 358         struct rte_eth_conf *dev_conf = &eth_dev->data->dev_conf;
 359
 360         if (rte_mempool_get(mempool, (void **)&ic_session)) {
 361                 PMD_DRV_LOG(ERR, "Cannot get object from ic_session mempool");
 362                 return -ENOMEM;
 363         }
 364
 365         if (conf->crypto_xform->type != RTE_CRYPTO_SYM_XFORM_AEAD ||
 366                         conf->crypto_xform->aead.algo !=
 367                                         RTE_CRYPTO_AEAD_AES_GCM) {
 368                 PMD_DRV_LOG(ERR, "Unsupported crypto transformation mode\n");
 369                 rte_mempool_put(mempool, (void *)ic_session);
 370                 return -ENOTSUP;
 371         }
 372         aead_xform = &conf->crypto_xform->aead;
 373
 374         if (conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
 375                 if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_SECURITY) {
 376                         ic_session->op = TXGBE_OP_AUTHENTICATED_DECRYPTION;
 377                 } else {
 378                         PMD_DRV_LOG(ERR, "IPsec decryption not enabled\n");
 379                         rte_mempool_put(mempool, (void *)ic_session);
 380                         return -ENOTSUP;
 381                 }
 382         } else {
 383                 if (dev_conf->txmode.offloads & DEV_TX_OFFLOAD_SECURITY) {
 384                         ic_session->op = TXGBE_OP_AUTHENTICATED_ENCRYPTION;
 385                 } else {
 386                         PMD_DRV_LOG(ERR, "IPsec encryption not enabled\n");
 387                         rte_mempool_put(mempool, (void *)ic_session);
 388                         return -ENOTSUP;
 389                 }
 390         }
 391
 392         ic_session->key = aead_xform->key.data;
 393         ic_session->key_len = aead_xform->key.length;
 394         memcpy(&ic_session->salt,
 395                &aead_xform->key.data[aead_xform->key.length], 4);
 396         ic_session->spi = conf->ipsec.spi;
 397         ic_session->dev = eth_dev;
 398
 399         set_sec_session_private_data(session, ic_session);
 400
 401         if (ic_session->op == TXGBE_OP_AUTHENTICATED_ENCRYPTION) {
 402                 if (txgbe_crypto_add_sa(ic_session)) {
 403                         PMD_DRV_LOG(ERR, "Failed to add SA\n");
 404                         rte_mempool_put(mempool, (void *)ic_session);
 405                         return -EPERM;
 406                 }
 407         }
 408
 409         return 0;
 410 }
 411
 412 static unsigned int
 413 txgbe_crypto_session_get_size(__rte_unused void *device)
 414 {
 415         return sizeof(struct txgbe_crypto_session);
 416 }
 417
 418 static int
 419 txgbe_crypto_remove_session(void *device,
 420                 struct rte_security_session *session)
 421 {
 422         struct rte_eth_dev *eth_dev = device;
 423         struct txgbe_crypto_session *ic_session =
 424                 (struct txgbe_crypto_session *)
 425                 get_sec_session_private_data(session);
 426         struct rte_mempool *mempool = rte_mempool_from_obj(ic_session);
 427
 428         if (eth_dev != ic_session->dev) {
 429                 PMD_DRV_LOG(ERR, "Session not bound to this device\n");
 430                 return -ENODEV;
 431         }
 432
 433         if (txgbe_crypto_remove_sa(eth_dev, ic_session)) {
 434                 PMD_DRV_LOG(ERR, "Failed to remove session\n");
 435                 return -EFAULT;
 436         }
 437
 438         rte_mempool_put(mempool, (void *)ic_session);
 439
 440         return 0;
 441 }
 442
 443 static inline uint8_t
 444 txgbe_crypto_compute_pad_len(struct rte_mbuf *m)
 445 {
 446         if (m->nb_segs == 1) {
 447                 /* 16 bytes ICV + 2 bytes ESP trailer + payload padding size
 448                  * payload padding size is stored at <pkt_len - 18>
 449                  */
 450                 uint8_t *esp_pad_len = rte_pktmbuf_mtod_offset(m, uint8_t *,
 451                                         rte_pktmbuf_pkt_len(m) -
 452                                         (ESP_TRAILER_SIZE + ESP_ICV_SIZE));
 453                 return *esp_pad_len + ESP_TRAILER_SIZE + ESP_ICV_SIZE;
 454         }
 455         return 0;
 456 }
 457
 458 static int
 459 txgbe_crypto_update_mb(void *device __rte_unused,
 460                 struct rte_security_session *session,
 461                        struct rte_mbuf *m, void *params __rte_unused)
 462 {
 463         struct txgbe_crypto_session *ic_session =
 464                         get_sec_session_private_data(session);
 465         if (ic_session->op == TXGBE_OP_AUTHENTICATED_ENCRYPTION) {
 466                 union txgbe_crypto_tx_desc_md *mdata =
 467                         (union txgbe_crypto_tx_desc_md *)
 468                                 rte_security_dynfield(m);
 469                 mdata->enc = 1;
 470                 mdata->sa_idx = ic_session->sa_index;
 471                 mdata->pad_len = txgbe_crypto_compute_pad_len(m);
 472         }
 473         return 0;
 474 }
 475
 476 static const struct rte_security_capability *
 477 txgbe_crypto_capabilities_get(void *device __rte_unused)
 478 {
 479         static const struct rte_cryptodev_capabilities
 480         aes_gcm_gmac_crypto_capabilities[] = {
 481                 {       /* AES GMAC (128-bit) */
 482                         .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 483                         {.sym = {
 484                                 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 485                                 {.auth = {
 486                                         .algo = RTE_CRYPTO_AUTH_AES_GMAC,
 487                                         .block_size = 16,
 488                                         .key_size = {
 489                                                 .min = 16,
 490                                                 .max = 16,
 491                                                 .increment = 0
 492                                         },
 493                                         .digest_size = {
 494                                                 .min = 16,
 495                                                 .max = 16,
 496                                                 .increment = 0
 497                                         },
 498                                         .iv_size = {
 499                                                 .min = 12,
 500                                                 .max = 12,
 501                                                 .increment = 0
 502                                         }
 503                                 }, }
 504                         }, }
 505                 },
 506                 {       /* AES GCM (128-bit) */
 507                         .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 508                         {.sym = {
 509                                 .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
 510                                 {.aead = {
 511                                         .algo = RTE_CRYPTO_AEAD_AES_GCM,
 512                                         .block_size = 16,
 513                                         .key_size = {
 514                                                 .min = 16,
 515                                                 .max = 16,
 516                                                 .increment = 0
 517                                         },
 518                                         .digest_size = {
 519                                                 .min = 16,
 520                                                 .max = 16,
 521                                                 .increment = 0
 522                                         },
 523                                         .aad_size = {
 524                                                 .min = 0,
 525                                                 .max = 65535,
 526                                                 .increment = 1
 527                                         },
 528                                         .iv_size = {
 529                                                 .min = 12,
 530                                                 .max = 12,
 531                                                 .increment = 0
 532                                         }
 533                                 }, }
 534                         }, }
 535                 },
 536                 {
 537                         .op = RTE_CRYPTO_OP_TYPE_UNDEFINED,
 538                         {.sym = {
 539                                 .xform_type = RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED
 540                         }, }
 541                 },
 542         };
 543
 544         static const struct rte_security_capability
 545         txgbe_security_capabilities[] = {
 546                 { /* IPsec Inline Crypto ESP Transport Egress */
 547                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
 548                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 549                         {.ipsec = {
 550                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 551                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
 552                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
 553                                 .options = { 0 }
 554                         } },
 555                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
 556                         .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
 557                 },
 558                 { /* IPsec Inline Crypto ESP Transport Ingress */
 559                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
 560                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 561                         {.ipsec = {
 562                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 563                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
 564                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
 565                                 .options = { 0 }
 566                         } },
 567                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
 568                         .ol_flags = 0
 569                 },
 570                 { /* IPsec Inline Crypto ESP Tunnel Egress */
 571                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
 572                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 573                         {.ipsec = {
 574                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 575                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
 576                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
 577                                 .options = { 0 }
 578                         } },
 579                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
 580                         .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
 581                 },
 582                 { /* IPsec Inline Crypto ESP Tunnel Ingress */
 583                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
 584                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 585                         {.ipsec = {
 586                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 587                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
 588                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
 589                                 .options = { 0 }
 590                         } },
 591                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
 592                         .ol_flags = 0
 593                 },
 594                 {
 595                         .action = RTE_SECURITY_ACTION_TYPE_NONE
 596                 }
 597         };
 598
 599         return txgbe_security_capabilities;
 600 }
 601
 602 int
 603 txgbe_crypto_enable_ipsec(struct rte_eth_dev *dev)
 604 {
 605         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
 606         uint32_t reg;
 607         uint64_t rx_offloads;
 608         uint64_t tx_offloads;
 609
 610         rx_offloads = dev->data->dev_conf.rxmode.offloads;
 611         tx_offloads = dev->data->dev_conf.txmode.offloads;
 612
 613         /* sanity checks */
 614         if (rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) {
 615                 PMD_DRV_LOG(ERR, "RSC and IPsec not supported");
 616                 return -1;
 617         }
 618         if (rx_offloads & DEV_RX_OFFLOAD_KEEP_CRC) {
 619                 PMD_DRV_LOG(ERR, "HW CRC strip needs to be enabled for IPsec");
 620                 return -1;
 621         }
 622
 623         /* Set TXGBE_SECTXBUFFAF to 0x14 as required in the datasheet*/
 624         wr32(hw, TXGBE_SECTXBUFAF, 0x14);
 625
 626         /* IFG needs to be set to 3 when we are using security. Otherwise a Tx
 627          * hang will occur with heavy traffic.
 628          */
 629         reg = rd32(hw, TXGBE_SECTXIFG);
 630         reg = (reg & ~TXGBE_SECTXIFG_MIN_MASK) | TXGBE_SECTXIFG_MIN(0x3);
 631         wr32(hw, TXGBE_SECTXIFG, reg);
 632
 633         reg = rd32(hw, TXGBE_SECRXCTL);
 634         reg |= TXGBE_SECRXCTL_CRCSTRIP;
 635         wr32(hw, TXGBE_SECRXCTL, reg);
 636
 637         if (rx_offloads & DEV_RX_OFFLOAD_SECURITY) {
 638                 wr32m(hw, TXGBE_SECRXCTL, TXGBE_SECRXCTL_ODSA, 0);
 639                 reg = rd32m(hw, TXGBE_SECRXCTL, TXGBE_SECRXCTL_ODSA);
 640                 if (reg != 0) {
 641                         PMD_DRV_LOG(ERR, "Error enabling Rx Crypto");
 642                         return -1;
 643                 }
 644         }
 645         if (tx_offloads & DEV_TX_OFFLOAD_SECURITY) {
 646                 wr32(hw, TXGBE_SECTXCTL, TXGBE_SECTXCTL_STFWD);
 647                 reg = rd32(hw, TXGBE_SECTXCTL);
 648                 if (reg != TXGBE_SECTXCTL_STFWD) {
 649                         PMD_DRV_LOG(ERR, "Error enabling Rx Crypto");
 650                         return -1;
 651                 }
 652         }
 653
 654         txgbe_crypto_clear_ipsec_tables(dev);
 655
 656         return 0;
 657 }
 658
 659 int
 660 txgbe_crypto_add_ingress_sa_from_flow(const void *sess,
 661                                       const void *ip_spec,
 662                                       uint8_t is_ipv6)
 663 {
 664         struct txgbe_crypto_session *ic_session =
 665                         get_sec_session_private_data(sess);
 666
 667         if (ic_session->op == TXGBE_OP_AUTHENTICATED_DECRYPTION) {
 668                 if (is_ipv6) {
 669                         const struct rte_flow_item_ipv6 *ipv6 = ip_spec;
 670                         ic_session->src_ip.type = IPv6;
 671                         ic_session->dst_ip.type = IPv6;
 672                         rte_memcpy(ic_session->src_ip.ipv6,
 673                                    ipv6->hdr.src_addr, 16);
 674                         rte_memcpy(ic_session->dst_ip.ipv6,
 675                                    ipv6->hdr.dst_addr, 16);
 676                 } else {
 677                         const struct rte_flow_item_ipv4 *ipv4 = ip_spec;
 678                         ic_session->src_ip.type = IPv4;
 679                         ic_session->dst_ip.type = IPv4;
 680                         ic_session->src_ip.ipv4 = ipv4->hdr.src_addr;
 681                         ic_session->dst_ip.ipv4 = ipv4->hdr.dst_addr;
 682                 }
 683                 return txgbe_crypto_add_sa(ic_session);
 684         }
 685
 686         return 0;
 687 }
 688
 689 static struct rte_security_ops txgbe_security_ops = {
 690         .session_create = txgbe_crypto_create_session,
 691         .session_get_size = txgbe_crypto_session_get_size,
 692         .session_destroy = txgbe_crypto_remove_session,
 693         .set_pkt_metadata = txgbe_crypto_update_mb,
 694         .capabilities_get = txgbe_crypto_capabilities_get
 695 };
 696
 697 static int
 698 txgbe_crypto_capable(struct rte_eth_dev *dev)
 699 {
 700         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
 701         uint32_t reg_i, reg, capable = 1;
 702         /* test if rx crypto can be enabled and then write back initial value*/
 703         reg_i = rd32(hw, TXGBE_SECRXCTL);
 704         wr32m(hw, TXGBE_SECRXCTL, TXGBE_SECRXCTL_ODSA, 0);
 705         reg = rd32m(hw, TXGBE_SECRXCTL, TXGBE_SECRXCTL_ODSA);
 706         if (reg != 0)
 707                 capable = 0;
 708         wr32(hw, TXGBE_SECRXCTL, reg_i);
 709         return capable;
 710 }
 711
 712 int
 713 txgbe_ipsec_ctx_create(struct rte_eth_dev *dev)
 714 {
 715         struct rte_security_ctx *ctx = NULL;
 716
 717         if (txgbe_crypto_capable(dev)) {
 718                 ctx = rte_malloc("rte_security_instances_ops",
 719                                  sizeof(struct rte_security_ctx), 0);
 720                 if (ctx) {
 721                         ctx->device = (void *)dev;
 722                         ctx->ops = &txgbe_security_ops;
 723                         ctx->sess_cnt = 0;
 724                         dev->security_ctx = ctx;
 725                 } else {
 726                         return -ENOMEM;
 727                 }
 728         }
 729         if (rte_security_dynfield_register() < 0)
 730                 return -rte_errno;
 731         return 0;
 732 }