drivers/net/txgbe/txgbe_ipsec.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2015-2020
   3  */
   4
   5 #include <ethdev_pci.h>
   6 #include <rte_security_driver.h>
   7 #include <rte_cryptodev.h>
   8
   9 #include "base/txgbe.h"
  10 #include "txgbe_ethdev.h"
  11 #include "txgbe_ipsec.h"
  12
  13 #define CMP_IP(a, b) (\
  14         (a).ipv6[0] == (b).ipv6[0] && \
  15         (a).ipv6[1] == (b).ipv6[1] && \
  16         (a).ipv6[2] == (b).ipv6[2] && \
  17         (a).ipv6[3] == (b).ipv6[3])
  18
  19 static void
  20 txgbe_crypto_clear_ipsec_tables(struct rte_eth_dev *dev)
  21 {
  22         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
  23         struct txgbe_ipsec *priv = TXGBE_DEV_IPSEC(dev);
  24         int i = 0;
  25
  26         /* clear Rx IP table*/
  27         for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
  28                 uint16_t index = i << 3;
  29                 uint32_t reg_val = TXGBE_IPSRXIDX_WRITE |
  30                                 TXGBE_IPSRXIDX_TB_IP | index;
  31                 wr32(hw, TXGBE_IPSRXADDR(0), 0);
  32                 wr32(hw, TXGBE_IPSRXADDR(1), 0);
  33                 wr32(hw, TXGBE_IPSRXADDR(2), 0);
  34                 wr32(hw, TXGBE_IPSRXADDR(3), 0);
  35                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
  36         }
  37
  38         /* clear Rx SPI and Rx/Tx SA tables*/
  39         for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
  40                 uint32_t index = i << 3;
  41                 uint32_t reg_val = TXGBE_IPSRXIDX_WRITE |
  42                                 TXGBE_IPSRXIDX_TB_SPI | index;
  43                 wr32(hw, TXGBE_IPSRXSPI, 0);
  44                 wr32(hw, TXGBE_IPSRXADDRIDX, 0);
  45                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
  46                 reg_val = TXGBE_IPSRXIDX_WRITE | TXGBE_IPSRXIDX_TB_KEY | index;
  47                 wr32(hw, TXGBE_IPSRXKEY(0), 0);
  48                 wr32(hw, TXGBE_IPSRXKEY(1), 0);
  49                 wr32(hw, TXGBE_IPSRXKEY(2), 0);
  50                 wr32(hw, TXGBE_IPSRXKEY(3), 0);
  51                 wr32(hw, TXGBE_IPSRXSALT, 0);
  52                 wr32(hw, TXGBE_IPSRXMODE, 0);
  53                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
  54                 reg_val = TXGBE_IPSTXIDX_WRITE | index;
  55                 wr32(hw, TXGBE_IPSTXKEY(0), 0);
  56                 wr32(hw, TXGBE_IPSTXKEY(1), 0);
  57                 wr32(hw, TXGBE_IPSTXKEY(2), 0);
  58                 wr32(hw, TXGBE_IPSTXKEY(3), 0);
  59                 wr32(hw, TXGBE_IPSTXSALT, 0);
  60                 wr32w(hw, TXGBE_IPSTXIDX, reg_val, TXGBE_IPSTXIDX_WRITE, 1000);
  61         }
  62
  63         memset(priv->rx_ip_tbl, 0, sizeof(priv->rx_ip_tbl));
  64         memset(priv->rx_sa_tbl, 0, sizeof(priv->rx_sa_tbl));
  65         memset(priv->tx_sa_tbl, 0, sizeof(priv->tx_sa_tbl));
  66 }
  67
  68 static int
  69 txgbe_crypto_add_sa(struct txgbe_crypto_session *ic_session)
  70 {
  71         struct rte_eth_dev *dev = ic_session->dev;
  72         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
  73         struct txgbe_ipsec *priv = TXGBE_DEV_IPSEC(dev);
  74         uint32_t reg_val;
  75         int sa_index = -1;
  76
  77         if (ic_session->op == TXGBE_OP_AUTHENTICATED_DECRYPTION) {
  78                 int i, ip_index = -1;
  79                 uint8_t *key;
  80
  81                 /* Find a match in the IP table*/
  82                 for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
  83                         if (CMP_IP(priv->rx_ip_tbl[i].ip,
  84                                    ic_session->dst_ip)) {
  85                                 ip_index = i;
  86                                 break;
  87                         }
  88                 }
  89                 /* If no match, find a free entry in the IP table*/
  90                 if (ip_index < 0) {
  91                         for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
  92                                 if (priv->rx_ip_tbl[i].ref_count == 0) {
  93                                         ip_index = i;
  94                                         break;
  95                                 }
  96                         }
  97                 }
  98
  99                 /* Fail if no match and no free entries*/
 100                 if (ip_index < 0) {
 101                         PMD_DRV_LOG(ERR,
 102                                     "No free entry left in the Rx IP table\n");
 103                         return -1;
 104                 }
 105
 106                 /* Find a free entry in the SA table*/
 107                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
 108                         if (priv->rx_sa_tbl[i].used == 0) {
 109                                 sa_index = i;
 110                                 break;
 111                         }
 112                 }
 113                 /* Fail if no free entries*/
 114                 if (sa_index < 0) {
 115                         PMD_DRV_LOG(ERR,
 116                                     "No free entry left in the Rx SA table\n");
 117                         return -1;
 118                 }
 119
 120                 priv->rx_ip_tbl[ip_index].ip.ipv6[0] =
 121                                 ic_session->dst_ip.ipv6[0];
 122                 priv->rx_ip_tbl[ip_index].ip.ipv6[1] =
 123                                 ic_session->dst_ip.ipv6[1];
 124                 priv->rx_ip_tbl[ip_index].ip.ipv6[2] =
 125                                 ic_session->dst_ip.ipv6[2];
 126                 priv->rx_ip_tbl[ip_index].ip.ipv6[3] =
 127                                 ic_session->dst_ip.ipv6[3];
 128                 priv->rx_ip_tbl[ip_index].ref_count++;
 129
 130                 priv->rx_sa_tbl[sa_index].spi = ic_session->spi;
 131                 priv->rx_sa_tbl[sa_index].ip_index = ip_index;
 132                 priv->rx_sa_tbl[sa_index].mode = IPSRXMOD_VALID;
 133                 if (ic_session->op == TXGBE_OP_AUTHENTICATED_DECRYPTION)
 134                         priv->rx_sa_tbl[sa_index].mode |=
 135                                         (IPSRXMOD_PROTO | IPSRXMOD_DECRYPT);
 136                 if (ic_session->dst_ip.type == IPv6) {
 137                         priv->rx_sa_tbl[sa_index].mode |= IPSRXMOD_IPV6;
 138                         priv->rx_ip_tbl[ip_index].ip.type = IPv6;
 139                 } else if (ic_session->dst_ip.type == IPv4) {
 140                         priv->rx_ip_tbl[ip_index].ip.type = IPv4;
 141                 }
 142                 priv->rx_sa_tbl[sa_index].used = 1;
 143
 144                 /* write IP table entry*/
 145                 reg_val = TXGBE_IPSRXIDX_ENA | TXGBE_IPSRXIDX_WRITE |
 146                                 TXGBE_IPSRXIDX_TB_IP | (ip_index << 3);
 147                 if (priv->rx_ip_tbl[ip_index].ip.type == IPv4) {
 148                         uint32_t ipv4 = priv->rx_ip_tbl[ip_index].ip.ipv4;
 149                         wr32(hw, TXGBE_IPSRXADDR(0), rte_cpu_to_be_32(ipv4));
 150                         wr32(hw, TXGBE_IPSRXADDR(1), 0);
 151                         wr32(hw, TXGBE_IPSRXADDR(2), 0);
 152                         wr32(hw, TXGBE_IPSRXADDR(3), 0);
 153                 } else {
 154                         wr32(hw, TXGBE_IPSRXADDR(0),
 155                                         priv->rx_ip_tbl[ip_index].ip.ipv6[0]);
 156                         wr32(hw, TXGBE_IPSRXADDR(1),
 157                                         priv->rx_ip_tbl[ip_index].ip.ipv6[1]);
 158                         wr32(hw, TXGBE_IPSRXADDR(2),
 159                                         priv->rx_ip_tbl[ip_index].ip.ipv6[2]);
 160                         wr32(hw, TXGBE_IPSRXADDR(3),
 161                                         priv->rx_ip_tbl[ip_index].ip.ipv6[3]);
 162                 }
 163                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
 164
 165                 /* write SPI table entry*/
 166                 reg_val = TXGBE_IPSRXIDX_ENA | TXGBE_IPSRXIDX_WRITE |
 167                                 TXGBE_IPSRXIDX_TB_SPI | (sa_index << 3);
 168                 wr32(hw, TXGBE_IPSRXSPI,
 169                                 priv->rx_sa_tbl[sa_index].spi);
 170                 wr32(hw, TXGBE_IPSRXADDRIDX,
 171                                 priv->rx_sa_tbl[sa_index].ip_index);
 172                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
 173
 174                 /* write Key table entry*/
 175                 key = malloc(ic_session->key_len);
 176                 if (!key)
 177                         return -ENOMEM;
 178
 179                 memcpy(key, ic_session->key, ic_session->key_len);
 180
 181                 reg_val = TXGBE_IPSRXIDX_ENA | TXGBE_IPSRXIDX_WRITE |
 182                                 TXGBE_IPSRXIDX_TB_KEY | (sa_index << 3);
 183                 wr32(hw, TXGBE_IPSRXKEY(0),
 184                         rte_cpu_to_be_32(*(uint32_t *)&key[12]));
 185                 wr32(hw, TXGBE_IPSRXKEY(1),
 186                         rte_cpu_to_be_32(*(uint32_t *)&key[8]));
 187                 wr32(hw, TXGBE_IPSRXKEY(2),
 188                         rte_cpu_to_be_32(*(uint32_t *)&key[4]));
 189                 wr32(hw, TXGBE_IPSRXKEY(3),
 190                         rte_cpu_to_be_32(*(uint32_t *)&key[0]));
 191                 wr32(hw, TXGBE_IPSRXSALT,
 192                                 rte_cpu_to_be_32(ic_session->salt));
 193                 wr32(hw, TXGBE_IPSRXMODE,
 194                                 priv->rx_sa_tbl[sa_index].mode);
 195                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
 196
 197                 free(key);
 198         } else { /* sess->dir == RTE_CRYPTO_OUTBOUND */
 199                 uint8_t *key;
 200                 int i;
 201
 202                 /* Find a free entry in the SA table*/
 203                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
 204                         if (priv->tx_sa_tbl[i].used == 0) {
 205                                 sa_index = i;
 206                                 break;
 207                         }
 208                 }
 209                 /* Fail if no free entries*/
 210                 if (sa_index < 0) {
 211                         PMD_DRV_LOG(ERR,
 212                                     "No free entry left in the Tx SA table\n");
 213                         return -1;
 214                 }
 215
 216                 priv->tx_sa_tbl[sa_index].spi =
 217                         rte_cpu_to_be_32(ic_session->spi);
 218                 priv->tx_sa_tbl[i].used = 1;
 219                 ic_session->sa_index = sa_index;
 220
 221                 key = malloc(ic_session->key_len);
 222                 if (!key)
 223                         return -ENOMEM;
 224
 225                 memcpy(key, ic_session->key, ic_session->key_len);
 226
 227                 /* write Key table entry*/
 228                 reg_val = TXGBE_IPSRXIDX_ENA |
 229                         TXGBE_IPSRXIDX_WRITE | (sa_index << 3);
 230                 wr32(hw, TXGBE_IPSTXKEY(0),
 231                         rte_cpu_to_be_32(*(uint32_t *)&key[12]));
 232                 wr32(hw, TXGBE_IPSTXKEY(1),
 233                         rte_cpu_to_be_32(*(uint32_t *)&key[8]));
 234                 wr32(hw, TXGBE_IPSTXKEY(2),
 235                         rte_cpu_to_be_32(*(uint32_t *)&key[4]));
 236                 wr32(hw, TXGBE_IPSTXKEY(3),
 237                         rte_cpu_to_be_32(*(uint32_t *)&key[0]));
 238                 wr32(hw, TXGBE_IPSTXSALT,
 239                                 rte_cpu_to_be_32(ic_session->salt));
 240                 wr32w(hw, TXGBE_IPSTXIDX, reg_val, TXGBE_IPSTXIDX_WRITE, 1000);
 241
 242                 free(key);
 243         }
 244
 245         return 0;
 246 }
 247
 248 static int
 249 txgbe_crypto_remove_sa(struct rte_eth_dev *dev,
 250                        struct txgbe_crypto_session *ic_session)
 251 {
 252         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
 253         struct txgbe_ipsec *priv = TXGBE_DEV_IPSEC(dev);
 254         uint32_t reg_val;
 255         int sa_index = -1;
 256
 257         if (ic_session->op == TXGBE_OP_AUTHENTICATED_DECRYPTION) {
 258                 int i, ip_index = -1;
 259
 260                 /* Find a match in the IP table*/
 261                 for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
 262                         if (CMP_IP(priv->rx_ip_tbl[i].ip, ic_session->dst_ip)) {
 263                                 ip_index = i;
 264                                 break;
 265                         }
 266                 }
 267
 268                 /* Fail if no match*/
 269                 if (ip_index < 0) {
 270                         PMD_DRV_LOG(ERR,
 271                                     "Entry not found in the Rx IP table\n");
 272                         return -1;
 273                 }
 274
 275                 /* Find a free entry in the SA table*/
 276                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
 277                         if (priv->rx_sa_tbl[i].spi ==
 278                                   rte_cpu_to_be_32(ic_session->spi)) {
 279                                 sa_index = i;
 280                                 break;
 281                         }
 282                 }
 283                 /* Fail if no match*/
 284                 if (sa_index < 0) {
 285                         PMD_DRV_LOG(ERR,
 286                                     "Entry not found in the Rx SA table\n");
 287                         return -1;
 288                 }
 289
 290                 /* Disable and clear Rx SPI and key table entryes*/
 291                 reg_val = TXGBE_IPSRXIDX_WRITE |
 292                         TXGBE_IPSRXIDX_TB_SPI | (sa_index << 3);
 293                 wr32(hw, TXGBE_IPSRXSPI, 0);
 294                 wr32(hw, TXGBE_IPSRXADDRIDX, 0);
 295                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
 296                 reg_val = TXGBE_IPSRXIDX_WRITE |
 297                         TXGBE_IPSRXIDX_TB_KEY | (sa_index << 3);
 298                 wr32(hw, TXGBE_IPSRXKEY(0), 0);
 299                 wr32(hw, TXGBE_IPSRXKEY(1), 0);
 300                 wr32(hw, TXGBE_IPSRXKEY(2), 0);
 301                 wr32(hw, TXGBE_IPSRXKEY(3), 0);
 302                 wr32(hw, TXGBE_IPSRXSALT, 0);
 303                 wr32(hw, TXGBE_IPSRXMODE, 0);
 304                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
 305                 priv->rx_sa_tbl[sa_index].used = 0;
 306
 307                 /* If last used then clear the IP table entry*/
 308                 priv->rx_ip_tbl[ip_index].ref_count--;
 309                 if (priv->rx_ip_tbl[ip_index].ref_count == 0) {
 310                         reg_val = TXGBE_IPSRXIDX_WRITE | TXGBE_IPSRXIDX_TB_IP |
 311                                         (ip_index << 3);
 312                         wr32(hw, TXGBE_IPSRXADDR(0), 0);
 313                         wr32(hw, TXGBE_IPSRXADDR(1), 0);
 314                         wr32(hw, TXGBE_IPSRXADDR(2), 0);
 315                         wr32(hw, TXGBE_IPSRXADDR(3), 0);
 316                 }
 317         } else { /* session->dir == RTE_CRYPTO_OUTBOUND */
 318                 int i;
 319
 320                 /* Find a match in the SA table*/
 321                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
 322                         if (priv->tx_sa_tbl[i].spi ==
 323                                     rte_cpu_to_be_32(ic_session->spi)) {
 324                                 sa_index = i;
 325                                 break;
 326                         }
 327                 }
 328                 /* Fail if no match entries*/
 329                 if (sa_index < 0) {
 330                         PMD_DRV_LOG(ERR,
 331                                     "Entry not found in the Tx SA table\n");
 332                         return -1;
 333                 }
 334                 reg_val = TXGBE_IPSRXIDX_WRITE | (sa_index << 3);
 335                 wr32(hw, TXGBE_IPSTXKEY(0), 0);
 336                 wr32(hw, TXGBE_IPSTXKEY(1), 0);
 337                 wr32(hw, TXGBE_IPSTXKEY(2), 0);
 338                 wr32(hw, TXGBE_IPSTXKEY(3), 0);
 339                 wr32(hw, TXGBE_IPSTXSALT, 0);
 340                 wr32w(hw, TXGBE_IPSTXIDX, reg_val, TXGBE_IPSTXIDX_WRITE, 1000);
 341
 342                 priv->tx_sa_tbl[sa_index].used = 0;
 343         }
 344
 345         return 0;
 346 }
 347
 348 static int
 349 txgbe_crypto_create_session(void *device,
 350                 struct rte_security_session_conf *conf,
 351                 struct rte_security_session *session,
 352                 struct rte_mempool *mempool)
 353 {
 354         struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)device;
 355         struct txgbe_crypto_session *ic_session = NULL;
 356         struct rte_crypto_aead_xform *aead_xform;
 357         struct rte_eth_conf *dev_conf = &eth_dev->data->dev_conf;
 358
 359         if (rte_mempool_get(mempool, (void **)&ic_session)) {
 360                 PMD_DRV_LOG(ERR, "Cannot get object from ic_session mempool");
 361                 return -ENOMEM;
 362         }
 363
 364         if (conf->crypto_xform->type != RTE_CRYPTO_SYM_XFORM_AEAD ||
 365                         conf->crypto_xform->aead.algo !=
 366                                         RTE_CRYPTO_AEAD_AES_GCM) {
 367                 PMD_DRV_LOG(ERR, "Unsupported crypto transformation mode\n");
 368                 rte_mempool_put(mempool, (void *)ic_session);
 369                 return -ENOTSUP;
 370         }
 371         aead_xform = &conf->crypto_xform->aead;
 372
 373         if (conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
 374                 if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_SECURITY) {
 375                         ic_session->op = TXGBE_OP_AUTHENTICATED_DECRYPTION;
 376                 } else {
 377                         PMD_DRV_LOG(ERR, "IPsec decryption not enabled\n");
 378                         rte_mempool_put(mempool, (void *)ic_session);
 379                         return -ENOTSUP;
 380                 }
 381         } else {
 382                 if (dev_conf->txmode.offloads & DEV_TX_OFFLOAD_SECURITY) {
 383                         ic_session->op = TXGBE_OP_AUTHENTICATED_ENCRYPTION;
 384                 } else {
 385                         PMD_DRV_LOG(ERR, "IPsec encryption not enabled\n");
 386                         rte_mempool_put(mempool, (void *)ic_session);
 387                         return -ENOTSUP;
 388                 }
 389         }
 390
 391         ic_session->key = aead_xform->key.data;
 392         ic_session->key_len = aead_xform->key.length;
 393         memcpy(&ic_session->salt,
 394                &aead_xform->key.data[aead_xform->key.length], 4);
 395         ic_session->spi = conf->ipsec.spi;
 396         ic_session->dev = eth_dev;
 397
 398         set_sec_session_private_data(session, ic_session);
 399
 400         if (ic_session->op == TXGBE_OP_AUTHENTICATED_ENCRYPTION) {
 401                 if (txgbe_crypto_add_sa(ic_session)) {
 402                         PMD_DRV_LOG(ERR, "Failed to add SA\n");
 403                         rte_mempool_put(mempool, (void *)ic_session);
 404                         return -EPERM;
 405                 }
 406         }
 407
 408         return 0;
 409 }
 410
 411 static unsigned int
 412 txgbe_crypto_session_get_size(__rte_unused void *device)
 413 {
 414         return sizeof(struct txgbe_crypto_session);
 415 }
 416
 417 static int
 418 txgbe_crypto_remove_session(void *device,
 419                 struct rte_security_session *session)
 420 {
 421         struct rte_eth_dev *eth_dev = device;
 422         struct txgbe_crypto_session *ic_session =
 423                 (struct txgbe_crypto_session *)
 424                 get_sec_session_private_data(session);
 425         struct rte_mempool *mempool = rte_mempool_from_obj(ic_session);
 426
 427         if (eth_dev != ic_session->dev) {
 428                 PMD_DRV_LOG(ERR, "Session not bound to this device\n");
 429                 return -ENODEV;
 430         }
 431
 432         if (txgbe_crypto_remove_sa(eth_dev, ic_session)) {
 433                 PMD_DRV_LOG(ERR, "Failed to remove session\n");
 434                 return -EFAULT;
 435         }
 436
 437         rte_mempool_put(mempool, (void *)ic_session);
 438
 439         return 0;
 440 }
 441
 442 static inline uint8_t
 443 txgbe_crypto_compute_pad_len(struct rte_mbuf *m)
 444 {
 445         if (m->nb_segs == 1) {
 446                 /* 16 bytes ICV + 2 bytes ESP trailer + payload padding size
 447                  * payload padding size is stored at <pkt_len - 18>
 448                  */
 449                 uint8_t *esp_pad_len = rte_pktmbuf_mtod_offset(m, uint8_t *,
 450                                         rte_pktmbuf_pkt_len(m) -
 451                                         (ESP_TRAILER_SIZE + ESP_ICV_SIZE));
 452                 return *esp_pad_len + ESP_TRAILER_SIZE + ESP_ICV_SIZE;
 453         }
 454         return 0;
 455 }
 456
 457 static int
 458 txgbe_crypto_update_mb(void *device __rte_unused,
 459                 struct rte_security_session *session,
 460                        struct rte_mbuf *m, void *params __rte_unused)
 461 {
 462         struct txgbe_crypto_session *ic_session =
 463                         get_sec_session_private_data(session);
 464         if (ic_session->op == TXGBE_OP_AUTHENTICATED_ENCRYPTION) {
 465                 union txgbe_crypto_tx_desc_md *mdata =
 466                         (union txgbe_crypto_tx_desc_md *)
 467                                 rte_security_dynfield(m);
 468                 mdata->enc = 1;
 469                 mdata->sa_idx = ic_session->sa_index;
 470                 mdata->pad_len = txgbe_crypto_compute_pad_len(m);
 471         }
 472         return 0;
 473 }
 474
 475 static const struct rte_security_capability *
 476 txgbe_crypto_capabilities_get(void *device __rte_unused)
 477 {
 478         static const struct rte_cryptodev_capabilities
 479         aes_gcm_gmac_crypto_capabilities[] = {
 480                 {       /* AES GMAC (128-bit) */
 481                         .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 482                         {.sym = {
 483                                 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 484                                 {.auth = {
 485                                         .algo = RTE_CRYPTO_AUTH_AES_GMAC,
 486                                         .block_size = 16,
 487                                         .key_size = {
 488                                                 .min = 16,
 489                                                 .max = 16,
 490                                                 .increment = 0
 491                                         },
 492                                         .digest_size = {
 493                                                 .min = 16,
 494                                                 .max = 16,
 495                                                 .increment = 0
 496                                         },
 497                                         .iv_size = {
 498                                                 .min = 12,
 499                                                 .max = 12,
 500                                                 .increment = 0
 501                                         }
 502                                 }, }
 503                         }, }
 504                 },
 505                 {       /* AES GCM (128-bit) */
 506                         .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 507                         {.sym = {
 508                                 .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
 509                                 {.aead = {
 510                                         .algo = RTE_CRYPTO_AEAD_AES_GCM,
 511                                         .block_size = 16,
 512                                         .key_size = {
 513                                                 .min = 16,
 514                                                 .max = 16,
 515                                                 .increment = 0
 516                                         },
 517                                         .digest_size = {
 518                                                 .min = 16,
 519                                                 .max = 16,
 520                                                 .increment = 0
 521                                         },
 522                                         .aad_size = {
 523                                                 .min = 0,
 524                                                 .max = 65535,
 525                                                 .increment = 1
 526                                         },
 527                                         .iv_size = {
 528                                                 .min = 12,
 529                                                 .max = 12,
 530                                                 .increment = 0
 531                                         }
 532                                 }, }
 533                         }, }
 534                 },
 535                 {
 536                         .op = RTE_CRYPTO_OP_TYPE_UNDEFINED,
 537                         {.sym = {
 538                                 .xform_type = RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED
 539                         }, }
 540                 },
 541         };
 542
 543         static const struct rte_security_capability
 544         txgbe_security_capabilities[] = {
 545                 { /* IPsec Inline Crypto ESP Transport Egress */
 546                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
 547                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 548                         {.ipsec = {
 549                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 550                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
 551                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
 552                                 .options = { 0 }
 553                         } },
 554                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
 555                         .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
 556                 },
 557                 { /* IPsec Inline Crypto ESP Transport Ingress */
 558                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
 559                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 560                         {.ipsec = {
 561                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 562                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
 563                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
 564                                 .options = { 0 }
 565                         } },
 566                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
 567                         .ol_flags = 0
 568                 },
 569                 { /* IPsec Inline Crypto ESP Tunnel Egress */
 570                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
 571                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 572                         {.ipsec = {
 573                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 574                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
 575                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
 576                                 .options = { 0 }
 577                         } },
 578                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
 579                         .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
 580                 },
 581                 { /* IPsec Inline Crypto ESP Tunnel Ingress */
 582                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
 583                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
 584                         {.ipsec = {
 585                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
 586                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
 587                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
 588                                 .options = { 0 }
 589                         } },
 590                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
 591                         .ol_flags = 0
 592                 },
 593                 {
 594                         .action = RTE_SECURITY_ACTION_TYPE_NONE
 595                 }
 596         };
 597
 598         return txgbe_security_capabilities;
 599 }
 600
 601 int
 602 txgbe_crypto_enable_ipsec(struct rte_eth_dev *dev)
 603 {
 604         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
 605         uint32_t reg;
 606         uint64_t rx_offloads;
 607         uint64_t tx_offloads;
 608
 609         rx_offloads = dev->data->dev_conf.rxmode.offloads;
 610         tx_offloads = dev->data->dev_conf.txmode.offloads;
 611
 612         /* sanity checks */
 613         if (rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) {
 614                 PMD_DRV_LOG(ERR, "RSC and IPsec not supported");
 615                 return -1;
 616         }
 617         if (rx_offloads & DEV_RX_OFFLOAD_KEEP_CRC) {
 618                 PMD_DRV_LOG(ERR, "HW CRC strip needs to be enabled for IPsec");
 619                 return -1;
 620         }
 621
 622         /* Set TXGBE_SECTXBUFFAF to 0x14 as required in the datasheet*/
 623         wr32(hw, TXGBE_SECTXBUFAF, 0x14);
 624
 625         /* IFG needs to be set to 3 when we are using security. Otherwise a Tx
 626          * hang will occur with heavy traffic.
 627          */
 628         reg = rd32(hw, TXGBE_SECTXIFG);
 629         reg = (reg & ~TXGBE_SECTXIFG_MIN_MASK) | TXGBE_SECTXIFG_MIN(0x3);
 630         wr32(hw, TXGBE_SECTXIFG, reg);
 631
 632         reg = rd32(hw, TXGBE_SECRXCTL);
 633         reg |= TXGBE_SECRXCTL_CRCSTRIP;
 634         wr32(hw, TXGBE_SECRXCTL, reg);
 635
 636         if (rx_offloads & DEV_RX_OFFLOAD_SECURITY) {
 637                 wr32m(hw, TXGBE_SECRXCTL, TXGBE_SECRXCTL_ODSA, 0);
 638                 reg = rd32m(hw, TXGBE_SECRXCTL, TXGBE_SECRXCTL_ODSA);
 639                 if (reg != 0) {
 640                         PMD_DRV_LOG(ERR, "Error enabling Rx Crypto");
 641                         return -1;
 642                 }
 643         }
 644         if (tx_offloads & DEV_TX_OFFLOAD_SECURITY) {
 645                 wr32(hw, TXGBE_SECTXCTL, TXGBE_SECTXCTL_STFWD);
 646                 reg = rd32(hw, TXGBE_SECTXCTL);
 647                 if (reg != TXGBE_SECTXCTL_STFWD) {
 648                         PMD_DRV_LOG(ERR, "Error enabling Rx Crypto");
 649                         return -1;
 650                 }
 651         }
 652
 653         txgbe_crypto_clear_ipsec_tables(dev);
 654
 655         return 0;
 656 }
 657
 658 int
 659 txgbe_crypto_add_ingress_sa_from_flow(const void *sess,
 660                                       const void *ip_spec,
 661                                       uint8_t is_ipv6)
 662 {
 663         struct txgbe_crypto_session *ic_session =
 664                         get_sec_session_private_data(sess);
 665
 666         if (ic_session->op == TXGBE_OP_AUTHENTICATED_DECRYPTION) {
 667                 if (is_ipv6) {
 668                         const struct rte_flow_item_ipv6 *ipv6 = ip_spec;
 669                         ic_session->src_ip.type = IPv6;
 670                         ic_session->dst_ip.type = IPv6;
 671                         rte_memcpy(ic_session->src_ip.ipv6,
 672                                    ipv6->hdr.src_addr, 16);
 673                         rte_memcpy(ic_session->dst_ip.ipv6,
 674                                    ipv6->hdr.dst_addr, 16);
 675                 } else {
 676                         const struct rte_flow_item_ipv4 *ipv4 = ip_spec;
 677                         ic_session->src_ip.type = IPv4;
 678                         ic_session->dst_ip.type = IPv4;
 679                         ic_session->src_ip.ipv4 = ipv4->hdr.src_addr;
 680                         ic_session->dst_ip.ipv4 = ipv4->hdr.dst_addr;
 681                 }
 682                 return txgbe_crypto_add_sa(ic_session);
 683         }
 684
 685         return 0;
 686 }
 687
 688 static struct rte_security_ops txgbe_security_ops = {
 689         .session_create = txgbe_crypto_create_session,
 690         .session_get_size = txgbe_crypto_session_get_size,
 691         .session_destroy = txgbe_crypto_remove_session,
 692         .set_pkt_metadata = txgbe_crypto_update_mb,
 693         .capabilities_get = txgbe_crypto_capabilities_get
 694 };
 695
 696 static int
 697 txgbe_crypto_capable(struct rte_eth_dev *dev)
 698 {
 699         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
 700         uint32_t reg_i, reg, capable = 1;
 701         /* test if rx crypto can be enabled and then write back initial value*/
 702         reg_i = rd32(hw, TXGBE_SECRXCTL);
 703         wr32m(hw, TXGBE_SECRXCTL, TXGBE_SECRXCTL_ODSA, 0);
 704         reg = rd32m(hw, TXGBE_SECRXCTL, TXGBE_SECRXCTL_ODSA);
 705         if (reg != 0)
 706                 capable = 0;
 707         wr32(hw, TXGBE_SECRXCTL, reg_i);
 708         return capable;
 709 }
 710
 711 int
 712 txgbe_ipsec_ctx_create(struct rte_eth_dev *dev)
 713 {
 714         struct rte_security_ctx *ctx = NULL;
 715
 716         if (txgbe_crypto_capable(dev)) {
 717                 ctx = rte_malloc("rte_security_instances_ops",
 718                                  sizeof(struct rte_security_ctx), 0);
 719                 if (ctx) {
 720                         ctx->device = (void *)dev;
 721                         ctx->ops = &txgbe_security_ops;
 722                         ctx->sess_cnt = 0;
 723                         dev->security_ctx = ctx;
 724                 } else {
 725                         return -ENOMEM;
 726                 }
 727         }
 728         if (rte_security_dynfield_register() < 0)
 729                 return -rte_errno;
 730         return 0;
 731 }