net/i40e: fix Rx packet statistics
[dpdk.git] / drivers / net / txgbe / txgbe_ipsec.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2015-2020 Beijing WangXun Technology Co., Ltd.
3  * Copyright(c) 2010-2017 Intel Corporation
4  */
5
6 #include <ethdev_pci.h>
7 #include <rte_security_driver.h>
8 #include <rte_cryptodev.h>
9
10 #include "base/txgbe.h"
11 #include "txgbe_ethdev.h"
12 #include "txgbe_ipsec.h"
13
14 #define CMP_IP(a, b) (\
15         (a).ipv6[0] == (b).ipv6[0] && \
16         (a).ipv6[1] == (b).ipv6[1] && \
17         (a).ipv6[2] == (b).ipv6[2] && \
18         (a).ipv6[3] == (b).ipv6[3])
19
20 static void
21 txgbe_crypto_clear_ipsec_tables(struct rte_eth_dev *dev)
22 {
23         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
24         struct txgbe_ipsec *priv = TXGBE_DEV_IPSEC(dev);
25         int i = 0;
26
27         /* clear Rx IP table*/
28         for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
29                 uint16_t index = i << 3;
30                 uint32_t reg_val = TXGBE_IPSRXIDX_WRITE |
31                                 TXGBE_IPSRXIDX_TB_IP | index;
32                 wr32(hw, TXGBE_IPSRXADDR(0), 0);
33                 wr32(hw, TXGBE_IPSRXADDR(1), 0);
34                 wr32(hw, TXGBE_IPSRXADDR(2), 0);
35                 wr32(hw, TXGBE_IPSRXADDR(3), 0);
36                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
37         }
38
39         /* clear Rx SPI and Rx/Tx SA tables*/
40         for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
41                 uint32_t index = i << 3;
42                 uint32_t reg_val = TXGBE_IPSRXIDX_WRITE |
43                                 TXGBE_IPSRXIDX_TB_SPI | index;
44                 wr32(hw, TXGBE_IPSRXSPI, 0);
45                 wr32(hw, TXGBE_IPSRXADDRIDX, 0);
46                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
47                 reg_val = TXGBE_IPSRXIDX_WRITE | TXGBE_IPSRXIDX_TB_KEY | index;
48                 wr32(hw, TXGBE_IPSRXKEY(0), 0);
49                 wr32(hw, TXGBE_IPSRXKEY(1), 0);
50                 wr32(hw, TXGBE_IPSRXKEY(2), 0);
51                 wr32(hw, TXGBE_IPSRXKEY(3), 0);
52                 wr32(hw, TXGBE_IPSRXSALT, 0);
53                 wr32(hw, TXGBE_IPSRXMODE, 0);
54                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
55                 reg_val = TXGBE_IPSTXIDX_WRITE | index;
56                 wr32(hw, TXGBE_IPSTXKEY(0), 0);
57                 wr32(hw, TXGBE_IPSTXKEY(1), 0);
58                 wr32(hw, TXGBE_IPSTXKEY(2), 0);
59                 wr32(hw, TXGBE_IPSTXKEY(3), 0);
60                 wr32(hw, TXGBE_IPSTXSALT, 0);
61                 wr32w(hw, TXGBE_IPSTXIDX, reg_val, TXGBE_IPSTXIDX_WRITE, 1000);
62         }
63
64         memset(priv->rx_ip_tbl, 0, sizeof(priv->rx_ip_tbl));
65         memset(priv->rx_sa_tbl, 0, sizeof(priv->rx_sa_tbl));
66         memset(priv->tx_sa_tbl, 0, sizeof(priv->tx_sa_tbl));
67 }
68
69 static int
70 txgbe_crypto_add_sa(struct txgbe_crypto_session *ic_session)
71 {
72         struct rte_eth_dev *dev = ic_session->dev;
73         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
74         struct txgbe_ipsec *priv = TXGBE_DEV_IPSEC(dev);
75         uint32_t reg_val;
76         int sa_index = -1;
77
78         if (ic_session->op == TXGBE_OP_AUTHENTICATED_DECRYPTION) {
79                 int i, ip_index = -1;
80                 uint8_t *key;
81
82                 /* Find a match in the IP table*/
83                 for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
84                         if (CMP_IP(priv->rx_ip_tbl[i].ip,
85                                    ic_session->dst_ip)) {
86                                 ip_index = i;
87                                 break;
88                         }
89                 }
90                 /* If no match, find a free entry in the IP table*/
91                 if (ip_index < 0) {
92                         for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
93                                 if (priv->rx_ip_tbl[i].ref_count == 0) {
94                                         ip_index = i;
95                                         break;
96                                 }
97                         }
98                 }
99
100                 /* Fail if no match and no free entries*/
101                 if (ip_index < 0) {
102                         PMD_DRV_LOG(ERR,
103                                     "No free entry left in the Rx IP table\n");
104                         return -1;
105                 }
106
107                 /* Find a free entry in the SA table*/
108                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
109                         if (priv->rx_sa_tbl[i].used == 0) {
110                                 sa_index = i;
111                                 break;
112                         }
113                 }
114                 /* Fail if no free entries*/
115                 if (sa_index < 0) {
116                         PMD_DRV_LOG(ERR,
117                                     "No free entry left in the Rx SA table\n");
118                         return -1;
119                 }
120
121                 priv->rx_ip_tbl[ip_index].ip.ipv6[0] =
122                                 ic_session->dst_ip.ipv6[0];
123                 priv->rx_ip_tbl[ip_index].ip.ipv6[1] =
124                                 ic_session->dst_ip.ipv6[1];
125                 priv->rx_ip_tbl[ip_index].ip.ipv6[2] =
126                                 ic_session->dst_ip.ipv6[2];
127                 priv->rx_ip_tbl[ip_index].ip.ipv6[3] =
128                                 ic_session->dst_ip.ipv6[3];
129                 priv->rx_ip_tbl[ip_index].ref_count++;
130
131                 priv->rx_sa_tbl[sa_index].spi = ic_session->spi;
132                 priv->rx_sa_tbl[sa_index].ip_index = ip_index;
133                 priv->rx_sa_tbl[sa_index].mode = IPSRXMOD_VALID;
134                 if (ic_session->op == TXGBE_OP_AUTHENTICATED_DECRYPTION)
135                         priv->rx_sa_tbl[sa_index].mode |=
136                                         (IPSRXMOD_PROTO | IPSRXMOD_DECRYPT);
137                 if (ic_session->dst_ip.type == IPv6) {
138                         priv->rx_sa_tbl[sa_index].mode |= IPSRXMOD_IPV6;
139                         priv->rx_ip_tbl[ip_index].ip.type = IPv6;
140                 } else if (ic_session->dst_ip.type == IPv4) {
141                         priv->rx_ip_tbl[ip_index].ip.type = IPv4;
142                 }
143                 priv->rx_sa_tbl[sa_index].used = 1;
144
145                 /* write IP table entry*/
146                 reg_val = TXGBE_IPSRXIDX_ENA | TXGBE_IPSRXIDX_WRITE |
147                                 TXGBE_IPSRXIDX_TB_IP | (ip_index << 3);
148                 if (priv->rx_ip_tbl[ip_index].ip.type == IPv4) {
149                         uint32_t ipv4 = priv->rx_ip_tbl[ip_index].ip.ipv4;
150                         wr32(hw, TXGBE_IPSRXADDR(0), rte_cpu_to_be_32(ipv4));
151                         wr32(hw, TXGBE_IPSRXADDR(1), 0);
152                         wr32(hw, TXGBE_IPSRXADDR(2), 0);
153                         wr32(hw, TXGBE_IPSRXADDR(3), 0);
154                 } else {
155                         wr32(hw, TXGBE_IPSRXADDR(0),
156                                         priv->rx_ip_tbl[ip_index].ip.ipv6[0]);
157                         wr32(hw, TXGBE_IPSRXADDR(1),
158                                         priv->rx_ip_tbl[ip_index].ip.ipv6[1]);
159                         wr32(hw, TXGBE_IPSRXADDR(2),
160                                         priv->rx_ip_tbl[ip_index].ip.ipv6[2]);
161                         wr32(hw, TXGBE_IPSRXADDR(3),
162                                         priv->rx_ip_tbl[ip_index].ip.ipv6[3]);
163                 }
164                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
165
166                 /* write SPI table entry*/
167                 reg_val = TXGBE_IPSRXIDX_ENA | TXGBE_IPSRXIDX_WRITE |
168                                 TXGBE_IPSRXIDX_TB_SPI | (sa_index << 3);
169                 wr32(hw, TXGBE_IPSRXSPI,
170                                 priv->rx_sa_tbl[sa_index].spi);
171                 wr32(hw, TXGBE_IPSRXADDRIDX,
172                                 priv->rx_sa_tbl[sa_index].ip_index);
173                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
174
175                 /* write Key table entry*/
176                 key = malloc(ic_session->key_len);
177                 if (!key)
178                         return -ENOMEM;
179
180                 memcpy(key, ic_session->key, ic_session->key_len);
181
182                 reg_val = TXGBE_IPSRXIDX_ENA | TXGBE_IPSRXIDX_WRITE |
183                                 TXGBE_IPSRXIDX_TB_KEY | (sa_index << 3);
184                 wr32(hw, TXGBE_IPSRXKEY(0),
185                         rte_cpu_to_be_32(*(uint32_t *)&key[12]));
186                 wr32(hw, TXGBE_IPSRXKEY(1),
187                         rte_cpu_to_be_32(*(uint32_t *)&key[8]));
188                 wr32(hw, TXGBE_IPSRXKEY(2),
189                         rte_cpu_to_be_32(*(uint32_t *)&key[4]));
190                 wr32(hw, TXGBE_IPSRXKEY(3),
191                         rte_cpu_to_be_32(*(uint32_t *)&key[0]));
192                 wr32(hw, TXGBE_IPSRXSALT,
193                                 rte_cpu_to_be_32(ic_session->salt));
194                 wr32(hw, TXGBE_IPSRXMODE,
195                                 priv->rx_sa_tbl[sa_index].mode);
196                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
197
198                 free(key);
199         } else { /* sess->dir == RTE_CRYPTO_OUTBOUND */
200                 uint8_t *key;
201                 int i;
202
203                 /* Find a free entry in the SA table*/
204                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
205                         if (priv->tx_sa_tbl[i].used == 0) {
206                                 sa_index = i;
207                                 break;
208                         }
209                 }
210                 /* Fail if no free entries*/
211                 if (sa_index < 0) {
212                         PMD_DRV_LOG(ERR,
213                                     "No free entry left in the Tx SA table\n");
214                         return -1;
215                 }
216
217                 priv->tx_sa_tbl[sa_index].spi =
218                         rte_cpu_to_be_32(ic_session->spi);
219                 priv->tx_sa_tbl[i].used = 1;
220                 ic_session->sa_index = sa_index;
221
222                 key = malloc(ic_session->key_len);
223                 if (!key)
224                         return -ENOMEM;
225
226                 memcpy(key, ic_session->key, ic_session->key_len);
227
228                 /* write Key table entry*/
229                 reg_val = TXGBE_IPSRXIDX_ENA |
230                         TXGBE_IPSRXIDX_WRITE | (sa_index << 3);
231                 wr32(hw, TXGBE_IPSTXKEY(0),
232                         rte_cpu_to_be_32(*(uint32_t *)&key[12]));
233                 wr32(hw, TXGBE_IPSTXKEY(1),
234                         rte_cpu_to_be_32(*(uint32_t *)&key[8]));
235                 wr32(hw, TXGBE_IPSTXKEY(2),
236                         rte_cpu_to_be_32(*(uint32_t *)&key[4]));
237                 wr32(hw, TXGBE_IPSTXKEY(3),
238                         rte_cpu_to_be_32(*(uint32_t *)&key[0]));
239                 wr32(hw, TXGBE_IPSTXSALT,
240                                 rte_cpu_to_be_32(ic_session->salt));
241                 wr32w(hw, TXGBE_IPSTXIDX, reg_val, TXGBE_IPSTXIDX_WRITE, 1000);
242
243                 free(key);
244         }
245
246         return 0;
247 }
248
249 static int
250 txgbe_crypto_remove_sa(struct rte_eth_dev *dev,
251                        struct txgbe_crypto_session *ic_session)
252 {
253         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
254         struct txgbe_ipsec *priv = TXGBE_DEV_IPSEC(dev);
255         uint32_t reg_val;
256         int sa_index = -1;
257
258         if (ic_session->op == TXGBE_OP_AUTHENTICATED_DECRYPTION) {
259                 int i, ip_index = -1;
260
261                 /* Find a match in the IP table*/
262                 for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
263                         if (CMP_IP(priv->rx_ip_tbl[i].ip, ic_session->dst_ip)) {
264                                 ip_index = i;
265                                 break;
266                         }
267                 }
268
269                 /* Fail if no match*/
270                 if (ip_index < 0) {
271                         PMD_DRV_LOG(ERR,
272                                     "Entry not found in the Rx IP table\n");
273                         return -1;
274                 }
275
276                 /* Find a free entry in the SA table*/
277                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
278                         if (priv->rx_sa_tbl[i].spi ==
279                                   rte_cpu_to_be_32(ic_session->spi)) {
280                                 sa_index = i;
281                                 break;
282                         }
283                 }
284                 /* Fail if no match*/
285                 if (sa_index < 0) {
286                         PMD_DRV_LOG(ERR,
287                                     "Entry not found in the Rx SA table\n");
288                         return -1;
289                 }
290
291                 /* Disable and clear Rx SPI and key table entryes*/
292                 reg_val = TXGBE_IPSRXIDX_WRITE |
293                         TXGBE_IPSRXIDX_TB_SPI | (sa_index << 3);
294                 wr32(hw, TXGBE_IPSRXSPI, 0);
295                 wr32(hw, TXGBE_IPSRXADDRIDX, 0);
296                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
297                 reg_val = TXGBE_IPSRXIDX_WRITE |
298                         TXGBE_IPSRXIDX_TB_KEY | (sa_index << 3);
299                 wr32(hw, TXGBE_IPSRXKEY(0), 0);
300                 wr32(hw, TXGBE_IPSRXKEY(1), 0);
301                 wr32(hw, TXGBE_IPSRXKEY(2), 0);
302                 wr32(hw, TXGBE_IPSRXKEY(3), 0);
303                 wr32(hw, TXGBE_IPSRXSALT, 0);
304                 wr32(hw, TXGBE_IPSRXMODE, 0);
305                 wr32w(hw, TXGBE_IPSRXIDX, reg_val, TXGBE_IPSRXIDX_WRITE, 1000);
306                 priv->rx_sa_tbl[sa_index].used = 0;
307
308                 /* If last used then clear the IP table entry*/
309                 priv->rx_ip_tbl[ip_index].ref_count--;
310                 if (priv->rx_ip_tbl[ip_index].ref_count == 0) {
311                         reg_val = TXGBE_IPSRXIDX_WRITE | TXGBE_IPSRXIDX_TB_IP |
312                                         (ip_index << 3);
313                         wr32(hw, TXGBE_IPSRXADDR(0), 0);
314                         wr32(hw, TXGBE_IPSRXADDR(1), 0);
315                         wr32(hw, TXGBE_IPSRXADDR(2), 0);
316                         wr32(hw, TXGBE_IPSRXADDR(3), 0);
317                 }
318         } else { /* session->dir == RTE_CRYPTO_OUTBOUND */
319                 int i;
320
321                 /* Find a match in the SA table*/
322                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
323                         if (priv->tx_sa_tbl[i].spi ==
324                                     rte_cpu_to_be_32(ic_session->spi)) {
325                                 sa_index = i;
326                                 break;
327                         }
328                 }
329                 /* Fail if no match entries*/
330                 if (sa_index < 0) {
331                         PMD_DRV_LOG(ERR,
332                                     "Entry not found in the Tx SA table\n");
333                         return -1;
334                 }
335                 reg_val = TXGBE_IPSRXIDX_WRITE | (sa_index << 3);
336                 wr32(hw, TXGBE_IPSTXKEY(0), 0);
337                 wr32(hw, TXGBE_IPSTXKEY(1), 0);
338                 wr32(hw, TXGBE_IPSTXKEY(2), 0);
339                 wr32(hw, TXGBE_IPSTXKEY(3), 0);
340                 wr32(hw, TXGBE_IPSTXSALT, 0);
341                 wr32w(hw, TXGBE_IPSTXIDX, reg_val, TXGBE_IPSTXIDX_WRITE, 1000);
342
343                 priv->tx_sa_tbl[sa_index].used = 0;
344         }
345
346         return 0;
347 }
348
349 static int
350 txgbe_crypto_create_session(void *device,
351                 struct rte_security_session_conf *conf,
352                 struct rte_security_session *session,
353                 struct rte_mempool *mempool)
354 {
355         struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)device;
356         struct txgbe_crypto_session *ic_session = NULL;
357         struct rte_crypto_aead_xform *aead_xform;
358         struct rte_eth_conf *dev_conf = &eth_dev->data->dev_conf;
359
360         if (rte_mempool_get(mempool, (void **)&ic_session)) {
361                 PMD_DRV_LOG(ERR, "Cannot get object from ic_session mempool");
362                 return -ENOMEM;
363         }
364
365         if (conf->crypto_xform->type != RTE_CRYPTO_SYM_XFORM_AEAD ||
366                         conf->crypto_xform->aead.algo !=
367                                         RTE_CRYPTO_AEAD_AES_GCM) {
368                 PMD_DRV_LOG(ERR, "Unsupported crypto transformation mode\n");
369                 rte_mempool_put(mempool, (void *)ic_session);
370                 return -ENOTSUP;
371         }
372         aead_xform = &conf->crypto_xform->aead;
373
374         if (conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
375                 if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_SECURITY) {
376                         ic_session->op = TXGBE_OP_AUTHENTICATED_DECRYPTION;
377                 } else {
378                         PMD_DRV_LOG(ERR, "IPsec decryption not enabled\n");
379                         rte_mempool_put(mempool, (void *)ic_session);
380                         return -ENOTSUP;
381                 }
382         } else {
383                 if (dev_conf->txmode.offloads & DEV_TX_OFFLOAD_SECURITY) {
384                         ic_session->op = TXGBE_OP_AUTHENTICATED_ENCRYPTION;
385                 } else {
386                         PMD_DRV_LOG(ERR, "IPsec encryption not enabled\n");
387                         rte_mempool_put(mempool, (void *)ic_session);
388                         return -ENOTSUP;
389                 }
390         }
391
392         ic_session->key = aead_xform->key.data;
393         ic_session->key_len = aead_xform->key.length;
394         memcpy(&ic_session->salt,
395                &aead_xform->key.data[aead_xform->key.length], 4);
396         ic_session->spi = conf->ipsec.spi;
397         ic_session->dev = eth_dev;
398
399         set_sec_session_private_data(session, ic_session);
400
401         if (ic_session->op == TXGBE_OP_AUTHENTICATED_ENCRYPTION) {
402                 if (txgbe_crypto_add_sa(ic_session)) {
403                         PMD_DRV_LOG(ERR, "Failed to add SA\n");
404                         rte_mempool_put(mempool, (void *)ic_session);
405                         return -EPERM;
406                 }
407         }
408
409         return 0;
410 }
411
412 static unsigned int
413 txgbe_crypto_session_get_size(__rte_unused void *device)
414 {
415         return sizeof(struct txgbe_crypto_session);
416 }
417
418 static int
419 txgbe_crypto_remove_session(void *device,
420                 struct rte_security_session *session)
421 {
422         struct rte_eth_dev *eth_dev = device;
423         struct txgbe_crypto_session *ic_session =
424                 (struct txgbe_crypto_session *)
425                 get_sec_session_private_data(session);
426         struct rte_mempool *mempool = rte_mempool_from_obj(ic_session);
427
428         if (eth_dev != ic_session->dev) {
429                 PMD_DRV_LOG(ERR, "Session not bound to this device\n");
430                 return -ENODEV;
431         }
432
433         if (txgbe_crypto_remove_sa(eth_dev, ic_session)) {
434                 PMD_DRV_LOG(ERR, "Failed to remove session\n");
435                 return -EFAULT;
436         }
437
438         rte_mempool_put(mempool, (void *)ic_session);
439
440         return 0;
441 }
442
443 static inline uint8_t
444 txgbe_crypto_compute_pad_len(struct rte_mbuf *m)
445 {
446         if (m->nb_segs == 1) {
447                 /* 16 bytes ICV + 2 bytes ESP trailer + payload padding size
448                  * payload padding size is stored at <pkt_len - 18>
449                  */
450                 uint8_t *esp_pad_len = rte_pktmbuf_mtod_offset(m, uint8_t *,
451                                         rte_pktmbuf_pkt_len(m) -
452                                         (ESP_TRAILER_SIZE + ESP_ICV_SIZE));
453                 return *esp_pad_len + ESP_TRAILER_SIZE + ESP_ICV_SIZE;
454         }
455         return 0;
456 }
457
458 static int
459 txgbe_crypto_update_mb(void *device __rte_unused,
460                 struct rte_security_session *session,
461                        struct rte_mbuf *m, void *params __rte_unused)
462 {
463         struct txgbe_crypto_session *ic_session =
464                         get_sec_session_private_data(session);
465         if (ic_session->op == TXGBE_OP_AUTHENTICATED_ENCRYPTION) {
466                 union txgbe_crypto_tx_desc_md *mdata =
467                         (union txgbe_crypto_tx_desc_md *)
468                                 rte_security_dynfield(m);
469                 mdata->enc = 1;
470                 mdata->sa_idx = ic_session->sa_index;
471                 mdata->pad_len = txgbe_crypto_compute_pad_len(m);
472         }
473         return 0;
474 }
475
476 static const struct rte_security_capability *
477 txgbe_crypto_capabilities_get(void *device __rte_unused)
478 {
479         static const struct rte_cryptodev_capabilities
480         aes_gcm_gmac_crypto_capabilities[] = {
481                 {       /* AES GMAC (128-bit) */
482                         .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
483                         {.sym = {
484                                 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
485                                 {.auth = {
486                                         .algo = RTE_CRYPTO_AUTH_AES_GMAC,
487                                         .block_size = 16,
488                                         .key_size = {
489                                                 .min = 16,
490                                                 .max = 16,
491                                                 .increment = 0
492                                         },
493                                         .digest_size = {
494                                                 .min = 16,
495                                                 .max = 16,
496                                                 .increment = 0
497                                         },
498                                         .iv_size = {
499                                                 .min = 12,
500                                                 .max = 12,
501                                                 .increment = 0
502                                         }
503                                 }, }
504                         }, }
505                 },
506                 {       /* AES GCM (128-bit) */
507                         .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
508                         {.sym = {
509                                 .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
510                                 {.aead = {
511                                         .algo = RTE_CRYPTO_AEAD_AES_GCM,
512                                         .block_size = 16,
513                                         .key_size = {
514                                                 .min = 16,
515                                                 .max = 16,
516                                                 .increment = 0
517                                         },
518                                         .digest_size = {
519                                                 .min = 16,
520                                                 .max = 16,
521                                                 .increment = 0
522                                         },
523                                         .aad_size = {
524                                                 .min = 0,
525                                                 .max = 65535,
526                                                 .increment = 1
527                                         },
528                                         .iv_size = {
529                                                 .min = 12,
530                                                 .max = 12,
531                                                 .increment = 0
532                                         }
533                                 }, }
534                         }, }
535                 },
536                 {
537                         .op = RTE_CRYPTO_OP_TYPE_UNDEFINED,
538                         {.sym = {
539                                 .xform_type = RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED
540                         }, }
541                 },
542         };
543
544         static const struct rte_security_capability
545         txgbe_security_capabilities[] = {
546                 { /* IPsec Inline Crypto ESP Transport Egress */
547                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
548                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
549                         {.ipsec = {
550                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
551                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
552                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
553                                 .options = { 0 }
554                         } },
555                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
556                         .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
557                 },
558                 { /* IPsec Inline Crypto ESP Transport Ingress */
559                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
560                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
561                         {.ipsec = {
562                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
563                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
564                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
565                                 .options = { 0 }
566                         } },
567                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
568                         .ol_flags = 0
569                 },
570                 { /* IPsec Inline Crypto ESP Tunnel Egress */
571                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
572                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
573                         {.ipsec = {
574                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
575                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
576                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
577                                 .options = { 0 }
578                         } },
579                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
580                         .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
581                 },
582                 { /* IPsec Inline Crypto ESP Tunnel Ingress */
583                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
584                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
585                         {.ipsec = {
586                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
587                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
588                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
589                                 .options = { 0 }
590                         } },
591                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
592                         .ol_flags = 0
593                 },
594                 {
595                         .action = RTE_SECURITY_ACTION_TYPE_NONE
596                 }
597         };
598
599         return txgbe_security_capabilities;
600 }
601
602 int
603 txgbe_crypto_enable_ipsec(struct rte_eth_dev *dev)
604 {
605         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
606         uint32_t reg;
607         uint64_t rx_offloads;
608         uint64_t tx_offloads;
609
610         rx_offloads = dev->data->dev_conf.rxmode.offloads;
611         tx_offloads = dev->data->dev_conf.txmode.offloads;
612
613         /* sanity checks */
614         if (rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) {
615                 PMD_DRV_LOG(ERR, "RSC and IPsec not supported");
616                 return -1;
617         }
618         if (rx_offloads & DEV_RX_OFFLOAD_KEEP_CRC) {
619                 PMD_DRV_LOG(ERR, "HW CRC strip needs to be enabled for IPsec");
620                 return -1;
621         }
622
623         /* Set TXGBE_SECTXBUFFAF to 0x14 as required in the datasheet*/
624         wr32(hw, TXGBE_SECTXBUFAF, 0x14);
625
626         /* IFG needs to be set to 3 when we are using security. Otherwise a Tx
627          * hang will occur with heavy traffic.
628          */
629         reg = rd32(hw, TXGBE_SECTXIFG);
630         reg = (reg & ~TXGBE_SECTXIFG_MIN_MASK) | TXGBE_SECTXIFG_MIN(0x3);
631         wr32(hw, TXGBE_SECTXIFG, reg);
632
633         reg = rd32(hw, TXGBE_SECRXCTL);
634         reg |= TXGBE_SECRXCTL_CRCSTRIP;
635         wr32(hw, TXGBE_SECRXCTL, reg);
636
637         if (rx_offloads & DEV_RX_OFFLOAD_SECURITY) {
638                 wr32m(hw, TXGBE_SECRXCTL, TXGBE_SECRXCTL_ODSA, 0);
639                 reg = rd32m(hw, TXGBE_SECRXCTL, TXGBE_SECRXCTL_ODSA);
640                 if (reg != 0) {
641                         PMD_DRV_LOG(ERR, "Error enabling Rx Crypto");
642                         return -1;
643                 }
644         }
645         if (tx_offloads & DEV_TX_OFFLOAD_SECURITY) {
646                 wr32(hw, TXGBE_SECTXCTL, TXGBE_SECTXCTL_STFWD);
647                 reg = rd32(hw, TXGBE_SECTXCTL);
648                 if (reg != TXGBE_SECTXCTL_STFWD) {
649                         PMD_DRV_LOG(ERR, "Error enabling Rx Crypto");
650                         return -1;
651                 }
652         }
653
654         txgbe_crypto_clear_ipsec_tables(dev);
655
656         return 0;
657 }
658
659 int
660 txgbe_crypto_add_ingress_sa_from_flow(const void *sess,
661                                       const void *ip_spec,
662                                       uint8_t is_ipv6)
663 {
664         struct txgbe_crypto_session *ic_session =
665                         get_sec_session_private_data(sess);
666
667         if (ic_session->op == TXGBE_OP_AUTHENTICATED_DECRYPTION) {
668                 if (is_ipv6) {
669                         const struct rte_flow_item_ipv6 *ipv6 = ip_spec;
670                         ic_session->src_ip.type = IPv6;
671                         ic_session->dst_ip.type = IPv6;
672                         rte_memcpy(ic_session->src_ip.ipv6,
673                                    ipv6->hdr.src_addr, 16);
674                         rte_memcpy(ic_session->dst_ip.ipv6,
675                                    ipv6->hdr.dst_addr, 16);
676                 } else {
677                         const struct rte_flow_item_ipv4 *ipv4 = ip_spec;
678                         ic_session->src_ip.type = IPv4;
679                         ic_session->dst_ip.type = IPv4;
680                         ic_session->src_ip.ipv4 = ipv4->hdr.src_addr;
681                         ic_session->dst_ip.ipv4 = ipv4->hdr.dst_addr;
682                 }
683                 return txgbe_crypto_add_sa(ic_session);
684         }
685
686         return 0;
687 }
688
689 static struct rte_security_ops txgbe_security_ops = {
690         .session_create = txgbe_crypto_create_session,
691         .session_get_size = txgbe_crypto_session_get_size,
692         .session_destroy = txgbe_crypto_remove_session,
693         .set_pkt_metadata = txgbe_crypto_update_mb,
694         .capabilities_get = txgbe_crypto_capabilities_get
695 };
696
697 static int
698 txgbe_crypto_capable(struct rte_eth_dev *dev)
699 {
700         struct txgbe_hw *hw = TXGBE_DEV_HW(dev);
701         uint32_t reg_i, reg, capable = 1;
702         /* test if rx crypto can be enabled and then write back initial value*/
703         reg_i = rd32(hw, TXGBE_SECRXCTL);
704         wr32m(hw, TXGBE_SECRXCTL, TXGBE_SECRXCTL_ODSA, 0);
705         reg = rd32m(hw, TXGBE_SECRXCTL, TXGBE_SECRXCTL_ODSA);
706         if (reg != 0)
707                 capable = 0;
708         wr32(hw, TXGBE_SECRXCTL, reg_i);
709         return capable;
710 }
711
712 int
713 txgbe_ipsec_ctx_create(struct rte_eth_dev *dev)
714 {
715         struct rte_security_ctx *ctx = NULL;
716
717         if (txgbe_crypto_capable(dev)) {
718                 ctx = rte_malloc("rte_security_instances_ops",
719                                  sizeof(struct rte_security_ctx), 0);
720                 if (ctx) {
721                         ctx->device = (void *)dev;
722                         ctx->ops = &txgbe_security_ops;
723                         ctx->sess_cnt = 0;
724                         dev->security_ctx = ctx;
725                 } else {
726                         return -ENOMEM;
727                 }
728         }
729         if (rte_security_dynfield_register() < 0)
730                 return -rte_errno;
731         return 0;
732 }