vdpa/mlx5: support queue update
[dpdk.git] / drivers / net / ixgbe / ixgbe_ipsec.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2017 Intel Corporation
3  */
4
5 #include <rte_ethdev_driver.h>
6 #include <rte_ethdev_pci.h>
7 #include <rte_ip.h>
8 #include <rte_jhash.h>
9 #include <rte_security_driver.h>
10 #include <rte_cryptodev.h>
11 #include <rte_flow.h>
12
13 #include "base/ixgbe_type.h"
14 #include "base/ixgbe_api.h"
15 #include "ixgbe_ethdev.h"
16 #include "ixgbe_ipsec.h"
17
18 #define RTE_IXGBE_REGISTER_POLL_WAIT_5_MS  5
19
20 #define IXGBE_WAIT_RREAD \
21         IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSRXIDX, reg_val, \
22         IPSRXIDX_READ, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS)
23 #define IXGBE_WAIT_RWRITE \
24         IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSRXIDX, reg_val, \
25         IPSRXIDX_WRITE, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS)
26 #define IXGBE_WAIT_TREAD \
27         IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSTXIDX, reg_val, \
28         IPSRXIDX_READ, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS)
29 #define IXGBE_WAIT_TWRITE \
30         IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSTXIDX, reg_val, \
31         IPSRXIDX_WRITE, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS)
32
33 #define CMP_IP(a, b) (\
34         (a).ipv6[0] == (b).ipv6[0] && \
35         (a).ipv6[1] == (b).ipv6[1] && \
36         (a).ipv6[2] == (b).ipv6[2] && \
37         (a).ipv6[3] == (b).ipv6[3])
38
39
40 static void
41 ixgbe_crypto_clear_ipsec_tables(struct rte_eth_dev *dev)
42 {
43         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
44         struct ixgbe_ipsec *priv = IXGBE_DEV_PRIVATE_TO_IPSEC(
45                                 dev->data->dev_private);
46         int i = 0;
47
48         /* clear Rx IP table*/
49         for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
50                 uint16_t index = i << 3;
51                 uint32_t reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_IP | index;
52                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0), 0);
53                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1), 0);
54                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2), 0);
55                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3), 0);
56                 IXGBE_WAIT_RWRITE;
57         }
58
59         /* clear Rx SPI and Rx/Tx SA tables*/
60         for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
61                 uint32_t index = i << 3;
62                 uint32_t reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_SPI | index;
63                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSPI, 0);
64                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPIDX, 0);
65                 IXGBE_WAIT_RWRITE;
66                 reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_KEY | index;
67                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(0), 0);
68                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(1), 0);
69                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(2), 0);
70                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(3), 0);
71                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSALT, 0);
72                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXMOD, 0);
73                 IXGBE_WAIT_RWRITE;
74                 reg_val = IPSRXIDX_WRITE | index;
75                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(0), 0);
76                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(1), 0);
77                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(2), 0);
78                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(3), 0);
79                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXSALT, 0);
80                 IXGBE_WAIT_TWRITE;
81         }
82
83         memset(priv->rx_ip_tbl, 0, sizeof(priv->rx_ip_tbl));
84         memset(priv->rx_sa_tbl, 0, sizeof(priv->rx_sa_tbl));
85         memset(priv->tx_sa_tbl, 0, sizeof(priv->tx_sa_tbl));
86 }
87
88 static int
89 ixgbe_crypto_add_sa(struct ixgbe_crypto_session *ic_session)
90 {
91         struct rte_eth_dev *dev = ic_session->dev;
92         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
93         struct ixgbe_ipsec *priv = IXGBE_DEV_PRIVATE_TO_IPSEC(
94                         dev->data->dev_private);
95         uint32_t reg_val;
96         int sa_index = -1;
97
98         if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION) {
99                 int i, ip_index = -1;
100                 uint8_t *key;
101
102                 /* Find a match in the IP table*/
103                 for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
104                         if (CMP_IP(priv->rx_ip_tbl[i].ip,
105                                    ic_session->dst_ip)) {
106                                 ip_index = i;
107                                 break;
108                         }
109                 }
110                 /* If no match, find a free entry in the IP table*/
111                 if (ip_index < 0) {
112                         for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
113                                 if (priv->rx_ip_tbl[i].ref_count == 0) {
114                                         ip_index = i;
115                                         break;
116                                 }
117                         }
118                 }
119
120                 /* Fail if no match and no free entries*/
121                 if (ip_index < 0) {
122                         PMD_DRV_LOG(ERR,
123                                     "No free entry left in the Rx IP table\n");
124                         return -1;
125                 }
126
127                 /* Find a free entry in the SA table*/
128                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
129                         if (priv->rx_sa_tbl[i].used == 0) {
130                                 sa_index = i;
131                                 break;
132                         }
133                 }
134                 /* Fail if no free entries*/
135                 if (sa_index < 0) {
136                         PMD_DRV_LOG(ERR,
137                                     "No free entry left in the Rx SA table\n");
138                         return -1;
139                 }
140
141                 priv->rx_ip_tbl[ip_index].ip.ipv6[0] =
142                                 ic_session->dst_ip.ipv6[0];
143                 priv->rx_ip_tbl[ip_index].ip.ipv6[1] =
144                                 ic_session->dst_ip.ipv6[1];
145                 priv->rx_ip_tbl[ip_index].ip.ipv6[2] =
146                                 ic_session->dst_ip.ipv6[2];
147                 priv->rx_ip_tbl[ip_index].ip.ipv6[3] =
148                                 ic_session->dst_ip.ipv6[3];
149                 priv->rx_ip_tbl[ip_index].ref_count++;
150
151                 priv->rx_sa_tbl[sa_index].spi =
152                         rte_cpu_to_be_32(ic_session->spi);
153                 priv->rx_sa_tbl[sa_index].ip_index = ip_index;
154                 priv->rx_sa_tbl[sa_index].mode = IPSRXMOD_VALID;
155                 if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION)
156                         priv->rx_sa_tbl[sa_index].mode |=
157                                         (IPSRXMOD_PROTO | IPSRXMOD_DECRYPT);
158                 if (ic_session->dst_ip.type == IPv6) {
159                         priv->rx_sa_tbl[sa_index].mode |= IPSRXMOD_IPV6;
160                         priv->rx_ip_tbl[ip_index].ip.type = IPv6;
161                 } else if (ic_session->dst_ip.type == IPv4)
162                         priv->rx_ip_tbl[ip_index].ip.type = IPv4;
163
164                 priv->rx_sa_tbl[sa_index].used = 1;
165
166                 /* write IP table entry*/
167                 reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE |
168                                 IPSRXIDX_TABLE_IP | (ip_index << 3);
169                 if (priv->rx_ip_tbl[ip_index].ip.type == IPv4) {
170                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0), 0);
171                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1), 0);
172                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2), 0);
173                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3),
174                                         priv->rx_ip_tbl[ip_index].ip.ipv4);
175                 } else {
176                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0),
177                                         priv->rx_ip_tbl[ip_index].ip.ipv6[0]);
178                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1),
179                                         priv->rx_ip_tbl[ip_index].ip.ipv6[1]);
180                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2),
181                                         priv->rx_ip_tbl[ip_index].ip.ipv6[2]);
182                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3),
183                                         priv->rx_ip_tbl[ip_index].ip.ipv6[3]);
184                 }
185                 IXGBE_WAIT_RWRITE;
186
187                 /* write SPI table entry*/
188                 reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE |
189                                 IPSRXIDX_TABLE_SPI | (sa_index << 3);
190                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSPI,
191                                 priv->rx_sa_tbl[sa_index].spi);
192                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPIDX,
193                                 priv->rx_sa_tbl[sa_index].ip_index);
194                 IXGBE_WAIT_RWRITE;
195
196                 /* write Key table entry*/
197                 key = malloc(ic_session->key_len);
198                 if (!key)
199                         return -ENOMEM;
200
201                 memcpy(key, ic_session->key, ic_session->key_len);
202
203                 reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE |
204                                 IPSRXIDX_TABLE_KEY | (sa_index << 3);
205                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(0),
206                         rte_cpu_to_be_32(*(uint32_t *)&key[12]));
207                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(1),
208                         rte_cpu_to_be_32(*(uint32_t *)&key[8]));
209                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(2),
210                         rte_cpu_to_be_32(*(uint32_t *)&key[4]));
211                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(3),
212                         rte_cpu_to_be_32(*(uint32_t *)&key[0]));
213                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSALT,
214                                 rte_cpu_to_be_32(ic_session->salt));
215                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXMOD,
216                                 priv->rx_sa_tbl[sa_index].mode);
217                 IXGBE_WAIT_RWRITE;
218
219                 free(key);
220
221         } else { /* sess->dir == RTE_CRYPTO_OUTBOUND */
222                 uint8_t *key;
223                 int i;
224
225                 /* Find a free entry in the SA table*/
226                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
227                         if (priv->tx_sa_tbl[i].used == 0) {
228                                 sa_index = i;
229                                 break;
230                         }
231                 }
232                 /* Fail if no free entries*/
233                 if (sa_index < 0) {
234                         PMD_DRV_LOG(ERR,
235                                     "No free entry left in the Tx SA table\n");
236                         return -1;
237                 }
238
239                 priv->tx_sa_tbl[sa_index].spi =
240                         rte_cpu_to_be_32(ic_session->spi);
241                 priv->tx_sa_tbl[i].used = 1;
242                 ic_session->sa_index = sa_index;
243
244                 key = malloc(ic_session->key_len);
245                 if (!key)
246                         return -ENOMEM;
247
248                 memcpy(key, ic_session->key, ic_session->key_len);
249
250                 /* write Key table entry*/
251                 reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE | (sa_index << 3);
252                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(0),
253                         rte_cpu_to_be_32(*(uint32_t *)&key[12]));
254                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(1),
255                         rte_cpu_to_be_32(*(uint32_t *)&key[8]));
256                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(2),
257                         rte_cpu_to_be_32(*(uint32_t *)&key[4]));
258                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(3),
259                         rte_cpu_to_be_32(*(uint32_t *)&key[0]));
260                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXSALT,
261                                 rte_cpu_to_be_32(ic_session->salt));
262                 IXGBE_WAIT_TWRITE;
263
264                 free(key);
265         }
266
267         return 0;
268 }
269
270 static int
271 ixgbe_crypto_remove_sa(struct rte_eth_dev *dev,
272                        struct ixgbe_crypto_session *ic_session)
273 {
274         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
275         struct ixgbe_ipsec *priv =
276                         IXGBE_DEV_PRIVATE_TO_IPSEC(dev->data->dev_private);
277         uint32_t reg_val;
278         int sa_index = -1;
279
280         if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION) {
281                 int i, ip_index = -1;
282
283                 /* Find a match in the IP table*/
284                 for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
285                         if (CMP_IP(priv->rx_ip_tbl[i].ip, ic_session->dst_ip)) {
286                                 ip_index = i;
287                                 break;
288                         }
289                 }
290
291                 /* Fail if no match*/
292                 if (ip_index < 0) {
293                         PMD_DRV_LOG(ERR,
294                                     "Entry not found in the Rx IP table\n");
295                         return -1;
296                 }
297
298                 /* Find a free entry in the SA table*/
299                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
300                         if (priv->rx_sa_tbl[i].spi ==
301                                   rte_cpu_to_be_32(ic_session->spi)) {
302                                 sa_index = i;
303                                 break;
304                         }
305                 }
306                 /* Fail if no match*/
307                 if (sa_index < 0) {
308                         PMD_DRV_LOG(ERR,
309                                     "Entry not found in the Rx SA table\n");
310                         return -1;
311                 }
312
313                 /* Disable and clear Rx SPI and key table table entryes*/
314                 reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_SPI | (sa_index << 3);
315                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSPI, 0);
316                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPIDX, 0);
317                 IXGBE_WAIT_RWRITE;
318                 reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_KEY | (sa_index << 3);
319                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(0), 0);
320                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(1), 0);
321                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(2), 0);
322                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(3), 0);
323                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSALT, 0);
324                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXMOD, 0);
325                 IXGBE_WAIT_RWRITE;
326                 priv->rx_sa_tbl[sa_index].used = 0;
327
328                 /* If last used then clear the IP table entry*/
329                 priv->rx_ip_tbl[ip_index].ref_count--;
330                 if (priv->rx_ip_tbl[ip_index].ref_count == 0) {
331                         reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_IP |
332                                         (ip_index << 3);
333                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0), 0);
334                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1), 0);
335                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2), 0);
336                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3), 0);
337                 }
338         } else { /* session->dir == RTE_CRYPTO_OUTBOUND */
339                 int i;
340
341                 /* Find a match in the SA table*/
342                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
343                         if (priv->tx_sa_tbl[i].spi ==
344                                     rte_cpu_to_be_32(ic_session->spi)) {
345                                 sa_index = i;
346                                 break;
347                         }
348                 }
349                 /* Fail if no match entries*/
350                 if (sa_index < 0) {
351                         PMD_DRV_LOG(ERR,
352                                     "Entry not found in the Tx SA table\n");
353                         return -1;
354                 }
355                 reg_val = IPSRXIDX_WRITE | (sa_index << 3);
356                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(0), 0);
357                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(1), 0);
358                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(2), 0);
359                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(3), 0);
360                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXSALT, 0);
361                 IXGBE_WAIT_TWRITE;
362
363                 priv->tx_sa_tbl[sa_index].used = 0;
364         }
365
366         return 0;
367 }
368
369 static int
370 ixgbe_crypto_create_session(void *device,
371                 struct rte_security_session_conf *conf,
372                 struct rte_security_session *session,
373                 struct rte_mempool *mempool)
374 {
375         struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)device;
376         struct ixgbe_crypto_session *ic_session = NULL;
377         struct rte_crypto_aead_xform *aead_xform;
378         struct rte_eth_conf *dev_conf = &eth_dev->data->dev_conf;
379
380         if (rte_mempool_get(mempool, (void **)&ic_session)) {
381                 PMD_DRV_LOG(ERR, "Cannot get object from ic_session mempool");
382                 return -ENOMEM;
383         }
384
385         if (conf->crypto_xform->type != RTE_CRYPTO_SYM_XFORM_AEAD ||
386                         conf->crypto_xform->aead.algo !=
387                                         RTE_CRYPTO_AEAD_AES_GCM) {
388                 PMD_DRV_LOG(ERR, "Unsupported crypto transformation mode\n");
389                 rte_mempool_put(mempool, (void *)ic_session);
390                 return -ENOTSUP;
391         }
392         aead_xform = &conf->crypto_xform->aead;
393
394         if (conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
395                 if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_SECURITY) {
396                         ic_session->op = IXGBE_OP_AUTHENTICATED_DECRYPTION;
397                 } else {
398                         PMD_DRV_LOG(ERR, "IPsec decryption not enabled\n");
399                         rte_mempool_put(mempool, (void *)ic_session);
400                         return -ENOTSUP;
401                 }
402         } else {
403                 if (dev_conf->txmode.offloads & DEV_TX_OFFLOAD_SECURITY) {
404                         ic_session->op = IXGBE_OP_AUTHENTICATED_ENCRYPTION;
405                 } else {
406                         PMD_DRV_LOG(ERR, "IPsec encryption not enabled\n");
407                         rte_mempool_put(mempool, (void *)ic_session);
408                         return -ENOTSUP;
409                 }
410         }
411
412         ic_session->key = aead_xform->key.data;
413         ic_session->key_len = aead_xform->key.length;
414         memcpy(&ic_session->salt,
415                &aead_xform->key.data[aead_xform->key.length], 4);
416         ic_session->spi = conf->ipsec.spi;
417         ic_session->dev = eth_dev;
418
419         set_sec_session_private_data(session, ic_session);
420
421         if (ic_session->op == IXGBE_OP_AUTHENTICATED_ENCRYPTION) {
422                 if (ixgbe_crypto_add_sa(ic_session)) {
423                         PMD_DRV_LOG(ERR, "Failed to add SA\n");
424                         rte_mempool_put(mempool, (void *)ic_session);
425                         return -EPERM;
426                 }
427         }
428
429         return 0;
430 }
431
432 static unsigned int
433 ixgbe_crypto_session_get_size(__rte_unused void *device)
434 {
435         return sizeof(struct ixgbe_crypto_session);
436 }
437
438 static int
439 ixgbe_crypto_remove_session(void *device,
440                 struct rte_security_session *session)
441 {
442         struct rte_eth_dev *eth_dev = device;
443         struct ixgbe_crypto_session *ic_session =
444                 (struct ixgbe_crypto_session *)
445                 get_sec_session_private_data(session);
446         struct rte_mempool *mempool = rte_mempool_from_obj(ic_session);
447
448         if (eth_dev != ic_session->dev) {
449                 PMD_DRV_LOG(ERR, "Session not bound to this device\n");
450                 return -ENODEV;
451         }
452
453         if (ixgbe_crypto_remove_sa(eth_dev, ic_session)) {
454                 PMD_DRV_LOG(ERR, "Failed to remove session\n");
455                 return -EFAULT;
456         }
457
458         rte_mempool_put(mempool, (void *)ic_session);
459
460         return 0;
461 }
462
463 static inline uint8_t
464 ixgbe_crypto_compute_pad_len(struct rte_mbuf *m)
465 {
466         if (m->nb_segs == 1) {
467                 /* 16 bytes ICV + 2 bytes ESP trailer + payload padding size
468                  * payload padding size is stored at <pkt_len - 18>
469                  */
470                 uint8_t *esp_pad_len = rte_pktmbuf_mtod_offset(m, uint8_t *,
471                                         rte_pktmbuf_pkt_len(m) -
472                                         (ESP_TRAILER_SIZE + ESP_ICV_SIZE));
473                 return *esp_pad_len + ESP_TRAILER_SIZE + ESP_ICV_SIZE;
474         }
475         return 0;
476 }
477
478 static int
479 ixgbe_crypto_update_mb(void *device __rte_unused,
480                 struct rte_security_session *session,
481                        struct rte_mbuf *m, void *params __rte_unused)
482 {
483         struct ixgbe_crypto_session *ic_session =
484                         get_sec_session_private_data(session);
485         if (ic_session->op == IXGBE_OP_AUTHENTICATED_ENCRYPTION) {
486                 union ixgbe_crypto_tx_desc_md *mdata =
487                         (union ixgbe_crypto_tx_desc_md *)&m->udata64;
488                 mdata->enc = 1;
489                 mdata->sa_idx = ic_session->sa_index;
490                 mdata->pad_len = ixgbe_crypto_compute_pad_len(m);
491         }
492         return 0;
493 }
494
495
496 static const struct rte_security_capability *
497 ixgbe_crypto_capabilities_get(void *device __rte_unused)
498 {
499         static const struct rte_cryptodev_capabilities
500         aes_gcm_gmac_crypto_capabilities[] = {
501                 {       /* AES GMAC (128-bit) */
502                         .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
503                         {.sym = {
504                                 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
505                                 {.auth = {
506                                         .algo = RTE_CRYPTO_AUTH_AES_GMAC,
507                                         .block_size = 16,
508                                         .key_size = {
509                                                 .min = 16,
510                                                 .max = 16,
511                                                 .increment = 0
512                                         },
513                                         .digest_size = {
514                                                 .min = 16,
515                                                 .max = 16,
516                                                 .increment = 0
517                                         },
518                                         .iv_size = {
519                                                 .min = 12,
520                                                 .max = 12,
521                                                 .increment = 0
522                                         }
523                                 }, }
524                         }, }
525                 },
526                 {       /* AES GCM (128-bit) */
527                         .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
528                         {.sym = {
529                                 .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
530                                 {.aead = {
531                                         .algo = RTE_CRYPTO_AEAD_AES_GCM,
532                                         .block_size = 16,
533                                         .key_size = {
534                                                 .min = 16,
535                                                 .max = 16,
536                                                 .increment = 0
537                                         },
538                                         .digest_size = {
539                                                 .min = 16,
540                                                 .max = 16,
541                                                 .increment = 0
542                                         },
543                                         .aad_size = {
544                                                 .min = 0,
545                                                 .max = 65535,
546                                                 .increment = 1
547                                         },
548                                         .iv_size = {
549                                                 .min = 12,
550                                                 .max = 12,
551                                                 .increment = 0
552                                         }
553                                 }, }
554                         }, }
555                 },
556                 {
557                         .op = RTE_CRYPTO_OP_TYPE_UNDEFINED,
558                         {.sym = {
559                                 .xform_type = RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED
560                         }, }
561                 },
562         };
563
564         static const struct rte_security_capability
565         ixgbe_security_capabilities[] = {
566                 { /* IPsec Inline Crypto ESP Transport Egress */
567                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
568                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
569                         {.ipsec = {
570                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
571                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
572                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
573                                 .options = { 0 }
574                         } },
575                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
576                         .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
577                 },
578                 { /* IPsec Inline Crypto ESP Transport Ingress */
579                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
580                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
581                         {.ipsec = {
582                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
583                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
584                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
585                                 .options = { 0 }
586                         } },
587                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
588                         .ol_flags = 0
589                 },
590                 { /* IPsec Inline Crypto ESP Tunnel Egress */
591                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
592                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
593                         {.ipsec = {
594                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
595                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
596                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
597                                 .options = { 0 }
598                         } },
599                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
600                         .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
601                 },
602                 { /* IPsec Inline Crypto ESP Tunnel Ingress */
603                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
604                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
605                         {.ipsec = {
606                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
607                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
608                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
609                                 .options = { 0 }
610                         } },
611                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
612                         .ol_flags = 0
613                 },
614                 {
615                         .action = RTE_SECURITY_ACTION_TYPE_NONE
616                 }
617         };
618
619         return ixgbe_security_capabilities;
620 }
621
622
623 int
624 ixgbe_crypto_enable_ipsec(struct rte_eth_dev *dev)
625 {
626         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
627         uint32_t reg;
628         uint64_t rx_offloads;
629         uint64_t tx_offloads;
630
631         rx_offloads = dev->data->dev_conf.rxmode.offloads;
632         tx_offloads = dev->data->dev_conf.txmode.offloads;
633
634         /* sanity checks */
635         if (rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) {
636                 PMD_DRV_LOG(ERR, "RSC and IPsec not supported");
637                 return -1;
638         }
639         if (rx_offloads & DEV_RX_OFFLOAD_KEEP_CRC) {
640                 PMD_DRV_LOG(ERR, "HW CRC strip needs to be enabled for IPsec");
641                 return -1;
642         }
643
644
645         /* Set IXGBE_SECTXBUFFAF to 0x15 as required in the datasheet*/
646         IXGBE_WRITE_REG(hw, IXGBE_SECTXBUFFAF, 0x15);
647
648         /* IFG needs to be set to 3 when we are using security. Otherwise a Tx
649          * hang will occur with heavy traffic.
650          */
651         reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
652         reg = (reg & 0xFFFFFFF0) | 0x3;
653         IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg);
654
655         reg  = IXGBE_READ_REG(hw, IXGBE_HLREG0);
656         reg |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_RXCRCSTRP;
657         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg);
658
659         if (rx_offloads & DEV_RX_OFFLOAD_SECURITY) {
660                 IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, 0);
661                 reg = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL);
662                 if (reg != 0) {
663                         PMD_DRV_LOG(ERR, "Error enabling Rx Crypto");
664                         return -1;
665                 }
666         }
667         if (tx_offloads & DEV_TX_OFFLOAD_SECURITY) {
668                 IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL,
669                                 IXGBE_SECTXCTRL_STORE_FORWARD);
670                 reg = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL);
671                 if (reg != IXGBE_SECTXCTRL_STORE_FORWARD) {
672                         PMD_DRV_LOG(ERR, "Error enabling Rx Crypto");
673                         return -1;
674                 }
675         }
676
677         ixgbe_crypto_clear_ipsec_tables(dev);
678
679         return 0;
680 }
681
682 int
683 ixgbe_crypto_add_ingress_sa_from_flow(const void *sess,
684                                       const void *ip_spec,
685                                       uint8_t is_ipv6)
686 {
687         struct ixgbe_crypto_session *ic_session
688                 = get_sec_session_private_data(sess);
689
690         if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION) {
691                 if (is_ipv6) {
692                         const struct rte_flow_item_ipv6 *ipv6 = ip_spec;
693                         ic_session->src_ip.type = IPv6;
694                         ic_session->dst_ip.type = IPv6;
695                         rte_memcpy(ic_session->src_ip.ipv6,
696                                    ipv6->hdr.src_addr, 16);
697                         rte_memcpy(ic_session->dst_ip.ipv6,
698                                    ipv6->hdr.dst_addr, 16);
699                 } else {
700                         const struct rte_flow_item_ipv4 *ipv4 = ip_spec;
701                         ic_session->src_ip.type = IPv4;
702                         ic_session->dst_ip.type = IPv4;
703                         ic_session->src_ip.ipv4 = ipv4->hdr.src_addr;
704                         ic_session->dst_ip.ipv4 = ipv4->hdr.dst_addr;
705                 }
706                 return ixgbe_crypto_add_sa(ic_session);
707         }
708
709         return 0;
710 }
711
712 static struct rte_security_ops ixgbe_security_ops = {
713         .session_create = ixgbe_crypto_create_session,
714         .session_update = NULL,
715         .session_get_size = ixgbe_crypto_session_get_size,
716         .session_stats_get = NULL,
717         .session_destroy = ixgbe_crypto_remove_session,
718         .set_pkt_metadata = ixgbe_crypto_update_mb,
719         .capabilities_get = ixgbe_crypto_capabilities_get
720 };
721
722 static int
723 ixgbe_crypto_capable(struct rte_eth_dev *dev)
724 {
725         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
726         uint32_t reg_i, reg, capable = 1;
727         /* test if rx crypto can be enabled and then write back initial value*/
728         reg_i = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL);
729         IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, 0);
730         reg = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL);
731         if (reg != 0)
732                 capable = 0;
733         IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, reg_i);
734         return capable;
735 }
736
737 int
738 ixgbe_ipsec_ctx_create(struct rte_eth_dev *dev)
739 {
740         struct rte_security_ctx *ctx = NULL;
741
742         if (ixgbe_crypto_capable(dev)) {
743                 ctx = rte_malloc("rte_security_instances_ops",
744                                  sizeof(struct rte_security_ctx), 0);
745                 if (ctx) {
746                         ctx->device = (void *)dev;
747                         ctx->ops = &ixgbe_security_ops;
748                         ctx->sess_cnt = 0;
749                         dev->security_ctx = ctx;
750                 } else {
751                         return -ENOMEM;
752                 }
753         }
754         return 0;
755 }