net/ixgbe: enable inline IPsec
[dpdk.git] / drivers / net / ixgbe / ixgbe_ipsec.c
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright(c) 2010-2017 Intel Corporation. All rights reserved.
5  *   All rights reserved.
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of Intel Corporation nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33
34 #include <rte_ethdev.h>
35 #include <rte_ethdev_pci.h>
36 #include <rte_ip.h>
37 #include <rte_jhash.h>
38 #include <rte_security_driver.h>
39 #include <rte_cryptodev.h>
40 #include <rte_flow.h>
41
42 #include "base/ixgbe_type.h"
43 #include "base/ixgbe_api.h"
44 #include "ixgbe_ethdev.h"
45 #include "ixgbe_ipsec.h"
46
47 #define RTE_IXGBE_REGISTER_POLL_WAIT_5_MS  5
48
49 #define IXGBE_WAIT_RREAD \
50         IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSRXIDX, reg_val, \
51         IPSRXIDX_READ, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS)
52 #define IXGBE_WAIT_RWRITE \
53         IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSRXIDX, reg_val, \
54         IPSRXIDX_WRITE, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS)
55 #define IXGBE_WAIT_TREAD \
56         IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSTXIDX, reg_val, \
57         IPSRXIDX_READ, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS)
58 #define IXGBE_WAIT_TWRITE \
59         IXGBE_WRITE_REG_THEN_POLL_MASK(hw, IXGBE_IPSTXIDX, reg_val, \
60         IPSRXIDX_WRITE, RTE_IXGBE_REGISTER_POLL_WAIT_5_MS)
61
62 #define CMP_IP(a, b) (\
63         (a).ipv6[0] == (b).ipv6[0] && \
64         (a).ipv6[1] == (b).ipv6[1] && \
65         (a).ipv6[2] == (b).ipv6[2] && \
66         (a).ipv6[3] == (b).ipv6[3])
67
68
69 static void
70 ixgbe_crypto_clear_ipsec_tables(struct rte_eth_dev *dev)
71 {
72         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
73         int i = 0;
74
75         /* clear Rx IP table*/
76         for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
77                 uint16_t index = i << 3;
78                 uint32_t reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_IP | index;
79                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0), 0);
80                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1), 0);
81                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2), 0);
82                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3), 0);
83                 IXGBE_WAIT_RWRITE;
84         }
85
86         /* clear Rx SPI and Rx/Tx SA tables*/
87         for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
88                 uint32_t index = i << 3;
89                 uint32_t reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_SPI | index;
90                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSPI, 0);
91                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPIDX, 0);
92                 IXGBE_WAIT_RWRITE;
93                 reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_KEY | index;
94                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(0), 0);
95                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(1), 0);
96                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(2), 0);
97                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(3), 0);
98                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSALT, 0);
99                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXMOD, 0);
100                 IXGBE_WAIT_RWRITE;
101                 reg_val = IPSRXIDX_WRITE | index;
102                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(0), 0);
103                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(1), 0);
104                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(2), 0);
105                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(3), 0);
106                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXSALT, 0);
107                 IXGBE_WAIT_TWRITE;
108         }
109 }
110
111 static int
112 ixgbe_crypto_add_sa(struct ixgbe_crypto_session *ic_session)
113 {
114         struct rte_eth_dev *dev = ic_session->dev;
115         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
116         struct ixgbe_ipsec *priv = IXGBE_DEV_PRIVATE_TO_IPSEC(
117                         dev->data->dev_private);
118         uint32_t reg_val;
119         int sa_index = -1;
120
121         if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION) {
122                 int i, ip_index = -1;
123
124                 /* Find a match in the IP table*/
125                 for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
126                         if (CMP_IP(priv->rx_ip_tbl[i].ip,
127                                    ic_session->dst_ip)) {
128                                 ip_index = i;
129                                 break;
130                         }
131                 }
132                 /* If no match, find a free entry in the IP table*/
133                 if (ip_index < 0) {
134                         for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
135                                 if (priv->rx_ip_tbl[i].ref_count == 0) {
136                                         ip_index = i;
137                                         break;
138                                 }
139                         }
140                 }
141
142                 /* Fail if no match and no free entries*/
143                 if (ip_index < 0) {
144                         PMD_DRV_LOG(ERR,
145                                     "No free entry left in the Rx IP table\n");
146                         return -1;
147                 }
148
149                 /* Find a free entry in the SA table*/
150                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
151                         if (priv->rx_sa_tbl[i].used == 0) {
152                                 sa_index = i;
153                                 break;
154                         }
155                 }
156                 /* Fail if no free entries*/
157                 if (sa_index < 0) {
158                         PMD_DRV_LOG(ERR,
159                                     "No free entry left in the Rx SA table\n");
160                         return -1;
161                 }
162
163                 priv->rx_ip_tbl[ip_index].ip.ipv6[0] =
164                                 ic_session->dst_ip.ipv6[0];
165                 priv->rx_ip_tbl[ip_index].ip.ipv6[1] =
166                                 ic_session->dst_ip.ipv6[1];
167                 priv->rx_ip_tbl[ip_index].ip.ipv6[2] =
168                                 ic_session->dst_ip.ipv6[2];
169                 priv->rx_ip_tbl[ip_index].ip.ipv6[3] =
170                                 ic_session->dst_ip.ipv6[3];
171                 priv->rx_ip_tbl[ip_index].ref_count++;
172
173                 priv->rx_sa_tbl[sa_index].spi =
174                         rte_cpu_to_be_32(ic_session->spi);
175                 priv->rx_sa_tbl[sa_index].ip_index = ip_index;
176                 priv->rx_sa_tbl[sa_index].key[3] =
177                         rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[0]);
178                 priv->rx_sa_tbl[sa_index].key[2] =
179                         rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[4]);
180                 priv->rx_sa_tbl[sa_index].key[1] =
181                         rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[8]);
182                 priv->rx_sa_tbl[sa_index].key[0] =
183                         rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[12]);
184                 priv->rx_sa_tbl[sa_index].salt =
185                         rte_cpu_to_be_32(ic_session->salt);
186                 priv->rx_sa_tbl[sa_index].mode = IPSRXMOD_VALID;
187                 if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION)
188                         priv->rx_sa_tbl[sa_index].mode |=
189                                         (IPSRXMOD_PROTO | IPSRXMOD_DECRYPT);
190                 if (ic_session->dst_ip.type == IPv6)
191                         priv->rx_sa_tbl[sa_index].mode |= IPSRXMOD_IPV6;
192                 priv->rx_sa_tbl[sa_index].used = 1;
193
194                 /* write IP table entry*/
195                 reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE |
196                                 IPSRXIDX_TABLE_IP | (ip_index << 3);
197                 if (priv->rx_ip_tbl[ip_index].ip.type == IPv4) {
198                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0), 0);
199                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1), 0);
200                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2), 0);
201                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3),
202                                         priv->rx_ip_tbl[ip_index].ip.ipv4);
203                 } else {
204                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0),
205                                         priv->rx_ip_tbl[ip_index].ip.ipv6[0]);
206                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1),
207                                         priv->rx_ip_tbl[ip_index].ip.ipv6[1]);
208                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2),
209                                         priv->rx_ip_tbl[ip_index].ip.ipv6[2]);
210                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3),
211                                         priv->rx_ip_tbl[ip_index].ip.ipv6[3]);
212                 }
213                 IXGBE_WAIT_RWRITE;
214
215                 /* write SPI table entry*/
216                 reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE |
217                                 IPSRXIDX_TABLE_SPI | (sa_index << 3);
218                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSPI,
219                                 priv->rx_sa_tbl[sa_index].spi);
220                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPIDX,
221                                 priv->rx_sa_tbl[sa_index].ip_index);
222                 IXGBE_WAIT_RWRITE;
223
224                 /* write Key table entry*/
225                 reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE |
226                                 IPSRXIDX_TABLE_KEY | (sa_index << 3);
227                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(0),
228                                 priv->rx_sa_tbl[sa_index].key[0]);
229                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(1),
230                                 priv->rx_sa_tbl[sa_index].key[1]);
231                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(2),
232                                 priv->rx_sa_tbl[sa_index].key[2]);
233                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(3),
234                                 priv->rx_sa_tbl[sa_index].key[3]);
235                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSALT,
236                                 priv->rx_sa_tbl[sa_index].salt);
237                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXMOD,
238                                 priv->rx_sa_tbl[sa_index].mode);
239                 IXGBE_WAIT_RWRITE;
240
241         } else { /* sess->dir == RTE_CRYPTO_OUTBOUND */
242                 int i;
243
244                 /* Find a free entry in the SA table*/
245                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
246                         if (priv->tx_sa_tbl[i].used == 0) {
247                                 sa_index = i;
248                                 break;
249                         }
250                 }
251                 /* Fail if no free entries*/
252                 if (sa_index < 0) {
253                         PMD_DRV_LOG(ERR,
254                                     "No free entry left in the Tx SA table\n");
255                         return -1;
256                 }
257
258                 priv->tx_sa_tbl[sa_index].spi =
259                         rte_cpu_to_be_32(ic_session->spi);
260                 priv->tx_sa_tbl[sa_index].key[3] =
261                         rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[0]);
262                 priv->tx_sa_tbl[sa_index].key[2] =
263                         rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[4]);
264                 priv->tx_sa_tbl[sa_index].key[1] =
265                         rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[8]);
266                 priv->tx_sa_tbl[sa_index].key[0] =
267                         rte_cpu_to_be_32(*(uint32_t *)&ic_session->key[12]);
268                 priv->tx_sa_tbl[sa_index].salt =
269                         rte_cpu_to_be_32(ic_session->salt);
270
271                 reg_val = IPSRXIDX_RX_EN | IPSRXIDX_WRITE | (sa_index << 3);
272                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(0),
273                                 priv->tx_sa_tbl[sa_index].key[0]);
274                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(1),
275                                 priv->tx_sa_tbl[sa_index].key[1]);
276                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(2),
277                                 priv->tx_sa_tbl[sa_index].key[2]);
278                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(3),
279                                 priv->tx_sa_tbl[sa_index].key[3]);
280                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXSALT,
281                                 priv->tx_sa_tbl[sa_index].salt);
282                 IXGBE_WAIT_TWRITE;
283
284                 priv->tx_sa_tbl[i].used = 1;
285                 ic_session->sa_index = sa_index;
286         }
287
288         return 0;
289 }
290
291 static int
292 ixgbe_crypto_remove_sa(struct rte_eth_dev *dev,
293                        struct ixgbe_crypto_session *ic_session)
294 {
295         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
296         struct ixgbe_ipsec *priv =
297                         IXGBE_DEV_PRIVATE_TO_IPSEC(dev->data->dev_private);
298         uint32_t reg_val;
299         int sa_index = -1;
300
301         if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION) {
302                 int i, ip_index = -1;
303
304                 /* Find a match in the IP table*/
305                 for (i = 0; i < IPSEC_MAX_RX_IP_COUNT; i++) {
306                         if (CMP_IP(priv->rx_ip_tbl[i].ip, ic_session->dst_ip)) {
307                                 ip_index = i;
308                                 break;
309                         }
310                 }
311
312                 /* Fail if no match*/
313                 if (ip_index < 0) {
314                         PMD_DRV_LOG(ERR,
315                                     "Entry not found in the Rx IP table\n");
316                         return -1;
317                 }
318
319                 /* Find a free entry in the SA table*/
320                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
321                         if (priv->rx_sa_tbl[i].spi ==
322                                   rte_cpu_to_be_32(ic_session->spi)) {
323                                 sa_index = i;
324                                 break;
325                         }
326                 }
327                 /* Fail if no match*/
328                 if (sa_index < 0) {
329                         PMD_DRV_LOG(ERR,
330                                     "Entry not found in the Rx SA table\n");
331                         return -1;
332                 }
333
334                 /* Disable and clear Rx SPI and key table table entryes*/
335                 reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_SPI | (sa_index << 3);
336                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSPI, 0);
337                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPIDX, 0);
338                 IXGBE_WAIT_RWRITE;
339                 reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_KEY | (sa_index << 3);
340                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(0), 0);
341                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(1), 0);
342                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(2), 0);
343                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXKEY(3), 0);
344                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXSALT, 0);
345                 IXGBE_WRITE_REG(hw, IXGBE_IPSRXMOD, 0);
346                 IXGBE_WAIT_RWRITE;
347                 priv->rx_sa_tbl[sa_index].used = 0;
348
349                 /* If last used then clear the IP table entry*/
350                 priv->rx_ip_tbl[ip_index].ref_count--;
351                 if (priv->rx_ip_tbl[ip_index].ref_count == 0) {
352                         reg_val = IPSRXIDX_WRITE | IPSRXIDX_TABLE_IP |
353                                         (ip_index << 3);
354                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(0), 0);
355                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(1), 0);
356                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(2), 0);
357                         IXGBE_WRITE_REG(hw, IXGBE_IPSRXIPADDR(3), 0);
358                 }
359         } else { /* session->dir == RTE_CRYPTO_OUTBOUND */
360                 int i;
361
362                 /* Find a match in the SA table*/
363                 for (i = 0; i < IPSEC_MAX_SA_COUNT; i++) {
364                         if (priv->tx_sa_tbl[i].spi ==
365                                     rte_cpu_to_be_32(ic_session->spi)) {
366                                 sa_index = i;
367                                 break;
368                         }
369                 }
370                 /* Fail if no match entries*/
371                 if (sa_index < 0) {
372                         PMD_DRV_LOG(ERR,
373                                     "Entry not found in the Tx SA table\n");
374                         return -1;
375                 }
376                 reg_val = IPSRXIDX_WRITE | (sa_index << 3);
377                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(0), 0);
378                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(1), 0);
379                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(2), 0);
380                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXKEY(3), 0);
381                 IXGBE_WRITE_REG(hw, IXGBE_IPSTXSALT, 0);
382                 IXGBE_WAIT_TWRITE;
383
384                 priv->tx_sa_tbl[sa_index].used = 0;
385         }
386
387         return 0;
388 }
389
390 static int
391 ixgbe_crypto_create_session(void *device,
392                 struct rte_security_session_conf *conf,
393                 struct rte_security_session *session,
394                 struct rte_mempool *mempool)
395 {
396         struct rte_eth_dev *eth_dev = (struct rte_eth_dev *)device;
397         struct ixgbe_crypto_session *ic_session = NULL;
398         struct rte_crypto_aead_xform *aead_xform;
399         struct rte_eth_conf *dev_conf = &eth_dev->data->dev_conf;
400
401         if (rte_mempool_get(mempool, (void **)&ic_session)) {
402                 PMD_DRV_LOG(ERR, "Cannot get object from ic_session mempool");
403                 return -ENOMEM;
404         }
405
406         if (conf->crypto_xform->type != RTE_CRYPTO_SYM_XFORM_AEAD ||
407                         conf->crypto_xform->aead.algo !=
408                                         RTE_CRYPTO_AEAD_AES_GCM) {
409                 PMD_DRV_LOG(ERR, "Unsupported crypto transformation mode\n");
410                 return -ENOTSUP;
411         }
412         aead_xform = &conf->crypto_xform->aead;
413
414         if (conf->ipsec.direction == RTE_SECURITY_IPSEC_SA_DIR_INGRESS) {
415                 if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_SECURITY) {
416                         ic_session->op = IXGBE_OP_AUTHENTICATED_DECRYPTION;
417                 } else {
418                         PMD_DRV_LOG(ERR, "IPsec decryption not enabled\n");
419                         return -ENOTSUP;
420                 }
421         } else {
422                 if (dev_conf->txmode.offloads & DEV_TX_OFFLOAD_SECURITY) {
423                         ic_session->op = IXGBE_OP_AUTHENTICATED_ENCRYPTION;
424                 } else {
425                         PMD_DRV_LOG(ERR, "IPsec encryption not enabled\n");
426                         return -ENOTSUP;
427                 }
428         }
429
430         ic_session->key = aead_xform->key.data;
431         memcpy(&ic_session->salt,
432                &aead_xform->key.data[aead_xform->key.length], 4);
433         ic_session->spi = conf->ipsec.spi;
434         ic_session->dev = eth_dev;
435
436         set_sec_session_private_data(session, ic_session);
437
438         if (ic_session->op == IXGBE_OP_AUTHENTICATED_ENCRYPTION) {
439                 if (ixgbe_crypto_add_sa(ic_session)) {
440                         PMD_DRV_LOG(ERR, "Failed to add SA\n");
441                         return -EPERM;
442                 }
443         }
444
445         return 0;
446 }
447
448 static int
449 ixgbe_crypto_remove_session(void *device,
450                 struct rte_security_session *session)
451 {
452         struct rte_eth_dev *eth_dev = device;
453         struct ixgbe_crypto_session *ic_session =
454                 (struct ixgbe_crypto_session *)
455                 get_sec_session_private_data(session);
456         struct rte_mempool *mempool = rte_mempool_from_obj(ic_session);
457
458         if (eth_dev != ic_session->dev) {
459                 PMD_DRV_LOG(ERR, "Session not bound to this device\n");
460                 return -ENODEV;
461         }
462
463         if (ixgbe_crypto_remove_sa(eth_dev, ic_session)) {
464                 PMD_DRV_LOG(ERR, "Failed to remove session\n");
465                 return -EFAULT;
466         }
467
468         rte_mempool_put(mempool, (void *)ic_session);
469
470         return 0;
471 }
472
473 static inline uint8_t
474 ixgbe_crypto_compute_pad_len(struct rte_mbuf *m)
475 {
476         if (m->nb_segs == 1) {
477                 /* 16 bytes ICV + 2 bytes ESP trailer + payload padding size
478                  * payload padding size is stored at <pkt_len - 18>
479                  */
480                 uint8_t *esp_pad_len = rte_pktmbuf_mtod_offset(m, uint8_t *,
481                                         rte_pktmbuf_pkt_len(m) -
482                                         (ESP_TRAILER_SIZE + ESP_ICV_SIZE));
483                 return *esp_pad_len + ESP_TRAILER_SIZE + ESP_ICV_SIZE;
484         }
485         return 0;
486 }
487
488 static int
489 ixgbe_crypto_update_mb(void *device __rte_unused,
490                 struct rte_security_session *session,
491                        struct rte_mbuf *m, void *params __rte_unused)
492 {
493         struct ixgbe_crypto_session *ic_session =
494                         get_sec_session_private_data(session);
495         if (ic_session->op == IXGBE_OP_AUTHENTICATED_ENCRYPTION) {
496                 union ixgbe_crypto_tx_desc_md *mdata =
497                         (union ixgbe_crypto_tx_desc_md *)&m->udata64;
498                 mdata->enc = 1;
499                 mdata->sa_idx = ic_session->sa_index;
500                 mdata->pad_len = ixgbe_crypto_compute_pad_len(m);
501         }
502         return 0;
503 }
504
505
506 static const struct rte_security_capability *
507 ixgbe_crypto_capabilities_get(void *device __rte_unused)
508 {
509         static const struct rte_cryptodev_capabilities
510         aes_gcm_gmac_crypto_capabilities[] = {
511                 {       /* AES GMAC (128-bit) */
512                         .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
513                         {.sym = {
514                                 .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
515                                 {.auth = {
516                                         .algo = RTE_CRYPTO_AUTH_AES_GMAC,
517                                         .block_size = 16,
518                                         .key_size = {
519                                                 .min = 16,
520                                                 .max = 16,
521                                                 .increment = 0
522                                         },
523                                         .digest_size = {
524                                                 .min = 16,
525                                                 .max = 16,
526                                                 .increment = 0
527                                         },
528                                         .iv_size = {
529                                                 .min = 12,
530                                                 .max = 12,
531                                                 .increment = 0
532                                         }
533                                 }, }
534                         }, }
535                 },
536                 {       /* AES GCM (128-bit) */
537                         .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
538                         {.sym = {
539                                 .xform_type = RTE_CRYPTO_SYM_XFORM_AEAD,
540                                 {.aead = {
541                                         .algo = RTE_CRYPTO_AEAD_AES_GCM,
542                                         .block_size = 16,
543                                         .key_size = {
544                                                 .min = 16,
545                                                 .max = 16,
546                                                 .increment = 0
547                                         },
548                                         .digest_size = {
549                                                 .min = 16,
550                                                 .max = 16,
551                                                 .increment = 0
552                                         },
553                                         .aad_size = {
554                                                 .min = 0,
555                                                 .max = 65535,
556                                                 .increment = 1
557                                         },
558                                         .iv_size = {
559                                                 .min = 12,
560                                                 .max = 12,
561                                                 .increment = 0
562                                         }
563                                 }, }
564                         }, }
565                 },
566                 {
567                         .op = RTE_CRYPTO_OP_TYPE_UNDEFINED,
568                         {.sym = {
569                                 .xform_type = RTE_CRYPTO_SYM_XFORM_NOT_SPECIFIED
570                         }, }
571                 },
572         };
573
574         static const struct rte_security_capability
575         ixgbe_security_capabilities[] = {
576                 { /* IPsec Inline Crypto ESP Transport Egress */
577                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
578                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
579                         .ipsec = {
580                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
581                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
582                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
583                                 .options = { 0 }
584                         },
585                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
586                         .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
587                 },
588                 { /* IPsec Inline Crypto ESP Transport Ingress */
589                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
590                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
591                         .ipsec = {
592                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
593                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TRANSPORT,
594                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
595                                 .options = { 0 }
596                         },
597                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
598                         .ol_flags = 0
599                 },
600                 { /* IPsec Inline Crypto ESP Tunnel Egress */
601                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
602                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
603                         .ipsec = {
604                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
605                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
606                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_EGRESS,
607                                 .options = { 0 }
608                         },
609                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
610                         .ol_flags = RTE_SECURITY_TX_OLOAD_NEED_MDATA
611                 },
612                 { /* IPsec Inline Crypto ESP Tunnel Ingress */
613                         .action = RTE_SECURITY_ACTION_TYPE_INLINE_CRYPTO,
614                         .protocol = RTE_SECURITY_PROTOCOL_IPSEC,
615                         .ipsec = {
616                                 .proto = RTE_SECURITY_IPSEC_SA_PROTO_ESP,
617                                 .mode = RTE_SECURITY_IPSEC_SA_MODE_TUNNEL,
618                                 .direction = RTE_SECURITY_IPSEC_SA_DIR_INGRESS,
619                                 .options = { 0 }
620                         },
621                         .crypto_capabilities = aes_gcm_gmac_crypto_capabilities,
622                         .ol_flags = 0
623                 },
624                 {
625                         .action = RTE_SECURITY_ACTION_TYPE_NONE
626                 }
627         };
628
629         return ixgbe_security_capabilities;
630 }
631
632
633 int
634 ixgbe_crypto_enable_ipsec(struct rte_eth_dev *dev)
635 {
636         struct ixgbe_hw *hw = IXGBE_DEV_PRIVATE_TO_HW(dev->data->dev_private);
637         uint32_t reg;
638
639         /* sanity checks */
640         if (dev->data->dev_conf.rxmode.enable_lro) {
641                 PMD_DRV_LOG(ERR, "RSC and IPsec not supported");
642                 return -1;
643         }
644         if (!dev->data->dev_conf.rxmode.hw_strip_crc) {
645                 PMD_DRV_LOG(ERR, "HW CRC strip needs to be enabled for IPsec");
646                 return -1;
647         }
648
649
650         /* Set IXGBE_SECTXBUFFAF to 0x15 as required in the datasheet*/
651         IXGBE_WRITE_REG(hw, IXGBE_SECTXBUFFAF, 0x15);
652
653         /* IFG needs to be set to 3 when we are using security. Otherwise a Tx
654          * hang will occur with heavy traffic.
655          */
656         reg = IXGBE_READ_REG(hw, IXGBE_SECTXMINIFG);
657         reg = (reg & 0xFFFFFFF0) | 0x3;
658         IXGBE_WRITE_REG(hw, IXGBE_SECTXMINIFG, reg);
659
660         reg  = IXGBE_READ_REG(hw, IXGBE_HLREG0);
661         reg |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_RXCRCSTRP;
662         IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg);
663
664         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_SECURITY) {
665                 IXGBE_WRITE_REG(hw, IXGBE_SECRXCTRL, 0);
666                 reg = IXGBE_READ_REG(hw, IXGBE_SECRXCTRL);
667                 if (reg != 0) {
668                         PMD_DRV_LOG(ERR, "Error enabling Rx Crypto");
669                         return -1;
670                 }
671         }
672         if (dev->data->dev_conf.txmode.offloads & DEV_TX_OFFLOAD_SECURITY) {
673                 IXGBE_WRITE_REG(hw, IXGBE_SECTXCTRL,
674                                 IXGBE_SECTXCTRL_STORE_FORWARD);
675                 reg = IXGBE_READ_REG(hw, IXGBE_SECTXCTRL);
676                 if (reg != IXGBE_SECTXCTRL_STORE_FORWARD) {
677                         PMD_DRV_LOG(ERR, "Error enabling Rx Crypto");
678                         return -1;
679                 }
680         }
681
682         ixgbe_crypto_clear_ipsec_tables(dev);
683
684         return 0;
685 }
686
687 int
688 ixgbe_crypto_add_ingress_sa_from_flow(const void *sess,
689                                       const void *ip_spec,
690                                       uint8_t is_ipv6)
691 {
692         struct ixgbe_crypto_session *ic_session
693                 = get_sec_session_private_data(sess);
694
695         if (ic_session->op == IXGBE_OP_AUTHENTICATED_DECRYPTION) {
696                 if (is_ipv6) {
697                         const struct rte_flow_item_ipv6 *ipv6 = ip_spec;
698                         ic_session->src_ip.type = IPv6;
699                         ic_session->dst_ip.type = IPv6;
700                         rte_memcpy(ic_session->src_ip.ipv6,
701                                    ipv6->hdr.src_addr, 16);
702                         rte_memcpy(ic_session->dst_ip.ipv6,
703                                    ipv6->hdr.dst_addr, 16);
704                 } else {
705                         const struct rte_flow_item_ipv4 *ipv4 = ip_spec;
706                         ic_session->src_ip.type = IPv4;
707                         ic_session->dst_ip.type = IPv4;
708                         ic_session->src_ip.ipv4 = ipv4->hdr.src_addr;
709                         ic_session->dst_ip.ipv4 = ipv4->hdr.dst_addr;
710                 }
711                 return ixgbe_crypto_add_sa(ic_session);
712         }
713
714         return 0;
715 }
716
717 static struct rte_security_ops ixgbe_security_ops = {
718         .session_create = ixgbe_crypto_create_session,
719         .session_update = NULL,
720         .session_stats_get = NULL,
721         .session_destroy = ixgbe_crypto_remove_session,
722         .set_pkt_metadata = ixgbe_crypto_update_mb,
723         .capabilities_get = ixgbe_crypto_capabilities_get
724 };
725
726 struct rte_security_ctx *
727 ixgbe_ipsec_ctx_create(struct rte_eth_dev *dev)
728 {
729         struct rte_security_ctx *ctx = rte_malloc("rte_security_instances_ops",
730                                         sizeof(struct rte_security_ctx), 0);
731         if (ctx) {
732                 ctx->device = (void *)dev;
733                 ctx->ops = &ixgbe_security_ops;
734                 ctx->sess_cnt = 0;
735         }
736         return ctx;
737 }