e71bce51bc8fce75ac88d748a6bae2bd9eea3a1a
[dpdk.git] / drivers / net / szedata2 / rte_eth_szedata2.c
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright (c) 2015 - 2016 CESNET
5  *   All rights reserved.
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of CESNET nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33
34 #include <stdint.h>
35 #include <unistd.h>
36 #include <stdbool.h>
37 #include <err.h>
38 #include <sys/types.h>
39 #include <dirent.h>
40 #include <sys/stat.h>
41 #include <fcntl.h>
42 #include <sys/mman.h>
43
44 #include <libsze2.h>
45
46 #include <rte_mbuf.h>
47 #include <rte_ethdev.h>
48 #include <rte_malloc.h>
49 #include <rte_memcpy.h>
50 #include <rte_kvargs.h>
51 #include <rte_dev.h>
52 #include <rte_atomic.h>
53
54 #include "rte_eth_szedata2.h"
55
56 #define RTE_ETH_SZEDATA2_MAX_RX_QUEUES 32
57 #define RTE_ETH_SZEDATA2_MAX_TX_QUEUES 32
58 #define RTE_ETH_SZEDATA2_TX_LOCK_SIZE (32 * 1024 * 1024)
59
60 /**
61  * size of szedata2_packet header with alignment
62  */
63 #define RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED 8
64
65 #define RTE_SZEDATA2_DRIVER_NAME "rte_szedata2_pmd"
66 #define RTE_SZEDATA2_PCI_DRIVER_NAME "rte_szedata2_pmd"
67
68 #define SZEDATA2_DEV_PATH_FMT "/dev/szedataII%u"
69
70 struct szedata2_rx_queue {
71         struct szedata *sze;
72         uint8_t rx_channel;
73         uint8_t in_port;
74         struct rte_mempool *mb_pool;
75         volatile uint64_t rx_pkts;
76         volatile uint64_t rx_bytes;
77         volatile uint64_t err_pkts;
78 };
79
80 struct szedata2_tx_queue {
81         struct szedata *sze;
82         uint8_t tx_channel;
83         volatile uint64_t tx_pkts;
84         volatile uint64_t tx_bytes;
85         volatile uint64_t err_pkts;
86 };
87
88 struct pmd_internals {
89         struct szedata2_rx_queue rx_queue[RTE_ETH_SZEDATA2_MAX_RX_QUEUES];
90         struct szedata2_tx_queue tx_queue[RTE_ETH_SZEDATA2_MAX_TX_QUEUES];
91         uint16_t max_rx_queues;
92         uint16_t max_tx_queues;
93         char sze_dev[PATH_MAX];
94 };
95
96 static struct ether_addr eth_addr = {
97         .addr_bytes = { 0x00, 0x11, 0x17, 0x00, 0x00, 0x00 }
98 };
99
100 static uint16_t
101 eth_szedata2_rx(void *queue,
102                 struct rte_mbuf **bufs,
103                 uint16_t nb_pkts)
104 {
105         unsigned int i;
106         struct rte_mbuf *mbuf;
107         struct szedata2_rx_queue *sze_q = queue;
108         struct rte_pktmbuf_pool_private *mbp_priv;
109         uint16_t num_rx = 0;
110         uint16_t buf_size;
111         uint16_t sg_size;
112         uint16_t hw_size;
113         uint16_t packet_size;
114         uint64_t num_bytes = 0;
115         struct szedata *sze = sze_q->sze;
116         uint8_t *header_ptr = NULL; /* header of packet */
117         uint8_t *packet_ptr1 = NULL;
118         uint8_t *packet_ptr2 = NULL;
119         uint16_t packet_len1 = 0;
120         uint16_t packet_len2 = 0;
121         uint16_t hw_data_align;
122
123         if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
124                 return 0;
125
126         /*
127          * Reads the given number of packets from szedata2 channel given
128          * by queue and copies the packet data into a newly allocated mbuf
129          * to return.
130          */
131         for (i = 0; i < nb_pkts; i++) {
132                 mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
133
134                 if (unlikely(mbuf == NULL))
135                         break;
136
137                 /* get the next sze packet */
138                 if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
139                                 sze->ct_rx_lck->next == NULL) {
140                         /* unlock old data */
141                         szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
142                         sze->ct_rx_lck_orig = NULL;
143                         sze->ct_rx_lck = NULL;
144                 }
145
146                 if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
147                         /* nothing to read, lock new data */
148                         sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
149                         sze->ct_rx_lck_orig = sze->ct_rx_lck;
150
151                         if (sze->ct_rx_lck == NULL) {
152                                 /* nothing to lock */
153                                 rte_pktmbuf_free(mbuf);
154                                 break;
155                         }
156
157                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
158                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
159
160                         if (!sze->ct_rx_rem_bytes) {
161                                 rte_pktmbuf_free(mbuf);
162                                 break;
163                         }
164                 }
165
166                 if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
167                         /*
168                          * cut in header
169                          * copy parts of header to merge buffer
170                          */
171                         if (sze->ct_rx_lck->next == NULL) {
172                                 rte_pktmbuf_free(mbuf);
173                                 break;
174                         }
175
176                         /* copy first part of header */
177                         rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
178                                         sze->ct_rx_rem_bytes);
179
180                         /* copy second part of header */
181                         sze->ct_rx_lck = sze->ct_rx_lck->next;
182                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
183                         rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
184                                 sze->ct_rx_cur_ptr,
185                                 RTE_SZE2_PACKET_HEADER_SIZE -
186                                 sze->ct_rx_rem_bytes);
187
188                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
189                                 sze->ct_rx_rem_bytes;
190                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
191                                 RTE_SZE2_PACKET_HEADER_SIZE +
192                                 sze->ct_rx_rem_bytes;
193
194                         header_ptr = (uint8_t *)sze->ct_rx_buffer;
195                 } else {
196                         /* not cut */
197                         header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
198                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
199                         sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
200                 }
201
202                 sg_size = le16toh(*((uint16_t *)header_ptr));
203                 hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
204                 packet_size = sg_size -
205                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
206
207
208                 /* checks if packet all right */
209                 if (!sg_size)
210                         errx(5, "Zero segsize");
211
212                 /* check sg_size and hwsize */
213                 if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
214                         errx(10, "Hwsize bigger than expected. Segsize: %d, "
215                                 "hwsize: %d", sg_size, hw_size);
216                 }
217
218                 hw_data_align =
219                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size) -
220                         RTE_SZE2_PACKET_HEADER_SIZE;
221
222                 if (sze->ct_rx_rem_bytes >=
223                                 (uint16_t)(sg_size -
224                                 RTE_SZE2_PACKET_HEADER_SIZE)) {
225                         /* no cut */
226                         /* one packet ready - go to another */
227                         packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
228                         packet_len1 = packet_size;
229                         packet_ptr2 = NULL;
230                         packet_len2 = 0;
231
232                         sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
233                                 RTE_SZE2_PACKET_HEADER_SIZE;
234                         sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
235                                 RTE_SZE2_PACKET_HEADER_SIZE;
236                 } else {
237                         /* cut in data */
238                         if (sze->ct_rx_lck->next == NULL) {
239                                 errx(6, "Need \"next\" lock, "
240                                         "but it is missing: %u",
241                                         sze->ct_rx_rem_bytes);
242                         }
243
244                         /* skip hw data */
245                         if (sze->ct_rx_rem_bytes <= hw_data_align) {
246                                 uint16_t rem_size = hw_data_align -
247                                         sze->ct_rx_rem_bytes;
248
249                                 /* MOVE to next lock */
250                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
251                                 sze->ct_rx_cur_ptr =
252                                         (void *)(((uint8_t *)
253                                         (sze->ct_rx_lck->start)) + rem_size);
254
255                                 packet_ptr1 = sze->ct_rx_cur_ptr;
256                                 packet_len1 = packet_size;
257                                 packet_ptr2 = NULL;
258                                 packet_len2 = 0;
259
260                                 sze->ct_rx_cur_ptr +=
261                                         RTE_SZE2_ALIGN8(packet_size);
262                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
263                                         rem_size - RTE_SZE2_ALIGN8(packet_size);
264                         } else {
265                                 /* get pointer and length from first part */
266                                 packet_ptr1 = sze->ct_rx_cur_ptr +
267                                         hw_data_align;
268                                 packet_len1 = sze->ct_rx_rem_bytes -
269                                         hw_data_align;
270
271                                 /* MOVE to next lock */
272                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
273                                 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
274
275                                 /* get pointer and length from second part */
276                                 packet_ptr2 = sze->ct_rx_cur_ptr;
277                                 packet_len2 = packet_size - packet_len1;
278
279                                 sze->ct_rx_cur_ptr +=
280                                         RTE_SZE2_ALIGN8(packet_size) -
281                                         packet_len1;
282                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
283                                         (RTE_SZE2_ALIGN8(packet_size) -
284                                          packet_len1);
285                         }
286                 }
287
288                 if (unlikely(packet_ptr1 == NULL)) {
289                         rte_pktmbuf_free(mbuf);
290                         break;
291                 }
292
293                 /* get the space available for data in the mbuf */
294                 mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
295                 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
296                                 RTE_PKTMBUF_HEADROOM);
297
298                 if (packet_size <= buf_size) {
299                         /* sze packet will fit in one mbuf, go ahead and copy */
300                         rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
301                                         packet_ptr1, packet_len1);
302                         if (packet_ptr2 != NULL) {
303                                 rte_memcpy((void *)(rte_pktmbuf_mtod(mbuf,
304                                         uint8_t *) + packet_len1),
305                                         packet_ptr2, packet_len2);
306                         }
307                         mbuf->data_len = (uint16_t)packet_size;
308
309                         mbuf->pkt_len = packet_size;
310                         mbuf->port = sze_q->in_port;
311                         bufs[num_rx] = mbuf;
312                         num_rx++;
313                         num_bytes += packet_size;
314                 } else {
315                         /*
316                          * sze packet will not fit in one mbuf,
317                          * scattered mode is not enabled, drop packet
318                          */
319                         RTE_LOG(ERR, PMD,
320                                 "SZE segment %d bytes will not fit in one mbuf "
321                                 "(%d bytes), scattered mode is not enabled, "
322                                 "drop packet!!\n",
323                                 packet_size, buf_size);
324                         rte_pktmbuf_free(mbuf);
325                 }
326         }
327
328         sze_q->rx_pkts += num_rx;
329         sze_q->rx_bytes += num_bytes;
330         return num_rx;
331 }
332
333 static uint16_t
334 eth_szedata2_rx_scattered(void *queue,
335                 struct rte_mbuf **bufs,
336                 uint16_t nb_pkts)
337 {
338         unsigned int i;
339         struct rte_mbuf *mbuf;
340         struct szedata2_rx_queue *sze_q = queue;
341         struct rte_pktmbuf_pool_private *mbp_priv;
342         uint16_t num_rx = 0;
343         uint16_t buf_size;
344         uint16_t sg_size;
345         uint16_t hw_size;
346         uint16_t packet_size;
347         uint64_t num_bytes = 0;
348         struct szedata *sze = sze_q->sze;
349         uint8_t *header_ptr = NULL; /* header of packet */
350         uint8_t *packet_ptr1 = NULL;
351         uint8_t *packet_ptr2 = NULL;
352         uint16_t packet_len1 = 0;
353         uint16_t packet_len2 = 0;
354         uint16_t hw_data_align;
355
356         if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
357                 return 0;
358
359         /*
360          * Reads the given number of packets from szedata2 channel given
361          * by queue and copies the packet data into a newly allocated mbuf
362          * to return.
363          */
364         for (i = 0; i < nb_pkts; i++) {
365                 const struct szedata_lock *ct_rx_lck_backup;
366                 unsigned int ct_rx_rem_bytes_backup;
367                 unsigned char *ct_rx_cur_ptr_backup;
368
369                 /* get the next sze packet */
370                 if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
371                                 sze->ct_rx_lck->next == NULL) {
372                         /* unlock old data */
373                         szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
374                         sze->ct_rx_lck_orig = NULL;
375                         sze->ct_rx_lck = NULL;
376                 }
377
378                 /*
379                  * Store items from sze structure which can be changed
380                  * before mbuf allocating. Use these items in case of mbuf
381                  * allocating failure.
382                  */
383                 ct_rx_lck_backup = sze->ct_rx_lck;
384                 ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
385                 ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
386
387                 if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
388                         /* nothing to read, lock new data */
389                         sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
390                         sze->ct_rx_lck_orig = sze->ct_rx_lck;
391
392                         /*
393                          * Backup items from sze structure must be updated
394                          * after locking to contain pointers to new locks.
395                          */
396                         ct_rx_lck_backup = sze->ct_rx_lck;
397                         ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
398                         ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
399
400                         if (sze->ct_rx_lck == NULL)
401                                 /* nothing to lock */
402                                 break;
403
404                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
405                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
406
407                         if (!sze->ct_rx_rem_bytes)
408                                 break;
409                 }
410
411                 if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
412                         /*
413                          * cut in header - copy parts of header to merge buffer
414                          */
415                         if (sze->ct_rx_lck->next == NULL)
416                                 break;
417
418                         /* copy first part of header */
419                         rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
420                                         sze->ct_rx_rem_bytes);
421
422                         /* copy second part of header */
423                         sze->ct_rx_lck = sze->ct_rx_lck->next;
424                         sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
425                         rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
426                                 sze->ct_rx_cur_ptr,
427                                 RTE_SZE2_PACKET_HEADER_SIZE -
428                                 sze->ct_rx_rem_bytes);
429
430                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
431                                 sze->ct_rx_rem_bytes;
432                         sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
433                                 RTE_SZE2_PACKET_HEADER_SIZE +
434                                 sze->ct_rx_rem_bytes;
435
436                         header_ptr = (uint8_t *)sze->ct_rx_buffer;
437                 } else {
438                         /* not cut */
439                         header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
440                         sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
441                         sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
442                 }
443
444                 sg_size = le16toh(*((uint16_t *)header_ptr));
445                 hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
446                 packet_size = sg_size -
447                         RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
448
449
450                 /* checks if packet all right */
451                 if (!sg_size)
452                         errx(5, "Zero segsize");
453
454                 /* check sg_size and hwsize */
455                 if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
456                         errx(10, "Hwsize bigger than expected. Segsize: %d, "
457                                         "hwsize: %d", sg_size, hw_size);
458                 }
459
460                 hw_data_align =
461                         RTE_SZE2_ALIGN8((RTE_SZE2_PACKET_HEADER_SIZE +
462                         hw_size)) - RTE_SZE2_PACKET_HEADER_SIZE;
463
464                 if (sze->ct_rx_rem_bytes >=
465                                 (uint16_t)(sg_size -
466                                 RTE_SZE2_PACKET_HEADER_SIZE)) {
467                         /* no cut */
468                         /* one packet ready - go to another */
469                         packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
470                         packet_len1 = packet_size;
471                         packet_ptr2 = NULL;
472                         packet_len2 = 0;
473
474                         sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
475                                 RTE_SZE2_PACKET_HEADER_SIZE;
476                         sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
477                                 RTE_SZE2_PACKET_HEADER_SIZE;
478                 } else {
479                         /* cut in data */
480                         if (sze->ct_rx_lck->next == NULL) {
481                                 errx(6, "Need \"next\" lock, but it is "
482                                         "missing: %u", sze->ct_rx_rem_bytes);
483                         }
484
485                         /* skip hw data */
486                         if (sze->ct_rx_rem_bytes <= hw_data_align) {
487                                 uint16_t rem_size = hw_data_align -
488                                         sze->ct_rx_rem_bytes;
489
490                                 /* MOVE to next lock */
491                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
492                                 sze->ct_rx_cur_ptr =
493                                         (void *)(((uint8_t *)
494                                         (sze->ct_rx_lck->start)) + rem_size);
495
496                                 packet_ptr1 = sze->ct_rx_cur_ptr;
497                                 packet_len1 = packet_size;
498                                 packet_ptr2 = NULL;
499                                 packet_len2 = 0;
500
501                                 sze->ct_rx_cur_ptr +=
502                                         RTE_SZE2_ALIGN8(packet_size);
503                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
504                                         rem_size - RTE_SZE2_ALIGN8(packet_size);
505                         } else {
506                                 /* get pointer and length from first part */
507                                 packet_ptr1 = sze->ct_rx_cur_ptr +
508                                         hw_data_align;
509                                 packet_len1 = sze->ct_rx_rem_bytes -
510                                         hw_data_align;
511
512                                 /* MOVE to next lock */
513                                 sze->ct_rx_lck = sze->ct_rx_lck->next;
514                                 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
515
516                                 /* get pointer and length from second part */
517                                 packet_ptr2 = sze->ct_rx_cur_ptr;
518                                 packet_len2 = packet_size - packet_len1;
519
520                                 sze->ct_rx_cur_ptr +=
521                                         RTE_SZE2_ALIGN8(packet_size) -
522                                         packet_len1;
523                                 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
524                                         (RTE_SZE2_ALIGN8(packet_size) -
525                                          packet_len1);
526                         }
527                 }
528
529                 if (unlikely(packet_ptr1 == NULL))
530                         break;
531
532                 mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
533
534                 if (unlikely(mbuf == NULL)) {
535                         /*
536                          * Restore items from sze structure to state after
537                          * unlocking (eventually locking).
538                          */
539                         sze->ct_rx_lck = ct_rx_lck_backup;
540                         sze->ct_rx_rem_bytes = ct_rx_rem_bytes_backup;
541                         sze->ct_rx_cur_ptr = ct_rx_cur_ptr_backup;
542                         break;
543                 }
544
545                 /* get the space available for data in the mbuf */
546                 mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
547                 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
548                                 RTE_PKTMBUF_HEADROOM);
549
550                 if (packet_size <= buf_size) {
551                         /* sze packet will fit in one mbuf, go ahead and copy */
552                         rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
553                                         packet_ptr1, packet_len1);
554                         if (packet_ptr2 != NULL) {
555                                 rte_memcpy((void *)
556                                         (rte_pktmbuf_mtod(mbuf, uint8_t *) +
557                                         packet_len1), packet_ptr2, packet_len2);
558                         }
559                         mbuf->data_len = (uint16_t)packet_size;
560                 } else {
561                         /*
562                          * sze packet will not fit in one mbuf,
563                          * scatter packet into more mbufs
564                          */
565                         struct rte_mbuf *m = mbuf;
566                         uint16_t len = rte_pktmbuf_tailroom(mbuf);
567
568                         /* copy first part of packet */
569                         /* fill first mbuf */
570                         rte_memcpy(rte_pktmbuf_append(mbuf, len), packet_ptr1,
571                                 len);
572                         packet_len1 -= len;
573                         packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
574
575                         while (packet_len1 > 0) {
576                                 /* fill new mbufs */
577                                 m->next = rte_pktmbuf_alloc(sze_q->mb_pool);
578
579                                 if (unlikely(m->next == NULL)) {
580                                         rte_pktmbuf_free(mbuf);
581                                         /*
582                                          * Restore items from sze structure
583                                          * to state after unlocking (eventually
584                                          * locking).
585                                          */
586                                         sze->ct_rx_lck = ct_rx_lck_backup;
587                                         sze->ct_rx_rem_bytes =
588                                                 ct_rx_rem_bytes_backup;
589                                         sze->ct_rx_cur_ptr =
590                                                 ct_rx_cur_ptr_backup;
591                                         goto finish;
592                                 }
593
594                                 m = m->next;
595
596                                 len = RTE_MIN(rte_pktmbuf_tailroom(m),
597                                         packet_len1);
598                                 rte_memcpy(rte_pktmbuf_append(mbuf, len),
599                                         packet_ptr1, len);
600
601                                 (mbuf->nb_segs)++;
602                                 packet_len1 -= len;
603                                 packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
604                         }
605
606                         if (packet_ptr2 != NULL) {
607                                 /* copy second part of packet, if exists */
608                                 /* fill the rest of currently last mbuf */
609                                 len = rte_pktmbuf_tailroom(m);
610                                 rte_memcpy(rte_pktmbuf_append(mbuf, len),
611                                         packet_ptr2, len);
612                                 packet_len2 -= len;
613                                 packet_ptr2 = ((uint8_t *)packet_ptr2) + len;
614
615                                 while (packet_len2 > 0) {
616                                         /* fill new mbufs */
617                                         m->next = rte_pktmbuf_alloc(
618                                                         sze_q->mb_pool);
619
620                                         if (unlikely(m->next == NULL)) {
621                                                 rte_pktmbuf_free(mbuf);
622                                                 /*
623                                                  * Restore items from sze
624                                                  * structure to state after
625                                                  * unlocking (eventually
626                                                  * locking).
627                                                  */
628                                                 sze->ct_rx_lck =
629                                                         ct_rx_lck_backup;
630                                                 sze->ct_rx_rem_bytes =
631                                                         ct_rx_rem_bytes_backup;
632                                                 sze->ct_rx_cur_ptr =
633                                                         ct_rx_cur_ptr_backup;
634                                                 goto finish;
635                                         }
636
637                                         m = m->next;
638
639                                         len = RTE_MIN(rte_pktmbuf_tailroom(m),
640                                                 packet_len2);
641                                         rte_memcpy(
642                                                 rte_pktmbuf_append(mbuf, len),
643                                                 packet_ptr2, len);
644
645                                         (mbuf->nb_segs)++;
646                                         packet_len2 -= len;
647                                         packet_ptr2 = ((uint8_t *)packet_ptr2) +
648                                                 len;
649                                 }
650                         }
651                 }
652                 mbuf->pkt_len = packet_size;
653                 mbuf->port = sze_q->in_port;
654                 bufs[num_rx] = mbuf;
655                 num_rx++;
656                 num_bytes += packet_size;
657         }
658
659 finish:
660         sze_q->rx_pkts += num_rx;
661         sze_q->rx_bytes += num_bytes;
662         return num_rx;
663 }
664
665 static uint16_t
666 eth_szedata2_tx(void *queue,
667                 struct rte_mbuf **bufs,
668                 uint16_t nb_pkts)
669 {
670         struct rte_mbuf *mbuf;
671         struct szedata2_tx_queue *sze_q = queue;
672         uint16_t num_tx = 0;
673         uint64_t num_bytes = 0;
674
675         const struct szedata_lock *lck;
676         uint32_t lock_size;
677         uint32_t lock_size2;
678         void *dst;
679         uint32_t pkt_len;
680         uint32_t hwpkt_len;
681         uint32_t unlock_size;
682         uint32_t rem_len;
683         uint8_t mbuf_segs;
684         uint16_t pkt_left = nb_pkts;
685
686         if (sze_q->sze == NULL || nb_pkts == 0)
687                 return 0;
688
689         while (pkt_left > 0) {
690                 unlock_size = 0;
691                 lck = szedata_tx_lock_data(sze_q->sze,
692                         RTE_ETH_SZEDATA2_TX_LOCK_SIZE,
693                         sze_q->tx_channel);
694                 if (lck == NULL)
695                         continue;
696
697                 dst = lck->start;
698                 lock_size = lck->len;
699                 lock_size2 = lck->next ? lck->next->len : 0;
700
701 next_packet:
702                 mbuf = bufs[nb_pkts - pkt_left];
703
704                 pkt_len = mbuf->pkt_len;
705                 mbuf_segs = mbuf->nb_segs;
706
707                 hwpkt_len = RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
708                         RTE_SZE2_ALIGN8(pkt_len);
709
710                 if (lock_size + lock_size2 < hwpkt_len) {
711                         szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
712                         continue;
713                 }
714
715                 num_bytes += pkt_len;
716
717                 if (lock_size > hwpkt_len) {
718                         void *tmp_dst;
719
720                         rem_len = 0;
721
722                         /* write packet length at first 2 bytes in 8B header */
723                         *((uint16_t *)dst) = htole16(
724                                         RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
725                                         pkt_len);
726                         *(((uint16_t *)dst) + 1) = htole16(0);
727
728                         /* copy packet from mbuf */
729                         tmp_dst = ((uint8_t *)(dst)) +
730                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
731                         if (mbuf_segs == 1) {
732                                 /*
733                                  * non-scattered packet,
734                                  * transmit from one mbuf
735                                  */
736                                 rte_memcpy(tmp_dst,
737                                         rte_pktmbuf_mtod(mbuf, const void *),
738                                         pkt_len);
739                         } else {
740                                 /* scattered packet, transmit from more mbufs */
741                                 struct rte_mbuf *m = mbuf;
742                                 while (m) {
743                                         rte_memcpy(tmp_dst,
744                                                 rte_pktmbuf_mtod(m,
745                                                 const void *),
746                                                 m->data_len);
747                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
748                                                 m->data_len;
749                                         m = m->next;
750                                 }
751                         }
752
753
754                         dst = ((uint8_t *)dst) + hwpkt_len;
755                         unlock_size += hwpkt_len;
756                         lock_size -= hwpkt_len;
757
758                         rte_pktmbuf_free(mbuf);
759                         num_tx++;
760                         pkt_left--;
761                         if (pkt_left == 0) {
762                                 szedata_tx_unlock_data(sze_q->sze, lck,
763                                         unlock_size);
764                                 break;
765                         }
766                         goto next_packet;
767                 } else if (lock_size + lock_size2 >= hwpkt_len) {
768                         void *tmp_dst;
769                         uint16_t write_len;
770
771                         /* write packet length at first 2 bytes in 8B header */
772                         *((uint16_t *)dst) =
773                                 htole16(RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
774                                         pkt_len);
775                         *(((uint16_t *)dst) + 1) = htole16(0);
776
777                         /*
778                          * If the raw packet (pkt_len) is smaller than lock_size
779                          * get the correct length for memcpy
780                          */
781                         write_len =
782                                 pkt_len < lock_size -
783                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED ?
784                                 pkt_len :
785                                 lock_size - RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
786
787                         rem_len = hwpkt_len - lock_size;
788
789                         tmp_dst = ((uint8_t *)(dst)) +
790                                 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
791                         if (mbuf_segs == 1) {
792                                 /*
793                                  * non-scattered packet,
794                                  * transmit from one mbuf
795                                  */
796                                 /* copy part of packet to first area */
797                                 rte_memcpy(tmp_dst,
798                                         rte_pktmbuf_mtod(mbuf, const void *),
799                                         write_len);
800
801                                 if (lck->next)
802                                         dst = lck->next->start;
803
804                                 /* copy part of packet to second area */
805                                 rte_memcpy(dst,
806                                         (const void *)(rte_pktmbuf_mtod(mbuf,
807                                                         const uint8_t *) +
808                                         write_len), pkt_len - write_len);
809                         } else {
810                                 /* scattered packet, transmit from more mbufs */
811                                 struct rte_mbuf *m = mbuf;
812                                 uint16_t written = 0;
813                                 uint16_t to_write = 0;
814                                 bool new_mbuf = true;
815                                 uint16_t write_off = 0;
816
817                                 /* copy part of packet to first area */
818                                 while (m && written < write_len) {
819                                         to_write = RTE_MIN(m->data_len,
820                                                         write_len - written);
821                                         rte_memcpy(tmp_dst,
822                                                 rte_pktmbuf_mtod(m,
823                                                         const void *),
824                                                 to_write);
825
826                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
827                                                 to_write;
828                                         if (m->data_len <= write_len -
829                                                         written) {
830                                                 m = m->next;
831                                                 new_mbuf = true;
832                                         } else {
833                                                 new_mbuf = false;
834                                         }
835                                         written += to_write;
836                                 }
837
838                                 if (lck->next)
839                                         dst = lck->next->start;
840
841                                 tmp_dst = dst;
842                                 written = 0;
843                                 write_off = new_mbuf ? 0 : to_write;
844
845                                 /* copy part of packet to second area */
846                                 while (m && written < pkt_len - write_len) {
847                                         rte_memcpy(tmp_dst, (const void *)
848                                                 (rte_pktmbuf_mtod(m,
849                                                 uint8_t *) + write_off),
850                                                 m->data_len - write_off);
851
852                                         tmp_dst = ((uint8_t *)(tmp_dst)) +
853                                                 (m->data_len - write_off);
854                                         written += m->data_len - write_off;
855                                         m = m->next;
856                                         write_off = 0;
857                                 }
858                         }
859
860                         dst = ((uint8_t *)dst) + rem_len;
861                         unlock_size += hwpkt_len;
862                         lock_size = lock_size2 - rem_len;
863                         lock_size2 = 0;
864
865                         rte_pktmbuf_free(mbuf);
866                         num_tx++;
867                 }
868
869                 szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
870                 pkt_left--;
871         }
872
873         sze_q->tx_pkts += num_tx;
874         sze_q->err_pkts += nb_pkts - num_tx;
875         sze_q->tx_bytes += num_bytes;
876         return num_tx;
877 }
878
879 static int
880 eth_rx_queue_start(struct rte_eth_dev *dev, uint16_t rxq_id)
881 {
882         struct szedata2_rx_queue *rxq = dev->data->rx_queues[rxq_id];
883         int ret;
884         struct pmd_internals *internals = (struct pmd_internals *)
885                 dev->data->dev_private;
886
887         if (rxq->sze == NULL) {
888                 uint32_t rx = 1 << rxq->rx_channel;
889                 uint32_t tx = 0;
890                 rxq->sze = szedata_open(internals->sze_dev);
891                 if (rxq->sze == NULL)
892                         return -EINVAL;
893                 ret = szedata_subscribe3(rxq->sze, &rx, &tx);
894                 if (ret != 0 || rx == 0)
895                         goto err;
896         }
897
898         ret = szedata_start(rxq->sze);
899         if (ret != 0)
900                 goto err;
901         dev->data->rx_queue_state[rxq_id] = RTE_ETH_QUEUE_STATE_STARTED;
902         return 0;
903
904 err:
905         szedata_close(rxq->sze);
906         rxq->sze = NULL;
907         return -EINVAL;
908 }
909
910 static int
911 eth_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rxq_id)
912 {
913         struct szedata2_rx_queue *rxq = dev->data->rx_queues[rxq_id];
914
915         if (rxq->sze != NULL) {
916                 szedata_close(rxq->sze);
917                 rxq->sze = NULL;
918         }
919
920         dev->data->rx_queue_state[rxq_id] = RTE_ETH_QUEUE_STATE_STOPPED;
921         return 0;
922 }
923
924 static int
925 eth_tx_queue_start(struct rte_eth_dev *dev, uint16_t txq_id)
926 {
927         struct szedata2_tx_queue *txq = dev->data->tx_queues[txq_id];
928         int ret;
929         struct pmd_internals *internals = (struct pmd_internals *)
930                 dev->data->dev_private;
931
932         if (txq->sze == NULL) {
933                 uint32_t rx = 0;
934                 uint32_t tx = 1 << txq->tx_channel;
935                 txq->sze = szedata_open(internals->sze_dev);
936                 if (txq->sze == NULL)
937                         return -EINVAL;
938                 ret = szedata_subscribe3(txq->sze, &rx, &tx);
939                 if (ret != 0 || tx == 0)
940                         goto err;
941         }
942
943         ret = szedata_start(txq->sze);
944         if (ret != 0)
945                 goto err;
946         dev->data->tx_queue_state[txq_id] = RTE_ETH_QUEUE_STATE_STARTED;
947         return 0;
948
949 err:
950         szedata_close(txq->sze);
951         txq->sze = NULL;
952         return -EINVAL;
953 }
954
955 static int
956 eth_tx_queue_stop(struct rte_eth_dev *dev, uint16_t txq_id)
957 {
958         struct szedata2_tx_queue *txq = dev->data->tx_queues[txq_id];
959
960         if (txq->sze != NULL) {
961                 szedata_close(txq->sze);
962                 txq->sze = NULL;
963         }
964
965         dev->data->tx_queue_state[txq_id] = RTE_ETH_QUEUE_STATE_STOPPED;
966         return 0;
967 }
968
969 static int
970 eth_dev_start(struct rte_eth_dev *dev)
971 {
972         int ret;
973         uint16_t i;
974         uint16_t nb_rx = dev->data->nb_rx_queues;
975         uint16_t nb_tx = dev->data->nb_tx_queues;
976
977         for (i = 0; i < nb_rx; i++) {
978                 ret = eth_rx_queue_start(dev, i);
979                 if (ret != 0)
980                         goto err_rx;
981         }
982
983         for (i = 0; i < nb_tx; i++) {
984                 ret = eth_tx_queue_start(dev, i);
985                 if (ret != 0)
986                         goto err_tx;
987         }
988
989         return 0;
990
991 err_tx:
992         for (i = 0; i < nb_tx; i++)
993                 eth_tx_queue_stop(dev, i);
994 err_rx:
995         for (i = 0; i < nb_rx; i++)
996                 eth_rx_queue_stop(dev, i);
997         return ret;
998 }
999
1000 static void
1001 eth_dev_stop(struct rte_eth_dev *dev)
1002 {
1003         uint16_t i;
1004         uint16_t nb_rx = dev->data->nb_rx_queues;
1005         uint16_t nb_tx = dev->data->nb_tx_queues;
1006
1007         for (i = 0; i < nb_tx; i++)
1008                 eth_tx_queue_stop(dev, i);
1009
1010         for (i = 0; i < nb_rx; i++)
1011                 eth_rx_queue_stop(dev, i);
1012 }
1013
1014 static int
1015 eth_dev_configure(struct rte_eth_dev *dev)
1016 {
1017         struct rte_eth_dev_data *data = dev->data;
1018         if (data->dev_conf.rxmode.enable_scatter == 1) {
1019                 dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1020                 data->scattered_rx = 1;
1021         } else {
1022                 dev->rx_pkt_burst = eth_szedata2_rx;
1023                 data->scattered_rx = 0;
1024         }
1025         return 0;
1026 }
1027
1028 static void
1029 eth_dev_info(struct rte_eth_dev *dev,
1030                 struct rte_eth_dev_info *dev_info)
1031 {
1032         struct pmd_internals *internals = dev->data->dev_private;
1033         dev_info->if_index = 0;
1034         dev_info->max_mac_addrs = 1;
1035         dev_info->max_rx_pktlen = (uint32_t)-1;
1036         dev_info->max_rx_queues = internals->max_rx_queues;
1037         dev_info->max_tx_queues = internals->max_tx_queues;
1038         dev_info->min_rx_bufsize = 0;
1039         dev_info->speed_capa = ETH_LINK_SPEED_100G;
1040 }
1041
1042 static void
1043 eth_stats_get(struct rte_eth_dev *dev,
1044                 struct rte_eth_stats *stats)
1045 {
1046         uint16_t i;
1047         uint16_t nb_rx = dev->data->nb_rx_queues;
1048         uint16_t nb_tx = dev->data->nb_tx_queues;
1049         uint64_t rx_total = 0;
1050         uint64_t tx_total = 0;
1051         uint64_t tx_err_total = 0;
1052         uint64_t rx_total_bytes = 0;
1053         uint64_t tx_total_bytes = 0;
1054         const struct pmd_internals *internals = dev->data->dev_private;
1055
1056         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && i < nb_rx; i++) {
1057                 stats->q_ipackets[i] = internals->rx_queue[i].rx_pkts;
1058                 stats->q_ibytes[i] = internals->rx_queue[i].rx_bytes;
1059                 rx_total += stats->q_ipackets[i];
1060                 rx_total_bytes += stats->q_ibytes[i];
1061         }
1062
1063         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS && i < nb_tx; i++) {
1064                 stats->q_opackets[i] = internals->tx_queue[i].tx_pkts;
1065                 stats->q_obytes[i] = internals->tx_queue[i].tx_bytes;
1066                 stats->q_errors[i] = internals->tx_queue[i].err_pkts;
1067                 tx_total += stats->q_opackets[i];
1068                 tx_total_bytes += stats->q_obytes[i];
1069                 tx_err_total += stats->q_errors[i];
1070         }
1071
1072         stats->ipackets = rx_total;
1073         stats->opackets = tx_total;
1074         stats->ibytes = rx_total_bytes;
1075         stats->obytes = tx_total_bytes;
1076         stats->oerrors = tx_err_total;
1077 }
1078
1079 static void
1080 eth_stats_reset(struct rte_eth_dev *dev)
1081 {
1082         uint16_t i;
1083         uint16_t nb_rx = dev->data->nb_rx_queues;
1084         uint16_t nb_tx = dev->data->nb_tx_queues;
1085         struct pmd_internals *internals = dev->data->dev_private;
1086
1087         for (i = 0; i < nb_rx; i++) {
1088                 internals->rx_queue[i].rx_pkts = 0;
1089                 internals->rx_queue[i].rx_bytes = 0;
1090                 internals->rx_queue[i].err_pkts = 0;
1091         }
1092         for (i = 0; i < nb_tx; i++) {
1093                 internals->tx_queue[i].tx_pkts = 0;
1094                 internals->tx_queue[i].tx_bytes = 0;
1095                 internals->tx_queue[i].err_pkts = 0;
1096         }
1097 }
1098
1099 static void
1100 eth_rx_queue_release(void *q)
1101 {
1102         struct szedata2_rx_queue *rxq = (struct szedata2_rx_queue *)q;
1103         if (rxq->sze != NULL) {
1104                 szedata_close(rxq->sze);
1105                 rxq->sze = NULL;
1106         }
1107 }
1108
1109 static void
1110 eth_tx_queue_release(void *q)
1111 {
1112         struct szedata2_tx_queue *txq = (struct szedata2_tx_queue *)q;
1113         if (txq->sze != NULL) {
1114                 szedata_close(txq->sze);
1115                 txq->sze = NULL;
1116         }
1117 }
1118
1119 static void
1120 eth_dev_close(struct rte_eth_dev *dev)
1121 {
1122         uint16_t i;
1123         uint16_t nb_rx = dev->data->nb_rx_queues;
1124         uint16_t nb_tx = dev->data->nb_tx_queues;
1125
1126         eth_dev_stop(dev);
1127
1128         for (i = 0; i < nb_rx; i++) {
1129                 eth_rx_queue_release(dev->data->rx_queues[i]);
1130                 dev->data->rx_queues[i] = NULL;
1131         }
1132         dev->data->nb_rx_queues = 0;
1133         for (i = 0; i < nb_tx; i++) {
1134                 eth_tx_queue_release(dev->data->tx_queues[i]);
1135                 dev->data->tx_queues[i] = NULL;
1136         }
1137         dev->data->nb_tx_queues = 0;
1138 }
1139
1140 static int
1141 eth_link_update(struct rte_eth_dev *dev,
1142                 int wait_to_complete __rte_unused)
1143 {
1144         struct rte_eth_link link;
1145         struct rte_eth_link *link_ptr = &link;
1146         struct rte_eth_link *dev_link = &dev->data->dev_link;
1147         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1148                         dev, SZEDATA2_CGMII_IBUF_BASE_OFF,
1149                         volatile struct szedata2_cgmii_ibuf *);
1150
1151         switch (cgmii_link_speed(ibuf)) {
1152         case SZEDATA2_LINK_SPEED_10G:
1153                 link.link_speed = ETH_SPEED_NUM_10G;
1154                 break;
1155         case SZEDATA2_LINK_SPEED_40G:
1156                 link.link_speed = ETH_SPEED_NUM_40G;
1157                 break;
1158         case SZEDATA2_LINK_SPEED_100G:
1159                 link.link_speed = ETH_SPEED_NUM_100G;
1160                 break;
1161         default:
1162                 link.link_speed = ETH_SPEED_NUM_10G;
1163                 break;
1164         }
1165
1166         /* szedata2 uses only full duplex */
1167         link.link_duplex = ETH_LINK_FULL_DUPLEX;
1168
1169         link.link_status = (cgmii_ibuf_is_enabled(ibuf) &&
1170                         cgmii_ibuf_is_link_up(ibuf)) ? ETH_LINK_UP : ETH_LINK_DOWN;
1171
1172         link.link_autoneg = ETH_LINK_SPEED_FIXED;
1173
1174         rte_atomic64_cmpset((uint64_t *)dev_link, *(uint64_t *)dev_link,
1175                         *(uint64_t *)link_ptr);
1176
1177         return 0;
1178 }
1179
1180 static int
1181 eth_dev_set_link_up(struct rte_eth_dev *dev)
1182 {
1183         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1184                         dev, SZEDATA2_CGMII_IBUF_BASE_OFF,
1185                         volatile struct szedata2_cgmii_ibuf *);
1186         volatile struct szedata2_cgmii_obuf *obuf = SZEDATA2_PCI_RESOURCE_PTR(
1187                         dev, SZEDATA2_CGMII_OBUF_BASE_OFF,
1188                         volatile struct szedata2_cgmii_obuf *);
1189
1190         cgmii_ibuf_enable(ibuf);
1191         cgmii_obuf_enable(obuf);
1192         return 0;
1193 }
1194
1195 static int
1196 eth_dev_set_link_down(struct rte_eth_dev *dev)
1197 {
1198         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1199                         dev, SZEDATA2_CGMII_IBUF_BASE_OFF,
1200                         volatile struct szedata2_cgmii_ibuf *);
1201         volatile struct szedata2_cgmii_obuf *obuf = SZEDATA2_PCI_RESOURCE_PTR(
1202                         dev, SZEDATA2_CGMII_OBUF_BASE_OFF,
1203                         volatile struct szedata2_cgmii_obuf *);
1204
1205         cgmii_ibuf_disable(ibuf);
1206         cgmii_obuf_disable(obuf);
1207         return 0;
1208 }
1209
1210 static int
1211 eth_rx_queue_setup(struct rte_eth_dev *dev,
1212                 uint16_t rx_queue_id,
1213                 uint16_t nb_rx_desc __rte_unused,
1214                 unsigned int socket_id __rte_unused,
1215                 const struct rte_eth_rxconf *rx_conf __rte_unused,
1216                 struct rte_mempool *mb_pool)
1217 {
1218         struct pmd_internals *internals = dev->data->dev_private;
1219         struct szedata2_rx_queue *rxq = &internals->rx_queue[rx_queue_id];
1220         int ret;
1221         uint32_t rx = 1 << rx_queue_id;
1222         uint32_t tx = 0;
1223
1224         rxq->sze = szedata_open(internals->sze_dev);
1225         if (rxq->sze == NULL)
1226                 return -EINVAL;
1227         ret = szedata_subscribe3(rxq->sze, &rx, &tx);
1228         if (ret != 0 || rx == 0) {
1229                 szedata_close(rxq->sze);
1230                 rxq->sze = NULL;
1231                 return -EINVAL;
1232         }
1233         rxq->rx_channel = rx_queue_id;
1234         rxq->in_port = dev->data->port_id;
1235         rxq->mb_pool = mb_pool;
1236         rxq->rx_pkts = 0;
1237         rxq->rx_bytes = 0;
1238         rxq->err_pkts = 0;
1239
1240         dev->data->rx_queues[rx_queue_id] = rxq;
1241         return 0;
1242 }
1243
1244 static int
1245 eth_tx_queue_setup(struct rte_eth_dev *dev,
1246                 uint16_t tx_queue_id,
1247                 uint16_t nb_tx_desc __rte_unused,
1248                 unsigned int socket_id __rte_unused,
1249                 const struct rte_eth_txconf *tx_conf __rte_unused)
1250 {
1251         struct pmd_internals *internals = dev->data->dev_private;
1252         struct szedata2_tx_queue *txq = &internals->tx_queue[tx_queue_id];
1253         int ret;
1254         uint32_t rx = 0;
1255         uint32_t tx = 1 << tx_queue_id;
1256
1257         txq->sze = szedata_open(internals->sze_dev);
1258         if (txq->sze == NULL)
1259                 return -EINVAL;
1260         ret = szedata_subscribe3(txq->sze, &rx, &tx);
1261         if (ret != 0 || tx == 0) {
1262                 szedata_close(txq->sze);
1263                 txq->sze = NULL;
1264                 return -EINVAL;
1265         }
1266         txq->tx_channel = tx_queue_id;
1267         txq->tx_pkts = 0;
1268         txq->tx_bytes = 0;
1269         txq->err_pkts = 0;
1270
1271         dev->data->tx_queues[tx_queue_id] = txq;
1272         return 0;
1273 }
1274
1275 static void
1276 eth_mac_addr_set(struct rte_eth_dev *dev __rte_unused,
1277                 struct ether_addr *mac_addr __rte_unused)
1278 {
1279 }
1280
1281 static void
1282 eth_promiscuous_enable(struct rte_eth_dev *dev)
1283 {
1284         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1285                         dev, SZEDATA2_CGMII_IBUF_BASE_OFF,
1286                         volatile struct szedata2_cgmii_ibuf *);
1287         cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_PROMISC);
1288 }
1289
1290 static void
1291 eth_promiscuous_disable(struct rte_eth_dev *dev)
1292 {
1293         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1294                         dev, SZEDATA2_CGMII_IBUF_BASE_OFF,
1295                         volatile struct szedata2_cgmii_ibuf *);
1296         cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_ONLY_VALID);
1297 }
1298
1299 static void
1300 eth_allmulticast_enable(struct rte_eth_dev *dev)
1301 {
1302         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1303                         dev, SZEDATA2_CGMII_IBUF_BASE_OFF,
1304                         volatile struct szedata2_cgmii_ibuf *);
1305         cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_ALL_MULTICAST);
1306 }
1307
1308 static void
1309 eth_allmulticast_disable(struct rte_eth_dev *dev)
1310 {
1311         volatile struct szedata2_cgmii_ibuf *ibuf = SZEDATA2_PCI_RESOURCE_PTR(
1312                         dev, SZEDATA2_CGMII_IBUF_BASE_OFF,
1313                         volatile struct szedata2_cgmii_ibuf *);
1314         cgmii_ibuf_mac_mode_write(ibuf, SZEDATA2_MAC_CHMODE_ONLY_VALID);
1315 }
1316
1317 static const struct eth_dev_ops ops = {
1318         .dev_start          = eth_dev_start,
1319         .dev_stop           = eth_dev_stop,
1320         .dev_set_link_up    = eth_dev_set_link_up,
1321         .dev_set_link_down  = eth_dev_set_link_down,
1322         .dev_close          = eth_dev_close,
1323         .dev_configure      = eth_dev_configure,
1324         .dev_infos_get      = eth_dev_info,
1325         .promiscuous_enable   = eth_promiscuous_enable,
1326         .promiscuous_disable  = eth_promiscuous_disable,
1327         .allmulticast_enable  = eth_allmulticast_enable,
1328         .allmulticast_disable = eth_allmulticast_disable,
1329         .rx_queue_start     = eth_rx_queue_start,
1330         .rx_queue_stop      = eth_rx_queue_stop,
1331         .tx_queue_start     = eth_tx_queue_start,
1332         .tx_queue_stop      = eth_tx_queue_stop,
1333         .rx_queue_setup     = eth_rx_queue_setup,
1334         .tx_queue_setup     = eth_tx_queue_setup,
1335         .rx_queue_release   = eth_rx_queue_release,
1336         .tx_queue_release   = eth_tx_queue_release,
1337         .link_update        = eth_link_update,
1338         .stats_get          = eth_stats_get,
1339         .stats_reset        = eth_stats_reset,
1340         .mac_addr_set       = eth_mac_addr_set,
1341 };
1342
1343 /*
1344  * This function goes through sysfs and looks for an index of szedata2
1345  * device file (/dev/szedataIIX, where X is the index).
1346  *
1347  * @return
1348  *           0 on success
1349  *          -1 on error
1350  */
1351 static int
1352 get_szedata2_index(struct rte_eth_dev *dev, uint32_t *index)
1353 {
1354         DIR *dir;
1355         struct dirent *entry;
1356         int ret;
1357         uint32_t tmp_index;
1358         FILE *fd;
1359         char pcislot_path[PATH_MAX];
1360         struct rte_pci_addr pcislot_addr = dev->pci_dev->addr;
1361         uint32_t domain;
1362         uint32_t bus;
1363         uint32_t devid;
1364         uint32_t function;
1365
1366         dir = opendir("/sys/class/combo");
1367         if (dir == NULL)
1368                 return -1;
1369
1370         /*
1371          * Iterate through all combosixX directories.
1372          * When the value in /sys/class/combo/combosixX/device/pcislot
1373          * file is the location of the ethernet device dev, "X" is the
1374          * index of the device.
1375          */
1376         while ((entry = readdir(dir)) != NULL) {
1377                 ret = sscanf(entry->d_name, "combosix%u", &tmp_index);
1378                 if (ret != 1)
1379                         continue;
1380
1381                 snprintf(pcislot_path, PATH_MAX,
1382                         "/sys/class/combo/combosix%u/device/pcislot",
1383                         tmp_index);
1384
1385                 fd = fopen(pcislot_path, "r");
1386                 if (fd == NULL)
1387                         continue;
1388
1389                 ret = fscanf(fd, "%4" PRIx16 ":%2" PRIx8 ":%2" PRIx8 ".%" PRIx8,
1390                                 &domain, &bus, &devid, &function);
1391                 fclose(fd);
1392                 if (ret != 4)
1393                         continue;
1394
1395                 if (pcislot_addr.domain == domain &&
1396                                 pcislot_addr.bus == bus &&
1397                                 pcislot_addr.devid == devid &&
1398                                 pcislot_addr.function == function) {
1399                         *index = tmp_index;
1400                         closedir(dir);
1401                         return 0;
1402                 }
1403         }
1404
1405         closedir(dir);
1406         return -1;
1407 }
1408
1409 static int
1410 rte_szedata2_eth_dev_init(struct rte_eth_dev *dev)
1411 {
1412         struct rte_eth_dev_data *data = dev->data;
1413         struct pmd_internals *internals = (struct pmd_internals *)
1414                 data->dev_private;
1415         struct szedata *szedata_temp;
1416         int ret;
1417         uint32_t szedata2_index;
1418         struct rte_pci_addr *pci_addr = &dev->pci_dev->addr;
1419         struct rte_pci_resource *pci_rsc =
1420                 &dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER];
1421         char rsc_filename[PATH_MAX];
1422         void *pci_resource_ptr = NULL;
1423         int fd;
1424
1425         RTE_LOG(INFO, PMD, "Initializing szedata2 device (" PCI_PRI_FMT ")\n",
1426                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1427                         pci_addr->function);
1428
1429         /* Get index of szedata2 device file and create path to device file */
1430         ret = get_szedata2_index(dev, &szedata2_index);
1431         if (ret != 0) {
1432                 RTE_LOG(ERR, PMD, "Failed to get szedata2 device index!\n");
1433                 return -ENODEV;
1434         }
1435         snprintf(internals->sze_dev, PATH_MAX, SZEDATA2_DEV_PATH_FMT,
1436                         szedata2_index);
1437
1438         RTE_LOG(INFO, PMD, "SZEDATA2 path: %s\n", internals->sze_dev);
1439
1440         /*
1441          * Get number of available DMA RX and TX channels, which is maximum
1442          * number of queues that can be created and store it in private device
1443          * data structure.
1444          */
1445         szedata_temp = szedata_open(internals->sze_dev);
1446         if (szedata_temp == NULL) {
1447                 RTE_LOG(ERR, PMD, "szedata_open(): failed to open %s",
1448                                 internals->sze_dev);
1449                 return -EINVAL;
1450         }
1451         internals->max_rx_queues = szedata_ifaces_available(szedata_temp,
1452                         SZE2_DIR_RX);
1453         internals->max_tx_queues = szedata_ifaces_available(szedata_temp,
1454                         SZE2_DIR_TX);
1455         szedata_close(szedata_temp);
1456
1457         RTE_LOG(INFO, PMD, "Available DMA channels RX: %u TX: %u\n",
1458                         internals->max_rx_queues, internals->max_tx_queues);
1459
1460         /* Set rx, tx burst functions */
1461         if (data->dev_conf.rxmode.enable_scatter == 1 ||
1462                 data->scattered_rx == 1) {
1463                 dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1464                 data->scattered_rx = 1;
1465         } else {
1466                 dev->rx_pkt_burst = eth_szedata2_rx;
1467                 data->scattered_rx = 0;
1468         }
1469         dev->tx_pkt_burst = eth_szedata2_tx;
1470
1471         /* Set function callbacks for Ethernet API */
1472         dev->dev_ops = &ops;
1473
1474         rte_eth_copy_pci_info(dev, dev->pci_dev);
1475
1476         /* mmap pci resource0 file to rte_pci_resource structure */
1477         if (dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].phys_addr ==
1478                         0) {
1479                 RTE_LOG(ERR, PMD, "Missing resource%u file\n",
1480                                 PCI_RESOURCE_NUMBER);
1481                 return -EINVAL;
1482         }
1483         snprintf(rsc_filename, PATH_MAX,
1484                 "%s/" PCI_PRI_FMT "/resource%u", pci_get_sysfs_path(),
1485                 pci_addr->domain, pci_addr->bus,
1486                 pci_addr->devid, pci_addr->function, PCI_RESOURCE_NUMBER);
1487         fd = open(rsc_filename, O_RDWR);
1488         if (fd < 0) {
1489                 RTE_LOG(ERR, PMD, "Could not open file %s\n", rsc_filename);
1490                 return -EINVAL;
1491         }
1492
1493         pci_resource_ptr = mmap(0,
1494                         dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len,
1495                         PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
1496         close(fd);
1497         if (pci_resource_ptr == NULL) {
1498                 RTE_LOG(ERR, PMD, "Could not mmap file %s (fd = %d)\n",
1499                                 rsc_filename, fd);
1500                 return -EINVAL;
1501         }
1502         dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr =
1503                 pci_resource_ptr;
1504
1505         RTE_LOG(DEBUG, PMD, "resource%u phys_addr = 0x%llx len = %llu "
1506                         "virt addr = %llx\n", PCI_RESOURCE_NUMBER,
1507                         (unsigned long long)pci_rsc->phys_addr,
1508                         (unsigned long long)pci_rsc->len,
1509                         (unsigned long long)pci_rsc->addr);
1510
1511         /* Get link state */
1512         eth_link_update(dev, 0);
1513
1514         /* Allocate space for one mac address */
1515         data->mac_addrs = rte_zmalloc(data->name, sizeof(struct ether_addr),
1516                         RTE_CACHE_LINE_SIZE);
1517         if (data->mac_addrs == NULL) {
1518                 RTE_LOG(ERR, PMD, "Could not alloc space for MAC address!\n");
1519                 munmap(dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr,
1520                         dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len);
1521                 return -EINVAL;
1522         }
1523
1524         ether_addr_copy(&eth_addr, data->mac_addrs);
1525
1526         /* At initial state COMBO card is in promiscuous mode so disable it */
1527         eth_promiscuous_disable(dev);
1528
1529         RTE_LOG(INFO, PMD, "szedata2 device ("
1530                         PCI_PRI_FMT ") successfully initialized\n",
1531                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1532                         pci_addr->function);
1533
1534         return 0;
1535 }
1536
1537 static int
1538 rte_szedata2_eth_dev_uninit(struct rte_eth_dev *dev)
1539 {
1540         struct rte_pci_addr *pci_addr = &dev->pci_dev->addr;
1541
1542         rte_free(dev->data->mac_addrs);
1543         dev->data->mac_addrs = NULL;
1544         munmap(dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].addr,
1545                 dev->pci_dev->mem_resource[PCI_RESOURCE_NUMBER].len);
1546
1547         RTE_LOG(INFO, PMD, "szedata2 device ("
1548                         PCI_PRI_FMT ") successfully uninitialized\n",
1549                         pci_addr->domain, pci_addr->bus, pci_addr->devid,
1550                         pci_addr->function);
1551
1552         return 0;
1553 }
1554
1555 static const struct rte_pci_id rte_szedata2_pci_id_table[] = {
1556         {
1557                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1558                                 PCI_DEVICE_ID_NETCOPE_COMBO80G)
1559         },
1560         {
1561                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1562                                 PCI_DEVICE_ID_NETCOPE_COMBO100G)
1563         },
1564         {
1565                 RTE_PCI_DEVICE(PCI_VENDOR_ID_NETCOPE,
1566                                 PCI_DEVICE_ID_NETCOPE_COMBO100G2)
1567         },
1568         {
1569                 .vendor_id = 0,
1570         }
1571 };
1572
1573 static struct eth_driver szedata2_eth_driver = {
1574         .pci_drv = {
1575                 .name     = RTE_SZEDATA2_PCI_DRIVER_NAME,
1576                 .id_table = rte_szedata2_pci_id_table,
1577         },
1578         .eth_dev_init     = rte_szedata2_eth_dev_init,
1579         .eth_dev_uninit   = rte_szedata2_eth_dev_uninit,
1580         .dev_private_size = sizeof(struct pmd_internals),
1581 };
1582
1583 static int
1584 rte_szedata2_init(const char *name __rte_unused,
1585                 const char *args __rte_unused)
1586 {
1587         rte_eth_driver_register(&szedata2_eth_driver);
1588         return 0;
1589 }
1590
1591 static int
1592 rte_szedata2_uninit(const char *name __rte_unused)
1593 {
1594         return 0;
1595 }
1596
1597 static struct rte_driver rte_szedata2_driver = {
1598         .type = PMD_PDEV,
1599         .name = RTE_SZEDATA2_DRIVER_NAME,
1600         .init = rte_szedata2_init,
1601         .uninit = rte_szedata2_uninit,
1602 };
1603
1604 PMD_REGISTER_DRIVER(rte_szedata2_driver, rte_szedata2_pmd);
1605 DRIVER_REGISTER_PCI_TABLE(rte_szedata2_pmd, rte_szedata2_pci_id_table);