4 * Copyright (c) 2015 CESNET
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of CESNET nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42 #include <rte_ethdev.h>
43 #include <rte_malloc.h>
44 #include <rte_memcpy.h>
45 #include <rte_kvargs.h>
48 #include "rte_eth_szedata2.h"
50 #define RTE_ETH_SZEDATA2_DEV_PATH_ARG "dev_path"
51 #define RTE_ETH_SZEDATA2_RX_IFACES_ARG "rx_ifaces"
52 #define RTE_ETH_SZEDATA2_TX_IFACES_ARG "tx_ifaces"
54 #define RTE_ETH_SZEDATA2_MAX_RX_QUEUES 32
55 #define RTE_ETH_SZEDATA2_MAX_TX_QUEUES 32
56 #define RTE_ETH_SZEDATA2_TX_LOCK_SIZE (32 * 1024 * 1024)
59 * size of szedata2_packet header with alignment
61 #define RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED 8
63 struct szedata2_rx_queue {
67 struct rte_mempool *mb_pool;
68 volatile uint64_t rx_pkts;
69 volatile uint64_t rx_bytes;
70 volatile uint64_t err_pkts;
73 struct szedata2_tx_queue {
76 volatile uint64_t tx_pkts;
77 volatile uint64_t err_pkts;
78 volatile uint64_t tx_bytes;
81 struct rxtx_szedata2 {
84 uint32_t sze_rx_mask_req;
85 uint32_t sze_tx_mask_req;
89 struct pmd_internals {
90 struct szedata2_rx_queue rx_queue[RTE_ETH_SZEDATA2_MAX_RX_QUEUES];
91 struct szedata2_tx_queue tx_queue[RTE_ETH_SZEDATA2_MAX_TX_QUEUES];
92 unsigned nb_rx_queues;
93 unsigned nb_tx_queues;
102 static const char *valid_arguments[] = {
103 RTE_ETH_SZEDATA2_DEV_PATH_ARG,
104 RTE_ETH_SZEDATA2_RX_IFACES_ARG,
105 RTE_ETH_SZEDATA2_TX_IFACES_ARG,
109 static struct ether_addr eth_addr = {
110 .addr_bytes = { 0x00, 0x11, 0x17, 0x00, 0x00, 0x00 }
112 static const char *drivername = "SZEdata2 PMD";
113 static struct rte_eth_link pmd_link = {
114 .link_speed = ETH_LINK_SPEED_10G,
115 .link_duplex = ETH_LINK_FULL_DUPLEX,
121 count_ones(uint32_t num)
123 num = num - ((num >> 1) & 0x55555555); /* reuse input as temporary */
124 num = (num & 0x33333333) + ((num >> 2) & 0x33333333); /* temp */
125 return (((num + (num >> 4)) & 0xF0F0F0F) * 0x1010101) >> 24; /* count */
129 eth_szedata2_rx(void *queue,
130 struct rte_mbuf **bufs,
134 struct rte_mbuf *mbuf;
135 struct szedata2_rx_queue *sze_q = queue;
136 struct rte_pktmbuf_pool_private *mbp_priv;
141 uint16_t packet_size;
142 uint64_t num_bytes = 0;
143 struct szedata *sze = sze_q->sze;
144 uint8_t *header_ptr = NULL; /* header of packet */
145 uint8_t *packet_ptr1 = NULL;
146 uint8_t *packet_ptr2 = NULL;
147 uint16_t packet_len1 = 0;
148 uint16_t packet_len2 = 0;
149 uint16_t hw_data_align;
151 if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
155 * Reads the given number of packets from szedata2 channel given
156 * by queue and copies the packet data into a newly allocated mbuf
159 for (i = 0; i < nb_pkts; i++) {
160 mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
162 if (unlikely(mbuf == NULL))
165 /* get the next sze packet */
166 if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
167 sze->ct_rx_lck->next == NULL) {
168 /* unlock old data */
169 szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
170 sze->ct_rx_lck_orig = NULL;
171 sze->ct_rx_lck = NULL;
174 if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
175 /* nothing to read, lock new data */
176 sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
177 sze->ct_rx_lck_orig = sze->ct_rx_lck;
179 if (sze->ct_rx_lck == NULL) {
180 /* nothing to lock */
181 rte_pktmbuf_free(mbuf);
185 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
186 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
188 if (!sze->ct_rx_rem_bytes) {
189 rte_pktmbuf_free(mbuf);
194 if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
197 * copy parts of header to merge buffer
199 if (sze->ct_rx_lck->next == NULL) {
200 rte_pktmbuf_free(mbuf);
204 /* copy first part of header */
205 rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
206 sze->ct_rx_rem_bytes);
208 /* copy second part of header */
209 sze->ct_rx_lck = sze->ct_rx_lck->next;
210 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
211 rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
213 RTE_SZE2_PACKET_HEADER_SIZE -
214 sze->ct_rx_rem_bytes);
216 sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
217 sze->ct_rx_rem_bytes;
218 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
219 RTE_SZE2_PACKET_HEADER_SIZE +
220 sze->ct_rx_rem_bytes;
222 header_ptr = (uint8_t *)sze->ct_rx_buffer;
225 header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
226 sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
227 sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
230 sg_size = le16toh(*((uint16_t *)header_ptr));
231 hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
232 packet_size = sg_size -
233 RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
236 /* checks if packet all right */
238 errx(5, "Zero segsize");
240 /* check sg_size and hwsize */
241 if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
242 errx(10, "Hwsize bigger than expected. Segsize: %d, "
243 "hwsize: %d", sg_size, hw_size);
247 RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size) -
248 RTE_SZE2_PACKET_HEADER_SIZE;
250 if (sze->ct_rx_rem_bytes >=
252 RTE_SZE2_PACKET_HEADER_SIZE)) {
254 /* one packet ready - go to another */
255 packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
256 packet_len1 = packet_size;
260 sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
261 RTE_SZE2_PACKET_HEADER_SIZE;
262 sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
263 RTE_SZE2_PACKET_HEADER_SIZE;
266 if (sze->ct_rx_lck->next == NULL) {
267 errx(6, "Need \"next\" lock, "
268 "but it is missing: %u",
269 sze->ct_rx_rem_bytes);
273 if (sze->ct_rx_rem_bytes <= hw_data_align) {
274 uint16_t rem_size = hw_data_align -
275 sze->ct_rx_rem_bytes;
277 /* MOVE to next lock */
278 sze->ct_rx_lck = sze->ct_rx_lck->next;
280 (void *)(((uint8_t *)
281 (sze->ct_rx_lck->start)) + rem_size);
283 packet_ptr1 = sze->ct_rx_cur_ptr;
284 packet_len1 = packet_size;
288 sze->ct_rx_cur_ptr +=
289 RTE_SZE2_ALIGN8(packet_size);
290 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
291 rem_size - RTE_SZE2_ALIGN8(packet_size);
293 /* get pointer and length from first part */
294 packet_ptr1 = sze->ct_rx_cur_ptr +
296 packet_len1 = sze->ct_rx_rem_bytes -
299 /* MOVE to next lock */
300 sze->ct_rx_lck = sze->ct_rx_lck->next;
301 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
303 /* get pointer and length from second part */
304 packet_ptr2 = sze->ct_rx_cur_ptr;
305 packet_len2 = packet_size - packet_len1;
307 sze->ct_rx_cur_ptr +=
308 RTE_SZE2_ALIGN8(packet_size) -
310 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
311 (RTE_SZE2_ALIGN8(packet_size) -
316 if (unlikely(packet_ptr1 == NULL)) {
317 rte_pktmbuf_free(mbuf);
321 /* get the space available for data in the mbuf */
322 mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
323 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
324 RTE_PKTMBUF_HEADROOM);
326 if (packet_size <= buf_size) {
327 /* sze packet will fit in one mbuf, go ahead and copy */
328 rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
329 packet_ptr1, packet_len1);
330 if (packet_ptr2 != NULL) {
331 rte_memcpy((void *)(rte_pktmbuf_mtod(mbuf,
332 uint8_t *) + packet_len1),
333 packet_ptr2, packet_len2);
335 mbuf->data_len = (uint16_t)packet_size;
337 mbuf->pkt_len = packet_size;
338 mbuf->port = sze_q->in_port;
341 num_bytes += packet_size;
344 * sze packet will not fit in one mbuf,
345 * scattered mode is not enabled, drop packet
348 "SZE segment %d bytes will not fit in one mbuf "
349 "(%d bytes), scattered mode is not enabled, "
351 packet_size, buf_size);
352 rte_pktmbuf_free(mbuf);
356 sze_q->rx_pkts += num_rx;
357 sze_q->rx_bytes += num_bytes;
362 eth_szedata2_rx_scattered(void *queue,
363 struct rte_mbuf **bufs,
367 struct rte_mbuf *mbuf;
368 struct szedata2_rx_queue *sze_q = queue;
369 struct rte_pktmbuf_pool_private *mbp_priv;
374 uint16_t packet_size;
375 uint64_t num_bytes = 0;
376 struct szedata *sze = sze_q->sze;
377 uint8_t *header_ptr = NULL; /* header of packet */
378 uint8_t *packet_ptr1 = NULL;
379 uint8_t *packet_ptr2 = NULL;
380 uint16_t packet_len1 = 0;
381 uint16_t packet_len2 = 0;
382 uint16_t hw_data_align;
384 if (unlikely(sze_q->sze == NULL || nb_pkts == 0))
388 * Reads the given number of packets from szedata2 channel given
389 * by queue and copies the packet data into a newly allocated mbuf
392 for (i = 0; i < nb_pkts; i++) {
393 const struct szedata_lock *ct_rx_lck_backup;
394 unsigned int ct_rx_rem_bytes_backup;
395 unsigned char *ct_rx_cur_ptr_backup;
397 /* get the next sze packet */
398 if (sze->ct_rx_lck != NULL && !sze->ct_rx_rem_bytes &&
399 sze->ct_rx_lck->next == NULL) {
400 /* unlock old data */
401 szedata_rx_unlock_data(sze_q->sze, sze->ct_rx_lck_orig);
402 sze->ct_rx_lck_orig = NULL;
403 sze->ct_rx_lck = NULL;
407 * Store items from sze structure which can be changed
408 * before mbuf allocating. Use these items in case of mbuf
409 * allocating failure.
411 ct_rx_lck_backup = sze->ct_rx_lck;
412 ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
413 ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
415 if (!sze->ct_rx_rem_bytes && sze->ct_rx_lck_orig == NULL) {
416 /* nothing to read, lock new data */
417 sze->ct_rx_lck = szedata_rx_lock_data(sze_q->sze, ~0U);
418 sze->ct_rx_lck_orig = sze->ct_rx_lck;
421 * Backup items from sze structure must be updated
422 * after locking to contain pointers to new locks.
424 ct_rx_lck_backup = sze->ct_rx_lck;
425 ct_rx_rem_bytes_backup = sze->ct_rx_rem_bytes;
426 ct_rx_cur_ptr_backup = sze->ct_rx_cur_ptr;
428 if (sze->ct_rx_lck == NULL)
429 /* nothing to lock */
432 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
433 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len;
435 if (!sze->ct_rx_rem_bytes)
439 if (sze->ct_rx_rem_bytes < RTE_SZE2_PACKET_HEADER_SIZE) {
441 * cut in header - copy parts of header to merge buffer
443 if (sze->ct_rx_lck->next == NULL)
446 /* copy first part of header */
447 rte_memcpy(sze->ct_rx_buffer, sze->ct_rx_cur_ptr,
448 sze->ct_rx_rem_bytes);
450 /* copy second part of header */
451 sze->ct_rx_lck = sze->ct_rx_lck->next;
452 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
453 rte_memcpy(sze->ct_rx_buffer + sze->ct_rx_rem_bytes,
455 RTE_SZE2_PACKET_HEADER_SIZE -
456 sze->ct_rx_rem_bytes);
458 sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE -
459 sze->ct_rx_rem_bytes;
460 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
461 RTE_SZE2_PACKET_HEADER_SIZE +
462 sze->ct_rx_rem_bytes;
464 header_ptr = (uint8_t *)sze->ct_rx_buffer;
467 header_ptr = (uint8_t *)sze->ct_rx_cur_ptr;
468 sze->ct_rx_cur_ptr += RTE_SZE2_PACKET_HEADER_SIZE;
469 sze->ct_rx_rem_bytes -= RTE_SZE2_PACKET_HEADER_SIZE;
472 sg_size = le16toh(*((uint16_t *)header_ptr));
473 hw_size = le16toh(*(((uint16_t *)header_ptr) + 1));
474 packet_size = sg_size -
475 RTE_SZE2_ALIGN8(RTE_SZE2_PACKET_HEADER_SIZE + hw_size);
478 /* checks if packet all right */
480 errx(5, "Zero segsize");
482 /* check sg_size and hwsize */
483 if (hw_size > sg_size - RTE_SZE2_PACKET_HEADER_SIZE) {
484 errx(10, "Hwsize bigger than expected. Segsize: %d, "
485 "hwsize: %d", sg_size, hw_size);
489 RTE_SZE2_ALIGN8((RTE_SZE2_PACKET_HEADER_SIZE +
490 hw_size)) - RTE_SZE2_PACKET_HEADER_SIZE;
492 if (sze->ct_rx_rem_bytes >=
494 RTE_SZE2_PACKET_HEADER_SIZE)) {
496 /* one packet ready - go to another */
497 packet_ptr1 = sze->ct_rx_cur_ptr + hw_data_align;
498 packet_len1 = packet_size;
502 sze->ct_rx_cur_ptr += RTE_SZE2_ALIGN8(sg_size) -
503 RTE_SZE2_PACKET_HEADER_SIZE;
504 sze->ct_rx_rem_bytes -= RTE_SZE2_ALIGN8(sg_size) -
505 RTE_SZE2_PACKET_HEADER_SIZE;
508 if (sze->ct_rx_lck->next == NULL) {
509 errx(6, "Need \"next\" lock, but it is "
510 "missing: %u", sze->ct_rx_rem_bytes);
514 if (sze->ct_rx_rem_bytes <= hw_data_align) {
515 uint16_t rem_size = hw_data_align -
516 sze->ct_rx_rem_bytes;
518 /* MOVE to next lock */
519 sze->ct_rx_lck = sze->ct_rx_lck->next;
521 (void *)(((uint8_t *)
522 (sze->ct_rx_lck->start)) + rem_size);
524 packet_ptr1 = sze->ct_rx_cur_ptr;
525 packet_len1 = packet_size;
529 sze->ct_rx_cur_ptr +=
530 RTE_SZE2_ALIGN8(packet_size);
531 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
532 rem_size - RTE_SZE2_ALIGN8(packet_size);
534 /* get pointer and length from first part */
535 packet_ptr1 = sze->ct_rx_cur_ptr +
537 packet_len1 = sze->ct_rx_rem_bytes -
540 /* MOVE to next lock */
541 sze->ct_rx_lck = sze->ct_rx_lck->next;
542 sze->ct_rx_cur_ptr = sze->ct_rx_lck->start;
544 /* get pointer and length from second part */
545 packet_ptr2 = sze->ct_rx_cur_ptr;
546 packet_len2 = packet_size - packet_len1;
548 sze->ct_rx_cur_ptr +=
549 RTE_SZE2_ALIGN8(packet_size) -
551 sze->ct_rx_rem_bytes = sze->ct_rx_lck->len -
552 (RTE_SZE2_ALIGN8(packet_size) -
557 if (unlikely(packet_ptr1 == NULL))
560 mbuf = rte_pktmbuf_alloc(sze_q->mb_pool);
562 if (unlikely(mbuf == NULL)) {
564 * Restore items from sze structure to state after
565 * unlocking (eventually locking).
567 sze->ct_rx_lck = ct_rx_lck_backup;
568 sze->ct_rx_rem_bytes = ct_rx_rem_bytes_backup;
569 sze->ct_rx_cur_ptr = ct_rx_cur_ptr_backup;
573 /* get the space available for data in the mbuf */
574 mbp_priv = rte_mempool_get_priv(sze_q->mb_pool);
575 buf_size = (uint16_t)(mbp_priv->mbuf_data_room_size -
576 RTE_PKTMBUF_HEADROOM);
578 if (packet_size <= buf_size) {
579 /* sze packet will fit in one mbuf, go ahead and copy */
580 rte_memcpy(rte_pktmbuf_mtod(mbuf, void *),
581 packet_ptr1, packet_len1);
582 if (packet_ptr2 != NULL) {
584 (rte_pktmbuf_mtod(mbuf, uint8_t *) +
585 packet_len1), packet_ptr2, packet_len2);
587 mbuf->data_len = (uint16_t)packet_size;
590 * sze packet will not fit in one mbuf,
591 * scatter packet into more mbufs
593 struct rte_mbuf *m = mbuf;
594 uint16_t len = rte_pktmbuf_tailroom(mbuf);
596 /* copy first part of packet */
597 /* fill first mbuf */
598 rte_memcpy(rte_pktmbuf_append(mbuf, len), packet_ptr1,
601 packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
603 while (packet_len1 > 0) {
605 m->next = rte_pktmbuf_alloc(sze_q->mb_pool);
607 if (unlikely(m->next == NULL)) {
608 rte_pktmbuf_free(mbuf);
610 * Restore items from sze structure
611 * to state after unlocking (eventually
614 sze->ct_rx_lck = ct_rx_lck_backup;
615 sze->ct_rx_rem_bytes =
616 ct_rx_rem_bytes_backup;
618 ct_rx_cur_ptr_backup;
624 len = RTE_MIN(rte_pktmbuf_tailroom(m),
626 rte_memcpy(rte_pktmbuf_append(mbuf, len),
631 packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
634 if (packet_ptr2 != NULL) {
635 /* copy second part of packet, if exists */
636 /* fill the rest of currently last mbuf */
637 len = rte_pktmbuf_tailroom(m);
638 rte_memcpy(rte_pktmbuf_append(mbuf, len),
641 packet_ptr2 = ((uint8_t *)packet_ptr2) + len;
643 while (packet_len2 > 0) {
645 m->next = rte_pktmbuf_alloc(
648 if (unlikely(m->next == NULL)) {
649 rte_pktmbuf_free(mbuf);
651 * Restore items from sze
652 * structure to state after
653 * unlocking (eventually
658 sze->ct_rx_rem_bytes =
659 ct_rx_rem_bytes_backup;
661 ct_rx_cur_ptr_backup;
667 len = RTE_MIN(rte_pktmbuf_tailroom(m),
670 rte_pktmbuf_append(mbuf, len),
675 packet_ptr2 = ((uint8_t *)packet_ptr2) +
680 mbuf->pkt_len = packet_size;
681 mbuf->port = sze_q->in_port;
684 num_bytes += packet_size;
688 sze_q->rx_pkts += num_rx;
689 sze_q->rx_bytes += num_bytes;
694 eth_szedata2_tx(void *queue,
695 struct rte_mbuf **bufs,
698 struct rte_mbuf *mbuf;
699 struct szedata2_tx_queue *sze_q = queue;
701 uint64_t num_bytes = 0;
703 const struct szedata_lock *lck;
709 uint32_t unlock_size;
712 uint16_t pkt_left = nb_pkts;
714 if (sze_q->sze == NULL || nb_pkts == 0)
717 while (pkt_left > 0) {
719 lck = szedata_tx_lock_data(sze_q->sze,
720 RTE_ETH_SZEDATA2_TX_LOCK_SIZE,
726 lock_size = lck->len;
727 lock_size2 = lck->next ? lck->next->len : 0;
730 mbuf = bufs[nb_pkts - pkt_left];
732 pkt_len = mbuf->pkt_len;
733 mbuf_segs = mbuf->nb_segs;
735 hwpkt_len = RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
736 RTE_SZE2_ALIGN8(pkt_len);
738 if (lock_size + lock_size2 < hwpkt_len) {
739 szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
743 num_bytes += pkt_len;
745 if (lock_size > hwpkt_len) {
750 /* write packet length at first 2 bytes in 8B header */
751 *((uint16_t *)dst) = htole16(
752 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
754 *(((uint16_t *)dst) + 1) = htole16(0);
756 /* copy packet from mbuf */
757 tmp_dst = ((uint8_t *)(dst)) +
758 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
759 if (mbuf_segs == 1) {
761 * non-scattered packet,
762 * transmit from one mbuf
765 rte_pktmbuf_mtod(mbuf, const void *),
768 /* scattered packet, transmit from more mbufs */
769 struct rte_mbuf *m = mbuf;
775 tmp_dst = ((uint8_t *)(tmp_dst)) +
782 dst = ((uint8_t *)dst) + hwpkt_len;
783 unlock_size += hwpkt_len;
784 lock_size -= hwpkt_len;
786 rte_pktmbuf_free(mbuf);
790 szedata_tx_unlock_data(sze_q->sze, lck,
795 } else if (lock_size + lock_size2 >= hwpkt_len) {
799 /* write packet length at first 2 bytes in 8B header */
801 htole16(RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
803 *(((uint16_t *)dst) + 1) = htole16(0);
806 * If the raw packet (pkt_len) is smaller than lock_size
807 * get the correct length for memcpy
810 pkt_len < lock_size -
811 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED ?
813 lock_size - RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
815 rem_len = hwpkt_len - lock_size;
817 tmp_dst = ((uint8_t *)(dst)) +
818 RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
819 if (mbuf_segs == 1) {
821 * non-scattered packet,
822 * transmit from one mbuf
824 /* copy part of packet to first area */
826 rte_pktmbuf_mtod(mbuf, const void *),
830 dst = lck->next->start;
832 /* copy part of packet to second area */
834 (const void *)(rte_pktmbuf_mtod(mbuf,
836 write_len), pkt_len - write_len);
838 /* scattered packet, transmit from more mbufs */
839 struct rte_mbuf *m = mbuf;
840 uint16_t written = 0;
841 uint16_t to_write = 0;
842 bool new_mbuf = true;
843 uint16_t write_off = 0;
845 /* copy part of packet to first area */
846 while (m && written < write_len) {
847 to_write = RTE_MIN(m->data_len,
848 write_len - written);
854 tmp_dst = ((uint8_t *)(tmp_dst)) +
856 if (m->data_len <= write_len -
867 dst = lck->next->start;
871 write_off = new_mbuf ? 0 : to_write;
873 /* copy part of packet to second area */
874 while (m && written < pkt_len - write_len) {
875 rte_memcpy(tmp_dst, (const void *)
877 uint8_t *) + write_off),
878 m->data_len - write_off);
880 tmp_dst = ((uint8_t *)(tmp_dst)) +
881 (m->data_len - write_off);
882 written += m->data_len - write_off;
888 dst = ((uint8_t *)dst) + rem_len;
889 unlock_size += hwpkt_len;
890 lock_size = lock_size2 - rem_len;
893 rte_pktmbuf_free(mbuf);
897 szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
901 sze_q->tx_pkts += num_tx;
902 sze_q->err_pkts += nb_pkts - num_tx;
903 sze_q->tx_bytes += num_bytes;
908 init_rx_channels(struct rte_eth_dev *dev, int v)
910 struct pmd_internals *internals = dev->data->dev_private;
913 uint32_t count = internals->num_of_rx;
914 uint32_t num_sub = 0;
919 rx = internals->sze_rx_req;
922 for (i = 0; i < count; i++) {
924 * Open, subscribe rx,tx channels and start device
927 RTE_LOG(INFO, PMD, "Opening SZE device %u. time\n", i);
929 internals->rx_queue[num_sub].sze =
930 szedata_open(internals->sze_dev);
931 if (internals->rx_queue[num_sub].sze == NULL)
934 /* separate least significant non-zero bit */
935 x = rx & ((~rx) + 1);
938 RTE_LOG(INFO, PMD, "Subscribing rx channel: 0x%x "
939 "tx channel: 0x%x\n", x, tx);
941 ret = szedata_subscribe3(internals->rx_queue[num_sub].sze,
944 szedata_close(internals->rx_queue[num_sub].sze);
945 internals->rx_queue[num_sub].sze = NULL;
950 RTE_LOG(INFO, PMD, "Subscribed rx channel: 0x%x "
951 "tx channel: 0x%x\n", x, tx);
955 RTE_LOG(INFO, PMD, "Starting SZE device for "
956 "rx queue: %u\n", num_sub);
958 ret = szedata_start(internals->rx_queue[num_sub].sze);
960 szedata_close(internals->rx_queue[num_sub].sze);
961 internals->rx_queue[num_sub].sze = NULL;
966 * set to 1 all bits lower than bit set to 1
970 internals->rx_queue[num_sub].rx_channel =
974 RTE_LOG(INFO, PMD, "Subscribed rx channel "
976 internals->rx_queue[num_sub].rx_channel
980 internals->nb_rx_queues = num_sub;
984 "Could not subscribe any rx channel. "
985 "Closing SZE device\n");
987 szedata_close(internals->rx_queue[num_sub].sze);
988 internals->rx_queue[num_sub].sze = NULL;
991 /* set least significant non-zero bit to zero */
995 dev->data->nb_rx_queues = (uint16_t)num_sub;
998 RTE_LOG(INFO, PMD, "Successfully opened rx channels: %u\n",
1005 init_tx_channels(struct rte_eth_dev *dev, int v)
1007 struct pmd_internals *internals = dev->data->dev_private;
1010 uint32_t count = internals->num_of_tx;
1011 uint32_t num_sub = 0;
1017 tx = internals->sze_tx_req;
1019 for (i = 0; i < count; i++) {
1021 * Open, subscribe rx,tx channels and start device
1024 RTE_LOG(INFO, PMD, "Opening SZE device %u. time\n",
1025 i + internals->num_of_rx);
1027 internals->tx_queue[num_sub].sze =
1028 szedata_open(internals->sze_dev);
1029 if (internals->tx_queue[num_sub].sze == NULL)
1032 /* separate least significant non-zero bit */
1033 x = tx & ((~tx) + 1);
1036 RTE_LOG(INFO, PMD, "Subscribing rx channel: 0x%x "
1037 "tx channel: 0x%x\n", rx, x);
1039 ret = szedata_subscribe3(internals->tx_queue[num_sub].sze,
1042 szedata_close(internals->tx_queue[num_sub].sze);
1043 internals->tx_queue[num_sub].sze = NULL;
1048 RTE_LOG(INFO, PMD, "Subscribed rx channel: 0x%x "
1049 "tx channel: 0x%x\n", rx, x);
1053 RTE_LOG(INFO, PMD, "Starting SZE device for "
1054 "tx queue: %u\n", num_sub);
1056 ret = szedata_start(internals->tx_queue[num_sub].sze);
1058 szedata_close(internals->tx_queue[num_sub].sze);
1059 internals->tx_queue[num_sub].sze = NULL;
1064 * set to 1 all bits lower than bit set to 1
1068 internals->tx_queue[num_sub].tx_channel =
1072 RTE_LOG(INFO, PMD, "Subscribed tx channel "
1074 internals->tx_queue[num_sub].tx_channel
1078 internals->nb_tx_queues = num_sub;
1082 "Could not subscribe any tx channel. "
1083 "Closing SZE device\n");
1085 szedata_close(internals->tx_queue[num_sub].sze);
1086 internals->tx_queue[num_sub].sze = NULL;
1089 /* set least significant non-zero bit to zero */
1093 dev->data->nb_tx_queues = (uint16_t)num_sub;
1096 RTE_LOG(INFO, PMD, "Successfully opened tx channels: %u\n",
1103 close_rx_channels(struct rte_eth_dev *dev)
1105 struct pmd_internals *internals = dev->data->dev_private;
1107 uint32_t num_sub = internals->nb_rx_queues;
1109 for (i = 0; i < num_sub; i++) {
1110 if (internals->rx_queue[i].sze != NULL) {
1111 szedata_close(internals->rx_queue[i].sze);
1112 internals->rx_queue[i].sze = NULL;
1115 /* set number of rx queues to zero */
1116 internals->nb_rx_queues = 0;
1117 dev->data->nb_rx_queues = (uint16_t)0;
1121 close_tx_channels(struct rte_eth_dev *dev)
1123 struct pmd_internals *internals = dev->data->dev_private;
1125 uint32_t num_sub = internals->nb_tx_queues;
1127 for (i = 0; i < num_sub; i++) {
1128 if (internals->tx_queue[i].sze != NULL) {
1129 szedata_close(internals->tx_queue[i].sze);
1130 internals->tx_queue[i].sze = NULL;
1133 /* set number of rx queues to zero */
1134 internals->nb_tx_queues = 0;
1135 dev->data->nb_tx_queues = (uint16_t)0;
1139 eth_dev_start(struct rte_eth_dev *dev)
1141 struct pmd_internals *internals = dev->data->dev_private;
1144 if (internals->nb_rx_queues == 0) {
1145 ret = init_rx_channels(dev, 0);
1147 close_rx_channels(dev);
1152 if (internals->nb_tx_queues == 0) {
1153 ret = init_tx_channels(dev, 0);
1155 close_tx_channels(dev);
1156 close_rx_channels(dev);
1161 dev->data->dev_link.link_status = 1;
1166 eth_dev_stop(struct rte_eth_dev *dev)
1169 struct pmd_internals *internals = dev->data->dev_private;
1171 for (i = 0; i < internals->nb_rx_queues; i++) {
1172 if (internals->rx_queue[i].sze != NULL) {
1173 szedata_close(internals->rx_queue[i].sze);
1174 internals->rx_queue[i].sze = NULL;
1178 for (i = 0; i < internals->nb_tx_queues; i++) {
1179 if (internals->tx_queue[i].sze != NULL) {
1180 szedata_close(internals->tx_queue[i].sze);
1181 internals->tx_queue[i].sze = NULL;
1185 internals->nb_rx_queues = 0;
1186 internals->nb_tx_queues = 0;
1188 dev->data->nb_rx_queues = (uint16_t)0;
1189 dev->data->nb_tx_queues = (uint16_t)0;
1191 dev->data->dev_link.link_status = 0;
1195 eth_dev_configure(struct rte_eth_dev *dev)
1197 struct rte_eth_dev_data *data = dev->data;
1198 if (data->dev_conf.rxmode.enable_scatter == 1) {
1199 dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1200 data->scattered_rx = 1;
1202 dev->rx_pkt_burst = eth_szedata2_rx;
1203 data->scattered_rx = 0;
1209 eth_dev_info(struct rte_eth_dev *dev,
1210 struct rte_eth_dev_info *dev_info)
1212 struct pmd_internals *internals = dev->data->dev_private;
1213 dev_info->driver_name = drivername;
1214 dev_info->if_index = internals->if_index;
1215 dev_info->max_mac_addrs = 1;
1216 dev_info->max_rx_pktlen = (uint32_t)-1;
1217 dev_info->max_rx_queues = (uint16_t)internals->nb_rx_queues;
1218 dev_info->max_tx_queues = (uint16_t)internals->nb_tx_queues;
1219 dev_info->min_rx_bufsize = 0;
1220 dev_info->pci_dev = NULL;
1224 eth_stats_get(struct rte_eth_dev *dev,
1225 struct rte_eth_stats *stats)
1228 uint64_t rx_total = 0;
1229 uint64_t tx_total = 0;
1230 uint64_t tx_err_total = 0;
1231 uint64_t rx_total_bytes = 0;
1232 uint64_t tx_total_bytes = 0;
1233 const struct pmd_internals *internal = dev->data->dev_private;
1235 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS &&
1236 i < internal->nb_rx_queues; i++) {
1237 stats->q_ipackets[i] = internal->rx_queue[i].rx_pkts;
1238 stats->q_ibytes[i] = internal->rx_queue[i].rx_bytes;
1239 rx_total += stats->q_ipackets[i];
1240 rx_total_bytes += stats->q_ibytes[i];
1243 for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS &&
1244 i < internal->nb_tx_queues; i++) {
1245 stats->q_opackets[i] = internal->tx_queue[i].tx_pkts;
1246 stats->q_errors[i] = internal->tx_queue[i].err_pkts;
1247 stats->q_obytes[i] = internal->tx_queue[i].tx_bytes;
1248 tx_total += stats->q_opackets[i];
1249 tx_err_total += stats->q_errors[i];
1250 tx_total_bytes += stats->q_obytes[i];
1253 stats->ipackets = rx_total;
1254 stats->opackets = tx_total;
1255 stats->ibytes = rx_total_bytes;
1256 stats->obytes = tx_total_bytes;
1257 stats->oerrors = tx_err_total;
1261 eth_stats_reset(struct rte_eth_dev *dev)
1264 struct pmd_internals *internal = dev->data->dev_private;
1265 for (i = 0; i < internal->nb_rx_queues; i++) {
1266 internal->rx_queue[i].rx_pkts = 0;
1267 internal->rx_queue[i].rx_bytes = 0;
1269 for (i = 0; i < internal->nb_tx_queues; i++) {
1270 internal->tx_queue[i].tx_pkts = 0;
1271 internal->tx_queue[i].err_pkts = 0;
1272 internal->tx_queue[i].tx_bytes = 0;
1277 eth_dev_close(struct rte_eth_dev *dev)
1280 struct pmd_internals *internals = dev->data->dev_private;
1282 for (i = 0; i < internals->nb_rx_queues; i++) {
1283 if (internals->rx_queue[i].sze != NULL) {
1284 szedata_close(internals->rx_queue[i].sze);
1285 internals->rx_queue[i].sze = NULL;
1289 for (i = 0; i < internals->nb_tx_queues; i++) {
1290 if (internals->tx_queue[i].sze != NULL) {
1291 szedata_close(internals->tx_queue[i].sze);
1292 internals->tx_queue[i].sze = NULL;
1296 internals->nb_rx_queues = 0;
1297 internals->nb_tx_queues = 0;
1299 dev->data->nb_rx_queues = (uint16_t)0;
1300 dev->data->nb_tx_queues = (uint16_t)0;
1304 eth_queue_release(void *q __rte_unused)
1309 eth_link_update(struct rte_eth_dev *dev __rte_unused,
1310 int wait_to_complete __rte_unused)
1316 eth_rx_queue_setup(struct rte_eth_dev *dev,
1317 uint16_t rx_queue_id,
1318 uint16_t nb_rx_desc __rte_unused,
1319 unsigned int socket_id __rte_unused,
1320 const struct rte_eth_rxconf *rx_conf __rte_unused,
1321 struct rte_mempool *mb_pool)
1323 struct pmd_internals *internals = dev->data->dev_private;
1324 struct szedata2_rx_queue *szedata2_q =
1325 &internals->rx_queue[rx_queue_id];
1326 szedata2_q->mb_pool = mb_pool;
1327 dev->data->rx_queues[rx_queue_id] = szedata2_q;
1328 szedata2_q->in_port = dev->data->port_id;
1333 eth_tx_queue_setup(struct rte_eth_dev *dev,
1334 uint16_t tx_queue_id,
1335 uint16_t nb_tx_desc __rte_unused,
1336 unsigned int socket_id __rte_unused,
1337 const struct rte_eth_txconf *tx_conf __rte_unused)
1339 struct pmd_internals *internals = dev->data->dev_private;
1340 dev->data->tx_queues[tx_queue_id] = &internals->tx_queue[tx_queue_id];
1345 eth_mac_addr_set(struct rte_eth_dev *dev __rte_unused,
1346 struct ether_addr *mac_addr __rte_unused)
1350 static struct eth_dev_ops ops = {
1351 .dev_start = eth_dev_start,
1352 .dev_stop = eth_dev_stop,
1353 .dev_close = eth_dev_close,
1354 .dev_configure = eth_dev_configure,
1355 .dev_infos_get = eth_dev_info,
1356 .rx_queue_setup = eth_rx_queue_setup,
1357 .tx_queue_setup = eth_tx_queue_setup,
1358 .rx_queue_release = eth_queue_release,
1359 .tx_queue_release = eth_queue_release,
1360 .link_update = eth_link_update,
1361 .stats_get = eth_stats_get,
1362 .stats_reset = eth_stats_reset,
1363 .mac_addr_set = eth_mac_addr_set,
1367 parse_mask(const char *mask_str, uint32_t *mask_num)
1372 value = strtol(mask_str, &endptr, 0);
1373 if (*endptr != '\0' || value > UINT32_MAX || value < 0)
1376 *mask_num = (uint32_t)value;
1381 add_rx_mask(const char *key __rte_unused, const char *value, void *extra_args)
1383 struct rxtx_szedata2 *szedata2 = extra_args;
1386 if (parse_mask(value, &mask) != 0)
1389 szedata2->sze_rx_mask_req |= mask;
1394 add_tx_mask(const char *key __rte_unused, const char *value, void *extra_args)
1396 struct rxtx_szedata2 *szedata2 = extra_args;
1399 if (parse_mask(value, &mask) != 0)
1402 szedata2->sze_tx_mask_req |= mask;
1407 rte_pmd_init_internals(const char *name, const unsigned nb_rx_queues,
1408 const unsigned nb_tx_queues,
1409 const unsigned numa_node,
1410 struct pmd_internals **internals,
1411 struct rte_eth_dev **eth_dev)
1413 struct rte_eth_dev_data *data = NULL;
1416 "Creating szedata2-backed ethdev on numa socket %u\n",
1420 * now do all data allocation - for eth_dev structure
1421 * and internal (private) data
1423 data = rte_zmalloc_socket(name, sizeof(*data), 0, numa_node);
1427 *internals = rte_zmalloc_socket(name, sizeof(**internals), 0,
1429 if (*internals == NULL)
1432 /* reserve an ethdev entry */
1433 *eth_dev = rte_eth_dev_allocate(name, RTE_ETH_DEV_VIRTUAL);
1434 if (*eth_dev == NULL)
1438 * now put it all together
1439 * - store queue data in internals,
1440 * - store numa_node info in pci_driver
1441 * - point eth_dev_data to internals
1442 * - and point eth_dev structure to new eth_dev_data structure
1444 * NOTE: we'll replace the data element, of originally allocated eth_dev
1445 * so the rings are local per-process
1448 (*internals)->nb_rx_queues = nb_rx_queues;
1449 (*internals)->nb_tx_queues = nb_tx_queues;
1451 (*internals)->if_index = 0;
1453 data->dev_private = *internals;
1454 data->port_id = (*eth_dev)->data->port_id;
1455 snprintf(data->name, sizeof(data->name), "%s", (*eth_dev)->data->name);
1456 data->nb_rx_queues = (uint16_t)nb_rx_queues;
1457 data->nb_tx_queues = (uint16_t)nb_tx_queues;
1458 data->dev_link = pmd_link;
1459 data->mac_addrs = ð_addr;
1461 (*eth_dev)->data = data;
1462 (*eth_dev)->dev_ops = &ops;
1463 (*eth_dev)->data->dev_flags = RTE_ETH_DEV_DETACHABLE;
1464 (*eth_dev)->driver = NULL;
1465 (*eth_dev)->data->kdrv = RTE_KDRV_NONE;
1466 (*eth_dev)->data->drv_name = drivername;
1467 (*eth_dev)->data->numa_node = numa_node;
1473 rte_free(*internals);
1478 rte_eth_from_szedata2(const char *name,
1479 struct rxtx_szedata2 *szedata2,
1480 const unsigned numa_node)
1482 struct pmd_internals *internals = NULL;
1483 struct rte_eth_dev *eth_dev = NULL;
1484 struct rte_eth_dev_data *data = NULL;
1487 if (rte_pmd_init_internals(name, 0, 0, numa_node,
1488 &internals, ð_dev) < 0)
1491 data = eth_dev->data;
1493 internals->sze_dev = szedata2->sze_dev;
1494 internals->sze_rx_req = szedata2->sze_rx_mask_req;
1495 internals->sze_tx_req = szedata2->sze_tx_mask_req;
1496 internals->num_of_rx = szedata2->num_of_rx;
1497 internals->num_of_tx = szedata2->num_of_tx;
1499 RTE_LOG(INFO, PMD, "Number of rx channels to open: %u mask: 0x%x\n",
1500 internals->num_of_rx, internals->sze_rx_req);
1501 RTE_LOG(INFO, PMD, "Number of tx channels to open: %u mask: 0x%x\n",
1502 internals->num_of_tx, internals->sze_tx_req);
1504 ret = init_rx_channels(eth_dev, 1);
1506 close_rx_channels(eth_dev);
1510 ret = init_tx_channels(eth_dev, 1);
1512 close_tx_channels(eth_dev);
1513 close_rx_channels(eth_dev);
1517 if (data->dev_conf.rxmode.enable_scatter == 1 ||
1518 data->scattered_rx == 1) {
1519 eth_dev->rx_pkt_burst = eth_szedata2_rx_scattered;
1520 data->scattered_rx = 1;
1522 eth_dev->rx_pkt_burst = eth_szedata2_rx;
1523 data->scattered_rx = 0;
1525 eth_dev->tx_pkt_burst = eth_szedata2_tx;
1532 rte_pmd_szedata2_devinit(const char *name, const char *params)
1536 struct rte_kvargs *kvlist;
1538 struct rte_kvargs_pair *pair = NULL;
1539 struct rxtx_szedata2 szedata2 = { 0, 0, 0, 0, NULL };
1540 bool dev_path_missing = true;
1542 RTE_LOG(INFO, PMD, "Initializing pmd_szedata2 for %s\n", name);
1544 numa_node = rte_socket_id();
1546 kvlist = rte_kvargs_parse(params, valid_arguments);
1551 * Get szedata2 device path and rx,tx channels from passed arguments.
1554 if (rte_kvargs_count(kvlist, RTE_ETH_SZEDATA2_DEV_PATH_ARG) != 1)
1557 if (rte_kvargs_count(kvlist, RTE_ETH_SZEDATA2_RX_IFACES_ARG) < 1)
1560 if (rte_kvargs_count(kvlist, RTE_ETH_SZEDATA2_TX_IFACES_ARG) < 1)
1563 for (k_idx = 0; k_idx < kvlist->count; k_idx++) {
1564 pair = &kvlist->pairs[k_idx];
1565 if (strstr(pair->key, RTE_ETH_SZEDATA2_DEV_PATH_ARG) != NULL) {
1566 szedata2.sze_dev = pair->value;
1567 dev_path_missing = false;
1572 if (dev_path_missing)
1575 ret = rte_kvargs_process(kvlist, RTE_ETH_SZEDATA2_RX_IFACES_ARG,
1576 &add_rx_mask, &szedata2);
1580 ret = rte_kvargs_process(kvlist, RTE_ETH_SZEDATA2_TX_IFACES_ARG,
1581 &add_tx_mask, &szedata2);
1585 szedata2.num_of_rx = count_ones(szedata2.sze_rx_mask_req);
1586 szedata2.num_of_tx = count_ones(szedata2.sze_tx_mask_req);
1588 RTE_LOG(INFO, PMD, "SZE device found at path %s\n", szedata2.sze_dev);
1590 return rte_eth_from_szedata2(name, &szedata2, numa_node);
1592 rte_kvargs_free(kvlist);
1597 rte_pmd_szedata2_devuninit(const char *name)
1599 struct rte_eth_dev *dev = NULL;
1601 RTE_LOG(INFO, PMD, "Uninitializing pmd_szedata2 for %s "
1602 "on numa socket %u\n", name, rte_socket_id());
1607 dev = rte_eth_dev_allocated(name);
1611 rte_free(dev->data->dev_private);
1612 rte_free(dev->data);
1613 rte_eth_dev_release_port(dev);
1617 static struct rte_driver pmd_szedata2_drv = {
1618 .name = "eth_szedata2",
1620 .init = rte_pmd_szedata2_devinit,
1621 .uninit = rte_pmd_szedata2_devuninit,
1624 PMD_REGISTER_DRIVER(pmd_szedata2_drv);