+ /*
+ * sze packet will not fit in one mbuf,
+ * scatter packet into more mbufs
+ */
+ struct rte_mbuf *m = mbuf;
+ uint16_t len = rte_pktmbuf_tailroom(mbuf);
+
+ /* copy first part of packet */
+ /* fill first mbuf */
+ rte_memcpy(rte_pktmbuf_append(mbuf, len), packet_ptr1,
+ len);
+ packet_len1 -= len;
+ packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
+
+ while (packet_len1 > 0) {
+ /* fill new mbufs */
+ m->next = rte_pktmbuf_alloc(sze_q->mb_pool);
+
+ if (unlikely(m->next == NULL)) {
+ rte_pktmbuf_free(mbuf);
+ /*
+ * Restore items from sze structure
+ * to state after unlocking (eventually
+ * locking).
+ */
+ sze->ct_rx_lck = ct_rx_lck_backup;
+ sze->ct_rx_rem_bytes =
+ ct_rx_rem_bytes_backup;
+ sze->ct_rx_cur_ptr =
+ ct_rx_cur_ptr_backup;
+ (*mbuf_failed_ptr)++;
+ goto finish;
+ }
+
+ m = m->next;
+
+ len = RTE_MIN(rte_pktmbuf_tailroom(m),
+ packet_len1);
+ rte_memcpy(rte_pktmbuf_append(mbuf, len),
+ packet_ptr1, len);
+
+ (mbuf->nb_segs)++;
+ packet_len1 -= len;
+ packet_ptr1 = ((uint8_t *)packet_ptr1) + len;
+ }
+
+ if (packet_ptr2 != NULL) {
+ /* copy second part of packet, if exists */
+ /* fill the rest of currently last mbuf */
+ len = rte_pktmbuf_tailroom(m);
+ rte_memcpy(rte_pktmbuf_append(mbuf, len),
+ packet_ptr2, len);
+ packet_len2 -= len;
+ packet_ptr2 = ((uint8_t *)packet_ptr2) + len;
+
+ while (packet_len2 > 0) {
+ /* fill new mbufs */
+ m->next = rte_pktmbuf_alloc(
+ sze_q->mb_pool);
+
+ if (unlikely(m->next == NULL)) {
+ rte_pktmbuf_free(mbuf);
+ /*
+ * Restore items from sze
+ * structure to state after
+ * unlocking (eventually
+ * locking).
+ */
+ sze->ct_rx_lck =
+ ct_rx_lck_backup;
+ sze->ct_rx_rem_bytes =
+ ct_rx_rem_bytes_backup;
+ sze->ct_rx_cur_ptr =
+ ct_rx_cur_ptr_backup;
+ (*mbuf_failed_ptr)++;
+ goto finish;
+ }
+
+ m = m->next;
+
+ len = RTE_MIN(rte_pktmbuf_tailroom(m),
+ packet_len2);
+ rte_memcpy(
+ rte_pktmbuf_append(mbuf, len),
+ packet_ptr2, len);
+
+ (mbuf->nb_segs)++;
+ packet_len2 -= len;
+ packet_ptr2 = ((uint8_t *)packet_ptr2) +
+ len;
+ }
+ }
+ }
+ mbuf->pkt_len = packet_size;
+ mbuf->port = sze_q->in_port;
+ bufs[num_rx] = mbuf;
+ num_rx++;
+ num_bytes += packet_size;
+ }
+
+finish:
+ sze_q->rx_pkts += num_rx;
+ sze_q->rx_bytes += num_bytes;
+ return num_rx;
+}
+
+static uint16_t
+eth_szedata2_tx(void *queue,
+ struct rte_mbuf **bufs,
+ uint16_t nb_pkts)
+{
+ struct rte_mbuf *mbuf;
+ struct szedata2_tx_queue *sze_q = queue;
+ uint16_t num_tx = 0;
+ uint64_t num_bytes = 0;
+
+ const struct szedata_lock *lck;
+ uint32_t lock_size;
+ uint32_t lock_size2;
+ void *dst;
+ uint32_t pkt_len;
+ uint32_t hwpkt_len;
+ uint32_t unlock_size;
+ uint32_t rem_len;
+ uint16_t mbuf_segs;
+ uint16_t pkt_left = nb_pkts;
+
+ if (sze_q->sze == NULL || nb_pkts == 0)
+ return 0;
+
+ while (pkt_left > 0) {
+ unlock_size = 0;
+ lck = szedata_tx_lock_data(sze_q->sze,
+ RTE_ETH_SZEDATA2_TX_LOCK_SIZE,
+ sze_q->tx_channel);
+ if (lck == NULL)
+ continue;
+
+ dst = lck->start;
+ lock_size = lck->len;
+ lock_size2 = lck->next ? lck->next->len : 0;
+
+next_packet:
+ mbuf = bufs[nb_pkts - pkt_left];
+
+ pkt_len = mbuf->pkt_len;
+ mbuf_segs = mbuf->nb_segs;
+
+ hwpkt_len = RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
+ RTE_SZE2_ALIGN8(pkt_len);
+
+ if (lock_size + lock_size2 < hwpkt_len) {
+ szedata_tx_unlock_data(sze_q->sze, lck, unlock_size);
+ continue;
+ }
+
+ num_bytes += pkt_len;
+
+ if (lock_size > hwpkt_len) {
+ void *tmp_dst;
+
+ rem_len = 0;
+
+ /* write packet length at first 2 bytes in 8B header */
+ *((uint16_t *)dst) = htole16(
+ RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED +
+ pkt_len);
+ *(((uint16_t *)dst) + 1) = htole16(0);
+
+ /* copy packet from mbuf */
+ tmp_dst = ((uint8_t *)(dst)) +
+ RTE_SZE2_PACKET_HEADER_SIZE_ALIGNED;
+ if (mbuf_segs == 1) {
+ /*
+ * non-scattered packet,
+ * transmit from one mbuf
+ */
+ rte_memcpy(tmp_dst,
+ rte_pktmbuf_mtod(mbuf, const void *),
+ pkt_len);
+ } else {
+ /* scattered packet, transmit from more mbufs */
+ struct rte_mbuf *m = mbuf;
+ while (m) {
+ rte_memcpy(tmp_dst,
+ rte_pktmbuf_mtod(m,
+ const void *),
+ m->data_len);
+ tmp_dst = ((uint8_t *)(tmp_dst)) +
+ m->data_len;
+ m = m->next;
+ }
+ }
+
+
+ dst = ((uint8_t *)dst) + hwpkt_len;
+ unlock_size += hwpkt_len;
+ lock_size -= hwpkt_len;
+
+ rte_pktmbuf_free(mbuf);
+ num_tx++;
+ pkt_left--;
+ if (pkt_left == 0) {
+ szedata_tx_unlock_data(sze_q->sze, lck,
+ unlock_size);
+ break;
+ }
+ goto next_packet;
+ } else if (lock_size + lock_size2 >= hwpkt_len) {
+ void *tmp_dst;
+ uint16_t write_len;