4 * Copyright (c) 2016-2017 Solarflare Communications Inc.
7 * This software was jointly developed between OKTET Labs (under contract
8 * for Solarflare) and Solarflare Communications, Inc.
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are met:
13 * 1. Redistributions of source code must retain the above copyright notice,
14 * this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright notice,
16 * this list of conditions and the following disclaimer in the documentation
17 * and/or other materials provided with the distribution.
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 #include "sfc_debug.h"
40 /** Standard TSO header length */
41 #define SFC_TSOH_STD_LEN 256
43 /** The number of TSO option descriptors that precede the packet descriptors */
44 #define SFC_TSO_OPDESCS_IDX_SHIFT 2
47 sfc_tso_alloc_tsoh_objs(struct sfc_tx_sw_desc *sw_ring,
48 unsigned int txq_entries, unsigned int socket_id)
52 for (i = 0; i < txq_entries; ++i) {
53 sw_ring[i].tsoh = rte_malloc_socket("sfc-txq-tsoh-obj",
57 if (sw_ring[i].tsoh == NULL)
58 goto fail_alloc_tsoh_objs;
65 rte_free(sw_ring[--i].tsoh);
71 sfc_tso_free_tsoh_objs(struct sfc_tx_sw_desc *sw_ring, unsigned int txq_entries)
75 for (i = 0; i < txq_entries; ++i) {
76 rte_free(sw_ring[i].tsoh);
77 sw_ring[i].tsoh = NULL;
82 sfc_tso_prepare_header(struct sfc_txq *txq, struct rte_mbuf **in_seg,
83 size_t *in_off, unsigned int idx, size_t bytes_left)
85 struct rte_mbuf *m = *in_seg;
86 size_t bytes_to_copy = 0;
87 uint8_t *tsoh = txq->sw_ring[idx & txq->ptr_mask].tsoh;
90 bytes_to_copy = MIN(bytes_left, m->data_len);
92 rte_memcpy(tsoh, rte_pktmbuf_mtod(m, uint8_t *),
95 bytes_left -= bytes_to_copy;
96 tsoh += bytes_to_copy;
100 SFC_ASSERT(m != NULL);
102 } while (bytes_left > 0);
104 if (bytes_to_copy == m->data_len) {
109 *in_off = bytes_to_copy;
114 sfc_tso_do(struct sfc_txq *txq, unsigned int idx, struct rte_mbuf **in_seg,
115 size_t *in_off, efx_desc_t **pend, unsigned int *pkt_descs,
119 const struct tcp_hdr *th;
120 efsys_dma_addr_t header_paddr;
123 struct rte_mbuf *m = *in_seg;
124 size_t nh_off = m->l2_len; /* IP header offset */
125 size_t tcph_off = m->l2_len + m->l3_len; /* TCP header offset */
126 size_t header_len = m->l2_len + m->l3_len + m->l4_len;
127 const efx_nic_cfg_t *encp = efx_nic_cfg_get(txq->evq->sa->nic);
129 idx += SFC_TSO_OPDESCS_IDX_SHIFT;
131 /* Packets which have too big headers should be discarded */
132 if (unlikely(header_len > SFC_TSOH_STD_LEN))
136 * The TCP header must start at most 208 bytes into the frame.
137 * If it starts later than this then the NIC won't realise
138 * it's a TCP packet and TSO edits won't be applied
140 if (unlikely(tcph_off > encp->enc_tx_tso_tcp_header_offset_limit))
143 header_paddr = rte_pktmbuf_mtophys(m);
146 * Sometimes headers may be split across multiple mbufs. In such cases
147 * we need to glue those pieces and store them in some temporary place.
148 * Also, packet headers must be contiguous in memory, so that
149 * they can be referred to with a single DMA descriptor. EF10 has no
150 * limitations on address boundaries crossing by DMA descriptor data.
152 if (m->data_len < header_len) {
153 sfc_tso_prepare_header(txq, in_seg, in_off, idx, header_len);
154 tsoh = txq->sw_ring[idx & txq->ptr_mask].tsoh;
156 header_paddr = rte_malloc_virt2phy((void *)tsoh);
158 if (m->data_len == header_len) {
162 *in_off = header_len;
165 tsoh = rte_pktmbuf_mtod(m, uint8_t *);
168 /* Handle IP header */
169 if (m->ol_flags & PKT_TX_IPV4) {
170 const struct ipv4_hdr *iphe4;
172 iphe4 = (const struct ipv4_hdr *)(tsoh + nh_off);
173 rte_memcpy(&packet_id, &iphe4->packet_id, sizeof(uint16_t));
174 packet_id = rte_be_to_cpu_16(packet_id);
175 } else if (m->ol_flags & PKT_TX_IPV6) {
181 /* Handle TCP header */
182 th = (const struct tcp_hdr *)(tsoh + tcph_off);
184 rte_memcpy(&sent_seq, &th->sent_seq, sizeof(uint32_t));
185 sent_seq = rte_be_to_cpu_32(sent_seq);
187 efx_tx_qdesc_tso2_create(txq->common, packet_id, sent_seq, m->tso_segsz,
188 *pend, EFX_TX_FATSOV2_OPT_NDESCS);
190 *pend += EFX_TX_FATSOV2_OPT_NDESCS;
191 *pkt_descs += EFX_TX_FATSOV2_OPT_NDESCS;
193 efx_tx_qdesc_dma_create(txq->common, header_paddr, header_len,
196 *pkt_len -= header_len;