4 * Copyright (c) 2015-2016 Amazon.com, Inc. or its affiliates.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of copyright holder nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
34 #include "ena_eth_com.h"
36 static inline struct ena_eth_io_rx_cdesc_base *ena_com_get_next_rx_cdesc(
37 struct ena_com_io_cq *io_cq)
39 struct ena_eth_io_rx_cdesc_base *cdesc;
40 u16 expected_phase, head_masked;
43 head_masked = io_cq->head & (io_cq->q_depth - 1);
44 expected_phase = io_cq->phase;
46 cdesc = (struct ena_eth_io_rx_cdesc_base *)
47 ((unsigned char *)io_cq->cdesc_addr.virt_addr
48 + (head_masked * io_cq->cdesc_entry_size_in_bytes));
50 desc_phase = (cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK) >>
51 ENA_ETH_IO_RX_CDESC_BASE_PHASE_SHIFT;
53 if (desc_phase != expected_phase)
59 static inline void ena_com_cq_inc_head(struct ena_com_io_cq *io_cq)
63 /* Switch phase bit in case of wrap around */
64 if (unlikely((io_cq->head & (io_cq->q_depth - 1)) == 0))
65 io_cq->phase = 1 - io_cq->phase;
68 static inline void *get_sq_desc(struct ena_com_io_sq *io_sq)
73 tail_masked = io_sq->tail & (io_sq->q_depth - 1);
75 offset = tail_masked * io_sq->desc_entry_size;
77 return (unsigned char *)io_sq->desc_addr.virt_addr + offset;
80 static inline void ena_com_copy_curr_sq_desc_to_dev(struct ena_com_io_sq *io_sq)
82 u16 tail_masked = io_sq->tail & (io_sq->q_depth - 1);
83 u32 offset = tail_masked * io_sq->desc_entry_size;
85 /* In case this queue isn't a LLQ */
86 if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
89 memcpy_toio((unsigned char *)io_sq->desc_addr.pbuf_dev_addr + offset,
90 (unsigned char *)io_sq->desc_addr.virt_addr + offset,
91 io_sq->desc_entry_size);
94 static inline void ena_com_sq_update_tail(struct ena_com_io_sq *io_sq)
98 /* Switch phase bit in case of wrap around */
99 if (unlikely((io_sq->tail & (io_sq->q_depth - 1)) == 0))
100 io_sq->phase = 1 - io_sq->phase;
103 static inline int ena_com_write_header(struct ena_com_io_sq *io_sq,
104 u8 *head_src, u16 header_len)
106 u16 tail_masked = io_sq->tail & (io_sq->q_depth - 1);
107 u8 __iomem *dev_head_addr =
108 io_sq->header_addr + (tail_masked * io_sq->tx_max_header_size);
110 if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
113 ENA_ASSERT(io_sq->header_addr, "header address is NULL\n");
115 memcpy_toio(dev_head_addr, head_src, header_len);
120 static inline struct ena_eth_io_rx_cdesc_base *
121 ena_com_rx_cdesc_idx_to_ptr(struct ena_com_io_cq *io_cq, u16 idx)
123 idx &= (io_cq->q_depth - 1);
124 return (struct ena_eth_io_rx_cdesc_base *)
125 ((unsigned char *)io_cq->cdesc_addr.virt_addr +
126 idx * io_cq->cdesc_entry_size_in_bytes);
129 static inline int ena_com_cdesc_rx_pkt_get(struct ena_com_io_cq *io_cq,
130 u16 *first_cdesc_idx,
133 struct ena_eth_io_rx_cdesc_base *cdesc;
134 u16 count = 0, head_masked;
138 cdesc = ena_com_get_next_rx_cdesc(io_cq);
142 ena_com_cq_inc_head(io_cq);
144 last = (cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK) >>
145 ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT;
149 *first_cdesc_idx = io_cq->cur_rx_pkt_cdesc_start_idx;
150 count += io_cq->cur_rx_pkt_cdesc_count;
152 head_masked = io_cq->head & (io_cq->q_depth - 1);
154 io_cq->cur_rx_pkt_cdesc_count = 0;
155 io_cq->cur_rx_pkt_cdesc_start_idx = head_masked;
157 ena_trc_dbg("ena q_id: %d packets were completed. first desc idx %u descs# %d\n",
158 io_cq->qid, *first_cdesc_idx, count);
160 io_cq->cur_rx_pkt_cdesc_count += count;
168 static inline bool ena_com_meta_desc_changed(struct ena_com_io_sq *io_sq,
169 struct ena_com_tx_ctx *ena_tx_ctx)
173 if (ena_tx_ctx->meta_valid) {
174 rc = memcmp(&io_sq->cached_tx_meta,
175 &ena_tx_ctx->ena_meta,
176 sizeof(struct ena_com_tx_meta));
178 if (unlikely(rc != 0))
185 static inline void ena_com_create_and_store_tx_meta_desc(
186 struct ena_com_io_sq *io_sq,
187 struct ena_com_tx_ctx *ena_tx_ctx)
189 struct ena_eth_io_tx_meta_desc *meta_desc = NULL;
190 struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta;
192 meta_desc = get_sq_desc(io_sq);
193 memset(meta_desc, 0x0, sizeof(struct ena_eth_io_tx_meta_desc));
195 meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_META_DESC_MASK;
197 meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_EXT_VALID_MASK;
199 /* bits 0-9 of the mss */
200 meta_desc->word2 |= (ena_meta->mss <<
201 ENA_ETH_IO_TX_META_DESC_MSS_LO_SHIFT) &
202 ENA_ETH_IO_TX_META_DESC_MSS_LO_MASK;
203 /* bits 10-13 of the mss */
204 meta_desc->len_ctrl |= ((ena_meta->mss >> 10) <<
205 ENA_ETH_IO_TX_META_DESC_MSS_HI_PTP_SHIFT) &
206 ENA_ETH_IO_TX_META_DESC_MSS_HI_PTP_MASK;
208 /* Extended meta desc */
209 meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_ETH_META_TYPE_MASK;
210 meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_META_STORE_MASK;
211 meta_desc->len_ctrl |= (io_sq->phase <<
212 ENA_ETH_IO_TX_META_DESC_PHASE_SHIFT) &
213 ENA_ETH_IO_TX_META_DESC_PHASE_MASK;
215 meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_FIRST_MASK;
216 meta_desc->word2 |= ena_meta->l3_hdr_len &
217 ENA_ETH_IO_TX_META_DESC_L3_HDR_LEN_MASK;
218 meta_desc->word2 |= (ena_meta->l3_hdr_offset <<
219 ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_SHIFT) &
220 ENA_ETH_IO_TX_META_DESC_L3_HDR_OFF_MASK;
222 meta_desc->word2 |= (ena_meta->l4_hdr_len <<
223 ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_SHIFT) &
224 ENA_ETH_IO_TX_META_DESC_L4_HDR_LEN_IN_WORDS_MASK;
226 meta_desc->len_ctrl |= ENA_ETH_IO_TX_META_DESC_META_STORE_MASK;
228 /* Cached the meta desc */
229 memcpy(&io_sq->cached_tx_meta, ena_meta,
230 sizeof(struct ena_com_tx_meta));
232 ena_com_copy_curr_sq_desc_to_dev(io_sq);
233 ena_com_sq_update_tail(io_sq);
236 static inline void ena_com_rx_set_flags(struct ena_com_rx_ctx *ena_rx_ctx,
237 struct ena_eth_io_rx_cdesc_base *cdesc)
239 ena_rx_ctx->l3_proto = (enum ena_eth_io_l3_proto_index)(cdesc->status &
240 ENA_ETH_IO_RX_CDESC_BASE_L3_PROTO_IDX_MASK);
241 ena_rx_ctx->l4_proto = (enum ena_eth_io_l4_proto_index)
242 ((cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_MASK) >>
243 ENA_ETH_IO_RX_CDESC_BASE_L4_PROTO_IDX_SHIFT);
244 ena_rx_ctx->l3_csum_err =
245 (cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_MASK) >>
246 ENA_ETH_IO_RX_CDESC_BASE_L3_CSUM_ERR_SHIFT;
247 ena_rx_ctx->l4_csum_err =
248 (cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_MASK) >>
249 ENA_ETH_IO_RX_CDESC_BASE_L4_CSUM_ERR_SHIFT;
250 ena_rx_ctx->hash = cdesc->hash;
252 (cdesc->status & ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_MASK) >>
253 ENA_ETH_IO_RX_CDESC_BASE_IPV4_FRAG_SHIFT;
255 ena_trc_dbg("ena_rx_ctx->l3_proto %d ena_rx_ctx->l4_proto %d\nena_rx_ctx->l3_csum_err %d ena_rx_ctx->l4_csum_err %d\nhash frag %d frag: %d cdesc_status: %x\n",
256 ena_rx_ctx->l3_proto,
257 ena_rx_ctx->l4_proto,
258 ena_rx_ctx->l3_csum_err,
259 ena_rx_ctx->l4_csum_err,
265 /*****************************************************************************/
266 /***************************** API **********************************/
267 /*****************************************************************************/
269 int ena_com_prepare_tx(struct ena_com_io_sq *io_sq,
270 struct ena_com_tx_ctx *ena_tx_ctx,
273 struct ena_eth_io_tx_desc *desc = NULL;
274 struct ena_com_buf *ena_bufs = ena_tx_ctx->ena_bufs;
275 void *push_header = ena_tx_ctx->push_header;
276 u16 header_len = ena_tx_ctx->header_len;
277 u16 num_bufs = ena_tx_ctx->num_bufs;
278 int total_desc, i, rc;
282 ENA_ASSERT(io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_TX,
285 /* num_bufs +1 for potential meta desc */
286 if (ena_com_sq_empty_space(io_sq) < (num_bufs + 1)) {
287 ena_trc_err("Not enough space in the tx queue\n");
288 return ENA_COM_NO_MEM;
291 if (unlikely(header_len > io_sq->tx_max_header_size)) {
292 ena_trc_err("header size is too large %d max header: %d\n",
293 header_len, io_sq->tx_max_header_size);
294 return ENA_COM_INVAL;
297 /* start with pushing the header (if needed) */
298 rc = ena_com_write_header(io_sq, push_header, header_len);
302 have_meta = ena_tx_ctx->meta_valid && ena_com_meta_desc_changed(io_sq,
305 ena_com_create_and_store_tx_meta_desc(io_sq, ena_tx_ctx);
307 /* If the caller doesn't want send packets */
308 if (unlikely(!num_bufs && !header_len)) {
309 *nb_hw_desc = have_meta ? 0 : 1;
313 desc = get_sq_desc(io_sq);
314 memset(desc, 0x0, sizeof(struct ena_eth_io_tx_desc));
316 /* Set first desc when we don't have meta descriptor */
318 desc->len_ctrl |= ENA_ETH_IO_TX_DESC_FIRST_MASK;
320 desc->buff_addr_hi_hdr_sz |= (header_len <<
321 ENA_ETH_IO_TX_DESC_HEADER_LENGTH_SHIFT) &
322 ENA_ETH_IO_TX_DESC_HEADER_LENGTH_MASK;
323 desc->len_ctrl |= (io_sq->phase << ENA_ETH_IO_TX_DESC_PHASE_SHIFT) &
324 ENA_ETH_IO_TX_DESC_PHASE_MASK;
326 desc->len_ctrl |= ENA_ETH_IO_TX_DESC_COMP_REQ_MASK;
329 desc->meta_ctrl |= (ena_tx_ctx->req_id <<
330 ENA_ETH_IO_TX_DESC_REQ_ID_LO_SHIFT) &
331 ENA_ETH_IO_TX_DESC_REQ_ID_LO_MASK;
333 desc->meta_ctrl |= (ena_tx_ctx->df <<
334 ENA_ETH_IO_TX_DESC_DF_SHIFT) &
335 ENA_ETH_IO_TX_DESC_DF_MASK;
338 desc->len_ctrl |= ((ena_tx_ctx->req_id >> 10) <<
339 ENA_ETH_IO_TX_DESC_REQ_ID_HI_SHIFT) &
340 ENA_ETH_IO_TX_DESC_REQ_ID_HI_MASK;
342 if (ena_tx_ctx->meta_valid) {
343 desc->meta_ctrl |= (ena_tx_ctx->tso_enable <<
344 ENA_ETH_IO_TX_DESC_TSO_EN_SHIFT) &
345 ENA_ETH_IO_TX_DESC_TSO_EN_MASK;
346 desc->meta_ctrl |= ena_tx_ctx->l3_proto &
347 ENA_ETH_IO_TX_DESC_L3_PROTO_IDX_MASK;
348 desc->meta_ctrl |= (ena_tx_ctx->l4_proto <<
349 ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_SHIFT) &
350 ENA_ETH_IO_TX_DESC_L4_PROTO_IDX_MASK;
351 desc->meta_ctrl |= (ena_tx_ctx->l3_csum_enable <<
352 ENA_ETH_IO_TX_DESC_L3_CSUM_EN_SHIFT) &
353 ENA_ETH_IO_TX_DESC_L3_CSUM_EN_MASK;
354 desc->meta_ctrl |= (ena_tx_ctx->l4_csum_enable <<
355 ENA_ETH_IO_TX_DESC_L4_CSUM_EN_SHIFT) &
356 ENA_ETH_IO_TX_DESC_L4_CSUM_EN_MASK;
357 desc->meta_ctrl |= (ena_tx_ctx->l4_csum_partial <<
358 ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_SHIFT) &
359 ENA_ETH_IO_TX_DESC_L4_CSUM_PARTIAL_MASK;
362 for (i = 0; i < num_bufs; i++) {
363 /* The first desc share the same desc as the header */
364 if (likely(i != 0)) {
365 ena_com_copy_curr_sq_desc_to_dev(io_sq);
366 ena_com_sq_update_tail(io_sq);
368 desc = get_sq_desc(io_sq);
369 memset(desc, 0x0, sizeof(struct ena_eth_io_tx_desc));
371 desc->len_ctrl |= (io_sq->phase <<
372 ENA_ETH_IO_TX_DESC_PHASE_SHIFT) &
373 ENA_ETH_IO_TX_DESC_PHASE_MASK;
376 desc->len_ctrl |= ena_bufs->len &
377 ENA_ETH_IO_TX_DESC_LENGTH_MASK;
379 addr_hi = ((ena_bufs->paddr &
380 GENMASK_ULL(io_sq->dma_addr_bits - 1, 32)) >> 32);
382 desc->buff_addr_lo = (u32)ena_bufs->paddr;
383 desc->buff_addr_hi_hdr_sz |= addr_hi &
384 ENA_ETH_IO_TX_DESC_ADDR_HI_MASK;
388 /* set the last desc indicator */
389 desc->len_ctrl |= ENA_ETH_IO_TX_DESC_LAST_MASK;
391 ena_com_copy_curr_sq_desc_to_dev(io_sq);
393 ena_com_sq_update_tail(io_sq);
395 total_desc = ENA_MAX16(num_bufs, 1);
396 total_desc += have_meta ? 1 : 0;
398 *nb_hw_desc = total_desc;
402 int ena_com_rx_pkt(struct ena_com_io_cq *io_cq,
403 struct ena_com_io_sq *io_sq,
404 struct ena_com_rx_ctx *ena_rx_ctx)
406 struct ena_com_rx_buf_info *ena_buf = &ena_rx_ctx->ena_bufs[0];
407 struct ena_eth_io_rx_cdesc_base *cdesc = NULL;
413 ENA_ASSERT(io_cq->direction == ENA_COM_IO_QUEUE_DIRECTION_RX,
416 rc = ena_com_cdesc_rx_pkt_get(io_cq, &cdesc_idx, &nb_hw_desc);
417 if (rc || (nb_hw_desc == 0)) {
418 ena_rx_ctx->descs = nb_hw_desc;
422 ena_trc_dbg("fetch rx packet: queue %d completed desc: %d\n",
423 io_cq->qid, nb_hw_desc);
425 if (unlikely(nb_hw_desc >= ena_rx_ctx->max_bufs)) {
426 ena_trc_err("Too many RX cdescs (%d) > MAX(%d)\n",
427 nb_hw_desc, ena_rx_ctx->max_bufs);
428 return ENA_COM_NO_SPACE;
431 for (i = 0; i < nb_hw_desc; i++) {
432 cdesc = ena_com_rx_cdesc_idx_to_ptr(io_cq, cdesc_idx + i);
434 ena_buf->len = cdesc->length;
435 ena_buf->req_id = cdesc->req_id;
439 /* Update SQ head ptr */
440 io_sq->next_to_comp += nb_hw_desc;
442 ena_trc_dbg("[%s][QID#%d] Updating SQ head to: %d\n", __func__,
443 io_sq->qid, io_sq->next_to_comp);
445 /* Get rx flags from the last pkt */
446 ena_com_rx_set_flags(ena_rx_ctx, cdesc);
448 ena_rx_ctx->descs = nb_hw_desc;
452 int ena_com_add_single_rx_desc(struct ena_com_io_sq *io_sq,
453 struct ena_com_buf *ena_buf,
456 struct ena_eth_io_rx_desc *desc;
458 ENA_ASSERT(io_sq->direction == ENA_COM_IO_QUEUE_DIRECTION_RX,
461 if (unlikely(ena_com_sq_empty_space(io_sq) == 0))
464 desc = get_sq_desc(io_sq);
465 memset(desc, 0x0, sizeof(struct ena_eth_io_rx_desc));
467 desc->length = ena_buf->len;
469 desc->ctrl |= ENA_ETH_IO_RX_DESC_FIRST_MASK;
470 desc->ctrl |= ENA_ETH_IO_RX_DESC_LAST_MASK;
471 desc->ctrl |= io_sq->phase & ENA_ETH_IO_RX_DESC_PHASE_MASK;
472 desc->ctrl |= ENA_ETH_IO_RX_DESC_COMP_REQ_MASK;
474 desc->req_id = req_id;
476 desc->buff_addr_lo = (u32)ena_buf->paddr;
479 GENMASK_ULL(io_sq->dma_addr_bits - 1, 32)) >> 32);
481 ena_com_sq_update_tail(io_sq);
486 int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq, u16 *req_id)
488 u8 expected_phase, cdesc_phase;
489 struct ena_eth_io_tx_cdesc *cdesc;
492 masked_head = io_cq->head & (io_cq->q_depth - 1);
493 expected_phase = io_cq->phase;
495 cdesc = (struct ena_eth_io_tx_cdesc *)
496 ((unsigned char *)io_cq->cdesc_addr.virt_addr
497 + (masked_head * io_cq->cdesc_entry_size_in_bytes));
499 cdesc_phase = cdesc->flags & ENA_ETH_IO_TX_CDESC_PHASE_MASK;
500 if (cdesc_phase != expected_phase)
503 ena_com_cq_inc_head(io_cq);
505 *req_id = cdesc->req_id;