X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fena%2Fena_ethdev.c;h=807751973543bf624cb1ed019eed6eb43b1b7182;hb=4ad61424aaab783002947e4348afcd16ecb58327;hp=7804a5c85dbb2c686df0db84cb49fbd8ee224451;hpb=7755060798de9c0146df8809926030ed47fc9086;p=dpdk.git diff --git a/drivers/net/ena/ena_ethdev.c b/drivers/net/ena/ena_ethdev.c index 7804a5c85d..8077519735 100644 --- a/drivers/net/ena/ena_ethdev.c +++ b/drivers/net/ena/ena_ethdev.c @@ -27,8 +27,8 @@ #include #define DRV_MODULE_VER_MAJOR 2 -#define DRV_MODULE_VER_MINOR 0 -#define DRV_MODULE_VER_SUBMINOR 3 +#define DRV_MODULE_VER_MINOR 1 +#define DRV_MODULE_VER_SUBMINOR 0 #define ENA_IO_TXQ_IDX(q) (2 * (q)) #define ENA_IO_RXQ_IDX(q) (2 * (q) + 1) @@ -141,22 +141,6 @@ static const struct ena_stats ena_stats_rx_strings[] = { #define ENA_TX_OFFLOAD_NOTSUP_MASK \ (PKT_TX_OFFLOAD_MASK ^ ENA_TX_OFFLOAD_MASK) -int ena_logtype_init; -int ena_logtype_driver; - -#ifdef RTE_LIBRTE_ENA_DEBUG_RX -int ena_logtype_rx; -#endif -#ifdef RTE_LIBRTE_ENA_DEBUG_TX -int ena_logtype_tx; -#endif -#ifdef RTE_LIBRTE_ENA_DEBUG_TX_FREE -int ena_logtype_tx_free; -#endif -#ifdef RTE_LIBRTE_ENA_COM_DEBUG -int ena_logtype_com; -#endif - static const struct rte_pci_id pci_id_ena_map[] = { { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_VF) }, { RTE_PCI_DEVICE(PCI_VENDOR_ID_AMAZON, PCI_DEVICE_ID_ENA_LLQ_VF) }, @@ -169,6 +153,13 @@ static int ena_device_init(struct ena_com_dev *ena_dev, struct ena_com_dev_get_features_ctx *get_feat_ctx, bool *wd_state); static int ena_dev_configure(struct rte_eth_dev *dev); +static void ena_tx_map_mbuf(struct ena_ring *tx_ring, + struct ena_tx_buffer *tx_info, + struct rte_mbuf *mbuf, + void **push_header, + uint16_t *header_len); +static int ena_xmit_mbuf(struct ena_ring *tx_ring, struct rte_mbuf *mbuf); +static void ena_tx_cleanup(struct ena_ring *tx_ring); static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts); static uint16_t eth_ena_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, @@ -188,6 +179,8 @@ static struct rte_mbuf *ena_rx_mbuf(struct ena_ring *rx_ring, uint8_t offset); static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts); +static int ena_add_single_rx_desc(struct ena_com_io_sq *io_sq, + struct rte_mbuf *mbuf, uint16_t id); static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count); static void ena_init_rings(struct ena_adapter *adapter, bool disable_meta_caching); @@ -1266,6 +1259,7 @@ static int ena_tx_queue_setup(struct rte_eth_dev *dev, txq->next_to_clean = 0; txq->next_to_use = 0; txq->ring_size = nb_desc; + txq->size_mask = nb_desc - 1; txq->numa_socket_id = socket_id; txq->tx_buffer_info = rte_zmalloc("txq->tx_buffer_info", @@ -1361,6 +1355,7 @@ static int ena_rx_queue_setup(struct rte_eth_dev *dev, rxq->next_to_clean = 0; rxq->next_to_use = 0; rxq->ring_size = nb_desc; + rxq->size_mask = nb_desc - 1; rxq->numa_socket_id = socket_id; rxq->mb_pool = mp; @@ -1405,12 +1400,28 @@ static int ena_rx_queue_setup(struct rte_eth_dev *dev, return 0; } +static int ena_add_single_rx_desc(struct ena_com_io_sq *io_sq, + struct rte_mbuf *mbuf, uint16_t id) +{ + struct ena_com_buf ebuf; + int rc; + + /* prepare physical address for DMA transaction */ + ebuf.paddr = mbuf->buf_iova + RTE_PKTMBUF_HEADROOM; + ebuf.len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM; + + /* pass resource to device */ + rc = ena_com_add_single_rx_desc(io_sq, &ebuf, id); + if (unlikely(rc != 0)) + PMD_DRV_LOG(WARNING, "failed adding rx desc\n"); + + return rc; +} + static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count) { unsigned int i; int rc; - uint16_t ring_size = rxq->ring_size; - uint16_t ring_mask = ring_size - 1; uint16_t next_to_use = rxq->next_to_use; uint16_t in_use, req_id; struct rte_mbuf **mbufs = rxq->rx_refill_buffer; @@ -1418,9 +1429,10 @@ static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count) if (unlikely(!count)) return 0; - in_use = ring_size - ena_com_free_q_entries(rxq->ena_com_io_sq) - 1; - - ena_assert_msg(((in_use + count) < ring_size), "bad ring state\n"); + in_use = rxq->ring_size - 1 - + ena_com_free_q_entries(rxq->ena_com_io_sq); + ena_assert_msg(((in_use + count) < rxq->ring_size), + "bad ring state\n"); /* get resources for incoming packets */ rc = rte_mempool_get_bulk(rxq->mb_pool, (void **)mbufs, count); @@ -1432,33 +1444,25 @@ static int ena_populate_rx_queue(struct ena_ring *rxq, unsigned int count) } for (i = 0; i < count; i++) { - uint16_t next_to_use_masked = next_to_use & ring_mask; struct rte_mbuf *mbuf = mbufs[i]; - struct ena_com_buf ebuf; struct ena_rx_buffer *rx_info; if (likely((i + 4) < count)) rte_prefetch0(mbufs[i + 4]); - req_id = rxq->empty_rx_reqs[next_to_use_masked]; + req_id = rxq->empty_rx_reqs[next_to_use]; rc = validate_rx_req_id(rxq, req_id); if (unlikely(rc)) break; rx_info = &rxq->rx_buffer_info[req_id]; - /* prepare physical address for DMA transaction */ - ebuf.paddr = mbuf->buf_iova + RTE_PKTMBUF_HEADROOM; - ebuf.len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM; - /* pass resource to device */ - rc = ena_com_add_single_rx_desc(rxq->ena_com_io_sq, - &ebuf, req_id); - if (unlikely(rc)) { - PMD_DRV_LOG(WARNING, "failed adding rx desc\n"); + rc = ena_add_single_rx_desc(rxq->ena_com_io_sq, mbuf, req_id); + if (unlikely(rc != 0)) break; - } + rx_info->mbuf = mbuf; - next_to_use++; + next_to_use = ENA_IDX_NEXT_MASKED(next_to_use, rxq->size_mask); } if (unlikely(i < count)) { @@ -1617,7 +1621,7 @@ static void ena_timer_wd_callback(__rte_unused struct rte_timer *timer, if (unlikely(adapter->trigger_reset)) { PMD_DRV_LOG(ERR, "Trigger reset is on\n"); - _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, + rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_RESET, NULL); } } @@ -1736,7 +1740,7 @@ static int eth_ena_dev_init(struct rte_eth_dev *eth_dev) int rc; static int adapters_found; bool disable_meta_caching; - bool wd_state; + bool wd_state = false; eth_dev->dev_ops = &ena_dev_ops; eth_dev->rx_pkt_burst = ð_ena_recv_pkts; @@ -2072,7 +2076,7 @@ static struct rte_mbuf *ena_rx_mbuf(struct ena_ring *rx_ring, struct rte_mbuf *mbuf; struct rte_mbuf *mbuf_head; struct ena_rx_buffer *rx_info; - unsigned int ring_mask = rx_ring->ring_size - 1; + int rc; uint16_t ntc, len, req_id, buf = 0; if (unlikely(descs == 0)) @@ -2100,8 +2104,8 @@ static struct rte_mbuf *ena_rx_mbuf(struct ena_ring *rx_ring, mbuf_head->data_off += offset; rx_info->mbuf = NULL; - rx_ring->empty_rx_reqs[ntc & ring_mask] = req_id; - ++ntc; + rx_ring->empty_rx_reqs[ntc] = req_id; + ntc = ENA_IDX_NEXT_MASKED(ntc, rx_ring->size_mask); while (--descs) { ++buf; @@ -2115,16 +2119,47 @@ static struct rte_mbuf *ena_rx_mbuf(struct ena_ring *rx_ring, rx_info = &rx_ring->rx_buffer_info[req_id]; RTE_ASSERT(rx_info->mbuf != NULL); - /* Create an mbuf chain. */ - mbuf->next = rx_info->mbuf; - mbuf = mbuf->next; + if (unlikely(len == 0)) { + /* + * Some devices can pass descriptor with the length 0. + * To avoid confusion, the PMD is simply putting the + * descriptor back, as it was never used. We'll avoid + * mbuf allocation that way. + */ + rc = ena_add_single_rx_desc(rx_ring->ena_com_io_sq, + rx_info->mbuf, req_id); + if (unlikely(rc != 0)) { + /* Free the mbuf in case of an error. */ + rte_mbuf_raw_free(rx_info->mbuf); + } else { + /* + * If there was no error, just exit the loop as + * 0 length descriptor is always the last one. + */ + break; + } + } else { + /* Create an mbuf chain. */ + mbuf->next = rx_info->mbuf; + mbuf = mbuf->next; - ena_init_rx_mbuf(mbuf, len); - mbuf_head->pkt_len += len; + ena_init_rx_mbuf(mbuf, len); + mbuf_head->pkt_len += len; + } + /* + * Mark the descriptor as depleted and perform necessary + * cleanup. + * This code will execute in two cases: + * 1. Descriptor len was greater than 0 - normal situation. + * 2. Descriptor len was 0 and we failed to add the descriptor + * to the device. In that situation, we should try to add + * the mbuf again in the populate routine and mark the + * descriptor as used up by the device. + */ rx_info->mbuf = NULL; - rx_ring->empty_rx_reqs[ntc & ring_mask] = req_id; - ++ntc; + rx_ring->empty_rx_reqs[ntc] = req_id; + ntc = ENA_IDX_NEXT_MASKED(ntc, rx_ring->size_mask); } *next_to_clean = ntc; @@ -2136,8 +2171,6 @@ static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts) { struct ena_ring *rx_ring = (struct ena_ring *)(rx_queue); - unsigned int ring_size = rx_ring->ring_size; - unsigned int ring_mask = ring_size - 1; unsigned int free_queue_entries; unsigned int refill_threshold; uint16_t next_to_clean = rx_ring->next_to_clean; @@ -2154,7 +2187,7 @@ static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, return 0; } - descs_in_use = ring_size - + descs_in_use = rx_ring->ring_size - ena_com_free_q_entries(rx_ring->ena_com_io_sq) - 1; nb_pkts = RTE_MIN(descs_in_use, nb_pkts); @@ -2183,9 +2216,10 @@ static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, ena_rx_ctx.pkt_offset); if (unlikely(mbuf == NULL)) { for (i = 0; i < ena_rx_ctx.descs; ++i) { - rx_ring->empty_rx_reqs[next_to_clean & ring_mask] = + rx_ring->empty_rx_reqs[next_to_clean] = rx_ring->ena_bufs[i].req_id; - ++next_to_clean; + next_to_clean = ENA_IDX_NEXT_MASKED( + next_to_clean, rx_ring->size_mask); } break; } @@ -2210,7 +2244,7 @@ static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, free_queue_entries = ena_com_free_q_entries(rx_ring->ena_com_io_sq); refill_threshold = - RTE_MIN(ring_size / ENA_REFILL_THRESH_DIVIDER, + RTE_MIN(rx_ring->ring_size / ENA_REFILL_THRESH_DIVIDER, (unsigned int)ENA_REFILL_THRESH_PACKET); /* Burst refill to save doorbells, memory barriers, const interval */ @@ -2345,192 +2379,221 @@ static int ena_check_and_linearize_mbuf(struct ena_ring *tx_ring, return rc; } -static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, - uint16_t nb_pkts) +static void ena_tx_map_mbuf(struct ena_ring *tx_ring, + struct ena_tx_buffer *tx_info, + struct rte_mbuf *mbuf, + void **push_header, + uint16_t *header_len) { - struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); - uint16_t next_to_use = tx_ring->next_to_use; - uint16_t next_to_clean = tx_ring->next_to_clean; - struct rte_mbuf *mbuf; - uint16_t seg_len; - unsigned int ring_size = tx_ring->ring_size; - unsigned int ring_mask = ring_size - 1; - unsigned int cleanup_budget; - struct ena_com_tx_ctx ena_tx_ctx; - struct ena_tx_buffer *tx_info; - struct ena_com_buf *ebuf; - uint16_t rc, req_id, total_tx_descs = 0; - uint16_t sent_idx = 0; - uint16_t push_len = 0; - uint16_t delta = 0; - int nb_hw_desc; - uint32_t total_length; + struct ena_com_buf *ena_buf; + uint16_t delta, seg_len, push_len; - /* Check adapter state */ - if (unlikely(tx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { - PMD_DRV_LOG(ALERT, - "Trying to xmit pkts while device is NOT running\n"); - return 0; - } + delta = 0; + seg_len = mbuf->data_len; - nb_pkts = RTE_MIN(ena_com_free_q_entries(tx_ring->ena_com_io_sq), - nb_pkts); + tx_info->mbuf = mbuf; + ena_buf = tx_info->bufs; - for (sent_idx = 0; sent_idx < nb_pkts; sent_idx++) { - mbuf = tx_pkts[sent_idx]; - total_length = 0; + if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { + /* + * Tx header might be (and will be in most cases) smaller than + * tx_max_header_size. But it's not an issue to send more data + * to the device, than actually needed if the mbuf size is + * greater than tx_max_header_size. + */ + push_len = RTE_MIN(mbuf->pkt_len, tx_ring->tx_max_header_size); + *header_len = push_len; - rc = ena_check_and_linearize_mbuf(tx_ring, mbuf); - if (unlikely(rc)) - break; + if (likely(push_len <= seg_len)) { + /* If the push header is in the single segment, then + * just point it to the 1st mbuf data. + */ + *push_header = rte_pktmbuf_mtod(mbuf, uint8_t *); + } else { + /* If the push header lays in the several segments, copy + * it to the intermediate buffer. + */ + rte_pktmbuf_read(mbuf, 0, push_len, + tx_ring->push_buf_intermediate_buf); + *push_header = tx_ring->push_buf_intermediate_buf; + delta = push_len - seg_len; + } + } else { + *push_header = NULL; + *header_len = 0; + push_len = 0; + } - req_id = tx_ring->empty_tx_reqs[next_to_use & ring_mask]; - tx_info = &tx_ring->tx_buffer_info[req_id]; - tx_info->mbuf = mbuf; - tx_info->num_of_bufs = 0; - ebuf = tx_info->bufs; + /* Process first segment taking into consideration pushed header */ + if (seg_len > push_len) { + ena_buf->paddr = mbuf->buf_iova + + mbuf->data_off + + push_len; + ena_buf->len = seg_len - push_len; + ena_buf++; + tx_info->num_of_bufs++; + } + + while ((mbuf = mbuf->next) != NULL) { + seg_len = mbuf->data_len; - /* Prepare TX context */ - memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx)); - memset(&ena_tx_ctx.ena_meta, 0x0, - sizeof(struct ena_com_tx_meta)); - ena_tx_ctx.ena_bufs = ebuf; - ena_tx_ctx.req_id = req_id; + /* Skip mbufs if whole data is pushed as a header */ + if (unlikely(delta > seg_len)) { + delta -= seg_len; + continue; + } + + ena_buf->paddr = mbuf->buf_iova + mbuf->data_off + delta; + ena_buf->len = seg_len - delta; + ena_buf++; + tx_info->num_of_bufs++; delta = 0; - seg_len = mbuf->data_len; + } +} - if (tx_ring->tx_mem_queue_type == - ENA_ADMIN_PLACEMENT_POLICY_DEV) { - push_len = RTE_MIN(mbuf->pkt_len, - tx_ring->tx_max_header_size); - ena_tx_ctx.header_len = push_len; +static int ena_xmit_mbuf(struct ena_ring *tx_ring, struct rte_mbuf *mbuf) +{ + struct ena_tx_buffer *tx_info; + struct ena_com_tx_ctx ena_tx_ctx = { { 0 } }; + uint16_t next_to_use; + uint16_t header_len; + uint16_t req_id; + void *push_header; + int nb_hw_desc; + int rc; - if (likely(push_len <= seg_len)) { - /* If the push header is in the single segment, - * then just point it to the 1st mbuf data. - */ - ena_tx_ctx.push_header = - rte_pktmbuf_mtod(mbuf, uint8_t *); - } else { - /* If the push header lays in the several - * segments, copy it to the intermediate buffer. - */ - rte_pktmbuf_read(mbuf, 0, push_len, - tx_ring->push_buf_intermediate_buf); - ena_tx_ctx.push_header = - tx_ring->push_buf_intermediate_buf; - delta = push_len - seg_len; - } - } /* there's no else as we take advantage of memset zeroing */ + rc = ena_check_and_linearize_mbuf(tx_ring, mbuf); + if (unlikely(rc)) + return rc; - /* Set TX offloads flags, if applicable */ - ena_tx_mbuf_prepare(mbuf, &ena_tx_ctx, tx_ring->offloads, - tx_ring->disable_meta_caching); + next_to_use = tx_ring->next_to_use; - rte_prefetch0(tx_pkts[(sent_idx + 4) & ring_mask]); + req_id = tx_ring->empty_tx_reqs[next_to_use]; + tx_info = &tx_ring->tx_buffer_info[req_id]; + tx_info->num_of_bufs = 0; - /* Process first segment taking into - * consideration pushed header - */ - if (seg_len > push_len) { - ebuf->paddr = mbuf->buf_iova + - mbuf->data_off + - push_len; - ebuf->len = seg_len - push_len; - ebuf++; - tx_info->num_of_bufs++; - } - total_length += mbuf->data_len; + ena_tx_map_mbuf(tx_ring, tx_info, mbuf, &push_header, &header_len); - while ((mbuf = mbuf->next) != NULL) { - seg_len = mbuf->data_len; + ena_tx_ctx.ena_bufs = tx_info->bufs; + ena_tx_ctx.push_header = push_header; + ena_tx_ctx.num_bufs = tx_info->num_of_bufs; + ena_tx_ctx.req_id = req_id; + ena_tx_ctx.header_len = header_len; - /* Skip mbufs if whole data is pushed as a header */ - if (unlikely(delta > seg_len)) { - delta -= seg_len; - continue; - } + /* Set Tx offloads flags, if applicable */ + ena_tx_mbuf_prepare(mbuf, &ena_tx_ctx, tx_ring->offloads, + tx_ring->disable_meta_caching); - ebuf->paddr = mbuf->buf_iova + mbuf->data_off + delta; - ebuf->len = seg_len - delta; - total_length += ebuf->len; - ebuf++; - tx_info->num_of_bufs++; + if (unlikely(ena_com_is_doorbell_needed(tx_ring->ena_com_io_sq, + &ena_tx_ctx))) { + PMD_DRV_LOG(DEBUG, + "llq tx max burst size of queue %d achieved, writing doorbell to send burst\n", + tx_ring->id); + ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); + } - delta = 0; - } + /* prepare the packet's descriptors to dma engine */ + rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq, &ena_tx_ctx, + &nb_hw_desc); + if (unlikely(rc)) { + ++tx_ring->tx_stats.prepare_ctx_err; + return rc; + } - ena_tx_ctx.num_bufs = tx_info->num_of_bufs; + tx_info->tx_descs = nb_hw_desc; - if (ena_com_is_doorbell_needed(tx_ring->ena_com_io_sq, - &ena_tx_ctx)) { - PMD_DRV_LOG(DEBUG, "llq tx max burst size of queue %d" - " achieved, writing doorbell to send burst\n", - tx_ring->id); - ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); - } + tx_ring->tx_stats.cnt++; + tx_ring->tx_stats.bytes += mbuf->pkt_len; - /* prepare the packet's descriptors to dma engine */ - rc = ena_com_prepare_tx(tx_ring->ena_com_io_sq, - &ena_tx_ctx, &nb_hw_desc); - if (unlikely(rc)) { - ++tx_ring->tx_stats.prepare_ctx_err; - break; - } - tx_info->tx_descs = nb_hw_desc; + tx_ring->next_to_use = ENA_IDX_NEXT_MASKED(next_to_use, + tx_ring->size_mask); - next_to_use++; - tx_ring->tx_stats.cnt++; - tx_ring->tx_stats.bytes += total_length; - } - tx_ring->tx_stats.available_desc = - ena_com_free_q_entries(tx_ring->ena_com_io_sq); + return 0; +} - /* If there are ready packets to be xmitted... */ - if (sent_idx > 0) { - /* ...let HW do its best :-) */ - ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); - tx_ring->tx_stats.doorbells++; - tx_ring->next_to_use = next_to_use; - } +static void ena_tx_cleanup(struct ena_ring *tx_ring) +{ + unsigned int cleanup_budget; + unsigned int total_tx_descs = 0; + uint16_t next_to_clean = tx_ring->next_to_clean; - /* Clear complete packets */ - while (ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, &req_id) >= 0) { - rc = validate_tx_req_id(tx_ring, req_id); - if (rc) + cleanup_budget = RTE_MIN(tx_ring->ring_size / ENA_REFILL_THRESH_DIVIDER, + (unsigned int)ENA_REFILL_THRESH_PACKET); + + while (likely(total_tx_descs < cleanup_budget)) { + struct rte_mbuf *mbuf; + struct ena_tx_buffer *tx_info; + uint16_t req_id; + + if (ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq, &req_id) != 0) + break; + + if (unlikely(validate_tx_req_id(tx_ring, req_id) != 0)) break; /* Get Tx info & store how many descs were processed */ tx_info = &tx_ring->tx_buffer_info[req_id]; - total_tx_descs += tx_info->tx_descs; - /* Free whole mbuf chain */ mbuf = tx_info->mbuf; rte_pktmbuf_free(mbuf); + tx_info->mbuf = NULL; + tx_ring->empty_tx_reqs[next_to_clean] = req_id; + + total_tx_descs += tx_info->tx_descs; /* Put back descriptor to the ring for reuse */ - tx_ring->empty_tx_reqs[next_to_clean & ring_mask] = req_id; - next_to_clean++; - cleanup_budget = - RTE_MIN(ring_size / ENA_REFILL_THRESH_DIVIDER, - (unsigned int)ENA_REFILL_THRESH_PACKET); - - /* If too many descs to clean, leave it for another run */ - if (unlikely(total_tx_descs > cleanup_budget)) - break; + next_to_clean = ENA_IDX_NEXT_MASKED(next_to_clean, + tx_ring->size_mask); } - tx_ring->tx_stats.available_desc = - ena_com_free_q_entries(tx_ring->ena_com_io_sq); - if (total_tx_descs > 0) { + if (likely(total_tx_descs > 0)) { /* acknowledge completion of sent packets */ tx_ring->next_to_clean = next_to_clean; ena_com_comp_ack(tx_ring->ena_com_io_sq, total_tx_descs); ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq); } +} +static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, + uint16_t nb_pkts) +{ + struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue); + uint16_t sent_idx = 0; + + /* Check adapter state */ + if (unlikely(tx_ring->adapter->state != ENA_ADAPTER_STATE_RUNNING)) { + PMD_DRV_LOG(ALERT, + "Trying to xmit pkts while device is NOT running\n"); + return 0; + } + + nb_pkts = RTE_MIN(ena_com_free_q_entries(tx_ring->ena_com_io_sq), + nb_pkts); + + for (sent_idx = 0; sent_idx < nb_pkts; sent_idx++) { + if (ena_xmit_mbuf(tx_ring, tx_pkts[sent_idx])) + break; + + rte_prefetch0(tx_pkts[ENA_IDX_ADD_MASKED(sent_idx, 4, + tx_ring->size_mask)]); + } + + tx_ring->tx_stats.available_desc = + ena_com_free_q_entries(tx_ring->ena_com_io_sq); + + /* If there are ready packets to be xmitted... */ + if (sent_idx > 0) { + /* ...let HW do its best :-) */ + ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq); + tx_ring->tx_stats.doorbells++; + } + + ena_tx_cleanup(tx_ring); + + tx_ring->tx_stats.available_desc = + ena_com_free_q_entries(tx_ring->ena_com_io_sq); tx_ring->tx_stats.tx_poll++; return sent_idx; @@ -2770,40 +2833,20 @@ RTE_PMD_REGISTER_PCI(net_ena, rte_ena_pmd); RTE_PMD_REGISTER_PCI_TABLE(net_ena, pci_id_ena_map); RTE_PMD_REGISTER_KMOD_DEP(net_ena, "* igb_uio | uio_pci_generic | vfio-pci"); RTE_PMD_REGISTER_PARAM_STRING(net_ena, ENA_DEVARG_LARGE_LLQ_HDR "=<0|1>"); - -RTE_INIT(ena_init_log) -{ - ena_logtype_init = rte_log_register("pmd.net.ena.init"); - if (ena_logtype_init >= 0) - rte_log_set_level(ena_logtype_init, RTE_LOG_NOTICE); - ena_logtype_driver = rte_log_register("pmd.net.ena.driver"); - if (ena_logtype_driver >= 0) - rte_log_set_level(ena_logtype_driver, RTE_LOG_NOTICE); - +RTE_LOG_REGISTER(ena_logtype_init, pmd.net.ena.init, NOTICE); +RTE_LOG_REGISTER(ena_logtype_driver, pmd.net.ena.driver, NOTICE); #ifdef RTE_LIBRTE_ENA_DEBUG_RX - ena_logtype_rx = rte_log_register("pmd.net.ena.rx"); - if (ena_logtype_rx >= 0) - rte_log_set_level(ena_logtype_rx, RTE_LOG_NOTICE); +RTE_LOG_REGISTER(ena_logtype_rx, pmd.net.ena.rx, NOTICE); #endif - #ifdef RTE_LIBRTE_ENA_DEBUG_TX - ena_logtype_tx = rte_log_register("pmd.net.ena.tx"); - if (ena_logtype_tx >= 0) - rte_log_set_level(ena_logtype_tx, RTE_LOG_NOTICE); +RTE_LOG_REGISTER(ena_logtype_tx, pmd.net.ena.tx, NOTICE); #endif - #ifdef RTE_LIBRTE_ENA_DEBUG_TX_FREE - ena_logtype_tx_free = rte_log_register("pmd.net.ena.tx_free"); - if (ena_logtype_tx_free >= 0) - rte_log_set_level(ena_logtype_tx_free, RTE_LOG_NOTICE); +RTE_LOG_REGISTER(ena_logtype_tx_free, pmd.net.ena.tx_free, NOTICE); #endif - #ifdef RTE_LIBRTE_ENA_COM_DEBUG - ena_logtype_com = rte_log_register("pmd.net.ena.com"); - if (ena_logtype_com >= 0) - rte_log_set_level(ena_logtype_com, RTE_LOG_NOTICE); +RTE_LOG_REGISTER(ena_logtype_com, pmd.net.ena.com, NOTICE); #endif -} /****************************************************************************** ******************************** AENQ Handlers ******************************* @@ -2824,7 +2867,7 @@ static void ena_update_on_link_change(void *adapter_data, adapter->link_status = status; ena_link_update(eth_dev, 0); - _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL); + rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_INTR_LSC, NULL); } static void ena_notification(void *data,