#include <ena_eth_io_defs.h>
#define DRV_MODULE_VER_MAJOR 2
-#define DRV_MODULE_VER_MINOR 5
+#define DRV_MODULE_VER_MINOR 6
#define DRV_MODULE_VER_SUBMINOR 0
#define __MERGE_64B_H_L(h, l) (((uint64_t)h << 32) | l)
/* Device arguments */
#define ENA_DEVARG_LARGE_LLQ_HDR "large_llq_hdr"
+/* Timeout in seconds after which a single uncompleted Tx packet should be
+ * considered as a missing.
+ */
+#define ENA_DEVARG_MISS_TXC_TO "miss_txc_to"
/*
* Each rte_memzone should have unique name.
void **push_header,
uint16_t *header_len);
static int ena_xmit_mbuf(struct ena_ring *tx_ring, struct rte_mbuf *mbuf);
-static void ena_tx_cleanup(struct ena_ring *tx_ring);
+static int ena_tx_cleanup(void *txp, uint32_t free_pkt_cnt);
static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
uint16_t nb_pkts);
static uint16_t eth_ena_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
static int ena_xstats_get_names(struct rte_eth_dev *dev,
struct rte_eth_xstat_name *xstats_names,
unsigned int n);
+static int ena_xstats_get_names_by_id(struct rte_eth_dev *dev,
+ const uint64_t *ids,
+ struct rte_eth_xstat_name *xstats_names,
+ unsigned int size);
static int ena_xstats_get(struct rte_eth_dev *dev,
struct rte_eth_xstat *stats,
unsigned int n);
const void *peer);
static const struct eth_dev_ops ena_dev_ops = {
- .dev_configure = ena_dev_configure,
- .dev_infos_get = ena_infos_get,
- .rx_queue_setup = ena_rx_queue_setup,
- .tx_queue_setup = ena_tx_queue_setup,
- .dev_start = ena_start,
- .dev_stop = ena_stop,
- .link_update = ena_link_update,
- .stats_get = ena_stats_get,
- .xstats_get_names = ena_xstats_get_names,
- .xstats_get = ena_xstats_get,
- .xstats_get_by_id = ena_xstats_get_by_id,
- .mtu_set = ena_mtu_set,
- .rx_queue_release = ena_rx_queue_release,
- .tx_queue_release = ena_tx_queue_release,
- .dev_close = ena_close,
- .dev_reset = ena_dev_reset,
- .reta_update = ena_rss_reta_update,
- .reta_query = ena_rss_reta_query,
- .rx_queue_intr_enable = ena_rx_queue_intr_enable,
- .rx_queue_intr_disable = ena_rx_queue_intr_disable,
- .rss_hash_update = ena_rss_hash_update,
- .rss_hash_conf_get = ena_rss_hash_conf_get,
+ .dev_configure = ena_dev_configure,
+ .dev_infos_get = ena_infos_get,
+ .rx_queue_setup = ena_rx_queue_setup,
+ .tx_queue_setup = ena_tx_queue_setup,
+ .dev_start = ena_start,
+ .dev_stop = ena_stop,
+ .link_update = ena_link_update,
+ .stats_get = ena_stats_get,
+ .xstats_get_names = ena_xstats_get_names,
+ .xstats_get_names_by_id = ena_xstats_get_names_by_id,
+ .xstats_get = ena_xstats_get,
+ .xstats_get_by_id = ena_xstats_get_by_id,
+ .mtu_set = ena_mtu_set,
+ .rx_queue_release = ena_rx_queue_release,
+ .tx_queue_release = ena_tx_queue_release,
+ .dev_close = ena_close,
+ .dev_reset = ena_dev_reset,
+ .reta_update = ena_rss_reta_update,
+ .reta_query = ena_rss_reta_query,
+ .rx_queue_intr_enable = ena_rx_queue_intr_enable,
+ .rx_queue_intr_disable = ena_rx_queue_intr_disable,
+ .rss_hash_update = ena_rss_hash_update,
+ .rss_hash_conf_get = ena_rss_hash_conf_get,
+ .tx_done_cleanup = ena_tx_cleanup,
};
/*********************************************************************
}),
struct ena_com_dev *ena_dev, u32 *ind_tbl);
+static inline void ena_trigger_reset(struct ena_adapter *adapter,
+ enum ena_regs_reset_reason_types reason)
+{
+ if (likely(!adapter->trigger_reset)) {
+ adapter->reset_reason = reason;
+ adapter->trigger_reset = true;
+ }
+}
+
static inline void ena_rx_mbuf_prepare(struct ena_ring *rx_ring,
struct rte_mbuf *mbuf,
struct ena_com_rx_ctx *ena_rx_ctx,
} else {
if (unlikely(ena_rx_ctx->l4_csum_err)) {
++rx_stats->l4_csum_bad;
- ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_BAD;
+ /*
+ * For the L4 Rx checksum offload the HW may indicate
+ * bad checksum although it's valid. Because of that,
+ * we're setting the UNKNOWN flag to let the app
+ * re-verify the checksum.
+ */
+ ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_UNKNOWN;
} else {
++rx_stats->l4_csum_good;
ol_flags |= RTE_MBUF_F_RX_L4_CKSUM_GOOD;
if (mbuf->ol_flags & RTE_MBUF_F_TX_IPV6) {
ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6;
+ /* For the IPv6 packets, DF always needs to be true. */
+ ena_tx_ctx->df = 1;
} else {
ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4;
if (mbuf->packet_type &
(RTE_PTYPE_L4_NONFRAG
| RTE_PTYPE_INNER_L4_NONFRAG))
- ena_tx_ctx->df = true;
+ ena_tx_ctx->df = 1;
}
/* check if L4 checksum is needed */
}
if (tx_info)
- PMD_TX_LOG(ERR, "tx_info doesn't have valid mbuf\n");
+ PMD_TX_LOG(ERR, "tx_info doesn't have valid mbuf. queue %d:%d req_id %u\n",
+ tx_ring->port_id, tx_ring->id, req_id);
else
- PMD_TX_LOG(ERR, "Invalid req_id: %hu\n", req_id);
+ PMD_TX_LOG(ERR, "Invalid req_id: %hu in queue %d:%d\n",
+ req_id, tx_ring->port_id, tx_ring->id);
/* Trigger device reset */
++tx_ring->tx_stats.bad_req_id;
- tx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_TX_REQ_ID;
- tx_ring->adapter->trigger_reset = true;
+ ena_trigger_reset(tx_ring->adapter, ENA_REGS_RESET_INV_TX_REQ_ID);
return -EFAULT;
}
if (unlikely((rte_get_timer_cycles() - adapter->timestamp_wd) >=
adapter->keep_alive_timeout)) {
PMD_DRV_LOG(ERR, "Keep alive timeout\n");
- adapter->reset_reason = ENA_REGS_RESET_KEEP_ALIVE_TO;
- adapter->trigger_reset = true;
+ ena_trigger_reset(adapter, ENA_REGS_RESET_KEEP_ALIVE_TO);
++adapter->dev_stats.wd_expired;
}
}
{
if (unlikely(!ena_com_get_admin_running_state(&adapter->ena_dev))) {
PMD_DRV_LOG(ERR, "ENA admin queue is not in running state\n");
- adapter->reset_reason = ENA_REGS_RESET_ADMIN_TO;
- adapter->trigger_reset = true;
+ ena_trigger_reset(adapter, ENA_REGS_RESET_ADMIN_TO);
}
}
return 0;
}
+ if (adapter->dev_mem_base == NULL) {
+ PMD_DRV_LOG(ERR,
+ "LLQ is advertised as supported, but device doesn't expose mem bar\n");
+ ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
+ return 0;
+ }
+
rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations);
if (unlikely(rc)) {
PMD_INIT_LOG(WARNING,
if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST)
return 0;
- if (!adapter->dev_mem_base) {
- PMD_DRV_LOG(ERR,
- "Unable to access LLQ BAR resource. Fallback to host mode policy.\n");
- ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
- return 0;
- }
-
ena_dev->mem_bar = adapter->dev_mem_base;
return 0;
}
ena_dev->reg_bar = adapter->regs;
- /* This is a dummy pointer for ena_com functions. */
- ena_dev->dmadev = adapter;
+ /* Pass device data as a pointer which can be passed to the IO functions
+ * by the ena_com (for example - the memory allocation).
+ */
+ ena_dev->dmadev = eth_dev->data;
adapter->id_number = adapters_found;
snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d",
adapter->id_number);
+ adapter->missing_tx_completion_to = ENA_TX_TIMEOUT;
+
rc = ena_parse_devargs(adapter, pci_dev->device.devargs);
if (rc != 0) {
PMD_INIT_LOG(CRIT, "Failed to parse devargs\n");
adapter->missing_tx_completion_budget =
RTE_MIN(ENA_MONITORED_TX_QUEUES, dev->data->nb_tx_queues);
- adapter->missing_tx_completion_to = ENA_TX_TIMEOUT;
/* To avoid detection of the spurious Tx completion timeout due to
* application not calling the Tx cleanup function, set timeout for the
* Tx queue which should be half of the missing completion timeout for a
rc);
if (rc == ENA_COM_NO_SPACE) {
++rx_ring->rx_stats.bad_desc_num;
- rx_ring->adapter->reset_reason =
- ENA_REGS_RESET_TOO_MANY_RX_DESCS;
+ ena_trigger_reset(rx_ring->adapter,
+ ENA_REGS_RESET_TOO_MANY_RX_DESCS);
} else {
++rx_ring->rx_stats.bad_req_id;
- rx_ring->adapter->reset_reason =
- ENA_REGS_RESET_INV_RX_REQ_ID;
+ ena_trigger_reset(rx_ring->adapter,
+ ENA_REGS_RESET_INV_RX_REQ_ID);
}
- rx_ring->adapter->trigger_reset = true;
return 0;
}
adapter->ena_dev.mmio_read.reg_read_to =
hints->mmio_read_timeout * 1000;
- if (hints->missing_tx_completion_timeout) {
- if (hints->missing_tx_completion_timeout ==
- ENA_HW_HINTS_NO_TIMEOUT) {
- adapter->missing_tx_completion_to =
- ENA_HW_HINTS_NO_TIMEOUT;
- } else {
- /* Convert from msecs to ticks */
- adapter->missing_tx_completion_to = rte_get_timer_hz() *
- hints->missing_tx_completion_timeout / 1000;
- adapter->tx_cleanup_stall_delay =
- adapter->missing_tx_completion_to / 2;
- }
- }
-
if (hints->driver_watchdog_timeout) {
if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT)
adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT;
if (unlikely(rc)) {
PMD_DRV_LOG(ERR, "Failed to prepare Tx buffers, rc: %d\n", rc);
++tx_ring->tx_stats.prepare_ctx_err;
- tx_ring->adapter->reset_reason =
- ENA_REGS_RESET_DRIVER_INVALID_STATE;
- tx_ring->adapter->trigger_reset = true;
+ ena_trigger_reset(tx_ring->adapter,
+ ENA_REGS_RESET_DRIVER_INVALID_STATE);
return rc;
}
return 0;
}
-static void ena_tx_cleanup(struct ena_ring *tx_ring)
+static int ena_tx_cleanup(void *txp, uint32_t free_pkt_cnt)
{
+ struct ena_ring *tx_ring = (struct ena_ring *)txp;
unsigned int total_tx_descs = 0;
+ unsigned int total_tx_pkts = 0;
uint16_t cleanup_budget;
uint16_t next_to_clean = tx_ring->next_to_clean;
- /* Attempt to release all Tx descriptors (ring_size - 1 -> size_mask) */
- cleanup_budget = tx_ring->size_mask;
+ /*
+ * If free_pkt_cnt is equal to 0, it means that the user requested
+ * full cleanup, so attempt to release all Tx descriptors
+ * (ring_size - 1 -> size_mask)
+ */
+ cleanup_budget = (free_pkt_cnt == 0) ? tx_ring->size_mask : free_pkt_cnt;
- while (likely(total_tx_descs < cleanup_budget)) {
+ while (likely(total_tx_pkts < cleanup_budget)) {
struct rte_mbuf *mbuf;
struct ena_tx_buffer *tx_info;
uint16_t req_id;
tx_ring->empty_tx_reqs[next_to_clean] = req_id;
total_tx_descs += tx_info->tx_descs;
+ total_tx_pkts++;
/* Put back descriptor to the ring for reuse */
next_to_clean = ENA_IDX_NEXT_MASKED(next_to_clean,
ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq);
}
- /* Notify completion handler that the cleanup was just called */
- tx_ring->last_cleanup_ticks = rte_get_timer_cycles();
+ /* Notify completion handler that full cleanup was performed */
+ if (free_pkt_cnt == 0 || total_tx_pkts < cleanup_budget)
+ tx_ring->last_cleanup_ticks = rte_get_timer_cycles();
+
+ return total_tx_pkts;
}
static uint16_t eth_ena_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
available_desc = ena_com_free_q_entries(tx_ring->ena_com_io_sq);
if (available_desc < tx_ring->tx_free_thresh)
- ena_tx_cleanup(tx_ring);
+ ena_tx_cleanup((void *)tx_ring, 0);
for (sent_idx = 0; sent_idx < nb_pkts; sent_idx++) {
if (ena_xmit_mbuf(tx_ring, tx_pkts[sent_idx]))
return xstats_count;
}
+/**
+ * DPDK callback to retrieve names of extended device statistics for the given
+ * ids.
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ * @param[out] xstats_names
+ * Buffer to insert names into.
+ * @param ids
+ * IDs array for which the names should be retrieved.
+ * @param size
+ * Number of ids.
+ *
+ * @return
+ * Positive value: number of xstats names. Negative value: error code.
+ */
+static int ena_xstats_get_names_by_id(struct rte_eth_dev *dev,
+ const uint64_t *ids,
+ struct rte_eth_xstat_name *xstats_names,
+ unsigned int size)
+{
+ uint64_t xstats_count = ena_xstats_calc_num(dev->data);
+ uint64_t id, qid;
+ unsigned int i;
+
+ if (xstats_names == NULL)
+ return xstats_count;
+
+ for (i = 0; i < size; ++i) {
+ id = ids[i];
+ if (id > xstats_count) {
+ PMD_DRV_LOG(ERR,
+ "ID value out of range: id=%" PRIu64 ", xstats_num=%" PRIu64 "\n",
+ id, xstats_count);
+ return -EINVAL;
+ }
+
+ if (id < ENA_STATS_ARRAY_GLOBAL) {
+ strcpy(xstats_names[i].name,
+ ena_stats_global_strings[id].name);
+ continue;
+ }
+
+ id -= ENA_STATS_ARRAY_GLOBAL;
+ if (id < ENA_STATS_ARRAY_ENI) {
+ strcpy(xstats_names[i].name,
+ ena_stats_eni_strings[id].name);
+ continue;
+ }
+
+ id -= ENA_STATS_ARRAY_ENI;
+ if (id < ENA_STATS_ARRAY_RX) {
+ qid = id / dev->data->nb_rx_queues;
+ id %= dev->data->nb_rx_queues;
+ snprintf(xstats_names[i].name,
+ sizeof(xstats_names[i].name),
+ "rx_q%" PRIu64 "d_%s",
+ qid, ena_stats_rx_strings[id].name);
+ continue;
+ }
+
+ id -= ENA_STATS_ARRAY_RX;
+ /* Although this condition is not needed, it was added for
+ * compatibility if new xstat structure would be ever added.
+ */
+ if (id < ENA_STATS_ARRAY_TX) {
+ qid = id / dev->data->nb_tx_queues;
+ id %= dev->data->nb_tx_queues;
+ snprintf(xstats_names[i].name,
+ sizeof(xstats_names[i].name),
+ "tx_q%" PRIu64 "_%s",
+ qid, ena_stats_tx_strings[id].name);
+ continue;
+ }
+ }
+
+ return i;
+}
+
/**
* DPDK callback to get extended device statistics.
*
return valid;
}
+static int ena_process_uint_devarg(const char *key,
+ const char *value,
+ void *opaque)
+{
+ struct ena_adapter *adapter = opaque;
+ char *str_end;
+ uint64_t uint_value;
+
+ uint_value = strtoull(value, &str_end, 10);
+ if (value == str_end) {
+ PMD_INIT_LOG(ERR,
+ "Invalid value for key '%s'. Only uint values are accepted.\n",
+ key);
+ return -EINVAL;
+ }
+
+ if (strcmp(key, ENA_DEVARG_MISS_TXC_TO) == 0) {
+ if (uint_value > ENA_MAX_TX_TIMEOUT_SECONDS) {
+ PMD_INIT_LOG(ERR,
+ "Tx timeout too high: %" PRIu64 " sec. Maximum allowed: %d sec.\n",
+ uint_value, ENA_MAX_TX_TIMEOUT_SECONDS);
+ return -EINVAL;
+ } else if (uint_value == 0) {
+ PMD_INIT_LOG(INFO,
+ "Check for missing Tx completions has been disabled.\n");
+ adapter->missing_tx_completion_to =
+ ENA_HW_HINTS_NO_TIMEOUT;
+ } else {
+ PMD_INIT_LOG(INFO,
+ "Tx packet completion timeout set to %" PRIu64 " seconds.\n",
+ uint_value);
+ adapter->missing_tx_completion_to =
+ uint_value * rte_get_timer_hz();
+ }
+ }
+
+ return 0;
+}
+
static int ena_process_bool_devarg(const char *key,
const char *value,
void *opaque)
{
static const char * const allowed_args[] = {
ENA_DEVARG_LARGE_LLQ_HDR,
+ ENA_DEVARG_MISS_TXC_TO,
NULL,
};
struct rte_kvargs *kvlist;
rc = rte_kvargs_process(kvlist, ENA_DEVARG_LARGE_LLQ_HDR,
ena_process_bool_devarg, adapter);
+ if (rc != 0)
+ goto exit;
+ rc = rte_kvargs_process(kvlist, ENA_DEVARG_MISS_TXC_TO,
+ ena_process_uint_devarg, adapter);
+exit:
rte_kvargs_free(kvlist);
return rc;
indirect_table);
}
+/*********************************************************************
+ * ena_plat_dpdk.h functions implementations
+ *********************************************************************/
+
+const struct rte_memzone *
+ena_mem_alloc_coherent(struct rte_eth_dev_data *data, size_t size,
+ int socket_id, unsigned int alignment, void **virt_addr,
+ dma_addr_t *phys_addr)
+{
+ char z_name[RTE_MEMZONE_NAMESIZE];
+ struct ena_adapter *adapter = data->dev_private;
+ const struct rte_memzone *memzone;
+ int rc;
+
+ rc = snprintf(z_name, RTE_MEMZONE_NAMESIZE, "ena_p%d_mz%" PRIu64 "",
+ data->port_id, adapter->memzone_cnt);
+ if (rc >= RTE_MEMZONE_NAMESIZE) {
+ PMD_DRV_LOG(ERR,
+ "Name for the ena_com memzone is too long. Port: %d, mz_num: %" PRIu64 "\n",
+ data->port_id, adapter->memzone_cnt);
+ goto error;
+ }
+ adapter->memzone_cnt++;
+
+ memzone = rte_memzone_reserve_aligned(z_name, size, socket_id,
+ RTE_MEMZONE_IOVA_CONTIG, alignment);
+ if (memzone == NULL) {
+ PMD_DRV_LOG(ERR, "Failed to allocate ena_com memzone: %s\n",
+ z_name);
+ goto error;
+ }
+
+ memset(memzone->addr, 0, size);
+ *virt_addr = memzone->addr;
+ *phys_addr = memzone->iova;
+
+ return memzone;
+
+error:
+ *virt_addr = NULL;
+ *phys_addr = 0;
+
+ return NULL;
+}
+
+
/*********************************************************************
* PMD configuration
*********************************************************************/
git.droids-corp.org / dpdk.git / blobdiff