* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
+#include <rte_string_fns.h>
#include <rte_ether.h>
#include <rte_ethdev_driver.h>
#include <rte_ethdev_pci.h>
#include <rte_dev.h>
#include <rte_errno.h>
#include <rte_version.h>
-#include <rte_eal_memconfig.h>
#include <rte_net.h>
#include "ena_ethdev.h"
#include <ena_admin_defs.h>
#include <ena_eth_io_defs.h>
-#define DRV_MODULE_VER_MAJOR 1
-#define DRV_MODULE_VER_MINOR 1
+#define DRV_MODULE_VER_MAJOR 2
+#define DRV_MODULE_VER_MINOR 0
#define DRV_MODULE_VER_SUBMINOR 1
#define ENA_IO_TXQ_IDX(q) (2 * (q))
#define TEST_BIT(val, bit_shift) (val & (1UL << bit_shift))
#define GET_L4_HDR_LEN(mbuf) \
- ((rte_pktmbuf_mtod_offset(mbuf, struct tcp_hdr *, \
+ ((rte_pktmbuf_mtod_offset(mbuf, struct rte_tcp_hdr *, \
mbuf->l3_len + mbuf->l2_len)->data_off) >> 4)
#define ENA_RX_RSS_TABLE_LOG_SIZE 7
static int ena_queue_start_all(struct rte_eth_dev *dev,
enum ena_ring_type ring_type);
static void ena_stats_restart(struct rte_eth_dev *dev);
-static void ena_infos_get(struct rte_eth_dev *dev,
- struct rte_eth_dev_info *dev_info);
+static int ena_infos_get(struct rte_eth_dev *dev,
+ struct rte_eth_dev_info *dev_info);
static int ena_rss_reta_update(struct rte_eth_dev *dev,
struct rte_eth_rss_reta_entry64 *reta_conf,
uint16_t reta_size);
.reta_query = ena_rss_reta_query,
};
-#define NUMA_NO_NODE SOCKET_ID_ANY
-
-static inline int ena_cpu_to_node(int cpu)
-{
- struct rte_config *config = rte_eal_get_configuration();
- struct rte_fbarray *arr = &config->mem_config->memzones;
- const struct rte_memzone *mz;
-
- if (unlikely(cpu >= RTE_MAX_MEMZONE))
- return NUMA_NO_NODE;
-
- mz = rte_fbarray_get(arr, cpu);
-
- return mz->socket_id;
-}
-
static inline void ena_rx_mbuf_prepare(struct rte_mbuf *mbuf,
struct ena_com_rx_ctx *ena_rx_ctx)
{
}
/* check if L4 checksum is needed */
- if ((mbuf->ol_flags & PKT_TX_TCP_CKSUM) &&
+ if (((mbuf->ol_flags & PKT_TX_L4_MASK) == PKT_TX_TCP_CKSUM) &&
(queue_offloads & DEV_TX_OFFLOAD_TCP_CKSUM)) {
ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP;
ena_tx_ctx->l4_csum_enable = true;
- } else if ((mbuf->ol_flags & PKT_TX_UDP_CKSUM) &&
- (queue_offloads & DEV_TX_OFFLOAD_UDP_CKSUM)) {
+ } else if (((mbuf->ol_flags & PKT_TX_L4_MASK) ==
+ PKT_TX_UDP_CKSUM) &&
+ (queue_offloads & DEV_TX_OFFLOAD_UDP_CKSUM)) {
ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP;
ena_tx_ctx->l4_csum_enable = true;
} else {
host_info->os_type = ENA_ADMIN_OS_DPDK;
host_info->kernel_ver = RTE_VERSION;
- snprintf((char *)host_info->kernel_ver_str,
- sizeof(host_info->kernel_ver_str),
- "%s", rte_version());
+ strlcpy((char *)host_info->kernel_ver_str, rte_version(),
+ sizeof(host_info->kernel_ver_str));
host_info->os_dist = RTE_VERSION;
- snprintf((char *)host_info->os_dist_str,
- sizeof(host_info->os_dist_str),
- "%s", rte_version());
+ strlcpy((char *)host_info->os_dist_str, rte_version(),
+ sizeof(host_info->os_dist_str));
host_info->driver_version =
(DRV_MODULE_VER_MAJOR) |
(DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) |
{
struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
- struct ena_adapter *adapter =
- (struct ena_adapter *)(dev->data->dev_private);
+ struct ena_adapter *adapter = dev->data->dev_private;
if (adapter->state == ENA_ADAPTER_STATE_RUNNING)
ena_stop(dev);
struct rte_eth_rss_reta_entry64 *reta_conf,
uint16_t reta_size)
{
- struct ena_adapter *adapter =
- (struct ena_adapter *)(dev->data->dev_private);
+ struct ena_adapter *adapter = dev->data->dev_private;
struct ena_com_dev *ena_dev = &adapter->ena_dev;
int rc, i;
u16 entry_value;
struct rte_eth_rss_reta_entry64 *reta_conf,
uint16_t reta_size)
{
- struct ena_adapter *adapter =
- (struct ena_adapter *)(dev->data->dev_private);
+ struct ena_adapter *adapter = dev->data->dev_private;
struct ena_com_dev *ena_dev = &adapter->ena_dev;
int rc;
int i;
__rte_unused int wait_to_complete)
{
struct rte_eth_link *link = &dev->data->dev_link;
- struct ena_adapter *adapter;
-
- adapter = (struct ena_adapter *)(dev->data->dev_private);
+ struct ena_adapter *adapter = dev->data->dev_private;
link->link_status = adapter->link_status ? ETH_LINK_UP : ETH_LINK_DOWN;
link->link_speed = ETH_SPEED_NUM_NONE;
static int ena_queue_start_all(struct rte_eth_dev *dev,
enum ena_ring_type ring_type)
{
- struct ena_adapter *adapter =
- (struct ena_adapter *)(dev->data->dev_private);
+ struct ena_adapter *adapter = dev->data->dev_private;
struct ena_ring *queues = NULL;
int nb_queues;
int i = 0;
static void ena_stats_restart(struct rte_eth_dev *dev)
{
- struct ena_adapter *adapter =
- (struct ena_adapter *)(dev->data->dev_private);
+ struct ena_adapter *adapter = dev->data->dev_private;
rte_atomic64_init(&adapter->drv_stats->ierrors);
rte_atomic64_init(&adapter->drv_stats->oerrors);
struct rte_eth_stats *stats)
{
struct ena_admin_basic_stats ena_stats;
- struct ena_adapter *adapter =
- (struct ena_adapter *)(dev->data->dev_private);
+ struct ena_adapter *adapter = dev->data->dev_private;
struct ena_com_dev *ena_dev = &adapter->ena_dev;
int rc;
int i;
ena_assert_msg(dev->data != NULL, "Uninitialized device\n");
ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device\n");
- adapter = (struct ena_adapter *)(dev->data->dev_private);
+ adapter = dev->data->dev_private;
ena_dev = &adapter->ena_dev;
ena_assert_msg(ena_dev != NULL, "Uninitialized device\n");
static int ena_start(struct rte_eth_dev *dev)
{
- struct ena_adapter *adapter =
- (struct ena_adapter *)(dev->data->dev_private);
+ struct ena_adapter *adapter = dev->data->dev_private;
uint64_t ticks;
int rc = 0;
static void ena_stop(struct rte_eth_dev *dev)
{
- struct ena_adapter *adapter =
- (struct ena_adapter *)(dev->data->dev_private);
+ struct ena_adapter *adapter = dev->data->dev_private;
struct ena_com_dev *ena_dev = &adapter->ena_dev;
int rc;
}
ctx.qid = ena_qid;
ctx.msix_vector = -1; /* interrupts not used */
- ctx.numa_node = ena_cpu_to_node(ring->id);
+ ctx.numa_node = ring->numa_socket_id;
rc = ena_com_create_io_queue(ena_dev, &ctx);
if (rc) {
static void ena_queue_stop_all(struct rte_eth_dev *dev,
enum ena_ring_type ring_type)
{
- struct ena_adapter *adapter =
- (struct ena_adapter *)(dev->data->dev_private);
+ struct ena_adapter *adapter = dev->data->dev_private;
struct ena_ring *queues = NULL;
uint16_t nb_queues, i;
static int ena_tx_queue_setup(struct rte_eth_dev *dev,
uint16_t queue_idx,
uint16_t nb_desc,
- __rte_unused unsigned int socket_id,
+ unsigned int socket_id,
const struct rte_eth_txconf *tx_conf)
{
struct ena_ring *txq = NULL;
- struct ena_adapter *adapter =
- (struct ena_adapter *)(dev->data->dev_private);
+ struct ena_adapter *adapter = dev->data->dev_private;
unsigned int i;
txq = &adapter->tx_ring[queue_idx];
txq->next_to_clean = 0;
txq->next_to_use = 0;
txq->ring_size = nb_desc;
+ txq->numa_socket_id = socket_id;
txq->tx_buffer_info = rte_zmalloc("txq->tx_buffer_info",
sizeof(struct ena_tx_buffer) *
static int ena_rx_queue_setup(struct rte_eth_dev *dev,
uint16_t queue_idx,
uint16_t nb_desc,
- __rte_unused unsigned int socket_id,
+ unsigned int socket_id,
__rte_unused const struct rte_eth_rxconf *rx_conf,
struct rte_mempool *mp)
{
- struct ena_adapter *adapter =
- (struct ena_adapter *)(dev->data->dev_private);
+ struct ena_adapter *adapter = dev->data->dev_private;
struct ena_ring *rxq = NULL;
int i;
rxq->next_to_clean = 0;
rxq->next_to_use = 0;
rxq->ring_size = nb_desc;
+ rxq->numa_socket_id = socket_id;
rxq->mb_pool = mp;
rxq->rx_buffer_info = rte_zmalloc("rxq->buffer_info",
static void ena_interrupt_handler_rte(void *cb_arg)
{
- struct ena_adapter *adapter = (struct ena_adapter *)cb_arg;
+ struct ena_adapter *adapter = cb_arg;
struct ena_com_dev *ena_dev = &adapter->ena_dev;
ena_com_admin_q_comp_intr_handler(ena_dev);
static void ena_timer_wd_callback(__rte_unused struct rte_timer *timer,
void *arg)
{
- struct ena_adapter *adapter = (struct ena_adapter *)arg;
+ struct ena_adapter *adapter = arg;
struct rte_eth_dev *dev = adapter->rte_dev;
check_for_missing_keep_alive(adapter);
if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
io_tx_sq_num = get_feat_ctx->llq.max_llq_num;
- io_queue_num = RTE_MIN(rte_lcore_count(), ENA_MAX_NUM_IO_QUEUES);
- io_queue_num = RTE_MIN(io_queue_num, io_rx_num);
+ io_queue_num = RTE_MIN(ENA_MAX_NUM_IO_QUEUES, io_rx_num);
io_queue_num = RTE_MIN(io_queue_num, io_tx_sq_num);
io_queue_num = RTE_MIN(io_queue_num, io_tx_cq_num);
struct ena_calc_queue_size_ctx calc_queue_ctx = { 0 };
struct rte_pci_device *pci_dev;
struct rte_intr_handle *intr_handle;
- struct ena_adapter *adapter =
- (struct ena_adapter *)(eth_dev->data->dev_private);
+ struct ena_adapter *adapter = eth_dev->data->dev_private;
struct ena_com_dev *ena_dev = &adapter->ena_dev;
struct ena_com_dev_get_features_ctx get_feat_ctx;
struct ena_llq_configurations llq_config;
static int adapters_found;
bool wd_state;
- memset(adapter, 0, sizeof(struct ena_adapter));
- ena_dev = &adapter->ena_dev;
-
eth_dev->dev_ops = &ena_dev_ops;
eth_dev->rx_pkt_burst = ð_ena_recv_pkts;
eth_dev->tx_pkt_burst = ð_ena_xmit_pkts;
eth_dev->tx_pkt_prepare = ð_ena_prep_pkts;
- adapter->rte_eth_dev_data = eth_dev->data;
- adapter->rte_dev = eth_dev;
if (rte_eal_process_type() != RTE_PROC_PRIMARY)
return 0;
+ memset(adapter, 0, sizeof(struct ena_adapter));
+ ena_dev = &adapter->ena_dev;
+
+ adapter->rte_eth_dev_data = eth_dev->data;
+ adapter->rte_dev = eth_dev;
+
pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
adapter->pdev = pci_dev;
/* Set max MTU for this device */
adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu;
- /* set device support for TSO */
- adapter->tso4_supported = get_feat_ctx.offload.tx &
- ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK;
+ /* set device support for offloads */
+ adapter->offloads.tso4_supported = (get_feat_ctx.offload.tx &
+ ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) != 0;
+ adapter->offloads.tx_csum_supported = (get_feat_ctx.offload.tx &
+ ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK) != 0;
+ adapter->offloads.rx_csum_supported =
+ (get_feat_ctx.offload.rx_supported &
+ ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK) != 0;
/* Copy MAC address and point DPDK to it */
- eth_dev->data->mac_addrs = (struct ether_addr *)adapter->mac_addr;
- ether_addr_copy((struct ether_addr *)get_feat_ctx.dev_attr.mac_addr,
- (struct ether_addr *)adapter->mac_addr);
+ eth_dev->data->mac_addrs = (struct rte_ether_addr *)adapter->mac_addr;
+ rte_ether_addr_copy((struct rte_ether_addr *)
+ get_feat_ctx.dev_attr.mac_addr,
+ (struct rte_ether_addr *)adapter->mac_addr);
/*
* Pass the information to the rte_eth_dev_close() that it should also
static void ena_destroy_device(struct rte_eth_dev *eth_dev)
{
- struct ena_adapter *adapter =
- (struct ena_adapter *)(eth_dev->data->dev_private);
+ struct ena_adapter *adapter = eth_dev->data->dev_private;
struct ena_com_dev *ena_dev = &adapter->ena_dev;
if (adapter->state == ENA_ADAPTER_STATE_FREE)
static int ena_dev_configure(struct rte_eth_dev *dev)
{
- struct ena_adapter *adapter =
- (struct ena_adapter *)(dev->data->dev_private);
+ struct ena_adapter *adapter = dev->data->dev_private;
adapter->state = ENA_ADAPTER_STATE_CONFIG;
}
}
-static void ena_infos_get(struct rte_eth_dev *dev,
+static int ena_infos_get(struct rte_eth_dev *dev,
struct rte_eth_dev_info *dev_info)
{
struct ena_adapter *adapter;
struct ena_com_dev *ena_dev;
- struct ena_com_dev_get_features_ctx feat;
uint64_t rx_feat = 0, tx_feat = 0;
- int rc = 0;
ena_assert_msg(dev->data != NULL, "Uninitialized device\n");
ena_assert_msg(dev->data->dev_private != NULL, "Uninitialized device\n");
- adapter = (struct ena_adapter *)(dev->data->dev_private);
+ adapter = dev->data->dev_private;
ena_dev = &adapter->ena_dev;
ena_assert_msg(ena_dev != NULL, "Uninitialized device\n");
ETH_LINK_SPEED_50G |
ETH_LINK_SPEED_100G;
- /* Get supported features from HW */
- rc = ena_com_get_dev_attr_feat(ena_dev, &feat);
- if (unlikely(rc)) {
- RTE_LOG(ERR, PMD,
- "Cannot get attribute for ena device rc= %d\n", rc);
- return;
- }
-
/* Set Tx & Rx features available for device */
- if (feat.offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK)
+ if (adapter->offloads.tso4_supported)
tx_feat |= DEV_TX_OFFLOAD_TCP_TSO;
- if (feat.offload.tx &
- ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK)
+ if (adapter->offloads.tx_csum_supported)
tx_feat |= DEV_TX_OFFLOAD_IPV4_CKSUM |
DEV_TX_OFFLOAD_UDP_CKSUM |
DEV_TX_OFFLOAD_TCP_CKSUM;
- if (feat.offload.rx_supported &
- ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK)
+ if (adapter->offloads.rx_csum_supported)
rx_feat |= DEV_RX_OFFLOAD_IPV4_CKSUM |
DEV_RX_OFFLOAD_UDP_CKSUM |
DEV_RX_OFFLOAD_TCP_CKSUM;
adapter->max_tx_sgl_size);
dev_info->tx_desc_lim.nb_mtu_seg_max = RTE_MIN(ENA_PKT_MAX_BUFS,
adapter->max_tx_sgl_size);
+
+ return 0;
}
static uint16_t eth_ena_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
ena_rx_mbuf_prepare(mbuf_head, &ena_rx_ctx);
if (unlikely(mbuf_head->ol_flags &
- (PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD)))
+ (PKT_RX_IP_CKSUM_BAD | PKT_RX_L4_CKSUM_BAD))) {
+ rte_atomic64_inc(&rx_ring->adapter->drv_stats->ierrors);
++rx_ring->rx_stats.bad_csum;
+ }
mbuf_head->hash.rss = ena_rx_ctx.hash;
desc_in_use = desc_in_use - completed + 1;
/* Burst refill to save doorbells, memory barriers, const interval */
- if (ring_size - desc_in_use > ENA_RING_DESCS_RATIO(ring_size))
+ if (ring_size - desc_in_use > ENA_RING_DESCS_RATIO(ring_size)) {
+ ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq);
ena_populate_rx_queue(rx_ring, ring_size - desc_in_use);
+ }
return recv_idx;
}
uint32_t i;
struct rte_mbuf *m;
struct ena_ring *tx_ring = (struct ena_ring *)(tx_queue);
- struct ipv4_hdr *ip_hdr;
+ struct rte_ipv4_hdr *ip_hdr;
uint64_t ol_flags;
uint16_t frag_field;
* length of the ethernet header.
*/
if (unlikely(m->l2_len == 0))
- m->l2_len = sizeof(struct ether_hdr);
+ m->l2_len = sizeof(struct rte_ether_hdr);
- ip_hdr = rte_pktmbuf_mtod_offset(m, struct ipv4_hdr *,
+ ip_hdr = rte_pktmbuf_mtod_offset(m, struct rte_ipv4_hdr *,
m->l2_len);
frag_field = rte_be_to_cpu_16(ip_hdr->fragment_offset);
- if ((frag_field & IPV4_HDR_DF_FLAG) != 0) {
+ if ((frag_field & RTE_IPV4_HDR_DF_FLAG) != 0) {
m->packet_type |= RTE_PTYPE_L4_NONFRAG;
/* If IPv4 header has DF flag enabled and TSO support is
* disabled, partial chcecksum should not be calculated.
*/
- if (!tx_ring->adapter->tso4_supported)
+ if (!tx_ring->adapter->offloads.tso4_supported)
continue;
}
if ((ol_flags & ENA_TX_OFFLOAD_NOTSUP_MASK) != 0 ||
(ol_flags & PKT_TX_L4_MASK) ==
PKT_TX_SCTP_CKSUM) {
- rte_errno = -ENOTSUP;
+ rte_errno = ENOTSUP;
return i;
}
#ifdef RTE_LIBRTE_ETHDEV_DEBUG
ret = rte_validate_tx_offload(m);
if (ret != 0) {
- rte_errno = ret;
+ rte_errno = -ret;
return i;
}
#endif
ret = rte_net_intel_cksum_flags_prepare(m,
ol_flags & ~PKT_TX_TCP_SEG);
if (ret != 0) {
- rte_errno = ret;
+ rte_errno = -ret;
return i;
}
}
/* Set TX offloads flags, if applicable */
ena_tx_mbuf_prepare(mbuf, &ena_tx_ctx, tx_ring->offloads);
- if (unlikely(mbuf->ol_flags &
- (PKT_RX_L4_CKSUM_BAD | PKT_RX_IP_CKSUM_BAD)))
- rte_atomic64_inc(&tx_ring->adapter->drv_stats->ierrors);
-
rte_prefetch0(tx_pkts[(sent_idx + 4) & ring_mask]);
/* Process first segment taking into
tx_info->tx_descs = nb_hw_desc;
next_to_use++;
- tx_ring->tx_stats.cnt += tx_info->num_of_bufs;
+ tx_ring->tx_stats.cnt++;
tx_ring->tx_stats.bytes += total_length;
}
tx_ring->tx_stats.available_desc =
if (total_tx_descs > 0) {
/* acknowledge completion of sent packets */
- ena_com_comp_ack(tx_ring->ena_com_io_sq, total_tx_descs);
tx_ring->next_to_clean = next_to_clean;
+ ena_com_comp_ack(tx_ring->ena_com_io_sq, total_tx_descs);
+ ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq);
}
tx_ring->tx_stats.tx_poll++;
struct rte_eth_xstat *xstats,
unsigned int n)
{
- struct ena_adapter *adapter =
- (struct ena_adapter *)(dev->data->dev_private);
+ struct ena_adapter *adapter = dev->data->dev_private;
unsigned int xstats_count = ena_xstats_calc_num(dev);
unsigned int stat, i, count = 0;
int stat_offset;
uint64_t *values,
unsigned int n)
{
- struct ena_adapter *adapter =
- (struct ena_adapter *)(dev->data->dev_private);
+ struct ena_adapter *adapter = dev->data->dev_private;
uint64_t id;
uint64_t rx_entries, tx_entries;
unsigned int i;
struct ena_admin_aenq_link_change_desc *aenq_link_desc;
uint32_t status;
- adapter = (struct ena_adapter *)adapter_data;
+ adapter = adapter_data;
aenq_link_desc = (struct ena_admin_aenq_link_change_desc *)aenq_e;
eth_dev = adapter->rte_dev;
static void ena_notification(void *data,
struct ena_admin_aenq_entry *aenq_e)
{
- struct ena_adapter *adapter = (struct ena_adapter *)data;
+ struct ena_adapter *adapter = data;
struct ena_admin_ena_hw_hints *hints;
if (aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION)
static void ena_keep_alive(void *adapter_data,
__rte_unused struct ena_admin_aenq_entry *aenq_e)
{
- struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
+ struct ena_adapter *adapter = adapter_data;
struct ena_admin_aenq_keep_alive_desc *desc;
uint64_t rx_drops;
git.droids-corp.org / dpdk.git / blobdiff