1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (c) 2015-2020 Amazon.com, Inc. or its affiliates.
9 #include <rte_atomic.h>
10 #include <rte_ether.h>
11 #include <ethdev_driver.h>
12 #include <ethdev_pci.h>
13 #include <rte_cycles.h>
15 #include <rte_bus_pci.h>
16 #include <rte_timer.h>
22 #define ENA_REGS_BAR 0
25 #define ENA_MAX_NUM_QUEUES 128
26 #define ENA_MIN_FRAME_LEN 64
27 #define ENA_NAME_MAX_LEN 20
28 #define ENA_PKT_MAX_BUFS 17
29 #define ENA_RX_BUF_MIN_SIZE 1400
30 #define ENA_DEFAULT_RING_SIZE 1024
32 #define ENA_RX_RSS_TABLE_LOG_SIZE 7
33 #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE)
35 #define ENA_MIN_MTU 128
37 #define ENA_MMIO_DISABLE_REG_READ BIT(0)
39 #define ENA_WD_TIMEOUT_SEC 3
40 #define ENA_DEVICE_KALIVE_TIMEOUT (ENA_WD_TIMEOUT_SEC * rte_get_timer_hz())
42 #define ENA_TX_TIMEOUT (5 * rte_get_timer_hz())
43 #define ENA_MAX_TX_TIMEOUT_SECONDS 60
44 #define ENA_MONITORED_TX_QUEUES 3
45 #define ENA_DEFAULT_MISSING_COMP 256U
47 /* While processing submitted and completed descriptors (rx and tx path
48 * respectively) in a loop it is desired to:
49 * - perform batch submissions while populating submission queue
50 * - avoid blocking transmission of other packets during cleanup phase
51 * Hence the utilization ratio of 1/8 of a queue size or max value if the size
52 * of the ring is very big - like 8k Rx rings.
54 #define ENA_REFILL_THRESH_DIVIDER 8
55 #define ENA_REFILL_THRESH_PACKET 256
57 #define ENA_IDX_NEXT_MASKED(idx, mask) (((idx) + 1) & (mask))
58 #define ENA_IDX_ADD_MASKED(idx, n, mask) (((idx) + (n)) & (mask))
60 #define ENA_RX_RSS_TABLE_LOG_SIZE 7
61 #define ENA_RX_RSS_TABLE_SIZE (1 << ENA_RX_RSS_TABLE_LOG_SIZE)
63 #define ENA_HASH_KEY_SIZE 40
65 #define ENA_ALL_RSS_HF (RTE_ETH_RSS_NONFRAG_IPV4_TCP | RTE_ETH_RSS_NONFRAG_IPV4_UDP | \
66 RTE_ETH_RSS_NONFRAG_IPV6_TCP | RTE_ETH_RSS_NONFRAG_IPV6_UDP)
68 #define ENA_IO_TXQ_IDX(q) (2 * (q))
69 #define ENA_IO_RXQ_IDX(q) (2 * (q) + 1)
70 /* Reversed version of ENA_IO_RXQ_IDX */
71 #define ENA_IO_RXQ_IDX_REV(q) (((q) - 1) / 2)
73 extern struct ena_shared_data *ena_shared_data;
82 struct ena_tx_buffer {
83 struct rte_mbuf *mbuf;
84 unsigned int tx_descs;
85 unsigned int num_of_bufs;
88 struct ena_com_buf bufs[ENA_PKT_MAX_BUFS];
91 /* Rx buffer holds only pointer to the mbuf - may be expanded in the future */
92 struct ena_rx_buffer {
93 struct rte_mbuf *mbuf;
94 struct ena_com_buf ena_buf;
97 struct ena_calc_queue_size_ctx {
98 struct ena_com_dev_get_features_ctx *get_feat_ctx;
99 struct ena_com_dev *ena_dev;
100 u32 max_rx_queue_size;
101 u32 max_tx_queue_size;
106 struct ena_stats_tx {
117 struct ena_stats_rx {
132 uint64_t last_cleanup_ticks;
134 enum ena_ring_type type;
135 enum ena_admin_placement_policy_type tx_mem_queue_type;
137 /* Indicate there are Tx packets pushed to the device and wait for db */
138 bool pkts_without_db;
140 /* Holds the empty requests for TX/RX OOO completions */
142 uint16_t *empty_tx_reqs;
143 uint16_t *empty_rx_reqs;
147 struct ena_tx_buffer *tx_buffer_info; /* contex of tx packet */
148 struct ena_rx_buffer *rx_buffer_info; /* contex of rx packet */
150 struct rte_mbuf **rx_refill_buffer;
151 unsigned int ring_size; /* number of tx/rx_buffer_info's entries */
152 unsigned int size_mask;
154 struct ena_com_io_cq *ena_com_io_cq;
155 struct ena_com_io_sq *ena_com_io_sq;
158 uint16_t tx_free_thresh;
159 uint16_t rx_free_thresh;
162 struct ena_com_rx_buf_info ena_bufs[ENA_PKT_MAX_BUFS]
165 struct rte_mempool *mb_pool;
166 unsigned int port_id;
168 /* Max length PMD can push to device for LLQ */
169 uint8_t tx_max_header_size;
172 uint8_t *push_buf_intermediate_buf;
174 struct ena_adapter *adapter;
178 bool disable_meta_caching;
181 struct ena_stats_rx rx_stats;
182 struct ena_stats_tx tx_stats;
185 unsigned int numa_socket_id;
187 uint32_t missing_tx_completion_threshold;
188 } __rte_cache_aligned;
190 enum ena_adapter_state {
191 ENA_ADAPTER_STATE_FREE = 0,
192 ENA_ADAPTER_STATE_INIT = 1,
193 ENA_ADAPTER_STATE_RUNNING = 2,
194 ENA_ADAPTER_STATE_STOPPED = 3,
195 ENA_ADAPTER_STATE_CONFIG = 4,
196 ENA_ADAPTER_STATE_CLOSED = 5,
199 struct ena_driver_stats {
200 rte_atomic64_t ierrors;
201 rte_atomic64_t oerrors;
202 rte_atomic64_t rx_nombuf;
206 struct ena_stats_dev {
211 * Tx drops cannot be reported as the driver statistic, because DPDK
212 * rte_eth_stats structure isn't providing appropriate field for that.
213 * As a workaround it is being published as an extended statistic.
218 struct ena_stats_eni {
220 * The number of packets shaped due to inbound aggregate BW
221 * allowance being exceeded
223 uint64_t bw_in_allowance_exceeded;
225 * The number of packets shaped due to outbound aggregate BW
226 * allowance being exceeded
228 uint64_t bw_out_allowance_exceeded;
229 /* The number of packets shaped due to PPS allowance being exceeded */
230 uint64_t pps_allowance_exceeded;
232 * The number of packets shaped due to connection tracking
233 * allowance being exceeded and leading to failure in establishment
236 uint64_t conntrack_allowance_exceeded;
238 * The number of packets shaped due to linklocal packet rate
239 * allowance being exceeded
241 uint64_t linklocal_allowance_exceeded;
244 struct ena_offloads {
245 uint32_t tx_offloads;
246 uint32_t rx_offloads;
249 /* board specific private data structure */
251 /* OS defined structs */
252 struct rte_eth_dev_data *edev_data;
254 struct ena_com_dev ena_dev __rte_cache_aligned;
257 struct ena_ring tx_ring[ENA_MAX_NUM_QUEUES] __rte_cache_aligned;
258 u32 max_tx_ring_size;
262 struct ena_ring rx_ring[ENA_MAX_NUM_QUEUES] __rte_cache_aligned;
263 u32 max_rx_ring_size;
266 u32 max_num_io_queues;
268 struct ena_offloads offloads;
270 /* The admin queue isn't protected by the lock and is used to
271 * retrieve statistics from the device. As there is no guarantee that
272 * application won't try to get statistics from multiple threads, it is
273 * safer to lock the queue to avoid admin queue failure.
275 rte_spinlock_t admin_lock;
278 char name[ENA_NAME_MAX_LEN];
279 u8 mac_addr[RTE_ETHER_ADDR_LEN];
284 struct ena_driver_stats *drv_stats;
285 enum ena_adapter_state state;
289 enum ena_regs_reset_reason_types reset_reason;
291 struct rte_timer timer_wd;
292 uint64_t timestamp_wd;
293 uint64_t keep_alive_timeout;
295 struct ena_stats_dev dev_stats;
296 struct ena_stats_eni eni_stats;
297 struct ena_admin_basic_stats basic_stats;
299 u32 indirect_table[ENA_RX_RSS_TABLE_SIZE];
301 uint32_t all_aenq_groups;
302 uint32_t active_aenq_groups;
306 bool use_large_llq_hdr;
308 uint32_t last_tx_comp_qid;
309 uint64_t missing_tx_completion_to;
310 uint64_t missing_tx_completion_budget;
311 uint64_t tx_cleanup_stall_delay;
313 uint64_t memzone_cnt;
316 int ena_mp_indirect_table_set(struct ena_adapter *adapter);
317 int ena_mp_indirect_table_get(struct ena_adapter *adapter,
318 uint32_t *indirect_table);
319 int ena_rss_reta_update(struct rte_eth_dev *dev,
320 struct rte_eth_rss_reta_entry64 *reta_conf,
322 int ena_rss_reta_query(struct rte_eth_dev *dev,
323 struct rte_eth_rss_reta_entry64 *reta_conf,
325 int ena_rss_hash_update(struct rte_eth_dev *dev,
326 struct rte_eth_rss_conf *rss_conf);
327 int ena_rss_hash_conf_get(struct rte_eth_dev *dev,
328 struct rte_eth_rss_conf *rss_conf);
329 int ena_rss_configure(struct ena_adapter *adapter);
331 #endif /* _ENA_ETHDEV_H_ */