extern struct rte_pci_driver mlx5_driver;
/* Dev ops structs */
-extern const struct eth_dev_ops mlx5_dev_sec_ops;
-extern const struct eth_dev_ops mlx5_dev_ops;
+extern const struct eth_dev_ops mlx5_os_dev_ops;
+extern const struct eth_dev_ops mlx5_os_dev_sec_ops;
+extern const struct eth_dev_ops mlx5_os_dev_ops_isolate;
struct mlx5_counter_ctrl {
/* Name of the counter. */
char dpdk_name[RTE_ETH_XSTATS_NAME_SIZE];
/* Name of the counter on the device table. */
char ctr_name[RTE_ETH_XSTATS_NAME_SIZE];
- uint32_t ib:1; /**< Nonzero for IB counters. */
+ uint32_t dev:1; /**< Nonzero for dev counters. */
};
struct mlx5_xstats_ctrl {
unsigned int dv_esw_en:1; /* Enable E-Switch DV flow. */
unsigned int dv_flow_en:1; /* Enable DV flow. */
unsigned int dv_xmeta_en:2; /* Enable extensive flow metadata. */
+ unsigned int lacp_by_user:1;
+ /* Enable user to manage LACP traffic. */
unsigned int swp:1; /* Tx generic tunnel checksum and TSO offload. */
unsigned int devx:1; /* Whether devx interface is available or not. */
unsigned int dest_tir:1; /* Whether advanced DR API is available. */
unsigned int reclaim_mode:2; /* Memory reclaim mode. */
+ unsigned int rt_timestamp:1; /* realtime timestamp format. */
struct {
unsigned int enabled:1; /* Whether MPRQ is enabled. */
unsigned int stride_num_n; /* Number of strides. */
int txq_inline_min; /* Minimal amount of data bytes to inline. */
int txq_inline_max; /* Max packet size for inlining with SEND. */
int txq_inline_mpw; /* Max packet size for inlining with eMPW. */
+ int tx_pp; /* Timestamp scheduling granularity in nanoseconds. */
+ int tx_skew; /* Tx scheduling skew between WQE and data on wire. */
struct mlx5_hca_attr hca_attr; /* HCA attributes. */
struct mlx5_lro_config lro; /* LRO configuration. */
};
MLX5_CNT_TO_CNT_EXT(pool, MLX5_POOL_GET_CNT((pool), (offset)))
#define MLX5_CNT_TO_AGE(cnt) \
((struct mlx5_age_param *)((cnt) + 1))
+/*
+ * The maximum single counter is 0x800000 as MLX5_CNT_BATCH_OFFSET
+ * defines. The pool size is 512, pool index should never reach
+ * INT16_MAX.
+ */
+#define POOL_IDX_INVALID UINT16_MAX
struct mlx5_flow_counter_pool;
uint64_t bytes;
};
+struct mlx5_flow_counter_pool;
/* Generic counters information. */
struct mlx5_flow_counter {
TAILQ_ENTRY(mlx5_flow_counter) next;
/**< Pointer to the next flow counter structure. */
union {
uint64_t hits; /**< Reset value of hits packets. */
- int64_t query_gen; /**< Generation of the last release. */
+ struct mlx5_flow_counter_pool *pool; /**< Counter pool. */
};
uint64_t bytes; /**< Reset value of bytes. */
void *action; /**< Pointer to the dv action. */
/* Generic counter pool structure - query is in pool resolution. */
struct mlx5_flow_counter_pool {
TAILQ_ENTRY(mlx5_flow_counter_pool) next;
- struct mlx5_counters counters; /* Free counter list. */
+ struct mlx5_counters counters[2]; /* Free counter list. */
union {
struct mlx5_devx_obj *min_dcs;
rte_atomic64_t a64_dcs;
};
/* The devx object of the minimum counter ID. */
- rte_atomic64_t start_query_gen; /* Query start round. */
- rte_atomic64_t end_query_gen; /* Query end round. */
- uint32_t index; /* Pool index in container. */
- uint8_t type; /* Memory type behind the counter array. */
+ uint32_t index:29; /* Pool index in container. */
+ uint32_t type:2; /* Memory type behind the counter array. */
+ volatile uint32_t query_gen:1; /* Query round. */
rte_spinlock_t sl; /* The pool lock. */
struct mlx5_counter_stats_raw *raw;
struct mlx5_counter_stats_raw *raw_hw; /* The raw on HW working. */
struct mlx5_pools_container {
rte_atomic16_t n_valid; /* Number of valid pools. */
uint16_t n; /* Number of pools. */
+ uint16_t last_pool_idx; /* Last used pool index */
+ int min_id; /* The minimum counter ID in the pools. */
+ int max_id; /* The maximum counter ID in the pools. */
rte_spinlock_t resize_sl; /* The resize lock. */
+ rte_spinlock_t csl; /* The counter free list lock. */
+ struct mlx5_counters counters; /* Free counter list. */
struct mlx5_counter_pools pool_list; /* Counter pool list. */
struct mlx5_flow_counter_pool **pools; /* Counter pool array. */
struct mlx5_counter_stats_mem_mng *mem_mng;
LIST_HEAD(stat_raws, mlx5_counter_stats_raw) free_stat_raws;
};
+/* Default miss action resource structure. */
+struct mlx5_flow_default_miss_resource {
+ void *action; /* Pointer to the rdma-core action. */
+ rte_atomic32_t refcnt; /* Default miss action reference counter. */
+};
+
#define MLX5_AGE_EVENT_NEW 1
#define MLX5_AGE_TRIGGER 2
#define MLX5_AGE_SET(age_info, BIT) \
#define MLX5_MAX_TABLES_EXTERNAL (MLX5_MAX_TABLES - 3)
#define MLX5_MAX_TABLES_FDB UINT16_MAX
-#define MLX5_DBR_PAGE_SIZE 4096 /* Must be >= 512. */
-#define MLX5_DBR_SIZE 8
-#define MLX5_DBR_PER_PAGE (MLX5_DBR_PAGE_SIZE / MLX5_DBR_SIZE)
-#define MLX5_DBR_BITMAP_SIZE (MLX5_DBR_PER_PAGE / 64)
-
-struct mlx5_devx_dbr_page {
- /* Door-bell records, must be first member in structure. */
- uint8_t dbrs[MLX5_DBR_PAGE_SIZE];
- LIST_ENTRY(mlx5_devx_dbr_page) next; /* Pointer to the next element. */
- void *umem;
- uint32_t dbr_count; /* Number of door-bell records in use. */
- /* 1 bit marks matching door-bell is in use. */
- uint64_t dbr_bitmap[MLX5_DBR_BITMAP_SIZE];
-};
-
/* ID generation structure. */
struct mlx5_flow_id_pool {
uint32_t *free_arr; /**< Pointer to the a array of free values. */
uint32_t max_id; /**< Maximum id can be allocated from the pool. */
};
+/* Tx pacing queue structure - for Clock and Rearm queues. */
+struct mlx5_txpp_wq {
+ /* Completion Queue related data.*/
+ struct mlx5_devx_obj *cq;
+ struct mlx5dv_devx_umem *cq_umem;
+ union {
+ volatile void *cq_buf;
+ volatile struct mlx5_cqe *cqes;
+ };
+ volatile uint32_t *cq_dbrec;
+ uint32_t cq_ci:24;
+ uint32_t arm_sn:2;
+ /* Send Queue related data.*/
+ struct mlx5_devx_obj *sq;
+ struct mlx5dv_devx_umem *sq_umem;
+ union {
+ volatile void *sq_buf;
+ volatile struct mlx5_wqe *wqes;
+ };
+ uint16_t sq_size; /* Number of WQEs in the queue. */
+ uint16_t sq_ci; /* Next WQE to execute. */
+ volatile uint32_t *sq_dbrec;
+};
+
+/* Tx packet pacing internal timestamp. */
+struct mlx5_txpp_ts {
+ rte_atomic64_t ci_ts;
+ rte_atomic64_t ts;
+};
+
+/* Tx packet pacing structure. */
+struct mlx5_dev_txpp {
+ pthread_mutex_t mutex; /* Pacing create/destroy mutex. */
+ uint32_t refcnt; /* Pacing reference counter. */
+ uint32_t freq; /* Timestamp frequency, Hz. */
+ uint32_t tick; /* Completion tick duration in nanoseconds. */
+ uint32_t test; /* Packet pacing test mode. */
+ int32_t skew; /* Scheduling skew. */
+ uint32_t eqn; /* Event Queue number. */
+ struct rte_intr_handle intr_handle; /* Periodic interrupt. */
+ struct mlx5dv_devx_event_channel *echan; /* Event Channel. */
+ struct mlx5_txpp_wq clock_queue; /* Clock Queue. */
+ struct mlx5_txpp_wq rearm_queue; /* Clock Queue. */
+ struct mlx5dv_pp *pp; /* Packet pacing context. */
+ uint16_t pp_id; /* Packet pacing context index. */
+ uint16_t ts_n; /* Number of captured timestamps. */
+ uint16_t ts_p; /* Pointer to statisticks timestamp. */
+ struct mlx5_txpp_ts *tsa; /* Timestamps sliding window stats. */
+ struct mlx5_txpp_ts ts; /* Cached completion id/timestamp. */
+ uint32_t sync_lost:1; /* ci/timestamp synchronization lost. */
+ /* Statistics counters. */
+ rte_atomic32_t err_miss_int; /* Missed service interrupt. */
+ rte_atomic32_t err_rearm_queue; /* Rearm Queue errors. */
+ rte_atomic32_t err_clock_queue; /* Clock Queue errors. */
+ rte_atomic32_t err_ts_past; /* Timestamp in the past. */
+ rte_atomic32_t err_ts_future; /* Timestamp in the distant future. */
+};
+
+/* Supported flex parser profile ID. */
+enum mlx5_flex_parser_profile_id {
+ MLX5_FLEX_PARSER_ECPRI_0 = 0,
+ MLX5_FLEX_PARSER_MAX = 8,
+};
+
+/* Sample ID information of flex parser structure. */
+struct mlx5_flex_parser_profiles {
+ uint32_t num; /* Actual number of samples. */
+ uint32_t ids[8]; /* Sample IDs for this profile. */
+ uint8_t offset[8]; /* Bytes offset of each parser. */
+ void *obj; /* Flex parser node object. */
+};
+
/*
* Shared Infiniband device context for Master/Representors
* which belong to same IB device with multiple IB ports.
char ibdev_name[DEV_SYSFS_NAME_MAX]; /* SYSFS dev name. */
char ibdev_path[DEV_SYSFS_PATH_MAX]; /* SYSFS dev path for secondary */
struct mlx5_dev_attr device_attr; /* Device properties. */
+ int numa_node; /* Numa node of backing physical device. */
LIST_ENTRY(mlx5_dev_ctx_shared) mem_event_cb;
/**< Called by memory event callback. */
struct mlx5_mr_share_cache share_cache;
+ /* Packet pacing related structure. */
+ struct mlx5_dev_txpp txpp;
/* Shared DV/DR flow data section. */
pthread_mutex_t dv_mutex; /* DV context mutex. */
uint32_t dv_meta_mask; /* flow META metadata supported mask. */
void *fdb_domain; /* FDB Direct Rules name space handle. */
void *rx_domain; /* RX Direct Rules name space handle. */
void *tx_domain; /* TX Direct Rules name space handle. */
+#ifndef RTE_ARCH_64
+ rte_spinlock_t uar_lock_cq; /* CQs share a common distinct UAR */
+ rte_spinlock_t uar_lock[MLX5_UAR_PAGE_NUM_MAX];
+ /* UAR same-page access control required in 32bit implementations. */
+#endif
struct mlx5_hlist *flow_tbls;
/* Direct Rules tables for FDB, NIC TX+RX */
void *esw_drop_action; /* Pointer to DR E-Switch drop action. */
uint32_t port_id_action_list; /* List of port ID actions. */
uint32_t push_vlan_action_list; /* List of push VLAN actions. */
struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
+ struct mlx5_flow_default_miss_resource default_miss;
+ /* Default miss action resource structure. */
struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX];
/* Memory Pool for mlx5 flow resources. */
+ struct mlx5_l3t_tbl *cnt_id_tbl; /* Shared counter lookup table. */
/* Shared interrupt handler section. */
struct rte_intr_handle intr_handle; /* Interrupt handler for device. */
struct rte_intr_handle intr_handle_devx; /* DEVX interrupt handler. */
struct mlx5_devx_obj *tis; /* TIS object. */
struct mlx5_devx_obj *td; /* Transport domain. */
struct mlx5_flow_id_pool *flow_id_pool; /* Flow ID pool. */
+ struct mlx5dv_devx_uar *tx_uar; /* Tx/packer pacing shared UAR. */
+ struct mlx5_flex_parser_profiles fp[MLX5_FLEX_PARSER_MAX];
+ /* Flex parser profiles information. */
struct mlx5_dev_shared_port port[]; /* per device port data array. */
};
unsigned int representor:1; /* Device is a port representor. */
unsigned int master:1; /* Device is a E-Switch master. */
unsigned int dr_shared:1; /* DV/DR data is shared. */
+ unsigned int txpp_en:1; /* Tx packet pacing enabled. */
unsigned int counter_fallback:1; /* Use counter fallback management. */
unsigned int mtr_en:1; /* Whether support meter. */
unsigned int mtr_reg_share:1; /* Whether support meter REG_C share. */
/* Context for Verbs allocator. */
int nl_socket_rdma; /* Netlink socket (NETLINK_RDMA). */
int nl_socket_route; /* Netlink socket (NETLINK_ROUTE). */
- LIST_HEAD(dbrpage, mlx5_devx_dbr_page) dbrpgs; /* Door-bell pages. */
+ struct mlx5_dbr_page_list dbrpgs; /* Door-bell pages. */
struct mlx5_nl_vlan_vmwa_context *vmwa_context; /* VLAN WA context. */
struct mlx5_flow_id_pool *qrss_id_pool;
struct mlx5_hlist *mreg_cp_tbl;
uint8_t mtr_color_reg; /* Meter color match REG_C. */
struct mlx5_mtr_profiles flow_meter_profiles; /* MTR profile list. */
struct mlx5_flow_meters flow_meters; /* MTR list. */
-#ifndef RTE_ARCH_64
- rte_spinlock_t uar_lock_cq; /* CQs share a common distinct UAR */
- rte_spinlock_t uar_lock[MLX5_UAR_PAGE_NUM_MAX];
- /* UAR same-page access control required in 32bit implementations. */
-#endif
uint8_t skip_default_rss_reta; /* Skip configuration of default reta. */
uint8_t fdb_def_rule; /* Whether fdb jump to table 1 is configured. */
struct mlx5_mp_id mp_id; /* ID of a multi-process process */
int mlx5_getenv_int(const char *);
int mlx5_proc_priv_init(struct rte_eth_dev *dev);
-int64_t mlx5_get_dbr(struct rte_eth_dev *dev,
- struct mlx5_devx_dbr_page **dbr_page);
-int32_t mlx5_release_dbr(struct rte_eth_dev *dev, uint32_t umem_id,
- uint64_t offset);
int mlx5_udp_tunnel_port_add(struct rte_eth_dev *dev,
struct rte_eth_udp_tunnel *udp_tunnel);
uint16_t mlx5_eth_find_next(uint16_t port_id, struct rte_pci_device *pci_dev);
int mlx5_dev_check_sibling_config(struct mlx5_priv *priv,
struct mlx5_dev_config *config);
int mlx5_init_once(void);
+int mlx5_dev_configure(struct rte_eth_dev *dev);
+int mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info);
+int mlx5_fw_version_get(struct rte_eth_dev *dev, char *fw_ver, size_t fw_size);
+int mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu);
+int mlx5_hairpin_cap_get(struct rte_eth_dev *dev,
+ struct rte_eth_hairpin_cap *cap);
+bool mlx5_flex_parser_ecpri_exist(struct rte_eth_dev *dev);
+int mlx5_flex_parser_ecpri_alloc(struct rte_eth_dev *dev);
/* mlx5_ethdev.c */
/* mlx5_ethdev_os.c */
int mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE]);
-int mlx5_get_master_ifname(const char *ibdev_path, char (*ifname)[IF_NAMESIZE]);
unsigned int mlx5_ifindex(const struct rte_eth_dev *dev);
int mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr);
int mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu);
/* mlx5_stats.c */
-void mlx5_stats_init(struct rte_eth_dev *dev);
int mlx5_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats);
int mlx5_stats_reset(struct rte_eth_dev *dev);
int mlx5_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *stats,
int mlx5_ctrl_flow(struct rte_eth_dev *dev,
struct rte_flow_item_eth *eth_spec,
struct rte_flow_item_eth *eth_mask);
+int mlx5_flow_lacp_miss(struct rte_eth_dev *dev);
struct rte_flow *mlx5_flow_create_esw_table_zero_flow(struct rte_eth_dev *dev);
int mlx5_flow_create_drop_queue(struct rte_eth_dev *dev);
void mlx5_flow_delete_drop_queue(struct rte_eth_dev *dev);
/* mlx5_os.c */
struct rte_pci_driver;
-const char *mlx5_os_get_ctx_device_name(void *ctx);
-const char *mlx5_os_get_ctx_device_path(void *ctx);
-const char *mlx5_os_get_dev_device_name(void *dev);
-uint32_t mlx5_os_get_umem_id(void *umem);
int mlx5_os_get_dev_attr(void *ctx, struct mlx5_dev_attr *dev_attr);
void mlx5_os_free_shared_dr(struct mlx5_priv *priv);
int mlx5_os_open_device(const struct mlx5_dev_spawn_data *spawn,
struct rte_pci_device *pci_dev);
void mlx5_os_dev_shared_handler_install(struct mlx5_dev_ctx_shared *sh);
void mlx5_os_dev_shared_handler_uninstall(struct mlx5_dev_ctx_shared *sh);
+int mlx5_os_read_dev_stat(struct mlx5_priv *priv,
+ const char *ctr_name, uint64_t *stat);
+int mlx5_os_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats);
+int mlx5_os_get_stats_n(struct rte_eth_dev *dev);
+void mlx5_os_stats_init(struct rte_eth_dev *dev);
+void mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb,
+ mlx5_dereg_mr_t *dereg_mr_cb);
+/* mlx5_txpp.c */
+
+int mlx5_txpp_start(struct rte_eth_dev *dev);
+void mlx5_txpp_stop(struct rte_eth_dev *dev);
+int mlx5_txpp_read_clock(struct rte_eth_dev *dev, uint64_t *timestamp);
+int mlx5_txpp_xstats_get(struct rte_eth_dev *dev,
+ struct rte_eth_xstat *stats,
+ unsigned int n, unsigned int n_used);
+int mlx5_txpp_xstats_reset(struct rte_eth_dev *dev);
+int mlx5_txpp_xstats_get_names(struct rte_eth_dev *dev,
+ struct rte_eth_xstat_name *xstats_names,
+ unsigned int n, unsigned int n_used);
+void mlx5_txpp_interrupt_handler(void *cb_arg);
+
#endif /* RTE_PMD_MLX5_H_ */
git.droids-corp.org / dpdk.git / blobdiff