#include <stdbool.h>
#include <stdint.h>
#include <limits.h>
-#include <net/if.h>
#include <netinet/in.h>
#include <sys/queue.h>
-/* Verbs header. */
-/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
-#ifdef PEDANTIC
-#pragma GCC diagnostic ignored "-Wpedantic"
-#endif
-#include <infiniband/verbs.h>
-#ifdef PEDANTIC
-#pragma GCC diagnostic error "-Wpedantic"
-#endif
-
#include <rte_pci.h>
#include <rte_ether.h>
#include <rte_ethdev_driver.h>
#include <mlx5_glue.h>
#include <mlx5_devx_cmds.h>
#include <mlx5_prm.h>
-#include <mlx5_nl.h>
#include <mlx5_common_mp.h>
#include <mlx5_common_mr.h>
};
extern struct mlx5_shared_data *mlx5_shared_data;
-extern struct rte_pci_driver mlx5_driver;
/* Dev ops structs */
extern const struct eth_dev_ops mlx5_os_dev_ops;
unsigned int devx:1; /* Whether devx interface is available or not. */
unsigned int dest_tir:1; /* Whether advanced DR API is available. */
unsigned int reclaim_mode:2; /* Memory reclaim mode. */
+ unsigned int rt_timestamp:1; /* realtime timestamp format. */
+ unsigned int sys_mem_en:1; /* The default memory allocator. */
+ unsigned int decap_en:1; /* Whether decap will be used or not. */
struct {
unsigned int enabled:1; /* Whether MPRQ is enabled. */
unsigned int stride_num_n; /* Number of strides. */
struct mlx5_flow_counter_ext {
uint32_t shared:1; /**< Share counter ID with other flow rules. */
uint32_t batch: 1;
+ uint32_t skipped:1; /* This counter is skipped or not. */
/**< Whether the counter was allocated by batch command. */
- uint32_t ref_cnt:30; /**< Reference counter. */
+ uint32_t ref_cnt:29; /**< Reference counter. */
uint32_t id; /**< User counter ID. */
union { /**< Holds the counters for the rule. */
#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
rte_atomic64_t a64_dcs;
};
/* The devx object of the minimum counter ID. */
- uint32_t index:29; /* Pool index in container. */
+ uint32_t index:28; /* Pool index in container. */
uint32_t type:2; /* Memory type behind the counter array. */
+ uint32_t skip_cnt:1; /* Pool contains skipped counter. */
volatile uint32_t query_gen:1; /* Query round. */
rte_spinlock_t sl; /* The pool lock. */
struct mlx5_counter_stats_raw *raw;
uint32_t max_id; /**< Maximum id can be allocated from the pool. */
};
+/* Tx pacing queue structure - for Clock and Rearm queues. */
+struct mlx5_txpp_wq {
+ /* Completion Queue related data.*/
+ struct mlx5_devx_obj *cq;
+ void *cq_umem;
+ union {
+ volatile void *cq_buf;
+ volatile struct mlx5_cqe *cqes;
+ };
+ volatile uint32_t *cq_dbrec;
+ uint32_t cq_ci:24;
+ uint32_t arm_sn:2;
+ /* Send Queue related data.*/
+ struct mlx5_devx_obj *sq;
+ void *sq_umem;
+ union {
+ volatile void *sq_buf;
+ volatile struct mlx5_wqe *wqes;
+ };
+ uint16_t sq_size; /* Number of WQEs in the queue. */
+ uint16_t sq_ci; /* Next WQE to execute. */
+ volatile uint32_t *sq_dbrec;
+};
+
+/* Tx packet pacing internal timestamp. */
+struct mlx5_txpp_ts {
+ rte_atomic64_t ci_ts;
+ rte_atomic64_t ts;
+};
+
+/* Tx packet pacing structure. */
+struct mlx5_dev_txpp {
+ pthread_mutex_t mutex; /* Pacing create/destroy mutex. */
+ uint32_t refcnt; /* Pacing reference counter. */
+ uint32_t freq; /* Timestamp frequency, Hz. */
+ uint32_t tick; /* Completion tick duration in nanoseconds. */
+ uint32_t test; /* Packet pacing test mode. */
+ int32_t skew; /* Scheduling skew. */
+ uint32_t eqn; /* Event Queue number. */
+ struct rte_intr_handle intr_handle; /* Periodic interrupt. */
+ void *echan; /* Event Channel. */
+ struct mlx5_txpp_wq clock_queue; /* Clock Queue. */
+ struct mlx5_txpp_wq rearm_queue; /* Clock Queue. */
+ void *pp; /* Packet pacing context. */
+ uint16_t pp_id; /* Packet pacing context index. */
+ uint16_t ts_n; /* Number of captured timestamps. */
+ uint16_t ts_p; /* Pointer to statisticks timestamp. */
+ struct mlx5_txpp_ts *tsa; /* Timestamps sliding window stats. */
+ struct mlx5_txpp_ts ts; /* Cached completion id/timestamp. */
+ uint32_t sync_lost:1; /* ci/timestamp synchronization lost. */
+ /* Statistics counters. */
+ rte_atomic32_t err_miss_int; /* Missed service interrupt. */
+ rte_atomic32_t err_rearm_queue; /* Rearm Queue errors. */
+ rte_atomic32_t err_clock_queue; /* Clock Queue errors. */
+ rte_atomic32_t err_ts_past; /* Timestamp in the past. */
+ rte_atomic32_t err_ts_future; /* Timestamp in the distant future. */
+};
+
+/* Supported flex parser profile ID. */
+enum mlx5_flex_parser_profile_id {
+ MLX5_FLEX_PARSER_ECPRI_0 = 0,
+ MLX5_FLEX_PARSER_MAX = 8,
+};
+
+/* Sample ID information of flex parser structure. */
+struct mlx5_flex_parser_profiles {
+ uint32_t num; /* Actual number of samples. */
+ uint32_t ids[8]; /* Sample IDs for this profile. */
+ uint8_t offset[8]; /* Bytes offset of each parser. */
+ void *obj; /* Flex parser node object. */
+};
+
/*
* Shared Infiniband device context for Master/Representors
* which belong to same IB device with multiple IB ports.
char ibdev_name[DEV_SYSFS_NAME_MAX]; /* SYSFS dev name. */
char ibdev_path[DEV_SYSFS_PATH_MAX]; /* SYSFS dev path for secondary */
struct mlx5_dev_attr device_attr; /* Device properties. */
+ int numa_node; /* Numa node of backing physical device. */
LIST_ENTRY(mlx5_dev_ctx_shared) mem_event_cb;
/**< Called by memory event callback. */
struct mlx5_mr_share_cache share_cache;
+ /* Packet pacing related structure. */
+ struct mlx5_dev_txpp txpp;
/* Shared DV/DR flow data section. */
pthread_mutex_t dv_mutex; /* DV context mutex. */
uint32_t dv_meta_mask; /* flow META metadata supported mask. */
void *esw_drop_action; /* Pointer to DR E-Switch drop action. */
void *pop_vlan_action; /* Pointer to DR pop VLAN action. */
uint32_t encaps_decaps; /* Encap/decap action indexed memory list. */
- LIST_HEAD(modify_cmd, mlx5_flow_dv_modify_hdr_resource) modify_cmds;
+ struct mlx5_hlist *modify_cmds;
struct mlx5_hlist *tag_table;
uint32_t port_id_action_list; /* List of port ID actions. */
uint32_t push_vlan_action_list; /* List of push VLAN actions. */
struct mlx5_devx_obj *tis; /* TIS object. */
struct mlx5_devx_obj *td; /* Transport domain. */
struct mlx5_flow_id_pool *flow_id_pool; /* Flow ID pool. */
+ void *tx_uar; /* Tx/packet pacing shared UAR. */
+ struct mlx5_flex_parser_profiles fp[MLX5_FLEX_PARSER_MAX];
+ /* Flex parser profiles information. */
+ void *devx_rx_uar; /* DevX UAR for Rx. */
struct mlx5_dev_shared_port port[]; /* per device port data array. */
};
#define MLX5_PROC_PRIV(port_id) \
((struct mlx5_proc_priv *)rte_eth_devices[port_id].process_private)
+/* HW objects operations structure. */
+struct mlx5_obj_ops {
+ int (*rxq_obj_modify_vlan_strip)(struct mlx5_rxq_obj *rxq_obj, int on);
+};
+
struct mlx5_priv {
struct rte_eth_dev_data *dev_data; /* Pointer to device data. */
struct mlx5_dev_ctx_shared *sh; /* Shared device context. */
unsigned int representor:1; /* Device is a port representor. */
unsigned int master:1; /* Device is a E-Switch master. */
unsigned int dr_shared:1; /* DV/DR data is shared. */
+ unsigned int txpp_en:1; /* Tx packet pacing enabled. */
unsigned int counter_fallback:1; /* Use counter fallback management. */
unsigned int mtr_en:1; /* Whether support meter. */
unsigned int mtr_reg_share:1; /* Whether support meter REG_C share. */
void *rss_desc; /* Intermediate rss description resources. */
int flow_idx; /* Intermediate device flow index. */
int flow_nested_idx; /* Intermediate device flow index, nested. */
+ struct mlx5_obj_ops *obj_ops; /* HW objects operations. */
LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
uint32_t hrxqs; /* Verbs Hash Rx queues. */
LIST_HEAD(ind_tables, mlx5_ind_table_obj) ind_tbls;
/* Pointer to next element. */
rte_atomic32_t refcnt; /**< Reference counter. */
- struct ibv_flow_action *verbs_action;
/**< Verbs modify header action object. */
uint8_t ft_type; /**< Flow table type, Rx or Tx. */
uint8_t max_lro_msg_size;
void mlx5_set_metadata_mask(struct rte_eth_dev *dev);
int mlx5_dev_check_sibling_config(struct mlx5_priv *priv,
struct mlx5_dev_config *config);
-int mlx5_init_once(void);
int mlx5_dev_configure(struct rte_eth_dev *dev);
int mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info);
int mlx5_fw_version_get(struct rte_eth_dev *dev, char *fw_ver, size_t fw_size);
int mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu);
int mlx5_hairpin_cap_get(struct rte_eth_dev *dev,
struct rte_eth_hairpin_cap *cap);
+bool mlx5_flex_parser_ecpri_exist(struct rte_eth_dev *dev);
+int mlx5_flex_parser_ecpri_alloc(struct rte_eth_dev *dev);
/* mlx5_ethdev.c */
int mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu);
int mlx5_hairpin_cap_get(struct rte_eth_dev *dev,
struct rte_eth_hairpin_cap *cap);
+eth_rx_burst_t mlx5_select_rx_function(struct rte_eth_dev *dev);
+struct mlx5_priv *mlx5_port_to_eswitch_info(uint16_t port, bool valid);
+struct mlx5_priv *mlx5_dev_to_eswitch_info(struct rte_eth_dev *dev);
+int mlx5_dev_configure_rss_reta(struct rte_eth_dev *dev);
/* mlx5_ethdev_os.c */
-int mlx5_get_ifname(const struct rte_eth_dev *dev, char (*ifname)[IF_NAMESIZE]);
unsigned int mlx5_ifindex(const struct rte_eth_dev *dev);
-int mlx5_ifreq(const struct rte_eth_dev *dev, int req, struct ifreq *ifr);
+int mlx5_get_mac(struct rte_eth_dev *dev, uint8_t (*mac)[RTE_ETHER_ADDR_LEN]);
int mlx5_get_mtu(struct rte_eth_dev *dev, uint16_t *mtu);
-int mlx5_set_flags(struct rte_eth_dev *dev, unsigned int keep,
- unsigned int flags);
int mlx5_set_mtu(struct rte_eth_dev *dev, uint16_t mtu);
int mlx5_read_clock(struct rte_eth_dev *dev, uint64_t *clock);
int mlx5_link_update(struct rte_eth_dev *dev, int wait_to_complete);
-int mlx5_force_link_status_change(struct rte_eth_dev *dev, int status);
int mlx5_dev_get_flow_ctrl(struct rte_eth_dev *dev,
struct rte_eth_fc_conf *fc_conf);
int mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev,
struct rte_eth_fc_conf *fc_conf);
-void mlx5_dev_link_status_handler(void *arg);
void mlx5_dev_interrupt_handler(void *arg);
void mlx5_dev_interrupt_handler_devx(void *arg);
-void mlx5_dev_interrupt_handler_uninstall(struct rte_eth_dev *dev);
-void mlx5_dev_interrupt_handler_install(struct rte_eth_dev *dev);
int mlx5_set_link_down(struct rte_eth_dev *dev);
int mlx5_set_link_up(struct rte_eth_dev *dev);
int mlx5_is_removed(struct rte_eth_dev *dev);
-eth_tx_burst_t mlx5_select_tx_function(struct rte_eth_dev *dev);
-eth_rx_burst_t mlx5_select_rx_function(struct rte_eth_dev *dev);
-struct mlx5_priv *mlx5_port_to_eswitch_info(uint16_t port, bool valid);
-struct mlx5_priv *mlx5_dev_to_eswitch_info(struct rte_eth_dev *dev);
int mlx5_sysfs_switch_info(unsigned int ifindex,
struct mlx5_switch_info *info);
-void mlx5_sysfs_check_switch_info(bool device_dir,
- struct mlx5_switch_info *switch_info);
void mlx5_translate_port_name(const char *port_name_in,
struct mlx5_switch_info *port_info_out);
void mlx5_intr_callback_unregister(const struct rte_intr_handle *handle,
struct rte_eth_dev_module_info *modinfo);
int mlx5_get_module_eeprom(struct rte_eth_dev *dev,
struct rte_dev_eeprom_info *info);
-int mlx5_dev_configure_rss_reta(struct rte_eth_dev *dev);
+int mlx5_os_read_dev_stat(struct mlx5_priv *priv,
+ const char *ctr_name, uint64_t *stat);
+int mlx5_os_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats);
+int mlx5_os_get_stats_n(struct rte_eth_dev *dev);
+void mlx5_os_stats_init(struct rte_eth_dev *dev);
/* mlx5_mac.c */
-int mlx5_get_mac(struct rte_eth_dev *dev, uint8_t (*mac)[RTE_ETHER_ADDR_LEN]);
void mlx5_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index);
int mlx5_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac,
uint32_t index, uint32_t vmdq);
-struct mlx5_nl_vlan_vmwa_context *mlx5_vlan_vmwa_init
- (struct rte_eth_dev *dev, uint32_t ifindex);
int mlx5_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr);
int mlx5_set_mc_addr_list(struct rte_eth_dev *dev,
struct rte_ether_addr *mc_addr_set,
int mlx5_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
void mlx5_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on);
int mlx5_vlan_offload_set(struct rte_eth_dev *dev, int mask);
-void mlx5_vlan_vmwa_exit(struct mlx5_nl_vlan_vmwa_context *ctx);
+
+/* mlx5_vlan_os.c */
+
+void mlx5_vlan_vmwa_exit(void *ctx);
void mlx5_vlan_vmwa_release(struct rte_eth_dev *dev,
struct mlx5_vf_vlan *vf_vlan);
void mlx5_vlan_vmwa_acquire(struct rte_eth_dev *dev,
struct mlx5_vf_vlan *vf_vlan);
+void *mlx5_vlan_vmwa_init(struct rte_eth_dev *dev, uint32_t ifindex);
/* mlx5_trigger.c */
int mlx5_flow_discover_mreg_c(struct rte_eth_dev *eth_dev);
bool mlx5_flow_ext_mreg_supported(struct rte_eth_dev *dev);
-int mlx5_flow_discover_priorities(struct rte_eth_dev *dev);
void mlx5_flow_print(struct rte_flow *flow);
int mlx5_flow_validate(struct rte_eth_dev *dev,
const struct rte_flow_attr *attr,
int mlx5_flow_get_aged_flows(struct rte_eth_dev *dev, void **contexts,
uint32_t nb_contexts, struct rte_flow_error *error);
-/* mlx5_mp.c */
-int mlx5_mp_primary_handle(const struct rte_mp_msg *mp_msg, const void *peer);
-int mlx5_mp_secondary_handle(const struct rte_mp_msg *mp_msg, const void *peer);
-void mlx5_mp_req_start_rxtx(struct rte_eth_dev *dev);
-void mlx5_mp_req_stop_rxtx(struct rte_eth_dev *dev);
+/* mlx5_mp_os.c */
+
+int mlx5_mp_os_primary_handle(const struct rte_mp_msg *mp_msg,
+ const void *peer);
+int mlx5_mp_os_secondary_handle(const struct rte_mp_msg *mp_msg,
+ const void *peer);
+void mlx5_mp_os_req_start_rxtx(struct rte_eth_dev *dev);
+void mlx5_mp_os_req_stop_rxtx(struct rte_eth_dev *dev);
+int mlx5_mp_os_req_queue_control(struct rte_eth_dev *dev, uint16_t queue_id,
+ enum mlx5_mp_req_type req_type);
/* mlx5_socket.c */
struct rte_pci_device *pci_dev);
void mlx5_os_dev_shared_handler_install(struct mlx5_dev_ctx_shared *sh);
void mlx5_os_dev_shared_handler_uninstall(struct mlx5_dev_ctx_shared *sh);
-int mlx5_os_read_dev_stat(struct mlx5_priv *priv,
- const char *ctr_name, uint64_t *stat);
-int mlx5_os_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats);
-int mlx5_os_get_stats_n(struct rte_eth_dev *dev);
-void mlx5_os_stats_init(struct rte_eth_dev *dev);
void mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb,
mlx5_dereg_mr_t *dereg_mr_cb);
+void mlx5_os_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index);
+int mlx5_os_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac,
+ uint32_t index);
+int mlx5_os_vf_mac_addr_modify(struct mlx5_priv *priv, unsigned int iface_idx,
+ struct rte_ether_addr *mac_addr,
+ int vf_index);
+int mlx5_os_set_promisc(struct rte_eth_dev *dev, int enable);
+int mlx5_os_set_allmulti(struct rte_eth_dev *dev, int enable);
+int mlx5_os_set_nonblock_channel_fd(int fd);
+void mlx5_os_mac_addr_flush(struct rte_eth_dev *dev);
+
+/* mlx5_txpp.c */
+
+int mlx5_txpp_start(struct rte_eth_dev *dev);
+void mlx5_txpp_stop(struct rte_eth_dev *dev);
+int mlx5_txpp_read_clock(struct rte_eth_dev *dev, uint64_t *timestamp);
+int mlx5_txpp_xstats_get(struct rte_eth_dev *dev,
+ struct rte_eth_xstat *stats,
+ unsigned int n, unsigned int n_used);
+int mlx5_txpp_xstats_reset(struct rte_eth_dev *dev);
+int mlx5_txpp_xstats_get_names(struct rte_eth_dev *dev,
+ struct rte_eth_xstat_name *xstats_names,
+ unsigned int n, unsigned int n_used);
+void mlx5_txpp_interrupt_handler(void *cb_arg);
+
+/* mlx5_rxtx.c */
+
+eth_tx_burst_t mlx5_select_tx_function(struct rte_eth_dev *dev);
+
#endif /* RTE_PMD_MLX5_H_ */
git.droids-corp.org / dpdk.git / blobdiff