#include <mlx5_glue.h>
#include <mlx5_devx_cmds.h>
#include <mlx5_prm.h>
+#include <mlx5_nl.h>
+#include <mlx5_common_mp.h>
+#include <mlx5_common_mr.h>
#include "mlx5_defs.h"
#include "mlx5_utils.h"
-#include "mlx5_mr.h"
#include "mlx5_autoconf.h"
-enum {
- PCI_VENDOR_ID_MELLANOX = 0x15b3,
-};
-
-enum {
- PCI_DEVICE_ID_MELLANOX_CONNECTX4 = 0x1013,
- PCI_DEVICE_ID_MELLANOX_CONNECTX4VF = 0x1014,
- PCI_DEVICE_ID_MELLANOX_CONNECTX4LX = 0x1015,
- PCI_DEVICE_ID_MELLANOX_CONNECTX4LXVF = 0x1016,
- PCI_DEVICE_ID_MELLANOX_CONNECTX5 = 0x1017,
- PCI_DEVICE_ID_MELLANOX_CONNECTX5VF = 0x1018,
- PCI_DEVICE_ID_MELLANOX_CONNECTX5EX = 0x1019,
- PCI_DEVICE_ID_MELLANOX_CONNECTX5EXVF = 0x101a,
- PCI_DEVICE_ID_MELLANOX_CONNECTX5BF = 0xa2d2,
- PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF = 0xa2d3,
- PCI_DEVICE_ID_MELLANOX_CONNECTX6 = 0x101b,
- PCI_DEVICE_ID_MELLANOX_CONNECTX6VF = 0x101c,
- PCI_DEVICE_ID_MELLANOX_CONNECTX6DX = 0x101d,
- PCI_DEVICE_ID_MELLANOX_CONNECTX6DXVF = 0x101e,
-};
-
-/* Request types for IPC. */
-enum mlx5_mp_req_type {
- MLX5_MP_REQ_VERBS_CMD_FD = 1,
- MLX5_MP_REQ_CREATE_MR,
- MLX5_MP_REQ_START_RXTX,
- MLX5_MP_REQ_STOP_RXTX,
- MLX5_MP_REQ_QUEUE_STATE_MODIFY,
-};
-
-struct mlx5_mp_arg_queue_state_modify {
- uint8_t is_wq; /* Set if WQ. */
- uint16_t queue_id; /* DPDK queue ID. */
- enum ibv_wq_state state; /* WQ requested state. */
-};
-/* Pameters for IPC. */
-struct mlx5_mp_param {
- enum mlx5_mp_req_type type;
- int port_id;
- int result;
- RTE_STD_C11
- union {
- uintptr_t addr; /* MLX5_MP_REQ_CREATE_MR */
- struct mlx5_mp_arg_queue_state_modify state_modify;
- /* MLX5_MP_REQ_QUEUE_STATE_MODIFY */
- } args;
+enum mlx5_ipool_index {
+#ifdef HAVE_IBV_FLOW_DV_SUPPORT
+ MLX5_IPOOL_DECAP_ENCAP = 0, /* Pool for encap/decap resource. */
+ MLX5_IPOOL_PUSH_VLAN, /* Pool for push vlan resource. */
+ MLX5_IPOOL_TAG, /* Pool for tag resource. */
+ MLX5_IPOOL_PORT_ID, /* Pool for port id resource. */
+ MLX5_IPOOL_JUMP, /* Pool for jump resource. */
+#endif
+ MLX5_IPOOL_MTR, /* Pool for meter resource. */
+ MLX5_IPOOL_MCP, /* Pool for metadata resource. */
+ MLX5_IPOOL_HRXQ, /* Pool for hrxq resource. */
+ MLX5_IPOOL_MLX5_FLOW, /* Pool for mlx5 flow handle. */
+ MLX5_IPOOL_RTE_FLOW, /* Pool for rte_flow. */
+ MLX5_IPOOL_MAX,
};
-/** Request timeout for IPC. */
-#define MLX5_MP_REQ_TIMEOUT_SEC 5
-
/** Key string for IPC. */
#define MLX5_MP_NAME "net_mlx5_mp"
-/* Recognized Infiniband device physical port name types. */
-enum mlx5_phys_port_name_type {
- MLX5_PHYS_PORT_NAME_TYPE_NOTSET = 0, /* Not set. */
- MLX5_PHYS_PORT_NAME_TYPE_LEGACY, /* before kernel ver < 5.0 */
- MLX5_PHYS_PORT_NAME_TYPE_UPLINK, /* p0, kernel ver >= 5.0 */
- MLX5_PHYS_PORT_NAME_TYPE_PFVF, /* pf0vf0, kernel ver >= 5.0 */
- MLX5_PHYS_PORT_NAME_TYPE_UNKNOWN, /* Unrecognized. */
-};
-
-/** Switch information returned by mlx5_nl_switch_info(). */
-struct mlx5_switch_info {
- uint32_t master:1; /**< Master device. */
- uint32_t representor:1; /**< Representor device. */
- enum mlx5_phys_port_name_type name_type; /** < Port name type. */
- int32_t pf_num; /**< PF number (valid for pfxvfx format only). */
- int32_t port_name; /**< Representor port name. */
- uint64_t switch_id; /**< Switch identifier. */
-};
LIST_HEAD(mlx5_dev_list, mlx5_ibv_shared);
/* Index in the device counters table. */
uint16_t dev_table_idx[MLX5_MAX_XSTATS];
uint64_t base[MLX5_MAX_XSTATS];
+ uint64_t xstats[MLX5_MAX_XSTATS];
+ uint64_t hw_stats[MLX5_MAX_XSTATS];
struct mlx5_counter_ctrl info[MLX5_MAX_XSTATS];
};
struct mlx5_stats_ctrl {
/* Base for imissed counter. */
uint64_t imissed_base;
+ uint64_t imissed;
};
-/* Flow list . */
-TAILQ_HEAD(mlx5_flows, rte_flow);
-
/* Default PMD specific parameter value. */
#define MLX5_ARG_UNSET (-1)
struct {
unsigned int enabled:1; /* Whether MPRQ is enabled. */
unsigned int stride_num_n; /* Number of strides. */
+ unsigned int stride_size_n; /* Size of a stride. */
unsigned int min_stride_size_n; /* Min size of a stride. */
unsigned int max_stride_size_n; /* Max size of a stride. */
unsigned int max_memcpy_len;
unsigned int tso_max_payload_sz; /* Maximum TCP payload for TSO. */
unsigned int ind_table_max_size; /* Maximum indirection table size. */
unsigned int max_dump_files_num; /* Maximum dump files per queue. */
+ unsigned int log_hp_size; /* Single hairpin queue data size in total. */
int txqs_inline; /* Queue number threshold for inlining. */
int txq_inline_min; /* Minimal amount of data bytes to inline. */
int txq_inline_max; /* Max packet size for inlining with SEND. */
MLX5_VERBS_ALLOC_TYPE_RX_QUEUE,
};
-/* VLAN netdev for VLAN workaround. */
-struct mlx5_vlan_dev {
- uint32_t refcnt;
- uint32_t ifindex; /**< Own interface index. */
-};
-
/* Structure for VF VLAN workaround. */
struct mlx5_vf_vlan {
uint32_t tag:12;
uint32_t created:1;
};
-/*
- * Array of VLAN devices created on the base of VF
- * used for workaround in virtual environments.
- */
-struct mlx5_vlan_vmwa_context {
- int nl_socket;
- uint32_t nl_sn;
- uint32_t vf_ifindex;
- struct rte_eth_dev *dev;
- struct mlx5_vlan_dev vlan_dev[4096];
-};
-
/**
* Verbs allocator needs a context to know in the callback which kind of
* resources it is allocating.
const void *obj; /* Pointer to the DPDK object. */
};
-LIST_HEAD(mlx5_mr_list, mlx5_mr);
-
/* Flow drop context necessary due to Verbs API. */
struct mlx5_drop {
struct mlx5_hrxq *hrxq; /* Hash Rx queue queue. */
#define MLX5_COUNTERS_PER_POOL 512
#define MLX5_MAX_PENDING_QUERIES 4
+#define MLX5_CNT_CONTAINER_RESIZE 64
+/*
+ * The pool index and offset of counter in the pool array makes up the
+ * counter index. In case the counter is from pool 0 and offset 0, it
+ * should plus 1 to avoid index 0, since 0 means invalid counter index
+ * currently.
+ */
+#define MLX5_MAKE_CNT_IDX(pi, offset) \
+ ((pi) * MLX5_COUNTERS_PER_POOL + (offset) + 1)
+#define MLX5_CNT_TO_CNT_EXT(pool, cnt) (&((struct mlx5_flow_counter_ext *) \
+ ((pool) + 1))[((cnt) - (pool)->counters_raw)])
+#define MLX5_GET_POOL_CNT_EXT(pool, offset) \
+ (&((struct mlx5_flow_counter_ext *) \
+ ((pool) + 1))[offset])
struct mlx5_flow_counter_pool;
uint64_t bytes;
};
-/* Counters information. */
+/* Generic counters information. */
struct mlx5_flow_counter {
TAILQ_ENTRY(mlx5_flow_counter) next;
/**< Pointer to the next flow counter structure. */
+ union {
+ uint64_t hits; /**< Reset value of hits packets. */
+ int64_t query_gen; /**< Generation of the last release. */
+ };
+ uint64_t bytes; /**< Reset value of bytes. */
+ void *action; /**< Pointer to the dv action. */
+};
+
+/* Extend counters information for none batch counters. */
+struct mlx5_flow_counter_ext {
uint32_t shared:1; /**< Share counter ID with other flow rules. */
uint32_t batch: 1;
/**< Whether the counter was allocated by batch command. */
uint32_t ref_cnt:30; /**< Reference counter. */
- uint32_t id; /**< Counter ID. */
+ uint32_t id; /**< User counter ID. */
union { /**< Holds the counters for the rule. */
#if defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42)
struct ibv_counter_set *cs;
struct ibv_counters *cs;
#endif
struct mlx5_devx_obj *dcs; /**< Counter Devx object. */
- struct mlx5_flow_counter_pool *pool; /**< The counter pool. */
- };
- union {
- uint64_t hits; /**< Reset value of hits packets. */
- int64_t query_gen; /**< Generation of the last release. */
};
- uint64_t bytes; /**< Reset value of bytes. */
- void *action; /**< Pointer to the dv action. */
};
+
TAILQ_HEAD(mlx5_counters, mlx5_flow_counter);
-/* Counter pool structure - query is in pool resolution. */
+/* Generic counter pool structure - query is in pool resolution. */
struct mlx5_flow_counter_pool {
TAILQ_ENTRY(mlx5_flow_counter_pool) next;
struct mlx5_counters counters; /* Free counter list. */
rte_atomic64_t a64_dcs;
};
/* The devx object of the minimum counter ID. */
- rte_atomic64_t query_gen;
- uint32_t n_counters: 16; /* Number of devx allocated counters. */
+ rte_atomic64_t start_query_gen; /* Query start round. */
+ rte_atomic64_t end_query_gen; /* Query end round. */
+ uint32_t index; /* Pool index in container. */
rte_spinlock_t sl; /* The pool lock. */
struct mlx5_counter_stats_raw *raw;
struct mlx5_counter_stats_raw *raw_hw; /* The raw on HW working. */
- struct mlx5_flow_counter counters_raw[]; /* The pool counters memory. */
+ struct mlx5_flow_counter counters_raw[MLX5_COUNTERS_PER_POOL];
+ /* The pool counters memory. */
};
struct mlx5_counter_stats_raw;
struct ibv_device_attr_ex device_attr; /* Device properties. */
LIST_ENTRY(mlx5_ibv_shared) mem_event_cb;
/**< Called by memory event callback. */
- struct {
- uint32_t dev_gen; /* Generation number to flush local caches. */
- rte_rwlock_t rwlock; /* MR Lock. */
- struct mlx5_mr_btree cache; /* Global MR cache table. */
- struct mlx5_mr_list mr_list; /* Registered MR list. */
- struct mlx5_mr_list mr_free_list; /* Freed MR list. */
- } mr;
+ struct mlx5_mr_share_cache share_cache;
/* Shared DV/DR flow data section. */
pthread_mutex_t dv_mutex; /* DV context mutex. */
uint32_t dv_meta_mask; /* flow META metadata supported mask. */
uint32_t dv_regc0_mask; /* available bits of metatada reg_c[0]. */
uint32_t dv_refcnt; /* DV/DR data reference counter. */
void *fdb_domain; /* FDB Direct Rules name space handle. */
- struct mlx5_flow_tbl_resource *fdb_mtr_sfx_tbl;
- /* FDB meter suffix rules table. */
void *rx_domain; /* RX Direct Rules name space handle. */
- struct mlx5_flow_tbl_resource *rx_mtr_sfx_tbl;
- /* RX meter suffix rules table. */
void *tx_domain; /* TX Direct Rules name space handle. */
- struct mlx5_flow_tbl_resource *tx_mtr_sfx_tbl;
- /* TX meter suffix rules table. */
struct mlx5_hlist *flow_tbls;
/* Direct Rules tables for FDB, NIC TX+RX */
void *esw_drop_action; /* Pointer to DR E-Switch drop action. */
void *pop_vlan_action; /* Pointer to DR pop VLAN action. */
- LIST_HEAD(encap_decap, mlx5_flow_dv_encap_decap_resource) encaps_decaps;
+ uint32_t encaps_decaps; /* Encap/decap action indexed memory list. */
LIST_HEAD(modify_cmd, mlx5_flow_dv_modify_hdr_resource) modify_cmds;
struct mlx5_hlist *tag_table;
- LIST_HEAD(port_id_action_list, mlx5_flow_dv_port_id_action_resource)
- port_id_action_list; /* List of port ID actions. */
- LIST_HEAD(push_vlan_action_list, mlx5_flow_dv_push_vlan_action_resource)
- push_vlan_action_list; /* List of push VLAN actions. */
+ uint32_t port_id_action_list; /* List of port ID actions. */
+ uint32_t push_vlan_action_list; /* List of push VLAN actions. */
struct mlx5_flow_counter_mng cmng; /* Counters management structure. */
+ struct mlx5_indexed_pool *ipool[MLX5_IPOOL_MAX];
+ /* Memory Pool for mlx5 flow resources. */
/* Shared interrupt handler section. */
pthread_mutex_t intr_mutex; /* Interrupt config mutex. */
uint32_t intr_cnt; /* Interrupt handler reference counter. */
unsigned int (*reta_idx)[]; /* RETA index table. */
unsigned int reta_idx_n; /* RETA index size. */
struct mlx5_drop drop_queue; /* Flow drop queues. */
- struct mlx5_flows flows; /* RTE Flow rules. */
- struct mlx5_flows ctrl_flows; /* Control flow rules. */
+ uint32_t flows; /* RTE Flow rules. */
+ uint32_t ctrl_flows; /* Control flow rules. */
+ void *inter_flows; /* Intermediate resources for flow creation. */
+ void *rss_desc; /* Intermediate rss description resources. */
+ int flow_idx; /* Intermediate device flow index. */
+ int flow_nested_idx; /* Intermediate device flow index, nested. */
LIST_HEAD(rxq, mlx5_rxq_ctrl) rxqsctrl; /* DPDK Rx queues. */
LIST_HEAD(rxqobj, mlx5_rxq_obj) rxqsobj; /* Verbs/DevX Rx queues. */
- LIST_HEAD(hrxq, mlx5_hrxq) hrxqs; /* Verbs Hash Rx queues. */
+ uint32_t hrxqs; /* Verbs Hash Rx queues. */
LIST_HEAD(txq, mlx5_txq_ctrl) txqsctrl; /* DPDK Tx queues. */
LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */
/* Indirection tables. */
/* Context for Verbs allocator. */
int nl_socket_rdma; /* Netlink socket (NETLINK_RDMA). */
int nl_socket_route; /* Netlink socket (NETLINK_ROUTE). */
- uint32_t nl_sn; /* Netlink message sequence number. */
LIST_HEAD(dbrpage, mlx5_devx_dbr_page) dbrpgs; /* Door-bell pages. */
- struct mlx5_vlan_vmwa_context *vmwa_context; /* VLAN WA context. */
+ struct mlx5_nl_vlan_vmwa_context *vmwa_context; /* VLAN WA context. */
struct mlx5_flow_id_pool *qrss_id_pool;
struct mlx5_hlist *mreg_cp_tbl;
/* Hash table of Rx metadata register copy table. */
/* UAR same-page access control required in 32bit implementations. */
#endif
uint8_t skip_default_rss_reta; /* Skip configuration of default reta. */
+ uint8_t fdb_def_rule; /* Whether fdb jump to table 1 is configured. */
+ struct mlx5_mp_id mp_id; /* ID of a multi-process process */
+ LIST_HEAD(fdir, mlx5_fdir_flow) fdir_flows; /* fdir flows. */
};
#define PORT_ID(priv) ((priv)->dev_data->port_id)
struct mlx5_switch_info *info);
void mlx5_sysfs_check_switch_info(bool device_dir,
struct mlx5_switch_info *switch_info);
-void mlx5_nl_check_switch_info(bool nun_vf_set,
- struct mlx5_switch_info *switch_info);
void mlx5_translate_port_name(const char *port_name_in,
struct mlx5_switch_info *port_info_out);
void mlx5_intr_callback_unregister(const struct rte_intr_handle *handle,
void mlx5_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index);
int mlx5_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac,
uint32_t index, uint32_t vmdq);
+struct mlx5_nl_vlan_vmwa_context *mlx5_vlan_vmwa_init
+ (struct rte_eth_dev *dev, uint32_t ifindex);
int mlx5_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr);
int mlx5_set_mc_addr_list(struct rte_eth_dev *dev,
struct rte_ether_addr *mc_addr_set,
int mlx5_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
void mlx5_vlan_strip_queue_set(struct rte_eth_dev *dev, uint16_t queue, int on);
int mlx5_vlan_offload_set(struct rte_eth_dev *dev, int mask);
+void mlx5_vlan_vmwa_exit(struct mlx5_nl_vlan_vmwa_context *ctx);
+void mlx5_vlan_vmwa_release(struct rte_eth_dev *dev,
+ struct mlx5_vf_vlan *vf_vlan);
+void mlx5_vlan_vmwa_acquire(struct rte_eth_dev *dev,
+ struct mlx5_vf_vlan *vf_vlan);
/* mlx5_trigger.c */
struct rte_flow_error *error);
int mlx5_flow_destroy(struct rte_eth_dev *dev, struct rte_flow *flow,
struct rte_flow_error *error);
-void mlx5_flow_list_flush(struct rte_eth_dev *dev, struct mlx5_flows *list);
+void mlx5_flow_list_flush(struct rte_eth_dev *dev, uint32_t *list, bool active);
int mlx5_flow_flush(struct rte_eth_dev *dev, struct rte_flow_error *error);
int mlx5_flow_query(struct rte_eth_dev *dev, struct rte_flow *flow,
const struct rte_flow_action *action, void *data,
enum rte_filter_type filter_type,
enum rte_filter_op filter_op,
void *arg);
-int mlx5_flow_start(struct rte_eth_dev *dev, struct mlx5_flows *list);
-void mlx5_flow_stop(struct rte_eth_dev *dev, struct mlx5_flows *list);
+int mlx5_flow_start(struct rte_eth_dev *dev, uint32_t *list);
+void mlx5_flow_stop(struct rte_eth_dev *dev, uint32_t *list);
+int mlx5_flow_start_default(struct rte_eth_dev *dev);
+void mlx5_flow_stop_default(struct rte_eth_dev *dev);
+void mlx5_flow_alloc_intermediate(struct rte_eth_dev *dev);
+void mlx5_flow_free_intermediate(struct rte_eth_dev *dev);
int mlx5_flow_verify(struct rte_eth_dev *dev);
int mlx5_ctrl_flow_source_queue(struct rte_eth_dev *dev, uint32_t queue);
int mlx5_ctrl_flow_vlan(struct rte_eth_dev *dev,
uint64_t async_id, int status);
void mlx5_set_query_alarm(struct mlx5_ibv_shared *sh);
void mlx5_flow_query_alarm(void *arg);
-struct mlx5_flow_counter *mlx5_counter_alloc(struct rte_eth_dev *dev);
-void mlx5_counter_free(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt);
-int mlx5_counter_query(struct rte_eth_dev *dev, struct mlx5_flow_counter *cnt,
+uint32_t mlx5_counter_alloc(struct rte_eth_dev *dev);
+void mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt);
+int mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt,
bool clear, uint64_t *pkts, uint64_t *bytes);
int mlx5_flow_dev_dump(struct rte_eth_dev *dev, FILE *file,
struct rte_flow_error *error);
+void mlx5_flow_rxq_dynf_metadata_set(struct rte_eth_dev *dev);
/* mlx5_mp.c */
+int mlx5_mp_primary_handle(const struct rte_mp_msg *mp_msg, const void *peer);
+int mlx5_mp_secondary_handle(const struct rte_mp_msg *mp_msg, const void *peer);
void mlx5_mp_req_start_rxtx(struct rte_eth_dev *dev);
void mlx5_mp_req_stop_rxtx(struct rte_eth_dev *dev);
-int mlx5_mp_req_mr_create(struct rte_eth_dev *dev, uintptr_t addr);
-int mlx5_mp_req_verbs_cmd_fd(struct rte_eth_dev *dev);
-int mlx5_mp_req_queue_state_modify(struct rte_eth_dev *dev,
- struct mlx5_mp_arg_queue_state_modify *sm);
-int mlx5_mp_init_primary(void);
-void mlx5_mp_uninit_primary(void);
-int mlx5_mp_init_secondary(void);
-void mlx5_mp_uninit_secondary(void);
/* mlx5_socket.c */
int mlx5_pmd_socket_init(void);
-void mlx5_pmd_socket_uninit(void);
-
-/* mlx5_nl.c */
-
-int mlx5_nl_init(int protocol);
-int mlx5_nl_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac,
- uint32_t index);
-int mlx5_nl_mac_addr_remove(struct rte_eth_dev *dev, struct rte_ether_addr *mac,
- uint32_t index);
-void mlx5_nl_mac_addr_sync(struct rte_eth_dev *dev);
-void mlx5_nl_mac_addr_flush(struct rte_eth_dev *dev);
-int mlx5_nl_promisc(struct rte_eth_dev *dev, int enable);
-int mlx5_nl_allmulti(struct rte_eth_dev *dev, int enable);
-unsigned int mlx5_nl_portnum(int nl, const char *name);
-unsigned int mlx5_nl_ifindex(int nl, const char *name, uint32_t pindex);
-int mlx5_nl_vf_mac_addr_modify(struct rte_eth_dev *dev,
- struct rte_ether_addr *mac, int vf_index);
-int mlx5_nl_switch_info(int nl, unsigned int ifindex,
- struct mlx5_switch_info *info);
-
-struct mlx5_vlan_vmwa_context *mlx5_vlan_vmwa_init(struct rte_eth_dev *dev,
- uint32_t ifindex);
-void mlx5_vlan_vmwa_exit(struct mlx5_vlan_vmwa_context *ctx);
-void mlx5_vlan_vmwa_release(struct rte_eth_dev *dev,
- struct mlx5_vf_vlan *vf_vlan);
-void mlx5_vlan_vmwa_acquire(struct rte_eth_dev *dev,
- struct mlx5_vf_vlan *vf_vlan);
/* mlx5_flow_meter.c */
git.droids-corp.org / dpdk.git / blobdiff