X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5.h;h=ef755ee8cf7c876235f01a53290069189592917e;hb=75315881c23f3aa7c04fb19c50915e64dd97bd38;hp=b2022f3300691ffc564329f9f7fa33d8cd9ce594;hpb=febcac7b46cdff5e3e12dd049ea191963cabbff7;p=dpdk.git diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index b2022f3300..ef755ee8cf 100644 --- a/drivers/net/mlx5/mlx5.h +++ b/drivers/net/mlx5/mlx5.h @@ -33,7 +33,9 @@ #include "mlx5_utils.h" #include "mlx5_os.h" #include "mlx5_autoconf.h" - +#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) +#include "mlx5_dr.h" +#endif #define MLX5_SH(dev) (((struct mlx5_priv *)(dev)->data->dev_private)->sh) @@ -62,7 +64,9 @@ enum mlx5_ipool_index { MLX5_IPOOL_PUSH_VLAN, /* Pool for push vlan resource. */ MLX5_IPOOL_TAG, /* Pool for tag resource. */ MLX5_IPOOL_PORT_ID, /* Pool for port id resource. */ - MLX5_IPOOL_JUMP, /* Pool for jump resource. */ + MLX5_IPOOL_JUMP, /* Pool for SWS jump resource. */ + /* Pool for HWS group. Jump action will be created internally. */ + MLX5_IPOOL_HW_GRP = MLX5_IPOOL_JUMP, MLX5_IPOOL_SAMPLE, /* Pool for sample resource. */ MLX5_IPOOL_DEST_ARRAY, /* Pool for destination array resource. */ MLX5_IPOOL_TUNNEL_ID, /* Pool for tunnel offload context */ @@ -106,6 +110,13 @@ enum mlx5_delay_drop_mode { MLX5_DELAY_DROP_HAIRPIN = RTE_BIT32(1), /* Hairpin queues enable. */ }; +/* The HWS action type root/non-root. */ +enum mlx5_hw_action_flag_type { + MLX5_HW_ACTION_FLAG_ROOT, /* Root action. */ + MLX5_HW_ACTION_FLAG_NONE_ROOT, /* Non-root ation. */ + MLX5_HW_ACTION_FLAG_MAX, /* Maximum action flag. */ +}; + /* Hlist and list callback context. */ struct mlx5_flow_cb_ctx { struct rte_eth_dev *dev; @@ -114,32 +125,50 @@ struct mlx5_flow_cb_ctx { void *data2; }; -/* Device attributes used in mlx5 PMD */ -struct mlx5_dev_attr { - uint64_t device_cap_flags_ex; - int max_qp_wr; - int max_sge; - int max_cq; - int max_qp; - int max_cqe; - uint32_t max_pd; - uint32_t max_mr; - uint32_t max_srq; - uint32_t max_srq_wr; - uint32_t raw_packet_caps; - uint32_t max_rwq_indirection_table_size; - uint32_t max_tso; - uint32_t tso_supported_qpts; - uint64_t flags; - uint64_t comp_mask; - uint32_t sw_parsing_offloads; - uint32_t min_single_stride_log_num_of_bytes; - uint32_t max_single_stride_log_num_of_bytes; - uint32_t min_single_wqe_log_num_of_strides; - uint32_t max_single_wqe_log_num_of_strides; - uint32_t stride_supported_qpts; - uint32_t tunnel_offloads_caps; - char fw_ver[64]; +/* Device capabilities structure which isn't changed in any stage. */ +struct mlx5_dev_cap { + int max_cq; /* Maximum number of supported CQs */ + int max_qp; /* Maximum number of supported QPs. */ + int max_qp_wr; /* Maximum number of outstanding WR on any WQ. */ + int max_sge; + /* Maximum number of s/g per WR for SQ & RQ of QP for non RDMA Read + * operations. + */ + int mps; /* Multi-packet send supported mode. */ + uint32_t vf:1; /* This is a VF. */ + uint32_t sf:1; /* This is a SF. */ + uint32_t txpp_en:1; /* Tx packet pacing is supported. */ + uint32_t mpls_en:1; /* MPLS over GRE/UDP is supported. */ + uint32_t cqe_comp:1; /* CQE compression is supported. */ + uint32_t hw_csum:1; /* Checksum offload is supported. */ + uint32_t hw_padding:1; /* End alignment padding is supported. */ + uint32_t dest_tir:1; /* Whether advanced DR API is available. */ + uint32_t dv_esw_en:1; /* E-Switch DV flow is supported. */ + uint32_t dv_flow_en:1; /* DV flow is supported. */ + uint32_t swp:3; /* Tx generic tunnel checksum and TSO offload. */ + uint32_t hw_vlan_strip:1; /* VLAN stripping is supported. */ + uint32_t scatter_fcs_w_decap_disable:1; + /* HW has bug working with tunnel packet decap and scatter FCS. */ + uint32_t hw_fcs_strip:1; /* FCS stripping is supported. */ + uint32_t rt_timestamp:1; /* Realtime timestamp format. */ + uint32_t lro_supported:1; /* Whether LRO is supported. */ + uint32_t rq_delay_drop_en:1; /* Enable RxQ delay drop. */ + uint32_t tunnel_en:3; + /* Whether tunnel stateless offloads are supported. */ + uint32_t ind_table_max_size; + /* Maximum receive WQ indirection table size. */ + uint32_t tso:1; /* Whether TSO is supported. */ + uint32_t tso_max_payload_sz; /* Maximum TCP payload for TSO. */ + struct { + uint32_t enabled:1; /* Whether MPRQ is enabled. */ + uint32_t log_min_stride_size; /* Log min size of a stride. */ + uint32_t log_max_stride_size; /* Log max size of a stride. */ + uint32_t log_min_stride_num; /* Log min num of strides. */ + uint32_t log_max_stride_num; /* Log max num of strides. */ + uint32_t log_min_stride_wqe_size; + /* Log min WQE size, (size of single stride)*(num of strides).*/ + } mprq; /* Capability for Multi-Packet RQ. */ + char fw_ver[64]; /* Firmware version of this device. */ }; /** Data associated with devices to spawn. */ @@ -215,9 +244,6 @@ struct mlx5_stats_ctrl { uint64_t imissed; }; -#define MLX5_LRO_SUPPORTED(dev) \ - (((struct mlx5_priv *)((dev)->data->dev_private))->config.lro.supported) - /* Maximal size of coalesced segment for LRO is set in chunks of 256 Bytes. */ #define MLX5_LRO_SEG_CHUNK_SIZE 256u @@ -227,76 +253,62 @@ struct mlx5_stats_ctrl { /* Maximal number of segments to split. */ #define MLX5_MAX_RXQ_NSEG (1u << MLX5_MAX_LOG_RQ_SEGS) -/* LRO configurations structure. */ -struct mlx5_lro_config { - uint32_t supported:1; /* Whether LRO is supported. */ - uint32_t timeout; /* User configuration. */ -}; - /* - * Device configuration structure. - * - * Merged configuration from: - * - * - Device capabilities, - * - User device parameters disabled features. + * Port configuration structure. + * User device parameters disabled features. + * This structure contains all configurations coming from devargs which + * oriented to port. When probing again, devargs doesn't have to be compatible + * with primary devargs. It is updated for each port in spawn function. */ -struct mlx5_dev_config { - unsigned int hw_csum:1; /* Checksum offload is supported. */ - unsigned int hw_vlan_strip:1; /* VLAN stripping is supported. */ +struct mlx5_port_config { unsigned int hw_vlan_insert:1; /* VLAN insertion in WQE is supported. */ - unsigned int hw_fcs_strip:1; /* FCS stripping is supported. */ unsigned int hw_padding:1; /* End alignment padding is supported. */ - unsigned int vf:1; /* This is a VF. */ - unsigned int sf:1; /* This is a SF. */ - unsigned int tunnel_en:3; - /* Whether tunnel stateless offloads are supported. */ - unsigned int mpls_en:1; /* MPLS over GRE/UDP is enabled. */ unsigned int cqe_comp:1; /* CQE compression is enabled. */ unsigned int cqe_comp_fmt:3; /* CQE compression format. */ - unsigned int tso:1; /* Whether TSO is supported. */ unsigned int rx_vec_en:1; /* Rx vector is enabled. */ - unsigned int l3_vxlan_en:1; /* Enable L3 VXLAN flow creation. */ - unsigned int vf_nl_en:1; /* Enable Netlink requests in VF mode. */ - unsigned int dv_esw_en:1; /* Enable E-Switch DV flow. */ - unsigned int dv_flow_en:1; /* Enable DV flow. */ - unsigned int dv_xmeta_en:2; /* Enable extensive flow metadata. */ - unsigned int lacp_by_user:1; - /* Enable user to manage LACP traffic. */ - unsigned int swp:3; /* Tx generic tunnel checksum and TSO offload. */ - unsigned int dest_tir:1; /* Whether advanced DR API is available. */ - unsigned int reclaim_mode:2; /* Memory reclaim mode. */ - unsigned int rt_timestamp:1; /* realtime timestamp format. */ - unsigned int decap_en:1; /* Whether decap will be used or not. */ - unsigned int dv_miss_info:1; /* restore packet after partial hw miss */ - unsigned int allow_duplicate_pattern:1; - /* Allow/Prevent the duplicate rules pattern. */ unsigned int std_delay_drop:1; /* Enable standard Rxq delay drop. */ unsigned int hp_delay_drop:1; /* Enable hairpin Rxq delay drop. */ struct { unsigned int enabled:1; /* Whether MPRQ is enabled. */ - unsigned int stride_num_n; /* Number of strides. */ - unsigned int stride_size_n; /* Size of a stride. */ - unsigned int min_stride_size_n; /* Min size of a stride. */ - unsigned int max_stride_size_n; /* Max size of a stride. */ + unsigned int log_stride_num; /* Log number of strides. */ + unsigned int log_stride_size; /* Log size of a stride. */ unsigned int max_memcpy_len; /* Maximum packet size to memcpy Rx packets. */ unsigned int min_rxqs_num; /* Rx queue count threshold to enable MPRQ. */ } mprq; /* Configurations for Multi-Packet RQ. */ int mps; /* Multi-packet send supported mode. */ - unsigned int tso_max_payload_sz; /* Maximum TCP payload for TSO. */ - unsigned int ind_table_max_size; /* Maximum indirection table size. */ unsigned int max_dump_files_num; /* Maximum dump files per queue. */ unsigned int log_hp_size; /* Single hairpin queue data size in total. */ + unsigned int lro_timeout; /* LRO user configuration. */ int txqs_inline; /* Queue number threshold for inlining. */ int txq_inline_min; /* Minimal amount of data bytes to inline. */ int txq_inline_max; /* Max packet size for inlining with SEND. */ int txq_inline_mpw; /* Max packet size for inlining with eMPW. */ +}; + +/* + * Share context device configuration structure. + * User device parameters disabled features. + * This structure updated once for device in mlx5_alloc_shared_dev_ctx() + * function and cannot change even when probing again. + */ +struct mlx5_sh_config { int tx_pp; /* Timestamp scheduling granularity in nanoseconds. */ int tx_skew; /* Tx scheduling skew between WQE and data on wire. */ - struct mlx5_hca_attr hca_attr; /* HCA attributes. */ - struct mlx5_lro_config lro; /* LRO configuration. */ + uint32_t reclaim_mode:2; /* Memory reclaim mode. */ + uint32_t dv_esw_en:1; /* Enable E-Switch DV flow. */ + /* Enable DV flow. 1 means SW steering, 2 means HW steering. */ + unsigned int dv_flow_en:2; + uint32_t dv_xmeta_en:2; /* Enable extensive flow metadata. */ + uint32_t dv_miss_info:1; /* Restore packet after partial hw miss. */ + uint32_t l3_vxlan_en:1; /* Enable L3 VXLAN flow creation. */ + uint32_t vf_nl_en:1; /* Enable Netlink requests in VF mode. */ + uint32_t lacp_by_user:1; /* Enable user to manage LACP traffic. */ + uint32_t decap_en:1; /* Whether decap will be used or not. */ + uint32_t hw_fcs_strip:1; /* FCS stripping is supported. */ + uint32_t allow_duplicate_pattern:1; + /* Allow/Prevent the duplicate rules pattern. */ }; @@ -319,6 +331,27 @@ struct mlx5_lb_ctx { uint16_t refcnt; /* Reference count for representors. */ }; +/* HW steering queue job descriptor type. */ +enum { + MLX5_HW_Q_JOB_TYPE_CREATE, /* Flow create job type. */ + MLX5_HW_Q_JOB_TYPE_DESTROY, /* Flow destroy job type. */ +}; + +/* HW steering flow management job descriptor. */ +struct mlx5_hw_q_job { + uint32_t type; /* Job type. */ + struct rte_flow_hw *flow; /* Flow attached to the job. */ + void *user_data; /* Job user data. */ + uint8_t *encap_data; /* Encap data. */ +}; + +/* HW steering job descriptor LIFO pool. */ +struct mlx5_hw_q { + uint32_t job_idx; /* Free job index. */ + uint32_t size; /* LIFO size. */ + struct mlx5_hw_q_job **job; /* LIFO header. */ +} __rte_cache_aligned; + #define MLX5_COUNTERS_PER_POOL 512 #define MLX5_MAX_PENDING_QUERIES 4 #define MLX5_CNT_CONTAINER_RESIZE 64 @@ -337,6 +370,9 @@ struct mlx5_lb_ctx { #define MLX5_CNT_ARRAY_IDX(pool, cnt) \ ((int)(((uint8_t *)(cnt) - (uint8_t *)((pool) + 1)) / \ MLX5_CNT_LEN(pool))) +#define MLX5_TS_MASK_SECS 8ull +/* timestamp wrapping in seconds, must be power of 2. */ + /* * The pool index and offset of counter in the pool array makes up the * counter index. In case the counter is from pool 0 and offset 0, it @@ -462,8 +498,7 @@ struct mlx5_flow_counter_pool { struct mlx5_counter_stats_mem_mng { LIST_ENTRY(mlx5_counter_stats_mem_mng) next; struct mlx5_counter_stats_raw *raws; - struct mlx5_devx_obj *dm; - void *umem; + struct mlx5_pmd_wrapped_mr wm; }; /* Raw memory structure for the counter statistics values of a pool. */ @@ -494,8 +529,6 @@ struct mlx5_flow_counter_mng { uint8_t pending_queries; uint16_t pool_index; uint8_t query_thread_on; - bool relaxed_ordering_read; - bool relaxed_ordering_write; bool counter_fallback; /* Use counter fallback management. */ LIST_HEAD(mem_mngs, mlx5_counter_stats_mem_mng) mem_mngs; LIST_HEAD(stat_raws, mlx5_counter_stats_raw) free_stat_raws; @@ -530,7 +563,6 @@ struct mlx5_aso_sq { rte_spinlock_t sqsl; struct mlx5_aso_cq cq; struct mlx5_devx_sq sq_obj; - volatile uint64_t *uar_addr; struct mlx5_pmd_mr mr; uint16_t pi; uint32_t head; @@ -605,6 +637,7 @@ struct mlx5_age_info { struct mlx5_dev_shared_port { uint32_t ih_port_id; uint32_t devx_ih_port_id; + uint32_t nl_ih_port_id; /* * Interrupt handler port_id. Used by shared interrupt * handler to find the corresponding rte_eth device @@ -637,6 +670,8 @@ struct mlx5_dev_shared_port { #define MLX5_MTR_TABLE_ID_DROP 2 /* Priority of the meter policy matcher. */ #define MLX5_MTR_POLICY_MATCHER_PRIO 0 +/* Green & yellow color valid for now. */ +#define MLX5_MTR_POLICY_MODE_ALL 0 /* Default policy. */ #define MLX5_MTR_POLICY_MODE_DEF 1 /* Only green color valid. */ @@ -744,6 +779,8 @@ struct mlx5_flow_meter_policy { /* If yellow color policy is skipped. */ uint32_t skip_g:1; /* If green color policy is skipped. */ + uint32_t mark:1; + /* If policy contains mark action. */ rte_spinlock_t sl; uint32_t ref_cnt; /* Use count. */ @@ -822,6 +859,8 @@ struct mlx5_flow_meter_info { uint32_t transfer:1; uint32_t def_policy:1; /* Meter points to default policy. */ + uint32_t color_aware:1; + /* Meter is color aware mode. */ void *drop_rule[MLX5_MTR_DOMAIN_MAX]; /* Meter drop rule in drop table. */ uint32_t drop_cnt; @@ -830,8 +869,10 @@ struct mlx5_flow_meter_info { /**< Use count. */ struct mlx5_indexed_pool *flow_ipool; /**< Index pool for flow id. */ - void *meter_action; + void *meter_action_g; /**< Flow meter action. */ + void *meter_action_y; + /**< Flow meter action for yellow init_color. */ }; /* PPS(packets per second) map to BPS(Bytes per second). @@ -958,7 +999,6 @@ union mlx5_flow_tbl_key { /* Table structure. */ struct mlx5_flow_tbl_resource { void *obj; /**< Pointer to DR table object. */ - uint32_t refcnt; /**< Reference counter. */ }; #define MLX5_MAX_TABLES UINT16_MAX @@ -979,7 +1019,7 @@ struct mlx5_flow_id_pool { uint32_t base_index; /**< The next index that can be used without any free elements. */ uint32_t *curr; /**< Pointer to the index to pop. */ - uint32_t *last; /**< Pointer to the last element in the empty arrray. */ + uint32_t *last; /**< Pointer to the last element in the empty array. */ uint32_t max_id; /**< Maximum id can be allocated from the pool. */ }; @@ -1016,7 +1056,7 @@ struct mlx5_dev_txpp { void *pp; /* Packet pacing context. */ uint16_t pp_id; /* Packet pacing context index. */ uint16_t ts_n; /* Number of captured timestamps. */ - uint16_t ts_p; /* Pointer to statisticks timestamp. */ + uint16_t ts_p; /* Pointer to statistics timestamp. */ struct mlx5_txpp_ts *tsa; /* Timestamps sliding window stats. */ struct mlx5_txpp_ts ts; /* Cached completion id/timestamp. */ uint32_t sync_lost:1; /* ci/timestamp synchronization lost. */ @@ -1120,7 +1160,7 @@ struct mlx5_flex_parser_devx { uint32_t sample_ids[MLX5_GRAPH_NODE_SAMPLE_NUM]; }; -/* Pattern field dscriptor - how to translate flex pattern into samples. */ +/* Pattern field descriptor - how to translate flex pattern into samples. */ __extension__ struct mlx5_flex_pattern_field { uint16_t width:6; @@ -1145,15 +1185,15 @@ struct mlx5_flex_item { struct mlx5_dev_ctx_shared { LIST_ENTRY(mlx5_dev_ctx_shared) next; uint32_t refcnt; - uint32_t devx:1; /* Opened with DV. */ + uint32_t esw_mode:1; /* Whether is E-Switch mode. */ uint32_t flow_hit_aso_en:1; /* Flow Hit ASO is supported. */ uint32_t steering_format_version:4; /* Indicates the device steering logic format. */ uint32_t meter_aso_en:1; /* Flow Meter ASO is supported. */ uint32_t ct_aso_en:1; /* Connection Tracking ASO is supported. */ uint32_t tunnel_header_0_1:1; /* tunnel_header_0_1 is supported. */ + uint32_t tunnel_header_2_3:1; /* tunnel_header_2_3 is supported. */ uint32_t misc5_cap:1; /* misc5 matcher parameter is supported. */ - uint32_t reclaim_mode:1; /* Reclaim memory. */ uint32_t dr_drop_action_en:1; /* Use DR drop action. */ uint32_t drop_action_check_flag:1; /* Check Flag for drop action. */ uint32_t flow_priority_check_flag:1; /* Check Flag for flow priority. */ @@ -1164,23 +1204,27 @@ struct mlx5_dev_ctx_shared { uint32_t tdn; /* Transport Domain number. */ char ibdev_name[MLX5_FS_NAME_MAX]; /* SYSFS dev name. */ char ibdev_path[MLX5_FS_PATH_MAX]; /* SYSFS dev path for secondary */ - struct mlx5_dev_attr device_attr; /* Device properties. */ + struct mlx5_dev_cap dev_cap; /* Device capabilities. */ + struct mlx5_sh_config config; /* Device configuration. */ int numa_node; /* Numa node of backing physical device. */ /* Packet pacing related structure. */ struct mlx5_dev_txpp txpp; /* Shared DV/DR flow data section. */ uint32_t dv_meta_mask; /* flow META metadata supported mask. */ uint32_t dv_mark_mask; /* flow MARK metadata supported mask. */ - uint32_t dv_regc0_mask; /* available bits of metatada reg_c[0]. */ + uint32_t dv_regc0_mask; /* available bits of metadata reg_c[0]. */ void *fdb_domain; /* FDB Direct Rules name space handle. */ void *rx_domain; /* RX Direct Rules name space handle. */ void *tx_domain; /* TX Direct Rules name space handle. */ #ifndef RTE_ARCH_64 - rte_spinlock_t uar_lock_cq; /* CQs share a common distinct UAR */ + rte_spinlock_t uar_lock_cq; /* CQs share a common distinct UAR. */ rte_spinlock_t uar_lock[MLX5_UAR_PAGE_NUM_MAX]; /* UAR same-page access control required in 32bit implementations. */ #endif - struct mlx5_hlist *flow_tbls; + union { + struct mlx5_hlist *flow_tbls; /* SWS flow table. */ + struct mlx5_hlist *groups; /* HWS flow group. */ + }; struct mlx5_flow_tunnel_hub *tunnel_hub; /* Direct Rules tables for FDB, NIC TX+RX */ void *dr_drop_action; /* Pointer to DR drop action, any domain. */ @@ -1201,14 +1245,16 @@ struct mlx5_dev_ctx_shared { /* Shared interrupt handler section. */ struct rte_intr_handle *intr_handle; /* Interrupt handler for device. */ struct rte_intr_handle *intr_handle_devx; /* DEVX interrupt handler. */ + struct rte_intr_handle *intr_handle_nl; /* Netlink interrupt handler. */ void *devx_comp; /* DEVX async comp obj. */ struct mlx5_devx_obj *tis[16]; /* TIS object. */ struct mlx5_devx_obj *td; /* Transport domain. */ struct mlx5_lag lag; /* LAG attributes */ - void *tx_uar; /* Tx/packet pacing shared UAR. */ + struct mlx5_uar tx_uar; /* DevX UAR for Tx and Txpp and ASO SQs. */ + struct mlx5_uar rx_uar; /* DevX UAR for Rx. */ + struct mlx5_proc_priv *pppriv; /* Pointer to primary private process. */ struct mlx5_ecpri_parser_profile ecpri_parser; /* Flex parser profiles information. */ - void *devx_rx_uar; /* DevX UAR for Rx. */ LIST_HEAD(shared_rxqs, mlx5_rxq_ctrl) shared_rxqs; /* Shared RXQs. */ struct mlx5_aso_age_mng *aso_age_mng; /* Management data for aging mechanism using ASO Flow Hit. */ @@ -1233,7 +1279,7 @@ struct mlx5_dev_ctx_shared { struct mlx5_proc_priv { size_t uar_table_sz; /* Size of UAR register table. */ - void *uar_table[]; + struct mlx5_uar_data uar_table[]; /* Table of UAR registers for each process. */ }; @@ -1250,6 +1296,7 @@ struct mlx5_flow_rss_desc { uint64_t hash_fields; /* Verbs Hash fields. */ uint8_t key[MLX5_RSS_HASH_KEY_LEN]; /**< RSS hash key. */ uint32_t key_len; /**< RSS hash key len. */ + uint32_t hws_flags; /**< HW steering action. */ uint32_t tunnel; /**< Queue in tunnel. */ uint32_t shared_rss; /**< Shared RSS index. */ struct mlx5_ind_table_obj *ind_tbl; @@ -1311,6 +1358,7 @@ struct mlx5_hrxq { #if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) void *action; /* DV QP action pointer. */ #endif + uint32_t hws_flags; /* Hw steering flags. */ uint64_t hash_fields; /* Verbs Hash fields. */ uint32_t rss_key_len; /* Hash key length in bytes. */ uint32_t idx; /* Hash Rx queue index. */ @@ -1410,6 +1458,7 @@ struct mlx5_priv { unsigned int mtr_en:1; /* Whether support meter. */ unsigned int mtr_reg_share:1; /* Whether support meter REG_C share. */ unsigned int lb_used:1; /* Loopback queue is referred to. */ + uint32_t mark_enabled:1; /* If mark action is enabled on rxqs. */ uint16_t domain_id; /* Switch domain identifier. */ uint16_t vport_id; /* Associated VF vport index (if any). */ uint32_t vport_meta_tag; /* Used for vport index match ove VF LAG. */ @@ -1420,6 +1469,7 @@ struct mlx5_priv { /* RX/TX queues. */ unsigned int rxqs_n; /* RX queues array size. */ unsigned int txqs_n; /* TX queues array size. */ + struct mlx5_external_rxq *ext_rxqs; /* External RX queues array. */ struct mlx5_rxq_priv *(*rxq_privs)[]; /* RX queue non-shared data. */ struct mlx5_txq_data *(*txqs)[]; /* TX queues. */ struct rte_mempool *mprq_mp; /* Mempool for Multi-Packet RQ. */ @@ -1440,6 +1490,8 @@ struct mlx5_priv { LIST_HEAD(txqobj, mlx5_txq_obj) txqsobj; /* Verbs/DevX Tx queues. */ /* Indirection tables. */ LIST_HEAD(ind_tables, mlx5_ind_table_obj) ind_tbls; + /* Standalone indirect tables. */ + LIST_HEAD(stdl_ind_tables, mlx5_ind_table_obj) standalone_ind_tbls; /* Pointer to next element. */ rte_rwlock_t ind_tbls_lock; uint32_t refcnt; /**< Reference counter. */ @@ -1449,7 +1501,7 @@ struct mlx5_priv { uint32_t link_speed_capa; /* Link speed capabilities. */ struct mlx5_xstats_ctrl xstats_ctrl; /* Extended stats control. */ struct mlx5_stats_ctrl stats_ctrl; /* Stats control. */ - struct mlx5_dev_config config; /* Device configuration. */ + struct mlx5_port_config config; /* Port configuration. */ /* Context for Verbs allocator. */ int nl_socket_rdma; /* Netlink socket (NETLINK_RDMA). */ int nl_socket_route; /* Netlink socket (NETLINK_ROUTE). */ @@ -1475,6 +1527,24 @@ struct mlx5_priv { struct mlx5_flex_item flex_item[MLX5_PORT_FLEX_ITEM_NUM]; /* Flex items have been created on the port. */ uint32_t flex_item_map; /* Map of allocated flex item elements. */ +#if defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_INFINIBAND_VERBS_H) + /* Item template list. */ + LIST_HEAD(flow_hw_itt, rte_flow_pattern_template) flow_hw_itt; + /* Action template list. */ + LIST_HEAD(flow_hw_at, rte_flow_actions_template) flow_hw_at; + struct mlx5dr_context *dr_ctx; /**< HW steering DR context. */ + uint32_t nb_queue; /* HW steering queue number. */ + /* HW steering queue polling mechanism job descriptor LIFO. */ + struct mlx5_hw_q *hw_q; + /* HW steering rte flow table list header. */ + LIST_HEAD(flow_hw_tbl, rte_flow_template_table) flow_hw_tbl; + /* HW steering global drop action. */ + struct mlx5dr_action *hw_drop[MLX5_HW_ACTION_FLAG_MAX] + [MLX5DR_TABLE_TYPE_MAX]; + /* HW steering global drop action. */ + struct mlx5dr_action *hw_tag[MLX5_HW_ACTION_FLAG_MAX]; + struct mlx5_indexed_pool *acts_ipool; /* Action data indexed pool. */ +#endif }; #define PORT_ID(priv) ((priv)->dev_data->port_id) @@ -1495,6 +1565,30 @@ enum dr_dump_rec_type { DR_DUMP_REC_TYPE_PMD_COUNTER = 4430, }; +/** + * Indicates whether HW objects operations can be created by DevX. + * + * This function is used for both: + * Before creation - deciding whether to create HW objects operations by DevX. + * After creation - indicator if HW objects operations were created by DevX. + * + * @param sh + * Pointer to shared device context. + * + * @return + * True if HW objects were created by DevX, False otherwise. + */ +static inline bool +mlx5_devx_obj_ops_en(struct mlx5_dev_ctx_shared *sh) +{ + /* + * When advanced DR API is available and DV flow is supported and + * DevX is supported, HW objects operations are created by DevX. + */ + return (sh->cdev->config.devx && sh->config.dv_flow_en && + sh->dev_cap.dest_tir); +} + /* mlx5.c */ int mlx5_getenv_int(const char *); @@ -1514,28 +1608,27 @@ void mlx5_age_event_prepare(struct mlx5_dev_ctx_shared *sh); for (port_id = mlx5_eth_find_next(0, dev); \ port_id < RTE_MAX_ETHPORTS; \ port_id = mlx5_eth_find_next(port_id + 1, dev)) -int mlx5_args(struct mlx5_dev_config *config, struct rte_devargs *devargs); +void mlx5_rt_timestamp_config(struct mlx5_dev_ctx_shared *sh, + struct mlx5_hca_attr *hca_attr); struct mlx5_dev_ctx_shared * mlx5_alloc_shared_dev_ctx(const struct mlx5_dev_spawn_data *spawn, - const struct mlx5_dev_config *config); + struct mlx5_kvargs_ctrl *mkvlist); void mlx5_free_shared_dev_ctx(struct mlx5_dev_ctx_shared *sh); int mlx5_dev_ctx_shared_mempool_subscribe(struct rte_eth_dev *dev); void mlx5_free_table_hash_list(struct mlx5_priv *priv); int mlx5_alloc_table_hash_list(struct mlx5_priv *priv); -void mlx5_set_min_inline(struct mlx5_dev_spawn_data *spawn, - struct mlx5_dev_config *config); +void mlx5_set_min_inline(struct mlx5_priv *priv); void mlx5_set_metadata_mask(struct rte_eth_dev *dev); -int mlx5_dev_check_sibling_config(struct mlx5_priv *priv, - struct mlx5_dev_config *config, - struct rte_device *dpdk_dev); -int mlx5_dev_configure(struct rte_eth_dev *dev); -int mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info); -int mlx5_fw_version_get(struct rte_eth_dev *dev, char *fw_ver, size_t fw_size); -int mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu); -int mlx5_hairpin_cap_get(struct rte_eth_dev *dev, - struct rte_eth_hairpin_cap *cap); +int mlx5_probe_again_args_validate(struct mlx5_common_device *cdev, + struct mlx5_kvargs_ctrl *mkvlist); +int mlx5_port_args_config(struct mlx5_priv *priv, + struct mlx5_kvargs_ctrl *mkvlist, + struct mlx5_port_config *config); +void mlx5_port_args_set_used(const char *name, uint16_t port_id, + struct mlx5_kvargs_ctrl *mkvlist); bool mlx5_flex_parser_ecpri_exist(struct rte_eth_dev *dev); int mlx5_flex_parser_ecpri_alloc(struct rte_eth_dev *dev); +void mlx5_flow_counter_mode_config(struct rte_eth_dev *dev); int mlx5_flow_aso_age_mng_init(struct mlx5_dev_ctx_shared *sh); int mlx5_aso_flow_mtrs_mng_init(struct mlx5_dev_ctx_shared *sh); int mlx5_flow_aso_ct_mng_init(struct mlx5_dev_ctx_shared *sh); @@ -1553,10 +1646,8 @@ int mlx5_representor_info_get(struct rte_eth_dev *dev, (((repr_id) >> 12) & 3) uint16_t mlx5_representor_id_encode(const struct mlx5_switch_info *info, enum rte_eth_representor_type hpf_type); -int mlx5_fw_version_get(struct rte_eth_dev *dev, char *fw_ver, - size_t fw_size); -int mlx5_dev_infos_get(struct rte_eth_dev *dev, - struct rte_eth_dev_info *info); +int mlx5_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info); +int mlx5_fw_version_get(struct rte_eth_dev *dev, char *fw_ver, size_t fw_size); const uint32_t *mlx5_dev_supported_ptypes_get(struct rte_eth_dev *dev); int mlx5_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu); int mlx5_hairpin_cap_get(struct rte_eth_dev *dev, @@ -1582,6 +1673,7 @@ int mlx5_dev_set_flow_ctrl(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf); void mlx5_dev_interrupt_handler(void *arg); void mlx5_dev_interrupt_handler_devx(void *arg); +void mlx5_dev_interrupt_handler_nl(void *arg); int mlx5_set_link_down(struct rte_eth_dev *dev); int mlx5_set_link_up(struct rte_eth_dev *dev); int mlx5_is_removed(struct rte_eth_dev *dev); @@ -1602,6 +1694,7 @@ int mlx5_os_read_dev_stat(struct mlx5_priv *priv, int mlx5_os_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats); int mlx5_os_get_stats_n(struct rte_eth_dev *dev); void mlx5_os_stats_init(struct rte_eth_dev *dev); +int mlx5_get_flag_dropless_rq(struct rte_eth_dev *dev); /* mlx5_mac.c */ @@ -1730,7 +1823,7 @@ void mlx5_flow_query_alarm(void *arg); uint32_t mlx5_counter_alloc(struct rte_eth_dev *dev); void mlx5_counter_free(struct rte_eth_dev *dev, uint32_t cnt); int mlx5_counter_query(struct rte_eth_dev *dev, uint32_t cnt, - bool clear, uint64_t *pkts, uint64_t *bytes); + bool clear, uint64_t *pkts, uint64_t *bytes, void **action); int mlx5_flow_dev_dump(struct rte_eth_dev *dev, struct rte_flow *flow, FILE *file, struct rte_flow_error *error); int save_dump_file(const unsigned char *data, uint32_t size, @@ -1782,6 +1875,10 @@ struct mlx5_flow_meter_policy *mlx5_flow_meter_policy_find (struct rte_eth_dev *dev, uint32_t policy_id, uint32_t *policy_idx); +struct mlx5_flow_meter_info * +mlx5_flow_meter_hierarchy_next_meter(struct mlx5_priv *priv, + struct mlx5_flow_meter_policy *policy, + uint32_t *mtr_idx); struct mlx5_flow_meter_policy * mlx5_flow_meter_hierarchy_get_final_policy(struct rte_eth_dev *dev, struct mlx5_flow_meter_policy *policy); @@ -1790,11 +1887,12 @@ int mlx5_flow_meter_flush(struct rte_eth_dev *dev, void mlx5_flow_meter_rxq_flush(struct rte_eth_dev *dev); /* mlx5_os.c */ + struct rte_pci_driver; -int mlx5_os_get_dev_attr(struct mlx5_common_device *dev, - struct mlx5_dev_attr *dev_attr); +int mlx5_os_capabilities_prepare(struct mlx5_dev_ctx_shared *sh); void mlx5_os_free_shared_dr(struct mlx5_priv *priv); -int mlx5_os_net_probe(struct mlx5_common_device *cdev); +int mlx5_os_net_probe(struct mlx5_common_device *cdev, + struct mlx5_kvargs_ctrl *mkvlist); void mlx5_os_dev_shared_handler_install(struct mlx5_dev_ctx_shared *sh); void mlx5_os_dev_shared_handler_uninstall(struct mlx5_dev_ctx_shared *sh); void mlx5_os_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index);