From 79a7e409a2f6d8ed1bc836b0456141860220225c Mon Sep 17 00:00:00 2001 From: Viacheslav Ovsiienko Date: Thu, 16 Jul 2020 08:23:04 +0000 Subject: [PATCH] common/mlx5: prepare support of packet pacing This patch prepares the common part of the mlx5 PMDs to support packet send scheduling on mbuf timestamps: - the DevX routine to query the packet pacing HCA capabilities - packet pacing Send Queue attributes support - the hardware related definitions Signed-off-by: Viacheslav Ovsiienko Acked-by: Matan Azrad --- drivers/common/mlx5/Makefile | 25 ++++++++++ drivers/common/mlx5/linux/meson.build | 10 ++++ drivers/common/mlx5/linux/mlx5_glue.c | 31 +++++++++++- drivers/common/mlx5/linux/mlx5_glue.h | 5 ++ drivers/common/mlx5/mlx5_devx_cmds.c | 19 +++++++- drivers/common/mlx5/mlx5_devx_cmds.h | 10 ++++ drivers/common/mlx5/mlx5_prm.h | 69 +++++++++++++++++++++++++-- 7 files changed, 161 insertions(+), 8 deletions(-) diff --git a/drivers/common/mlx5/Makefile b/drivers/common/mlx5/Makefile index 5f12be3f52..53afffa244 100644 --- a/drivers/common/mlx5/Makefile +++ b/drivers/common/mlx5/Makefile @@ -171,6 +171,16 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh infiniband/mlx5dv.h \ func mlx5dv_devx_qp_query \ $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_MLX5DV_DEVX_UAR_OFFSET \ + infiniband/mlx5dv.h \ + field "struct mlx5dv_devx_uar.mmap_off" \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_MLX5DV_PP_ALLOC \ + infiniband/mlx5dv.h \ + func mlx5dv_pp_alloc \ + $(AUTOCONF_OUTPUT) $Q sh -- '$<' '$@' \ HAVE_MLX5DV_DR_ACTION_DEST_DEVX_TIR \ infiniband/mlx5dv.h \ @@ -206,6 +216,21 @@ mlx5_autoconf.h.new: $(RTE_SDK)/buildtools/auto-config-h.sh infiniband/mlx5dv.h \ func mlx5dv_dr_domain_set_reclaim_device_memory \ $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_MLX5_OPCODE_ENHANCED_MPSW \ + infiniband/mlx5dv.h \ + enum MLX5_OPCODE_ENHANCED_MPSW \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_MLX5_OPCODE_SEND_EN \ + infiniband/mlx5dv.h \ + enum MLX5_OPCODE_SEND_EN \ + $(AUTOCONF_OUTPUT) + $Q sh -- '$<' '$@' \ + HAVE_MLX5_OPCODE_WAIT \ + infiniband/mlx5dv.h \ + enum MLX5_OPCODE_WAIT \ + $(AUTOCONF_OUTPUT) $Q sh -- '$<' '$@' \ HAVE_ETHTOOL_LINK_MODE_25G \ /usr/include/linux/ethtool.h \ diff --git a/drivers/common/mlx5/linux/meson.build b/drivers/common/mlx5/linux/meson.build index 2294213f1d..48e8ad6a4b 100644 --- a/drivers/common/mlx5/linux/meson.build +++ b/drivers/common/mlx5/linux/meson.build @@ -64,6 +64,8 @@ has_member_args = [ 'struct ibv_counter_set_init_attr', 'counter_set_id' ], [ 'HAVE_IBV_DEVICE_COUNTERS_SET_V45', 'infiniband/verbs.h', 'struct ibv_counters_init_attr', 'comp_mask' ], + [ 'HAVE_MLX5DV_DEVX_UAR_OFFSET', 'infiniband/mlx5dv.h', + 'struct mlx5dv_devx_uar', 'mmap_off' ], ] # input array for meson symbol search: # [ "MACRO to define if found", "header for the search", @@ -101,6 +103,8 @@ has_sym_args = [ 'mlx5dv_devx_obj_query_async' ], [ 'HAVE_IBV_DEVX_QP', 'infiniband/mlx5dv.h', 'mlx5dv_devx_qp_query' ], + [ 'HAVE_MLX5DV_PP_ALLOC', 'infiniband/mlx5dv.h', + 'mlx5dv_pp_alloc' ], [ 'HAVE_MLX5DV_DR_ACTION_DEST_DEVX_TIR', 'infiniband/mlx5dv.h', 'mlx5dv_dr_action_create_dest_devx_tir' ], [ 'HAVE_IBV_DEVX_EVENT', 'infiniband/mlx5dv.h', @@ -116,6 +120,12 @@ has_sym_args = [ [ 'HAVE_MLX5DV_DR_VLAN', 'infiniband/mlx5dv.h', 'mlx5dv_dr_action_create_push_vlan' ], [ 'HAVE_IBV_VAR', 'infiniband/mlx5dv.h', 'mlx5dv_alloc_var' ], + [ 'HAVE_MLX5_OPCODE_ENHANCED_MPSW', 'infiniband/mlx5dv.h', + 'MLX5_OPCODE_ENHANCED_MPSW' ], + [ 'HAVE_MLX5_OPCODE_SEND_EN', 'infiniband/mlx5dv.h', + 'MLX5_OPCODE_SEND_EN' ], + [ 'HAVE_MLX5_OPCODE_WAIT', 'infiniband/mlx5dv.h', + 'MLX5_OPCODE_WAIT' ], [ 'HAVE_SUPPORTED_40000baseKR4_Full', 'linux/ethtool.h', 'SUPPORTED_40000baseKR4_Full' ], [ 'HAVE_SUPPORTED_40000baseCR4_Full', 'linux/ethtool.h', diff --git a/drivers/common/mlx5/linux/mlx5_glue.c b/drivers/common/mlx5/linux/mlx5_glue.c index 395519d0ab..4d3875fa27 100644 --- a/drivers/common/mlx5/linux/mlx5_glue.c +++ b/drivers/common/mlx5/linux/mlx5_glue.c @@ -1195,7 +1195,6 @@ mlx5_glue_dv_free_var(struct mlx5dv_var *var) #endif } - static void mlx5_glue_dr_reclaim_domain_memory(void *domain, uint32_t enable) { @@ -1207,6 +1206,34 @@ mlx5_glue_dr_reclaim_domain_memory(void *domain, uint32_t enable) #endif } +static struct mlx5dv_pp * +mlx5_glue_dv_alloc_pp(struct ibv_context *context, + size_t pp_context_sz, + const void *pp_context, + uint32_t flags) +{ +#ifdef HAVE_MLX5DV_PP_ALLOC + return mlx5dv_pp_alloc(context, pp_context_sz, pp_context, flags); +#else + RTE_SET_USED(context); + RTE_SET_USED(pp_context_sz); + RTE_SET_USED(pp_context); + RTE_SET_USED(flags); + errno = ENOTSUP; + return NULL; +#endif +} + +static void +mlx5_glue_dv_free_pp(struct mlx5dv_pp *pp) +{ +#ifdef HAVE_MLX5DV_PP_ALLOC + mlx5dv_pp_free(pp); +#else + RTE_SET_USED(pp); +#endif +} + __rte_cache_aligned const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue) { .version = MLX5_GLUE_VERSION, @@ -1319,4 +1346,6 @@ const struct mlx5_glue *mlx5_glue = &(const struct mlx5_glue) { .devx_free_uar = mlx5_glue_devx_free_uar, .dv_alloc_var = mlx5_glue_dv_alloc_var, .dv_free_var = mlx5_glue_dv_free_var, + .dv_alloc_pp = mlx5_glue_dv_alloc_pp, + .dv_free_pp = mlx5_glue_dv_free_pp, }; diff --git a/drivers/common/mlx5/linux/mlx5_glue.h b/drivers/common/mlx5/linux/mlx5_glue.h index 069d8540cc..c4f9b006ad 100644 --- a/drivers/common/mlx5/linux/mlx5_glue.h +++ b/drivers/common/mlx5/linux/mlx5_glue.h @@ -304,6 +304,11 @@ struct mlx5_glue { struct mlx5dv_devx_async_event_hdr *event_data, size_t event_resp_len); void (*dr_reclaim_domain_memory)(void *domain, uint32_t enable); + struct mlx5dv_pp *(*dv_alloc_pp)(struct ibv_context *context, + size_t pp_context_sz, + const void *pp_context, + uint32_t flags); + void (*dv_free_pp)(struct mlx5dv_pp *pp); }; extern const struct mlx5_glue *mlx5_glue; diff --git a/drivers/common/mlx5/mlx5_devx_cmds.c b/drivers/common/mlx5/mlx5_devx_cmds.c index 54b20a7a36..bb356e2631 100644 --- a/drivers/common/mlx5/mlx5_devx_cmds.c +++ b/drivers/common/mlx5/mlx5_devx_cmds.c @@ -467,6 +467,14 @@ mlx5_devx_cmd_query_hca_attr(void *ctx, attr->vdpa.queue_counters_valid = !!(MLX5_GET64(cmd_hca_cap, hcattr, general_obj_types) & MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_Q_COUNTERS); + attr->wqe_index_ignore = MLX5_GET(cmd_hca_cap, hcattr, + wqe_index_ignore_cap); + attr->cross_channel = MLX5_GET(cmd_hca_cap, hcattr, cd); + attr->non_wire_sq = MLX5_GET(cmd_hca_cap, hcattr, non_wire_sq); + attr->log_max_static_sq_wq = MLX5_GET(cmd_hca_cap, hcattr, + log_max_static_sq_wq); + attr->dev_freq_khz = MLX5_GET(cmd_hca_cap, hcattr, + device_frequency_khz); attr->regex = MLX5_GET(cmd_hca_cap, hcattr, regexp); attr->regexp_num_of_engines = MLX5_GET(cmd_hca_cap, hcattr, regexp_num_of_engines); @@ -490,9 +498,13 @@ mlx5_devx_cmd_query_hca_attr(void *ctx, attr->qos.log_max_flow_meter = MLX5_GET(qos_cap, hcattr, log_max_flow_meter); attr->qos.flow_meter_reg_c_ids = - MLX5_GET(qos_cap, hcattr, flow_meter_reg_id); + MLX5_GET(qos_cap, hcattr, flow_meter_reg_id); attr->qos.flow_meter_reg_share = - MLX5_GET(qos_cap, hcattr, flow_meter_reg_share); + MLX5_GET(qos_cap, hcattr, flow_meter_reg_share); + attr->qos.packet_pacing = + MLX5_GET(qos_cap, hcattr, packet_pacing); + attr->qos.wqe_rate_pp = + MLX5_GET(qos_cap, hcattr, wqe_rate_pp); } if (attr->vdpa.valid) mlx5_devx_cmd_query_hca_vdpa_attr(ctx, &attr->vdpa); @@ -974,6 +986,8 @@ mlx5_devx_cmd_create_sq(void *ctx, MLX5_SET(sqc, sq_ctx, reg_umr, sq_attr->reg_umr); MLX5_SET(sqc, sq_ctx, allow_swp, sq_attr->allow_swp); MLX5_SET(sqc, sq_ctx, hairpin, sq_attr->hairpin); + MLX5_SET(sqc, sq_ctx, non_wire, sq_attr->non_wire); + MLX5_SET(sqc, sq_ctx, static_sq_wq, sq_attr->static_sq_wq); MLX5_SET(sqc, sq_ctx, user_index, sq_attr->user_index); MLX5_SET(sqc, sq_ctx, cqn, sq_attr->cqn); MLX5_SET(sqc, sq_ctx, packet_pacing_rate_limit_index, @@ -1188,6 +1202,7 @@ mlx5_devx_cmd_create_cq(void *ctx, struct mlx5_devx_cq_attr *attr) } else { MLX5_SET64(cqc, cqctx, dbr_addr, attr->db_addr); } + MLX5_SET(cqc, cqctx, cqe_sz, attr->cqe_size); MLX5_SET(cqc, cqctx, cc, attr->use_first_only); MLX5_SET(cqc, cqctx, oi, attr->overrun_ignore); MLX5_SET(cqc, cqctx, log_cq_size, attr->log_cq_size); diff --git a/drivers/common/mlx5/mlx5_devx_cmds.h b/drivers/common/mlx5/mlx5_devx_cmds.h index bb14ca5850..7b15b5e0ca 100644 --- a/drivers/common/mlx5/mlx5_devx_cmds.h +++ b/drivers/common/mlx5/mlx5_devx_cmds.h @@ -29,6 +29,8 @@ struct mlx5_devx_mkey_attr { struct mlx5_hca_qos_attr { uint32_t sup:1; /* Whether QOS is supported. */ uint32_t srtcm_sup:1; /* Whether srTCM mode is supported. */ + uint32_t packet_pacing:1; /* Packet pacing is supported. */ + uint32_t wqe_rate_pp:1; /* Packet pacing WQE rate mode. */ uint32_t flow_meter_reg_share:1; /* Whether reg_c share is supported. */ uint8_t log_max_flow_meter; @@ -90,6 +92,11 @@ struct mlx5_hca_attr { uint32_t vhca_id:16; uint32_t relaxed_ordering_write:1; uint32_t relaxed_ordering_read:1; + uint32_t wqe_index_ignore:1; + uint32_t cross_channel:1; + uint32_t non_wire_sq:1; /* SQ with non-wire ops is supported. */ + uint32_t log_max_static_sq_wq:5; /* Static WQE size SQ. */ + uint32_t dev_freq_khz; /* Timestamp counter frequency, kHz. */ uint32_t regex:1; uint32_t regexp_num_of_engines; struct mlx5_hca_qos_attr qos; @@ -209,6 +216,8 @@ struct mlx5_devx_create_sq_attr { uint32_t reg_umr:1; uint32_t allow_swp:1; uint32_t hairpin:1; + uint32_t non_wire:1; + uint32_t static_sq_wq:1; uint32_t user_index:24; uint32_t cqn:24; uint32_t packet_pacing_rate_limit_index:16; @@ -232,6 +241,7 @@ struct mlx5_devx_cq_attr { uint32_t db_umem_valid:1; uint32_t use_first_only:1; uint32_t overrun_ignore:1; + uint32_t cqe_size:3; uint32_t log_cq_size:5; uint32_t log_page_size:5; uint32_t uar_page_id; diff --git a/drivers/common/mlx5/mlx5_prm.h b/drivers/common/mlx5/mlx5_prm.h index c05891f1ae..2f92663780 100644 --- a/drivers/common/mlx5/mlx5_prm.h +++ b/drivers/common/mlx5/mlx5_prm.h @@ -41,6 +41,10 @@ /* Invalidate a CQE. */ #define MLX5_CQE_INVALIDATE (MLX5_CQE_INVALID << 4) +/* Hardware index widths. */ +#define MLX5_CQ_INDEX_WIDTH 24 +#define MLX5_WQ_INDEX_WIDTH 16 + /* WQE Segment sizes in bytes. */ #define MLX5_WSEG_SIZE 16u #define MLX5_WQE_CSEG_SIZE sizeof(struct mlx5_wqe_cseg) @@ -126,7 +130,17 @@ MLX5_ESEG_MIN_INLINE_SIZE) /* Missed in mlv5dv.h, should define here. */ +#ifndef HAVE_MLX5_OPCODE_ENHANCED_MPSW #define MLX5_OPCODE_ENHANCED_MPSW 0x29u +#endif + +#ifndef HAVE_MLX5_OPCODE_SEND_EN +#define MLX5_OPCODE_SEND_EN 0x17u +#endif + +#ifndef HAVE_MLX5_OPCODE_WAIT +#define MLX5_OPCODE_WAIT 0x0fu +#endif /* CQE value to inform that VLAN is stripped. */ #define MLX5_CQE_VLAN_STRIPPED (1u << 0) @@ -255,6 +269,9 @@ /* The alignment needed for WQ buffer. */ #define MLX5_WQE_BUF_ALIGNMENT sysconf(_SC_PAGESIZE) +/* The alignment needed for CQ buffer. */ +#define MLX5_CQE_BUF_ALIGNMENT sysconf(_SC_PAGESIZE) + /* Completion mode. */ enum mlx5_completion_mode { MLX5_COMP_ONLY_ERR = 0x0, @@ -314,6 +331,13 @@ struct mlx5_wqe_eseg { }; } __rte_packed; +struct mlx5_wqe_qseg { + uint32_t reserved0; + uint32_t reserved1; + uint32_t max_index; + uint32_t qpn_cqn; +} __rte_packed; + /* The title WQEBB, header of WQE. */ struct mlx5_wqe { union { @@ -373,6 +397,14 @@ struct mlx5_cqe { uint8_t op_own; }; +struct mlx5_cqe_ts { + uint64_t timestamp; + uint32_t sop_drop_qpn; + uint16_t wqe_counter; + uint8_t rsvd5; + uint8_t op_own; +}; + /* MMO metadata segment */ #define MLX5_OPCODE_MMO 0x2f @@ -1042,7 +1074,9 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 reserved_at_40[0x40]; u8 log_max_srq_sz[0x8]; u8 log_max_qp_sz[0x8]; - u8 reserved_at_90[0xb]; + u8 reserved_at_90[0x9]; + u8 wqe_index_ignore_cap[0x1]; + u8 dynamic_qp_allocation[0x1]; u8 log_max_qp[0x5]; u8 regexp[0x1]; u8 reserved_at_a1[0x3]; @@ -1073,9 +1107,12 @@ struct mlx5_ifc_cmd_hca_cap_bits { u8 umr_extended_translation_offset[0x1]; u8 null_mkey[0x1]; u8 log_max_klm_list_size[0x6]; - u8 reserved_at_120[0xa]; + u8 non_wire_sq[0x1]; + u8 reserved_at_121[0x9]; u8 log_max_ra_req_dc[0x6]; - u8 reserved_at_130[0xa]; + u8 reserved_at_130[0x3]; + u8 log_max_static_sq_wq[0x5]; + u8 reserved_at_138[0x2]; u8 log_max_ra_res_dc[0x6]; u8 reserved_at_140[0xa]; u8 log_max_ra_req_qp[0x6]; @@ -1326,7 +1363,8 @@ struct mlx5_ifc_qos_cap_bits { u8 reserved_at_8[0x8]; u8 log_max_flow_meter[0x8]; u8 flow_meter_reg_id[0x8]; - u8 reserved_at_25[0x8]; + u8 wqe_rate_pp[0x1]; + u8 reserved_at_25[0x7]; u8 flow_meter_reg_share[0x1]; u8 reserved_at_2e[0x17]; u8 packet_pacing_max_rate[0x20]; @@ -1890,7 +1928,9 @@ struct mlx5_ifc_sqc_bits { u8 reg_umr[0x1]; u8 allow_swp[0x1]; u8 hairpin[0x1]; - u8 reserved_at_f[0x11]; + u8 non_wire[0x1]; + u8 static_sq_wq[0x1]; + u8 reserved_at_11[0xf]; u8 reserved_at_20[0x8]; u8 user_index[0x18]; u8 reserved_at_40[0x8]; @@ -1990,6 +2030,11 @@ struct mlx5_ifc_flow_meter_parameters_bits { u8 reserved_at_8[0x60]; // 14h-1Ch }; +enum { + MLX5_CQE_SIZE_64B = 0x0, + MLX5_CQE_SIZE_128B = 0x1, +}; + struct mlx5_ifc_cqc_bits { u8 status[0x4]; u8 as_notify[0x1]; @@ -2541,6 +2586,20 @@ struct mlx5_ifc_query_qp_in_bits { u8 reserved_at_60[0x20]; }; +enum { + MLX5_DATA_RATE = 0x0, + MLX5_WQE_RATE = 0x1, +}; + +struct mlx5_ifc_set_pp_rate_limit_context_bits { + u8 rate_limit[0x20]; + u8 burst_upper_bound[0x20]; + u8 reserved_at_40[0xC]; + u8 rate_mode[0x4]; + u8 typical_packet_size[0x10]; + u8 reserved_at_60[0x120]; +}; + struct regexp_params_field_select_bits { u8 reserved_at_0[0x1e]; u8 stop_engine[0x1]; -- 2.20.1