X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Flinux%2Fmlx5_os.c;h=fc339affa5e7fe1d6c0aeccff5561ce1c7c25628;hb=d462a83c65d8b1ab38db68202cf1c74f75ebfbec;hp=14af468d6f00ce85462863f30fddead7a865c5e4;hpb=8f848f32fc24f6db2dc4366b8f4111cdc3e67356;p=dpdk.git diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c index 14af468d6f..fc339affa5 100644 --- a/drivers/net/mlx5/linux/mlx5_os.c +++ b/drivers/net/mlx5/linux/mlx5_os.c @@ -10,22 +10,11 @@ #include #include #include -#include #include #include #include #include -/* Verbs header. */ -/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif -#include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif - #include #include #include @@ -37,12 +26,14 @@ #include #include #include +#include #include #include #include #include #include +#include #include "mlx5_defs.h" #include "mlx5.h" @@ -66,6 +57,35 @@ #define MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP (1 << 4) #endif +static const char *MZ_MLX5_PMD_SHARED_DATA = "mlx5_pmd_shared_data"; + +/* Spinlock for mlx5_shared_data allocation. */ +static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER; + +/* Process local data for secondary processes. */ +static struct mlx5_local_data mlx5_local_data; + +/** + * Set the completion channel file descriptor interrupt as non-blocking. + * + * @param[in] rxq_obj + * Pointer to RQ channel object, which includes the channel fd + * + * @param[out] fd + * The file descriptor (representing the intetrrupt) used in this channel. + * + * @return + * 0 on successfully setting the fd to non-blocking, non-zero otherwise. + */ +int +mlx5_os_set_nonblock_channel_fd(int fd) +{ + int flags; + + flags = fcntl(fd, F_GETFL); + return fcntl(fd, F_SETFL, flags | O_NONBLOCK); +} + /** * Get mlx5 device attributes. The glue function query_device_ex() is called * with out parameter of type 'struct ibv_device_attr_ex *'. Then fill in mlx5 @@ -133,7 +153,7 @@ mlx5_os_get_dev_attr(void *ctx, struct mlx5_dev_attr *device_attr) * Verbs callback to allocate a memory. This function should allocate the space * according to the size provided residing inside a huge page. * Please note that all allocation must respect the alignment from libmlx5 - * (i.e. currently sysconf(_SC_PAGESIZE)). + * (i.e. currently rte_mem_page_size()). * * @param[in] size * The size in bytes of the memory to allocate. @@ -148,8 +168,13 @@ mlx5_alloc_verbs_buf(size_t size, void *data) { struct mlx5_priv *priv = data; void *ret; - size_t alignment = sysconf(_SC_PAGESIZE); unsigned int socket = SOCKET_ID_ANY; + size_t alignment = rte_mem_page_size(); + if (alignment == (size_t)-1) { + DRV_LOG(ERR, "Failed to get mem page size"); + rte_errno = ENOMEM; + return NULL; + } if (priv->verbs_alloc_ctx.type == MLX5_VERBS_ALLOC_TYPE_TX_QUEUE) { const struct mlx5_txq_ctrl *ctrl = priv->verbs_alloc_ctx.obj; @@ -162,7 +187,7 @@ mlx5_alloc_verbs_buf(size_t size, void *data) socket = ctrl->socket; } MLX5_ASSERT(data != NULL); - ret = rte_malloc_socket(__func__, size, alignment, socket); + ret = mlx5_malloc(0, size, alignment, socket); if (!ret && size) rte_errno = ENOMEM; return ret; @@ -180,7 +205,7 @@ static void mlx5_free_verbs_buf(void *ptr, void *data __rte_unused) { MLX5_ASSERT(data != NULL); - rte_free(ptr); + mlx5_free(ptr); } /** @@ -350,6 +375,109 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv) mlx5_free_table_hash_list(priv); } +/** + * Initialize shared data between primary and secondary process. + * + * A memzone is reserved by primary process and secondary processes attach to + * the memzone. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx5_init_shared_data(void) +{ + const struct rte_memzone *mz; + int ret = 0; + + rte_spinlock_lock(&mlx5_shared_data_lock); + if (mlx5_shared_data == NULL) { + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + /* Allocate shared memory. */ + mz = rte_memzone_reserve(MZ_MLX5_PMD_SHARED_DATA, + sizeof(*mlx5_shared_data), + SOCKET_ID_ANY, 0); + if (mz == NULL) { + DRV_LOG(ERR, + "Cannot allocate mlx5 shared data"); + ret = -rte_errno; + goto error; + } + mlx5_shared_data = mz->addr; + memset(mlx5_shared_data, 0, sizeof(*mlx5_shared_data)); + rte_spinlock_init(&mlx5_shared_data->lock); + } else { + /* Lookup allocated shared memory. */ + mz = rte_memzone_lookup(MZ_MLX5_PMD_SHARED_DATA); + if (mz == NULL) { + DRV_LOG(ERR, + "Cannot attach mlx5 shared data"); + ret = -rte_errno; + goto error; + } + mlx5_shared_data = mz->addr; + memset(&mlx5_local_data, 0, sizeof(mlx5_local_data)); + } + } +error: + rte_spinlock_unlock(&mlx5_shared_data_lock); + return ret; +} + +/** + * PMD global initialization. + * + * Independent from individual device, this function initializes global + * per-PMD data structures distinguishing primary and secondary processes. + * Hence, each initialization is called once per a process. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx5_init_once(void) +{ + struct mlx5_shared_data *sd; + struct mlx5_local_data *ld = &mlx5_local_data; + int ret = 0; + + if (mlx5_init_shared_data()) + return -rte_errno; + sd = mlx5_shared_data; + MLX5_ASSERT(sd); + rte_spinlock_lock(&sd->lock); + switch (rte_eal_process_type()) { + case RTE_PROC_PRIMARY: + if (sd->init_done) + break; + LIST_INIT(&sd->mem_event_cb_list); + rte_rwlock_init(&sd->mem_event_rwlock); + rte_mem_event_callback_register("MLX5_MEM_EVENT_CB", + mlx5_mr_mem_event_cb, NULL); + ret = mlx5_mp_init_primary(MLX5_MP_NAME, + mlx5_mp_os_primary_handle); + if (ret) + goto out; + sd->init_done = true; + break; + case RTE_PROC_SECONDARY: + if (ld->init_done) + break; + ret = mlx5_mp_init_secondary(MLX5_MP_NAME, + mlx5_mp_os_secondary_handle); + if (ret) + goto out; + ++sd->secondary_cnt; + ld->init_done = true; + break; + default: + break; + } +out: + rte_spinlock_unlock(&sd->lock); + return ret; +} + /** * Spawn an Ethernet device from Verbs information. * @@ -370,7 +498,7 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv) static struct rte_eth_dev * mlx5_dev_spawn(struct rte_device *dpdk_dev, struct mlx5_dev_spawn_data *spawn, - struct mlx5_dev_config config) + struct mlx5_dev_config *config) { const struct mlx5_switch_info *switch_info = &spawn->info; struct mlx5_dev_ctx_shared *sh = NULL; @@ -488,19 +616,20 @@ err_secondary: * devargs here to get ones, and later proceed devargs again * to override some hardware settings. */ - err = mlx5_args(&config, dpdk_dev->devargs); + err = mlx5_args(config, dpdk_dev->devargs); if (err) { err = rte_errno; DRV_LOG(ERR, "failed to process device arguments: %s", strerror(rte_errno)); goto error; } - sh = mlx5_alloc_shared_dev_ctx(spawn, &config); + mlx5_malloc_mem_select(config->sys_mem_en); + sh = mlx5_alloc_shared_dev_ctx(spawn, config); if (!sh) return NULL; - config.devx = sh->devx; + config->devx = sh->devx; #ifdef HAVE_MLX5DV_DR_ACTION_DEST_DEVX_TIR - config.dest_tir = 1; + config->dest_tir = 1; #endif #ifdef HAVE_IBV_MLX5_MOD_SWP dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_SWP; @@ -533,7 +662,7 @@ err_secondary: swp = dv_attr.sw_parsing_caps.sw_parsing_offloads; DRV_LOG(DEBUG, "SWP support: %u", swp); #endif - config.swp = !!swp; + config->swp = !!swp; #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_STRIDING_RQ) { struct mlx5dv_striding_rq_caps mprq_caps = @@ -566,7 +695,7 @@ err_secondary: cqe_comp = 0; else cqe_comp = 1; - config.cqe_comp = cqe_comp; + config->cqe_comp = cqe_comp; #ifdef HAVE_IBV_MLX5_MOD_CQE_128B_PAD /* Whether device supports 128B Rx CQE padding. */ cqe_pad = RTE_CACHE_LINE_SIZE == 128 && @@ -587,7 +716,7 @@ err_secondary: DRV_LOG(WARNING, "tunnel offloading disabled due to old OFED/rdma-core version"); #endif - config.tunnel_en = tunnel_en; + config->tunnel_en = tunnel_en; #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT mpls_en = ((dv_attr.tunnel_offloads_caps & MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_GRE) && @@ -599,7 +728,7 @@ err_secondary: DRV_LOG(WARNING, "MPLS over GRE/UDP tunnel offloading disabled due to" " old OFED/rdma-core version or firmware configuration"); #endif - config.mpls_en = mpls_en; + config->mpls_en = mpls_en; /* Check port status. */ err = mlx5_glue->query_port(sh->ctx, spawn->phys_port, &port_attr); if (err) { @@ -616,9 +745,9 @@ err_secondary: mlx5_glue->port_state_str(port_attr.state), port_attr.state); /* Allocate private eth device data. */ - priv = rte_zmalloc("ethdev private structure", + priv = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE, sizeof(*priv), - RTE_CACHE_LINE_SIZE); + RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); if (priv == NULL) { DRV_LOG(ERR, "priv allocation failure"); err = ENOMEM; @@ -630,12 +759,6 @@ err_secondary: priv->mtu = RTE_ETHER_MTU; priv->mp_id.port_id = port_id; strlcpy(priv->mp_id.name, MLX5_MP_NAME, RTE_MP_MAX_NAME_LEN); -#ifndef RTE_ARCH_64 - /* Initialize UAR access locks for 32bit implementations. */ - rte_spinlock_init(&priv->uar_lock_cq); - for (i = 0; i < MLX5_UAR_PAGE_NUM_MAX; i++) - rte_spinlock_init(&priv->uar_lock[i]); -#endif /* Some internal functions rely on Netlink sockets, open them now. */ priv->nl_socket_rdma = mlx5_nl_init(NETLINK_RDMA); priv->nl_socket_route = mlx5_nl_init(NETLINK_ROUTE); @@ -746,90 +869,88 @@ err_secondary: own_domain_id = 1; } /* Override some values set by hardware configuration. */ - mlx5_args(&config, dpdk_dev->devargs); - err = mlx5_dev_check_sibling_config(priv, &config); + mlx5_args(config, dpdk_dev->devargs); + err = mlx5_dev_check_sibling_config(priv, config); if (err) goto error; - config.hw_csum = !!(sh->device_attr.device_cap_flags_ex & + config->hw_csum = !!(sh->device_attr.device_cap_flags_ex & IBV_DEVICE_RAW_IP_CSUM); DRV_LOG(DEBUG, "checksum offloading is %ssupported", - (config.hw_csum ? "" : "not ")); + (config->hw_csum ? "" : "not ")); #if !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) && \ !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) DRV_LOG(DEBUG, "counters are not supported"); #endif #if !defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_MLX5DV_DR) - if (config.dv_flow_en) { + if (config->dv_flow_en) { DRV_LOG(WARNING, "DV flow is not supported"); - config.dv_flow_en = 0; + config->dv_flow_en = 0; } #endif - config.ind_table_max_size = + config->ind_table_max_size = sh->device_attr.max_rwq_indirection_table_size; /* * Remove this check once DPDK supports larger/variable * indirection tables. */ - if (config.ind_table_max_size > (unsigned int)ETH_RSS_RETA_SIZE_512) - config.ind_table_max_size = ETH_RSS_RETA_SIZE_512; + if (config->ind_table_max_size > (unsigned int)ETH_RSS_RETA_SIZE_512) + config->ind_table_max_size = ETH_RSS_RETA_SIZE_512; DRV_LOG(DEBUG, "maximum Rx indirection table size is %u", - config.ind_table_max_size); - config.hw_vlan_strip = !!(sh->device_attr.raw_packet_caps & + config->ind_table_max_size); + config->hw_vlan_strip = !!(sh->device_attr.raw_packet_caps & IBV_RAW_PACKET_CAP_CVLAN_STRIPPING); DRV_LOG(DEBUG, "VLAN stripping is %ssupported", - (config.hw_vlan_strip ? "" : "not ")); - config.hw_fcs_strip = !!(sh->device_attr.raw_packet_caps & + (config->hw_vlan_strip ? "" : "not ")); + config->hw_fcs_strip = !!(sh->device_attr.raw_packet_caps & IBV_RAW_PACKET_CAP_SCATTER_FCS); - DRV_LOG(DEBUG, "FCS stripping configuration is %ssupported", - (config.hw_fcs_strip ? "" : "not ")); #if defined(HAVE_IBV_WQ_FLAG_RX_END_PADDING) hw_padding = !!sh->device_attr.rx_pad_end_addr_align; #elif defined(HAVE_IBV_WQ_FLAGS_PCI_WRITE_END_PADDING) hw_padding = !!(sh->device_attr.device_cap_flags_ex & IBV_DEVICE_PCI_WRITE_END_PADDING); #endif - if (config.hw_padding && !hw_padding) { + if (config->hw_padding && !hw_padding) { DRV_LOG(DEBUG, "Rx end alignment padding isn't supported"); - config.hw_padding = 0; - } else if (config.hw_padding) { + config->hw_padding = 0; + } else if (config->hw_padding) { DRV_LOG(DEBUG, "Rx end alignment padding is enabled"); } - config.tso = (sh->device_attr.max_tso > 0 && + config->tso = (sh->device_attr.max_tso > 0 && (sh->device_attr.tso_supported_qpts & (1 << IBV_QPT_RAW_PACKET))); - if (config.tso) - config.tso_max_payload_sz = sh->device_attr.max_tso; + if (config->tso) + config->tso_max_payload_sz = sh->device_attr.max_tso; /* * MPW is disabled by default, while the Enhanced MPW is enabled * by default. */ - if (config.mps == MLX5_ARG_UNSET) - config.mps = (mps == MLX5_MPW_ENHANCED) ? MLX5_MPW_ENHANCED : + if (config->mps == MLX5_ARG_UNSET) + config->mps = (mps == MLX5_MPW_ENHANCED) ? MLX5_MPW_ENHANCED : MLX5_MPW_DISABLED; else - config.mps = config.mps ? mps : MLX5_MPW_DISABLED; + config->mps = config->mps ? mps : MLX5_MPW_DISABLED; DRV_LOG(INFO, "%sMPS is %s", - config.mps == MLX5_MPW_ENHANCED ? "enhanced " : - config.mps == MLX5_MPW ? "legacy " : "", - config.mps != MLX5_MPW_DISABLED ? "enabled" : "disabled"); - if (config.cqe_comp && !cqe_comp) { + config->mps == MLX5_MPW_ENHANCED ? "enhanced " : + config->mps == MLX5_MPW ? "legacy " : "", + config->mps != MLX5_MPW_DISABLED ? "enabled" : "disabled"); + if (config->cqe_comp && !cqe_comp) { DRV_LOG(WARNING, "Rx CQE compression isn't supported"); - config.cqe_comp = 0; + config->cqe_comp = 0; } - if (config.cqe_pad && !cqe_pad) { + if (config->cqe_pad && !cqe_pad) { DRV_LOG(WARNING, "Rx CQE padding isn't supported"); - config.cqe_pad = 0; - } else if (config.cqe_pad) { + config->cqe_pad = 0; + } else if (config->cqe_pad) { DRV_LOG(INFO, "Rx CQE padding is enabled"); } - if (config.devx) { + if (config->devx) { priv->counter_fallback = 0; - err = mlx5_devx_cmd_query_hca_attr(sh->ctx, &config.hca_attr); + err = mlx5_devx_cmd_query_hca_attr(sh->ctx, &config->hca_attr); if (err) { err = -err; goto error; } - if (!config.hca_attr.flow_counters_dump) + if (!config->hca_attr.flow_counters_dump) priv->counter_fallback = 1; #ifndef HAVE_IBV_DEVX_ASYNC priv->counter_fallback = 1; @@ -837,26 +958,27 @@ err_secondary: if (priv->counter_fallback) DRV_LOG(INFO, "Use fall-back DV counter management"); /* Check for LRO support. */ - if (config.dest_tir && config.hca_attr.lro_cap && - config.dv_flow_en) { + if (config->dest_tir && config->hca_attr.lro_cap && + config->dv_flow_en) { /* TBD check tunnel lro caps. */ - config.lro.supported = config.hca_attr.lro_cap; + config->lro.supported = config->hca_attr.lro_cap; DRV_LOG(DEBUG, "Device supports LRO"); /* * If LRO timeout is not configured by application, * use the minimal supported value. */ - if (!config.lro.timeout) - config.lro.timeout = - config.hca_attr.lro_timer_supported_periods[0]; + if (!config->lro.timeout) + config->lro.timeout = + config->hca_attr.lro_timer_supported_periods[0]; DRV_LOG(DEBUG, "LRO session timeout set to %d usec", - config.lro.timeout); + config->lro.timeout); } #if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER) - if (config.hca_attr.qos.sup && config.hca_attr.qos.srtcm_sup && - config.dv_flow_en) { + if (config->hca_attr.qos.sup && + config->hca_attr.qos.srtcm_sup && + config->dv_flow_en) { uint8_t reg_c_mask = - config.hca_attr.qos.flow_meter_reg_c_ids; + config->hca_attr.qos.flow_meter_reg_c_ids; /* * Meter needs two REG_C's for color match and pre-sfx * flow match. Here get the REG_C for color match. @@ -872,64 +994,64 @@ err_secondary: REG_C_0; priv->mtr_en = 1; priv->mtr_reg_share = - config.hca_attr.qos.flow_meter_reg_share; + config->hca_attr.qos.flow_meter_reg_share; DRV_LOG(DEBUG, "The REG_C meter uses is %d", priv->mtr_color_reg); } } #endif } - if (config.tx_pp) { + if (config->tx_pp) { DRV_LOG(DEBUG, "Timestamp counter frequency %u kHz", - config.hca_attr.dev_freq_khz); + config->hca_attr.dev_freq_khz); DRV_LOG(DEBUG, "Packet pacing is %ssupported", - config.hca_attr.qos.packet_pacing ? "" : "not "); + config->hca_attr.qos.packet_pacing ? "" : "not "); DRV_LOG(DEBUG, "Cross channel ops are %ssupported", - config.hca_attr.cross_channel ? "" : "not "); + config->hca_attr.cross_channel ? "" : "not "); DRV_LOG(DEBUG, "WQE index ignore is %ssupported", - config.hca_attr.wqe_index_ignore ? "" : "not "); + config->hca_attr.wqe_index_ignore ? "" : "not "); DRV_LOG(DEBUG, "Non-wire SQ feature is %ssupported", - config.hca_attr.non_wire_sq ? "" : "not "); + config->hca_attr.non_wire_sq ? "" : "not "); DRV_LOG(DEBUG, "Static WQE SQ feature is %ssupported (%d)", - config.hca_attr.log_max_static_sq_wq ? "" : "not ", - config.hca_attr.log_max_static_sq_wq); + config->hca_attr.log_max_static_sq_wq ? "" : "not ", + config->hca_attr.log_max_static_sq_wq); DRV_LOG(DEBUG, "WQE rate PP mode is %ssupported", - config.hca_attr.qos.wqe_rate_pp ? "" : "not "); - if (!config.devx) { + config->hca_attr.qos.wqe_rate_pp ? "" : "not "); + if (!config->devx) { DRV_LOG(ERR, "DevX is required for packet pacing"); err = ENODEV; goto error; } - if (!config.hca_attr.qos.packet_pacing) { + if (!config->hca_attr.qos.packet_pacing) { DRV_LOG(ERR, "Packet pacing is not supported"); err = ENODEV; goto error; } - if (!config.hca_attr.cross_channel) { + if (!config->hca_attr.cross_channel) { DRV_LOG(ERR, "Cross channel operations are" " required for packet pacing"); err = ENODEV; goto error; } - if (!config.hca_attr.wqe_index_ignore) { + if (!config->hca_attr.wqe_index_ignore) { DRV_LOG(ERR, "WQE index ignore feature is" " required for packet pacing"); err = ENODEV; goto error; } - if (!config.hca_attr.non_wire_sq) { + if (!config->hca_attr.non_wire_sq) { DRV_LOG(ERR, "Non-wire SQ feature is" " required for packet pacing"); err = ENODEV; goto error; } - if (!config.hca_attr.log_max_static_sq_wq) { + if (!config->hca_attr.log_max_static_sq_wq) { DRV_LOG(ERR, "Static WQE SQ feature is" " required for packet pacing"); err = ENODEV; goto error; } - if (!config.hca_attr.qos.wqe_rate_pp) { + if (!config->hca_attr.qos.wqe_rate_pp) { DRV_LOG(ERR, "WQE rate mode is required" " for packet pacing"); err = ENODEV; @@ -942,11 +1064,41 @@ err_secondary: goto error; #endif } - if (config.mprq.enabled && mprq) { - if (config.mprq.stride_num_n && - (config.mprq.stride_num_n > mprq_max_stride_num_n || - config.mprq.stride_num_n < mprq_min_stride_num_n)) { - config.mprq.stride_num_n = + if (config->devx) { + uint32_t reg[MLX5_ST_SZ_DW(register_mtutc)]; + + err = mlx5_devx_cmd_register_read + (sh->ctx, MLX5_REGISTER_ID_MTUTC, 0, + reg, MLX5_ST_SZ_DW(register_mtutc)); + if (!err) { + uint32_t ts_mode; + + /* MTUTC register is read successfully. */ + ts_mode = MLX5_GET(register_mtutc, reg, + time_stamp_mode); + if (ts_mode == MLX5_MTUTC_TIMESTAMP_MODE_REAL_TIME) + config->rt_timestamp = 1; + } else { + /* Kernel does not support register reading. */ + if (config->hca_attr.dev_freq_khz == + (NS_PER_S / MS_PER_S)) + config->rt_timestamp = 1; + } + } + /* + * If HW has bug working with tunnel packet decapsulation and + * scatter FCS, and decapsulation is needed, clear the hw_fcs_strip + * bit. Then DEV_RX_OFFLOAD_KEEP_CRC bit will not be set anymore. + */ + if (config->hca_attr.scatter_fcs_w_decap_disable && config->decap_en) + config->hw_fcs_strip = 0; + DRV_LOG(DEBUG, "FCS stripping configuration is %ssupported", + (config->hw_fcs_strip ? "" : "not ")); + if (config->mprq.enabled && mprq) { + if (config->mprq.stride_num_n && + (config->mprq.stride_num_n > mprq_max_stride_num_n || + config->mprq.stride_num_n < mprq_min_stride_num_n)) { + config->mprq.stride_num_n = RTE_MIN(RTE_MAX(MLX5_MPRQ_STRIDE_NUM_N, mprq_min_stride_num_n), mprq_max_stride_num_n); @@ -954,12 +1106,12 @@ err_secondary: "the number of strides" " for Multi-Packet RQ is out of range," " setting default value (%u)", - 1 << config.mprq.stride_num_n); + 1 << config->mprq.stride_num_n); } - if (config.mprq.stride_size_n && - (config.mprq.stride_size_n > mprq_max_stride_size_n || - config.mprq.stride_size_n < mprq_min_stride_size_n)) { - config.mprq.stride_size_n = + if (config->mprq.stride_size_n && + (config->mprq.stride_size_n > mprq_max_stride_size_n || + config->mprq.stride_size_n < mprq_min_stride_size_n)) { + config->mprq.stride_size_n = RTE_MIN(RTE_MAX(MLX5_MPRQ_STRIDE_SIZE_N, mprq_min_stride_size_n), mprq_max_stride_size_n); @@ -967,16 +1119,16 @@ err_secondary: "the size of a stride" " for Multi-Packet RQ is out of range," " setting default value (%u)", - 1 << config.mprq.stride_size_n); + 1 << config->mprq.stride_size_n); } - config.mprq.min_stride_size_n = mprq_min_stride_size_n; - config.mprq.max_stride_size_n = mprq_max_stride_size_n; - } else if (config.mprq.enabled && !mprq) { + config->mprq.min_stride_size_n = mprq_min_stride_size_n; + config->mprq.max_stride_size_n = mprq_max_stride_size_n; + } else if (config->mprq.enabled && !mprq) { DRV_LOG(WARNING, "Multi-Packet RQ isn't supported"); - config.mprq.enabled = 0; + config->mprq.enabled = 0; } - if (config.max_dump_files_num == 0) - config.max_dump_files_num = 128; + if (config->max_dump_files_num == 0) + config->max_dump_files_num = 128; eth_dev = rte_eth_dev_allocate(name); if (eth_dev == NULL) { DRV_LOG(ERR, "can not allocate rte ethdev"); @@ -1041,7 +1193,7 @@ err_secondary: eth_dev->dev_ops = &mlx5_os_dev_ops; /* Register MAC address. */ claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0)); - if (config.vf && config.vf_nl_en) + if (config->vf && config->vf_nl_en) mlx5_nl_mac_addr_sync(priv->nl_socket_route, mlx5_ifindex(eth_dev), eth_dev->data->mac_addrs, @@ -1069,19 +1221,19 @@ err_secondary: */ mlx5_link_update(eth_dev, 0); #ifdef HAVE_MLX5DV_DR_ESWITCH - if (!(config.hca_attr.eswitch_manager && config.dv_flow_en && + if (!(config->hca_attr.eswitch_manager && config->dv_flow_en && (switch_info->representor || switch_info->master))) - config.dv_esw_en = 0; + config->dv_esw_en = 0; #else - config.dv_esw_en = 0; + config->dv_esw_en = 0; #endif /* Detect minimal data bytes to inline. */ - mlx5_set_min_inline(spawn, &config); + mlx5_set_min_inline(spawn, config); /* Store device configuration on private structure. */ - priv->config = config; + priv->config = *config; /* Create context for virtual machine VLAN workaround. */ priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex); - if (config.dv_flow_en) { + if (config->dv_flow_en) { err = mlx5_alloc_shared_dr(priv); if (err) goto error; @@ -1170,7 +1322,7 @@ error: mlx5_flow_id_pool_release(priv->qrss_id_pool); if (own_domain_id) claim_zero(rte_eth_switch_domain_free(priv->domain_id)); - rte_free(priv); + mlx5_free(priv); if (eth_dev != NULL) eth_dev->data->dev_private = NULL; } @@ -1369,13 +1521,9 @@ mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, int bd = -1; struct mlx5_dev_spawn_data *list = NULL; struct mlx5_dev_config dev_config; + unsigned int dev_config_vf; int ret; - if (mlx5_class_get(pci_dev->device.devargs) != MLX5_CLASS_NET) { - DRV_LOG(DEBUG, "Skip probing - should be probed by other mlx5" - " driver."); - return 1; - } if (rte_eal_process_type() == RTE_PROC_PRIMARY) mlx5_pmd_socket_init(); ret = mlx5_init_once(); @@ -1384,7 +1532,6 @@ mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, strerror(rte_errno)); return -rte_errno; } - MLX5_ASSERT(pci_drv == &mlx5_driver); errno = 0; ibv_list = mlx5_glue->get_device_list(&ret); if (!ibv_list) { @@ -1489,10 +1636,10 @@ mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, * Now we can determine the maximal * amount of devices to be spawned. */ - list = rte_zmalloc("device spawn data", - sizeof(struct mlx5_dev_spawn_data) * - (np ? np : nd), - RTE_CACHE_LINE_SIZE); + list = mlx5_malloc(MLX5_MEM_ZERO, + sizeof(struct mlx5_dev_spawn_data) * + (np ? np : nd), + RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); if (!list) { DRV_LOG(ERR, "spawn data array allocation failure"); rte_errno = ENOMEM; @@ -1694,29 +1841,6 @@ mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, * (i.e. master first, then representors from lowest to highest ID). */ qsort(list, ns, sizeof(*list), mlx5_dev_spawn_data_cmp); - /* Default configuration. */ - dev_config = (struct mlx5_dev_config){ - .hw_padding = 0, - .mps = MLX5_ARG_UNSET, - .dbnc = MLX5_ARG_UNSET, - .rx_vec_en = 1, - .txq_inline_max = MLX5_ARG_UNSET, - .txq_inline_min = MLX5_ARG_UNSET, - .txq_inline_mpw = MLX5_ARG_UNSET, - .txqs_inline = MLX5_ARG_UNSET, - .vf_nl_en = 1, - .mr_ext_memseg_en = 1, - .mprq = { - .enabled = 0, /* Disabled by default. */ - .stride_num_n = 0, - .stride_size_n = 0, - .max_memcpy_len = MLX5_MPRQ_MEMCPY_DEFAULT_LEN, - .min_rxqs_num = MLX5_MPRQ_MIN_RXQS, - }, - .dv_esw_en = 1, - .dv_flow_en = 1, - .log_hp_size = MLX5_ARG_UNSET, - }; /* Device specific configuration. */ switch (pci_dev->id.device_id) { case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: @@ -1726,17 +1850,36 @@ mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, case PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF: case PCI_DEVICE_ID_MELLANOX_CONNECTX6VF: case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXVF: - dev_config.vf = 1; + dev_config_vf = 1; break; default: + dev_config_vf = 0; break; } for (i = 0; i != ns; ++i) { uint32_t restore; + /* Default configuration. */ + memset(&dev_config, 0, sizeof(struct mlx5_dev_config)); + dev_config.vf = dev_config_vf; + dev_config.mps = MLX5_ARG_UNSET; + dev_config.dbnc = MLX5_ARG_UNSET; + dev_config.rx_vec_en = 1; + dev_config.txq_inline_max = MLX5_ARG_UNSET; + dev_config.txq_inline_min = MLX5_ARG_UNSET; + dev_config.txq_inline_mpw = MLX5_ARG_UNSET; + dev_config.txqs_inline = MLX5_ARG_UNSET; + dev_config.vf_nl_en = 1; + dev_config.mr_ext_memseg_en = 1; + dev_config.mprq.max_memcpy_len = MLX5_MPRQ_MEMCPY_DEFAULT_LEN; + dev_config.mprq.min_rxqs_num = MLX5_MPRQ_MIN_RXQS; + dev_config.dv_esw_en = 1; + dev_config.dv_flow_en = 1; + dev_config.decap_en = 1; + dev_config.log_hp_size = MLX5_ARG_UNSET; list[i].eth_dev = mlx5_dev_spawn(&pci_dev->device, &list[i], - dev_config); + &dev_config); if (!list[i].eth_dev) { if (rte_errno != EBUSY && rte_errno != EEXIST) break; @@ -1783,7 +1926,7 @@ exit: if (nl_route >= 0) close(nl_route); if (list) - rte_free(list); + mlx5_free(list); MLX5_ASSERT(ibv_list); mlx5_glue->free_device_list(ibv_list); return ret; @@ -1875,6 +2018,9 @@ mlx5_os_open_device(const struct mlx5_dev_spawn_data *spawn, { int dbmap_env; int err = 0; + + sh->numa_node = spawn->pci_dev->device.numa_node; + pthread_mutex_init(&sh->txpp.mutex, NULL); /* * Configure environment variable "MLX5_BF_SHUT_UP" * before the device creation. The rdma_core library @@ -2011,10 +2157,20 @@ mlx5_os_read_dev_stat(struct mlx5_priv *priv, const char *ctr_name, if (priv->sh) { MKSTR(path, "%s/ports/%d/hw_counters/%s", - priv->sh->ibdev_path, - priv->dev_port, - ctr_name); + priv->sh->ibdev_path, + priv->dev_port, + ctr_name); fd = open(path, O_RDONLY); + /* + * in switchdev the file location is not per port + * but rather in /hw_counters/. + */ + if (fd == -1) { + MKSTR(path1, "%s/hw_counters/%s", + priv->sh->ibdev_path, + ctr_name); + fd = open(path1, O_RDONLY); + } if (fd != -1) { char buf[21] = {'\0'}; ssize_t n = read(fd, buf, sizeof(buf)); @@ -2031,305 +2187,133 @@ mlx5_os_read_dev_stat(struct mlx5_priv *priv, const char *ctr_name, } /** - * Read device counters table. + * Set the reg_mr and dereg_mr call backs * - * @param dev - * Pointer to Ethernet device. - * @param[out] stats - * Counters table output buffer. + * @param reg_mr_cb[out] + * Pointer to reg_mr func + * @param dereg_mr_cb[out] + * Pointer to dereg_mr func * - * @return - * 0 on success and stats is filled, negative errno value otherwise and - * rte_errno is set. */ -int -mlx5_os_read_dev_counters(struct rte_eth_dev *dev, uint64_t *stats) +void +mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, + mlx5_dereg_mr_t *dereg_mr_cb) +{ + *reg_mr_cb = mlx5_verbs_ops.reg_mr; + *dereg_mr_cb = mlx5_verbs_ops.dereg_mr; +} + +/** + * Remove a MAC address from device + * + * @param dev + * Pointer to Ethernet device structure. + * @param index + * MAC address index. + */ +void +mlx5_os_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; - unsigned int i; - struct ifreq ifr; - unsigned int stats_sz = xstats_ctrl->stats_n * sizeof(uint64_t); - unsigned char et_stat_buf[sizeof(struct ethtool_stats) + stats_sz]; - struct ethtool_stats *et_stats = (struct ethtool_stats *)et_stat_buf; - int ret; + const int vf = priv->config.vf; - et_stats->cmd = ETHTOOL_GSTATS; - et_stats->n_stats = xstats_ctrl->stats_n; - ifr.ifr_data = (caddr_t)et_stats; - ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); - if (ret) { - DRV_LOG(WARNING, - "port %u unable to read statistic values from device", - dev->data->port_id); - return ret; - } - for (i = 0; i != xstats_ctrl->mlx5_stats_n; ++i) { - if (xstats_ctrl->info[i].dev) { - ret = mlx5_os_read_dev_stat(priv, - xstats_ctrl->info[i].ctr_name, - &stats[i]); - /* return last xstats counter if fail to read. */ - if (ret == 0) - xstats_ctrl->xstats[i] = stats[i]; - else - stats[i] = xstats_ctrl->xstats[i]; - } else { - stats[i] = (uint64_t) - et_stats->data[xstats_ctrl->dev_table_idx[i]]; - } - } - return 0; + if (vf) + mlx5_nl_mac_addr_remove(priv->nl_socket_route, + mlx5_ifindex(dev), priv->mac_own, + &dev->data->mac_addrs[index], index); } /** - * Query the number of statistics provided by ETHTOOL. + * Adds a MAC address to the device * * @param dev - * Pointer to Ethernet device. + * Pointer to Ethernet device structure. + * @param mac_addr + * MAC address to register. + * @param index + * MAC address index. * * @return - * Number of statistics on success, negative errno value otherwise and - * rte_errno is set. + * 0 on success, a negative errno value otherwise */ int -mlx5_os_get_stats_n(struct rte_eth_dev *dev) +mlx5_os_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac, + uint32_t index) { - struct ethtool_drvinfo drvinfo; - struct ifreq ifr; - int ret; + struct mlx5_priv *priv = dev->data->dev_private; + const int vf = priv->config.vf; + int ret = 0; - drvinfo.cmd = ETHTOOL_GDRVINFO; - ifr.ifr_data = (caddr_t)&drvinfo; - ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); - if (ret) { - DRV_LOG(WARNING, "port %u unable to query number of statistics", - dev->data->port_id); - return ret; - } - return drvinfo.n_stats; + if (vf) + ret = mlx5_nl_mac_addr_add(priv->nl_socket_route, + mlx5_ifindex(dev), priv->mac_own, + mac, index); + return ret; } -static const struct mlx5_counter_ctrl mlx5_counters_init[] = { - { - .dpdk_name = "rx_port_unicast_bytes", - .ctr_name = "rx_vport_unicast_bytes", - }, - { - .dpdk_name = "rx_port_multicast_bytes", - .ctr_name = "rx_vport_multicast_bytes", - }, - { - .dpdk_name = "rx_port_broadcast_bytes", - .ctr_name = "rx_vport_broadcast_bytes", - }, - { - .dpdk_name = "rx_port_unicast_packets", - .ctr_name = "rx_vport_unicast_packets", - }, - { - .dpdk_name = "rx_port_multicast_packets", - .ctr_name = "rx_vport_multicast_packets", - }, - { - .dpdk_name = "rx_port_broadcast_packets", - .ctr_name = "rx_vport_broadcast_packets", - }, - { - .dpdk_name = "tx_port_unicast_bytes", - .ctr_name = "tx_vport_unicast_bytes", - }, - { - .dpdk_name = "tx_port_multicast_bytes", - .ctr_name = "tx_vport_multicast_bytes", - }, - { - .dpdk_name = "tx_port_broadcast_bytes", - .ctr_name = "tx_vport_broadcast_bytes", - }, - { - .dpdk_name = "tx_port_unicast_packets", - .ctr_name = "tx_vport_unicast_packets", - }, - { - .dpdk_name = "tx_port_multicast_packets", - .ctr_name = "tx_vport_multicast_packets", - }, - { - .dpdk_name = "tx_port_broadcast_packets", - .ctr_name = "tx_vport_broadcast_packets", - }, - { - .dpdk_name = "rx_wqe_err", - .ctr_name = "rx_wqe_err", - }, - { - .dpdk_name = "rx_crc_errors_phy", - .ctr_name = "rx_crc_errors_phy", - }, - { - .dpdk_name = "rx_in_range_len_errors_phy", - .ctr_name = "rx_in_range_len_errors_phy", - }, - { - .dpdk_name = "rx_symbol_err_phy", - .ctr_name = "rx_symbol_err_phy", - }, - { - .dpdk_name = "tx_errors_phy", - .ctr_name = "tx_errors_phy", - }, - { - .dpdk_name = "rx_out_of_buffer", - .ctr_name = "out_of_buffer", - .dev = 1, - }, - { - .dpdk_name = "tx_packets_phy", - .ctr_name = "tx_packets_phy", - }, - { - .dpdk_name = "rx_packets_phy", - .ctr_name = "rx_packets_phy", - }, - { - .dpdk_name = "tx_discards_phy", - .ctr_name = "tx_discards_phy", - }, - { - .dpdk_name = "rx_discards_phy", - .ctr_name = "rx_discards_phy", - }, - { - .dpdk_name = "tx_bytes_phy", - .ctr_name = "tx_bytes_phy", - }, - { - .dpdk_name = "rx_bytes_phy", - .ctr_name = "rx_bytes_phy", - }, - /* Representor only */ - { - .dpdk_name = "rx_packets", - .ctr_name = "vport_rx_packets", - }, - { - .dpdk_name = "rx_bytes", - .ctr_name = "vport_rx_bytes", - }, - { - .dpdk_name = "tx_packets", - .ctr_name = "vport_tx_packets", - }, - { - .dpdk_name = "tx_bytes", - .ctr_name = "vport_tx_bytes", - }, -}; - -static const unsigned int xstats_n = RTE_DIM(mlx5_counters_init); +/** + * Modify a VF MAC address + * + * @param priv + * Pointer to device private data. + * @param mac_addr + * MAC address to modify into. + * @param iface_idx + * Net device interface index + * @param vf_index + * VF index + * + * @return + * 0 on success, a negative errno value otherwise + */ +int +mlx5_os_vf_mac_addr_modify(struct mlx5_priv *priv, + unsigned int iface_idx, + struct rte_ether_addr *mac_addr, + int vf_index) +{ + return mlx5_nl_vf_mac_addr_modify + (priv->nl_socket_route, iface_idx, mac_addr, vf_index); +} /** - * Init the structures to read device counters. + * Set device promiscuous mode * * @param dev - * Pointer to Ethernet device. + * Pointer to Ethernet device structure. + * @param enable + * 0 - promiscuous is disabled, otherwise - enabled + * + * @return + * 0 on success, a negative error value otherwise */ -void -mlx5_os_stats_init(struct rte_eth_dev *dev) +int +mlx5_os_set_promisc(struct rte_eth_dev *dev, int enable) { struct mlx5_priv *priv = dev->data->dev_private; - struct mlx5_xstats_ctrl *xstats_ctrl = &priv->xstats_ctrl; - struct mlx5_stats_ctrl *stats_ctrl = &priv->stats_ctrl; - unsigned int i; - unsigned int j; - struct ifreq ifr; - struct ethtool_gstrings *strings = NULL; - unsigned int dev_stats_n; - unsigned int str_sz; - int ret; - - /* So that it won't aggregate for each init. */ - xstats_ctrl->mlx5_stats_n = 0; - ret = mlx5_os_get_stats_n(dev); - if (ret < 0) { - DRV_LOG(WARNING, "port %u no extended statistics available", - dev->data->port_id); - return; - } - dev_stats_n = ret; - /* Allocate memory to grab stat names and values. */ - str_sz = dev_stats_n * ETH_GSTRING_LEN; - strings = (struct ethtool_gstrings *) - rte_malloc("xstats_strings", - str_sz + sizeof(struct ethtool_gstrings), 0); - if (!strings) { - DRV_LOG(WARNING, "port %u unable to allocate memory for xstats", - dev->data->port_id); - return; - } - strings->cmd = ETHTOOL_GSTRINGS; - strings->string_set = ETH_SS_STATS; - strings->len = dev_stats_n; - ifr.ifr_data = (caddr_t)strings; - ret = mlx5_ifreq(dev, SIOCETHTOOL, &ifr); - if (ret) { - DRV_LOG(WARNING, "port %u unable to get statistic names", - dev->data->port_id); - goto free; - } - for (i = 0; i != dev_stats_n; ++i) { - const char *curr_string = (const char *) - &strings->data[i * ETH_GSTRING_LEN]; - for (j = 0; j != xstats_n; ++j) { - if (!strcmp(mlx5_counters_init[j].ctr_name, - curr_string)) { - unsigned int idx = xstats_ctrl->mlx5_stats_n++; - - xstats_ctrl->dev_table_idx[idx] = i; - xstats_ctrl->info[idx] = mlx5_counters_init[j]; - break; - } - } - } - /* Add dev counters. */ - for (i = 0; i != xstats_n; ++i) { - if (mlx5_counters_init[i].dev) { - unsigned int idx = xstats_ctrl->mlx5_stats_n++; - - xstats_ctrl->info[idx] = mlx5_counters_init[i]; - xstats_ctrl->hw_stats[idx] = 0; - } - } - MLX5_ASSERT(xstats_ctrl->mlx5_stats_n <= MLX5_MAX_XSTATS); - xstats_ctrl->stats_n = dev_stats_n; - /* Copy to base at first time. */ - ret = mlx5_os_read_dev_counters(dev, xstats_ctrl->base); - if (ret) - DRV_LOG(ERR, "port %u cannot read device counters: %s", - dev->data->port_id, strerror(rte_errno)); - mlx5_os_read_dev_stat(priv, "out_of_buffer", &stats_ctrl->imissed_base); - stats_ctrl->imissed = 0; -free: - rte_free(strings); + return mlx5_nl_promisc(priv->nl_socket_route, + mlx5_ifindex(dev), !!enable); } /** - * Set the reg_mr and dereg_mr call backs + * Set device promiscuous mode * - * @param reg_mr_cb[out] - * Pointer to reg_mr func - * @param dereg_mr_cb[out] - * Pointer to dereg_mr func + * @param dev + * Pointer to Ethernet device structure. + * @param enable + * 0 - all multicase is disabled, otherwise - enabled * + * @return + * 0 on success, a negative error value otherwise */ -void -mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, - mlx5_dereg_mr_t *dereg_mr_cb) +int +mlx5_os_set_allmulti(struct rte_eth_dev *dev, int enable) { - *reg_mr_cb = mlx5_verbs_ops.reg_mr; - *dereg_mr_cb = mlx5_verbs_ops.dereg_mr; + struct mlx5_priv *priv = dev->data->dev_private; + + return mlx5_nl_allmulti(priv->nl_socket_route, + mlx5_ifindex(dev), !!enable); } const struct eth_dev_ops mlx5_os_dev_ops = { @@ -2351,7 +2335,7 @@ const struct eth_dev_ops mlx5_os_dev_ops = { .xstats_get_names = mlx5_xstats_get_names, .fw_version_get = mlx5_fw_version_get, .dev_infos_get = mlx5_dev_infos_get, - .read_clock = mlx5_read_clock, + .read_clock = mlx5_txpp_read_clock, .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, .vlan_filter_set = mlx5_vlan_filter_set, .rx_queue_setup = mlx5_rx_queue_setup, @@ -2360,6 +2344,10 @@ const struct eth_dev_ops mlx5_os_dev_ops = { .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, .rx_queue_release = mlx5_rx_queue_release, .tx_queue_release = mlx5_tx_queue_release, + .rx_queue_start = mlx5_rx_queue_start, + .rx_queue_stop = mlx5_rx_queue_stop, + .tx_queue_start = mlx5_tx_queue_start, + .tx_queue_stop = mlx5_tx_queue_stop, .flow_ctrl_get = mlx5_dev_get_flow_ctrl, .flow_ctrl_set = mlx5_dev_set_flow_ctrl, .mac_addr_remove = mlx5_mac_addr_remove, @@ -2400,6 +2388,11 @@ const struct eth_dev_ops mlx5_os_dev_sec_ops = { .xstats_get_names = mlx5_xstats_get_names, .fw_version_get = mlx5_fw_version_get, .dev_infos_get = mlx5_dev_infos_get, + .read_clock = mlx5_txpp_read_clock, + .rx_queue_start = mlx5_rx_queue_start, + .rx_queue_stop = mlx5_rx_queue_stop, + .tx_queue_start = mlx5_tx_queue_start, + .tx_queue_stop = mlx5_tx_queue_stop, .rx_descriptor_status = mlx5_rx_descriptor_status, .tx_descriptor_status = mlx5_tx_descriptor_status, .rxq_info_get = mlx5_rxq_info_get, @@ -2430,6 +2423,7 @@ const struct eth_dev_ops mlx5_os_dev_ops_isolate = { .xstats_get_names = mlx5_xstats_get_names, .fw_version_get = mlx5_fw_version_get, .dev_infos_get = mlx5_dev_infos_get, + .read_clock = mlx5_txpp_read_clock, .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, .vlan_filter_set = mlx5_vlan_filter_set, .rx_queue_setup = mlx5_rx_queue_setup, @@ -2438,6 +2432,10 @@ const struct eth_dev_ops mlx5_os_dev_ops_isolate = { .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, .rx_queue_release = mlx5_rx_queue_release, .tx_queue_release = mlx5_tx_queue_release, + .rx_queue_start = mlx5_rx_queue_start, + .rx_queue_stop = mlx5_rx_queue_stop, + .tx_queue_start = mlx5_tx_queue_start, + .tx_queue_stop = mlx5_tx_queue_stop, .flow_ctrl_get = mlx5_dev_get_flow_ctrl, .flow_ctrl_set = mlx5_dev_set_flow_ctrl, .mac_addr_remove = mlx5_mac_addr_remove,