X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Flinux%2Fmlx5_os.c;h=69123e12c368cc4fc675eec81350afea3609f8ea;hb=972a1bf8120d971c04c065221c46c37c266b1f62;hp=2ecb1a56e3ee8ec0790ee0d6423dec9781e6e94f;hpb=2eb4d0107acc2515e0ee1eceb49cf45e6d49fe76;p=dpdk.git diff --git a/drivers/net/mlx5/linux/mlx5_os.c b/drivers/net/mlx5/linux/mlx5_os.c index 2ecb1a56e3..69123e12c3 100644 --- a/drivers/net/mlx5/linux/mlx5_os.c +++ b/drivers/net/mlx5/linux/mlx5_os.c @@ -10,20 +10,11 @@ #include #include #include -#include #include +#include +#include #include -/* Verbs header. */ -/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */ -#ifdef PEDANTIC -#pragma GCC diagnostic ignored "-Wpedantic" -#endif -#include -#ifdef PEDANTIC -#pragma GCC diagnostic error "-Wpedantic" -#endif - #include #include #include @@ -35,20 +26,25 @@ #include #include #include +#include #include #include #include #include +#include +#include #include "mlx5_defs.h" #include "mlx5.h" +#include "mlx5_common_os.h" #include "mlx5_utils.h" #include "mlx5_rxtx.h" #include "mlx5_autoconf.h" #include "mlx5_mr.h" #include "mlx5_flow.h" #include "rte_pmd_mlx5.h" +#include "mlx5_verbs.h" #define MLX5_TAGS_HLIST_ARRAY_SIZE 8192 @@ -61,59 +57,33 @@ #define MLX5DV_CONTEXT_FLAGS_CQE_128B_COMP (1 << 4) #endif -/** - * Get ibv device name. Given an ibv_context pointer - return a - * pointer to the corresponding device name. - * - * @param[in] ctx - * Pointer to ibv context. - * - * @return - * Pointer to device name if ctx is valid, NULL otherwise. - */ -const char * -mlx5_os_get_ctx_device_name(void *ctx) -{ - if (!ctx) - return NULL; - return ((struct ibv_context *)ctx)->device->name; -} +static const char *MZ_MLX5_PMD_SHARED_DATA = "mlx5_pmd_shared_data"; -/** - * Get ibv device path name. Given an ibv_context pointer - return a - * pointer to the corresponding device path name. - * - * @param[in] ctx - * Pointer to ibv context. - * - * @return - * Pointer to device path name if ctx is valid, NULL otherwise. - */ -const char * -mlx5_os_get_ctx_device_path(void *ctx) -{ - if (!ctx) - return NULL; +/* Spinlock for mlx5_shared_data allocation. */ +static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER; - return ((struct ibv_context *)ctx)->device->ibdev_path; -} +/* Process local data for secondary processes. */ +static struct mlx5_local_data mlx5_local_data; /** - * Get umem id. Given a pointer to umem object of type - * 'struct mlx5dv_devx_umem *' - return its id. + * Set the completion channel file descriptor interrupt as non-blocking. * - * @param[in] umem - * Pointer to umem object. + * @param[in] rxq_obj + * Pointer to RQ channel object, which includes the channel fd + * + * @param[out] fd + * The file descriptor (representing the intetrrupt) used in this channel. * * @return - * The umem id if umem is valid, 0 otherwise. + * 0 on successfully setting the fd to non-blocking, non-zero otherwise. */ -uint32_t -mlx5_os_get_umem_id(void *umem) +int +mlx5_os_set_nonblock_channel_fd(int fd) { - if (!umem) - return 0; - return ((struct mlx5dv_devx_umem *)umem)->umem_id; + int flags; + + flags = fcntl(fd, F_GETFL); + return fcntl(fd, F_SETFL, flags | O_NONBLOCK); } /** @@ -183,7 +153,7 @@ mlx5_os_get_dev_attr(void *ctx, struct mlx5_dev_attr *device_attr) * Verbs callback to allocate a memory. This function should allocate the space * according to the size provided residing inside a huge page. * Please note that all allocation must respect the alignment from libmlx5 - * (i.e. currently sysconf(_SC_PAGESIZE)). + * (i.e. currently rte_mem_page_size()). * * @param[in] size * The size in bytes of the memory to allocate. @@ -198,8 +168,13 @@ mlx5_alloc_verbs_buf(size_t size, void *data) { struct mlx5_priv *priv = data; void *ret; - size_t alignment = sysconf(_SC_PAGESIZE); unsigned int socket = SOCKET_ID_ANY; + size_t alignment = rte_mem_page_size(); + if (alignment == (size_t)-1) { + DRV_LOG(ERR, "Failed to get mem page size"); + rte_errno = ENOMEM; + return NULL; + } if (priv->verbs_alloc_ctx.type == MLX5_VERBS_ALLOC_TYPE_TX_QUEUE) { const struct mlx5_txq_ctrl *ctrl = priv->verbs_alloc_ctx.obj; @@ -212,7 +187,7 @@ mlx5_alloc_verbs_buf(size_t size, void *data) socket = ctrl->socket; } MLX5_ASSERT(data != NULL); - ret = rte_malloc_socket(__func__, size, alignment, socket); + ret = mlx5_malloc(0, size, alignment, socket); if (!ret && size) rte_errno = ENOMEM; return ret; @@ -230,7 +205,7 @@ static void mlx5_free_verbs_buf(void *ptr, void *data __rte_unused) { MLX5_ASSERT(data != NULL); - rte_free(ptr); + mlx5_free(ptr); } /** @@ -262,7 +237,7 @@ mlx5_alloc_shared_dr(struct mlx5_priv *priv) snprintf(s, sizeof(s), "%s_tags", sh->ibdev_name); sh->tag_table = mlx5_hlist_create(s, MLX5_TAGS_HLIST_ARRAY_SIZE); if (!sh->tag_table) { - DRV_LOG(ERR, "tags with hash creation failed.\n"); + DRV_LOG(ERR, "tags with hash creation failed."); err = ENOMEM; goto error; } @@ -400,6 +375,109 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv) mlx5_free_table_hash_list(priv); } +/** + * Initialize shared data between primary and secondary process. + * + * A memzone is reserved by primary process and secondary processes attach to + * the memzone. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx5_init_shared_data(void) +{ + const struct rte_memzone *mz; + int ret = 0; + + rte_spinlock_lock(&mlx5_shared_data_lock); + if (mlx5_shared_data == NULL) { + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + /* Allocate shared memory. */ + mz = rte_memzone_reserve(MZ_MLX5_PMD_SHARED_DATA, + sizeof(*mlx5_shared_data), + SOCKET_ID_ANY, 0); + if (mz == NULL) { + DRV_LOG(ERR, + "Cannot allocate mlx5 shared data"); + ret = -rte_errno; + goto error; + } + mlx5_shared_data = mz->addr; + memset(mlx5_shared_data, 0, sizeof(*mlx5_shared_data)); + rte_spinlock_init(&mlx5_shared_data->lock); + } else { + /* Lookup allocated shared memory. */ + mz = rte_memzone_lookup(MZ_MLX5_PMD_SHARED_DATA); + if (mz == NULL) { + DRV_LOG(ERR, + "Cannot attach mlx5 shared data"); + ret = -rte_errno; + goto error; + } + mlx5_shared_data = mz->addr; + memset(&mlx5_local_data, 0, sizeof(mlx5_local_data)); + } + } +error: + rte_spinlock_unlock(&mlx5_shared_data_lock); + return ret; +} + +/** + * PMD global initialization. + * + * Independent from individual device, this function initializes global + * per-PMD data structures distinguishing primary and secondary processes. + * Hence, each initialization is called once per a process. + * + * @return + * 0 on success, a negative errno value otherwise and rte_errno is set. + */ +static int +mlx5_init_once(void) +{ + struct mlx5_shared_data *sd; + struct mlx5_local_data *ld = &mlx5_local_data; + int ret = 0; + + if (mlx5_init_shared_data()) + return -rte_errno; + sd = mlx5_shared_data; + MLX5_ASSERT(sd); + rte_spinlock_lock(&sd->lock); + switch (rte_eal_process_type()) { + case RTE_PROC_PRIMARY: + if (sd->init_done) + break; + LIST_INIT(&sd->mem_event_cb_list); + rte_rwlock_init(&sd->mem_event_rwlock); + rte_mem_event_callback_register("MLX5_MEM_EVENT_CB", + mlx5_mr_mem_event_cb, NULL); + ret = mlx5_mp_init_primary(MLX5_MP_NAME, + mlx5_mp_os_primary_handle); + if (ret) + goto out; + sd->init_done = true; + break; + case RTE_PROC_SECONDARY: + if (ld->init_done) + break; + ret = mlx5_mp_init_secondary(MLX5_MP_NAME, + mlx5_mp_os_secondary_handle); + if (ret) + goto out; + ++sd->secondary_cnt; + ld->init_done = true; + break; + default: + break; + } +out: + rte_spinlock_unlock(&sd->lock); + return ret; +} + /** * Spawn an Ethernet device from Verbs information. * @@ -420,7 +498,7 @@ mlx5_os_free_shared_dr(struct mlx5_priv *priv) static struct rte_eth_dev * mlx5_dev_spawn(struct rte_device *dpdk_dev, struct mlx5_dev_spawn_data *spawn, - struct mlx5_dev_config config) + struct mlx5_dev_config *config) { const struct mlx5_switch_info *switch_info = &spawn->info; struct mlx5_dev_ctx_shared *sh = NULL; @@ -482,10 +560,12 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, /* Bonding device. */ if (!switch_info->representor) snprintf(name, sizeof(name), "%s_%s", - dpdk_dev->name, spawn->ibv_dev->name); + dpdk_dev->name, + mlx5_os_get_dev_device_name(spawn->phys_dev)); else snprintf(name, sizeof(name), "%s_%s_representor_%u", - dpdk_dev->name, spawn->ibv_dev->name, + dpdk_dev->name, + mlx5_os_get_dev_device_name(spawn->phys_dev), switch_info->port_name); } /* check if the device is already spawned */ @@ -504,7 +584,7 @@ mlx5_dev_spawn(struct rte_device *dpdk_dev, return NULL; } eth_dev->device = dpdk_dev; - eth_dev->dev_ops = &mlx5_dev_sec_ops; + eth_dev->dev_ops = &mlx5_os_dev_sec_ops; err = mlx5_proc_priv_init(eth_dev); if (err) return NULL; @@ -536,19 +616,20 @@ err_secondary: * devargs here to get ones, and later proceed devargs again * to override some hardware settings. */ - err = mlx5_args(&config, dpdk_dev->devargs); + err = mlx5_args(config, dpdk_dev->devargs); if (err) { err = rte_errno; DRV_LOG(ERR, "failed to process device arguments: %s", strerror(rte_errno)); goto error; } - sh = mlx5_alloc_shared_ibctx(spawn, &config); + mlx5_malloc_mem_select(config->sys_mem_en); + sh = mlx5_alloc_shared_dev_ctx(spawn, config); if (!sh) return NULL; - config.devx = sh->devx; + config->devx = sh->devx; #ifdef HAVE_MLX5DV_DR_ACTION_DEST_DEVX_TIR - config.dest_tir = 1; + config->dest_tir = 1; #endif #ifdef HAVE_IBV_MLX5_MOD_SWP dv_attr.comp_mask |= MLX5DV_CONTEXT_MASK_SWP; @@ -581,7 +662,7 @@ err_secondary: swp = dv_attr.sw_parsing_caps.sw_parsing_offloads; DRV_LOG(DEBUG, "SWP support: %u", swp); #endif - config.swp = !!swp; + config->swp = !!swp; #ifdef HAVE_IBV_DEVICE_STRIDING_RQ_SUPPORT if (dv_attr.comp_mask & MLX5DV_CONTEXT_MASK_STRIDING_RQ) { struct mlx5dv_striding_rq_caps mprq_caps = @@ -614,7 +695,7 @@ err_secondary: cqe_comp = 0; else cqe_comp = 1; - config.cqe_comp = cqe_comp; + config->cqe_comp = cqe_comp; #ifdef HAVE_IBV_MLX5_MOD_CQE_128B_PAD /* Whether device supports 128B Rx CQE padding. */ cqe_pad = RTE_CACHE_LINE_SIZE == 128 && @@ -635,7 +716,7 @@ err_secondary: DRV_LOG(WARNING, "tunnel offloading disabled due to old OFED/rdma-core version"); #endif - config.tunnel_en = tunnel_en; + config->tunnel_en = tunnel_en; #ifdef HAVE_IBV_DEVICE_MPLS_SUPPORT mpls_en = ((dv_attr.tunnel_offloads_caps & MLX5DV_RAW_PACKET_CAP_TUNNELED_OFFLOAD_CW_MPLS_OVER_GRE) && @@ -647,9 +728,9 @@ err_secondary: DRV_LOG(WARNING, "MPLS over GRE/UDP tunnel offloading disabled due to" " old OFED/rdma-core version or firmware configuration"); #endif - config.mpls_en = mpls_en; + config->mpls_en = mpls_en; /* Check port status. */ - err = mlx5_glue->query_port(sh->ctx, spawn->ibv_port, &port_attr); + err = mlx5_glue->query_port(sh->ctx, spawn->phys_port, &port_attr); if (err) { DRV_LOG(ERR, "port query failed: %s", strerror(err)); goto error; @@ -664,26 +745,20 @@ err_secondary: mlx5_glue->port_state_str(port_attr.state), port_attr.state); /* Allocate private eth device data. */ - priv = rte_zmalloc("ethdev private structure", + priv = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE, sizeof(*priv), - RTE_CACHE_LINE_SIZE); + RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); if (priv == NULL) { DRV_LOG(ERR, "priv allocation failure"); err = ENOMEM; goto error; } priv->sh = sh; - priv->ibv_port = spawn->ibv_port; + priv->dev_port = spawn->phys_port; priv->pci_dev = spawn->pci_dev; priv->mtu = RTE_ETHER_MTU; priv->mp_id.port_id = port_id; strlcpy(priv->mp_id.name, MLX5_MP_NAME, RTE_MP_MAX_NAME_LEN); -#ifndef RTE_ARCH_64 - /* Initialize UAR access locks for 32bit implementations. */ - rte_spinlock_init(&priv->uar_lock_cq); - for (i = 0; i < MLX5_UAR_PAGE_NUM_MAX; i++) - rte_spinlock_init(&priv->uar_lock[i]); -#endif /* Some internal functions rely on Netlink sockets, open them now. */ priv->nl_socket_rdma = mlx5_nl_init(NETLINK_RDMA); priv->nl_socket_route = mlx5_nl_init(NETLINK_ROUTE); @@ -703,12 +778,13 @@ err_secondary: if (switch_info->representor || switch_info->master) { devx_port.comp_mask = MLX5DV_DEVX_PORT_VPORT | MLX5DV_DEVX_PORT_MATCH_REG_C_0; - err = mlx5_glue->devx_port_query(sh->ctx, spawn->ibv_port, + err = mlx5_glue->devx_port_query(sh->ctx, spawn->phys_port, &devx_port); if (err) { DRV_LOG(WARNING, "can't query devx port %d on device %s", - spawn->ibv_port, spawn->ibv_dev->name); + spawn->phys_port, + mlx5_os_get_dev_device_name(spawn->phys_dev)); devx_port.comp_mask = 0; } } @@ -718,14 +794,18 @@ err_secondary: if (!priv->vport_meta_mask) { DRV_LOG(ERR, "vport zero mask for port %d" " on bonding device %s", - spawn->ibv_port, spawn->ibv_dev->name); + spawn->phys_port, + mlx5_os_get_dev_device_name + (spawn->phys_dev)); err = ENOTSUP; goto error; } if (priv->vport_meta_tag & ~priv->vport_meta_mask) { DRV_LOG(ERR, "invalid vport tag for port %d" " on bonding device %s", - spawn->ibv_port, spawn->ibv_dev->name); + spawn->phys_port, + mlx5_os_get_dev_device_name + (spawn->phys_dev)); err = ENOTSUP; goto error; } @@ -735,7 +815,8 @@ err_secondary: } else if (spawn->pf_bond >= 0) { DRV_LOG(ERR, "can't deduce vport index for port %d" " on bonding device %s", - spawn->ibv_port, spawn->ibv_dev->name); + spawn->phys_port, + mlx5_os_get_dev_device_name(spawn->phys_dev)); err = ENOTSUP; goto error; } else { @@ -788,90 +869,88 @@ err_secondary: own_domain_id = 1; } /* Override some values set by hardware configuration. */ - mlx5_args(&config, dpdk_dev->devargs); - err = mlx5_dev_check_sibling_config(priv, &config); + mlx5_args(config, dpdk_dev->devargs); + err = mlx5_dev_check_sibling_config(priv, config); if (err) goto error; - config.hw_csum = !!(sh->device_attr.device_cap_flags_ex & + config->hw_csum = !!(sh->device_attr.device_cap_flags_ex & IBV_DEVICE_RAW_IP_CSUM); DRV_LOG(DEBUG, "checksum offloading is %ssupported", - (config.hw_csum ? "" : "not ")); + (config->hw_csum ? "" : "not ")); #if !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V42) && \ !defined(HAVE_IBV_DEVICE_COUNTERS_SET_V45) DRV_LOG(DEBUG, "counters are not supported"); #endif #if !defined(HAVE_IBV_FLOW_DV_SUPPORT) || !defined(HAVE_MLX5DV_DR) - if (config.dv_flow_en) { + if (config->dv_flow_en) { DRV_LOG(WARNING, "DV flow is not supported"); - config.dv_flow_en = 0; + config->dv_flow_en = 0; } #endif - config.ind_table_max_size = + config->ind_table_max_size = sh->device_attr.max_rwq_indirection_table_size; /* * Remove this check once DPDK supports larger/variable * indirection tables. */ - if (config.ind_table_max_size > (unsigned int)ETH_RSS_RETA_SIZE_512) - config.ind_table_max_size = ETH_RSS_RETA_SIZE_512; + if (config->ind_table_max_size > (unsigned int)ETH_RSS_RETA_SIZE_512) + config->ind_table_max_size = ETH_RSS_RETA_SIZE_512; DRV_LOG(DEBUG, "maximum Rx indirection table size is %u", - config.ind_table_max_size); - config.hw_vlan_strip = !!(sh->device_attr.raw_packet_caps & + config->ind_table_max_size); + config->hw_vlan_strip = !!(sh->device_attr.raw_packet_caps & IBV_RAW_PACKET_CAP_CVLAN_STRIPPING); DRV_LOG(DEBUG, "VLAN stripping is %ssupported", - (config.hw_vlan_strip ? "" : "not ")); - config.hw_fcs_strip = !!(sh->device_attr.raw_packet_caps & + (config->hw_vlan_strip ? "" : "not ")); + config->hw_fcs_strip = !!(sh->device_attr.raw_packet_caps & IBV_RAW_PACKET_CAP_SCATTER_FCS); - DRV_LOG(DEBUG, "FCS stripping configuration is %ssupported", - (config.hw_fcs_strip ? "" : "not ")); #if defined(HAVE_IBV_WQ_FLAG_RX_END_PADDING) hw_padding = !!sh->device_attr.rx_pad_end_addr_align; #elif defined(HAVE_IBV_WQ_FLAGS_PCI_WRITE_END_PADDING) hw_padding = !!(sh->device_attr.device_cap_flags_ex & IBV_DEVICE_PCI_WRITE_END_PADDING); #endif - if (config.hw_padding && !hw_padding) { + if (config->hw_padding && !hw_padding) { DRV_LOG(DEBUG, "Rx end alignment padding isn't supported"); - config.hw_padding = 0; - } else if (config.hw_padding) { + config->hw_padding = 0; + } else if (config->hw_padding) { DRV_LOG(DEBUG, "Rx end alignment padding is enabled"); } - config.tso = (sh->device_attr.max_tso > 0 && + config->tso = (sh->device_attr.max_tso > 0 && (sh->device_attr.tso_supported_qpts & (1 << IBV_QPT_RAW_PACKET))); - if (config.tso) - config.tso_max_payload_sz = sh->device_attr.max_tso; + if (config->tso) + config->tso_max_payload_sz = sh->device_attr.max_tso; /* * MPW is disabled by default, while the Enhanced MPW is enabled * by default. */ - if (config.mps == MLX5_ARG_UNSET) - config.mps = (mps == MLX5_MPW_ENHANCED) ? MLX5_MPW_ENHANCED : + if (config->mps == MLX5_ARG_UNSET) + config->mps = (mps == MLX5_MPW_ENHANCED) ? MLX5_MPW_ENHANCED : MLX5_MPW_DISABLED; else - config.mps = config.mps ? mps : MLX5_MPW_DISABLED; + config->mps = config->mps ? mps : MLX5_MPW_DISABLED; DRV_LOG(INFO, "%sMPS is %s", - config.mps == MLX5_MPW_ENHANCED ? "enhanced " : - config.mps == MLX5_MPW ? "legacy " : "", - config.mps != MLX5_MPW_DISABLED ? "enabled" : "disabled"); - if (config.cqe_comp && !cqe_comp) { + config->mps == MLX5_MPW_ENHANCED ? "enhanced " : + config->mps == MLX5_MPW ? "legacy " : "", + config->mps != MLX5_MPW_DISABLED ? "enabled" : "disabled"); + if (config->cqe_comp && !cqe_comp) { DRV_LOG(WARNING, "Rx CQE compression isn't supported"); - config.cqe_comp = 0; + config->cqe_comp = 0; } - if (config.cqe_pad && !cqe_pad) { + if (config->cqe_pad && !cqe_pad) { DRV_LOG(WARNING, "Rx CQE padding isn't supported"); - config.cqe_pad = 0; - } else if (config.cqe_pad) { + config->cqe_pad = 0; + } else if (config->cqe_pad) { DRV_LOG(INFO, "Rx CQE padding is enabled"); } - if (config.devx) { + if (config->devx) { priv->counter_fallback = 0; - err = mlx5_devx_cmd_query_hca_attr(sh->ctx, &config.hca_attr); + err = mlx5_devx_cmd_query_hca_attr(sh->ctx, &config->hca_attr); if (err) { err = -err; goto error; } - if (!config.hca_attr.flow_counters_dump) + if (!config->hca_attr.flow_counters_dump) priv->counter_fallback = 1; #ifndef HAVE_IBV_DEVX_ASYNC priv->counter_fallback = 1; @@ -879,26 +958,27 @@ err_secondary: if (priv->counter_fallback) DRV_LOG(INFO, "Use fall-back DV counter management"); /* Check for LRO support. */ - if (config.dest_tir && config.hca_attr.lro_cap && - config.dv_flow_en) { + if (config->dest_tir && config->hca_attr.lro_cap && + config->dv_flow_en) { /* TBD check tunnel lro caps. */ - config.lro.supported = config.hca_attr.lro_cap; + config->lro.supported = config->hca_attr.lro_cap; DRV_LOG(DEBUG, "Device supports LRO"); /* * If LRO timeout is not configured by application, * use the minimal supported value. */ - if (!config.lro.timeout) - config.lro.timeout = - config.hca_attr.lro_timer_supported_periods[0]; + if (!config->lro.timeout) + config->lro.timeout = + config->hca_attr.lro_timer_supported_periods[0]; DRV_LOG(DEBUG, "LRO session timeout set to %d usec", - config.lro.timeout); + config->lro.timeout); } #if defined(HAVE_MLX5DV_DR) && defined(HAVE_MLX5_DR_CREATE_ACTION_FLOW_METER) - if (config.hca_attr.qos.sup && config.hca_attr.qos.srtcm_sup && - config.dv_flow_en) { + if (config->hca_attr.qos.sup && + config->hca_attr.qos.srtcm_sup && + config->dv_flow_en) { uint8_t reg_c_mask = - config.hca_attr.qos.flow_meter_reg_c_ids; + config->hca_attr.qos.flow_meter_reg_c_ids; /* * Meter needs two REG_C's for color match and pre-sfx * flow match. Here get the REG_C for color match. @@ -914,18 +994,112 @@ err_secondary: REG_C_0; priv->mtr_en = 1; priv->mtr_reg_share = - config.hca_attr.qos.flow_meter_reg_share; + config->hca_attr.qos.flow_meter_reg_share; DRV_LOG(DEBUG, "The REG_C meter uses is %d", priv->mtr_color_reg); } } #endif } - if (config.mprq.enabled && mprq) { - if (config.mprq.stride_num_n && - (config.mprq.stride_num_n > mprq_max_stride_num_n || - config.mprq.stride_num_n < mprq_min_stride_num_n)) { - config.mprq.stride_num_n = + if (config->tx_pp) { + DRV_LOG(DEBUG, "Timestamp counter frequency %u kHz", + config->hca_attr.dev_freq_khz); + DRV_LOG(DEBUG, "Packet pacing is %ssupported", + config->hca_attr.qos.packet_pacing ? "" : "not "); + DRV_LOG(DEBUG, "Cross channel ops are %ssupported", + config->hca_attr.cross_channel ? "" : "not "); + DRV_LOG(DEBUG, "WQE index ignore is %ssupported", + config->hca_attr.wqe_index_ignore ? "" : "not "); + DRV_LOG(DEBUG, "Non-wire SQ feature is %ssupported", + config->hca_attr.non_wire_sq ? "" : "not "); + DRV_LOG(DEBUG, "Static WQE SQ feature is %ssupported (%d)", + config->hca_attr.log_max_static_sq_wq ? "" : "not ", + config->hca_attr.log_max_static_sq_wq); + DRV_LOG(DEBUG, "WQE rate PP mode is %ssupported", + config->hca_attr.qos.wqe_rate_pp ? "" : "not "); + if (!config->devx) { + DRV_LOG(ERR, "DevX is required for packet pacing"); + err = ENODEV; + goto error; + } + if (!config->hca_attr.qos.packet_pacing) { + DRV_LOG(ERR, "Packet pacing is not supported"); + err = ENODEV; + goto error; + } + if (!config->hca_attr.cross_channel) { + DRV_LOG(ERR, "Cross channel operations are" + " required for packet pacing"); + err = ENODEV; + goto error; + } + if (!config->hca_attr.wqe_index_ignore) { + DRV_LOG(ERR, "WQE index ignore feature is" + " required for packet pacing"); + err = ENODEV; + goto error; + } + if (!config->hca_attr.non_wire_sq) { + DRV_LOG(ERR, "Non-wire SQ feature is" + " required for packet pacing"); + err = ENODEV; + goto error; + } + if (!config->hca_attr.log_max_static_sq_wq) { + DRV_LOG(ERR, "Static WQE SQ feature is" + " required for packet pacing"); + err = ENODEV; + goto error; + } + if (!config->hca_attr.qos.wqe_rate_pp) { + DRV_LOG(ERR, "WQE rate mode is required" + " for packet pacing"); + err = ENODEV; + goto error; + } +#ifndef HAVE_MLX5DV_DEVX_UAR_OFFSET + DRV_LOG(ERR, "DevX does not provide UAR offset," + " can't create queues for packet pacing"); + err = ENODEV; + goto error; +#endif + } + if (config->devx) { + uint32_t reg[MLX5_ST_SZ_DW(register_mtutc)]; + + err = config->hca_attr.access_register_user ? + mlx5_devx_cmd_register_read + (sh->ctx, MLX5_REGISTER_ID_MTUTC, 0, + reg, MLX5_ST_SZ_DW(register_mtutc)) : ENOTSUP; + if (!err) { + uint32_t ts_mode; + + /* MTUTC register is read successfully. */ + ts_mode = MLX5_GET(register_mtutc, reg, + time_stamp_mode); + if (ts_mode == MLX5_MTUTC_TIMESTAMP_MODE_REAL_TIME) + config->rt_timestamp = 1; + } else { + /* Kernel does not support register reading. */ + if (config->hca_attr.dev_freq_khz == + (NS_PER_S / MS_PER_S)) + config->rt_timestamp = 1; + } + } + /* + * If HW has bug working with tunnel packet decapsulation and + * scatter FCS, and decapsulation is needed, clear the hw_fcs_strip + * bit. Then DEV_RX_OFFLOAD_KEEP_CRC bit will not be set anymore. + */ + if (config->hca_attr.scatter_fcs_w_decap_disable && config->decap_en) + config->hw_fcs_strip = 0; + DRV_LOG(DEBUG, "FCS stripping configuration is %ssupported", + (config->hw_fcs_strip ? "" : "not ")); + if (config->mprq.enabled && mprq) { + if (config->mprq.stride_num_n && + (config->mprq.stride_num_n > mprq_max_stride_num_n || + config->mprq.stride_num_n < mprq_min_stride_num_n)) { + config->mprq.stride_num_n = RTE_MIN(RTE_MAX(MLX5_MPRQ_STRIDE_NUM_N, mprq_min_stride_num_n), mprq_max_stride_num_n); @@ -933,12 +1107,12 @@ err_secondary: "the number of strides" " for Multi-Packet RQ is out of range," " setting default value (%u)", - 1 << config.mprq.stride_num_n); + 1 << config->mprq.stride_num_n); } - if (config.mprq.stride_size_n && - (config.mprq.stride_size_n > mprq_max_stride_size_n || - config.mprq.stride_size_n < mprq_min_stride_size_n)) { - config.mprq.stride_size_n = + if (config->mprq.stride_size_n && + (config->mprq.stride_size_n > mprq_max_stride_size_n || + config->mprq.stride_size_n < mprq_min_stride_size_n)) { + config->mprq.stride_size_n = RTE_MIN(RTE_MAX(MLX5_MPRQ_STRIDE_SIZE_N, mprq_min_stride_size_n), mprq_max_stride_size_n); @@ -946,16 +1120,16 @@ err_secondary: "the size of a stride" " for Multi-Packet RQ is out of range," " setting default value (%u)", - 1 << config.mprq.stride_size_n); + 1 << config->mprq.stride_size_n); } - config.mprq.min_stride_size_n = mprq_min_stride_size_n; - config.mprq.max_stride_size_n = mprq_max_stride_size_n; - } else if (config.mprq.enabled && !mprq) { + config->mprq.min_stride_size_n = mprq_min_stride_size_n; + config->mprq.max_stride_size_n = mprq_max_stride_size_n; + } else if (config->mprq.enabled && !mprq) { DRV_LOG(WARNING, "Multi-Packet RQ isn't supported"); - config.mprq.enabled = 0; + config->mprq.enabled = 0; } - if (config.max_dump_files_num == 0) - config.max_dump_files_num = 128; + if (config->max_dump_files_num == 0) + config->max_dump_files_num = 128; eth_dev = rte_eth_dev_allocate(name); if (eth_dev == NULL) { DRV_LOG(ERR, "can not allocate rte ethdev"); @@ -1017,10 +1191,10 @@ err_secondary: /* Initialize burst functions to prevent crashes before link-up. */ eth_dev->rx_pkt_burst = removed_rx_burst; eth_dev->tx_pkt_burst = removed_tx_burst; - eth_dev->dev_ops = &mlx5_dev_ops; + eth_dev->dev_ops = &mlx5_os_dev_ops; /* Register MAC address. */ claim_zero(mlx5_mac_addr_add(eth_dev, &mac, 0, 0)); - if (config.vf && config.vf_nl_en) + if (config->vf && config->vf_nl_en) mlx5_nl_mac_addr_sync(priv->nl_socket_route, mlx5_ifindex(eth_dev), eth_dev->data->mac_addrs, @@ -1030,14 +1204,13 @@ err_secondary: TAILQ_INIT(&priv->flow_meters); TAILQ_INIT(&priv->flow_meter_profiles); /* Hint libmlx5 to use PMD allocator for data plane resources */ - struct mlx5dv_ctx_allocators alctr = { - .alloc = &mlx5_alloc_verbs_buf, - .free = &mlx5_free_verbs_buf, - .data = priv, - }; mlx5_glue->dv_set_context_attr(sh->ctx, - MLX5DV_CTX_ATTR_BUF_ALLOCATORS, - (void *)((uintptr_t)&alctr)); + MLX5DV_CTX_ATTR_BUF_ALLOCATORS, + (void *)((uintptr_t)&(struct mlx5dv_ctx_allocators){ + .alloc = &mlx5_alloc_verbs_buf, + .free = &mlx5_free_verbs_buf, + .data = priv, + })); /* Bring Ethernet device up. */ DRV_LOG(DEBUG, "port %u forcing Ethernet interface up", eth_dev->data->port_id); @@ -1049,19 +1222,19 @@ err_secondary: */ mlx5_link_update(eth_dev, 0); #ifdef HAVE_MLX5DV_DR_ESWITCH - if (!(config.hca_attr.eswitch_manager && config.dv_flow_en && + if (!(config->hca_attr.eswitch_manager && config->dv_flow_en && (switch_info->representor || switch_info->master))) - config.dv_esw_en = 0; + config->dv_esw_en = 0; #else - config.dv_esw_en = 0; + config->dv_esw_en = 0; #endif /* Detect minimal data bytes to inline. */ - mlx5_set_min_inline(spawn, &config); + mlx5_set_min_inline(spawn, config); /* Store device configuration on private structure. */ - priv->config = config; + priv->config = *config; /* Create context for virtual machine VLAN workaround. */ priv->vmwa_context = mlx5_vlan_vmwa_init(eth_dev, spawn->ifindex); - if (config.dv_flow_en) { + if (config->dv_flow_en) { err = mlx5_alloc_shared_dr(priv); if (err) goto error; @@ -1150,7 +1323,7 @@ error: mlx5_flow_id_pool_release(priv->qrss_id_pool); if (own_domain_id) claim_zero(rte_eth_switch_domain_free(priv->domain_id)); - rte_free(priv); + mlx5_free(priv); if (eth_dev != NULL) eth_dev->data->dev_private = NULL; } @@ -1162,7 +1335,7 @@ error: rte_eth_dev_release_port(eth_dev); } if (sh) - mlx5_free_shared_ibctx(sh); + mlx5_free_shared_dev_ctx(sh); MLX5_ASSERT(err > 0); rte_errno = err; return NULL; @@ -1349,13 +1522,9 @@ mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, int bd = -1; struct mlx5_dev_spawn_data *list = NULL; struct mlx5_dev_config dev_config; + unsigned int dev_config_vf; int ret; - if (mlx5_class_get(pci_dev->device.devargs) != MLX5_CLASS_NET) { - DRV_LOG(DEBUG, "Skip probing - should be probed by other mlx5" - " driver."); - return 1; - } if (rte_eal_process_type() == RTE_PROC_PRIMARY) mlx5_pmd_socket_init(); ret = mlx5_init_once(); @@ -1364,7 +1533,6 @@ mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, strerror(rte_errno)); return -rte_errno; } - MLX5_ASSERT(pci_drv == &mlx5_driver); errno = 0; ibv_list = mlx5_glue->get_device_list(&ret); if (!ibv_list) { @@ -1469,10 +1637,10 @@ mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, * Now we can determine the maximal * amount of devices to be spawned. */ - list = rte_zmalloc("device spawn data", - sizeof(struct mlx5_dev_spawn_data) * - (np ? np : nd), - RTE_CACHE_LINE_SIZE); + list = mlx5_malloc(MLX5_MEM_ZERO, + sizeof(struct mlx5_dev_spawn_data) * + (np ? np : nd), + RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY); if (!list) { DRV_LOG(ERR, "spawn data array allocation failure"); rte_errno = ENOMEM; @@ -1491,13 +1659,15 @@ mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, MLX5_ASSERT(np); for (i = 1; i <= np; ++i) { list[ns].max_port = np; - list[ns].ibv_port = i; - list[ns].ibv_dev = ibv_match[0]; + list[ns].phys_port = i; + list[ns].phys_dev = ibv_match[0]; list[ns].eth_dev = NULL; list[ns].pci_dev = pci_dev; list[ns].pf_bond = bd; list[ns].ifindex = mlx5_nl_ifindex - (nl_rdma, list[ns].ibv_dev->name, i); + (nl_rdma, + mlx5_os_get_dev_device_name + (list[ns].phys_dev), i); if (!list[ns].ifindex) { /* * No network interface index found for the @@ -1531,6 +1701,8 @@ mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, if (list[ns].info.port_name == bd) ns++; break; + case MLX5_PHYS_PORT_NAME_TYPE_PFHPF: + /* Fallthrough */ case MLX5_PHYS_PORT_NAME_TYPE_PFVF: if (list[ns].info.pf_num == bd) ns++; @@ -1573,15 +1745,17 @@ mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, for (i = 0; i != nd; ++i) { memset(&list[ns].info, 0, sizeof(list[ns].info)); list[ns].max_port = 1; - list[ns].ibv_port = 1; - list[ns].ibv_dev = ibv_match[i]; + list[ns].phys_port = 1; + list[ns].phys_dev = ibv_match[i]; list[ns].eth_dev = NULL; list[ns].pci_dev = pci_dev; list[ns].pf_bond = -1; list[ns].ifindex = 0; if (nl_rdma >= 0) list[ns].ifindex = mlx5_nl_ifindex - (nl_rdma, list[ns].ibv_dev->name, 1); + (nl_rdma, + mlx5_os_get_dev_device_name + (list[ns].phys_dev), 1); if (!list[ns].ifindex) { char ifname[IF_NAMESIZE]; @@ -1603,8 +1777,8 @@ mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, */ continue; } - ret = mlx5_get_master_ifname - (ibv_match[i]->ibdev_path, &ifname); + ret = mlx5_get_ifname_sysfs + (ibv_match[i]->ibdev_path, ifname); if (!ret) list[ns].ifindex = if_nametoindex(ifname); @@ -1668,29 +1842,6 @@ mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, * (i.e. master first, then representors from lowest to highest ID). */ qsort(list, ns, sizeof(*list), mlx5_dev_spawn_data_cmp); - /* Default configuration. */ - dev_config = (struct mlx5_dev_config){ - .hw_padding = 0, - .mps = MLX5_ARG_UNSET, - .dbnc = MLX5_ARG_UNSET, - .rx_vec_en = 1, - .txq_inline_max = MLX5_ARG_UNSET, - .txq_inline_min = MLX5_ARG_UNSET, - .txq_inline_mpw = MLX5_ARG_UNSET, - .txqs_inline = MLX5_ARG_UNSET, - .vf_nl_en = 1, - .mr_ext_memseg_en = 1, - .mprq = { - .enabled = 0, /* Disabled by default. */ - .stride_num_n = 0, - .stride_size_n = 0, - .max_memcpy_len = MLX5_MPRQ_MEMCPY_DEFAULT_LEN, - .min_rxqs_num = MLX5_MPRQ_MIN_RXQS, - }, - .dv_esw_en = 1, - .dv_flow_en = 1, - .log_hp_size = MLX5_ARG_UNSET, - }; /* Device specific configuration. */ switch (pci_dev->id.device_id) { case PCI_DEVICE_ID_MELLANOX_CONNECTX4VF: @@ -1700,17 +1851,36 @@ mlx5_os_pci_probe(struct rte_pci_driver *pci_drv __rte_unused, case PCI_DEVICE_ID_MELLANOX_CONNECTX5BFVF: case PCI_DEVICE_ID_MELLANOX_CONNECTX6VF: case PCI_DEVICE_ID_MELLANOX_CONNECTX6DXVF: - dev_config.vf = 1; + dev_config_vf = 1; break; default: + dev_config_vf = 0; break; } for (i = 0; i != ns; ++i) { uint32_t restore; + /* Default configuration. */ + memset(&dev_config, 0, sizeof(struct mlx5_dev_config)); + dev_config.vf = dev_config_vf; + dev_config.mps = MLX5_ARG_UNSET; + dev_config.dbnc = MLX5_ARG_UNSET; + dev_config.rx_vec_en = 1; + dev_config.txq_inline_max = MLX5_ARG_UNSET; + dev_config.txq_inline_min = MLX5_ARG_UNSET; + dev_config.txq_inline_mpw = MLX5_ARG_UNSET; + dev_config.txqs_inline = MLX5_ARG_UNSET; + dev_config.vf_nl_en = 1; + dev_config.mr_ext_memseg_en = 1; + dev_config.mprq.max_memcpy_len = MLX5_MPRQ_MEMCPY_DEFAULT_LEN; + dev_config.mprq.min_rxqs_num = MLX5_MPRQ_MIN_RXQS; + dev_config.dv_esw_en = 1; + dev_config.dv_flow_en = 1; + dev_config.decap_en = 1; + dev_config.log_hp_size = MLX5_ARG_UNSET; list[i].eth_dev = mlx5_dev_spawn(&pci_dev->device, &list[i], - dev_config); + &dev_config); if (!list[i].eth_dev) { if (rte_errno != EBUSY && rte_errno != EEXIST) break; @@ -1757,7 +1927,7 @@ exit: if (nl_route >= 0) close(nl_route); if (list) - rte_free(list); + mlx5_free(list); MLX5_ASSERT(ibv_list); mlx5_glue->free_device_list(ibv_list); return ret; @@ -1849,6 +2019,9 @@ mlx5_os_open_device(const struct mlx5_dev_spawn_data *spawn, { int dbmap_env; int err = 0; + + sh->numa_node = spawn->pci_dev->device.numa_node; + pthread_mutex_init(&sh->txpp.mutex, NULL); /* * Configure environment variable "MLX5_BF_SHUT_UP" * before the device creation. The rdma_core library @@ -1858,7 +2031,7 @@ mlx5_os_open_device(const struct mlx5_dev_spawn_data *spawn, dbmap_env = mlx5_config_doorbell_mapping_env(config); /* Try to open IB device with DV first, then usual Verbs. */ errno = 0; - sh->ctx = mlx5_glue->dv_open_device(spawn->ibv_dev); + sh->ctx = mlx5_glue->dv_open_device(spawn->phys_dev); if (sh->ctx) { sh->devx = 1; DRV_LOG(DEBUG, "DevX is supported"); @@ -1866,7 +2039,7 @@ mlx5_os_open_device(const struct mlx5_dev_spawn_data *spawn, mlx5_restore_doorbell_mapping_env(dbmap_env); } else { /* The environment variable is still configured. */ - sh->ctx = mlx5_glue->open_device(spawn->ibv_dev); + sh->ctx = mlx5_glue->open_device(spawn->phys_dev); err = errno ? errno : ENODEV; /* * The environment variable is not needed anymore, @@ -1914,19 +2087,21 @@ mlx5_os_dev_shared_handler_install(struct mlx5_dev_ctx_shared *sh) if (sh->devx) { #ifdef HAVE_IBV_DEVX_ASYNC sh->intr_handle_devx.fd = -1; - sh->devx_comp = mlx5_glue->devx_create_cmd_comp(sh->ctx); - if (!sh->devx_comp) { + sh->devx_comp = + (void *)mlx5_glue->devx_create_cmd_comp(sh->ctx); + struct mlx5dv_devx_cmd_comp *devx_comp = sh->devx_comp; + if (!devx_comp) { DRV_LOG(INFO, "failed to allocate devx_comp."); return; } - flags = fcntl(sh->devx_comp->fd, F_GETFL); - ret = fcntl(sh->devx_comp->fd, F_SETFL, flags | O_NONBLOCK); + flags = fcntl(devx_comp->fd, F_GETFL); + ret = fcntl(devx_comp->fd, F_SETFL, flags | O_NONBLOCK); if (ret) { DRV_LOG(INFO, "failed to change file descriptor" " devx comp"); return; } - sh->intr_handle_devx.fd = sh->devx_comp->fd; + sh->intr_handle_devx.fd = devx_comp->fd; sh->intr_handle_devx.type = RTE_INTR_HANDLE_EXT; if (rte_intr_callback_register(&sh->intr_handle_devx, mlx5_dev_interrupt_handler_devx, sh)) { @@ -1960,3 +2135,329 @@ mlx5_os_dev_shared_handler_uninstall(struct mlx5_dev_ctx_shared *sh) mlx5_glue->devx_destroy_cmd_comp(sh->devx_comp); #endif } + +/** + * Read statistics by a named counter. + * + * @param[in] priv + * Pointer to the private device data structure. + * @param[in] ctr_name + * Pointer to the name of the statistic counter to read + * @param[out] stat + * Pointer to read statistic value. + * @return + * 0 on success and stat is valud, 1 if failed to read the value + * rte_errno is set. + * + */ +int +mlx5_os_read_dev_stat(struct mlx5_priv *priv, const char *ctr_name, + uint64_t *stat) +{ + int fd; + + if (priv->sh) { + MKSTR(path, "%s/ports/%d/hw_counters/%s", + priv->sh->ibdev_path, + priv->dev_port, + ctr_name); + fd = open(path, O_RDONLY); + /* + * in switchdev the file location is not per port + * but rather in /hw_counters/. + */ + if (fd == -1) { + MKSTR(path1, "%s/hw_counters/%s", + priv->sh->ibdev_path, + ctr_name); + fd = open(path1, O_RDONLY); + } + if (fd != -1) { + char buf[21] = {'\0'}; + ssize_t n = read(fd, buf, sizeof(buf)); + + close(fd); + if (n != -1) { + *stat = strtoull(buf, NULL, 10); + return 0; + } + } + } + *stat = 0; + return 1; +} + +/** + * Set the reg_mr and dereg_mr call backs + * + * @param reg_mr_cb[out] + * Pointer to reg_mr func + * @param dereg_mr_cb[out] + * Pointer to dereg_mr func + * + */ +void +mlx5_os_set_reg_mr_cb(mlx5_reg_mr_t *reg_mr_cb, + mlx5_dereg_mr_t *dereg_mr_cb) +{ + *reg_mr_cb = mlx5_verbs_ops.reg_mr; + *dereg_mr_cb = mlx5_verbs_ops.dereg_mr; +} + +/** + * Remove a MAC address from device + * + * @param dev + * Pointer to Ethernet device structure. + * @param index + * MAC address index. + */ +void +mlx5_os_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) +{ + struct mlx5_priv *priv = dev->data->dev_private; + const int vf = priv->config.vf; + + if (vf) + mlx5_nl_mac_addr_remove(priv->nl_socket_route, + mlx5_ifindex(dev), priv->mac_own, + &dev->data->mac_addrs[index], index); +} + +/** + * Adds a MAC address to the device + * + * @param dev + * Pointer to Ethernet device structure. + * @param mac_addr + * MAC address to register. + * @param index + * MAC address index. + * + * @return + * 0 on success, a negative errno value otherwise + */ +int +mlx5_os_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac, + uint32_t index) +{ + struct mlx5_priv *priv = dev->data->dev_private; + const int vf = priv->config.vf; + int ret = 0; + + if (vf) + ret = mlx5_nl_mac_addr_add(priv->nl_socket_route, + mlx5_ifindex(dev), priv->mac_own, + mac, index); + return ret; +} + +/** + * Modify a VF MAC address + * + * @param priv + * Pointer to device private data. + * @param mac_addr + * MAC address to modify into. + * @param iface_idx + * Net device interface index + * @param vf_index + * VF index + * + * @return + * 0 on success, a negative errno value otherwise + */ +int +mlx5_os_vf_mac_addr_modify(struct mlx5_priv *priv, + unsigned int iface_idx, + struct rte_ether_addr *mac_addr, + int vf_index) +{ + return mlx5_nl_vf_mac_addr_modify + (priv->nl_socket_route, iface_idx, mac_addr, vf_index); +} + +/** + * Set device promiscuous mode + * + * @param dev + * Pointer to Ethernet device structure. + * @param enable + * 0 - promiscuous is disabled, otherwise - enabled + * + * @return + * 0 on success, a negative error value otherwise + */ +int +mlx5_os_set_promisc(struct rte_eth_dev *dev, int enable) +{ + struct mlx5_priv *priv = dev->data->dev_private; + + return mlx5_nl_promisc(priv->nl_socket_route, + mlx5_ifindex(dev), !!enable); +} + +/** + * Set device promiscuous mode + * + * @param dev + * Pointer to Ethernet device structure. + * @param enable + * 0 - all multicase is disabled, otherwise - enabled + * + * @return + * 0 on success, a negative error value otherwise + */ +int +mlx5_os_set_allmulti(struct rte_eth_dev *dev, int enable) +{ + struct mlx5_priv *priv = dev->data->dev_private; + + return mlx5_nl_allmulti(priv->nl_socket_route, + mlx5_ifindex(dev), !!enable); +} + +const struct eth_dev_ops mlx5_os_dev_ops = { + .dev_configure = mlx5_dev_configure, + .dev_start = mlx5_dev_start, + .dev_stop = mlx5_dev_stop, + .dev_set_link_down = mlx5_set_link_down, + .dev_set_link_up = mlx5_set_link_up, + .dev_close = mlx5_dev_close, + .promiscuous_enable = mlx5_promiscuous_enable, + .promiscuous_disable = mlx5_promiscuous_disable, + .allmulticast_enable = mlx5_allmulticast_enable, + .allmulticast_disable = mlx5_allmulticast_disable, + .link_update = mlx5_link_update, + .stats_get = mlx5_stats_get, + .stats_reset = mlx5_stats_reset, + .xstats_get = mlx5_xstats_get, + .xstats_reset = mlx5_xstats_reset, + .xstats_get_names = mlx5_xstats_get_names, + .fw_version_get = mlx5_fw_version_get, + .dev_infos_get = mlx5_dev_infos_get, + .read_clock = mlx5_txpp_read_clock, + .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, + .vlan_filter_set = mlx5_vlan_filter_set, + .rx_queue_setup = mlx5_rx_queue_setup, + .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup, + .tx_queue_setup = mlx5_tx_queue_setup, + .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, + .rx_queue_release = mlx5_rx_queue_release, + .tx_queue_release = mlx5_tx_queue_release, + .rx_queue_start = mlx5_rx_queue_start, + .rx_queue_stop = mlx5_rx_queue_stop, + .tx_queue_start = mlx5_tx_queue_start, + .tx_queue_stop = mlx5_tx_queue_stop, + .flow_ctrl_get = mlx5_dev_get_flow_ctrl, + .flow_ctrl_set = mlx5_dev_set_flow_ctrl, + .mac_addr_remove = mlx5_mac_addr_remove, + .mac_addr_add = mlx5_mac_addr_add, + .mac_addr_set = mlx5_mac_addr_set, + .set_mc_addr_list = mlx5_set_mc_addr_list, + .mtu_set = mlx5_dev_set_mtu, + .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, + .vlan_offload_set = mlx5_vlan_offload_set, + .reta_update = mlx5_dev_rss_reta_update, + .reta_query = mlx5_dev_rss_reta_query, + .rss_hash_update = mlx5_rss_hash_update, + .rss_hash_conf_get = mlx5_rss_hash_conf_get, + .filter_ctrl = mlx5_dev_filter_ctrl, + .rx_descriptor_status = mlx5_rx_descriptor_status, + .tx_descriptor_status = mlx5_tx_descriptor_status, + .rxq_info_get = mlx5_rxq_info_get, + .txq_info_get = mlx5_txq_info_get, + .rx_burst_mode_get = mlx5_rx_burst_mode_get, + .tx_burst_mode_get = mlx5_tx_burst_mode_get, + .rx_queue_count = mlx5_rx_queue_count, + .rx_queue_intr_enable = mlx5_rx_intr_enable, + .rx_queue_intr_disable = mlx5_rx_intr_disable, + .is_removed = mlx5_is_removed, + .udp_tunnel_port_add = mlx5_udp_tunnel_port_add, + .get_module_info = mlx5_get_module_info, + .get_module_eeprom = mlx5_get_module_eeprom, + .hairpin_cap_get = mlx5_hairpin_cap_get, + .mtr_ops_get = mlx5_flow_meter_ops_get, +}; + +/* Available operations from secondary process. */ +const struct eth_dev_ops mlx5_os_dev_sec_ops = { + .stats_get = mlx5_stats_get, + .stats_reset = mlx5_stats_reset, + .xstats_get = mlx5_xstats_get, + .xstats_reset = mlx5_xstats_reset, + .xstats_get_names = mlx5_xstats_get_names, + .fw_version_get = mlx5_fw_version_get, + .dev_infos_get = mlx5_dev_infos_get, + .read_clock = mlx5_txpp_read_clock, + .rx_queue_start = mlx5_rx_queue_start, + .rx_queue_stop = mlx5_rx_queue_stop, + .tx_queue_start = mlx5_tx_queue_start, + .tx_queue_stop = mlx5_tx_queue_stop, + .rx_descriptor_status = mlx5_rx_descriptor_status, + .tx_descriptor_status = mlx5_tx_descriptor_status, + .rxq_info_get = mlx5_rxq_info_get, + .txq_info_get = mlx5_txq_info_get, + .rx_burst_mode_get = mlx5_rx_burst_mode_get, + .tx_burst_mode_get = mlx5_tx_burst_mode_get, + .get_module_info = mlx5_get_module_info, + .get_module_eeprom = mlx5_get_module_eeprom, +}; + +/* Available operations in flow isolated mode. */ +const struct eth_dev_ops mlx5_os_dev_ops_isolate = { + .dev_configure = mlx5_dev_configure, + .dev_start = mlx5_dev_start, + .dev_stop = mlx5_dev_stop, + .dev_set_link_down = mlx5_set_link_down, + .dev_set_link_up = mlx5_set_link_up, + .dev_close = mlx5_dev_close, + .promiscuous_enable = mlx5_promiscuous_enable, + .promiscuous_disable = mlx5_promiscuous_disable, + .allmulticast_enable = mlx5_allmulticast_enable, + .allmulticast_disable = mlx5_allmulticast_disable, + .link_update = mlx5_link_update, + .stats_get = mlx5_stats_get, + .stats_reset = mlx5_stats_reset, + .xstats_get = mlx5_xstats_get, + .xstats_reset = mlx5_xstats_reset, + .xstats_get_names = mlx5_xstats_get_names, + .fw_version_get = mlx5_fw_version_get, + .dev_infos_get = mlx5_dev_infos_get, + .read_clock = mlx5_txpp_read_clock, + .dev_supported_ptypes_get = mlx5_dev_supported_ptypes_get, + .vlan_filter_set = mlx5_vlan_filter_set, + .rx_queue_setup = mlx5_rx_queue_setup, + .rx_hairpin_queue_setup = mlx5_rx_hairpin_queue_setup, + .tx_queue_setup = mlx5_tx_queue_setup, + .tx_hairpin_queue_setup = mlx5_tx_hairpin_queue_setup, + .rx_queue_release = mlx5_rx_queue_release, + .tx_queue_release = mlx5_tx_queue_release, + .rx_queue_start = mlx5_rx_queue_start, + .rx_queue_stop = mlx5_rx_queue_stop, + .tx_queue_start = mlx5_tx_queue_start, + .tx_queue_stop = mlx5_tx_queue_stop, + .flow_ctrl_get = mlx5_dev_get_flow_ctrl, + .flow_ctrl_set = mlx5_dev_set_flow_ctrl, + .mac_addr_remove = mlx5_mac_addr_remove, + .mac_addr_add = mlx5_mac_addr_add, + .mac_addr_set = mlx5_mac_addr_set, + .set_mc_addr_list = mlx5_set_mc_addr_list, + .mtu_set = mlx5_dev_set_mtu, + .vlan_strip_queue_set = mlx5_vlan_strip_queue_set, + .vlan_offload_set = mlx5_vlan_offload_set, + .filter_ctrl = mlx5_dev_filter_ctrl, + .rx_descriptor_status = mlx5_rx_descriptor_status, + .tx_descriptor_status = mlx5_tx_descriptor_status, + .rxq_info_get = mlx5_rxq_info_get, + .txq_info_get = mlx5_txq_info_get, + .rx_burst_mode_get = mlx5_rx_burst_mode_get, + .tx_burst_mode_get = mlx5_tx_burst_mode_get, + .rx_queue_intr_enable = mlx5_rx_intr_enable, + .rx_queue_intr_disable = mlx5_rx_intr_disable, + .is_removed = mlx5_is_removed, + .get_module_info = mlx5_get_module_info, + .get_module_eeprom = mlx5_get_module_eeprom, + .hairpin_cap_get = mlx5_hairpin_cap_get, + .mtr_ops_get = mlx5_flow_meter_ops_get, +};