#include <sys/mman.h>
#include <linux/rtnetlink.h>
-/* Verbs header. */
-/* ISO C doesn't support unnamed structs/unions, disabling -pedantic. */
-#ifdef PEDANTIC
-#pragma GCC diagnostic ignored "-Wpedantic"
-#endif
-#include <infiniband/verbs.h>
-#ifdef PEDANTIC
-#pragma GCC diagnostic error "-Wpedantic"
-#endif
-
#include <rte_malloc.h>
#include <rte_ethdev_driver.h>
#include <rte_ethdev_pci.h>
#include <mlx5_common.h>
#include <mlx5_common_os.h>
#include <mlx5_common_mp.h>
+#include <mlx5_common_pci.h>
+#include <mlx5_malloc.h>
#include "mlx5_defs.h"
#include "mlx5.h"
/* Flow memory reclaim mode. */
#define MLX5_RECLAIM_MEM "reclaim_mem_mode"
-static const char *MZ_MLX5_PMD_SHARED_DATA = "mlx5_pmd_shared_data";
+/* The default memory allocator used in PMD. */
+#define MLX5_SYS_MEM_EN "sys_mem_en"
+/* Decap will be used or not. */
+#define MLX5_DECAP_EN "decap_en"
/* Shared memory between primary and secondary processes. */
struct mlx5_shared_data *mlx5_shared_data;
-/* Spinlock for mlx5_shared_data allocation. */
-static rte_spinlock_t mlx5_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
-
-/* Process local data for secondary processes. */
-static struct mlx5_local_data mlx5_local_data;
+/** Driver-specific log messages type. */
+int mlx5_logtype;
static LIST_HEAD(, mlx5_dev_ctx_shared) mlx5_dev_ctx_list =
LIST_HEAD_INITIALIZER();
.grow_shift = 2,
.need_lock = 0,
.release_mem_en = 1,
- .malloc = rte_malloc_socket,
- .free = rte_free,
+ .malloc = mlx5_malloc,
+ .free = mlx5_free,
.type = "mlx5_encap_decap_ipool",
},
{
.grow_shift = 2,
.need_lock = 0,
.release_mem_en = 1,
- .malloc = rte_malloc_socket,
- .free = rte_free,
+ .malloc = mlx5_malloc,
+ .free = mlx5_free,
.type = "mlx5_push_vlan_ipool",
},
{
.grow_shift = 2,
.need_lock = 0,
.release_mem_en = 1,
- .malloc = rte_malloc_socket,
- .free = rte_free,
+ .malloc = mlx5_malloc,
+ .free = mlx5_free,
.type = "mlx5_tag_ipool",
},
{
.grow_shift = 2,
.need_lock = 0,
.release_mem_en = 1,
- .malloc = rte_malloc_socket,
- .free = rte_free,
+ .malloc = mlx5_malloc,
+ .free = mlx5_free,
.type = "mlx5_port_id_ipool",
},
{
.grow_shift = 2,
.need_lock = 0,
.release_mem_en = 1,
- .malloc = rte_malloc_socket,
- .free = rte_free,
+ .malloc = mlx5_malloc,
+ .free = mlx5_free,
.type = "mlx5_jump_ipool",
},
#endif
.grow_shift = 2,
.need_lock = 0,
.release_mem_en = 1,
- .malloc = rte_malloc_socket,
- .free = rte_free,
+ .malloc = mlx5_malloc,
+ .free = mlx5_free,
.type = "mlx5_meter_ipool",
},
{
.grow_shift = 2,
.need_lock = 0,
.release_mem_en = 1,
- .malloc = rte_malloc_socket,
- .free = rte_free,
+ .malloc = mlx5_malloc,
+ .free = mlx5_free,
.type = "mlx5_mcp_ipool",
},
{
.grow_shift = 2,
.need_lock = 0,
.release_mem_en = 1,
- .malloc = rte_malloc_socket,
- .free = rte_free,
+ .malloc = mlx5_malloc,
+ .free = mlx5_free,
.type = "mlx5_hrxq_ipool",
},
{
.grow_shift = 2,
.need_lock = 0,
.release_mem_en = 1,
- .malloc = rte_malloc_socket,
- .free = rte_free,
+ .malloc = mlx5_malloc,
+ .free = mlx5_free,
.type = "mlx5_flow_handle_ipool",
},
{
.trunk_size = 4096,
.need_lock = 1,
.release_mem_en = 1,
- .malloc = rte_malloc_socket,
- .free = rte_free,
+ .malloc = mlx5_malloc,
+ .free = mlx5_free,
.type = "rte_flow_ipool",
},
};
struct mlx5_flow_id_pool *pool;
void *mem;
- pool = rte_zmalloc("id pool allocation", sizeof(*pool),
- RTE_CACHE_LINE_SIZE);
+ pool = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*pool),
+ RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
if (!pool) {
DRV_LOG(ERR, "can't allocate id pool");
rte_errno = ENOMEM;
return NULL;
}
- mem = rte_zmalloc("", MLX5_FLOW_MIN_ID_POOL_SIZE * sizeof(uint32_t),
- RTE_CACHE_LINE_SIZE);
+ mem = mlx5_malloc(MLX5_MEM_ZERO,
+ MLX5_FLOW_MIN_ID_POOL_SIZE * sizeof(uint32_t),
+ RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
if (!mem) {
DRV_LOG(ERR, "can't allocate mem for id pool");
rte_errno = ENOMEM;
pool->max_id = max_id;
return pool;
error:
- rte_free(pool);
+ mlx5_free(pool);
return NULL;
}
void
mlx5_flow_id_pool_release(struct mlx5_flow_id_pool *pool)
{
- rte_free(pool->free_arr);
- rte_free(pool);
+ mlx5_free(pool->free_arr);
+ mlx5_free(pool);
}
/**
size = pool->curr - pool->free_arr;
size2 = size * MLX5_ID_GENERATION_ARRAY_FACTOR;
MLX5_ASSERT(size2 > size);
- mem = rte_malloc("", size2 * sizeof(uint32_t), 0);
+ mem = mlx5_malloc(0, size2 * sizeof(uint32_t), 0,
+ SOCKET_ID_ANY);
if (!mem) {
DRV_LOG(ERR, "can't allocate mem for id pool");
rte_errno = ENOMEM;
return -rte_errno;
}
memcpy(mem, pool->free_arr, size * sizeof(uint32_t));
- rte_free(pool->free_arr);
+ mlx5_free(pool->free_arr);
pool->free_arr = mem;
pool->curr = pool->free_arr + size;
pool->last = pool->free_arr + size2;
LIST_REMOVE(mng, next);
claim_zero(mlx5_devx_cmd_destroy(mng->dm));
claim_zero(mlx5_glue->devx_umem_dereg(mng->umem));
- rte_free(mem);
+ mlx5_free(mem);
}
/**
(pool, j)->dcs));
}
TAILQ_REMOVE(&sh->cmng.ccont[i].pool_list, pool, next);
- rte_free(pool);
+ mlx5_free(pool);
pool = TAILQ_FIRST(&sh->cmng.ccont[i].pool_list);
}
- rte_free(sh->cmng.ccont[i].pools);
+ mlx5_free(sh->cmng.ccont[i].pools);
}
mng = LIST_FIRST(&sh->cmng.mem_mngs);
while (mng) {
}
/* No device found, we have to create new shared context. */
MLX5_ASSERT(spawn->max_port);
- sh = rte_zmalloc("ethdev shared ib context",
+ sh = mlx5_malloc(MLX5_MEM_ZERO | MLX5_MEM_RTE,
sizeof(struct mlx5_dev_ctx_shared) +
spawn->max_port *
sizeof(struct mlx5_dev_shared_port),
- RTE_CACHE_LINE_SIZE);
+ RTE_CACHE_LINE_SIZE, SOCKET_ID_ANY);
if (!sh) {
DRV_LOG(ERR, "shared context allocation failure");
rte_errno = ENOMEM;
err = ENOMEM;
goto error;
}
+ sh->devx_rx_uar = mlx5_glue->devx_alloc_uar(sh->ctx, 0);
+ if (!sh->devx_rx_uar) {
+ DRV_LOG(ERR, "Failed to allocate Rx DevX UAR.");
+ err = ENOMEM;
+ goto error;
+ }
}
sh->flow_id_pool = mlx5_flow_id_pool_alloc
((1 << HAIRPIN_FLOW_ID_BITS) - 1);
claim_zero(mlx5_devx_cmd_destroy(sh->tis));
if (sh->td)
claim_zero(mlx5_devx_cmd_destroy(sh->td));
+ if (sh->devx_rx_uar)
+ mlx5_glue->devx_free_uar(sh->devx_rx_uar);
if (sh->pd)
claim_zero(mlx5_glue->dealloc_pd(sh->pd));
if (sh->ctx)
claim_zero(mlx5_glue->close_device(sh->ctx));
if (sh->flow_id_pool)
mlx5_flow_id_pool_release(sh->flow_id_pool);
- rte_free(sh);
+ mlx5_free(sh);
MLX5_ASSERT(err > 0);
rte_errno = err;
return NULL;
mlx5_mr_release_cache(&sh->share_cache);
/* Remove context from the global device list. */
LIST_REMOVE(sh, next);
+ pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex);
/*
* Ensure there is no async event handler installed.
* Only primary process handles async device events.
claim_zero(mlx5_devx_cmd_destroy(sh->tis));
if (sh->td)
claim_zero(mlx5_devx_cmd_destroy(sh->td));
+ if (sh->devx_rx_uar)
+ mlx5_glue->devx_free_uar(sh->devx_rx_uar);
if (sh->ctx)
claim_zero(mlx5_glue->close_device(sh->ctx));
if (sh->flow_id_pool)
mlx5_flow_id_pool_release(sh->flow_id_pool);
pthread_mutex_destroy(&sh->txpp.mutex);
- rte_free(sh);
+ mlx5_free(sh);
+ return;
exit:
pthread_mutex_unlock(&mlx5_dev_ctx_list_mutex);
}
entry);
MLX5_ASSERT(tbl_data);
mlx5_hlist_remove(sh->flow_tbls, pos);
- rte_free(tbl_data);
+ mlx5_free(tbl_data);
}
table_key.direction = 1;
pos = mlx5_hlist_lookup(sh->flow_tbls, table_key.v64);
entry);
MLX5_ASSERT(tbl_data);
mlx5_hlist_remove(sh->flow_tbls, pos);
- rte_free(tbl_data);
+ mlx5_free(tbl_data);
}
table_key.direction = 0;
table_key.domain = 1;
entry);
MLX5_ASSERT(tbl_data);
mlx5_hlist_remove(sh->flow_tbls, pos);
- rte_free(tbl_data);
+ mlx5_free(tbl_data);
}
mlx5_hlist_destroy(sh->flow_tbls, NULL, NULL);
}
.direction = 0,
}
};
- struct mlx5_flow_tbl_data_entry *tbl_data = rte_zmalloc(NULL,
- sizeof(*tbl_data), 0);
+ struct mlx5_flow_tbl_data_entry *tbl_data = mlx5_malloc(MLX5_MEM_ZERO,
+ sizeof(*tbl_data), 0,
+ SOCKET_ID_ANY);
if (!tbl_data) {
err = ENOMEM;
rte_atomic32_init(&tbl_data->tbl.refcnt);
rte_atomic32_inc(&tbl_data->tbl.refcnt);
table_key.direction = 1;
- tbl_data = rte_zmalloc(NULL, sizeof(*tbl_data), 0);
+ tbl_data = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*tbl_data), 0,
+ SOCKET_ID_ANY);
if (!tbl_data) {
err = ENOMEM;
goto error;
rte_atomic32_inc(&tbl_data->tbl.refcnt);
table_key.direction = 0;
table_key.domain = 1;
- tbl_data = rte_zmalloc(NULL, sizeof(*tbl_data), 0);
+ tbl_data = mlx5_malloc(MLX5_MEM_ZERO, sizeof(*tbl_data), 0,
+ SOCKET_ID_ANY);
if (!tbl_data) {
err = ENOMEM;
goto error;
return err;
}
-/**
- * Initialize shared data between primary and secondary process.
- *
- * A memzone is reserved by primary process and secondary processes attach to
- * the memzone.
- *
- * @return
- * 0 on success, a negative errno value otherwise and rte_errno is set.
- */
-static int
-mlx5_init_shared_data(void)
-{
- const struct rte_memzone *mz;
- int ret = 0;
-
- rte_spinlock_lock(&mlx5_shared_data_lock);
- if (mlx5_shared_data == NULL) {
- if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
- /* Allocate shared memory. */
- mz = rte_memzone_reserve(MZ_MLX5_PMD_SHARED_DATA,
- sizeof(*mlx5_shared_data),
- SOCKET_ID_ANY, 0);
- if (mz == NULL) {
- DRV_LOG(ERR,
- "Cannot allocate mlx5 shared data");
- ret = -rte_errno;
- goto error;
- }
- mlx5_shared_data = mz->addr;
- memset(mlx5_shared_data, 0, sizeof(*mlx5_shared_data));
- rte_spinlock_init(&mlx5_shared_data->lock);
- } else {
- /* Lookup allocated shared memory. */
- mz = rte_memzone_lookup(MZ_MLX5_PMD_SHARED_DATA);
- if (mz == NULL) {
- DRV_LOG(ERR,
- "Cannot attach mlx5 shared data");
- ret = -rte_errno;
- goto error;
- }
- mlx5_shared_data = mz->addr;
- memset(&mlx5_local_data, 0, sizeof(mlx5_local_data));
- }
- }
-error:
- rte_spinlock_unlock(&mlx5_shared_data_lock);
- return ret;
-}
-
/**
* Retrieve integer value from environment variable.
*
*/
ppriv_size =
sizeof(struct mlx5_proc_priv) + priv->txqs_n * sizeof(void *);
- ppriv = rte_malloc_socket("mlx5_proc_priv", ppriv_size,
- RTE_CACHE_LINE_SIZE, dev->device->numa_node);
+ ppriv = mlx5_malloc(MLX5_MEM_RTE, ppriv_size, RTE_CACHE_LINE_SIZE,
+ dev->device->numa_node);
if (!ppriv) {
rte_errno = ENOMEM;
return -rte_errno;
{
if (!dev->process_private)
return;
- rte_free(dev->process_private);
+ mlx5_free(dev->process_private);
dev->process_private = NULL;
}
dev->tx_pkt_burst = removed_tx_burst;
rte_wmb();
/* Disable datapath on secondary process. */
- mlx5_mp_req_stop_rxtx(dev);
+ mlx5_mp_os_req_stop_rxtx(dev);
/* Free the eCPRI flex parser resource. */
mlx5_flex_parser_ecpri_release(dev);
if (priv->rxqs != NULL) {
mlx5_mprq_free_mp(dev);
mlx5_os_free_shared_dr(priv);
if (priv->rss_conf.rss_key != NULL)
- rte_free(priv->rss_conf.rss_key);
+ mlx5_free(priv->rss_conf.rss_key);
if (priv->reta_idx != NULL)
- rte_free(priv->reta_idx);
+ mlx5_free(priv->reta_idx);
if (priv->config.vf)
mlx5_nl_mac_addr_flush(priv->nl_socket_route, mlx5_ifindex(dev),
dev->data->mac_addrs,
return -rte_errno;
}
config->reclaim_mode = tmp;
+ } else if (strcmp(MLX5_SYS_MEM_EN, key) == 0) {
+ config->sys_mem_en = !!tmp;
+ } else if (strcmp(MLX5_DECAP_EN, key) == 0) {
+ config->decap_en = !!tmp;
} else {
DRV_LOG(WARNING, "%s: unknown parameter", key);
rte_errno = EINVAL;
MLX5_CLASS_ARG_NAME,
MLX5_HP_BUF_SIZE,
MLX5_RECLAIM_MEM,
+ MLX5_SYS_MEM_EN,
+ MLX5_DECAP_EN,
NULL,
};
struct rte_kvargs *kvlist;
return 0;
}
-/**
- * PMD global initialization.
- *
- * Independent from individual device, this function initializes global
- * per-PMD data structures distinguishing primary and secondary processes.
- * Hence, each initialization is called once per a process.
- *
- * @return
- * 0 on success, a negative errno value otherwise and rte_errno is set.
- */
-int
-mlx5_init_once(void)
-{
- struct mlx5_shared_data *sd;
- struct mlx5_local_data *ld = &mlx5_local_data;
- int ret = 0;
-
- if (mlx5_init_shared_data())
- return -rte_errno;
- sd = mlx5_shared_data;
- MLX5_ASSERT(sd);
- rte_spinlock_lock(&sd->lock);
- switch (rte_eal_process_type()) {
- case RTE_PROC_PRIMARY:
- if (sd->init_done)
- break;
- LIST_INIT(&sd->mem_event_cb_list);
- rte_rwlock_init(&sd->mem_event_rwlock);
- rte_mem_event_callback_register("MLX5_MEM_EVENT_CB",
- mlx5_mr_mem_event_cb, NULL);
- ret = mlx5_mp_init_primary(MLX5_MP_NAME,
- mlx5_mp_primary_handle);
- if (ret)
- goto out;
- sd->init_done = true;
- break;
- case RTE_PROC_SECONDARY:
- if (ld->init_done)
- break;
- ret = mlx5_mp_init_secondary(MLX5_MP_NAME,
- mlx5_mp_secondary_handle);
- if (ret)
- goto out;
- ++sd->secondary_cnt;
- ld->init_done = true;
- break;
- default:
- break;
- }
-out:
- rte_spinlock_unlock(&sd->lock);
- return ret;
-}
-
/**
* Configures the minimal amount of data to inline into WQE
* while sending packets.
}
};
-struct rte_pci_driver mlx5_driver = {
- .driver = {
- .name = MLX5_DRIVER_NAME
+static struct mlx5_pci_driver mlx5_driver = {
+ .driver_class = MLX5_CLASS_NET,
+ .pci_driver = {
+ .driver = {
+ .name = MLX5_DRIVER_NAME,
+ },
+ .id_table = mlx5_pci_id_map,
+ .probe = mlx5_os_pci_probe,
+ .remove = mlx5_pci_remove,
+ .dma_map = mlx5_dma_map,
+ .dma_unmap = mlx5_dma_unmap,
+ .drv_flags = PCI_DRV_FLAGS,
},
- .id_table = mlx5_pci_id_map,
- .probe = mlx5_os_pci_probe,
- .remove = mlx5_pci_remove,
- .dma_map = mlx5_dma_map,
- .dma_unmap = mlx5_dma_unmap,
- .drv_flags = PCI_DRV_FLAGS,
};
/* Initialize driver log type. */
*/
RTE_INIT(rte_mlx5_pmd_init)
{
+ mlx5_common_init();
/* Build the static tables for Verbs conversion. */
mlx5_set_ptype_table();
mlx5_set_cksum_table();
mlx5_set_swp_types_table();
if (mlx5_glue)
- rte_pci_register(&mlx5_driver);
+ mlx5_pci_driver_register(&mlx5_driver);
}
RTE_PMD_EXPORT_NAME(net_mlx5, __COUNTER__);
git.droids-corp.org / dpdk.git / blobdiff