#include <rte_mbuf.h>
#include <rte_malloc.h>
-#include <rte_ethdev_driver.h>
+#include <ethdev_driver.h>
+#include <rte_bus_pci.h>
#include <rte_common.h>
#include <rte_eal_paging.h>
-#include <mlx5_glue.h>
-#include <mlx5_devx_cmds.h>
#include <mlx5_common.h>
#include <mlx5_common_mr.h>
#include <mlx5_malloc.h>
#include "mlx5_defs.h"
#include "mlx5_utils.h"
#include "mlx5.h"
+#include "mlx5_tx.h"
#include "mlx5_rxtx.h"
#include "mlx5_autoconf.h"
DEV_TX_OFFLOAD_GRE_TNL_TSO |
DEV_TX_OFFLOAD_GENEVE_TNL_TSO);
}
+ if (!config->mprq.enabled)
+ offloads |= DEV_TX_OFFLOAD_MBUF_FAST_FREE;
return offloads;
}
/* Resync CQE and WQE (WQ in reset state). */
rte_io_wmb();
*txq->cq_db = rte_cpu_to_be_32(txq->cq_ci);
+ txq->cq_pi = txq->cq_ci;
rte_io_wmb();
}
MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
/* Move QP to RESET state. */
- if (txq_ctrl->obj->type == MLX5_TXQ_OBJ_TYPE_DEVX_SQ) {
- struct mlx5_devx_modify_sq_attr msq_attr = { 0 };
-
- /* Change queue state to reset with DevX. */
- msq_attr.sq_state = MLX5_SQC_STATE_RDY;
- msq_attr.state = MLX5_SQC_STATE_RST;
- ret = mlx5_devx_cmd_modify_sq(txq_ctrl->obj->sq_devx,
- &msq_attr);
- if (ret) {
- DRV_LOG(ERR, "Cannot change the "
- "Tx QP state to RESET %s",
- strerror(errno));
- rte_errno = errno;
- return ret;
- }
- } else {
- struct ibv_qp_attr mod = {
- .qp_state = IBV_QPS_RESET,
- .port_num = (uint8_t)priv->dev_port,
- };
- struct ibv_qp *qp = txq_ctrl->obj->qp;
-
- /* Change queue state to reset with Verbs. */
- ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
- if (ret) {
- DRV_LOG(ERR, "Cannot change the Tx QP state to RESET "
- "%s", strerror(errno));
- rte_errno = errno;
- return ret;
- }
- }
+ ret = priv->obj_ops.txq_obj_modify(txq_ctrl->obj, MLX5_TXQ_MOD_RDY2RST,
+ (uint8_t)priv->dev_port);
+ if (ret)
+ return ret;
/* Handle all send completions. */
txq_sync_cq(txq);
/* Free elts stored in the SQ. */
int ret;
MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
- if (txq_ctrl->obj->type == MLX5_TXQ_OBJ_TYPE_DEVX_SQ) {
- struct mlx5_devx_modify_sq_attr msq_attr = { 0 };
- struct mlx5_txq_obj *obj = txq_ctrl->obj;
-
- msq_attr.sq_state = MLX5_SQC_STATE_RDY;
- msq_attr.state = MLX5_SQC_STATE_RST;
- ret = mlx5_devx_cmd_modify_sq(obj->sq_devx, &msq_attr);
- if (ret) {
- rte_errno = errno;
- DRV_LOG(ERR,
- "Cannot change the Tx QP state to RESET "
- "%s", strerror(errno));
- return ret;
- }
- msq_attr.sq_state = MLX5_SQC_STATE_RST;
- msq_attr.state = MLX5_SQC_STATE_RDY;
- ret = mlx5_devx_cmd_modify_sq(obj->sq_devx, &msq_attr);
- if (ret) {
- rte_errno = errno;
- DRV_LOG(ERR,
- "Cannot change the Tx QP state to READY "
- "%s", strerror(errno));
- return ret;
- }
- } else {
- struct ibv_qp_attr mod = {
- .qp_state = IBV_QPS_RESET,
- .port_num = (uint8_t)priv->dev_port,
- };
- struct ibv_qp *qp = txq_ctrl->obj->qp;
-
- ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
- if (ret) {
- DRV_LOG(ERR, "Cannot change the Tx QP state to RESET "
- "%s", strerror(errno));
- rte_errno = errno;
- return ret;
- }
- mod.qp_state = IBV_QPS_INIT;
- ret = mlx5_glue->modify_qp(qp, &mod,
- (IBV_QP_STATE | IBV_QP_PORT));
- if (ret) {
- DRV_LOG(ERR, "Cannot change Tx QP state to INIT %s",
- strerror(errno));
- rte_errno = errno;
- return ret;
- }
- mod.qp_state = IBV_QPS_RTR;
- ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
- if (ret) {
- DRV_LOG(ERR, "Cannot change Tx QP state to RTR %s",
- strerror(errno));
- rte_errno = errno;
- return ret;
- }
- mod.qp_state = IBV_QPS_RTS;
- ret = mlx5_glue->modify_qp(qp, &mod, IBV_QP_STATE);
- if (ret) {
- DRV_LOG(ERR, "Cannot change Tx QP state to RTS %s",
- strerror(errno));
- rte_errno = errno;
- return ret;
- }
- }
+ ret = priv->obj_ops.txq_obj_modify(txq_ctrl->obj,
+ MLX5_TXQ_MOD_RST2RDY,
+ (uint8_t)priv->dev_port);
+ if (ret)
+ return ret;
txq_ctrl->txq.wqe_ci = 0;
txq_ctrl->txq.wqe_pi = 0;
txq_ctrl->txq.elts_comp = 0;
DRV_LOG(DEBUG, "port %u adding Tx queue %u to list",
dev->data->port_id, idx);
(*priv->txqs)[idx] = &txq_ctrl->txq;
- dev->data->tx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED;
return 0;
}
res = mlx5_tx_queue_pre_setup(dev, idx, &desc);
if (res)
return res;
- if (hairpin_conf->peer_count != 1 ||
- hairpin_conf->peers[0].port != dev->data->port_id ||
- hairpin_conf->peers[0].queue >= priv->rxqs_n) {
- DRV_LOG(ERR, "port %u unable to setup hairpin queue index %u "
- " invalid hairpind configuration", dev->data->port_id,
- idx);
+ if (hairpin_conf->peer_count != 1) {
rte_errno = EINVAL;
+ DRV_LOG(ERR, "port %u unable to setup Tx hairpin queue index %u"
+ " peer count is %u", dev->data->port_id,
+ idx, hairpin_conf->peer_count);
return -rte_errno;
}
+ if (hairpin_conf->peers[0].port == dev->data->port_id) {
+ if (hairpin_conf->peers[0].queue >= priv->rxqs_n) {
+ rte_errno = EINVAL;
+ DRV_LOG(ERR, "port %u unable to setup Tx hairpin queue"
+ " index %u, Rx %u is larger than %u",
+ dev->data->port_id, idx,
+ hairpin_conf->peers[0].queue, priv->txqs_n);
+ return -rte_errno;
+ }
+ } else {
+ if (hairpin_conf->manual_bind == 0 ||
+ hairpin_conf->tx_explicit == 0) {
+ rte_errno = EINVAL;
+ DRV_LOG(ERR, "port %u unable to setup Tx hairpin queue"
+ " index %u peer port %u with attributes %u %u",
+ dev->data->port_id, idx,
+ hairpin_conf->peers[0].port,
+ hairpin_conf->manual_bind,
+ hairpin_conf->tx_explicit);
+ return -rte_errno;
+ }
+ }
txq_ctrl = mlx5_txq_hairpin_new(dev, idx, desc, hairpin_conf);
if (!txq_ctrl) {
DRV_LOG(ERR, "port %u unable to allocate queue index %u",
void
mlx5_tx_uar_uninit_secondary(struct rte_eth_dev *dev)
{
- struct mlx5_priv *priv = dev->data->dev_private;
- struct mlx5_txq_data *txq;
- struct mlx5_txq_ctrl *txq_ctrl;
+ struct mlx5_proc_priv *ppriv = (struct mlx5_proc_priv *)
+ dev->process_private;
+ const size_t page_size = rte_mem_page_size();
+ void *addr;
unsigned int i;
+ if (page_size == (size_t)-1) {
+ DRV_LOG(ERR, "Failed to get mem page size");
+ return;
+ }
MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_SECONDARY);
- for (i = 0; i != priv->txqs_n; ++i) {
- if (!(*priv->txqs)[i])
+ for (i = 0; i != ppriv->uar_table_sz; ++i) {
+ if (!ppriv->uar_table[i])
continue;
- txq = (*priv->txqs)[i];
- txq_ctrl = container_of(txq, struct mlx5_txq_ctrl, txq);
- txq_uar_uninit_secondary(txq_ctrl);
+ addr = ppriv->uar_table[i];
+ rte_mem_unmap(RTE_PTR_ALIGN_FLOOR(addr, page_size), page_size);
+
}
}
bool vlan_inline;
unsigned int temp;
+ txq_ctrl->txq.fast_free =
+ !!((txq_ctrl->txq.offloads & DEV_TX_OFFLOAD_MBUF_FAST_FREE) &&
+ !(txq_ctrl->txq.offloads & DEV_TX_OFFLOAD_MULTI_SEGS) &&
+ !config->mprq.enabled);
if (config->txqs_inline == MLX5_ARG_UNSET)
txqs_inline =
#if defined(RTE_ARCH_ARM64)
- (priv->pci_dev->id.device_id ==
+ (priv->pci_dev && priv->pci_dev->id.device_id ==
PCI_DEVICE_ID_MELLANOX_CONNECTX5BF) ?
MLX5_INLINE_MAX_TXQS_BLUEFIELD :
#endif
rte_errno = ENOMEM;
goto error;
}
- rte_atomic32_inc(&tmpl->refcnt);
+ __atomic_fetch_add(&tmpl->refcnt, 1, __ATOMIC_RELAXED);
tmpl->type = MLX5_TXQ_TYPE_STANDARD;
LIST_INSERT_HEAD(&priv->txqsctrl, tmpl, next);
return tmpl;
error:
+ mlx5_mr_btree_free(&tmpl->txq.mr_ctrl.cache_bh);
mlx5_free(tmpl);
return NULL;
}
tmpl->txq.idx = idx;
tmpl->hairpin_conf = *hairpin_conf;
tmpl->type = MLX5_TXQ_TYPE_HAIRPIN;
- rte_atomic32_inc(&tmpl->refcnt);
+ __atomic_fetch_add(&tmpl->refcnt, 1, __ATOMIC_RELAXED);
LIST_INSERT_HEAD(&priv->txqsctrl, tmpl, next);
return tmpl;
}
if (txq_data) {
ctrl = container_of(txq_data, struct mlx5_txq_ctrl, txq);
- rte_atomic32_inc(&ctrl->refcnt);
+ __atomic_fetch_add(&ctrl->refcnt, 1, __ATOMIC_RELAXED);
}
return ctrl;
}
struct mlx5_priv *priv = dev->data->dev_private;
struct mlx5_txq_ctrl *txq_ctrl;
- if (!(*priv->txqs)[idx])
+ if (priv->txqs == NULL || (*priv->txqs)[idx] == NULL)
return 0;
txq_ctrl = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq);
- if (!rte_atomic32_dec_and_test(&txq_ctrl->refcnt))
+ if (__atomic_sub_fetch(&txq_ctrl->refcnt, 1, __ATOMIC_RELAXED) > 1)
return 1;
if (txq_ctrl->obj) {
priv->obj_ops.txq_obj_release(txq_ctrl->obj);
txq_ctrl->txq.fcqs = NULL;
}
txq_free_elts(txq_ctrl);
- mlx5_mr_btree_free(&txq_ctrl->txq.mr_ctrl.cache_bh);
+ dev->data->tx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STOPPED;
+ }
+ if (!__atomic_load_n(&txq_ctrl->refcnt, __ATOMIC_RELAXED)) {
+ if (txq_ctrl->type == MLX5_TXQ_TYPE_STANDARD)
+ mlx5_mr_btree_free(&txq_ctrl->txq.mr_ctrl.cache_bh);
+ LIST_REMOVE(txq_ctrl, next);
+ mlx5_free(txq_ctrl);
+ (*priv->txqs)[idx] = NULL;
}
- LIST_REMOVE(txq_ctrl, next);
- mlx5_free(txq_ctrl);
- (*priv->txqs)[idx] = NULL;
- dev->data->tx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STOPPED;
return 0;
}
if (!(*priv->txqs)[idx])
return -1;
txq = container_of((*priv->txqs)[idx], struct mlx5_txq_ctrl, txq);
- return (rte_atomic32_read(&txq->refcnt) == 1);
+ return (__atomic_load_n(&txq->refcnt, __ATOMIC_RELAXED) == 1);
}
/**
(RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME, NULL);
off = rte_mbuf_dynfield_lookup
(RTE_MBUF_DYNFIELD_TIMESTAMP_NAME, NULL);
- if (nbit > 0 && off >= 0 && sh->txpp.refcnt)
+ if (nbit >= 0 && off >= 0 && sh->txpp.refcnt)
mask = 1ULL << nbit;
for (i = 0; i != priv->txqs_n; ++i) {
data = (*priv->txqs)[i];
git.droids-corp.org / dpdk.git / blobdiff