X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Fmlx5%2Fmlx5_rxq.c;h=946f7454f65a0843ecff91ef482afc21117d5b87;hb=4defbc8cbb6d5b520f13ee7f2396b0a31516d370;hp=bccdc5e57fda97fc2c905b67100e604266807e0a;hpb=08d1838f645aa02d07016a4641c85a27c8468510;p=dpdk.git

diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index bccdc5e57f..946f7454f6 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -431,6 +431,244 @@ mlx5_rxq_releasable(struct rte_eth_dev *dev, uint16_t idx)
 	return (rte_atomic32_read(&rxq_ctrl->refcnt) == 1);
 }
 
+/* Fetches and drops all SW-owned and error CQEs to synchronize CQ. */
+static void
+rxq_sync_cq(struct mlx5_rxq_data *rxq)
+{
+	const uint16_t cqe_n = 1 << rxq->cqe_n;
+	const uint16_t cqe_mask = cqe_n - 1;
+	volatile struct mlx5_cqe *cqe;
+	int ret, i;
+
+	i = cqe_n;
+	do {
+		cqe = &(*rxq->cqes)[rxq->cq_ci & cqe_mask];
+		ret = check_cqe(cqe, cqe_n, rxq->cq_ci);
+		if (ret == MLX5_CQE_STATUS_HW_OWN)
+			break;
+		if (ret == MLX5_CQE_STATUS_ERR) {
+			rxq->cq_ci++;
+			continue;
+		}
+		MLX5_ASSERT(ret == MLX5_CQE_STATUS_SW_OWN);
+		if (MLX5_CQE_FORMAT(cqe->op_own) != MLX5_COMPRESSED) {
+			rxq->cq_ci++;
+			continue;
+		}
+		/* Compute the next non compressed CQE. */
+		rxq->cq_ci += rte_be_to_cpu_32(cqe->byte_cnt);
+
+	} while (--i);
+	/* Move all CQEs to HW ownership, including possible MiniCQEs. */
+	for (i = 0; i < cqe_n; i++) {
+		cqe = &(*rxq->cqes)[i];
+		cqe->op_own = MLX5_CQE_INVALIDATE;
+	}
+	/* Resync CQE and WQE (WQ in RESET state). */
+	rte_cio_wmb();
+	*rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
+	rte_cio_wmb();
+	*rxq->rq_db = rte_cpu_to_be_32(0);
+	rte_cio_wmb();
+}
+
+/**
+ * Rx queue stop. Device queue goes to the RESET state,
+ * all involved mbufs are freed from WQ.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param idx
+ *   RX queue index.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_rx_queue_stop_primary(struct rte_eth_dev *dev, uint16_t idx)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_rxq_data *rxq = (*priv->rxqs)[idx];
+	struct mlx5_rxq_ctrl *rxq_ctrl =
+			container_of(rxq, struct mlx5_rxq_ctrl, rxq);
+	int ret;
+
+	MLX5_ASSERT(rte_eal_process_type() == RTE_PROC_PRIMARY);
+	if (rxq_ctrl->obj->type == MLX5_RXQ_OBJ_TYPE_IBV) {
+		struct ibv_wq_attr mod = {
+			.attr_mask = IBV_WQ_ATTR_STATE,
+			.wq_state = IBV_WQS_RESET,
+		};
+
+		ret = mlx5_glue->modify_wq(rxq_ctrl->obj->wq, &mod);
+	} else { /* rxq_ctrl->obj->type == MLX5_RXQ_OBJ_TYPE_DEVX_RQ. */
+		struct mlx5_devx_modify_rq_attr rq_attr;
+
+		memset(&rq_attr, 0, sizeof(rq_attr));
+		rq_attr.rq_state = MLX5_RQC_STATE_RST;
+		rq_attr.state = MLX5_RQC_STATE_RDY;
+		ret = mlx5_devx_cmd_modify_rq(rxq_ctrl->obj->rq, &rq_attr);
+	}
+	if (ret) {
+		DRV_LOG(ERR, "Cannot change Rx WQ state to RESET:  %s",
+			strerror(errno));
+		rte_errno = errno;
+		return ret;
+	}
+	/* Remove all processes CQEs. */
+	rxq_sync_cq(rxq);
+	/* Free all involved mbufs. */
+	rxq_free_elts(rxq_ctrl);
+	/* Set the actual queue state. */
+	dev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STOPPED;
+	return 0;
+}
+
+/**
+ * Rx queue stop. Device queue goes to the RESET state,
+ * all involved mbufs are freed from WQ.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param idx
+ *   RX queue index.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_rx_queue_stop(struct rte_eth_dev *dev, uint16_t idx)
+{
+	eth_rx_burst_t pkt_burst = dev->rx_pkt_burst;
+	int ret;
+
+	if (dev->data->rx_queue_state[idx] == RTE_ETH_QUEUE_STATE_HAIRPIN) {
+		DRV_LOG(ERR, "Hairpin queue can't be stopped");
+		rte_errno = EINVAL;
+		return -EINVAL;
+	}
+	if (dev->data->rx_queue_state[idx] == RTE_ETH_QUEUE_STATE_STOPPED)
+		return 0;
+	/*
+	 * Vectorized Rx burst requires the CQ and RQ indices
+	 * synchronized, that might be broken on RQ restart
+	 * and cause Rx malfunction, so queue stopping is
+	 * not supported if vectorized Rx burst is engaged.
+	 * The routine pointer depends on the process
+	 * type, should perform check there.
+	 */
+	if (pkt_burst == mlx5_rx_burst) {
+		DRV_LOG(ERR, "Rx queue stop is not supported "
+			"for vectorized Rx");
+		rte_errno = EINVAL;
+		return -EINVAL;
+	}
+	if (rte_eal_process_type() ==  RTE_PROC_SECONDARY) {
+		ret = mlx5_mp_os_req_queue_control(dev, idx,
+						   MLX5_MP_REQ_QUEUE_RX_STOP);
+	} else {
+		ret = mlx5_rx_queue_stop_primary(dev, idx);
+	}
+	return ret;
+}
+
+/**
+ * Rx queue start. Device queue goes to the ready state,
+ * all required mbufs are allocated and WQ is replenished.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param idx
+ *   RX queue index.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_rx_queue_start_primary(struct rte_eth_dev *dev, uint16_t idx)
+{
+	struct mlx5_priv *priv = dev->data->dev_private;
+	struct mlx5_rxq_data *rxq = (*priv->rxqs)[idx];
+	struct mlx5_rxq_ctrl *rxq_ctrl =
+			container_of(rxq, struct mlx5_rxq_ctrl, rxq);
+	int ret;
+
+	MLX5_ASSERT(rte_eal_process_type() ==  RTE_PROC_PRIMARY);
+	/* Allocate needed buffers. */
+	ret = rxq_alloc_elts(rxq_ctrl);
+	if (ret) {
+		DRV_LOG(ERR, "Cannot reallocate buffers for Rx WQ");
+		rte_errno = errno;
+		return ret;
+	}
+	rte_cio_wmb();
+	*rxq->cq_db = rte_cpu_to_be_32(rxq->cq_ci);
+	rte_cio_wmb();
+	/* Reset RQ consumer before moving queue ro READY state. */
+	*rxq->rq_db = rte_cpu_to_be_32(0);
+	rte_cio_wmb();
+	if (rxq_ctrl->obj->type == MLX5_RXQ_OBJ_TYPE_IBV) {
+		struct ibv_wq_attr mod = {
+			.attr_mask = IBV_WQ_ATTR_STATE,
+			.wq_state = IBV_WQS_RDY,
+		};
+
+		ret = mlx5_glue->modify_wq(rxq_ctrl->obj->wq, &mod);
+	} else { /* rxq_ctrl->obj->type == MLX5_RXQ_OBJ_TYPE_DEVX_RQ. */
+		struct mlx5_devx_modify_rq_attr rq_attr;
+
+		memset(&rq_attr, 0, sizeof(rq_attr));
+		rq_attr.rq_state = MLX5_RQC_STATE_RDY;
+		rq_attr.state = MLX5_RQC_STATE_RST;
+		ret = mlx5_devx_cmd_modify_rq(rxq_ctrl->obj->rq, &rq_attr);
+	}
+	if (ret) {
+		DRV_LOG(ERR, "Cannot change Rx WQ state to READY:  %s",
+			strerror(errno));
+		rte_errno = errno;
+		return ret;
+	}
+	/* Reinitialize RQ - set WQEs. */
+	mlx5_rxq_initialize(rxq);
+	rxq->err_state = MLX5_RXQ_ERR_STATE_NO_ERROR;
+	/* Set actual queue state. */
+	dev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED;
+	return 0;
+}
+
+/**
+ * Rx queue start. Device queue goes to the ready state,
+ * all required mbufs are allocated and WQ is replenished.
+ *
+ * @param dev
+ *   Pointer to Ethernet device structure.
+ * @param idx
+ *   RX queue index.
+ *
+ * @return
+ *   0 on success, a negative errno value otherwise and rte_errno is set.
+ */
+int
+mlx5_rx_queue_start(struct rte_eth_dev *dev, uint16_t idx)
+{
+	int ret;
+
+	if (dev->data->rx_queue_state[idx] == RTE_ETH_QUEUE_STATE_HAIRPIN) {
+		DRV_LOG(ERR, "Hairpin queue can't be started");
+		rte_errno = EINVAL;
+		return -EINVAL;
+	}
+	if (dev->data->rx_queue_state[idx] == RTE_ETH_QUEUE_STATE_STARTED)
+		return 0;
+	if (rte_eal_process_type() ==  RTE_PROC_SECONDARY) {
+		ret = mlx5_mp_os_req_queue_control(dev, idx,
+						   MLX5_MP_REQ_QUEUE_RX_START);
+	} else {
+		ret = mlx5_rx_queue_start_primary(dev, idx);
+	}
+	return ret;
+}
+
 /**
  * Rx queue presetup checks.
  *
@@ -950,34 +1188,43 @@ mlx5_rx_intr_disable(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 	if (rxq_obj->type == MLX5_RXQ_OBJ_TYPE_IBV) {
 		ret = mlx5_glue->get_cq_event(rxq_obj->ibv_channel, &ev_cq,
 					      &ev_ctx);
-		if (ret || ev_cq != rxq_obj->ibv_cq) {
-			rte_errno = EINVAL;
+		if (ret < 0 || ev_cq != rxq_obj->ibv_cq)
 			goto exit;
-		}
 		mlx5_glue->ack_cq_events(rxq_obj->ibv_cq, 1);
 	} else if (rxq_obj->type == MLX5_RXQ_OBJ_TYPE_DEVX_RQ) {
 #ifdef HAVE_IBV_DEVX_EVENT
-		struct mlx5dv_devx_async_event_hdr *event_data = NULL;
+		union {
+			struct mlx5dv_devx_async_event_hdr event_resp;
+			uint8_t buf[sizeof(struct mlx5dv_devx_async_event_hdr)
+				    + 128];
+		} out;
 
 		ret = mlx5_glue->devx_get_event
-				(rxq_obj->devx_channel, event_data,
-				 sizeof(struct mlx5dv_devx_async_event_hdr));
-		if (ret <= 0 || event_data->cookie !=
-				(uint64_t)(uintptr_t)rxq_obj->devx_cq) {
-			rte_errno = EINVAL;
+				(rxq_obj->devx_channel, &out.event_resp,
+				 sizeof(out.buf));
+		if (ret < 0 || out.event_resp.cookie !=
+				(uint64_t)(uintptr_t)rxq_obj->devx_cq)
 			goto exit;
-		}
 #endif /* HAVE_IBV_DEVX_EVENT */
 	}
 	rxq_data->cq_arm_sn++;
 	mlx5_rxq_obj_release(rxq_obj);
 	return 0;
 exit:
+	/**
+	 * For ret < 0 save the errno (may be EAGAIN which means the get_event
+	 * function was called before receiving one).
+	 */
+	if (ret < 0)
+		rte_errno = errno;
+	else
+		rte_errno = EINVAL;
 	ret = rte_errno; /* Save rte_errno before cleanup. */
 	if (rxq_obj)
 		mlx5_rxq_obj_release(rxq_obj);
-	DRV_LOG(WARNING, "port %u unable to disable interrupt on Rx queue %d",
-		dev->data->port_id, rx_queue_id);
+	if (ret != EAGAIN)
+		DRV_LOG(WARNING, "port %u unable to disable interrupt on Rx queue %d",
+			dev->data->port_id, rx_queue_id);
 	rte_errno = ret; /* Restore rte_errno. */
 	return -rte_errno;
 }
@@ -1197,7 +1444,7 @@ mlx5_devx_wq_attr_fill(struct mlx5_priv *priv, struct mlx5_rxq_ctrl *rxq_ctrl,
 	wq_attr->dbr_addr = rxq_ctrl->rq_dbr_offset;
 	wq_attr->dbr_umem_id = rxq_ctrl->rq_dbr_umem_id;
 	wq_attr->dbr_umem_valid = 1;
-	wq_attr->wq_umem_id = rxq_ctrl->wq_umem->umem_id;
+	wq_attr->wq_umem_id = mlx5_os_get_umem_id(rxq_ctrl->wq_umem);
 	wq_attr->wq_umem_valid = 1;
 }
 
@@ -1373,8 +1620,9 @@ mlx5_devx_cq_new(struct rte_eth_dev *dev, unsigned int cqe_n, uint16_t idx,
 		DRV_LOG(ERR, "Failed to register umem for CQ.");
 		goto error;
 	}
-	cq_attr.uar_page_id = priv->sh->devx_rx_uar->page_id;
-	cq_attr.q_umem_id = rxq_ctrl->cq_umem->umem_id;
+	cq_attr.uar_page_id =
+		mlx5_os_get_devx_uar_page_id(priv->sh->devx_rx_uar);
+	cq_attr.q_umem_id = mlx5_os_get_umem_id(rxq_ctrl->cq_umem);
 	cq_attr.q_umem_valid = 1;
 	cq_attr.log_cq_size = log_cqe_n;
 	cq_attr.log_page_size = rte_log2_u32(page_size);
@@ -1403,6 +1651,8 @@ mlx5_devx_cq_new(struct rte_eth_dev *dev, unsigned int cqe_n, uint16_t idx,
 	memset((void *)(uintptr_t)rxq_data->cqes, 0xFF, cq_size);
 	return cq_obj;
 error:
+	if (cq_obj)
+		mlx5_devx_cmd_destroy(cq_obj);
 	rxq_release_devx_cq_resources(rxq_ctrl);
 	return NULL;
 }
@@ -1479,6 +1729,7 @@ mlx5_rxq_obj_hairpin_new(struct rte_eth_dev *dev, uint16_t idx)
 	rte_atomic32_inc(&tmpl->refcnt);
 	LIST_INSERT_HEAD(&priv->rxqsobj, tmpl, next);
 	priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;
+	dev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_HAIRPIN;
 	return tmpl;
 }
 
@@ -1539,7 +1790,8 @@ mlx5_rxq_obj_new(struct rte_eth_dev *dev, uint16_t idx,
 				rte_errno = ENOMEM;
 				goto error;
 			}
-			tmpl->fd = tmpl->ibv_channel->fd;
+			tmpl->fd = ((struct ibv_comp_channel *)
+					(tmpl->ibv_channel))->fd;
 		} else if (tmpl->type == MLX5_RXQ_OBJ_TYPE_DEVX_RQ) {
 			int devx_ev_flag =
 			  MLX5DV_DEVX_CREATE_EVENT_CHANNEL_FLAGS_OMIT_EV_DATA;
@@ -1555,7 +1807,8 @@ mlx5_rxq_obj_new(struct rte_eth_dev *dev, uint16_t idx,
 					rte_errno);
 				goto error;
 			}
-			tmpl->fd = tmpl->devx_channel->fd;
+			tmpl->fd =
+				mlx5_os_get_devx_channel_fd(tmpl->devx_channel);
 		}
 	}
 	if (mlx5_rxq_mprq_enabled(rxq_data))
@@ -1647,7 +1900,8 @@ mlx5_rxq_obj_new(struct rte_eth_dev *dev, uint16_t idx,
 		rxq_data->cq_db =
 			(uint32_t *)((uintptr_t)dbr_page->dbrs +
 				     (uintptr_t)rxq_ctrl->cq_dbr_offset);
-		rxq_data->cq_uar = priv->sh->devx_rx_uar->base_addr;
+		rxq_data->cq_uar =
+			mlx5_os_get_devx_uar_base_addr(priv->sh->devx_rx_uar);
 		/* Create CQ using DevX API. */
 		tmpl->devx_cq = mlx5_devx_cq_new(dev, cqe_n, idx, tmpl);
 		if (!tmpl->devx_cq) {
@@ -1690,6 +1944,7 @@ mlx5_rxq_obj_new(struct rte_eth_dev *dev, uint16_t idx,
 	rte_atomic32_inc(&tmpl->refcnt);
 	LIST_INSERT_HEAD(&priv->rxqsobj, tmpl, next);
 	priv->verbs_alloc_ctx.type = MLX5_VERBS_ALLOC_TYPE_NONE;
+	dev->data->rx_queue_state[idx] = RTE_ETH_QUEUE_STATE_STARTED;
 	return tmpl;
 error:
 	if (tmpl) {
@@ -3045,7 +3300,7 @@ mlx5_ind_table_obj_drop_new(struct rte_eth_dev *dev)
 		(priv->sh->ctx,
 		 &(struct ibv_rwq_ind_table_init_attr){
 			.log_ind_tbl_size = 0,
-			.ind_tbl = &rxq->wq,
+			.ind_tbl = (struct ibv_wq **)&rxq->wq,
 			.comp_mask = 0,
 		 });
 	if (!tmpl.ind_table) {