+static uint16_t mlx4_tx_burst(void *, struct rte_mbuf **, uint16_t);
+static uint16_t removed_rx_burst(void *, struct rte_mbuf **, uint16_t);
+
+/**
+ * Configure secondary process queues from a private data pointer (primary
+ * or secondary) and update burst callbacks. Can take place only once.
+ *
+ * All queues must have been previously created by the primary process to
+ * avoid undefined behavior.
+ *
+ * @param priv
+ * Private data pointer from either primary or secondary process.
+ *
+ * @return
+ * Private data pointer from secondary process, NULL in case of error.
+ */
+static struct priv *
+mlx4_secondary_data_setup(struct priv *priv)
+{
+ unsigned int port_id = 0;
+ struct mlx4_secondary_data *sd;
+ void **tx_queues;
+ void **rx_queues;
+ unsigned int nb_tx_queues;
+ unsigned int nb_rx_queues;
+ unsigned int i;
+
+ /* priv must be valid at this point. */
+ assert(priv != NULL);
+ /* priv->dev must also be valid but may point to local memory from
+ * another process, possibly with the same address and must not
+ * be dereferenced yet. */
+ assert(priv->dev != NULL);
+ /* Determine port ID by finding out where priv comes from. */
+ while (1) {
+ sd = &mlx4_secondary_data[port_id];
+ rte_spinlock_lock(&sd->lock);
+ /* Primary process? */
+ if (sd->primary_priv == priv)
+ break;
+ /* Secondary process? */
+ if (sd->data.dev_private == priv)
+ break;
+ rte_spinlock_unlock(&sd->lock);
+ if (++port_id == RTE_DIM(mlx4_secondary_data))
+ port_id = 0;
+ }
+ /* Switch to secondary private structure. If private data has already
+ * been updated by another thread, there is nothing else to do. */
+ priv = sd->data.dev_private;
+ if (priv->dev->data == &sd->data)
+ goto end;
+ /* Sanity checks. Secondary private structure is supposed to point
+ * to local eth_dev, itself still pointing to the shared device data
+ * structure allocated by the primary process. */
+ assert(sd->shared_dev_data != &sd->data);
+ assert(sd->data.nb_tx_queues == 0);
+ assert(sd->data.tx_queues == NULL);
+ assert(sd->data.nb_rx_queues == 0);
+ assert(sd->data.rx_queues == NULL);
+ assert(priv != sd->primary_priv);
+ assert(priv->dev->data == sd->shared_dev_data);
+ assert(priv->txqs_n == 0);
+ assert(priv->txqs == NULL);
+ assert(priv->rxqs_n == 0);
+ assert(priv->rxqs == NULL);
+ nb_tx_queues = sd->shared_dev_data->nb_tx_queues;
+ nb_rx_queues = sd->shared_dev_data->nb_rx_queues;
+ /* Allocate local storage for queues. */
+ tx_queues = rte_zmalloc("secondary ethdev->tx_queues",
+ sizeof(sd->data.tx_queues[0]) * nb_tx_queues,
+ RTE_CACHE_LINE_SIZE);
+ rx_queues = rte_zmalloc("secondary ethdev->rx_queues",
+ sizeof(sd->data.rx_queues[0]) * nb_rx_queues,
+ RTE_CACHE_LINE_SIZE);
+ if (tx_queues == NULL || rx_queues == NULL)
+ goto error;
+ /* Lock to prevent control operations during setup. */
+ priv_lock(priv);
+ /* TX queues. */
+ for (i = 0; i != nb_tx_queues; ++i) {
+ struct txq *primary_txq = (*sd->primary_priv->txqs)[i];
+ struct txq *txq;
+
+ if (primary_txq == NULL)
+ continue;
+ txq = rte_calloc_socket("TXQ", 1, sizeof(*txq), 0,
+ primary_txq->socket);
+ if (txq != NULL) {
+ if (txq_setup(priv->dev,
+ txq,
+ primary_txq->elts_n * MLX4_PMD_SGE_WR_N,
+ primary_txq->socket,
+ NULL) == 0) {
+ txq->stats.idx = primary_txq->stats.idx;
+ tx_queues[i] = txq;
+ continue;
+ }
+ rte_free(txq);
+ }
+ while (i) {
+ txq = tx_queues[--i];
+ txq_cleanup(txq);
+ rte_free(txq);
+ }
+ goto error;
+ }
+ /* RX queues. */
+ for (i = 0; i != nb_rx_queues; ++i) {
+ struct rxq *primary_rxq = (*sd->primary_priv->rxqs)[i];
+
+ if (primary_rxq == NULL)
+ continue;
+ /* Not supported yet. */
+ rx_queues[i] = NULL;
+ }
+ /* Update everything. */
+ priv->txqs = (void *)tx_queues;
+ priv->txqs_n = nb_tx_queues;
+ priv->rxqs = (void *)rx_queues;
+ priv->rxqs_n = nb_rx_queues;
+ sd->data.rx_queues = rx_queues;
+ sd->data.tx_queues = tx_queues;
+ sd->data.nb_rx_queues = nb_rx_queues;
+ sd->data.nb_tx_queues = nb_tx_queues;
+ sd->data.dev_link = sd->shared_dev_data->dev_link;
+ sd->data.mtu = sd->shared_dev_data->mtu;
+ memcpy(sd->data.rx_queue_state, sd->shared_dev_data->rx_queue_state,
+ sizeof(sd->data.rx_queue_state));
+ memcpy(sd->data.tx_queue_state, sd->shared_dev_data->tx_queue_state,
+ sizeof(sd->data.tx_queue_state));
+ sd->data.dev_flags = sd->shared_dev_data->dev_flags;
+ /* Use local data from now on. */
+ rte_mb();
+ priv->dev->data = &sd->data;
+ rte_mb();
+ priv->dev->tx_pkt_burst = mlx4_tx_burst;
+ priv->dev->rx_pkt_burst = removed_rx_burst;
+ priv_unlock(priv);
+end:
+ /* More sanity checks. */
+ assert(priv->dev->tx_pkt_burst == mlx4_tx_burst);
+ assert(priv->dev->rx_pkt_burst == removed_rx_burst);
+ assert(priv->dev->data == &sd->data);
+ rte_spinlock_unlock(&sd->lock);
+ return priv;
+error:
+ priv_unlock(priv);
+ rte_free(tx_queues);
+ rte_free(rx_queues);
+ rte_spinlock_unlock(&sd->lock);
+ return NULL;
+}
+