/* DPDK headers don't like -pedantic. */
#ifdef PEDANTIC
-#pragma GCC diagnostic ignored "-pedantic"
+#pragma GCC diagnostic ignored "-Wpedantic"
#endif
#include <rte_atomic.h>
#include <rte_ethdev.h>
#include <rte_alarm.h>
#include <rte_malloc.h>
#ifdef PEDANTIC
-#pragma GCC diagnostic error "-pedantic"
+#pragma GCC diagnostic error "-Wpedantic"
#endif
#include "mlx5.h"
if (priv_get_sysfs_ulong(priv, "flags", &tmp) == -1)
return -1;
tmp &= keep;
- tmp |= flags;
+ tmp |= (flags & (~keep));
return priv_set_sysfs_ulong(priv, "flags", tmp);
}
(DEV_RX_OFFLOAD_IPV4_CKSUM |
DEV_RX_OFFLOAD_UDP_CKSUM |
DEV_RX_OFFLOAD_TCP_CKSUM) :
- 0);
- info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT;
+ 0) |
+ (priv->hw_vlan_strip ? DEV_RX_OFFLOAD_VLAN_STRIP : 0);
+ if (!priv->mps)
+ info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT;
if (priv->hw_csum)
info->tx_offload_capa |=
(DEV_TX_OFFLOAD_IPV4_CKSUM |
* size if it is not fixed.
* The API should be updated to solve this problem. */
info->reta_size = priv->ind_table_max_size;
- info->speed_capa =
- ETH_LINK_SPEED_1G |
- ETH_LINK_SPEED_10G |
- ETH_LINK_SPEED_20G |
- ETH_LINK_SPEED_25G |
- ETH_LINK_SPEED_40G |
- ETH_LINK_SPEED_50G |
- ETH_LINK_SPEED_56G |
- ETH_LINK_SPEED_100G;
+ info->hash_key_size = ((*priv->rss_conf) ?
+ (*priv->rss_conf)[0]->rss_key_len :
+ 0);
+ info->speed_capa = priv->link_speed_capa;
priv_unlock(priv);
}
};
- if (dev->rx_pkt_burst == mlx5_rx_burst ||
- dev->rx_pkt_burst == mlx5_rx_burst_sp)
+ if (dev->rx_pkt_burst == mlx5_rx_burst)
return ptypes;
return NULL;
}
/**
- * DPDK callback to retrieve physical link information (unlocked version).
+ * Retrieve physical link information (unlocked version using legacy ioctl).
*
* @param dev
* Pointer to Ethernet device structure.
* Wait for request completion (ignored).
*/
static int
-mlx5_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete)
+mlx5_link_update_unlocked_gset(struct rte_eth_dev *dev, int wait_to_complete)
{
struct priv *priv = mlx5_get_priv(dev);
struct ethtool_cmd edata = {
- .cmd = ETHTOOL_GSET
+ .cmd = ETHTOOL_GSET /* Deprecated since Linux v4.5. */
};
struct ifreq ifr;
struct rte_eth_link dev_link;
dev_link.link_speed = 0;
else
dev_link.link_speed = link_speed;
+ priv->link_speed_capa = 0;
+ if (edata.supported & SUPPORTED_Autoneg)
+ priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG;
+ if (edata.supported & (SUPPORTED_1000baseT_Full |
+ SUPPORTED_1000baseKX_Full))
+ priv->link_speed_capa |= ETH_LINK_SPEED_1G;
+ if (edata.supported & SUPPORTED_10000baseKR_Full)
+ priv->link_speed_capa |= ETH_LINK_SPEED_10G;
+ if (edata.supported & (SUPPORTED_40000baseKR4_Full |
+ SUPPORTED_40000baseCR4_Full |
+ SUPPORTED_40000baseSR4_Full |
+ SUPPORTED_40000baseLR4_Full))
+ priv->link_speed_capa |= ETH_LINK_SPEED_40G;
dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ?
ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
return -1;
}
+/**
+ * Retrieve physical link information (unlocked version using new ioctl from
+ * Linux 4.5).
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ * @param wait_to_complete
+ * Wait for request completion (ignored).
+ */
+static int
+mlx5_link_update_unlocked_gs(struct rte_eth_dev *dev, int wait_to_complete)
+{
+#ifdef ETHTOOL_GLINKSETTINGS
+ struct priv *priv = mlx5_get_priv(dev);
+ struct ethtool_link_settings edata = {
+ .cmd = ETHTOOL_GLINKSETTINGS,
+ };
+ struct ifreq ifr;
+ struct rte_eth_link dev_link;
+ uint64_t sc;
+
+ (void)wait_to_complete;
+ if (priv_ifreq(priv, SIOCGIFFLAGS, &ifr)) {
+ WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno));
+ return -1;
+ }
+ memset(&dev_link, 0, sizeof(dev_link));
+ dev_link.link_status = ((ifr.ifr_flags & IFF_UP) &&
+ (ifr.ifr_flags & IFF_RUNNING));
+ ifr.ifr_data = (void *)&edata;
+ if (priv_ifreq(priv, SIOCETHTOOL, &ifr)) {
+ DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s",
+ strerror(errno));
+ return -1;
+ }
+ dev_link.link_speed = edata.speed;
+ sc = edata.link_mode_masks[0] |
+ ((uint64_t)edata.link_mode_masks[1] << 32);
+ priv->link_speed_capa = 0;
+ /* Link speeds available in kernel v4.5. */
+ if (sc & ETHTOOL_LINK_MODE_Autoneg_BIT)
+ priv->link_speed_capa |= ETH_LINK_SPEED_AUTONEG;
+ if (sc & (ETHTOOL_LINK_MODE_1000baseT_Full_BIT |
+ ETHTOOL_LINK_MODE_1000baseKX_Full_BIT))
+ priv->link_speed_capa |= ETH_LINK_SPEED_1G;
+ if (sc & (ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT |
+ ETHTOOL_LINK_MODE_10000baseKR_Full_BIT |
+ ETHTOOL_LINK_MODE_10000baseR_FEC_BIT))
+ priv->link_speed_capa |= ETH_LINK_SPEED_10G;
+ if (sc & (ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT |
+ ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT))
+ priv->link_speed_capa |= ETH_LINK_SPEED_20G;
+ if (sc & (ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT |
+ ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT |
+ ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT |
+ ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT))
+ priv->link_speed_capa |= ETH_LINK_SPEED_40G;
+ if (sc & (ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT |
+ ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT |
+ ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT |
+ ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT))
+ priv->link_speed_capa |= ETH_LINK_SPEED_56G;
+ /* Link speeds available in kernel v4.6. */
+#ifdef HAVE_ETHTOOL_LINK_MODE_25G
+ if (sc & (ETHTOOL_LINK_MODE_25000baseCR_Full_BIT |
+ ETHTOOL_LINK_MODE_25000baseKR_Full_BIT |
+ ETHTOOL_LINK_MODE_25000baseSR_Full_BIT))
+ priv->link_speed_capa |= ETH_LINK_SPEED_25G;
+#endif
+#ifdef HAVE_ETHTOOL_LINK_MODE_50G
+ if (sc & (ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT |
+ ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT))
+ priv->link_speed_capa |= ETH_LINK_SPEED_50G;
+#endif
+#ifdef HAVE_ETHTOOL_LINK_MODE_100G
+ if (sc & (ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT |
+ ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT |
+ ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT |
+ ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT))
+ priv->link_speed_capa |= ETH_LINK_SPEED_100G;
+#endif
+ dev_link.link_duplex = ((edata.duplex == DUPLEX_HALF) ?
+ ETH_LINK_HALF_DUPLEX : ETH_LINK_FULL_DUPLEX);
+ dev_link.link_autoneg = !(dev->data->dev_conf.link_speeds &
+ ETH_LINK_SPEED_FIXED);
+ if (memcmp(&dev_link, &dev->data->dev_link, sizeof(dev_link))) {
+ /* Link status changed. */
+ dev->data->dev_link = dev_link;
+ return 0;
+ }
+#else
+ (void)dev;
+ (void)wait_to_complete;
+#endif
+ /* Link status is still the same. */
+ return -1;
+}
+
+/**
+ * DPDK callback to retrieve physical link information (unlocked version).
+ *
+ * @param dev
+ * Pointer to Ethernet device structure.
+ * @param wait_to_complete
+ * Wait for request completion (ignored).
+ */
+int
+mlx5_link_update_unlocked(struct rte_eth_dev *dev, int wait_to_complete)
+{
+ int ret;
+
+ ret = mlx5_link_update_unlocked_gs(dev, wait_to_complete);
+ if (ret < 0)
+ ret = mlx5_link_update_unlocked_gset(dev, wait_to_complete);
+ return ret;
+}
+
/**
* DPDK callback to retrieve physical link information.
*
unsigned int i;
uint16_t (*rx_func)(void *, struct rte_mbuf **, uint16_t) =
mlx5_rx_burst;
+ unsigned int max_frame_len;
+ int rehash;
+ int restart = priv->started;
if (mlx5_is_secondary())
return -E_RTE_SECONDARY;
goto out;
} else
DEBUG("adapter port %u MTU set to %u", priv->port, mtu);
- priv->mtu = mtu;
/* Temporarily replace RX handler with a fake one, assuming it has not
* been copied elsewhere. */
dev->rx_pkt_burst = removed_rx_burst;
* removed_rx_burst() instead. */
rte_wmb();
usleep(1000);
+ /* MTU does not include header and CRC. */
+ max_frame_len = ETHER_HDR_LEN + mtu + ETHER_CRC_LEN;
+ /* Check if at least one queue is going to need a SGE update. */
+ for (i = 0; i != priv->rxqs_n; ++i) {
+ struct rxq *rxq = (*priv->rxqs)[i];
+ unsigned int mb_len;
+ unsigned int size = RTE_PKTMBUF_HEADROOM + max_frame_len;
+ unsigned int sges_n;
+
+ if (rxq == NULL)
+ continue;
+ mb_len = rte_pktmbuf_data_room_size(rxq->mp);
+ assert(mb_len >= RTE_PKTMBUF_HEADROOM);
+ /*
+ * Determine the number of SGEs needed for a full packet
+ * and round it to the next power of two.
+ */
+ sges_n = log2above((size / mb_len) + !!(size % mb_len));
+ if (sges_n != rxq->sges_n)
+ break;
+ }
+ /*
+ * If all queues have the right number of SGEs, a simple rehash
+ * of their buffers is enough, otherwise SGE information can only
+ * be updated in a queue by recreating it. All resources that depend
+ * on queues (flows, indirection tables) must be recreated as well in
+ * that case.
+ */
+ rehash = (i == priv->rxqs_n);
+ if (!rehash) {
+ /* Clean up everything as with mlx5_dev_stop(). */
+ priv_special_flow_disable_all(priv);
+ priv_mac_addrs_disable(priv);
+ priv_destroy_hash_rxqs(priv);
+ priv_fdir_disable(priv);
+ priv_dev_interrupt_handler_uninstall(priv, dev);
+ }
+recover:
/* Reconfigure each RX queue. */
for (i = 0; (i != priv->rxqs_n); ++i) {
struct rxq *rxq = (*priv->rxqs)[i];
- unsigned int mb_len;
- unsigned int max_frame_len;
+ struct rxq_ctrl *rxq_ctrl =
+ container_of(rxq, struct rxq_ctrl, rxq);
int sp;
+ unsigned int mb_len;
+ unsigned int tmp;
if (rxq == NULL)
continue;
- /* Calculate new maximum frame length according to MTU and
- * toggle scattered support (sp) if necessary. */
- max_frame_len = (priv->mtu + ETHER_HDR_LEN +
- (ETHER_MAX_VLAN_FRAME_LEN - ETHER_MAX_LEN));
mb_len = rte_pktmbuf_data_room_size(rxq->mp);
assert(mb_len >= RTE_PKTMBUF_HEADROOM);
+ /* Toggle scattered support (sp) if necessary. */
sp = (max_frame_len > (mb_len - RTE_PKTMBUF_HEADROOM));
/* Provide new values to rxq_setup(). */
dev->data->dev_conf.rxmode.jumbo_frame = sp;
dev->data->dev_conf.rxmode.max_rx_pkt_len = max_frame_len;
- ret = rxq_rehash(dev, rxq);
- if (ret) {
- /* Force SP RX if that queue requires it and abort. */
- if (rxq->sp)
- rx_func = mlx5_rx_burst_sp;
- break;
+ if (rehash)
+ ret = rxq_rehash(dev, rxq_ctrl);
+ else
+ ret = rxq_ctrl_setup(dev, rxq_ctrl, 1 << rxq->elts_n,
+ rxq_ctrl->socket, NULL, rxq->mp);
+ if (!ret)
+ continue;
+ /* Attempt to roll back in case of error. */
+ tmp = (mb_len << rxq->sges_n) - RTE_PKTMBUF_HEADROOM;
+ if (max_frame_len != tmp) {
+ max_frame_len = tmp;
+ goto recover;
}
- /* Scattered burst function takes priority. */
- if (rxq->sp)
- rx_func = mlx5_rx_burst_sp;
+ /* Double fault, disable RX. */
+ break;
+ }
+ /*
+ * Use a safe RX burst function in case of error, otherwise mimic
+ * mlx5_dev_start().
+ */
+ if (ret) {
+ ERROR("unable to reconfigure RX queues, RX disabled");
+ rx_func = removed_rx_burst;
+ } else if (restart &&
+ !rehash &&
+ !priv_create_hash_rxqs(priv) &&
+ !priv_rehash_flows(priv)) {
+ if (dev->data->dev_conf.fdir_conf.mode == RTE_FDIR_MODE_NONE)
+ priv_fdir_enable(priv);
+ priv_dev_interrupt_handler_install(priv, dev);
}
+ priv->mtu = mtu;
/* Burst functions can now be called again. */
rte_wmb();
dev->rx_pkt_burst = rx_func;
ret = priv_dev_link_status_handler(priv, dev);
priv_unlock(priv);
if (ret)
- _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
+ _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
}
/**
ret = priv_dev_link_status_handler(priv, dev);
priv_unlock(priv);
if (ret)
- _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC);
+ _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
}
/**
rte_eal_alarm_cancel(mlx5_dev_link_status_handler, dev);
priv->pending_alarm = 0;
priv->intr_handle.fd = 0;
- priv->intr_handle.type = 0;
+ priv->intr_handle.type = RTE_INTR_HANDLE_UNKNOWN;
}
/**
/**
* Change the link state (UP / DOWN).
*
- * @param dev
+ * @param priv
* Pointer to Ethernet device structure.
* @param up
* Nonzero for link up, otherwise link down.
{
struct rte_eth_dev *dev = priv->dev;
int err;
- unsigned int i;
if (up) {
err = priv_set_flags(priv, ~IFF_UP, IFF_UP);
if (err)
return err;
- for (i = 0; i < priv->rxqs_n; i++)
- if ((*priv->rxqs)[i]->sp)
- break;
- /* Check if an sp queue exists.
- * Note: Some old frames might be received.
- */
- if (i == priv->rxqs_n)
- dev->rx_pkt_burst = mlx5_rx_burst;
- else
- dev->rx_pkt_burst = mlx5_rx_burst_sp;
- dev->tx_pkt_burst = mlx5_tx_burst;
+ priv_select_tx_function(priv);
+ priv_select_rx_function(priv);
} else {
err = priv_set_flags(priv, ~IFF_UP, ~IFF_UP);
if (err)
/* TX queues. */
for (i = 0; i != nb_tx_queues; ++i) {
struct txq *primary_txq = (*sd->primary_priv->txqs)[i];
- struct txq *txq;
+ struct txq_ctrl *primary_txq_ctrl;
+ struct txq_ctrl *txq_ctrl;
if (primary_txq == NULL)
continue;
- txq = rte_calloc_socket("TXQ", 1, sizeof(*txq), 0,
- primary_txq->socket);
- if (txq != NULL) {
- if (txq_setup(priv->dev,
- txq,
- primary_txq->elts_n * MLX5_PMD_SGE_WR_N,
- primary_txq->socket,
- NULL) == 0) {
- txq->stats.idx = primary_txq->stats.idx;
- tx_queues[i] = txq;
+ primary_txq_ctrl = container_of(primary_txq,
+ struct txq_ctrl, txq);
+ txq_ctrl = rte_calloc_socket("TXQ", 1, sizeof(*txq_ctrl) +
+ (1 << primary_txq->elts_n) *
+ sizeof(struct rte_mbuf *), 0,
+ primary_txq_ctrl->socket);
+ if (txq_ctrl != NULL) {
+ if (txq_ctrl_setup(priv->dev,
+ txq_ctrl,
+ 1 << primary_txq->elts_n,
+ primary_txq_ctrl->socket,
+ NULL) == 0) {
+ txq_ctrl->txq.stats.idx =
+ primary_txq->stats.idx;
+ tx_queues[i] = &txq_ctrl->txq;
continue;
}
- rte_free(txq);
+ rte_free(txq_ctrl);
}
while (i) {
- txq = tx_queues[--i];
- txq_cleanup(txq);
- rte_free(txq);
+ txq_ctrl = tx_queues[--i];
+ txq_cleanup(txq_ctrl);
+ rte_free(txq_ctrl);
}
goto error;
}
/* RX queues. */
for (i = 0; i != nb_rx_queues; ++i) {
- struct rxq *primary_rxq = (*sd->primary_priv->rxqs)[i];
+ struct rxq_ctrl *primary_rxq =
+ container_of((*sd->primary_priv->rxqs)[i],
+ struct rxq_ctrl, rxq);
if (primary_rxq == NULL)
continue;
rte_mb();
priv->dev->data = &sd->data;
rte_mb();
- priv->dev->tx_pkt_burst = mlx5_tx_burst;
- priv->dev->rx_pkt_burst = removed_rx_burst;
+ priv_select_tx_function(priv);
+ priv_select_rx_function(priv);
priv_unlock(priv);
end:
/* More sanity checks. */
- assert(priv->dev->tx_pkt_burst == mlx5_tx_burst);
- assert(priv->dev->rx_pkt_burst == removed_rx_burst);
assert(priv->dev->data == &sd->data);
rte_spinlock_unlock(&sd->lock);
return priv;
rte_spinlock_unlock(&sd->lock);
return NULL;
}
+
+/**
+ * Configure the TX function to use.
+ *
+ * @param priv
+ * Pointer to private structure.
+ */
+void
+priv_select_tx_function(struct priv *priv)
+{
+ priv->dev->tx_pkt_burst = mlx5_tx_burst;
+ /* Display warning for unsupported configurations. */
+ if (priv->sriov && priv->mps)
+ WARN("multi-packet send WQE cannot be used on a SR-IOV setup");
+ /* Select appropriate TX function. */
+ if ((priv->sriov == 0) && priv->mps && priv->txq_inline) {
+ priv->dev->tx_pkt_burst = mlx5_tx_burst_mpw_inline;
+ DEBUG("selected MPW inline TX function");
+ } else if ((priv->sriov == 0) && priv->mps) {
+ priv->dev->tx_pkt_burst = mlx5_tx_burst_mpw;
+ DEBUG("selected MPW TX function");
+ }
+}
+
+/**
+ * Configure the RX function to use.
+ *
+ * @param priv
+ * Pointer to private structure.
+ */
+void
+priv_select_rx_function(struct priv *priv)
+{
+ priv->dev->rx_pkt_burst = mlx5_rx_burst;
+}
git.droids-corp.org / dpdk.git / blobdiff