X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=drivers%2Fnet%2Ffailsafe%2Ffailsafe_ops.c;h=55e21d635c9eff3eb52a402fb169698a33393396;hb=f9f0b5121f62d8701165d20bfdd14638bc72654d;hp=e0f1b0b7e730360ef3245d42bf1100f716ee001a;hpb=6265ab51b827e6c33d5c01fe7dd13057b642c88e;p=dpdk.git diff --git a/drivers/net/failsafe/failsafe_ops.c b/drivers/net/failsafe/failsafe_ops.c index e0f1b0b7e7..55e21d635c 100644 --- a/drivers/net/failsafe/failsafe_ops.c +++ b/drivers/net/failsafe/failsafe_ops.c @@ -1,215 +1,38 @@ -/*- - * BSD LICENSE - * - * Copyright 2017 6WIND S.A. - * Copyright 2017 Mellanox. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of 6WIND S.A. nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright 2017 6WIND S.A. + * Copyright 2017 Mellanox Technologies, Ltd */ +#include #include +#include #include #include -#include +#include #include #include +#include +#include +#include #include "failsafe_private.h" -static struct rte_eth_dev_info default_infos = { - /* Max possible number of elements */ - .max_rx_pktlen = UINT32_MAX, - .max_rx_queues = RTE_MAX_QUEUES_PER_PORT, - .max_tx_queues = RTE_MAX_QUEUES_PER_PORT, - .max_mac_addrs = FAILSAFE_MAX_ETHADDR, - .max_hash_mac_addrs = UINT32_MAX, - .max_vfs = UINT16_MAX, - .max_vmdq_pools = UINT16_MAX, - .rx_desc_lim = { - .nb_max = UINT16_MAX, - .nb_min = 0, - .nb_align = 1, - .nb_seg_max = UINT16_MAX, - .nb_mtu_seg_max = UINT16_MAX, - }, - .tx_desc_lim = { - .nb_max = UINT16_MAX, - .nb_min = 0, - .nb_align = 1, - .nb_seg_max = UINT16_MAX, - .nb_mtu_seg_max = UINT16_MAX, - }, - /* - * Set of capabilities that can be verified upon - * configuring a sub-device. - */ - .rx_offload_capa = - DEV_RX_OFFLOAD_VLAN_STRIP | - DEV_RX_OFFLOAD_QINQ_STRIP | - DEV_RX_OFFLOAD_IPV4_CKSUM | - DEV_RX_OFFLOAD_UDP_CKSUM | - DEV_RX_OFFLOAD_TCP_CKSUM | - DEV_RX_OFFLOAD_TCP_LRO, - .tx_offload_capa = 0x0, - .flow_type_rss_offloads = 0x0, -}; - -/** - * Check whether a specific offloading capability - * is supported by a sub_device. - * - * @return - * 0: all requested capabilities are supported by the sub_device - * positive value: This flag at least is not supported by the sub_device - */ -static int -fs_port_offload_validate(struct rte_eth_dev *dev, - struct sub_device *sdev) -{ - struct rte_eth_dev_info infos = {0}; - struct rte_eth_conf *cf; - uint32_t cap; - - cf = &dev->data->dev_conf; - SUBOPS(sdev, dev_infos_get)(ETH(sdev), &infos); - /* RX capabilities */ - cap = infos.rx_offload_capa; - if (cf->rxmode.hw_vlan_strip && - ((cap & DEV_RX_OFFLOAD_VLAN_STRIP) == 0)) { - WARN("VLAN stripping offload requested but not supported by sub_device %d", - SUB_ID(sdev)); - return DEV_RX_OFFLOAD_VLAN_STRIP; - } - if (cf->rxmode.hw_ip_checksum && - ((cap & (DEV_RX_OFFLOAD_IPV4_CKSUM | - DEV_RX_OFFLOAD_UDP_CKSUM | - DEV_RX_OFFLOAD_TCP_CKSUM)) != - (DEV_RX_OFFLOAD_IPV4_CKSUM | - DEV_RX_OFFLOAD_UDP_CKSUM | - DEV_RX_OFFLOAD_TCP_CKSUM))) { - WARN("IP checksum offload requested but not supported by sub_device %d", - SUB_ID(sdev)); - return DEV_RX_OFFLOAD_IPV4_CKSUM | - DEV_RX_OFFLOAD_UDP_CKSUM | - DEV_RX_OFFLOAD_TCP_CKSUM; - } - if (cf->rxmode.enable_lro && - ((cap & DEV_RX_OFFLOAD_TCP_LRO) == 0)) { - WARN("TCP LRO offload requested but not supported by sub_device %d", - SUB_ID(sdev)); - return DEV_RX_OFFLOAD_TCP_LRO; - } - if (cf->rxmode.hw_vlan_extend && - ((cap & DEV_RX_OFFLOAD_QINQ_STRIP) == 0)) { - WARN("Stacked VLAN stripping offload requested but not supported by sub_device %d", - SUB_ID(sdev)); - return DEV_RX_OFFLOAD_QINQ_STRIP; - } - /* TX capabilities */ - /* Nothing to do, no tx capa supported */ - return 0; -} - -/* - * Disable the dev_conf flag related to an offload capability flag - * within an ethdev configuration. - */ -static int -fs_port_disable_offload(struct rte_eth_conf *cf, - uint32_t ol_cap) -{ - switch (ol_cap) { - case DEV_RX_OFFLOAD_VLAN_STRIP: - INFO("Disabling VLAN stripping offload"); - cf->rxmode.hw_vlan_strip = 0; - break; - case DEV_RX_OFFLOAD_IPV4_CKSUM: - case DEV_RX_OFFLOAD_UDP_CKSUM: - case DEV_RX_OFFLOAD_TCP_CKSUM: - case (DEV_RX_OFFLOAD_IPV4_CKSUM | - DEV_RX_OFFLOAD_UDP_CKSUM | - DEV_RX_OFFLOAD_TCP_CKSUM): - INFO("Disabling IP checksum offload"); - cf->rxmode.hw_ip_checksum = 0; - break; - case DEV_RX_OFFLOAD_TCP_LRO: - INFO("Disabling TCP LRO offload"); - cf->rxmode.enable_lro = 0; - break; - case DEV_RX_OFFLOAD_QINQ_STRIP: - INFO("Disabling stacked VLAN stripping offload"); - cf->rxmode.hw_vlan_extend = 0; - break; - default: - DEBUG("Unable to disable offload capability: %" PRIx32, - ol_cap); - return -1; - } - return 0; -} - static int fs_dev_configure(struct rte_eth_dev *dev) { struct sub_device *sdev; uint8_t i; - int capa_flag; int ret; - FOREACH_SUBDEV(sdev, i, dev) { - if (sdev->state != DEV_PROBED) - continue; - DEBUG("Checking capabilities for sub_device %d", i); - while ((capa_flag = fs_port_offload_validate(dev, sdev))) { - /* - * Refuse to change configuration if multiple devices - * are present and we already have configured at least - * some of them. - */ - if (PRIV(dev)->state >= DEV_ACTIVE && - PRIV(dev)->subs_tail > 1) { - ERROR("device already configured, cannot fix live configuration"); - return -1; - } - ret = fs_port_disable_offload(&dev->data->dev_conf, - capa_flag); - if (ret) { - ERROR("Unable to disable offload capability"); - return ret; - } - } - } + fs_lock(dev, 0); FOREACH_SUBDEV(sdev, i, dev) { int rmv_interrupt = 0; int lsc_interrupt = 0; int lsc_enabled; - if (sdev->state != DEV_PROBED) + if (sdev->state != DEV_PROBED && + !(PRIV(dev)->alarm_lock == 0 && sdev->state == DEV_ACTIVE)) continue; rmv_interrupt = ETH(sdev)->data->dev_flags & @@ -232,16 +55,18 @@ fs_dev_configure(struct rte_eth_dev *dev) dev->data->dev_conf.intr_conf.lsc = 0; } DEBUG("Configuring sub-device %d", i); - sdev->remove = 0; ret = rte_eth_dev_configure(PORT_ID(sdev), dev->data->nb_rx_queues, dev->data->nb_tx_queues, &dev->data->dev_conf); if (ret) { + if (!fs_err(sdev, ret)) + continue; ERROR("Could not configure sub_device %d", i); + fs_unlock(dev, 0); return ret; } - if (rmv_interrupt) { + if (rmv_interrupt && sdev->rmv_callback == 0) { ret = rte_eth_dev_callback_register(PORT_ID(sdev), RTE_ETH_EVENT_INTR_RMV, failsafe_eth_rmv_event_callback, @@ -249,9 +74,11 @@ fs_dev_configure(struct rte_eth_dev *dev) if (ret) WARN("Failed to register RMV callback for sub_device %d", SUB_ID(sdev)); + else + sdev->rmv_callback = 1; } dev->data->dev_conf.intr_conf.rmv = 0; - if (lsc_interrupt) { + if (lsc_interrupt && sdev->lsc_callback == 0) { ret = rte_eth_dev_callback_register(PORT_ID(sdev), RTE_ETH_EVENT_INTR_LSC, failsafe_eth_lsc_event_callback, @@ -259,15 +86,39 @@ fs_dev_configure(struct rte_eth_dev *dev) if (ret) WARN("Failed to register LSC callback for sub_device %d", SUB_ID(sdev)); + else + sdev->lsc_callback = 1; } dev->data->dev_conf.intr_conf.lsc = lsc_enabled; sdev->state = DEV_ACTIVE; } if (PRIV(dev)->state < DEV_ACTIVE) PRIV(dev)->state = DEV_ACTIVE; + fs_unlock(dev, 0); return 0; } +static void +fs_set_queues_state_start(struct rte_eth_dev *dev) +{ + struct rxq *rxq; + struct txq *txq; + uint16_t i; + + for (i = 0; i < dev->data->nb_rx_queues; i++) { + rxq = dev->data->rx_queues[i]; + if (rxq != NULL && !rxq->info.conf.rx_deferred_start) + dev->data->rx_queue_state[i] = + RTE_ETH_QUEUE_STATE_STARTED; + } + for (i = 0; i < dev->data->nb_tx_queues; i++) { + txq = dev->data->tx_queues[i]; + if (txq != NULL && !txq->info.conf.tx_deferred_start) + dev->data->tx_queue_state[i] = + RTE_ETH_QUEUE_STATE_STARTED; + } +} + static int fs_dev_start(struct rte_eth_dev *dev) { @@ -275,32 +126,85 @@ fs_dev_start(struct rte_eth_dev *dev) uint8_t i; int ret; + fs_lock(dev, 0); + ret = failsafe_rx_intr_install(dev); + if (ret) { + fs_unlock(dev, 0); + return ret; + } FOREACH_SUBDEV(sdev, i, dev) { if (sdev->state != DEV_ACTIVE) continue; DEBUG("Starting sub_device %d", i); ret = rte_eth_dev_start(PORT_ID(sdev)); - if (ret) + if (ret) { + if (!fs_err(sdev, ret)) + continue; + fs_unlock(dev, 0); + return ret; + } + ret = failsafe_rx_intr_install_subdevice(sdev); + if (ret) { + if (!fs_err(sdev, ret)) + continue; + if (fs_err(sdev, rte_eth_dev_stop(PORT_ID(sdev))) < 0) + ERROR("Failed to stop sub-device %u", + SUB_ID(sdev)); + fs_unlock(dev, 0); return ret; + } sdev->state = DEV_STARTED; } - if (PRIV(dev)->state < DEV_STARTED) + if (PRIV(dev)->state < DEV_STARTED) { PRIV(dev)->state = DEV_STARTED; + fs_set_queues_state_start(dev); + } fs_switch_dev(dev, NULL); + fs_unlock(dev, 0); return 0; } static void +fs_set_queues_state_stop(struct rte_eth_dev *dev) +{ + uint16_t i; + + for (i = 0; i < dev->data->nb_rx_queues; i++) + if (dev->data->rx_queues[i] != NULL) + dev->data->rx_queue_state[i] = + RTE_ETH_QUEUE_STATE_STOPPED; + for (i = 0; i < dev->data->nb_tx_queues; i++) + if (dev->data->tx_queues[i] != NULL) + dev->data->tx_queue_state[i] = + RTE_ETH_QUEUE_STATE_STOPPED; +} + +static int fs_dev_stop(struct rte_eth_dev *dev) { struct sub_device *sdev; uint8_t i; + int ret; + fs_lock(dev, 0); PRIV(dev)->state = DEV_STARTED - 1; FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_STARTED) { - rte_eth_dev_stop(PORT_ID(sdev)); + ret = rte_eth_dev_stop(PORT_ID(sdev)); + if (fs_err(sdev, ret) < 0) { + ERROR("Failed to stop device %u", + PORT_ID(sdev)); + PRIV(dev)->state = DEV_STARTED + 1; + fs_unlock(dev, 0); + return ret; + } + failsafe_rx_intr_uninstall_subdevice(sdev); sdev->state = DEV_STARTED - 1; } + failsafe_rx_intr_uninstall(dev); + fs_set_queues_state_stop(dev); + fs_unlock(dev, 0); + + return 0; } static int @@ -310,15 +214,18 @@ fs_dev_set_link_up(struct rte_eth_dev *dev) uint8_t i; int ret; + fs_lock(dev, 0); FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { DEBUG("Calling rte_eth_dev_set_link_up on sub_device %d", i); ret = rte_eth_dev_set_link_up(PORT_ID(sdev)); - if (ret) { + if ((ret = fs_err(sdev, ret))) { ERROR("Operation rte_eth_dev_set_link_up failed for sub_device %d" " with error %d", i, ret); + fs_unlock(dev, 0); return ret; } } + fs_unlock(dev, 0); return 0; } @@ -329,54 +236,147 @@ fs_dev_set_link_down(struct rte_eth_dev *dev) uint8_t i; int ret; + fs_lock(dev, 0); FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { DEBUG("Calling rte_eth_dev_set_link_down on sub_device %d", i); ret = rte_eth_dev_set_link_down(PORT_ID(sdev)); - if (ret) { + if ((ret = fs_err(sdev, ret))) { ERROR("Operation rte_eth_dev_set_link_down failed for sub_device %d" " with error %d", i, ret); + fs_unlock(dev, 0); return ret; } } + fs_unlock(dev, 0); return 0; } -static void fs_dev_free_queues(struct rte_eth_dev *dev); -static void -fs_dev_close(struct rte_eth_dev *dev) +static int +fs_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id) { struct sub_device *sdev; uint8_t i; + int ret; + int err = 0; + bool failure = true; - failsafe_hotplug_alarm_cancel(dev); - if (PRIV(dev)->state == DEV_STARTED) - dev->dev_ops->dev_stop(dev); - PRIV(dev)->state = DEV_ACTIVE - 1; + fs_lock(dev, 0); FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { - DEBUG("Closing sub_device %d", i); - rte_eth_dev_close(PORT_ID(sdev)); - sdev->state = DEV_ACTIVE - 1; + uint16_t port_id = ETH(sdev)->data->port_id; + + ret = rte_eth_dev_rx_queue_stop(port_id, rx_queue_id); + ret = fs_err(sdev, ret); + if (ret) { + ERROR("Rx queue stop failed for subdevice %d", i); + err = ret; + } else { + failure = false; + } } - fs_dev_free_queues(dev); + dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; + fs_unlock(dev, 0); + /* Return 0 in case of at least one successful queue stop */ + return (failure) ? err : 0; +} + +static int +fs_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + fs_lock(dev, 0); + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + uint16_t port_id = ETH(sdev)->data->port_id; + + ret = rte_eth_dev_rx_queue_start(port_id, rx_queue_id); + ret = fs_err(sdev, ret); + if (ret) { + ERROR("Rx queue start failed for subdevice %d", i); + fs_rx_queue_stop(dev, rx_queue_id); + fs_unlock(dev, 0); + return ret; + } + } + dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; + fs_unlock(dev, 0); + return 0; +} + +static int +fs_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + int err = 0; + bool failure = true; + + fs_lock(dev, 0); + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + uint16_t port_id = ETH(sdev)->data->port_id; + + ret = rte_eth_dev_tx_queue_stop(port_id, tx_queue_id); + ret = fs_err(sdev, ret); + if (ret) { + ERROR("Tx queue stop failed for subdevice %d", i); + err = ret; + } else { + failure = false; + } + } + dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED; + fs_unlock(dev, 0); + /* Return 0 in case of at least one successful queue stop */ + return (failure) ? err : 0; +} + +static int +fs_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + fs_lock(dev, 0); + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + uint16_t port_id = ETH(sdev)->data->port_id; + + ret = rte_eth_dev_tx_queue_start(port_id, tx_queue_id); + ret = fs_err(sdev, ret); + if (ret) { + ERROR("Tx queue start failed for subdevice %d", i); + fs_tx_queue_stop(dev, tx_queue_id); + fs_unlock(dev, 0); + return ret; + } + } + dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED; + fs_unlock(dev, 0); + return 0; } static void -fs_rx_queue_release(void *queue) +fs_rx_queue_release(struct rte_eth_dev *dev, uint16_t qid) { - struct rte_eth_dev *dev; struct sub_device *sdev; uint8_t i; - struct rxq *rxq; + struct rxq *rxq = dev->data->rx_queues[qid]; - if (queue == NULL) + if (rxq == NULL) return; - rxq = queue; - dev = rxq->priv->dev; - FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) - SUBOPS(sdev, rx_queue_release) - (ETH(sdev)->data->rx_queues[rxq->qid]); + fs_lock(dev, 0); + if (rxq->event_fd >= 0) + close(rxq->event_fd); + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + if (ETH(sdev)->data->rx_queues != NULL && + ETH(sdev)->data->rx_queues[rxq->qid] != NULL) + SUBOPS(sdev, rx_queue_release)(ETH(sdev), rxq->qid); + } dev->data->rx_queues[rxq->qid] = NULL; rte_free(rxq); + fs_unlock(dev, 0); } static int @@ -387,22 +387,52 @@ fs_rx_queue_setup(struct rte_eth_dev *dev, const struct rte_eth_rxconf *rx_conf, struct rte_mempool *mb_pool) { + /* + * FIXME: Add a proper interface in rte_eal_interrupts for + * allocating eventfd as an interrupt vector. + * For the time being, fake as if we are using MSIX interrupts, + * this will cause rte_intr_efd_enable to allocate an eventfd for us. + */ + struct rte_intr_handle *intr_handle; struct sub_device *sdev; struct rxq *rxq; uint8_t i; int ret; + intr_handle = rte_intr_instance_alloc(RTE_INTR_INSTANCE_F_PRIVATE); + if (intr_handle == NULL) + return -ENOMEM; + + if (rte_intr_type_set(intr_handle, RTE_INTR_HANDLE_VFIO_MSIX)) + return -rte_errno; + + if (rte_intr_efds_index_set(intr_handle, 0, -1)) + return -rte_errno; + + fs_lock(dev, 0); + if (rx_conf->rx_deferred_start) { + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) { + if (SUBOPS(sdev, rx_queue_start) == NULL) { + ERROR("Rx queue deferred start is not " + "supported for subdevice %d", i); + fs_unlock(dev, 0); + return -EINVAL; + } + } + } rxq = dev->data->rx_queues[rx_queue_id]; if (rxq != NULL) { - fs_rx_queue_release(rxq); + fs_rx_queue_release(dev, rx_queue_id); dev->data->rx_queues[rx_queue_id] = NULL; } rxq = rte_zmalloc(NULL, sizeof(*rxq) + sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail, RTE_CACHE_LINE_SIZE); - if (rxq == NULL) + if (rxq == NULL) { + fs_unlock(dev, 0); return -ENOMEM; + } FOREACH_SUBDEV(sdev, i, dev) rte_atomic64_init(&rxq->refcnt[i]); rxq->qid = rx_queue_id; @@ -411,40 +441,126 @@ fs_rx_queue_setup(struct rte_eth_dev *dev, rxq->info.conf = *rx_conf; rxq->info.nb_desc = nb_rx_desc; rxq->priv = PRIV(dev); + rxq->sdev = PRIV(dev)->subs; + ret = rte_intr_efd_enable(intr_handle, 1); + if (ret < 0) { + fs_unlock(dev, 0); + return ret; + } + rxq->event_fd = rte_intr_efds_index_get(intr_handle, 0); dev->data->rx_queues[rx_queue_id] = rxq; FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { ret = rte_eth_rx_queue_setup(PORT_ID(sdev), rx_queue_id, nb_rx_desc, socket_id, rx_conf, mb_pool); - if (ret) { + if ((ret = fs_err(sdev, ret))) { ERROR("RX queue setup failed for sub_device %d", i); goto free_rxq; } } + fs_unlock(dev, 0); return 0; free_rxq: - fs_rx_queue_release(rxq); + fs_rx_queue_release(dev, rx_queue_id); + fs_unlock(dev, 0); return ret; } +static int +fs_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx) +{ + struct rxq *rxq; + struct sub_device *sdev; + uint8_t i; + int ret; + int rc = 0; + + fs_lock(dev, 0); + if (idx >= dev->data->nb_rx_queues) { + rc = -EINVAL; + goto unlock; + } + rxq = dev->data->rx_queues[idx]; + if (rxq == NULL || rxq->event_fd <= 0) { + rc = -EINVAL; + goto unlock; + } + /* Fail if proxy service is nor running. */ + if (PRIV(dev)->rxp.sstate != SS_RUNNING) { + ERROR("failsafe interrupt services are not running"); + rc = -EAGAIN; + goto unlock; + } + rxq->enable_events = 1; + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + ret = rte_eth_dev_rx_intr_enable(PORT_ID(sdev), idx); + ret = fs_err(sdev, ret); + if (ret) + rc = ret; + } +unlock: + fs_unlock(dev, 0); + if (rc) + rte_errno = -rc; + return rc; +} + +static int +fs_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx) +{ + struct rxq *rxq; + struct sub_device *sdev; + uint64_t u64; + uint8_t i; + int rc = 0; + int ret; + + fs_lock(dev, 0); + if (idx >= dev->data->nb_rx_queues) { + rc = -EINVAL; + goto unlock; + } + rxq = dev->data->rx_queues[idx]; + if (rxq == NULL || rxq->event_fd <= 0) { + rc = -EINVAL; + goto unlock; + } + rxq->enable_events = 0; + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + ret = rte_eth_dev_rx_intr_disable(PORT_ID(sdev), idx); + ret = fs_err(sdev, ret); + if (ret) + rc = ret; + } + /* Clear pending events */ + while (read(rxq->event_fd, &u64, sizeof(uint64_t)) > 0) + ; +unlock: + fs_unlock(dev, 0); + if (rc) + rte_errno = -rc; + return rc; +} + static void -fs_tx_queue_release(void *queue) +fs_tx_queue_release(struct rte_eth_dev *dev, uint16_t qid) { - struct rte_eth_dev *dev; struct sub_device *sdev; uint8_t i; - struct txq *txq; + struct txq *txq = dev->data->tx_queues[qid]; - if (queue == NULL) + if (txq == NULL) return; - txq = queue; - dev = txq->priv->dev; - FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) - SUBOPS(sdev, tx_queue_release) - (ETH(sdev)->data->tx_queues[txq->qid]); + fs_lock(dev, 0); + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + if (ETH(sdev)->data->tx_queues != NULL && + ETH(sdev)->data->tx_queues[txq->qid] != NULL) + SUBOPS(sdev, tx_queue_release)(ETH(sdev), txq->qid); + } dev->data->tx_queues[txq->qid] = NULL; rte_free(txq); + fs_unlock(dev, 0); } static int @@ -459,17 +575,30 @@ fs_tx_queue_setup(struct rte_eth_dev *dev, uint8_t i; int ret; + fs_lock(dev, 0); + if (tx_conf->tx_deferred_start) { + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) { + if (SUBOPS(sdev, tx_queue_start) == NULL) { + ERROR("Tx queue deferred start is not " + "supported for subdevice %d", i); + fs_unlock(dev, 0); + return -EINVAL; + } + } + } txq = dev->data->tx_queues[tx_queue_id]; if (txq != NULL) { - fs_tx_queue_release(txq); + fs_tx_queue_release(dev, tx_queue_id); dev->data->tx_queues[tx_queue_id] = NULL; } txq = rte_zmalloc("ethdev TX queue", sizeof(*txq) + sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail, RTE_CACHE_LINE_SIZE); - if (txq == NULL) + if (txq == NULL) { + fs_unlock(dev, 0); return -ENOMEM; + } FOREACH_SUBDEV(sdev, i, dev) rte_atomic64_init(&txq->refcnt[i]); txq->qid = tx_queue_id; @@ -483,14 +612,16 @@ fs_tx_queue_setup(struct rte_eth_dev *dev, tx_queue_id, nb_tx_desc, socket_id, tx_conf); - if (ret) { + if ((ret = fs_err(sdev, ret))) { ERROR("TX queue setup failed for sub_device %d", i); goto free_txq; } } + fs_unlock(dev, 0); return 0; free_txq: - fs_tx_queue_release(txq); + fs_tx_queue_release(dev, tx_queue_id); + fs_unlock(dev, 0); return ret; } @@ -500,55 +631,197 @@ fs_dev_free_queues(struct rte_eth_dev *dev) uint16_t i; for (i = 0; i < dev->data->nb_rx_queues; i++) { - fs_rx_queue_release(dev->data->rx_queues[i]); + fs_rx_queue_release(dev, i); dev->data->rx_queues[i] = NULL; } dev->data->nb_rx_queues = 0; for (i = 0; i < dev->data->nb_tx_queues; i++) { - fs_tx_queue_release(dev->data->tx_queues[i]); + fs_tx_queue_release(dev, i); dev->data->tx_queues[i] = NULL; } dev->data->nb_tx_queues = 0; } -static void +int +failsafe_eth_dev_close(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + int err, ret = 0; + + fs_lock(dev, 0); + failsafe_hotplug_alarm_cancel(dev); + if (PRIV(dev)->state == DEV_STARTED) { + ret = dev->dev_ops->dev_stop(dev); + if (ret != 0) { + fs_unlock(dev, 0); + return ret; + } + } + PRIV(dev)->state = DEV_ACTIVE - 1; + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + DEBUG("Closing sub_device %d", i); + failsafe_eth_dev_unregister_callbacks(sdev); + err = rte_eth_dev_close(PORT_ID(sdev)); + if (err) { + ret = ret ? ret : err; + ERROR("Error while closing sub-device %u", + PORT_ID(sdev)); + } + sdev->state = DEV_ACTIVE - 1; + } + rte_eth_dev_callback_unregister(RTE_ETH_ALL, RTE_ETH_EVENT_NEW, + failsafe_eth_new_event_callback, dev); + if (rte_eal_process_type() != RTE_PROC_PRIMARY) { + fs_unlock(dev, 0); + return ret; + } + fs_dev_free_queues(dev); + err = failsafe_eal_uninit(dev); + if (err) { + ret = ret ? ret : err; + ERROR("Error while uninitializing sub-EAL"); + } + failsafe_args_free(dev); + rte_free(PRIV(dev)->subs); + rte_free(PRIV(dev)->mcast_addrs); + /* mac_addrs must not be freed alone because part of dev_private */ + dev->data->mac_addrs = NULL; + fs_unlock(dev, 0); + err = pthread_mutex_destroy(&PRIV(dev)->hotplug_mutex); + if (err) { + ret = ret ? ret : err; + ERROR("Error while destroying hotplug mutex"); + } + return ret; +} + +static int fs_promiscuous_enable(struct rte_eth_dev *dev) { struct sub_device *sdev; uint8_t i; + int ret = 0; - FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) - rte_eth_promiscuous_enable(PORT_ID(sdev)); + fs_lock(dev, 0); + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + ret = rte_eth_promiscuous_enable(PORT_ID(sdev)); + ret = fs_err(sdev, ret); + if (ret != 0) { + ERROR("Promiscuous mode enable failed for subdevice %d", + PORT_ID(sdev)); + break; + } + } + if (ret != 0) { + /* Rollback in the case of failure */ + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + ret = rte_eth_promiscuous_disable(PORT_ID(sdev)); + ret = fs_err(sdev, ret); + if (ret != 0) + ERROR("Promiscuous mode disable during rollback failed for subdevice %d", + PORT_ID(sdev)); + } + } + fs_unlock(dev, 0); + + return ret; } -static void +static int fs_promiscuous_disable(struct rte_eth_dev *dev) { struct sub_device *sdev; uint8_t i; + int ret = 0; - FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) - rte_eth_promiscuous_disable(PORT_ID(sdev)); + fs_lock(dev, 0); + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + ret = rte_eth_promiscuous_disable(PORT_ID(sdev)); + ret = fs_err(sdev, ret); + if (ret != 0) { + ERROR("Promiscuous mode disable failed for subdevice %d", + PORT_ID(sdev)); + break; + } + } + if (ret != 0) { + /* Rollback in the case of failure */ + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + ret = rte_eth_promiscuous_enable(PORT_ID(sdev)); + ret = fs_err(sdev, ret); + if (ret != 0) + ERROR("Promiscuous mode enable during rollback failed for subdevice %d", + PORT_ID(sdev)); + } + } + fs_unlock(dev, 0); + + return ret; } -static void +static int fs_allmulticast_enable(struct rte_eth_dev *dev) { struct sub_device *sdev; uint8_t i; + int ret = 0; - FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) - rte_eth_allmulticast_enable(PORT_ID(sdev)); + fs_lock(dev, 0); + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + ret = rte_eth_allmulticast_enable(PORT_ID(sdev)); + ret = fs_err(sdev, ret); + if (ret != 0) { + ERROR("All-multicast mode enable failed for subdevice %d", + PORT_ID(sdev)); + break; + } + } + if (ret != 0) { + /* Rollback in the case of failure */ + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + ret = rte_eth_allmulticast_disable(PORT_ID(sdev)); + ret = fs_err(sdev, ret); + if (ret != 0) + ERROR("All-multicast mode disable during rollback failed for subdevice %d", + PORT_ID(sdev)); + } + } + fs_unlock(dev, 0); + + return ret; } -static void +static int fs_allmulticast_disable(struct rte_eth_dev *dev) { struct sub_device *sdev; uint8_t i; + int ret = 0; - FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) - rte_eth_allmulticast_disable(PORT_ID(sdev)); + fs_lock(dev, 0); + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + ret = rte_eth_allmulticast_disable(PORT_ID(sdev)); + ret = fs_err(sdev, ret); + if (ret != 0) { + ERROR("All-multicast mode disable failed for subdevice %d", + PORT_ID(sdev)); + break; + } + } + if (ret != 0) { + /* Rollback in the case of failure */ + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + ret = rte_eth_allmulticast_enable(PORT_ID(sdev)); + ret = fs_err(sdev, ret); + if (ret != 0) + ERROR("All-multicast mode enable during rollback failed for subdevice %d", + PORT_ID(sdev)); + } + } + fs_unlock(dev, 0); + + return ret; } static int @@ -559,12 +832,15 @@ fs_link_update(struct rte_eth_dev *dev, uint8_t i; int ret; + fs_lock(dev, 0); FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { DEBUG("Calling link_update on sub_device %d", i); ret = (SUBOPS(sdev, link_update))(ETH(sdev), wait_to_complete); - if (ret && ret != -1) { + if (ret && ret != -1 && sdev->remove == 0 && + rte_eth_dev_is_removed(PORT_ID(sdev)) == 0) { ERROR("Link update failed for sub_device %d with error %d", i, ret); + fs_unlock(dev, 0); return ret; } } @@ -576,37 +852,274 @@ fs_link_update(struct rte_eth_dev *dev, l2 = Ð(TX_SUBDEV(dev))->data->dev_link; if (memcmp(l1, l2, sizeof(*l1))) { *l1 = *l2; + fs_unlock(dev, 0); return 0; } } + fs_unlock(dev, 0); return -1; } -static void +static int fs_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats) { + struct rte_eth_stats backup; struct sub_device *sdev; uint8_t i; + int ret; + fs_lock(dev, 0); rte_memcpy(stats, &PRIV(dev)->stats_accumulator, sizeof(*stats)); FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { - rte_eth_stats_get(PORT_ID(sdev), &sdev->stats_snapshot); - failsafe_stats_increment(stats, &sdev->stats_snapshot); + struct rte_eth_stats *snapshot = &sdev->stats_snapshot.stats; + uint64_t *timestamp = &sdev->stats_snapshot.timestamp; + + rte_memcpy(&backup, snapshot, sizeof(backup)); + ret = rte_eth_stats_get(PORT_ID(sdev), snapshot); + if (ret) { + if (!fs_err(sdev, ret)) { + rte_memcpy(snapshot, &backup, sizeof(backup)); + goto inc; + } + ERROR("Operation rte_eth_stats_get failed for sub_device %d with error %d", + i, ret); + *timestamp = 0; + fs_unlock(dev, 0); + return ret; + } + *timestamp = rte_rdtsc(); +inc: + failsafe_stats_increment(stats, snapshot); } + fs_unlock(dev, 0); + return 0; } -static void +static int fs_stats_reset(struct rte_eth_dev *dev) { struct sub_device *sdev; uint8_t i; + int ret; + fs_lock(dev, 0); FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { - rte_eth_stats_reset(PORT_ID(sdev)); + ret = rte_eth_stats_reset(PORT_ID(sdev)); + if (ret) { + if (!fs_err(sdev, ret)) + continue; + + ERROR("Operation rte_eth_stats_reset failed for sub_device %d with error %d", + i, ret); + fs_unlock(dev, 0); + return ret; + } memset(&sdev->stats_snapshot, 0, sizeof(struct rte_eth_stats)); } memset(&PRIV(dev)->stats_accumulator, 0, sizeof(struct rte_eth_stats)); + fs_unlock(dev, 0); + + return 0; +} + +static int +__fs_xstats_count(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + int count = 0; + uint8_t i; + int ret; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + ret = rte_eth_xstats_get_names(PORT_ID(sdev), NULL, 0); + if (ret < 0) + return ret; + count += ret; + } + + return count; +} + +static int +__fs_xstats_get_names(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, + unsigned int limit) +{ + struct sub_device *sdev; + unsigned int count = 0; + uint8_t i; + + /* Caller only cares about count */ + if (!xstats_names) + return __fs_xstats_count(dev); + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + struct rte_eth_xstat_name *sub_names = xstats_names + count; + int j, r; + + if (count >= limit) + break; + + r = rte_eth_xstats_get_names(PORT_ID(sdev), + sub_names, limit - count); + if (r < 0) + return r; + + /* add subN_ prefix to names */ + for (j = 0; j < r; j++) { + char *xname = sub_names[j].name; + char tmp[RTE_ETH_XSTATS_NAME_SIZE]; + + if ((xname[0] == 't' || xname[0] == 'r') && + xname[1] == 'x' && xname[2] == '_') + snprintf(tmp, sizeof(tmp), "%.3ssub%u_%s", + xname, i, xname + 3); + else + snprintf(tmp, sizeof(tmp), "sub%u_%s", + i, xname); + + strlcpy(xname, tmp, RTE_ETH_XSTATS_NAME_SIZE); + } + count += r; + } + return count; +} + +static int +fs_xstats_get_names(struct rte_eth_dev *dev, + struct rte_eth_xstat_name *xstats_names, + unsigned int limit) +{ + int ret; + + fs_lock(dev, 0); + ret = __fs_xstats_get_names(dev, xstats_names, limit); + fs_unlock(dev, 0); + return ret; +} + +static int +__fs_xstats_get(struct rte_eth_dev *dev, + struct rte_eth_xstat *xstats, + unsigned int n) +{ + unsigned int count = 0; + struct sub_device *sdev; + uint8_t i; + int j, ret; + + ret = __fs_xstats_count(dev); + /* + * if error + * or caller did not give enough space + * or just querying + */ + if (ret < 0 || ret > (int)n || xstats == NULL) + return ret; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + ret = rte_eth_xstats_get(PORT_ID(sdev), xstats, n); + if (ret < 0) + return ret; + + if (ret > (int)n) + return n + count; + + /* add offset to id's from sub-device */ + for (j = 0; j < ret; j++) + xstats[j].id += count; + + xstats += ret; + n -= ret; + count += ret; + } + + return count; +} + +static int +fs_xstats_get(struct rte_eth_dev *dev, + struct rte_eth_xstat *xstats, + unsigned int n) +{ + int ret; + + fs_lock(dev, 0); + ret = __fs_xstats_get(dev, xstats, n); + fs_unlock(dev, 0); + + return ret; +} + + +static int +fs_xstats_reset(struct rte_eth_dev *dev) +{ + struct sub_device *sdev; + uint8_t i; + int r = 0; + + fs_lock(dev, 0); + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + r = rte_eth_xstats_reset(PORT_ID(sdev)); + if (r < 0) + break; + } + fs_unlock(dev, 0); + + return r; +} + +static void +fs_dev_merge_desc_lim(struct rte_eth_desc_lim *to, + const struct rte_eth_desc_lim *from) +{ + to->nb_max = RTE_MIN(to->nb_max, from->nb_max); + to->nb_min = RTE_MAX(to->nb_min, from->nb_min); + to->nb_align = RTE_MAX(to->nb_align, from->nb_align); + + to->nb_seg_max = RTE_MIN(to->nb_seg_max, from->nb_seg_max); + to->nb_mtu_seg_max = RTE_MIN(to->nb_mtu_seg_max, from->nb_mtu_seg_max); +} + +/* + * Merge the information from sub-devices. + * + * The reported values must be the common subset of all sub devices + */ +static void +fs_dev_merge_info(struct rte_eth_dev_info *info, + const struct rte_eth_dev_info *sinfo) +{ + info->min_mtu = RTE_MAX(info->min_mtu, sinfo->min_mtu); + info->max_mtu = RTE_MIN(info->max_mtu, sinfo->max_mtu); + info->max_rx_pktlen = RTE_MIN(info->max_rx_pktlen, sinfo->max_rx_pktlen); + info->max_rx_queues = RTE_MIN(info->max_rx_queues, sinfo->max_rx_queues); + info->max_tx_queues = RTE_MIN(info->max_tx_queues, sinfo->max_tx_queues); + info->max_mac_addrs = RTE_MIN(info->max_mac_addrs, sinfo->max_mac_addrs); + info->max_hash_mac_addrs = RTE_MIN(info->max_hash_mac_addrs, + sinfo->max_hash_mac_addrs); + info->max_vmdq_pools = RTE_MIN(info->max_vmdq_pools, sinfo->max_vmdq_pools); + info->max_vfs = RTE_MIN(info->max_vfs, sinfo->max_vfs); + + fs_dev_merge_desc_lim(&info->rx_desc_lim, &sinfo->rx_desc_lim); + fs_dev_merge_desc_lim(&info->tx_desc_lim, &sinfo->tx_desc_lim); + + info->rx_offload_capa &= sinfo->rx_offload_capa; + info->tx_offload_capa &= sinfo->tx_offload_capa; + info->rx_queue_offload_capa &= sinfo->rx_queue_offload_capa; + info->tx_queue_offload_capa &= sinfo->tx_queue_offload_capa; + info->flow_type_rss_offloads &= sinfo->flow_type_rss_offloads; + + /* + * RETA size is a GCD of RETA sizes indicated by sub-devices. + * Each of these sizes is a power of 2, so use the lower one. + */ + info->reta_size = RTE_MIN(info->reta_size, sinfo->reta_size); + + info->hash_key_size = RTE_MIN(info->hash_key_size, + sinfo->hash_key_size); } /** @@ -636,38 +1149,98 @@ fs_stats_reset(struct rte_eth_dev *dev) * all sub_devices and the default capabilities. * Uses a logical AND of TX capabilities among * the active probed sub_device and the default capabilities. + * Uses a logical AND of device capabilities among + * all sub_devices and the default capabilities. * */ -static void +static int fs_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *infos) { struct sub_device *sdev; uint8_t i; + int ret; - sdev = TX_SUBDEV(dev); - if (sdev == NULL) { - DEBUG("No probed device, using default infos"); - rte_memcpy(&PRIV(dev)->infos, &default_infos, - sizeof(default_infos)); - } else { - uint32_t rx_offload_capa; + /* Use maximum upper bounds by default */ + infos->min_mtu = RTE_ETHER_MIN_MTU; + infos->max_mtu = UINT16_MAX; + infos->max_rx_pktlen = UINT32_MAX; + infos->max_rx_queues = RTE_MAX_QUEUES_PER_PORT; + infos->max_tx_queues = RTE_MAX_QUEUES_PER_PORT; + infos->max_mac_addrs = FAILSAFE_MAX_ETHADDR; + infos->max_hash_mac_addrs = UINT32_MAX; + infos->max_vfs = UINT16_MAX; + infos->max_vmdq_pools = UINT16_MAX; + infos->reta_size = UINT16_MAX; + infos->hash_key_size = UINT8_MAX; - rx_offload_capa = default_infos.rx_offload_capa; - FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) { - rte_eth_dev_info_get(PORT_ID(sdev), - &PRIV(dev)->infos); - rx_offload_capa &= PRIV(dev)->infos.rx_offload_capa; - } - sdev = TX_SUBDEV(dev); - rte_eth_dev_info_get(PORT_ID(sdev), &PRIV(dev)->infos); - PRIV(dev)->infos.rx_offload_capa = rx_offload_capa; - PRIV(dev)->infos.tx_offload_capa &= - default_infos.tx_offload_capa; - PRIV(dev)->infos.flow_type_rss_offloads &= - default_infos.flow_type_rss_offloads; + /* + * Set of capabilities that can be verified upon + * configuring a sub-device. + */ + infos->rx_offload_capa = + RTE_ETH_RX_OFFLOAD_VLAN_STRIP | + RTE_ETH_RX_OFFLOAD_IPV4_CKSUM | + RTE_ETH_RX_OFFLOAD_UDP_CKSUM | + RTE_ETH_RX_OFFLOAD_TCP_CKSUM | + RTE_ETH_RX_OFFLOAD_TCP_LRO | + RTE_ETH_RX_OFFLOAD_QINQ_STRIP | + RTE_ETH_RX_OFFLOAD_OUTER_IPV4_CKSUM | + RTE_ETH_RX_OFFLOAD_MACSEC_STRIP | + RTE_ETH_RX_OFFLOAD_HEADER_SPLIT | + RTE_ETH_RX_OFFLOAD_VLAN_FILTER | + RTE_ETH_RX_OFFLOAD_VLAN_EXTEND | + RTE_ETH_RX_OFFLOAD_SCATTER | + RTE_ETH_RX_OFFLOAD_TIMESTAMP | + RTE_ETH_RX_OFFLOAD_SECURITY | + RTE_ETH_RX_OFFLOAD_RSS_HASH; + + infos->rx_queue_offload_capa = + RTE_ETH_RX_OFFLOAD_VLAN_STRIP | + RTE_ETH_RX_OFFLOAD_IPV4_CKSUM | + RTE_ETH_RX_OFFLOAD_UDP_CKSUM | + RTE_ETH_RX_OFFLOAD_TCP_CKSUM | + RTE_ETH_RX_OFFLOAD_TCP_LRO | + RTE_ETH_RX_OFFLOAD_QINQ_STRIP | + RTE_ETH_RX_OFFLOAD_OUTER_IPV4_CKSUM | + RTE_ETH_RX_OFFLOAD_MACSEC_STRIP | + RTE_ETH_RX_OFFLOAD_HEADER_SPLIT | + RTE_ETH_RX_OFFLOAD_VLAN_FILTER | + RTE_ETH_RX_OFFLOAD_VLAN_EXTEND | + RTE_ETH_RX_OFFLOAD_SCATTER | + RTE_ETH_RX_OFFLOAD_TIMESTAMP | + RTE_ETH_RX_OFFLOAD_SECURITY | + RTE_ETH_RX_OFFLOAD_RSS_HASH; + + infos->tx_offload_capa = + RTE_ETH_TX_OFFLOAD_MULTI_SEGS | + RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE | + RTE_ETH_TX_OFFLOAD_IPV4_CKSUM | + RTE_ETH_TX_OFFLOAD_UDP_CKSUM | + RTE_ETH_TX_OFFLOAD_TCP_CKSUM | + RTE_ETH_TX_OFFLOAD_TCP_TSO; + + infos->flow_type_rss_offloads = + RTE_ETH_RSS_IP | + RTE_ETH_RSS_UDP | + RTE_ETH_RSS_TCP; + infos->dev_capa = + RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP | + RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP; + infos->dev_capa &= ~RTE_ETH_DEV_CAPA_FLOW_RULE_KEEP; + + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) { + struct rte_eth_dev_info sub_info; + + ret = rte_eth_dev_info_get(PORT_ID(sdev), &sub_info); + ret = fs_err(sdev, ret); + if (ret != 0) + return ret; + + fs_dev_merge_info(infos, &sub_info); } - rte_memcpy(infos, &PRIV(dev)->infos, sizeof(*infos)); + + return 0; } static const uint32_t * @@ -675,14 +1248,20 @@ fs_dev_supported_ptypes_get(struct rte_eth_dev *dev) { struct sub_device *sdev; struct rte_eth_dev *edev; + const uint32_t *ret; + fs_lock(dev, 0); sdev = TX_SUBDEV(dev); - if (sdev == NULL) - return NULL; + if (sdev == NULL) { + ret = NULL; + goto unlock; + } edev = ETH(sdev); /* ENOTSUP: counts as no supported ptypes */ - if (SUBOPS(sdev, dev_supported_ptypes_get) == NULL) - return NULL; + if (SUBOPS(sdev, dev_supported_ptypes_get) == NULL) { + ret = NULL; + goto unlock; + } /* * The API does not permit to do a clean AND of all ptypes, * It is also incomplete by design and we do not really care @@ -690,7 +1269,10 @@ fs_dev_supported_ptypes_get(struct rte_eth_dev *dev) * We just return the ptypes of the device of highest * priority, usually the PREFERRED device. */ - return SUBOPS(sdev, dev_supported_ptypes_get)(edev); + ret = SUBOPS(sdev, dev_supported_ptypes_get)(edev); +unlock: + fs_unlock(dev, 0); + return ret; } static int @@ -700,15 +1282,18 @@ fs_mtu_set(struct rte_eth_dev *dev, uint16_t mtu) uint8_t i; int ret; + fs_lock(dev, 0); FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { DEBUG("Calling rte_eth_dev_set_mtu on sub_device %d", i); ret = rte_eth_dev_set_mtu(PORT_ID(sdev), mtu); - if (ret) { + if ((ret = fs_err(sdev, ret))) { ERROR("Operation rte_eth_dev_set_mtu failed for sub_device %d with error %d", i, ret); + fs_unlock(dev, 0); return ret; } } + fs_unlock(dev, 0); return 0; } @@ -719,15 +1304,18 @@ fs_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on) uint8_t i; int ret; + fs_lock(dev, 0); FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { DEBUG("Calling rte_eth_dev_vlan_filter on sub_device %d", i); ret = rte_eth_dev_vlan_filter(PORT_ID(sdev), vlan_id, on); - if (ret) { + if ((ret = fs_err(sdev, ret))) { ERROR("Operation rte_eth_dev_vlan_filter failed for sub_device %d" " with error %d", i, ret); + fs_unlock(dev, 0); return ret; } } + fs_unlock(dev, 0); return 0; } @@ -736,13 +1324,22 @@ fs_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf) { struct sub_device *sdev; + int ret; + fs_lock(dev, 0); sdev = TX_SUBDEV(dev); - if (sdev == NULL) - return 0; - if (SUBOPS(sdev, flow_ctrl_get) == NULL) - return -ENOTSUP; - return SUBOPS(sdev, flow_ctrl_get)(ETH(sdev), fc_conf); + if (sdev == NULL) { + ret = 0; + goto unlock; + } + if (SUBOPS(sdev, flow_ctrl_get) == NULL) { + ret = -ENOTSUP; + goto unlock; + } + ret = SUBOPS(sdev, flow_ctrl_get)(ETH(sdev), fc_conf); +unlock: + fs_unlock(dev, 0); + return ret; } static int @@ -753,15 +1350,18 @@ fs_flow_ctrl_set(struct rte_eth_dev *dev, uint8_t i; int ret; + fs_lock(dev, 0); FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { DEBUG("Calling rte_eth_dev_flow_ctrl_set on sub_device %d", i); ret = rte_eth_dev_flow_ctrl_set(PORT_ID(sdev), fc_conf); - if (ret) { + if ((ret = fs_err(sdev, ret))) { ERROR("Operation rte_eth_dev_flow_ctrl_set failed for sub_device %d" " with error %d", i, ret); + fs_unlock(dev, 0); return ret; } } + fs_unlock(dev, 0); return 0; } @@ -771,6 +1371,7 @@ fs_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) struct sub_device *sdev; uint8_t i; + fs_lock(dev, 0); /* No check: already done within the rte_eth_dev_mac_addr_remove * call for the fail-safe device. */ @@ -778,11 +1379,12 @@ fs_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index) rte_eth_dev_mac_addr_remove(PORT_ID(sdev), &dev->data->mac_addrs[index]); PRIV(dev)->mac_addr_pool[index] = 0; + fs_unlock(dev, 0); } static int fs_mac_addr_add(struct rte_eth_dev *dev, - struct ether_addr *mac_addr, + struct rte_ether_addr *mac_addr, uint32_t index, uint32_t vmdq) { @@ -791,11 +1393,13 @@ fs_mac_addr_add(struct rte_eth_dev *dev, uint8_t i; RTE_ASSERT(index < FAILSAFE_MAX_ETHADDR); + fs_lock(dev, 0); FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { ret = rte_eth_dev_mac_addr_add(PORT_ID(sdev), mac_addr, vmdq); - if (ret) { + if ((ret = fs_err(sdev, ret))) { ERROR("Operation rte_eth_dev_mac_addr_add failed for sub_device %" PRIu8 " with error %d", i, ret); + fs_unlock(dev, 0); return ret; } } @@ -804,43 +1408,112 @@ fs_mac_addr_add(struct rte_eth_dev *dev, PRIV(dev)->nb_mac_addr = index; } PRIV(dev)->mac_addr_pool[index] = vmdq; + fs_unlock(dev, 0); return 0; } -static void -fs_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr) +static int +fs_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr) { struct sub_device *sdev; uint8_t i; + int ret; - FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) - rte_eth_dev_default_mac_addr_set(PORT_ID(sdev), mac_addr); + fs_lock(dev, 0); + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + ret = rte_eth_dev_default_mac_addr_set(PORT_ID(sdev), mac_addr); + ret = fs_err(sdev, ret); + if (ret) { + ERROR("Operation rte_eth_dev_mac_addr_set failed for sub_device %d with error %d", + i, ret); + fs_unlock(dev, 0); + return ret; + } + } + fs_unlock(dev, 0); + + return 0; } static int -fs_filter_ctrl(struct rte_eth_dev *dev, - enum rte_filter_type type, - enum rte_filter_op op, - void *arg) +fs_set_mc_addr_list(struct rte_eth_dev *dev, + struct rte_ether_addr *mc_addr_set, uint32_t nb_mc_addr) { struct sub_device *sdev; uint8_t i; int ret; + void *mcast_addrs; + + fs_lock(dev, 0); - if (type == RTE_ETH_FILTER_GENERIC && - op == RTE_ETH_FILTER_GET) { - *(const void **)arg = &fs_flow_ops; - return 0; + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + ret = rte_eth_dev_set_mc_addr_list(PORT_ID(sdev), + mc_addr_set, nb_mc_addr); + if (ret != 0) { + ERROR("Operation rte_eth_dev_set_mc_addr_list failed for sub_device %d with error %d", + i, ret); + goto rollback; + } } + + mcast_addrs = rte_realloc(PRIV(dev)->mcast_addrs, + nb_mc_addr * sizeof(PRIV(dev)->mcast_addrs[0]), 0); + if (mcast_addrs == NULL && nb_mc_addr > 0) { + ret = -ENOMEM; + goto rollback; + } + rte_memcpy(mcast_addrs, mc_addr_set, + nb_mc_addr * sizeof(PRIV(dev)->mcast_addrs[0])); + PRIV(dev)->nb_mcast_addr = nb_mc_addr; + PRIV(dev)->mcast_addrs = mcast_addrs; + + fs_unlock(dev, 0); + return 0; + +rollback: FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { - DEBUG("Calling rte_eth_dev_filter_ctrl on sub_device %d", i); - ret = rte_eth_dev_filter_ctrl(PORT_ID(sdev), type, op, arg); + int rc = rte_eth_dev_set_mc_addr_list(PORT_ID(sdev), + PRIV(dev)->mcast_addrs, PRIV(dev)->nb_mcast_addr); + if (rc != 0) { + ERROR("Multicast MAC address list rollback for sub_device %d failed with error %d", + i, rc); + } + } + + fs_unlock(dev, 0); + return ret; +} + +static int +fs_rss_hash_update(struct rte_eth_dev *dev, + struct rte_eth_rss_conf *rss_conf) +{ + struct sub_device *sdev; + uint8_t i; + int ret; + + fs_lock(dev, 0); + FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) { + ret = rte_eth_dev_rss_hash_update(PORT_ID(sdev), rss_conf); + ret = fs_err(sdev, ret); if (ret) { - ERROR("Operation rte_eth_dev_filter_ctrl failed for sub_device %d" - " with error %d", i, ret); + ERROR("Operation rte_eth_dev_rss_hash_update" + " failed for sub_device %d with error %d", + i, ret); + fs_unlock(dev, 0); return ret; } } + fs_unlock(dev, 0); + + return 0; +} + +static int +fs_flow_ops_get(struct rte_eth_dev *dev __rte_unused, + const struct rte_flow_ops **ops) +{ + *ops = &fs_flow_ops; return 0; } @@ -850,7 +1523,7 @@ const struct eth_dev_ops failsafe_ops = { .dev_stop = fs_dev_stop, .dev_set_link_down = fs_dev_set_link_down, .dev_set_link_up = fs_dev_set_link_up, - .dev_close = fs_dev_close, + .dev_close = failsafe_eth_dev_close, .promiscuous_enable = fs_promiscuous_enable, .promiscuous_disable = fs_promiscuous_disable, .allmulticast_enable = fs_allmulticast_enable, @@ -858,18 +1531,29 @@ const struct eth_dev_ops failsafe_ops = { .link_update = fs_link_update, .stats_get = fs_stats_get, .stats_reset = fs_stats_reset, + .xstats_get = fs_xstats_get, + .xstats_get_names = fs_xstats_get_names, + .xstats_reset = fs_xstats_reset, .dev_infos_get = fs_dev_infos_get, .dev_supported_ptypes_get = fs_dev_supported_ptypes_get, .mtu_set = fs_mtu_set, .vlan_filter_set = fs_vlan_filter_set, + .rx_queue_start = fs_rx_queue_start, + .rx_queue_stop = fs_rx_queue_stop, + .tx_queue_start = fs_tx_queue_start, + .tx_queue_stop = fs_tx_queue_stop, .rx_queue_setup = fs_rx_queue_setup, .tx_queue_setup = fs_tx_queue_setup, .rx_queue_release = fs_rx_queue_release, .tx_queue_release = fs_tx_queue_release, + .rx_queue_intr_enable = fs_rx_intr_enable, + .rx_queue_intr_disable = fs_rx_intr_disable, .flow_ctrl_get = fs_flow_ctrl_get, .flow_ctrl_set = fs_flow_ctrl_set, .mac_addr_remove = fs_mac_addr_remove, .mac_addr_add = fs_mac_addr_add, .mac_addr_set = fs_mac_addr_set, - .filter_ctrl = fs_filter_ctrl, + .set_mc_addr_list = fs_set_mc_addr_list, + .rss_hash_update = fs_rss_hash_update, + .flow_ops_get = fs_flow_ops_get, };