From cea37e513329fde070abb8b1f655836f550e54d3 Mon Sep 17 00:00:00 2001 From: Chunsong Feng Date: Fri, 25 Oct 2019 20:37:05 +0800 Subject: [PATCH] net/hns3: fix FLR reset PF FLR resets the PCIe ECAM space of all VFs under the PF and does not automatically recover. Therefore, the VF driver needs to restore the ECAM configuration, including bus_master_en, msix_enable to avoid FLR reset failure. Fixes: 2790c6464725 ("net/hns3: support device reset") Signed-off-by: Chunsong Feng Signed-off-by: Wei Hu (Xavier) --- drivers/net/hns3/hns3_cmd.c | 15 ---- drivers/net/hns3/hns3_ethdev.c | 5 ++ drivers/net/hns3/hns3_ethdev_vf.c | 135 +++++++++++++++++++++++++++++- drivers/net/hns3/hns3_intr.c | 5 +- 4 files changed, 139 insertions(+), 21 deletions(-) diff --git a/drivers/net/hns3/hns3_cmd.c b/drivers/net/hns3/hns3_cmd.c index 58776c2ec6..65a5af8e4f 100644 --- a/drivers/net/hns3/hns3_cmd.c +++ b/drivers/net/hns3/hns3_cmd.c @@ -216,24 +216,9 @@ hns3_cmd_csq_clean(struct hns3_hw *hw) if (!is_valid_csq_clean_head(csq, head)) { struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw); - uint32_t global; - uint32_t fun_rst; hns3_err(hw, "wrong cmd head (%u, %u-%u)", head, csq->next_to_use, csq->next_to_clean); rte_atomic16_set(&hw->reset.disable_cmd, 1); - if (hns->is_vf) { - global = hns3_read_dev(hw, HNS3_VF_RST_ING); - fun_rst = hns3_read_dev(hw, HNS3_FUN_RST_ING); - hns3_err(hw, "Delayed VF reset global: %x fun_rst: %x", - global, fun_rst); - hns3_atomic_set_bit(HNS3_VF_RESET, &hw->reset.pending); - } else { - global = hns3_read_dev(hw, HNS3_GLOBAL_RESET_REG); - fun_rst = hns3_read_dev(hw, HNS3_FUN_RST_ING); - hns3_err(hw, "Delayed IMP reset global: %x fun_rst: %x", - global, fun_rst); - hns3_atomic_set_bit(HNS3_IMP_RESET, &hw->reset.pending); - } hns3_schedule_delayed_reset(hns); diff --git a/drivers/net/hns3/hns3_ethdev.c b/drivers/net/hns3/hns3_ethdev.c index 862a717fdf..3435bce26e 100644 --- a/drivers/net/hns3/hns3_ethdev.c +++ b/drivers/net/hns3/hns3_ethdev.c @@ -4680,6 +4680,11 @@ hns3_reset_service(void *param) rte_atomic16_set(&hns->hw.reset.schedule, SCHEDULE_REQUESTED); hns3_err(hw, "Handling interrupts in delayed tasks"); hns3_interrupt_handler(&rte_eth_devices[hw->data->port_id]); + reset_level = hns3_get_reset_level(hns, &hw->reset.pending); + if (reset_level == HNS3_NONE_RESET) { + hns3_err(hw, "No reset level is set, try IMP reset"); + hns3_atomic_set_bit(HNS3_IMP_RESET, &hw->reset.pending); + } } rte_atomic16_set(&hns->hw.reset.schedule, SCHEDULE_NONE); diff --git a/drivers/net/hns3/hns3_ethdev_vf.c b/drivers/net/hns3/hns3_ethdev_vf.c index 121beb58dc..403674969c 100644 --- a/drivers/net/hns3/hns3_ethdev_vf.c +++ b/drivers/net/hns3/hns3_ethdev_vf.c @@ -9,6 +9,8 @@ #include #include #include +#include + #include #include #include @@ -24,6 +26,7 @@ #include #include #include +#include #include "hns3_ethdev.h" #include "hns3_logs.h" @@ -56,6 +59,81 @@ static enum hns3_reset_level hns3vf_get_reset_level(struct hns3_hw *hw, static int hns3vf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); static int hns3vf_dev_configure_vlan(struct rte_eth_dev *dev); +/* set PCI bus mastering */ +static void +hns3vf_set_bus_master(const struct rte_pci_device *device, bool op) +{ + uint16_t reg; + + rte_pci_read_config(device, ®, sizeof(reg), PCI_COMMAND); + + if (op) + /* set the master bit */ + reg |= PCI_COMMAND_MASTER; + else + reg &= ~(PCI_COMMAND_MASTER); + + rte_pci_write_config(device, ®, sizeof(reg), PCI_COMMAND); +} + +/** + * hns3vf_find_pci_capability - lookup a capability in the PCI capability list + * @cap: the capability + * + * Return the address of the given capability within the PCI capability list. + */ +static int +hns3vf_find_pci_capability(const struct rte_pci_device *device, int cap) +{ +#define MAX_PCIE_CAPABILITY 48 + uint16_t status; + uint8_t pos; + uint8_t id; + int ttl; + + rte_pci_read_config(device, &status, sizeof(status), PCI_STATUS); + if (!(status & PCI_STATUS_CAP_LIST)) + return 0; + + ttl = MAX_PCIE_CAPABILITY; + rte_pci_read_config(device, &pos, sizeof(pos), PCI_CAPABILITY_LIST); + while (ttl-- && pos >= PCI_STD_HEADER_SIZEOF) { + rte_pci_read_config(device, &id, sizeof(id), + (pos + PCI_CAP_LIST_ID)); + + if (id == 0xFF) + break; + + if (id == cap) + return (int)pos; + + rte_pci_read_config(device, &pos, sizeof(pos), + (pos + PCI_CAP_LIST_NEXT)); + } + return 0; +} + +static int +hns3vf_enable_msix(const struct rte_pci_device *device, bool op) +{ + uint16_t control; + int pos; + + pos = hns3vf_find_pci_capability(device, PCI_CAP_ID_MSIX); + if (pos) { + rte_pci_read_config(device, &control, sizeof(control), + (pos + PCI_MSIX_FLAGS)); + if (op) + control |= PCI_MSIX_FLAGS_ENABLE; + else + control &= ~PCI_MSIX_FLAGS_ENABLE; + rte_pci_write_config(device, &control, sizeof(control), + (pos + PCI_MSIX_FLAGS)); + return 0; + } + return -1; +} + static int hns3vf_add_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr, __attribute__ ((unused)) uint32_t idx, @@ -1308,9 +1386,30 @@ hns3vf_wait_hardware_ready(struct hns3_adapter *hns) struct hns3_wait_data *wait_data = hw->reset.wait_data; struct timeval tv; - if (wait_data->result == HNS3_WAIT_SUCCESS) - return 0; - else if (wait_data->result == HNS3_WAIT_TIMEOUT) { + if (wait_data->result == HNS3_WAIT_SUCCESS) { + /* + * After vf reset is ready, the PF may not have completed + * the reset processing. The vf sending mbox to PF may fail + * during the pf reset, so it is better to add extra delay. + */ + if (hw->reset.level == HNS3_VF_FUNC_RESET || + hw->reset.level == HNS3_FLR_RESET) + return 0; + /* Reset retry process, no need to add extra delay. */ + if (hw->reset.attempts) + return 0; + if (wait_data->check_completion == NULL) + return 0; + + wait_data->check_completion = NULL; + wait_data->interval = 1 * MSEC_PER_SEC * USEC_PER_MSEC; + wait_data->count = 1; + wait_data->result = HNS3_WAIT_REQUEST; + rte_eal_alarm_set(wait_data->interval, hns3_wait_callback, + wait_data); + hns3_warn(hw, "hardware is ready, delay 1 sec for PF reset complete"); + return -EAGAIN; + } else if (wait_data->result == HNS3_WAIT_TIMEOUT) { gettimeofday(&tv, NULL); hns3_warn(hw, "Reset step4 hardware not ready after reset time=%ld.%.6ld", tv.tv_sec, tv.tv_usec); @@ -1473,6 +1572,11 @@ hns3vf_reset_service(void *param) rte_atomic16_set(&hns->hw.reset.schedule, SCHEDULE_REQUESTED); hns3_err(hw, "Handling interrupts in delayed tasks"); hns3vf_interrupt_handler(&rte_eth_devices[hw->data->port_id]); + reset_level = hns3vf_get_reset_level(hw, &hw->reset.pending); + if (reset_level == HNS3_NONE_RESET) { + hns3_err(hw, "No reset level is set, try global reset"); + hns3_atomic_set_bit(HNS3_VF_RESET, &hw->reset.pending); + } } rte_atomic16_set(&hns->hw.reset.schedule, SCHEDULE_NONE); @@ -1498,14 +1602,35 @@ hns3vf_reset_service(void *param) static int hns3vf_reinit_dev(struct hns3_adapter *hns) { + struct rte_eth_dev *eth_dev = &rte_eth_devices[hns->hw.data->port_id]; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); struct hns3_hw *hw = &hns->hw; int ret; + if (hw->reset.level == HNS3_VF_FULL_RESET) { + rte_intr_disable(&pci_dev->intr_handle); + hns3vf_set_bus_master(pci_dev, true); + } + /* Firmware command initialize */ ret = hns3_cmd_init(hw); if (ret) { hns3_err(hw, "Failed to init cmd: %d", ret); - return ret; + goto err_cmd_init; + } + + if (hw->reset.level == HNS3_VF_FULL_RESET) { + /* + * UIO enables msix by writing the pcie configuration space + * vfio_pci enables msix in rte_intr_enable. + */ + if (pci_dev->kdrv == RTE_KDRV_IGB_UIO || + pci_dev->kdrv == RTE_KDRV_UIO_GENERIC) { + if (hns3vf_enable_msix(pci_dev, true)) + hns3_err(hw, "Failed to enable msix"); + } + + rte_intr_enable(&pci_dev->intr_handle); } ret = hns3_reset_all_queues(hns); @@ -1522,6 +1647,8 @@ hns3vf_reinit_dev(struct hns3_adapter *hns) return 0; +err_cmd_init: + hns3vf_set_bus_master(pci_dev, false); err_init: hns3_cmd_uninit(hw); return ret; diff --git a/drivers/net/hns3/hns3_intr.c b/drivers/net/hns3/hns3_intr.c index 9e2d81156f..6c3ebd3ee1 100644 --- a/drivers/net/hns3/hns3_intr.c +++ b/drivers/net/hns3/hns3_intr.c @@ -890,11 +890,12 @@ reset_fail: hns3_warn(hw, "%s reset fail fail_cnt:%" PRIx64 " success_cnt:%" PRIx64 " global_cnt:%" PRIx64 " imp_cnt:%" PRIx64 " request_cnt:%" PRIx64 " exec_cnt:%" PRIx64 - " merge_cnt:%" PRIx64, + " merge_cnt:%" PRIx64 "adapter_state:%d", reset_string[hw->reset.level], hw->reset.stats.fail_cnt, hw->reset.stats.success_cnt, hw->reset.stats.global_cnt, hw->reset.stats.imp_cnt, hw->reset.stats.request_cnt, - hw->reset.stats.exec_cnt, hw->reset.stats.merge_cnt); + hw->reset.stats.exec_cnt, hw->reset.stats.merge_cnt, + hw->adapter_state); /* IMP no longer waiting the ready flag */ hns3_notify_reset_ready(hw, true); -- 2.20.1