From: Kalesh AP Date: Wed, 2 Oct 2019 01:23:23 +0000 (-0700) Subject: net/bnxt: handle reset notify async event from FW X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=df6cd7c1f73a;p=dpdk.git net/bnxt: handle reset notify async event from FW When the FW upgrade is initiated the current instance of FW issues a HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY async notification to the driver. On receiving this notification, the PMD shall quiesce itself and poll on the HWRM_VER_GET FW command at regular intervals. Once the VER_GET command succeeds, the driver should go through the rediscovery process and re-initialize the device. Also register with FW for the reset notify async event. Signed-off-by: Kalesh AP Reviewed-by: Ajit Khaparde Reviewed-by: Somnath Kotur --- diff --git a/drivers/net/bnxt/bnxt.h b/drivers/net/bnxt/bnxt.h index 37b4c717d6..8797b032ee 100644 --- a/drivers/net/bnxt/bnxt.h +++ b/drivers/net/bnxt/bnxt.h @@ -333,6 +333,16 @@ struct bnxt_ctx_mem_info { struct bnxt_ctx_pg_info *tqm_mem[BNXT_MAX_TC_Q]; }; +/* Maximum Firmware Reset bail out value in milliseconds */ +#define BNXT_MAX_FW_RESET_TIMEOUT 6000 +/* Minimum time required for the firmware readiness in milliseconds */ +#define BNXT_MIN_FW_READY_TIMEOUT 2000 +/* Frequency for the firmware readiness check in milliseconds */ +#define BNXT_FW_READY_WAIT_INTERVAL 100 + +#define US_PER_MS 1000 +#define NS_PER_US 1000 + #define BNXT_HWRM_SHORT_REQ_LEN sizeof(struct hwrm_short_input) struct bnxt { void *bar0; @@ -463,6 +473,9 @@ struct bnxt { struct bnxt_ptp_cfg *ptp_cfg; uint16_t vf_resv_strategy; struct bnxt_ctx_mem_info *ctx; + + uint16_t fw_reset_min_msecs; + uint16_t fw_reset_max_msecs; }; int bnxt_link_update_op(struct rte_eth_dev *eth_dev, int wait_to_complete); diff --git a/drivers/net/bnxt/bnxt_cpr.c b/drivers/net/bnxt/bnxt_cpr.c index bbcdb42f10..1a23649b05 100644 --- a/drivers/net/bnxt/bnxt_cpr.c +++ b/drivers/net/bnxt/bnxt_cpr.c @@ -4,6 +4,7 @@ */ #include +#include #include "bnxt.h" #include "bnxt_cpr.h" @@ -40,6 +41,21 @@ void bnxt_handle_async_event(struct bnxt *bp, case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED: PMD_DRV_LOG(INFO, "Port conn async event\n"); break; + case HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY: + /* timestamp_lo/hi values are in units of 100ms */ + bp->fw_reset_max_msecs = async_cmp->timestamp_hi ? + rte_le_to_cpu_16(async_cmp->timestamp_hi) * 100 : + BNXT_MAX_FW_RESET_TIMEOUT; + bp->fw_reset_min_msecs = async_cmp->timestamp_lo ? + async_cmp->timestamp_lo * 100 : + BNXT_MIN_FW_READY_TIMEOUT; + PMD_DRV_LOG(INFO, + "Firmware non-fatal reset event received\n"); + + bp->flags |= BNXT_FLAG_FW_RESET; + rte_eal_alarm_set(US_PER_MS, bnxt_dev_reset_and_resume, + (void *)bp); + break; default: PMD_DRV_LOG(INFO, "handle_async_event id = 0x%x\n", event_id); break; diff --git a/drivers/net/bnxt/bnxt_cpr.h b/drivers/net/bnxt/bnxt_cpr.h index 8c6a34b611..f48293b963 100644 --- a/drivers/net/bnxt/bnxt_cpr.h +++ b/drivers/net/bnxt/bnxt_cpr.h @@ -106,5 +106,6 @@ struct bnxt; void bnxt_handle_async_event(struct bnxt *bp, struct cmpl_base *cmp); void bnxt_handle_fwd_req(struct bnxt *bp, struct cmpl_base *cmp); int bnxt_event_hwrm_resp_handler(struct bnxt *bp, struct cmpl_base *cmp); +void bnxt_dev_reset_and_resume(void *arg); #endif diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c index d90a6e4202..6bc006a719 100644 --- a/drivers/net/bnxt/bnxt_ethdev.c +++ b/drivers/net/bnxt/bnxt_ethdev.c @@ -11,6 +11,7 @@ #include #include #include +#include #include "bnxt.h" #include "bnxt_cpr.h" @@ -166,6 +167,8 @@ static int bnxt_vlan_offload_set_op(struct rte_eth_dev *dev, int mask); static void bnxt_print_link_info(struct rte_eth_dev *eth_dev); static int bnxt_mtu_set_op(struct rte_eth_dev *eth_dev, uint16_t new_mtu); static int bnxt_dev_uninit(struct rte_eth_dev *eth_dev); +static int bnxt_init_resources(struct bnxt *bp, bool reconfig_dev); +static int bnxt_uninit_resources(struct bnxt *bp, bool reconfig_dev); int is_bnxt_in_error(struct bnxt *bp) { @@ -201,19 +204,25 @@ static uint16_t bnxt_rss_hash_tbl_size(const struct bnxt *bp) return bnxt_rss_ctxts(bp) * BNXT_RSS_ENTRIES_PER_CTX_THOR; } -static void bnxt_free_mem(struct bnxt *bp) +static void bnxt_free_mem(struct bnxt *bp, bool reconfig) { bnxt_free_filter_mem(bp); bnxt_free_vnic_attributes(bp); bnxt_free_vnic_mem(bp); - bnxt_free_stats(bp); - bnxt_free_tx_rings(bp); - bnxt_free_rx_rings(bp); + /* tx/rx rings are configured as part of *_queue_setup callbacks. + * If the number of rings change across fw update, + * we don't have much choice except to warn the user. + */ + if (!reconfig) { + bnxt_free_stats(bp); + bnxt_free_tx_rings(bp); + bnxt_free_rx_rings(bp); + } bnxt_free_async_cp_ring(bp); } -static int bnxt_alloc_mem(struct bnxt *bp) +static int bnxt_alloc_mem(struct bnxt *bp, bool reconfig) { int rc; @@ -244,7 +253,7 @@ static int bnxt_alloc_mem(struct bnxt *bp) return 0; alloc_mem_err: - bnxt_free_mem(bp); + bnxt_free_mem(bp, reconfig); return rc; } @@ -3523,6 +3532,89 @@ static const struct eth_dev_ops bnxt_dev_ops = { .timesync_read_tx_timestamp = bnxt_timesync_read_tx_timestamp, }; +static void bnxt_dev_cleanup(struct bnxt *bp) +{ + bnxt_set_hwrm_link_config(bp, false); + bp->link_info.link_up = 0; + if (bp->dev_stopped == 0) + bnxt_dev_stop_op(bp->eth_dev); + + bnxt_uninit_resources(bp, true); +} + +static int bnxt_restore_filters(struct bnxt *bp) +{ + struct rte_eth_dev *dev = bp->eth_dev; + int ret = 0; + + if (dev->data->all_multicast) + ret = bnxt_allmulticast_enable_op(dev); + if (dev->data->promiscuous) + ret = bnxt_promiscuous_enable_op(dev); + + /* TODO restore other filters as well */ + return ret; +} + +static void bnxt_dev_recover(void *arg) +{ + struct bnxt *bp = arg; + int timeout = bp->fw_reset_max_msecs; + int rc = 0; + + do { + rc = bnxt_hwrm_ver_get(bp); + if (rc == 0) + break; + rte_delay_ms(BNXT_FW_READY_WAIT_INTERVAL); + timeout -= BNXT_FW_READY_WAIT_INTERVAL; + } while (rc && timeout); + + if (rc) { + PMD_DRV_LOG(ERR, "FW is not Ready after reset\n"); + goto err; + } + + rc = bnxt_init_resources(bp, true); + if (rc) { + PMD_DRV_LOG(ERR, + "Failed to initialize resources after reset\n"); + goto err; + } + /* clear reset flag as the device is initialized now */ + bp->flags &= ~BNXT_FLAG_FW_RESET; + + rc = bnxt_dev_start_op(bp->eth_dev); + if (rc) { + PMD_DRV_LOG(ERR, "Failed to start port after reset\n"); + goto err; + } + + rc = bnxt_restore_filters(bp); + if (rc) + goto err; + + PMD_DRV_LOG(INFO, "Recovered from FW reset\n"); + return; +err: + bp->flags |= BNXT_FLAG_FATAL_ERROR; + bnxt_uninit_resources(bp, false); + PMD_DRV_LOG(ERR, "Failed to recover from FW reset\n"); +} + +void bnxt_dev_reset_and_resume(void *arg) +{ + struct bnxt *bp = arg; + int rc; + + bnxt_dev_cleanup(bp); + + rc = rte_eal_alarm_set(US_PER_MS * bp->fw_reset_min_msecs, + bnxt_dev_recover, (void *)bp); + if (rc) + PMD_DRV_LOG(ERR, "Error setting recovery alarm"); +} + static bool bnxt_vf_pciid(uint16_t id) { if (id == BROADCOM_DEV_ID_57304_VF || @@ -3965,6 +4057,22 @@ static int bnxt_setup_mac_addr(struct rte_eth_dev *eth_dev) return rc; } +static int bnxt_restore_dflt_mac(struct bnxt *bp) +{ + int rc = 0; + + /* MAC is already configured in FW */ + if (!bnxt_check_zero_bytes(bp->dflt_mac_addr, RTE_ETHER_ADDR_LEN)) + return 0; + + /* Restore the old MAC configured */ + rc = bnxt_hwrm_set_mac(bp); + if (rc) + PMD_DRV_LOG(ERR, "Failed to restore MAC address\n"); + + return rc; +} + static void bnxt_config_vf_req_fwd(struct bnxt *bp) { if (!BNXT_PF(bp)) @@ -4038,7 +4146,7 @@ static int bnxt_init_fw(struct bnxt *bp) return 0; } -static int bnxt_init_resources(struct bnxt *bp) +static int bnxt_init_resources(struct bnxt *bp, bool reconfig_dev) { int rc; @@ -4046,9 +4154,15 @@ static int bnxt_init_resources(struct bnxt *bp) if (rc) return rc; - rc = bnxt_setup_mac_addr(bp->eth_dev); - if (rc) - return rc; + if (!reconfig_dev) { + rc = bnxt_setup_mac_addr(bp->eth_dev); + if (rc) + return rc; + } else { + rc = bnxt_restore_dflt_mac(bp); + if (rc) + return rc; + } bnxt_config_vf_req_fwd(bp); @@ -4075,7 +4189,7 @@ static int bnxt_init_resources(struct bnxt *bp) } } - rc = bnxt_alloc_mem(bp); + rc = bnxt_alloc_mem(bp, reconfig_dev); if (rc) return rc; @@ -4149,7 +4263,7 @@ bnxt_dev_init(struct rte_eth_dev *eth_dev) "Failed to allocate hwrm resource rc: %x\n", rc); goto error_free; } - rc = bnxt_init_resources(bp); + rc = bnxt_init_resources(bp, false); if (rc) goto error_free; @@ -4170,18 +4284,19 @@ error_free: } static int -bnxt_uninit_resources(struct bnxt *bp) +bnxt_uninit_resources(struct bnxt *bp, bool reconfig_dev) { int rc; bnxt_disable_int(bp); bnxt_free_int(bp); - bnxt_free_mem(bp); + bnxt_free_mem(bp, reconfig_dev); bnxt_hwrm_func_buf_unrgtr(bp); rc = bnxt_hwrm_func_driver_unregister(bp, 0); bp->flags &= ~BNXT_FLAG_REGISTERED; bnxt_free_ctx_mem(bp); - bnxt_free_hwrm_resources(bp); + if (!reconfig_dev) + bnxt_free_hwrm_resources(bp); return rc; } @@ -4197,7 +4312,7 @@ bnxt_dev_uninit(struct rte_eth_dev *eth_dev) PMD_DRV_LOG(DEBUG, "Calling Device uninit\n"); - rc = bnxt_uninit_resources(bp); + rc = bnxt_uninit_resources(bp, false); if (bp->grp_info != NULL) { rte_free(bp->grp_info); diff --git a/drivers/net/bnxt/bnxt_hwrm.c b/drivers/net/bnxt/bnxt_hwrm.c index 24a5a09147..abbad0152c 100644 --- a/drivers/net/bnxt/bnxt_hwrm.c +++ b/drivers/net/bnxt/bnxt_hwrm.c @@ -27,6 +27,7 @@ #include #define HWRM_CMD_TIMEOUT 6000000 +#define HWRM_SHORT_CMD_TIMEOUT 50000 #define HWRM_SPEC_CODE_1_8_3 0x10803 #define HWRM_VERSION_1_9_1 0x10901 #define HWRM_VERSION_1_9_2 0x10903 @@ -97,6 +98,17 @@ static int bnxt_hwrm_send_message(struct bnxt *bp, void *msg, GRCPF_REG_KONG_CHANNEL_OFFSET : GRCPF_REG_CHIMP_CHANNEL_OFFSET; uint16_t mb_trigger_offset = use_kong_mb ? GRCPF_REG_KONG_COMM_TRIGGER : GRCPF_REG_CHIMP_COMM_TRIGGER; + uint32_t timeout; + + /* Do not send HWRM commands to firmware in error state */ + if (bp->flags & BNXT_FLAG_FATAL_ERROR) + return 0; + + /* For VER_GET command, set timeout as 50ms */ + if (rte_cpu_to_le_16(req->req_type) == HWRM_VER_GET) + timeout = HWRM_SHORT_CMD_TIMEOUT; + else + timeout = HWRM_CMD_TIMEOUT; if (bp->flags & BNXT_FLAG_SHORT_CMD || msg_len > bp->max_req_len) { @@ -139,7 +151,7 @@ static int bnxt_hwrm_send_message(struct bnxt *bp, void *msg, rte_write32(1, bar); /* Poll for the valid bit */ - for (i = 0; i < HWRM_CMD_TIMEOUT; i++) { + for (i = 0; i < timeout; i++) { /* Sanity check on the resp->resp_len */ rte_rmb(); if (resp->resp_len && resp->resp_len <= bp->max_resp_len) { @@ -151,7 +163,12 @@ static int bnxt_hwrm_send_message(struct bnxt *bp, void *msg, rte_delay_us(1); } - if (i >= HWRM_CMD_TIMEOUT) { + if (i >= timeout) { + /* Suppress VER_GET timeout messages during reset recovery */ + if (bp->flags & BNXT_FLAG_FW_RESET && + rte_cpu_to_le_16(req->req_type) == HWRM_VER_GET) + return -ETIMEDOUT; + PMD_DRV_LOG(ERR, "Error(timeout) sending msg 0x%04x\n", req->req_type); return -ETIMEDOUT; @@ -657,12 +674,21 @@ int bnxt_hwrm_func_reset(struct bnxt *bp) int bnxt_hwrm_func_driver_register(struct bnxt *bp) { int rc; + uint32_t flags = 0; struct hwrm_func_drv_rgtr_input req = {.req_type = 0 }; struct hwrm_func_drv_rgtr_output *resp = bp->hwrm_cmd_resp_addr; if (bp->flags & BNXT_FLAG_REGISTERED) return 0; + flags = HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_HOT_RESET_SUPPORT; + + /* PFs and trusted VFs should indicate the support of the + * Master capability on non Stingray platform + */ + if ((BNXT_PF(bp) || BNXT_VF_IS_TRUSTED(bp)) && !BNXT_STINGRAY(bp)) + flags |= HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_MASTER_SUPPORT; + HWRM_PREP(req, FUNC_DRV_RGTR, BNXT_USE_CHIMP_MB); req.enables = rte_cpu_to_le_32(HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_VER | HWRM_FUNC_DRV_RGTR_INPUT_ENABLES_ASYNC_EVENT_FWD); @@ -683,14 +709,16 @@ int bnxt_hwrm_func_driver_register(struct bnxt *bp) * this HWRM sniffer list in FW because DPDK PF driver does * not support this. */ - req.flags = - rte_cpu_to_le_32(HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_FWD_NONE_MODE); + flags |= HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_FWD_NONE_MODE; } + req.flags = rte_cpu_to_le_32(flags); + req.async_event_fwd[0] |= rte_cpu_to_le_32(ASYNC_CMPL_EVENT_ID_LINK_STATUS_CHANGE | ASYNC_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED | - ASYNC_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE); + ASYNC_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE | + ASYNC_CMPL_EVENT_ID_RESET_NOTIFY); req.async_event_fwd[1] |= rte_cpu_to_le_32(ASYNC_CMPL_EVENT_ID_PF_DRVR_UNLOAD | ASYNC_CMPL_EVENT_ID_VF_CFG_CHANGE); @@ -837,7 +865,10 @@ int bnxt_hwrm_ver_get(struct bnxt *bp) rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB); - HWRM_CHECK_RESULT(); + if (bp->flags & BNXT_FLAG_FW_RESET) + HWRM_CHECK_RESULT_SILENT(); + else + HWRM_CHECK_RESULT(); PMD_DRV_LOG(INFO, "%d.%d.%d:%d.%d.%d\n", resp->hwrm_intf_maj_8b, resp->hwrm_intf_min_8b, @@ -2685,6 +2716,10 @@ int bnxt_hwrm_func_qcfg(struct bnxt *bp, uint16_t *mtu) if (BNXT_VF(bp) && (flags & HWRM_FUNC_QCFG_OUTPUT_FLAGS_TRUSTED_VF)) { bp->flags |= BNXT_FLAG_TRUSTED_VF_EN; PMD_DRV_LOG(INFO, "Trusted VF cap enabled\n"); + } else if (BNXT_VF(bp) && + !(flags & HWRM_FUNC_QCFG_OUTPUT_FLAGS_TRUSTED_VF)) { + bp->flags &= ~BNXT_FLAG_TRUSTED_VF_EN; + PMD_DRV_LOG(INFO, "Trusted VF cap disabled\n"); } if (mtu) diff --git a/drivers/net/bnxt/bnxt_hwrm.h b/drivers/net/bnxt/bnxt_hwrm.h index c882fc2a15..a036205329 100644 --- a/drivers/net/bnxt/bnxt_hwrm.h +++ b/drivers/net/bnxt/bnxt_hwrm.h @@ -21,6 +21,8 @@ struct bnxt_cp_ring_info; (1 << HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PORT_CONN_NOT_ALLOWED) #define ASYNC_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE \ (1 << HWRM_ASYNC_EVENT_CMPL_EVENT_ID_LINK_SPEED_CFG_CHANGE) +#define ASYNC_CMPL_EVENT_ID_RESET_NOTIFY \ + (1 << HWRM_ASYNC_EVENT_CMPL_EVENT_ID_RESET_NOTIFY) #define ASYNC_CMPL_EVENT_ID_PF_DRVR_UNLOAD \ (1 << (HWRM_ASYNC_EVENT_CMPL_EVENT_ID_PF_DRVR_UNLOAD - 32)) #define ASYNC_CMPL_EVENT_ID_VF_CFG_CHANGE \ diff --git a/drivers/net/bnxt/hsi_struct_def_dpdk.h b/drivers/net/bnxt/hsi_struct_def_dpdk.h index 0095717254..809ea48736 100644 --- a/drivers/net/bnxt/hsi_struct_def_dpdk.h +++ b/drivers/net/bnxt/hsi_struct_def_dpdk.h @@ -8918,6 +8918,17 @@ struct hwrm_func_drv_rgtr_input { */ #define HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_ERROR_RECOVERY_SUPPORT \ UINT32_C(0x20) + /* + * When this bit is 1, the function is indicating the support of the + * Master capability. The Firmware will use this capability to select + * the Master function. The master function will be used to initiate + * designated functionality like error recovery etc. If none of the + * registered PFs or trusted VFs indicate this support, then + * firmware will select the 1st registered PF as Master capable + * instance. + */ + #define HWRM_FUNC_DRV_RGTR_INPUT_FLAGS_MASTER_SUPPORT \ + UINT32_C(0x40) uint32_t enables; /* * This bit must be '1' for the os_type field to be