From: Mike Baucom Date: Sun, 30 May 2021 08:59:24 +0000 (+0530) Subject: net/bnxt: add HA support in ULP X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=3184b1ef666aaa8d0e4df7b032d7738fb86ad589;p=dpdk.git net/bnxt: add HA support in ULP Add the ability for cooperative applications to share resources and use the high availability functionality in the HW. Signed-off-by: Mike Baucom Signed-off-by: Venkat Duvvuru Reviewed-by: Shahaji Bhosle Reviewed-by: Randy Schacher Reviewed-by: Ajit Khaparde --- diff --git a/drivers/net/bnxt/tf_ulp/bnxt_ulp.c b/drivers/net/bnxt/tf_ulp/bnxt_ulp.c index 0daa8e4c29..972bf8b992 100644 --- a/drivers/net/bnxt/tf_ulp/bnxt_ulp.c +++ b/drivers/net/bnxt/tf_ulp/bnxt_ulp.c @@ -23,6 +23,7 @@ #include "ulp_mapper.h" #include "ulp_port_db.h" #include "ulp_tun.h" +#include "ulp_ha_mgr.h" /* Linked list of all TF sessions. */ STAILQ_HEAD(, bnxt_ulp_session_state) bnxt_ulp_session_list = @@ -315,6 +316,9 @@ bnxt_ulp_cntxt_app_caps_init(struct bnxt_ulp_context *ulp_ctx, if (info[i].flags & BNXT_ULP_APP_CAP_SHARED_EN) ulp_ctx->cfg_data->ulp_flags |= BNXT_ULP_SHARED_SESSION_ENABLED; + if (info[i].flags & BNXT_ULP_APP_CAP_HOT_UPGRADE_EN) + ulp_ctx->cfg_data->ulp_flags |= + BNXT_ULP_HIGH_AVAIL_ENABLED; } if (!found) { BNXT_TF_DBG(ERR, "APP ID %d, Device ID: 0x%x not supported.\n", @@ -1137,9 +1141,18 @@ static void bnxt_ulp_deinit(struct bnxt *bp, struct bnxt_ulp_session_state *session) { + bool ha_enabled; + if (!bp->ulp_ctx || !bp->ulp_ctx->cfg_data) return; + ha_enabled = bnxt_ulp_cntxt_ha_enabled(bp->ulp_ctx); + if (ha_enabled && session->session_opened) { + int32_t rc = ulp_ha_mgr_close(bp->ulp_ctx); + if (rc) + BNXT_TF_DBG(ERR, "Failed to close HA (%d)\n", rc); + } + /* clean up default flows */ bnxt_ulp_destroy_df_rules(bp, true); @@ -1179,6 +1192,9 @@ bnxt_ulp_deinit(struct bnxt *bp, /* free the flow db lock */ pthread_mutex_destroy(&bp->ulp_ctx->cfg_data->flow_db_lock); + if (ha_enabled) + ulp_ha_mgr_deinit(bp->ulp_ctx); + /* Delete the ulp context and tf session and free the ulp context */ ulp_ctx_deinit(bp, session); BNXT_TF_DBG(DEBUG, "ulp ctx has been deinitialized\n"); @@ -1275,6 +1291,19 @@ bnxt_ulp_init(struct bnxt *bp, BNXT_TF_DBG(ERR, "Failed to set tx global configuration\n"); goto jump_to_error; } + + if (bnxt_ulp_cntxt_ha_enabled(bp->ulp_ctx)) { + rc = ulp_ha_mgr_init(bp->ulp_ctx); + if (rc) { + BNXT_TF_DBG(ERR, "Failed to initialize HA %d\n", rc); + goto jump_to_error; + } + rc = ulp_ha_mgr_open(bp->ulp_ctx); + if (rc) { + BNXT_TF_DBG(ERR, "Failed to Process HA Open %d\n", rc); + goto jump_to_error; + } + } BNXT_TF_DBG(DEBUG, "ulp ctx has been initialized\n"); return rc; @@ -1828,3 +1857,33 @@ bnxt_ulp_cntxt_release_fdb_lock(struct bnxt_ulp_context *ulp_ctx) pthread_mutex_unlock(&ulp_ctx->cfg_data->flow_db_lock); } + +/* Function to set the ha info into the context */ +int32_t +bnxt_ulp_cntxt_ptr2_ha_info_set(struct bnxt_ulp_context *ulp_ctx, + struct bnxt_ulp_ha_mgr_info *ulp_ha_info) +{ + if (ulp_ctx == NULL || ulp_ctx->cfg_data == NULL) { + BNXT_TF_DBG(ERR, "Invalid ulp context data\n"); + return -EINVAL; + } + ulp_ctx->cfg_data->ha_info = ulp_ha_info; + return 0; +} + +/* Function to retrieve the ha info from the context. */ +struct bnxt_ulp_ha_mgr_info * +bnxt_ulp_cntxt_ptr2_ha_info_get(struct bnxt_ulp_context *ulp_ctx) +{ + if (ulp_ctx == NULL || ulp_ctx->cfg_data == NULL) + return NULL; + return ulp_ctx->cfg_data->ha_info; +} + +bool +bnxt_ulp_cntxt_ha_enabled(struct bnxt_ulp_context *ulp_ctx) +{ + if (ulp_ctx == NULL || ulp_ctx->cfg_data == NULL) + return false; + return !!ULP_HIGH_AVAIL_IS_ENABLED(ulp_ctx->cfg_data->ulp_flags); +} diff --git a/drivers/net/bnxt/tf_ulp/bnxt_ulp.h b/drivers/net/bnxt/tf_ulp/bnxt_ulp.h index 1ba67ed9f6..b1f090a5cb 100644 --- a/drivers/net/bnxt/tf_ulp/bnxt_ulp.h +++ b/drivers/net/bnxt/tf_ulp/bnxt_ulp.h @@ -67,6 +67,7 @@ struct bnxt_ulp_data { void *mapper_data; struct bnxt_ulp_port_db *port_db; struct bnxt_ulp_fc_info *fc_info; + struct bnxt_ulp_ha_mgr_info *ha_info; uint32_t ulp_flags; struct bnxt_ulp_df_rule_info df_rule_info[RTE_MAX_ETHPORTS]; struct bnxt_ulp_vfr_rule_info vfr_rule_info[RTE_MAX_ETHPORTS]; @@ -275,4 +276,13 @@ bnxt_ulp_cntxt_app_caps_init(struct bnxt_ulp_context *ulp_ctx, struct bnxt_ulp_resource_resv_info * bnxt_ulp_resource_resv_list_get(uint32_t *num_entries); +int32_t +bnxt_ulp_cntxt_ptr2_ha_info_set(struct bnxt_ulp_context *ulp_ctx, + struct bnxt_ulp_ha_mgr_info *ulp_ha_info); + +struct bnxt_ulp_ha_mgr_info * +bnxt_ulp_cntxt_ptr2_ha_info_get(struct bnxt_ulp_context *ulp_ctx); + +bool +bnxt_ulp_cntxt_ha_enabled(struct bnxt_ulp_context *ulp_ctx); #endif /* _BNXT_ULP_H_ */ diff --git a/drivers/net/bnxt/tf_ulp/bnxt_ulp_flow.c b/drivers/net/bnxt/tf_ulp/bnxt_ulp_flow.c index 9c27217573..96e6a76270 100644 --- a/drivers/net/bnxt/tf_ulp/bnxt_ulp_flow.c +++ b/drivers/net/bnxt/tf_ulp/bnxt_ulp_flow.c @@ -11,6 +11,7 @@ #include "ulp_mapper.h" #include "ulp_fc_mgr.h" #include "ulp_port_db.h" +#include "ulp_ha_mgr.h" #include static int32_t @@ -112,11 +113,17 @@ bnxt_ulp_init_mapper_params(struct bnxt_ulp_mapper_create_parms *mapper_cparms, /* update the WC Priority flag */ if (!bnxt_ulp_cntxt_ptr2_ulp_flags_get(params->ulp_ctx, &ulp_flags) && ULP_HIGH_AVAIL_IS_ENABLED(ulp_flags)) { - /* TBD: read the state and Set the WC priority */ - ULP_COMP_FLD_IDX_WR(params, - BNXT_ULP_CF_IDX_WC_IS_HA_HIGH_REG, 1); + enum ulp_ha_mgr_region region = ULP_HA_REGION_LOW; + int32_t rc; + + rc = ulp_ha_mgr_region_get(params->ulp_ctx, ®ion); + if (rc) + BNXT_TF_DBG(ERR, "Unable to get WC region\n"); + if (region == ULP_HA_REGION_HI) + ULP_COMP_FLD_IDX_WR(params, + BNXT_ULP_CF_IDX_WC_IS_HA_HIGH_REG, + 1); } - } /* Function to create the rte flow. */ diff --git a/drivers/net/bnxt/tf_ulp/meson.build b/drivers/net/bnxt/tf_ulp/meson.build index 5555350a57..49df42c610 100644 --- a/drivers/net/bnxt/tf_ulp/meson.build +++ b/drivers/net/bnxt/tf_ulp/meson.build @@ -24,6 +24,7 @@ sources += files( 'ulp_tun.c', 'ulp_gen_tbl.c', 'ulp_gen_hash.c', + 'ulp_ha_mgr.c', 'ulp_rte_handler_tbl.c', 'ulp_template_db_wh_plus_act.c', 'ulp_template_db_wh_plus_class.c', diff --git a/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.c b/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.c new file mode 100644 index 0000000000..dc71054f46 --- /dev/null +++ b/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.c @@ -0,0 +1,551 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2019-2021 Broadcom + * All rights reserved. + */ + +#include +#include +#include +#include +#include +#include "bnxt.h" +#include "bnxt_ulp.h" +#include "bnxt_tf_common.h" +#include "ulp_ha_mgr.h" +#include "ulp_flow_db.h" + +/* Local only MACROs and defines that aren't exported */ +#define ULP_HA_TIMER_THREAD (1 << 0) +#define ULP_HA_TIMER_IS_RUNNING(info) (!!((info)->flags & ULP_HA_TIMER_THREAD)) +#define ULP_HA_TIMER_SEC 1 +#define ULP_HA_WAIT_TIME (MS_PER_S / 10) +#define ULP_HA_WAIT_TIMEOUT (MS_PER_S * 2) + +#define ULP_HA_IF_TBL_DIR TF_DIR_RX +#define ULP_HA_IF_TBL_TYPE TF_IF_TBL_TYPE_PROF_PARIF_ERR_ACT_REC_PTR +#define ULP_HA_IF_TBL_IDX 10 + +static void ulp_ha_mgr_timer_cancel(struct bnxt_ulp_context *ulp_ctx); +static int32_t ulp_ha_mgr_timer_start(struct bnxt_ulp_context *ulp_ctx); +static void ulp_ha_mgr_timer_cb(void *arg); +static int32_t ulp_ha_mgr_app_type_set(struct bnxt_ulp_context *ulp_ctx, + enum ulp_ha_mgr_app_type app_type); +static int32_t +ulp_ha_mgr_region_set(struct bnxt_ulp_context *ulp_ctx, + enum ulp_ha_mgr_region region); +static int32_t +ulp_ha_mgr_state_set(struct bnxt_ulp_context *ulp_ctx, + enum ulp_ha_mgr_state state); + +static int32_t +ulp_ha_mgr_state_set(struct bnxt_ulp_context *ulp_ctx, + enum ulp_ha_mgr_state state) +{ + struct tf_set_if_tbl_entry_parms set_parms = { 0 }; + struct tf *tfp; + uint32_t val = 0; + int32_t rc = 0; + + if (ulp_ctx == NULL) { + BNXT_TF_DBG(ERR, "Invalid parms in state get.\n"); + return -EINVAL; + } + tfp = bnxt_ulp_cntxt_tfp_get(ulp_ctx, BNXT_ULP_SHARED_SESSION_NO); + if (tfp == NULL) { + BNXT_TF_DBG(ERR, "Unable to get the TFP.\n"); + return -EINVAL; + } + + val = (uint32_t)state; + + set_parms.dir = ULP_HA_IF_TBL_DIR; + set_parms.type = ULP_HA_IF_TBL_TYPE; + set_parms.data = (uint8_t *)&val; + set_parms.data_sz_in_bytes = sizeof(val); + set_parms.idx = ULP_HA_IF_TBL_IDX; + + rc = tf_set_if_tbl_entry(tfp, &set_parms); + if (rc) + BNXT_TF_DBG(ERR, "Failed to write the HA state\n"); + + return rc; +} + +static int32_t +ulp_ha_mgr_region_set(struct bnxt_ulp_context *ulp_ctx, + enum ulp_ha_mgr_region region) +{ + struct bnxt_ulp_ha_mgr_info *ha_info; + + if (ulp_ctx == NULL) { + BNXT_TF_DBG(ERR, "Invalid params in ha region get.\n"); + return -EINVAL; + } + + ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx); + if (ha_info == NULL) { + BNXT_TF_DBG(ERR, "Unable to get ha info\n"); + return -EINVAL; + } + ha_info->region = region; + + return 0; +} + +static int32_t +ulp_ha_mgr_app_type_set(struct bnxt_ulp_context *ulp_ctx, + enum ulp_ha_mgr_app_type app_type) +{ + struct bnxt_ulp_ha_mgr_info *ha_info; + + if (ulp_ctx == NULL) { + BNXT_TF_DBG(ERR, "Invalid Parms.\n"); + return -EINVAL; + } + + ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx); + if (ha_info == NULL) { + BNXT_TF_DBG(ERR, "Unable to get the ha info.\n"); + return -EINVAL; + } + ha_info->app_type = app_type; + + return 0; +} + +/* + * When a secondary opens, the timer is started and periodically checks for a + * close of the primary (state moved to SEC_TIMER_COPY). + * In SEC_TIMER_COPY: + * - The flow db must be locked to prevent flows from being added to the high + * region during a move. + * - Move the high entries to low + * - Set the region to low for subsequent flows + * - Switch our persona to Primary + * - Set the state to Primary Run + * - Release the flow db lock for flows to continue + */ +static void +ulp_ha_mgr_timer_cb(void *arg) +{ + struct tf_move_tcam_shared_entries_parms mparms = { 0 }; + struct bnxt_ulp_context *ulp_ctx; + enum ulp_ha_mgr_state curr_state; + struct tf *tfp; + int32_t rc; + + ulp_ctx = (struct bnxt_ulp_context *)arg; + rc = ulp_ha_mgr_state_get(ulp_ctx, &curr_state); + if (rc) { + /* + * This shouldn't happen, if it does, resetart the timer + * and try again next time. + */ + BNXT_TF_DBG(ERR, "On HA CB:Failed(%d) to get state.\n", rc); + goto cb_restart; + } + if (curr_state != ULP_HA_STATE_SEC_TIMER_COPY) + goto cb_restart; + + /* Protect the flow database during the copy */ + if (bnxt_ulp_cntxt_acquire_fdb_lock(ulp_ctx)) { + /* Should not fail, if we do, restart timer and try again */ + BNXT_TF_DBG(ERR, "Flow db lock acquire failed\n"); + goto cb_restart; + } + /* All paths after this point must release the fdb lock */ + + /* The Primary has issued a close and we are in the timer copy + * phase. Become the new Primary, Set state to Primary Run and + * move WC entries to Low Region. + */ + BNXT_TF_DBG(INFO, "On HA CB: Moving entries HI to LOW\n"); + mparms.dir = TF_DIR_RX; + mparms.tcam_tbl_type = TF_TCAM_TBL_TYPE_WC_TCAM_HIGH; + tfp = bnxt_ulp_cntxt_tfp_get(ulp_ctx, BNXT_ULP_SHARED_SESSION_YES); + if (tfp == NULL) { + BNXT_TF_DBG(ERR, "On HA CB: Unable to get the TFP.\n"); + goto unlock; + } + + rc = tf_move_tcam_shared_entries(tfp, &mparms); + if (rc) { + BNXT_TF_DBG(ERR, "On HA_CB: Failed to move entries\n"); + goto unlock; + } + + ulp_ha_mgr_region_set(ulp_ctx, ULP_HA_REGION_LOW); + ulp_ha_mgr_app_type_set(ulp_ctx, ULP_HA_APP_TYPE_PRIM); + ulp_ha_mgr_state_set(ulp_ctx, ULP_HA_STATE_PRIM_RUN); + BNXT_TF_DBG(INFO, "On HA CB: SEC[SEC_TIMER_COPY] => PRIM[PRIM_RUN]\n"); +unlock: + bnxt_ulp_cntxt_release_fdb_lock(ulp_ctx); + return; +cb_restart: + ulp_ha_mgr_timer_start(ulp_ctx); +} + +static int32_t +ulp_ha_mgr_timer_start(struct bnxt_ulp_context *ulp_ctx) +{ + struct bnxt_ulp_ha_mgr_info *ha_info; + + if (ulp_ctx == NULL) { + BNXT_TF_DBG(ERR, "Invalid parmsi for ha timer start.\n"); + return -EINVAL; + } + + ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx); + + if (ha_info == NULL) { + BNXT_TF_DBG(ERR, "Unable to get HA Info in timer start.\n"); + return -EINVAL; + } + ha_info->flags |= ULP_HA_TIMER_THREAD; + rte_eal_alarm_set(US_PER_S * ULP_HA_TIMER_SEC, + ulp_ha_mgr_timer_cb, + (void *)ulp_ctx); + return 0; +} + +static void +ulp_ha_mgr_timer_cancel(struct bnxt_ulp_context *ulp_ctx) +{ + struct bnxt_ulp_ha_mgr_info *ha_info; + + ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx); + if (ha_info == NULL) { + BNXT_TF_DBG(ERR, "Unable to get ha info\n"); + return; + } + + ha_info->flags &= ~ULP_HA_TIMER_THREAD; + rte_eal_alarm_cancel(ulp_ha_mgr_timer_cb, (void *)ulp_ctx); +} + +int32_t +ulp_ha_mgr_init(struct bnxt_ulp_context *ulp_ctx) +{ + struct bnxt_ulp_ha_mgr_info *ha_info; + int32_t rc; + ha_info = rte_zmalloc("ulp_ha_mgr_info", sizeof(*ha_info), 0); + if (!ha_info) + return -ENOMEM; + + /* Add the HA info tbl to the ulp context. */ + bnxt_ulp_cntxt_ptr2_ha_info_set(ulp_ctx, ha_info); + + rc = pthread_mutex_init(&ha_info->ha_lock, NULL); + if (rc) { + PMD_DRV_LOG(ERR, "Failed to initialize ha mutex\n"); + goto cleanup; + } + + return 0; +cleanup: + if (ha_info != NULL) + ulp_ha_mgr_deinit(ulp_ctx); + return -ENOMEM; +} + +void +ulp_ha_mgr_deinit(struct bnxt_ulp_context *ulp_ctx) +{ + struct bnxt_ulp_ha_mgr_info *ha_info; + + ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx); + if (ha_info == NULL) { + BNXT_TF_DBG(ERR, "Unable to get HA Info for deinit.\n"); + return; + } + + pthread_mutex_destroy(&ha_info->ha_lock); + rte_free(ha_info); + + bnxt_ulp_cntxt_ptr2_ha_info_set(ulp_ctx, NULL); +} + +int32_t +ulp_ha_mgr_app_type_get(struct bnxt_ulp_context *ulp_ctx, + enum ulp_ha_mgr_app_type *app_type) +{ + struct bnxt_ulp_ha_mgr_info *ha_info; + + if (ulp_ctx == NULL || app_type == NULL) { + BNXT_TF_DBG(ERR, "Invalid Parms.\n"); + return -EINVAL; + } + + ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx); + if (ha_info == NULL) { + BNXT_TF_DBG(ERR, "Unable to get the HA info.\n"); + return -EINVAL; + } + *app_type = ha_info->app_type; + + return 0; +} + +int32_t +ulp_ha_mgr_state_get(struct bnxt_ulp_context *ulp_ctx, + enum ulp_ha_mgr_state *state) +{ + struct tf_get_if_tbl_entry_parms get_parms = { 0 }; + struct tf *tfp; + uint32_t val = 0; + int32_t rc = 0; + + if (ulp_ctx == NULL || state == NULL) { + BNXT_TF_DBG(ERR, "Invalid parms in state get.\n"); + return -EINVAL; + } + tfp = bnxt_ulp_cntxt_tfp_get(ulp_ctx, BNXT_ULP_SHARED_SESSION_NO); + if (tfp == NULL) { + BNXT_TF_DBG(ERR, "Unable to get the TFP.\n"); + return -EINVAL; + } + + get_parms.dir = ULP_HA_IF_TBL_DIR; + get_parms.type = ULP_HA_IF_TBL_TYPE; + get_parms.idx = ULP_HA_IF_TBL_IDX; + get_parms.data = (uint8_t *)&val; + get_parms.data_sz_in_bytes = sizeof(val); + + rc = tf_get_if_tbl_entry(tfp, &get_parms); + if (rc) + BNXT_TF_DBG(ERR, "Failed to read the HA state\n"); + + *state = val; + return rc; +} + +int32_t +ulp_ha_mgr_open(struct bnxt_ulp_context *ulp_ctx) +{ + enum ulp_ha_mgr_state curr_state; + int32_t rc; + + rc = ulp_ha_mgr_state_get(ulp_ctx, &curr_state); + if (rc) { + BNXT_TF_DBG(ERR, "Failed to get HA state on Open (%d)\n", rc); + return -EINVAL; + } + + /* + * An Open can only occur during the Init and Primary Run states. During + * Init, the system attempting to Open will become the only system + * running. During Primary Run, the system attempting to Open will + * become the secondary system temporarily, and should eventually be + * transitioned to the primary system. + */ + switch (curr_state) { + case ULP_HA_STATE_INIT: + /* + * No system is running, as we are the primary. Since no other + * system is running, we start writing into the low region. By + * writing into the low region, we save room for the secondary + * system to override our entries by using the high region. + */ + ulp_ha_mgr_app_type_set(ulp_ctx, ULP_HA_APP_TYPE_PRIM); + ulp_ha_mgr_region_set(ulp_ctx, ULP_HA_REGION_LOW); + rc = ulp_ha_mgr_state_set(ulp_ctx, ULP_HA_STATE_PRIM_RUN); + if (rc) { + BNXT_TF_DBG(ERR, "On Open: Failed to set PRIM_RUN.\n"); + return -EINVAL; + } + + BNXT_TF_DBG(INFO, "On Open: [INIT] => PRIM[PRIM_RUN]\n"); + break; + case ULP_HA_STATE_PRIM_RUN: + /* + * The secondary system is starting in order to take over. + * The current primary is expected to eventually close and pass + * full control to this system;however, until the primary closes + * both are operational. + * + * The timer is started in order to determine when the + * primary has closed. + */ + ulp_ha_mgr_app_type_set(ulp_ctx, ULP_HA_APP_TYPE_SEC); + ulp_ha_mgr_region_set(ulp_ctx, ULP_HA_REGION_HI); + + /* + * TODO: + * Clear the high region so the secondary can begin overriding + * the current entries. + */ + rc = ulp_ha_mgr_timer_start(ulp_ctx); + if (rc) { + BNXT_TF_DBG(ERR, "Unable to start timer on HA Open.\n"); + return -EINVAL; + } + + rc = ulp_ha_mgr_state_set(ulp_ctx, ULP_HA_STATE_PRIM_SEC_RUN); + if (rc) { + BNXT_TF_DBG(ERR, "On Open: Failed to set PRIM_SEC_RUN\n"); + return -EINVAL; + } + BNXT_TF_DBG(INFO, "On Open: [PRIM_RUN] => [PRIM_SEC_RUN]\n"); + break; + default: + BNXT_TF_DBG(ERR, "On Open: Unknown state 0x%x\n", curr_state); + return -EINVAL; + } + + return 0; +} + +int32_t +ulp_ha_mgr_close(struct bnxt_ulp_context *ulp_ctx) +{ + enum ulp_ha_mgr_state curr_state, next_state, poll_state; + enum ulp_ha_mgr_app_type app_type; + int32_t timeout; + int32_t rc; + + rc = ulp_ha_mgr_state_get(ulp_ctx, &curr_state); + if (rc) { + BNXT_TF_DBG(ERR, "On Close: Failed(%d) to get HA state\n", rc); + return -EINVAL; + } + + rc = ulp_ha_mgr_app_type_get(ulp_ctx, &app_type); + if (rc) { + BNXT_TF_DBG(ERR, "On Close: Failed to get the app type.\n"); + return -EINVAL; + } + + if (curr_state == ULP_HA_STATE_PRIM_RUN && + app_type == ULP_HA_APP_TYPE_PRIM) { + /* + * Only the primary is running, so a close effectively moves the + * system back to INIT. + */ + next_state = ULP_HA_STATE_INIT; + ulp_ha_mgr_state_set(ulp_ctx, next_state); + BNXT_TF_DBG(INFO, "On Close: PRIM[PRIM_RUN] => [INIT]\n"); + } else if (curr_state == ULP_HA_STATE_PRIM_SEC_RUN && + app_type == ULP_HA_APP_TYPE_PRIM) { + /* + * While both are running, the primary received a close. + * Cleanup the flows, set the COPY state, and wait for the + * secondary to become the Primary. + */ + BNXT_TF_DBG(INFO, + "On Close: PRIM[PRIM_SEC_RUN] flushing flows.\n"); + + ulp_flow_db_flush_flows(ulp_ctx, BNXT_ULP_FDB_TYPE_REGULAR); + ulp_ha_mgr_state_set(ulp_ctx, ULP_HA_STATE_SEC_TIMER_COPY); + + /* + * TODO: This needs to be bounded in case the other system does + * not move to PRIM_RUN. + */ + BNXT_TF_DBG(INFO, + "On Close: PRIM[PRIM_SEC_RUN] => [Copy], enter wait.\n"); + timeout = ULP_HA_WAIT_TIMEOUT; + do { + rte_delay_ms(ULP_HA_WAIT_TIME); + rc = ulp_ha_mgr_state_get(ulp_ctx, &poll_state); + if (rc) { + BNXT_TF_DBG(ERR, + "Failed to get HA state on Close (%d)\n", + rc); + goto cleanup; + } + timeout -= ULP_HA_WAIT_TIME; + BNXT_TF_DBG(INFO, + "On Close: Waiting %d ms for PRIM_RUN\n", + timeout); + } while (poll_state != ULP_HA_STATE_PRIM_RUN && timeout > 0); + + if (timeout <= 0) { + BNXT_TF_DBG(ERR, "On Close: SEC[COPY] Timed out\n"); + goto cleanup; + } + + BNXT_TF_DBG(INFO, "On Close: PRIM[PRIM_SEC_RUN] => [COPY]\n"); + } else if (curr_state == ULP_HA_STATE_PRIM_SEC_RUN && + app_type == ULP_HA_APP_TYPE_SEC) { + /* + * While both are running, the secondary unexpectedly received a + * close. Cancel the timer, set the state to Primary RUN since + * it is the only one running. + */ + ulp_ha_mgr_timer_cancel(ulp_ctx); + ulp_ha_mgr_state_set(ulp_ctx, ULP_HA_STATE_PRIM_RUN); + + BNXT_TF_DBG(INFO, "On Close: SEC[PRIM_SEC_RUN] => [PRIM_RUN]\n"); + } else if (curr_state == ULP_HA_STATE_SEC_TIMER_COPY && + app_type == ULP_HA_APP_TYPE_SEC) { + /* + * While both were running and the Secondary went into copy, + * secondary received a close. Wait until the former Primary + * clears the copy stage, close, and set to INIT. + */ + BNXT_TF_DBG(INFO, "On Close: SEC[COPY] wait for PRIM_RUN\n"); + + timeout = ULP_HA_WAIT_TIMEOUT; + do { + rte_delay_ms(ULP_HA_WAIT_TIME); + rc = ulp_ha_mgr_state_get(ulp_ctx, &poll_state); + if (rc) { + BNXT_TF_DBG(ERR, + "Failed to get HA state on Close (%d)\n", + rc); + goto cleanup; + } + + timeout -= ULP_HA_WAIT_TIME; + BNXT_TF_DBG(INFO, + "On Close: Waiting %d ms for PRIM_RUN\n", + timeout); + } while (poll_state != ULP_HA_STATE_PRIM_RUN && + timeout >= 0); + + if (timeout <= 0) { + BNXT_TF_DBG(ERR, + "On Close: SEC[COPY] Timed out\n"); + goto cleanup; + } + + next_state = ULP_HA_STATE_INIT; + rc = ulp_ha_mgr_state_set(ulp_ctx, next_state); + if (rc) { + BNXT_TF_DBG(ERR, + "On Close: Failed to set state to INIT(%x)\n", + rc); + goto cleanup; + } + + BNXT_TF_DBG(INFO, + "On Close: SEC[COPY] => [INIT] after %d ms\n", + ULP_HA_WAIT_TIMEOUT - timeout); + } else { + BNXT_TF_DBG(ERR, "On Close: Invalid type/state %d/%d\n", + curr_state, app_type); + } +cleanup: + return rc; +} + +int32_t +ulp_ha_mgr_region_get(struct bnxt_ulp_context *ulp_ctx, + enum ulp_ha_mgr_region *region) +{ + struct bnxt_ulp_ha_mgr_info *ha_info; + + if (ulp_ctx == NULL || region == NULL) { + BNXT_TF_DBG(ERR, "Invalid params in ha region get.\n"); + return -EINVAL; + } + + ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx); + if (ha_info == NULL) { + BNXT_TF_DBG(ERR, "Unable to get ha info\n"); + return -EINVAL; + } + *region = ha_info->region; + + return 0; +} diff --git a/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.h b/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.h new file mode 100644 index 0000000000..793511564a --- /dev/null +++ b/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2014-2021 Broadcom + * All rights reserved. + */ + +#ifndef _ULP_HA_MGR_H_ +#define _ULP_HA_MGR_H_ + +#include "bnxt_ulp.h" + +enum ulp_ha_mgr_state { + ULP_HA_STATE_INIT, + ULP_HA_STATE_PRIM_RUN, + ULP_HA_STATE_PRIM_SEC_RUN, + ULP_HA_STATE_SEC_TIMER_COPY, + ULP_HA_PRIM_CLOSE +}; + +enum ulp_ha_mgr_app_type { + ULP_HA_APP_TYPE_NONE, + ULP_HA_APP_TYPE_PRIM, + ULP_HA_APP_TYPE_SEC +}; + +enum ulp_ha_mgr_region { + ULP_HA_REGION_LOW, + ULP_HA_REGION_HI +}; + +struct bnxt_ulp_ha_mgr_info { + enum ulp_ha_mgr_app_type app_type; + enum ulp_ha_mgr_region region; + uint32_t flags; + pthread_mutex_t ha_lock; +}; + +bool +ulp_ha_mgr_is_enabled(struct bnxt_ulp_context *ulp_ctx); + +int32_t +ulp_ha_mgr_enable(struct bnxt_ulp_context *ulp_ctx); + +int32_t +ulp_ha_mgr_init(struct bnxt_ulp_context *ulp_ctx); + +void +ulp_ha_mgr_deinit(struct bnxt_ulp_context *ulp_ctx); + +int32_t +ulp_ha_mgr_app_type_get(struct bnxt_ulp_context *ulp_ctx, + enum ulp_ha_mgr_app_type *app_type); + +int32_t +ulp_ha_mgr_state_get(struct bnxt_ulp_context *ulp_ctx, + enum ulp_ha_mgr_state *state); + +int32_t +ulp_ha_mgr_open(struct bnxt_ulp_context *ulp_ctx); + +int32_t +ulp_ha_mgr_close(struct bnxt_ulp_context *ulp_ctx); + +int32_t +ulp_ha_mgr_region_get(struct bnxt_ulp_context *ulp_ctx, + enum ulp_ha_mgr_region *region); + +#endif /* _ULP_HA_MGR_H_*/ diff --git a/drivers/net/bnxt/tf_ulp/ulp_mapper.c b/drivers/net/bnxt/tf_ulp/ulp_mapper.c index e2404c392b..05a43b6dc5 100644 --- a/drivers/net/bnxt/tf_ulp/ulp_mapper.c +++ b/drivers/net/bnxt/tf_ulp/ulp_mapper.c @@ -19,6 +19,7 @@ #include "tf_util.h" #include "ulp_template_db_tbl.h" #include "ulp_port_db.h" +#include "ulp_ha_mgr.h" static uint8_t mapper_fld_zeros[16] = { 0 }; @@ -419,7 +420,7 @@ ulp_mapper_ident_fields_get(struct bnxt_ulp_mapper_parms *mparms, } static inline int32_t -ulp_mapper_tcam_entry_free(struct bnxt_ulp_context *ulp __rte_unused, +ulp_mapper_tcam_entry_free(struct bnxt_ulp_context *ulp, struct tf *tfp, struct ulp_flow_db_res_params *res) { @@ -429,6 +430,30 @@ ulp_mapper_tcam_entry_free(struct bnxt_ulp_context *ulp __rte_unused, .idx = (uint16_t)res->resource_hndl }; + /* If HA is enabled, we may have to remap the TF Type */ + if (bnxt_ulp_cntxt_ha_enabled(ulp)) { + enum ulp_ha_mgr_region region; + int32_t rc; + + switch (res->resource_type) { + case TF_TCAM_TBL_TYPE_WC_TCAM_HIGH: + case TF_TCAM_TBL_TYPE_WC_TCAM_LOW: + rc = ulp_ha_mgr_region_get(ulp, ®ion); + if (rc) + /* Log this, but assume region is correct */ + BNXT_TF_DBG(ERR, + "Unable to get HA region (%d)\n", + rc); + else + fparms.tcam_tbl_type = + (region == ULP_HA_REGION_LOW) ? + TF_TCAM_TBL_TYPE_WC_TCAM_LOW : + TF_TCAM_TBL_TYPE_WC_TCAM_HIGH; + break; + default: + break; + } + } return tf_free_tcam_entry(tfp, &fparms); } @@ -2904,10 +2929,12 @@ static int32_t ulp_mapper_app_glb_resource_info_init(struct bnxt_ulp_context *ulp_ctx, struct bnxt_ulp_mapper_data *mapper_data) { + struct tf_get_shared_tbl_increment_parms iparms; struct bnxt_ulp_glb_resource_info *glb_res; struct tf_get_session_info_parms sparms; uint32_t num_entries, i, dev_id, res; struct tf_resource_info *res_info; + uint32_t addend; uint64_t regval; enum tf_dir dir; int32_t rc = 0; @@ -2915,13 +2942,11 @@ ulp_mapper_app_glb_resource_info_init(struct bnxt_ulp_context *ulp_ctx, uint8_t app_id; memset(&sparms, 0, sizeof(sparms)); - glb_res = bnxt_ulp_app_glb_resource_info_list_get(&num_entries); if (!glb_res || !num_entries) { BNXT_TF_DBG(ERR, "Invalid Arguments\n"); return -EINVAL; } - tfp = bnxt_ulp_cntxt_shared_tfp_get(ulp_ctx); if (!tfp) { BNXT_TF_DBG(ERR, "Failed to get tfp for app global init"); @@ -2958,12 +2983,29 @@ ulp_mapper_app_glb_resource_info_init(struct bnxt_ulp_context *ulp_ctx, continue; dir = glb_res[i].direction; res = glb_res[i].resource_type; + addend = 1; switch (glb_res[i].resource_func) { case BNXT_ULP_RESOURCE_FUNC_IDENTIFIER: res_info = &sparms.session_info.ident[dir].info[res]; break; case BNXT_ULP_RESOURCE_FUNC_INDEX_TABLE: + /* + * Tables may have various strides for the allocations. + * Need to account. + */ + memset(&iparms, 0, sizeof(iparms)); + iparms.dir = dir; + iparms.type = res; + rc = tf_get_shared_tbl_increment(tfp, &iparms); + if (rc) { + BNXT_TF_DBG(ERR, + "Failed to get addend for %s[%s] rc=(%d)\n", + tf_tbl_type_2_str(res), + tf_dir_2_str(dir), rc); + return rc; + } + addend = iparms.increment_cnt; res_info = &sparms.session_info.tbl[dir].info[res]; break; case BNXT_ULP_RESOURCE_FUNC_TCAM_TABLE: @@ -2977,10 +3019,8 @@ ulp_mapper_app_glb_resource_info_init(struct bnxt_ulp_context *ulp_ctx, glb_res[i].resource_func); continue; } - regval = tfp_cpu_to_be_64((uint64_t)res_info->start); - res_info->start++; - + res_info->start += addend; /* * All resources written to the global regfile are shared for * this function.