net/bnxt: add HA support in ULP
authorMike Baucom <michael.baucom@broadcom.com>
Sun, 30 May 2021 08:59:24 +0000 (14:29 +0530)
committerAjit Khaparde <ajit.khaparde@broadcom.com>
Thu, 8 Jul 2021 00:02:13 +0000 (02:02 +0200)
Add the ability for cooperative applications to share resources and
use the high availability functionality in the HW.

Signed-off-by: Mike Baucom <michael.baucom@broadcom.com>
Signed-off-by: Venkat Duvvuru <venkatkumar.duvvuru@broadcom.com>
Reviewed-by: Shahaji Bhosle <sbhosle@broadcom.com>
Reviewed-by: Randy Schacher <stuart.schacher@broadcom.com>
Reviewed-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
drivers/net/bnxt/tf_ulp/bnxt_ulp.c
drivers/net/bnxt/tf_ulp/bnxt_ulp.h
drivers/net/bnxt/tf_ulp/bnxt_ulp_flow.c
drivers/net/bnxt/tf_ulp/meson.build
drivers/net/bnxt/tf_ulp/ulp_ha_mgr.c [new file with mode: 0644]
drivers/net/bnxt/tf_ulp/ulp_ha_mgr.h [new file with mode: 0644]
drivers/net/bnxt/tf_ulp/ulp_mapper.c

index 0daa8e4..972bf8b 100644 (file)
@@ -23,6 +23,7 @@
 #include "ulp_mapper.h"
 #include "ulp_port_db.h"
 #include "ulp_tun.h"
+#include "ulp_ha_mgr.h"
 
 /* Linked list of all TF sessions. */
 STAILQ_HEAD(, bnxt_ulp_session_state) bnxt_ulp_session_list =
@@ -315,6 +316,9 @@ bnxt_ulp_cntxt_app_caps_init(struct bnxt_ulp_context *ulp_ctx,
                if (info[i].flags & BNXT_ULP_APP_CAP_SHARED_EN)
                        ulp_ctx->cfg_data->ulp_flags |=
                                BNXT_ULP_SHARED_SESSION_ENABLED;
+               if (info[i].flags & BNXT_ULP_APP_CAP_HOT_UPGRADE_EN)
+                       ulp_ctx->cfg_data->ulp_flags |=
+                               BNXT_ULP_HIGH_AVAIL_ENABLED;
        }
        if (!found) {
                BNXT_TF_DBG(ERR, "APP ID %d, Device ID: 0x%x not supported.\n",
@@ -1137,9 +1141,18 @@ static void
 bnxt_ulp_deinit(struct bnxt *bp,
                struct bnxt_ulp_session_state *session)
 {
+       bool ha_enabled;
+
        if (!bp->ulp_ctx || !bp->ulp_ctx->cfg_data)
                return;
 
+       ha_enabled = bnxt_ulp_cntxt_ha_enabled(bp->ulp_ctx);
+       if (ha_enabled && session->session_opened) {
+               int32_t rc = ulp_ha_mgr_close(bp->ulp_ctx);
+               if (rc)
+                       BNXT_TF_DBG(ERR, "Failed to close HA (%d)\n", rc);
+       }
+
        /* clean up default flows */
        bnxt_ulp_destroy_df_rules(bp, true);
 
@@ -1179,6 +1192,9 @@ bnxt_ulp_deinit(struct bnxt *bp,
        /* free the flow db lock */
        pthread_mutex_destroy(&bp->ulp_ctx->cfg_data->flow_db_lock);
 
+       if (ha_enabled)
+               ulp_ha_mgr_deinit(bp->ulp_ctx);
+
        /* Delete the ulp context and tf session and free the ulp context */
        ulp_ctx_deinit(bp, session);
        BNXT_TF_DBG(DEBUG, "ulp ctx has been deinitialized\n");
@@ -1275,6 +1291,19 @@ bnxt_ulp_init(struct bnxt *bp,
                BNXT_TF_DBG(ERR, "Failed to set tx global configuration\n");
                goto jump_to_error;
        }
+
+       if (bnxt_ulp_cntxt_ha_enabled(bp->ulp_ctx)) {
+               rc = ulp_ha_mgr_init(bp->ulp_ctx);
+               if (rc) {
+                       BNXT_TF_DBG(ERR, "Failed to initialize HA %d\n", rc);
+                       goto jump_to_error;
+               }
+               rc = ulp_ha_mgr_open(bp->ulp_ctx);
+               if (rc) {
+                       BNXT_TF_DBG(ERR, "Failed to Process HA Open %d\n", rc);
+                       goto jump_to_error;
+               }
+       }
        BNXT_TF_DBG(DEBUG, "ulp ctx has been initialized\n");
        return rc;
 
@@ -1828,3 +1857,33 @@ bnxt_ulp_cntxt_release_fdb_lock(struct bnxt_ulp_context  *ulp_ctx)
 
        pthread_mutex_unlock(&ulp_ctx->cfg_data->flow_db_lock);
 }
+
+/* Function to set the ha info into the context */
+int32_t
+bnxt_ulp_cntxt_ptr2_ha_info_set(struct bnxt_ulp_context *ulp_ctx,
+                               struct bnxt_ulp_ha_mgr_info *ulp_ha_info)
+{
+       if (ulp_ctx == NULL || ulp_ctx->cfg_data == NULL) {
+               BNXT_TF_DBG(ERR, "Invalid ulp context data\n");
+               return -EINVAL;
+       }
+       ulp_ctx->cfg_data->ha_info = ulp_ha_info;
+       return 0;
+}
+
+/* Function to retrieve the ha info from the context. */
+struct bnxt_ulp_ha_mgr_info *
+bnxt_ulp_cntxt_ptr2_ha_info_get(struct bnxt_ulp_context *ulp_ctx)
+{
+       if (ulp_ctx == NULL || ulp_ctx->cfg_data == NULL)
+               return NULL;
+       return ulp_ctx->cfg_data->ha_info;
+}
+
+bool
+bnxt_ulp_cntxt_ha_enabled(struct bnxt_ulp_context *ulp_ctx)
+{
+       if (ulp_ctx == NULL || ulp_ctx->cfg_data == NULL)
+               return false;
+       return !!ULP_HIGH_AVAIL_IS_ENABLED(ulp_ctx->cfg_data->ulp_flags);
+}
index 1ba67ed..b1f090a 100644 (file)
@@ -67,6 +67,7 @@ struct bnxt_ulp_data {
        void                            *mapper_data;
        struct bnxt_ulp_port_db         *port_db;
        struct bnxt_ulp_fc_info         *fc_info;
+       struct bnxt_ulp_ha_mgr_info     *ha_info;
        uint32_t                        ulp_flags;
        struct bnxt_ulp_df_rule_info    df_rule_info[RTE_MAX_ETHPORTS];
        struct bnxt_ulp_vfr_rule_info   vfr_rule_info[RTE_MAX_ETHPORTS];
@@ -275,4 +276,13 @@ bnxt_ulp_cntxt_app_caps_init(struct bnxt_ulp_context *ulp_ctx,
 struct bnxt_ulp_resource_resv_info *
 bnxt_ulp_resource_resv_list_get(uint32_t *num_entries);
 
+int32_t
+bnxt_ulp_cntxt_ptr2_ha_info_set(struct bnxt_ulp_context *ulp_ctx,
+                               struct bnxt_ulp_ha_mgr_info *ulp_ha_info);
+
+struct bnxt_ulp_ha_mgr_info *
+bnxt_ulp_cntxt_ptr2_ha_info_get(struct bnxt_ulp_context *ulp_ctx);
+
+bool
+bnxt_ulp_cntxt_ha_enabled(struct bnxt_ulp_context *ulp_ctx);
 #endif /* _BNXT_ULP_H_ */
index 9c27217..96e6a76 100644 (file)
@@ -11,6 +11,7 @@
 #include "ulp_mapper.h"
 #include "ulp_fc_mgr.h"
 #include "ulp_port_db.h"
+#include "ulp_ha_mgr.h"
 #include <rte_malloc.h>
 
 static int32_t
@@ -112,11 +113,17 @@ bnxt_ulp_init_mapper_params(struct bnxt_ulp_mapper_create_parms *mapper_cparms,
        /* update the WC Priority flag */
        if (!bnxt_ulp_cntxt_ptr2_ulp_flags_get(params->ulp_ctx, &ulp_flags) &&
            ULP_HIGH_AVAIL_IS_ENABLED(ulp_flags)) {
-               /* TBD: read the state and Set the WC priority */
-               ULP_COMP_FLD_IDX_WR(params,
-                                   BNXT_ULP_CF_IDX_WC_IS_HA_HIGH_REG, 1);
+               enum ulp_ha_mgr_region region = ULP_HA_REGION_LOW;
+               int32_t rc;
+
+               rc = ulp_ha_mgr_region_get(params->ulp_ctx, &region);
+               if (rc)
+                       BNXT_TF_DBG(ERR, "Unable to get WC region\n");
+               if (region == ULP_HA_REGION_HI)
+                       ULP_COMP_FLD_IDX_WR(params,
+                                           BNXT_ULP_CF_IDX_WC_IS_HA_HIGH_REG,
+                                           1);
        }
-
 }
 
 /* Function to create the rte flow. */
index 5555350..49df42c 100644 (file)
@@ -24,6 +24,7 @@ sources += files(
         'ulp_tun.c',
         'ulp_gen_tbl.c',
         'ulp_gen_hash.c',
+        'ulp_ha_mgr.c',
         'ulp_rte_handler_tbl.c',
         'ulp_template_db_wh_plus_act.c',
         'ulp_template_db_wh_plus_class.c',
diff --git a/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.c b/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.c
new file mode 100644 (file)
index 0000000..dc71054
--- /dev/null
@@ -0,0 +1,551 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019-2021 Broadcom
+ * All rights reserved.
+ */
+
+#include <rte_common.h>
+#include <rte_cycles.h>
+#include <rte_malloc.h>
+#include <rte_log.h>
+#include <rte_alarm.h>
+#include "bnxt.h"
+#include "bnxt_ulp.h"
+#include "bnxt_tf_common.h"
+#include "ulp_ha_mgr.h"
+#include "ulp_flow_db.h"
+
+/* Local only MACROs and defines that aren't exported */
+#define ULP_HA_TIMER_THREAD    (1 << 0)
+#define ULP_HA_TIMER_IS_RUNNING(info) (!!((info)->flags & ULP_HA_TIMER_THREAD))
+#define ULP_HA_TIMER_SEC 1
+#define ULP_HA_WAIT_TIME (MS_PER_S / 10)
+#define ULP_HA_WAIT_TIMEOUT (MS_PER_S * 2)
+
+#define ULP_HA_IF_TBL_DIR      TF_DIR_RX
+#define ULP_HA_IF_TBL_TYPE     TF_IF_TBL_TYPE_PROF_PARIF_ERR_ACT_REC_PTR
+#define ULP_HA_IF_TBL_IDX 10
+
+static void ulp_ha_mgr_timer_cancel(struct bnxt_ulp_context *ulp_ctx);
+static int32_t ulp_ha_mgr_timer_start(struct bnxt_ulp_context *ulp_ctx);
+static void ulp_ha_mgr_timer_cb(void *arg);
+static int32_t ulp_ha_mgr_app_type_set(struct bnxt_ulp_context *ulp_ctx,
+                               enum ulp_ha_mgr_app_type app_type);
+static int32_t
+ulp_ha_mgr_region_set(struct bnxt_ulp_context *ulp_ctx,
+                     enum ulp_ha_mgr_region region);
+static int32_t
+ulp_ha_mgr_state_set(struct bnxt_ulp_context *ulp_ctx,
+                    enum ulp_ha_mgr_state state);
+
+static int32_t
+ulp_ha_mgr_state_set(struct bnxt_ulp_context *ulp_ctx,
+                    enum ulp_ha_mgr_state state)
+{
+       struct tf_set_if_tbl_entry_parms set_parms = { 0 };
+       struct tf *tfp;
+       uint32_t val = 0;
+       int32_t rc = 0;
+
+       if (ulp_ctx == NULL) {
+               BNXT_TF_DBG(ERR, "Invalid parms in state get.\n");
+               return -EINVAL;
+       }
+       tfp = bnxt_ulp_cntxt_tfp_get(ulp_ctx, BNXT_ULP_SHARED_SESSION_NO);
+       if (tfp == NULL) {
+               BNXT_TF_DBG(ERR, "Unable to get the TFP.\n");
+               return -EINVAL;
+       }
+
+       val = (uint32_t)state;
+
+       set_parms.dir = ULP_HA_IF_TBL_DIR;
+       set_parms.type = ULP_HA_IF_TBL_TYPE;
+       set_parms.data = (uint8_t *)&val;
+       set_parms.data_sz_in_bytes = sizeof(val);
+       set_parms.idx = ULP_HA_IF_TBL_IDX;
+
+       rc = tf_set_if_tbl_entry(tfp, &set_parms);
+       if (rc)
+               BNXT_TF_DBG(ERR, "Failed to write the HA state\n");
+
+       return rc;
+}
+
+static int32_t
+ulp_ha_mgr_region_set(struct bnxt_ulp_context *ulp_ctx,
+                     enum ulp_ha_mgr_region region)
+{
+       struct bnxt_ulp_ha_mgr_info *ha_info;
+
+       if (ulp_ctx == NULL) {
+               BNXT_TF_DBG(ERR, "Invalid params in ha region get.\n");
+               return -EINVAL;
+       }
+
+       ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx);
+       if (ha_info == NULL) {
+               BNXT_TF_DBG(ERR, "Unable to get ha info\n");
+               return -EINVAL;
+       }
+       ha_info->region = region;
+
+       return 0;
+}
+
+static int32_t
+ulp_ha_mgr_app_type_set(struct bnxt_ulp_context *ulp_ctx,
+                       enum ulp_ha_mgr_app_type app_type)
+{
+       struct bnxt_ulp_ha_mgr_info *ha_info;
+
+       if (ulp_ctx == NULL) {
+               BNXT_TF_DBG(ERR, "Invalid Parms.\n");
+               return -EINVAL;
+       }
+
+       ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx);
+       if (ha_info == NULL) {
+               BNXT_TF_DBG(ERR, "Unable to get the ha info.\n");
+               return -EINVAL;
+       }
+       ha_info->app_type = app_type;
+
+       return 0;
+}
+
+/*
+ * When a secondary opens, the timer is started and periodically checks for a
+ * close of the primary (state moved to SEC_TIMER_COPY).
+ * In SEC_TIMER_COPY:
+ * - The flow db must be locked to prevent flows from being added to the high
+ *   region during a move.
+ * - Move the high entries to low
+ * - Set the region to low for subsequent flows
+ * - Switch our persona to Primary
+ * - Set the state to Primary Run
+ * - Release the flow db lock for flows to continue
+ */
+static void
+ulp_ha_mgr_timer_cb(void *arg)
+{
+       struct tf_move_tcam_shared_entries_parms mparms = { 0 };
+       struct bnxt_ulp_context *ulp_ctx;
+       enum ulp_ha_mgr_state curr_state;
+       struct tf *tfp;
+       int32_t rc;
+
+       ulp_ctx = (struct bnxt_ulp_context *)arg;
+       rc = ulp_ha_mgr_state_get(ulp_ctx, &curr_state);
+       if (rc) {
+               /*
+                * This shouldn't happen, if it does, resetart the timer
+                * and try again next time.
+                */
+               BNXT_TF_DBG(ERR, "On HA CB:Failed(%d) to get state.\n", rc);
+               goto cb_restart;
+       }
+       if (curr_state != ULP_HA_STATE_SEC_TIMER_COPY)
+               goto cb_restart;
+
+       /* Protect the flow database during the copy */
+       if (bnxt_ulp_cntxt_acquire_fdb_lock(ulp_ctx)) {
+               /* Should not fail, if we do, restart timer and try again */
+               BNXT_TF_DBG(ERR, "Flow db lock acquire failed\n");
+               goto cb_restart;
+       }
+       /* All paths after this point must release the fdb lock */
+
+       /* The Primary has issued a close and we are in the timer copy
+        * phase.  Become the new Primary, Set state to Primary Run and
+        * move WC entries to Low Region.
+        */
+       BNXT_TF_DBG(INFO, "On HA CB: Moving entries HI to LOW\n");
+       mparms.dir = TF_DIR_RX;
+       mparms.tcam_tbl_type = TF_TCAM_TBL_TYPE_WC_TCAM_HIGH;
+       tfp = bnxt_ulp_cntxt_tfp_get(ulp_ctx, BNXT_ULP_SHARED_SESSION_YES);
+       if (tfp == NULL) {
+               BNXT_TF_DBG(ERR, "On HA CB: Unable to get the TFP.\n");
+               goto unlock;
+       }
+
+       rc = tf_move_tcam_shared_entries(tfp, &mparms);
+       if (rc) {
+               BNXT_TF_DBG(ERR, "On HA_CB: Failed to move entries\n");
+               goto unlock;
+       }
+
+       ulp_ha_mgr_region_set(ulp_ctx, ULP_HA_REGION_LOW);
+       ulp_ha_mgr_app_type_set(ulp_ctx, ULP_HA_APP_TYPE_PRIM);
+       ulp_ha_mgr_state_set(ulp_ctx, ULP_HA_STATE_PRIM_RUN);
+       BNXT_TF_DBG(INFO, "On HA CB: SEC[SEC_TIMER_COPY] => PRIM[PRIM_RUN]\n");
+unlock:
+       bnxt_ulp_cntxt_release_fdb_lock(ulp_ctx);
+       return;
+cb_restart:
+       ulp_ha_mgr_timer_start(ulp_ctx);
+}
+
+static int32_t
+ulp_ha_mgr_timer_start(struct bnxt_ulp_context *ulp_ctx)
+{
+       struct bnxt_ulp_ha_mgr_info *ha_info;
+
+       if (ulp_ctx == NULL) {
+               BNXT_TF_DBG(ERR, "Invalid parmsi for ha timer start.\n");
+               return -EINVAL;
+       }
+
+       ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx);
+
+       if (ha_info == NULL) {
+               BNXT_TF_DBG(ERR, "Unable to get HA Info in timer start.\n");
+               return -EINVAL;
+       }
+       ha_info->flags |= ULP_HA_TIMER_THREAD;
+       rte_eal_alarm_set(US_PER_S * ULP_HA_TIMER_SEC,
+                         ulp_ha_mgr_timer_cb,
+                         (void *)ulp_ctx);
+       return 0;
+}
+
+static void
+ulp_ha_mgr_timer_cancel(struct bnxt_ulp_context *ulp_ctx)
+{
+       struct bnxt_ulp_ha_mgr_info *ha_info;
+
+       ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx);
+       if (ha_info == NULL) {
+               BNXT_TF_DBG(ERR, "Unable to get ha info\n");
+               return;
+       }
+
+       ha_info->flags &= ~ULP_HA_TIMER_THREAD;
+       rte_eal_alarm_cancel(ulp_ha_mgr_timer_cb, (void *)ulp_ctx);
+}
+
+int32_t
+ulp_ha_mgr_init(struct bnxt_ulp_context *ulp_ctx)
+{
+       struct bnxt_ulp_ha_mgr_info *ha_info;
+       int32_t rc;
+       ha_info = rte_zmalloc("ulp_ha_mgr_info", sizeof(*ha_info), 0);
+       if (!ha_info)
+               return -ENOMEM;
+
+       /* Add the HA info tbl to the ulp context. */
+       bnxt_ulp_cntxt_ptr2_ha_info_set(ulp_ctx, ha_info);
+
+       rc = pthread_mutex_init(&ha_info->ha_lock, NULL);
+       if (rc) {
+               PMD_DRV_LOG(ERR, "Failed to initialize ha mutex\n");
+               goto cleanup;
+       }
+
+       return 0;
+cleanup:
+       if (ha_info != NULL)
+               ulp_ha_mgr_deinit(ulp_ctx);
+       return -ENOMEM;
+}
+
+void
+ulp_ha_mgr_deinit(struct bnxt_ulp_context *ulp_ctx)
+{
+       struct bnxt_ulp_ha_mgr_info *ha_info;
+
+       ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx);
+       if (ha_info == NULL) {
+               BNXT_TF_DBG(ERR, "Unable to get HA Info for deinit.\n");
+               return;
+       }
+
+       pthread_mutex_destroy(&ha_info->ha_lock);
+       rte_free(ha_info);
+
+       bnxt_ulp_cntxt_ptr2_ha_info_set(ulp_ctx, NULL);
+}
+
+int32_t
+ulp_ha_mgr_app_type_get(struct bnxt_ulp_context *ulp_ctx,
+                       enum ulp_ha_mgr_app_type *app_type)
+{
+       struct bnxt_ulp_ha_mgr_info *ha_info;
+
+       if (ulp_ctx == NULL || app_type == NULL) {
+               BNXT_TF_DBG(ERR, "Invalid Parms.\n");
+               return -EINVAL;
+       }
+
+       ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx);
+       if (ha_info == NULL) {
+               BNXT_TF_DBG(ERR, "Unable to get the HA info.\n");
+               return -EINVAL;
+       }
+       *app_type = ha_info->app_type;
+
+       return 0;
+}
+
+int32_t
+ulp_ha_mgr_state_get(struct bnxt_ulp_context *ulp_ctx,
+                    enum ulp_ha_mgr_state *state)
+{
+       struct tf_get_if_tbl_entry_parms get_parms = { 0 };
+       struct tf *tfp;
+       uint32_t val = 0;
+       int32_t rc = 0;
+
+       if (ulp_ctx == NULL || state == NULL) {
+               BNXT_TF_DBG(ERR, "Invalid parms in state get.\n");
+               return -EINVAL;
+       }
+       tfp = bnxt_ulp_cntxt_tfp_get(ulp_ctx, BNXT_ULP_SHARED_SESSION_NO);
+       if (tfp == NULL) {
+               BNXT_TF_DBG(ERR, "Unable to get the TFP.\n");
+               return -EINVAL;
+       }
+
+       get_parms.dir = ULP_HA_IF_TBL_DIR;
+       get_parms.type = ULP_HA_IF_TBL_TYPE;
+       get_parms.idx = ULP_HA_IF_TBL_IDX;
+       get_parms.data = (uint8_t *)&val;
+       get_parms.data_sz_in_bytes = sizeof(val);
+
+       rc = tf_get_if_tbl_entry(tfp, &get_parms);
+       if (rc)
+               BNXT_TF_DBG(ERR, "Failed to read the HA state\n");
+
+       *state = val;
+       return rc;
+}
+
+int32_t
+ulp_ha_mgr_open(struct bnxt_ulp_context *ulp_ctx)
+{
+       enum ulp_ha_mgr_state curr_state;
+       int32_t rc;
+
+       rc = ulp_ha_mgr_state_get(ulp_ctx, &curr_state);
+       if (rc) {
+               BNXT_TF_DBG(ERR, "Failed to get HA state on Open (%d)\n", rc);
+               return -EINVAL;
+       }
+
+       /*
+        * An Open can only occur during the Init and Primary Run states. During
+        * Init, the system attempting to Open will become the only system
+        * running. During Primary Run, the system attempting to Open will
+        * become the secondary system temporarily, and should eventually be
+        * transitioned to the primary system.
+        */
+       switch (curr_state) {
+       case ULP_HA_STATE_INIT:
+               /*
+                * No system is running, as we are the primary.  Since no other
+                * system is running, we start writing into the low region.  By
+                * writing into the low region, we save room for the secondary
+                * system to override our entries by using the high region.
+                */
+               ulp_ha_mgr_app_type_set(ulp_ctx, ULP_HA_APP_TYPE_PRIM);
+               ulp_ha_mgr_region_set(ulp_ctx, ULP_HA_REGION_LOW);
+               rc = ulp_ha_mgr_state_set(ulp_ctx, ULP_HA_STATE_PRIM_RUN);
+               if (rc) {
+                       BNXT_TF_DBG(ERR, "On Open: Failed to set PRIM_RUN.\n");
+                       return -EINVAL;
+               }
+
+               BNXT_TF_DBG(INFO, "On Open: [INIT] => PRIM[PRIM_RUN]\n");
+               break;
+       case ULP_HA_STATE_PRIM_RUN:
+               /*
+                * The secondary system is starting in order to take over.
+                * The current primary is expected to eventually close and pass
+                * full control to this system;however, until the primary closes
+                * both are operational.
+                *
+                * The timer is started in order to determine when the
+                * primary has closed.
+                */
+               ulp_ha_mgr_app_type_set(ulp_ctx, ULP_HA_APP_TYPE_SEC);
+               ulp_ha_mgr_region_set(ulp_ctx, ULP_HA_REGION_HI);
+
+               /*
+                * TODO:
+                * Clear the high region so the secondary can begin overriding
+                * the current entries.
+                */
+               rc = ulp_ha_mgr_timer_start(ulp_ctx);
+               if (rc) {
+                       BNXT_TF_DBG(ERR, "Unable to start timer on HA Open.\n");
+                       return -EINVAL;
+               }
+
+               rc = ulp_ha_mgr_state_set(ulp_ctx, ULP_HA_STATE_PRIM_SEC_RUN);
+               if (rc) {
+                       BNXT_TF_DBG(ERR, "On Open: Failed to set PRIM_SEC_RUN\n");
+                       return -EINVAL;
+               }
+               BNXT_TF_DBG(INFO, "On Open: [PRIM_RUN] => [PRIM_SEC_RUN]\n");
+               break;
+       default:
+               BNXT_TF_DBG(ERR, "On Open: Unknown state 0x%x\n", curr_state);
+               return -EINVAL;
+       }
+
+       return 0;
+}
+
+int32_t
+ulp_ha_mgr_close(struct bnxt_ulp_context *ulp_ctx)
+{
+       enum ulp_ha_mgr_state curr_state, next_state, poll_state;
+       enum ulp_ha_mgr_app_type app_type;
+       int32_t timeout;
+       int32_t rc;
+
+       rc = ulp_ha_mgr_state_get(ulp_ctx, &curr_state);
+       if (rc) {
+               BNXT_TF_DBG(ERR, "On Close: Failed(%d) to get HA state\n", rc);
+               return -EINVAL;
+       }
+
+       rc = ulp_ha_mgr_app_type_get(ulp_ctx, &app_type);
+       if (rc) {
+               BNXT_TF_DBG(ERR, "On Close: Failed to get the app type.\n");
+               return -EINVAL;
+       }
+
+       if (curr_state == ULP_HA_STATE_PRIM_RUN &&
+           app_type == ULP_HA_APP_TYPE_PRIM) {
+               /*
+                * Only the primary is running, so a close effectively moves the
+                * system back to INIT.
+                */
+               next_state = ULP_HA_STATE_INIT;
+               ulp_ha_mgr_state_set(ulp_ctx, next_state);
+               BNXT_TF_DBG(INFO, "On Close: PRIM[PRIM_RUN] => [INIT]\n");
+       } else if (curr_state == ULP_HA_STATE_PRIM_SEC_RUN &&
+                 app_type == ULP_HA_APP_TYPE_PRIM) {
+               /*
+                * While both are running, the primary received a close.
+                * Cleanup the flows, set the COPY state, and wait for the
+                * secondary to become the Primary.
+                */
+               BNXT_TF_DBG(INFO,
+                           "On Close: PRIM[PRIM_SEC_RUN] flushing flows.\n");
+
+               ulp_flow_db_flush_flows(ulp_ctx, BNXT_ULP_FDB_TYPE_REGULAR);
+               ulp_ha_mgr_state_set(ulp_ctx, ULP_HA_STATE_SEC_TIMER_COPY);
+
+               /*
+                * TODO: This needs to be bounded in case the other system does
+                * not move to PRIM_RUN.
+                */
+               BNXT_TF_DBG(INFO,
+                           "On Close: PRIM[PRIM_SEC_RUN] => [Copy], enter wait.\n");
+               timeout = ULP_HA_WAIT_TIMEOUT;
+               do {
+                       rte_delay_ms(ULP_HA_WAIT_TIME);
+                       rc = ulp_ha_mgr_state_get(ulp_ctx, &poll_state);
+                       if (rc) {
+                               BNXT_TF_DBG(ERR,
+                                           "Failed to get HA state on Close (%d)\n",
+                                           rc);
+                               goto cleanup;
+                       }
+                       timeout -= ULP_HA_WAIT_TIME;
+                       BNXT_TF_DBG(INFO,
+                                   "On Close: Waiting %d ms for PRIM_RUN\n",
+                                   timeout);
+               } while (poll_state != ULP_HA_STATE_PRIM_RUN && timeout > 0);
+
+               if (timeout <= 0) {
+                       BNXT_TF_DBG(ERR, "On Close: SEC[COPY] Timed out\n");
+                       goto cleanup;
+               }
+
+               BNXT_TF_DBG(INFO, "On Close: PRIM[PRIM_SEC_RUN] => [COPY]\n");
+       } else if (curr_state == ULP_HA_STATE_PRIM_SEC_RUN &&
+                  app_type == ULP_HA_APP_TYPE_SEC) {
+               /*
+                * While both are running, the secondary unexpectedly received a
+                * close.  Cancel the timer, set the state to Primary RUN since
+                * it is the only one running.
+                */
+               ulp_ha_mgr_timer_cancel(ulp_ctx);
+               ulp_ha_mgr_state_set(ulp_ctx, ULP_HA_STATE_PRIM_RUN);
+
+               BNXT_TF_DBG(INFO, "On Close: SEC[PRIM_SEC_RUN] => [PRIM_RUN]\n");
+       } else if (curr_state == ULP_HA_STATE_SEC_TIMER_COPY &&
+                  app_type == ULP_HA_APP_TYPE_SEC) {
+               /*
+                * While both were running and the Secondary went into copy,
+                * secondary received a close.  Wait until the former Primary
+                * clears the copy stage, close, and set to INIT.
+                */
+               BNXT_TF_DBG(INFO, "On Close: SEC[COPY] wait for PRIM_RUN\n");
+
+               timeout = ULP_HA_WAIT_TIMEOUT;
+               do {
+                       rte_delay_ms(ULP_HA_WAIT_TIME);
+                       rc = ulp_ha_mgr_state_get(ulp_ctx, &poll_state);
+                       if (rc) {
+                               BNXT_TF_DBG(ERR,
+                                           "Failed to get HA state on Close (%d)\n",
+                                           rc);
+                               goto cleanup;
+                       }
+
+                       timeout -= ULP_HA_WAIT_TIME;
+                       BNXT_TF_DBG(INFO,
+                                   "On Close: Waiting %d ms for PRIM_RUN\n",
+                                   timeout);
+               } while (poll_state != ULP_HA_STATE_PRIM_RUN &&
+                        timeout >= 0);
+
+               if (timeout <= 0) {
+                       BNXT_TF_DBG(ERR,
+                                   "On Close: SEC[COPY] Timed out\n");
+                       goto cleanup;
+               }
+
+               next_state = ULP_HA_STATE_INIT;
+               rc = ulp_ha_mgr_state_set(ulp_ctx, next_state);
+               if (rc) {
+                       BNXT_TF_DBG(ERR,
+                                   "On Close: Failed to set state to INIT(%x)\n",
+                                   rc);
+                       goto cleanup;
+               }
+
+               BNXT_TF_DBG(INFO,
+                           "On Close: SEC[COPY] => [INIT] after %d ms\n",
+                           ULP_HA_WAIT_TIMEOUT - timeout);
+       } else {
+               BNXT_TF_DBG(ERR, "On Close: Invalid type/state %d/%d\n",
+                           curr_state, app_type);
+       }
+cleanup:
+       return rc;
+}
+
+int32_t
+ulp_ha_mgr_region_get(struct bnxt_ulp_context *ulp_ctx,
+                     enum ulp_ha_mgr_region *region)
+{
+       struct bnxt_ulp_ha_mgr_info *ha_info;
+
+       if (ulp_ctx == NULL || region == NULL) {
+               BNXT_TF_DBG(ERR, "Invalid params in ha region get.\n");
+               return -EINVAL;
+       }
+
+       ha_info = bnxt_ulp_cntxt_ptr2_ha_info_get(ulp_ctx);
+       if (ha_info == NULL) {
+               BNXT_TF_DBG(ERR, "Unable to get ha info\n");
+               return -EINVAL;
+       }
+       *region = ha_info->region;
+
+       return 0;
+}
diff --git a/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.h b/drivers/net/bnxt/tf_ulp/ulp_ha_mgr.h
new file mode 100644 (file)
index 0000000..7935115
--- /dev/null
@@ -0,0 +1,67 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2014-2021 Broadcom
+ * All rights reserved.
+ */
+
+#ifndef _ULP_HA_MGR_H_
+#define _ULP_HA_MGR_H_
+
+#include "bnxt_ulp.h"
+
+enum ulp_ha_mgr_state {
+       ULP_HA_STATE_INIT,
+       ULP_HA_STATE_PRIM_RUN,
+       ULP_HA_STATE_PRIM_SEC_RUN,
+       ULP_HA_STATE_SEC_TIMER_COPY,
+       ULP_HA_PRIM_CLOSE
+};
+
+enum ulp_ha_mgr_app_type {
+       ULP_HA_APP_TYPE_NONE,
+       ULP_HA_APP_TYPE_PRIM,
+       ULP_HA_APP_TYPE_SEC
+};
+
+enum ulp_ha_mgr_region {
+       ULP_HA_REGION_LOW,
+       ULP_HA_REGION_HI
+};
+
+struct bnxt_ulp_ha_mgr_info {
+       enum ulp_ha_mgr_app_type app_type;
+       enum ulp_ha_mgr_region region;
+       uint32_t flags;
+       pthread_mutex_t ha_lock;
+};
+
+bool
+ulp_ha_mgr_is_enabled(struct bnxt_ulp_context *ulp_ctx);
+
+int32_t
+ulp_ha_mgr_enable(struct bnxt_ulp_context *ulp_ctx);
+
+int32_t
+ulp_ha_mgr_init(struct bnxt_ulp_context *ulp_ctx);
+
+void
+ulp_ha_mgr_deinit(struct bnxt_ulp_context *ulp_ctx);
+
+int32_t
+ulp_ha_mgr_app_type_get(struct bnxt_ulp_context *ulp_ctx,
+                       enum ulp_ha_mgr_app_type *app_type);
+
+int32_t
+ulp_ha_mgr_state_get(struct bnxt_ulp_context *ulp_ctx,
+                    enum ulp_ha_mgr_state *state);
+
+int32_t
+ulp_ha_mgr_open(struct bnxt_ulp_context *ulp_ctx);
+
+int32_t
+ulp_ha_mgr_close(struct bnxt_ulp_context *ulp_ctx);
+
+int32_t
+ulp_ha_mgr_region_get(struct bnxt_ulp_context *ulp_ctx,
+                     enum ulp_ha_mgr_region *region);
+
+#endif /* _ULP_HA_MGR_H_*/
index e2404c3..05a43b6 100644 (file)
@@ -19,6 +19,7 @@
 #include "tf_util.h"
 #include "ulp_template_db_tbl.h"
 #include "ulp_port_db.h"
+#include "ulp_ha_mgr.h"
 
 static uint8_t mapper_fld_zeros[16] = { 0 };
 
@@ -419,7 +420,7 @@ ulp_mapper_ident_fields_get(struct bnxt_ulp_mapper_parms *mparms,
 }
 
 static inline int32_t
-ulp_mapper_tcam_entry_free(struct bnxt_ulp_context *ulp  __rte_unused,
+ulp_mapper_tcam_entry_free(struct bnxt_ulp_context *ulp,
                           struct tf *tfp,
                           struct ulp_flow_db_res_params *res)
 {
@@ -429,6 +430,30 @@ ulp_mapper_tcam_entry_free(struct bnxt_ulp_context *ulp  __rte_unused,
                .idx            = (uint16_t)res->resource_hndl
        };
 
+       /* If HA is enabled, we may have to remap the TF Type */
+       if (bnxt_ulp_cntxt_ha_enabled(ulp)) {
+               enum ulp_ha_mgr_region region;
+               int32_t rc;
+
+               switch (res->resource_type) {
+               case TF_TCAM_TBL_TYPE_WC_TCAM_HIGH:
+               case TF_TCAM_TBL_TYPE_WC_TCAM_LOW:
+                       rc = ulp_ha_mgr_region_get(ulp, &region);
+                       if (rc)
+                               /* Log this, but assume region is correct */
+                               BNXT_TF_DBG(ERR,
+                                           "Unable to get HA region (%d)\n",
+                                           rc);
+                       else
+                               fparms.tcam_tbl_type =
+                                       (region == ULP_HA_REGION_LOW) ?
+                                       TF_TCAM_TBL_TYPE_WC_TCAM_LOW :
+                                       TF_TCAM_TBL_TYPE_WC_TCAM_HIGH;
+                       break;
+               default:
+                       break;
+               }
+       }
        return tf_free_tcam_entry(tfp, &fparms);
 }
 
@@ -2904,10 +2929,12 @@ static int32_t
 ulp_mapper_app_glb_resource_info_init(struct bnxt_ulp_context *ulp_ctx,
                                      struct bnxt_ulp_mapper_data *mapper_data)
 {
+       struct tf_get_shared_tbl_increment_parms iparms;
        struct bnxt_ulp_glb_resource_info *glb_res;
        struct tf_get_session_info_parms sparms;
        uint32_t num_entries, i, dev_id, res;
        struct tf_resource_info *res_info;
+       uint32_t addend;
        uint64_t regval;
        enum tf_dir dir;
        int32_t rc = 0;
@@ -2915,13 +2942,11 @@ ulp_mapper_app_glb_resource_info_init(struct bnxt_ulp_context *ulp_ctx,
        uint8_t app_id;
 
        memset(&sparms, 0, sizeof(sparms));
-
        glb_res = bnxt_ulp_app_glb_resource_info_list_get(&num_entries);
        if (!glb_res || !num_entries) {
                BNXT_TF_DBG(ERR, "Invalid Arguments\n");
                return -EINVAL;
        }
-
        tfp = bnxt_ulp_cntxt_shared_tfp_get(ulp_ctx);
        if (!tfp) {
                BNXT_TF_DBG(ERR, "Failed to get tfp for app global init");
@@ -2958,12 +2983,29 @@ ulp_mapper_app_glb_resource_info_init(struct bnxt_ulp_context *ulp_ctx,
                        continue;
                dir = glb_res[i].direction;
                res = glb_res[i].resource_type;
+               addend = 1;
 
                switch (glb_res[i].resource_func) {
                case BNXT_ULP_RESOURCE_FUNC_IDENTIFIER:
                        res_info = &sparms.session_info.ident[dir].info[res];
                        break;
                case BNXT_ULP_RESOURCE_FUNC_INDEX_TABLE:
+                       /*
+                        * Tables may have various strides for the allocations.
+                        * Need to account.
+                        */
+                       memset(&iparms, 0, sizeof(iparms));
+                       iparms.dir = dir;
+                       iparms.type = res;
+                       rc = tf_get_shared_tbl_increment(tfp, &iparms);
+                       if (rc) {
+                               BNXT_TF_DBG(ERR,
+                                           "Failed to get addend for %s[%s] rc=(%d)\n",
+                                           tf_tbl_type_2_str(res),
+                                           tf_dir_2_str(dir), rc);
+                               return rc;
+                       }
+                       addend = iparms.increment_cnt;
                        res_info = &sparms.session_info.tbl[dir].info[res];
                        break;
                case BNXT_ULP_RESOURCE_FUNC_TCAM_TABLE:
@@ -2977,10 +3019,8 @@ ulp_mapper_app_glb_resource_info_init(struct bnxt_ulp_context *ulp_ctx,
                                    glb_res[i].resource_func);
                        continue;
                }
-
                regval = tfp_cpu_to_be_64((uint64_t)res_info->start);
-               res_info->start++;
-
+               res_info->start += addend;
                /*
                 * All resources written to the global regfile are shared for
                 * this function.