From: Kishore Padmanabha Date: Fri, 11 Sep 2020 01:55:39 +0000 (-0700) Subject: net/bnxt: fix port stop process and cleanup resources X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=09b23f8b9df6;p=dpdk.git net/bnxt: fix port stop process and cleanup resources The port deinitialization now cleans up all the resources properly. If all the ports are stopped then ULP context is freed. Added fix to update the correct tfp pointer in the ULP context with the changes to support multi control channels. Fixes: 70e64b27af5b ("net/bnxt: support ULP session manager cleanup") Cc: stable@dpdk.org Signed-off-by: Kishore Padmanabha Reviewed-by: Mike Baucom Reviewed-by: Shahaji Bhosle --- diff --git a/drivers/net/bnxt/bnxt.h b/drivers/net/bnxt/bnxt.h index ef5824cf9a..f0b0800151 100644 --- a/drivers/net/bnxt/bnxt.h +++ b/drivers/net/bnxt/bnxt.h @@ -822,8 +822,7 @@ struct bnxt_vf_representor { uint16_t dflt_vnic_id; uint16_t svif; uint16_t vfr_tx_cfa_action; - uint32_t rep2vf_flow_id; - uint32_t vf2rep_flow_id; + uint32_t dpdk_port_id; /* Private data store of associated PF/Trusted VF */ struct rte_eth_dev *parent_dev; uint8_t mac_addr[RTE_ETHER_ADDR_LEN]; @@ -893,11 +892,14 @@ extern int bnxt_logtype_driver; PMD_DRV_LOG_RAW(level, fmt, ## args) extern const struct rte_flow_ops bnxt_ulp_rte_flow_ops; -int32_t bnxt_ulp_init(struct bnxt *bp); -void bnxt_ulp_deinit(struct bnxt *bp); +int32_t bnxt_ulp_port_init(struct bnxt *bp); +void bnxt_ulp_port_deinit(struct bnxt *bp); int32_t bnxt_ulp_create_df_rules(struct bnxt *bp); void bnxt_ulp_destroy_df_rules(struct bnxt *bp, bool global); - +int32_t +bnxt_ulp_create_vfr_default_rules(struct rte_eth_dev *vfr_ethdev); +int32_t +bnxt_ulp_delete_vfr_default_rules(struct bnxt_vf_representor *vfr); uint16_t bnxt_get_vnic_id(uint16_t port, enum bnxt_ulp_intf_type type); uint16_t bnxt_get_svif(uint16_t port_id, bool func_svif, enum bnxt_ulp_intf_type type); diff --git a/drivers/net/bnxt/bnxt_ethdev.c b/drivers/net/bnxt/bnxt_ethdev.c index 2b2b2eeb80..073412de25 100644 --- a/drivers/net/bnxt/bnxt_ethdev.c +++ b/drivers/net/bnxt/bnxt_ethdev.c @@ -1236,6 +1236,11 @@ static int bnxt_dev_start_op(struct rte_eth_dev *eth_dev) if (rc) goto error; + /* Initialize bnxt ULP port details */ + rc = bnxt_ulp_port_init(bp); + if (rc) + goto error; + eth_dev->rx_pkt_burst = bnxt_receive_function(eth_dev); eth_dev->tx_pkt_burst = bnxt_transmit_function(eth_dev); @@ -1243,8 +1248,6 @@ static int bnxt_dev_start_op(struct rte_eth_dev *eth_dev) bnxt_schedule_fw_health_check(bp); pthread_mutex_unlock(&bp->def_cp_lock); - bnxt_ulp_init(bp); - return 0; error: @@ -1306,8 +1309,8 @@ static void bnxt_dev_stop_op(struct rte_eth_dev *eth_dev) /* disable uio/vfio intr/eventfd mapping */ rte_intr_disable(intr_handle); - bnxt_ulp_destroy_df_rules(bp, false); - bnxt_ulp_deinit(bp); + /* delete the bnxt ULP port details */ + bnxt_ulp_port_deinit(bp); bnxt_cancel_fw_health_check(bp); @@ -1601,8 +1604,6 @@ static int bnxt_promiscuous_disable_op(struct rte_eth_dev *eth_dev) if (rc != 0) vnic->flags = old_flags; - bnxt_ulp_create_df_rules(bp); - return rc; } @@ -3716,9 +3717,14 @@ bnxt_filter_ctrl_op(struct rte_eth_dev *dev, struct bnxt *bp = dev->data->dev_private; int ret = 0; + if (!bp) + return -EIO; + if (BNXT_ETH_DEV_IS_REPRESENTOR(dev)) { struct bnxt_vf_representor *vfr = dev->data->dev_private; bp = vfr->parent_dev->data->dev_private; + if (!bp) + return -EIO; } ret = is_bnxt_in_error(bp); diff --git a/drivers/net/bnxt/bnxt_reps.c b/drivers/net/bnxt/bnxt_reps.c index b4ed5d6ef2..00e44bce54 100644 --- a/drivers/net/bnxt/bnxt_reps.c +++ b/drivers/net/bnxt/bnxt_reps.c @@ -267,66 +267,37 @@ static int bnxt_tf_vfr_alloc(struct rte_eth_dev *vfr_ethdev) struct bnxt_vf_representor *vfr = vfr_ethdev->data->dev_private; struct rte_eth_dev *parent_dev = vfr->parent_dev; struct bnxt *parent_bp = parent_dev->data->dev_private; - uint16_t vfr_port_id = vfr_ethdev->data->port_id; - struct ulp_tlv_param param_list[] = { - { - .type = BNXT_ULP_DF_PARAM_TYPE_DEV_PORT_ID, - .length = 2, - .value = {(vfr_port_id >> 8) & 0xff, vfr_port_id & 0xff} - }, - { - .type = BNXT_ULP_DF_PARAM_TYPE_LAST, - .length = 0, - .value = {0} - } - }; - ulp_port_db_dev_port_intf_update(parent_bp->ulp_ctx, vfr_ethdev); + if (!parent_bp || !parent_bp->ulp_ctx) { + BNXT_TF_DBG(ERR, "Invalid arguments\n"); + return 0; + } - rc = ulp_default_flow_create(parent_dev, param_list, - BNXT_ULP_DF_TPL_VFREP_TO_VF, - &vfr->rep2vf_flow_id); + /* Update the ULP portdata base with the new VFR interface */ + rc = ulp_port_db_dev_port_intf_update(parent_bp->ulp_ctx, vfr_ethdev); if (rc) { - BNXT_TF_DBG(DEBUG, - "Default flow rule creation for VFR->VF failed!\n"); - goto err; + BNXT_TF_DBG(ERR, "Failed to update ulp port details vfr:%u\n", + vfr->vf_id); + return rc; } - BNXT_TF_DBG(DEBUG, "*** Default flow rule created for VFR->VF! ***\n"); - BNXT_TF_DBG(DEBUG, "rep2vf_flow_id = %d\n", vfr->rep2vf_flow_id); - rc = ulp_default_flow_db_cfa_action_get(parent_bp->ulp_ctx, - vfr->rep2vf_flow_id, - &vfr->vfr_tx_cfa_action); - if (rc) { - BNXT_TF_DBG(DEBUG, - "Failed to get action_ptr for VFR->VF dflt rule\n"); - goto rep2vf_free; - } - BNXT_TF_DBG(DEBUG, "tx_cfa_action = %d\n", vfr->vfr_tx_cfa_action); - rc = ulp_default_flow_create(parent_dev, param_list, - BNXT_ULP_DF_TPL_VF_TO_VFREP, - &vfr->vf2rep_flow_id); + /* Create the default rules for the VFR */ + rc = bnxt_ulp_create_vfr_default_rules(vfr_ethdev); if (rc) { - BNXT_TF_DBG(DEBUG, - "Default flow rule creation for VF->VFR failed!\n"); - goto rep2vf_free; + BNXT_TF_DBG(ERR, "Failed to create VFR default rules vfr:%u\n", + vfr->vf_id); + return rc; } - - BNXT_TF_DBG(DEBUG, "*** Default flow rule created for VF->VFR! ***\n"); - BNXT_TF_DBG(DEBUG, "vfr2rep_flow_id = %d\n", vfr->vf2rep_flow_id); - + /* update the port id so you can backtrack to ethdev */ + vfr->dpdk_port_id = vfr_ethdev->data->port_id; rc = bnxt_hwrm_cfa_vfr_alloc(parent_bp, vfr->vf_id); - if (rc) - goto vf2rep_free; - - return 0; + if (rc) { + BNXT_TF_DBG(ERR, "Failed in hwrm vfr alloc vfr:%u rc=%d\n", + vfr->vf_id, rc); + (void)bnxt_ulp_delete_vfr_default_rules(vfr); + } -vf2rep_free: - ulp_default_flow_destroy(vfr->parent_dev, vfr->vf2rep_flow_id); -rep2vf_free: - ulp_default_flow_destroy(vfr->parent_dev, vfr->rep2vf_flow_id); -err: - return -EIO; + return rc; } static int bnxt_vfr_alloc(struct rte_eth_dev *vfr_ethdev) @@ -337,7 +308,7 @@ static int bnxt_vfr_alloc(struct rte_eth_dev *vfr_ethdev) if (!vfr || !vfr->parent_dev) { PMD_DRV_LOG(ERR, - "No memory allocated for representor\n"); + "No memory allocated for representor\n"); return -ENOMEM; } @@ -391,14 +362,12 @@ int bnxt_vf_rep_dev_start_op(struct rte_eth_dev *eth_dev) rep_info = &parent_bp->rep_info[rep_bp->vf_id]; pthread_mutex_lock(&rep_info->vfr_start_lock); - if (rep_info->conduit_valid) { - pthread_mutex_unlock(&rep_info->vfr_start_lock); - return 0; - } - rc = bnxt_get_dflt_vnic_svif(parent_bp, rep_bp); - if (rc || !rep_info->conduit_valid) { - pthread_mutex_unlock(&rep_info->vfr_start_lock); - return rc; + if (!rep_info->conduit_valid) { + rc = bnxt_get_dflt_vnic_svif(parent_bp, rep_bp); + if (rc || !rep_info->conduit_valid) { + pthread_mutex_unlock(&rep_info->vfr_start_lock); + return rc; + } } pthread_mutex_unlock(&rep_info->vfr_start_lock); @@ -417,21 +386,7 @@ int bnxt_vf_rep_dev_start_op(struct rte_eth_dev *eth_dev) static int bnxt_tf_vfr_free(struct bnxt_vf_representor *vfr) { - int rc = 0; - - rc = ulp_default_flow_destroy(vfr->parent_dev, - vfr->rep2vf_flow_id); - if (rc) - PMD_DRV_LOG(ERR, - "default flow destroy failed rep2vf flowid: %d\n", - vfr->rep2vf_flow_id); - rc = ulp_default_flow_destroy(vfr->parent_dev, - vfr->vf2rep_flow_id); - if (rc) - PMD_DRV_LOG(ERR, - "default flow destroy failed vf2rep flowid: %d\n", - vfr->vf2rep_flow_id); - return 0; + return bnxt_ulp_delete_vfr_default_rules(vfr); } static int bnxt_vfr_free(struct bnxt_vf_representor *vfr) @@ -458,7 +413,6 @@ static int bnxt_vfr_free(struct bnxt_vf_representor *vfr) PMD_DRV_LOG(ERR, "Failed to free representor %d in FW\n", vfr->vf_id); - return rc; } PMD_DRV_LOG(DEBUG, "freed representor %d in FW\n", diff --git a/drivers/net/bnxt/tf_ulp/bnxt_ulp.c b/drivers/net/bnxt/tf_ulp/bnxt_ulp.c index 0d4a455134..21baed0488 100644 --- a/drivers/net/bnxt/tf_ulp/bnxt_ulp.c +++ b/drivers/net/bnxt/tf_ulp/bnxt_ulp.c @@ -32,23 +32,22 @@ static pthread_mutex_t bnxt_ulp_global_mutex = PTHREAD_MUTEX_INITIALIZER; /* * Allow the deletion of context only for the bnxt device that - * created the session - * TBD - The implementation of the function should change to - * using the reference count once tf_session_attach functionality - * is fixed. + * created the session. */ bool ulp_ctx_deinit_allowed(void *ptr) { struct bnxt *bp = (struct bnxt *)ptr; - if (!bp) - return 0; + if (!bp || !bp->ulp_ctx || !bp->ulp_ctx->cfg_data) + return false; - if (&bp->tfp == bp->ulp_ctx->g_tfp) - return 1; + if (!bp->ulp_ctx->cfg_data->ref_cnt) { + BNXT_TF_DBG(DEBUG, "ulp ctx shall initiate deinit\n"); + return true; + } - return 0; + return false; } /* @@ -155,8 +154,10 @@ ulp_ctx_session_open(struct bnxt *bp, params.ctrl_chan_name, rc); return -EINVAL; } - session->session_opened = 1; - session->g_tfp = &bp->tfp; + if (!session->session_opened) { + session->session_opened = 1; + session->g_tfp = &bp->tfp; + } return rc; } @@ -173,7 +174,6 @@ ulp_ctx_session_close(struct bnxt *bp, tf_close_session(&bp->tfp); session->session_opened = 0; session->g_tfp = NULL; - bp->ulp_ctx->g_tfp = NULL; } static void @@ -285,10 +285,6 @@ ulp_eem_tbl_scope_deinit(struct bnxt *bp, struct bnxt_ulp_context *ulp_ctx) if (!ulp_ctx || !ulp_ctx->cfg_data) return -EINVAL; - /* Free the resources for the last device */ - if (!ulp_ctx_deinit_allowed(bp)) - return rc; - tfp = bnxt_ulp_cntxt_tfp_get(ulp_ctx); if (!tfp) { BNXT_TF_DBG(ERR, "Failed to get the truflow pointer\n"); @@ -331,11 +327,6 @@ static int32_t ulp_ctx_deinit(struct bnxt *bp, struct bnxt_ulp_session_state *session) { - if (!session || !bp) { - BNXT_TF_DBG(ERR, "Invalid Arguments\n"); - return -EINVAL; - } - /* close the tf session */ ulp_ctx_session_close(bp, session); @@ -356,11 +347,6 @@ ulp_ctx_init(struct bnxt *bp, struct bnxt_ulp_data *ulp_data; int32_t rc = 0; - if (!session || !bp) { - BNXT_TF_DBG(ERR, "Invalid Arguments\n"); - return -EINVAL; - } - /* Allocate memory to hold ulp context data. */ ulp_data = rte_zmalloc("bnxt_ulp_data", sizeof(struct bnxt_ulp_data), 0); @@ -378,11 +364,12 @@ ulp_ctx_init(struct bnxt *bp, /* Open the ulp session. */ rc = ulp_ctx_session_open(bp, session); if (rc) { + session->session_opened = 1; (void)ulp_ctx_deinit(bp, session); return rc; } - bnxt_ulp_cntxt_tfp_set(bp->ulp_ctx, session->g_tfp); + bnxt_ulp_cntxt_tfp_set(bp->ulp_ctx, &bp->tfp); return rc; } @@ -395,7 +382,7 @@ ulp_dparms_init(struct bnxt *bp, uint32_t dev_id; if (!bp->max_num_kflows) - return -EINVAL; + return 0; if (bnxt_ulp_cntxt_dev_id_get(ulp_ctx, &dev_id)) { BNXT_TF_DBG(DEBUG, "Failed to get device id\n"); @@ -445,51 +432,37 @@ ulp_dparms_dev_port_intf_update(struct bnxt *bp, } static int32_t -ulp_ctx_attach(struct bnxt_ulp_context *ulp_ctx, +ulp_ctx_attach(struct bnxt *bp, struct bnxt_ulp_session_state *session) { - if (!ulp_ctx || !session) { - BNXT_TF_DBG(ERR, "Invalid Arguments\n"); - return -EINVAL; - } + int32_t rc = 0; /* Increment the ulp context data reference count usage. */ - ulp_ctx->cfg_data = session->cfg_data; - ulp_ctx->cfg_data->ref_cnt++; + bp->ulp_ctx->cfg_data = session->cfg_data; + bp->ulp_ctx->cfg_data->ref_cnt++; - /* TBD call TF_session_attach. */ - ulp_ctx->g_tfp = session->g_tfp; - return 0; -} - -static int32_t -ulp_ctx_detach(struct bnxt *bp, - struct bnxt_ulp_session_state *session) -{ - struct bnxt_ulp_context *ulp_ctx; + /* update the session details in bnxt tfp */ + bp->tfp.session = session->g_tfp->session; - if (!bp || !session) { - BNXT_TF_DBG(ERR, "Invalid Arguments\n"); - return -EINVAL; + /* Create a TF Client */ + rc = ulp_ctx_session_open(bp, session); + if (rc) { + PMD_DRV_LOG(ERR, "Failed to open ctxt session, rc:%d\n", rc); + bp->tfp.session = NULL; + return rc; } - ulp_ctx = bp->ulp_ctx; - if (!ulp_ctx->cfg_data) - return 0; - - /* TBD call TF_session_detach */ + bnxt_ulp_cntxt_tfp_set(bp->ulp_ctx, &bp->tfp); + return rc; +} - /* Increment the ulp context data reference count usage. */ - if (ulp_ctx->cfg_data->ref_cnt >= 1) { - ulp_ctx->cfg_data->ref_cnt--; - if (ulp_ctx_deinit_allowed(bp)) - ulp_ctx_deinit(bp, session); - ulp_ctx->cfg_data = NULL; - ulp_ctx->g_tfp = NULL; - return 0; +static void +ulp_ctx_detach(struct bnxt *bp) +{ + if (bp->tfp.session) { + tf_close_session(&bp->tfp); + bp->tfp.session = NULL; } - BNXT_TF_DBG(ERR, "context deatach on invalid data\n"); - return 0; } /* @@ -542,6 +515,7 @@ ulp_session_init(struct bnxt *bp, struct rte_pci_device *pci_dev; struct rte_pci_addr *pci_addr; struct bnxt_ulp_session_state *session; + int rc = 0; if (!bp) return NULL; @@ -567,7 +541,12 @@ ulp_session_init(struct bnxt *bp, /* Add it to the queue */ session->pci_info.domain = pci_addr->domain; session->pci_info.bus = pci_addr->bus; - pthread_mutex_init(&session->bnxt_ulp_mutex, NULL); + rc = pthread_mutex_init(&session->bnxt_ulp_mutex, NULL); + if (rc) { + BNXT_TF_DBG(ERR, "mutex create failed\n"); + pthread_mutex_unlock(&bnxt_ulp_global_mutex); + return NULL; + } STAILQ_INSERT_TAIL(&bnxt_ulp_session_list, session, next); } @@ -643,80 +622,122 @@ bnxt_ulp_global_cfg_update(struct bnxt *bp, return rc; } +/* Internal function to delete all the flows belonging to the given port */ +static void +bnxt_ulp_flush_port_flows(struct bnxt *bp) +{ + uint16_t func_id; + + func_id = bnxt_get_fw_func_id(bp->eth_dev->data->port_id, + BNXT_ULP_INTF_TYPE_INVALID); + ulp_flow_db_function_flow_flush(bp->ulp_ctx, func_id); +} + +/* Internal function to delete the VFR default flows */ +static void +bnxt_ulp_destroy_vfr_default_rules(struct bnxt *bp, bool global) +{ + struct bnxt_ulp_vfr_rule_info *info; + uint8_t port_id; + struct rte_eth_dev *vfr_eth_dev; + struct bnxt_vf_representor *vfr_bp; + + if (!BNXT_TRUFLOW_EN(bp) || BNXT_ETH_DEV_IS_REPRESENTOR(bp->eth_dev)) + return; + + if (!bp->ulp_ctx || !bp->ulp_ctx->cfg_data) + return; + + /* Delete default rules for all ports */ + for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++) { + info = &bp->ulp_ctx->cfg_data->vfr_rule_info[port_id]; + if (!info->valid) + continue; + + if (!global && info->parent_port_id != + bp->eth_dev->data->port_id) + continue; + + /* Destroy the flows */ + ulp_default_flow_destroy(bp->eth_dev, info->rep2vf_flow_id); + ulp_default_flow_destroy(bp->eth_dev, info->vf2rep_flow_id); + /* Clean up the tx action pointer */ + vfr_eth_dev = &rte_eth_devices[port_id]; + if (vfr_eth_dev) { + vfr_bp = vfr_eth_dev->data->dev_private; + vfr_bp->vfr_tx_cfa_action = 0; + } + memset(info, 0, sizeof(struct bnxt_ulp_vfr_rule_info)); + } +} + /* - * When a port is initialized by dpdk. This functions is called - * and this function initializes the ULP context and rest of the + * When a port is deinit'ed by dpdk. This function is called + * and this function clears the ULP context and rest of the * infrastructure associated with it. */ -int32_t -bnxt_ulp_init(struct bnxt *bp) +static void +bnxt_ulp_deinit(struct bnxt *bp, + struct bnxt_ulp_session_state *session) { - struct bnxt_ulp_session_state *session; - bool init; - int rc; + if (!bp->ulp_ctx || !bp->ulp_ctx->cfg_data) + return; - if (!BNXT_TRUFLOW_EN(bp)) - return 0; + /* clean up default flows */ + bnxt_ulp_destroy_df_rules(bp, true); - if (bp->ulp_ctx) { - BNXT_TF_DBG(DEBUG, "ulp ctx already allocated\n"); - return -EINVAL; - } + /* clean up default VFR flows */ + bnxt_ulp_destroy_vfr_default_rules(bp, true); - /* - * Multiple uplink ports can be associated with a single vswitch. - * Make sure only the port that is started first will initialize - * the TF session. - */ - session = ulp_session_init(bp, &init); - if (!session) { - BNXT_TF_DBG(ERR, "Failed to initialize the tf session\n"); - return -EINVAL; - } + /* clean up regular flows */ + ulp_flow_db_flush_flows(bp->ulp_ctx, BNXT_ULP_REGULAR_FLOW_TABLE); - bp->ulp_ctx = rte_zmalloc("bnxt_ulp_ctx", - sizeof(struct bnxt_ulp_context), 0); - if (!bp->ulp_ctx) { - BNXT_TF_DBG(ERR, "Failed to allocate ulp ctx\n"); - ulp_session_deinit(session); - return -ENOMEM; - } + /* cleanup the eem table scope */ + ulp_eem_tbl_scope_deinit(bp, bp->ulp_ctx); - /* - * If ULP is already initialized for a specific domain then simply - * assign the ulp context to this rte_eth_dev. - */ - if (init) { - rc = ulp_ctx_attach(bp->ulp_ctx, session); - if (rc) { - BNXT_TF_DBG(ERR, - "Failed to attach the ulp context\n"); - ulp_session_deinit(session); - rte_free(bp->ulp_ctx); - return rc; - } + /* cleanup the flow database */ + ulp_flow_db_deinit(bp->ulp_ctx); - /* Update bnxt driver flags */ - rc = ulp_dparms_dev_port_intf_update(bp, bp->ulp_ctx); - if (rc) { - BNXT_TF_DBG(ERR, "Failed to update driver flags\n"); - ulp_ctx_detach(bp, session); - ulp_session_deinit(session); - rte_free(bp->ulp_ctx); - return rc; - } + /* Delete the Mark database */ + ulp_mark_db_deinit(bp->ulp_ctx); - /* update the port database */ - rc = ulp_port_db_dev_port_intf_update(bp->ulp_ctx, bp->eth_dev); - if (rc) { - BNXT_TF_DBG(ERR, - "Failed to update port database\n"); - ulp_ctx_detach(bp, session); - ulp_session_deinit(session); - rte_free(bp->ulp_ctx); - } - return rc; - } + /* cleanup the ulp mapper */ + ulp_mapper_deinit(bp->ulp_ctx); + + /* Delete the Flow Counter Manager */ + ulp_fc_mgr_deinit(bp->ulp_ctx); + + /* Delete the Port database */ + ulp_port_db_deinit(bp->ulp_ctx); + + /* Disable NAT feature */ + (void)bnxt_ulp_global_cfg_update(bp, TF_DIR_RX, TF_TUNNEL_ENCAP, + TF_TUNNEL_ENCAP_NAT, + (BNXT_ULP_NAT_INNER_L2_HEADER_SMAC | + BNXT_ULP_NAT_INNER_L2_HEADER_DMAC), + 0); + + (void)bnxt_ulp_global_cfg_update(bp, TF_DIR_TX, TF_TUNNEL_ENCAP, + TF_TUNNEL_ENCAP_NAT, + (BNXT_ULP_NAT_INNER_L2_HEADER_SMAC | + BNXT_ULP_NAT_INNER_L2_HEADER_DMAC), + 0); + + /* Delete the ulp context and tf session and free the ulp context */ + ulp_ctx_deinit(bp, session); + BNXT_TF_DBG(DEBUG, "ulp ctx has been deinitialized\n"); +} + +/* + * When a port is initialized by dpdk. This functions is called + * and this function initializes the ULP context and rest of the + * infrastructure associated with it. + */ +static int32_t +bnxt_ulp_init(struct bnxt *bp, + struct bnxt_ulp_session_state *session) +{ + int rc; /* Allocate and Initialize the ulp context. */ rc = ulp_ctx_init(bp, session); @@ -727,25 +748,15 @@ bnxt_ulp_init(struct bnxt *bp) /* Initialize ulp dparms with values devargs passed */ rc = ulp_dparms_init(bp, bp->ulp_ctx); - - /* create the port database */ - rc = ulp_port_db_init(bp->ulp_ctx, bp->port_cnt); - if (rc) { - BNXT_TF_DBG(ERR, "Failed to create the port database\n"); - goto jump_to_error; - } - - /* Update bnxt driver flags */ - rc = ulp_dparms_dev_port_intf_update(bp, bp->ulp_ctx); if (rc) { - BNXT_TF_DBG(ERR, "Failed to update driver flags\n"); + BNXT_TF_DBG(ERR, "Failed to initialize the dparms\n"); goto jump_to_error; } - /* update the port database */ - rc = ulp_port_db_dev_port_intf_update(bp->ulp_ctx, bp->eth_dev); + /* create the port database */ + rc = ulp_port_db_init(bp->ulp_ctx, bp->port_cnt); if (rc) { - BNXT_TF_DBG(ERR, "Failed to update port database\n"); + BNXT_TF_DBG(ERR, "Failed to create the port database\n"); goto jump_to_error; } @@ -804,32 +815,131 @@ bnxt_ulp_init(struct bnxt *bp) BNXT_TF_DBG(ERR, "Failed to set tx global configuration\n"); goto jump_to_error; } - + BNXT_TF_DBG(DEBUG, "ulp ctx has been initialized\n"); return rc; jump_to_error: - bnxt_ulp_deinit(bp); - return -ENOMEM; + bnxt_ulp_deinit(bp, session); + return rc; } -/* Below are the access functions to access internal data of ulp context. */ +/* + * When a port is initialized by dpdk. This functions sets up + * the port specific details. + */ +int32_t +bnxt_ulp_port_init(struct bnxt *bp) +{ + struct bnxt_ulp_session_state *session; + bool initialized; + int32_t rc = 0; + + if (!bp || !BNXT_TRUFLOW_EN(bp)) + return rc; + + if (!BNXT_PF(bp) && !BNXT_VF_IS_TRUSTED(bp)) { + BNXT_TF_DBG(ERR, + "Skip ulp init for port: %d, not a TVF or PF\n", + bp->eth_dev->data->port_id); + return rc; + } + + if (bp->ulp_ctx) { + BNXT_TF_DBG(DEBUG, "ulp ctx already allocated\n"); + return rc; + } + + bp->ulp_ctx = rte_zmalloc("bnxt_ulp_ctx", + sizeof(struct bnxt_ulp_context), 0); + if (!bp->ulp_ctx) { + BNXT_TF_DBG(ERR, "Failed to allocate ulp ctx\n"); + return -ENOMEM; + } + + /* + * Multiple uplink ports can be associated with a single vswitch. + * Make sure only the port that is started first will initialize + * the TF session. + */ + session = ulp_session_init(bp, &initialized); + if (!session) { + BNXT_TF_DBG(ERR, "Failed to initialize the tf session\n"); + rc = -EIO; + goto jump_to_error; + } + + if (initialized) { + /* + * If ULP is already initialized for a specific domain then + * simply assign the ulp context to this rte_eth_dev. + */ + rc = ulp_ctx_attach(bp, session); + if (rc) { + BNXT_TF_DBG(ERR, "Failed to attach the ulp context\n"); + goto jump_to_error; + } + } else { + rc = bnxt_ulp_init(bp, session); + if (rc) { + BNXT_TF_DBG(ERR, "Failed to initialize the ulp init\n"); + goto jump_to_error; + } + } + + /* Update bnxt driver flags */ + rc = ulp_dparms_dev_port_intf_update(bp, bp->ulp_ctx); + if (rc) { + BNXT_TF_DBG(ERR, "Failed to update driver flags\n"); + goto jump_to_error; + } + + /* update the port database for the given interface */ + rc = ulp_port_db_dev_port_intf_update(bp->ulp_ctx, bp->eth_dev); + if (rc) { + BNXT_TF_DBG(ERR, "Failed to update port database\n"); + goto jump_to_error; + } + /* create the default rules */ + bnxt_ulp_create_df_rules(bp); + BNXT_TF_DBG(DEBUG, "ULP Port:%d created and initialized\n", + bp->eth_dev->data->port_id); + return rc; + +jump_to_error: + bnxt_ulp_port_deinit(bp); + return rc; +} /* - * When a port is deinit'ed by dpdk. This function is called - * and this function clears the ULP context and rest of the - * infrastructure associated with it. + * When a port is de-initialized by dpdk. This functions clears up + * the port specific details. */ void -bnxt_ulp_deinit(struct bnxt *bp) +bnxt_ulp_port_deinit(struct bnxt *bp) { - struct bnxt_ulp_session_state *session; - struct rte_pci_device *pci_dev; - struct rte_pci_addr *pci_addr; + struct bnxt_ulp_session_state *session; + struct rte_pci_device *pci_dev; + struct rte_pci_addr *pci_addr; if (!BNXT_TRUFLOW_EN(bp)) return; - /* Get the session first */ + if (!BNXT_PF(bp) && !BNXT_VF_IS_TRUSTED(bp)) { + BNXT_TF_DBG(ERR, + "Skip ULP deinit port:%d, not a TVF or PF\n", + bp->eth_dev->data->port_id); + return; + } + + if (!bp->ulp_ctx) { + BNXT_TF_DBG(DEBUG, "ulp ctx already de-allocated\n"); + return; + } + + BNXT_TF_DBG(DEBUG, "ULP Port:%d destroyed\n", + bp->eth_dev->data->port_id); + + /* Get the session details */ pci_dev = RTE_DEV_TO_PCI(bp->eth_dev->device); pci_addr = &pci_dev->addr; pthread_mutex_lock(&bnxt_ulp_global_mutex); @@ -837,57 +947,42 @@ bnxt_ulp_deinit(struct bnxt *bp) pthread_mutex_unlock(&bnxt_ulp_global_mutex); /* session not found then just exit */ - if (!session) + if (!session) { + /* Free the ulp context */ + rte_free(bp->ulp_ctx); + bp->ulp_ctx = NULL; return; + } - /* clean up default flows */ - bnxt_ulp_destroy_df_rules(bp, true); - - /* clean up regular flows */ - ulp_flow_db_flush_flows(bp->ulp_ctx, BNXT_ULP_REGULAR_FLOW_TABLE); - - /* cleanup the eem table scope */ - ulp_eem_tbl_scope_deinit(bp, bp->ulp_ctx); - - /* cleanup the flow database */ - ulp_flow_db_deinit(bp->ulp_ctx); - - /* Delete the Mark database */ - ulp_mark_db_deinit(bp->ulp_ctx); - - /* cleanup the ulp mapper */ - ulp_mapper_deinit(bp->ulp_ctx); - - /* Delete the Flow Counter Manager */ - ulp_fc_mgr_deinit(bp->ulp_ctx); - - /* Delete the Port database */ - ulp_port_db_deinit(bp->ulp_ctx); - - /* Disable NAT feature */ - (void)bnxt_ulp_global_cfg_update(bp, TF_DIR_RX, TF_TUNNEL_ENCAP, - TF_TUNNEL_ENCAP_NAT, - (BNXT_ULP_NAT_INNER_L2_HEADER_SMAC | - BNXT_ULP_NAT_INNER_L2_HEADER_DMAC), - 0); + /* Check the reference count to deinit or deattach*/ + if (bp->ulp_ctx->cfg_data && bp->ulp_ctx->cfg_data->ref_cnt) { + bp->ulp_ctx->cfg_data->ref_cnt--; + if (bp->ulp_ctx->cfg_data->ref_cnt) { + /* free the port details */ + /* Free the default flow rule associated to this port */ + bnxt_ulp_destroy_df_rules(bp, false); + bnxt_ulp_destroy_vfr_default_rules(bp, false); - (void)bnxt_ulp_global_cfg_update(bp, TF_DIR_TX, TF_TUNNEL_ENCAP, - TF_TUNNEL_ENCAP_NAT, - (BNXT_ULP_NAT_INNER_L2_HEADER_SMAC | - BNXT_ULP_NAT_INNER_L2_HEADER_DMAC), - 0); + /* free flows associated with this port */ + bnxt_ulp_flush_port_flows(bp); - /* Delete the ulp context and tf session */ - ulp_ctx_detach(bp, session); + /* close the session associated with this port */ + ulp_ctx_detach(bp); + } else { + /* Perform ulp ctx deinit */ + bnxt_ulp_deinit(bp, session); + } + } - /* Finally delete the bnxt session*/ + /* clean up the session */ ulp_session_deinit(session); + /* Free the ulp context */ rte_free(bp->ulp_ctx); - bp->ulp_ctx = NULL; } +/* Below are the access functions to access internal data of ulp context. */ /* Function to set the Mark DB into the context */ int32_t bnxt_ulp_cntxt_ptr2_mark_db_set(struct bnxt_ulp_context *ulp_ctx, @@ -974,7 +1069,6 @@ bnxt_ulp_cntxt_tfp_set(struct bnxt_ulp_context *ulp, struct tf *tfp) return -EINVAL; } - /* TBD The tfp should be removed once tf_attach is implemented. */ ulp->g_tfp = tfp; return 0; } @@ -987,7 +1081,6 @@ bnxt_ulp_cntxt_tfp_get(struct bnxt_ulp_context *ulp) BNXT_TF_DBG(ERR, "Invalid arguments\n"); return NULL; } - /* TBD The tfp should be removed once tf_attach is implemented. */ return ulp->g_tfp; } @@ -1129,3 +1222,14 @@ bnxt_ulp_cntxt_ptr2_ulp_flags_get(struct bnxt_ulp_context *ulp_ctx, *flags = ulp_ctx->cfg_data->ulp_flags; return 0; } + +/* Function to get the ulp vfr info from the ulp context. */ +struct bnxt_ulp_vfr_rule_info* +bnxt_ulp_cntxt_ptr2_ulp_vfr_info_get(struct bnxt_ulp_context *ulp_ctx, + uint32_t port_id) +{ + if (!ulp_ctx || !ulp_ctx->cfg_data || port_id >= RTE_MAX_ETHPORTS) + return NULL; + + return &ulp_ctx->cfg_data->vfr_rule_info[port_id]; +} diff --git a/drivers/net/bnxt/tf_ulp/bnxt_ulp.h b/drivers/net/bnxt/tf_ulp/bnxt_ulp.h index d532452157..8a2825ae5b 100644 --- a/drivers/net/bnxt/tf_ulp/bnxt_ulp.h +++ b/drivers/net/bnxt/tf_ulp/bnxt_ulp.h @@ -28,6 +28,13 @@ struct bnxt_ulp_df_rule_info { uint8_t valid; }; +struct bnxt_ulp_vfr_rule_info { + uint32_t rep2vf_flow_id; + uint32_t vf2rep_flow_id; + uint16_t parent_port_id; + uint8_t valid; +}; + struct bnxt_ulp_data { uint32_t tbl_scope_id; struct bnxt_ulp_mark_tbl *mark_tbl; @@ -38,12 +45,12 @@ struct bnxt_ulp_data { struct bnxt_ulp_port_db *port_db; struct bnxt_ulp_fc_info *fc_info; uint32_t ulp_flags; - struct bnxt_ulp_df_rule_info df_rule_info[RTE_MAX_ETHPORTS]; + struct bnxt_ulp_df_rule_info df_rule_info[RTE_MAX_ETHPORTS]; + struct bnxt_ulp_vfr_rule_info vfr_rule_info[RTE_MAX_ETHPORTS]; }; struct bnxt_ulp_context { struct bnxt_ulp_data *cfg_data; - /* TBD The tfp should be removed once tf_attach is implemented. */ struct tf *g_tfp; }; @@ -58,7 +65,6 @@ struct bnxt_ulp_session_state { pthread_mutex_t bnxt_ulp_mutex; struct bnxt_ulp_pci_info pci_info; struct bnxt_ulp_data *cfg_data; - /* TBD The tfp should be removed once tf_attach is implemented. */ struct tf *g_tfp; uint32_t session_opened; }; @@ -183,4 +189,8 @@ int32_t bnxt_ulp_get_df_rule_info(uint8_t port_id, struct bnxt_ulp_context *ulp_ctx, struct bnxt_ulp_df_rule_info *info); +struct bnxt_ulp_vfr_rule_info* +bnxt_ulp_cntxt_ptr2_ulp_vfr_info_get(struct bnxt_ulp_context *ulp_ctx, + uint32_t port_id); + #endif /* _BNXT_ULP_H_ */ diff --git a/drivers/net/bnxt/tf_ulp/bnxt_ulp_flow.c b/drivers/net/bnxt/tf_ulp/bnxt_ulp_flow.c index 89fffcf01b..2ab00453ac 100644 --- a/drivers/net/bnxt/tf_ulp/bnxt_ulp_flow.c +++ b/drivers/net/bnxt/tf_ulp/bnxt_ulp_flow.c @@ -87,19 +87,19 @@ bnxt_ulp_flow_create(struct rte_eth_dev *dev, uint32_t class_id, act_tmpl; struct rte_flow *flow_id; uint32_t fid; - int ret; + int ret = BNXT_TF_RC_ERROR; if (bnxt_ulp_flow_validate_args(attr, pattern, actions, error) == BNXT_TF_RC_ERROR) { BNXT_TF_DBG(ERR, "Invalid arguments being passed\n"); - return NULL; + goto parse_error; } ulp_ctx = bnxt_ulp_eth_dev_ptr2_cntxt_get(dev); if (!ulp_ctx) { BNXT_TF_DBG(ERR, "ULP context is not initialized\n"); - return NULL; + goto parse_error; } /* Initialize the parser params */ @@ -173,20 +173,20 @@ bnxt_ulp_flow_validate(struct rte_eth_dev *dev, { struct ulp_rte_parser_params params; uint32_t class_id, act_tmpl; - int ret; + int ret = BNXT_TF_RC_ERROR; struct bnxt_ulp_context *ulp_ctx; if (bnxt_ulp_flow_validate_args(attr, pattern, actions, error) == BNXT_TF_RC_ERROR) { BNXT_TF_DBG(ERR, "Invalid arguments being passed\n"); - return -EINVAL; + goto parse_error; } ulp_ctx = bnxt_ulp_eth_dev_ptr2_cntxt_get(dev); if (!ulp_ctx) { BNXT_TF_DBG(ERR, "ULP context is not initialized\n"); - return -EINVAL; + goto parse_error; } /* Initialize the parser params */ @@ -289,11 +289,8 @@ bnxt_ulp_flow_flush(struct rte_eth_dev *eth_dev, ulp_ctx = bnxt_ulp_eth_dev_ptr2_cntxt_get(eth_dev); if (!ulp_ctx) { - BNXT_TF_DBG(ERR, "ULP context is not initialized\n"); - rte_flow_error_set(error, EINVAL, - RTE_FLOW_ERROR_TYPE_HANDLE, NULL, - "Failed to flush flow."); - return -EINVAL; + BNXT_TF_DBG(DEBUG, "ULP context is not initialized\n"); + return ret; } bp = eth_dev->data->dev_private; diff --git a/drivers/net/bnxt/tf_ulp/ulp_def_rules.c b/drivers/net/bnxt/tf_ulp/ulp_def_rules.c index 46acc1d65a..2d0c3bccc7 100644 --- a/drivers/net/bnxt/tf_ulp/ulp_def_rules.c +++ b/drivers/net/bnxt/tf_ulp/ulp_def_rules.c @@ -377,7 +377,7 @@ int32_t ulp_default_flow_destroy(struct rte_eth_dev *eth_dev, uint32_t flow_id) { struct bnxt_ulp_context *ulp_ctx; - int rc; + int rc = 0; ulp_ctx = bnxt_ulp_eth_dev_ptr2_cntxt_get(eth_dev); if (!ulp_ctx) { @@ -385,6 +385,11 @@ ulp_default_flow_destroy(struct rte_eth_dev *eth_dev, uint32_t flow_id) return -EINVAL; } + if (!flow_id) { + BNXT_TF_DBG(DEBUG, "invalid flow id zero\n"); + return rc; + } + rc = ulp_mapper_flow_destroy(ulp_ctx, flow_id, BNXT_ULP_DEFAULT_FLOW_TABLE); if (rc) @@ -417,7 +422,7 @@ bnxt_ulp_destroy_df_rules(struct bnxt *bp, bool global) info->port_to_app_flow_id); ulp_default_flow_destroy(bp->eth_dev, info->app_to_port_flow_id); - info->valid = false; + memset(info, 0, sizeof(struct bnxt_ulp_df_rule_info)); return; } @@ -431,7 +436,7 @@ bnxt_ulp_destroy_df_rules(struct bnxt *bp, bool global) info->port_to_app_flow_id); ulp_default_flow_destroy(bp->eth_dev, info->app_to_port_flow_id); - info->valid = false; + memset(info, 0, sizeof(struct bnxt_ulp_df_rule_info)); } } @@ -470,22 +475,19 @@ bnxt_ulp_create_df_rules(struct bnxt *bp) port_id = bp->eth_dev->data->port_id; info = &bp->ulp_ctx->cfg_data->df_rule_info[port_id]; - BNXT_TF_DBG(INFO, "*** creating port to app default rule ***\n"); rc = bnxt_create_port_app_df_rule(bp, BNXT_ULP_DF_TPL_PORT_TO_VS, &info->port_to_app_flow_id); if (rc) { - PMD_DRV_LOG(ERR, + BNXT_TF_DBG(ERR, "Failed to create port to app default rule\n"); return rc; } - BNXT_TF_DBG(INFO, "*** created port to app default rule ***\n"); bp->tx_cfa_action = 0; - BNXT_TF_DBG(INFO, "*** creating app to port default rule ***\n"); rc = bnxt_create_port_app_df_rule(bp, BNXT_ULP_DF_TPL_VS_TO_PORT, &info->app_to_port_flow_id); if (rc) { - PMD_DRV_LOG(ERR, + BNXT_TF_DBG(ERR, "Failed to create app to port default rule\n"); goto port_to_app_free; } @@ -497,7 +499,6 @@ bnxt_ulp_create_df_rules(struct bnxt *bp) goto app_to_port_free; info->valid = true; - BNXT_TF_DBG(INFO, "*** created app to port default rule ***\n"); return 0; app_to_port_free: @@ -508,3 +509,115 @@ port_to_app_free: return rc; } + +static int32_t +bnxt_create_port_vfr_default_rule(struct bnxt *bp, + uint8_t flow_type, + uint16_t vfr_port_id, + uint32_t *flow_id) +{ + struct ulp_tlv_param param_list[] = { + { + .type = BNXT_ULP_DF_PARAM_TYPE_DEV_PORT_ID, + .length = 2, + .value = {(vfr_port_id >> 8) & 0xff, vfr_port_id & 0xff} + }, + { + .type = BNXT_ULP_DF_PARAM_TYPE_LAST, + .length = 0, + .value = {0} + } + }; + return ulp_default_flow_create(bp->eth_dev, param_list, flow_type, + flow_id); +} + +int32_t +bnxt_ulp_create_vfr_default_rules(struct rte_eth_dev *vfr_ethdev) +{ + struct bnxt_ulp_vfr_rule_info *info; + struct bnxt_vf_representor *vfr = vfr_ethdev->data->dev_private; + struct rte_eth_dev *parent_dev = vfr->parent_dev; + struct bnxt *bp = parent_dev->data->dev_private; + uint16_t vfr_port_id = vfr_ethdev->data->port_id; + uint8_t port_id; + int rc; + + if (!bp || !BNXT_TRUFLOW_EN(bp)) + return 0; + + port_id = vfr_ethdev->data->port_id; + info = bnxt_ulp_cntxt_ptr2_ulp_vfr_info_get(bp->ulp_ctx, port_id); + + if (!info) { + BNXT_TF_DBG(ERR, "Failed to get vfr ulp context\n"); + return -EINVAL; + } + + if (info->valid) { + BNXT_TF_DBG(ERR, "VFR already allocated\n"); + return -EINVAL; + } + + memset(info, 0, sizeof(struct bnxt_ulp_vfr_rule_info)); + rc = bnxt_create_port_vfr_default_rule(bp, BNXT_ULP_DF_TPL_VFREP_TO_VF, + vfr_port_id, + &info->rep2vf_flow_id); + if (rc) { + BNXT_TF_DBG(ERR, "Failed to create VFREP to VF default rule\n"); + goto error; + } + rc = bnxt_create_port_vfr_default_rule(bp, BNXT_ULP_DF_TPL_VF_TO_VFREP, + vfr_port_id, + &info->vf2rep_flow_id); + if (rc) { + BNXT_TF_DBG(ERR, "Failed to create VF to VFREP default rule\n"); + goto error; + } + rc = ulp_default_flow_db_cfa_action_get(bp->ulp_ctx, + info->rep2vf_flow_id, + &vfr->vfr_tx_cfa_action); + if (rc) { + BNXT_TF_DBG(ERR, "Failed to get the tx cfa action\n"); + goto error; + } + + /* Update the other details */ + info->valid = true; + info->parent_port_id = bp->eth_dev->data->port_id; + return 0; + +error: + if (info->rep2vf_flow_id) + ulp_default_flow_destroy(bp->eth_dev, info->rep2vf_flow_id); + if (info->vf2rep_flow_id) + ulp_default_flow_destroy(bp->eth_dev, info->vf2rep_flow_id); + return rc; +} + +int32_t +bnxt_ulp_delete_vfr_default_rules(struct bnxt_vf_representor *vfr) +{ + struct bnxt_ulp_vfr_rule_info *info; + struct rte_eth_dev *parent_dev = vfr->parent_dev; + struct bnxt *bp = parent_dev->data->dev_private; + + if (!bp || !BNXT_TRUFLOW_EN(bp)) + return 0; + info = bnxt_ulp_cntxt_ptr2_ulp_vfr_info_get(bp->ulp_ctx, + vfr->dpdk_port_id); + if (!info) { + BNXT_TF_DBG(ERR, "Failed to get vfr ulp context\n"); + return -EINVAL; + } + + if (!info->valid) { + BNXT_TF_DBG(ERR, "VFR already freed\n"); + return -EINVAL; + } + ulp_default_flow_destroy(bp->eth_dev, info->rep2vf_flow_id); + ulp_default_flow_destroy(bp->eth_dev, info->vf2rep_flow_id); + vfr->vfr_tx_cfa_action = 0; + memset(info, 0, sizeof(struct bnxt_ulp_vfr_rule_info)); + return 0; +} diff --git a/drivers/net/bnxt/tf_ulp/ulp_flow_db.c b/drivers/net/bnxt/tf_ulp/ulp_flow_db.c index 7144517409..cbdf5df68a 100644 --- a/drivers/net/bnxt/tf_ulp/ulp_flow_db.c +++ b/drivers/net/bnxt/tf_ulp/ulp_flow_db.c @@ -351,10 +351,8 @@ int32_t ulp_flow_db_deinit(struct bnxt_ulp_context *ulp_ctxt) struct bnxt_ulp_flow_db *flow_db; flow_db = bnxt_ulp_cntxt_ptr2_flow_db_get(ulp_ctxt); - if (!flow_db) { - BNXT_TF_DBG(ERR, "Invalid Arguments\n"); + if (!flow_db) return -EINVAL; - } /* Detach the flow database from the ulp context. */ bnxt_ulp_cntxt_ptr2_flow_db_set(ulp_ctxt, NULL); diff --git a/drivers/net/bnxt/tf_ulp/ulp_flow_db.h b/drivers/net/bnxt/tf_ulp/ulp_flow_db.h index 117e250d63..8c83664d02 100644 --- a/drivers/net/bnxt/tf_ulp/ulp_flow_db.h +++ b/drivers/net/bnxt/tf_ulp/ulp_flow_db.h @@ -9,8 +9,8 @@ #include "bnxt_ulp.h" #include "ulp_template_db_enum.h" -#define BNXT_FLOW_DB_DEFAULT_NUM_FLOWS 128 -#define BNXT_FLOW_DB_DEFAULT_NUM_RESOURCES 5 +#define BNXT_FLOW_DB_DEFAULT_NUM_FLOWS 512 +#define BNXT_FLOW_DB_DEFAULT_NUM_RESOURCES 8 /* * Structure for the flow database resource information