drivers/net/bnxt/bnxt_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2014-2021 Broadcom
   3  * All rights reserved.
   4  */
   5
   6 #include <inttypes.h>
   7 #include <stdbool.h>
   8
   9 #include <rte_dev.h>
  10 #include <ethdev_driver.h>
  11 #include <ethdev_pci.h>
  12 #include <rte_malloc.h>
  13 #include <rte_cycles.h>
  14 #include <rte_alarm.h>
  15 #include <rte_kvargs.h>
  16 #include <rte_vect.h>
  17
  18 #include "bnxt.h"
  19 #include "bnxt_filter.h"
  20 #include "bnxt_hwrm.h"
  21 #include "bnxt_irq.h"
  22 #include "bnxt_reps.h"
  23 #include "bnxt_ring.h"
  24 #include "bnxt_rxq.h"
  25 #include "bnxt_rxr.h"
  26 #include "bnxt_stats.h"
  27 #include "bnxt_txq.h"
  28 #include "bnxt_txr.h"
  29 #include "bnxt_vnic.h"
  30 #include "hsi_struct_def_dpdk.h"
  31 #include "bnxt_nvm_defs.h"
  32 #include "bnxt_tf_common.h"
  33 #include "ulp_flow_db.h"
  34 #include "rte_pmd_bnxt.h"
  35
  36 #define DRV_MODULE_NAME         "bnxt"
  37 static const char bnxt_version[] =
  38         "Broadcom NetXtreme driver " DRV_MODULE_NAME;
  39
  40 /*
  41  * The set of PCI devices this driver supports
  42  */
  43 static const struct rte_pci_id bnxt_pci_id_map[] = {
  44         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM,
  45                          BROADCOM_DEV_ID_STRATUS_NIC_VF1) },
  46         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM,
  47                          BROADCOM_DEV_ID_STRATUS_NIC_VF2) },
  48         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_STRATUS_NIC) },
  49         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57414_VF) },
  50         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57304_VF) },
  51         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_NS2) },
  52         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57406_VF) },
  53         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57407_MF) },
  54         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_5741X_VF) },
  55         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_5731X_VF) },
  56         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57417_MF) },
  57         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57412) },
  58         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57414) },
  59         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57416_RJ45) },
  60         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57417_RJ45) },
  61         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57412_MF) },
  62         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57317_RJ45) },
  63         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57417_SFP) },
  64         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57416_SFP) },
  65         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57317_SFP) },
  66         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57414_MF) },
  67         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57416_MF) },
  68         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58802) },
  69         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58804) },
  70         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58808) },
  71         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58802_VF) },
  72         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57508) },
  73         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57504) },
  74         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57502) },
  75         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57500_VF1) },
  76         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57500_VF2) },
  77         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57508_MF1) },
  78         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57504_MF1) },
  79         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57502_MF1) },
  80         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57508_MF2) },
  81         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57504_MF2) },
  82         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57502_MF2) },
  83         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58812) },
  84         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58814) },
  85         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58818) },
  86         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58818_VF) },
  87         { .vendor_id = 0, /* sentinel */ },
  88 };
  89
  90 #define BNXT_DEVARG_ACCUM_STATS "accum-stats"
  91 #define BNXT_DEVARG_FLOW_XSTAT  "flow-xstat"
  92 #define BNXT_DEVARG_MAX_NUM_KFLOWS  "max-num-kflows"
  93 #define BNXT_DEVARG_REPRESENTOR "representor"
  94 #define BNXT_DEVARG_REP_BASED_PF  "rep-based-pf"
  95 #define BNXT_DEVARG_REP_IS_PF  "rep-is-pf"
  96 #define BNXT_DEVARG_REP_Q_R2F  "rep-q-r2f"
  97 #define BNXT_DEVARG_REP_Q_F2R  "rep-q-f2r"
  98 #define BNXT_DEVARG_REP_FC_R2F  "rep-fc-r2f"
  99 #define BNXT_DEVARG_REP_FC_F2R  "rep-fc-f2r"
 100 #define BNXT_DEVARG_APP_ID      "app-id"
 101
 102 static const char *const bnxt_dev_args[] = {
 103         BNXT_DEVARG_REPRESENTOR,
 104         BNXT_DEVARG_ACCUM_STATS,
 105         BNXT_DEVARG_FLOW_XSTAT,
 106         BNXT_DEVARG_MAX_NUM_KFLOWS,
 107         BNXT_DEVARG_REP_BASED_PF,
 108         BNXT_DEVARG_REP_IS_PF,
 109         BNXT_DEVARG_REP_Q_R2F,
 110         BNXT_DEVARG_REP_Q_F2R,
 111         BNXT_DEVARG_REP_FC_R2F,
 112         BNXT_DEVARG_REP_FC_F2R,
 113         BNXT_DEVARG_APP_ID,
 114         NULL
 115 };
 116
 117 /*
 118  * accum-stats == false to disable flow counter accumulation
 119  * accum-stats == true to enable flow counter accumulation
 120  */
 121 #define BNXT_DEVARG_ACCUM_STATS_INVALID(accum_stats)    ((accum_stats) > 1)
 122
 123 /*
 124  * app-id = an non-negative 8-bit number
 125  */
 126 #define BNXT_DEVARG_APP_ID_INVALID(val)                 ((val) > 255)
 127
 128 /*
 129  * flow_xstat == false to disable the feature
 130  * flow_xstat == true to enable the feature
 131  */
 132 #define BNXT_DEVARG_FLOW_XSTAT_INVALID(flow_xstat)      ((flow_xstat) > 1)
 133
 134 /*
 135  * rep_is_pf == false to indicate VF representor
 136  * rep_is_pf == true to indicate PF representor
 137  */
 138 #define BNXT_DEVARG_REP_IS_PF_INVALID(rep_is_pf)        ((rep_is_pf) > 1)
 139
 140 /*
 141  * rep_based_pf == Physical index of the PF
 142  */
 143 #define BNXT_DEVARG_REP_BASED_PF_INVALID(rep_based_pf)  ((rep_based_pf) > 15)
 144 /*
 145  * rep_q_r2f == Logical COS Queue index for the rep to endpoint direction
 146  */
 147 #define BNXT_DEVARG_REP_Q_R2F_INVALID(rep_q_r2f)        ((rep_q_r2f) > 3)
 148
 149 /*
 150  * rep_q_f2r == Logical COS Queue index for the endpoint to rep direction
 151  */
 152 #define BNXT_DEVARG_REP_Q_F2R_INVALID(rep_q_f2r)        ((rep_q_f2r) > 3)
 153
 154 /*
 155  * rep_fc_r2f == Flow control for the representor to endpoint direction
 156  */
 157 #define BNXT_DEVARG_REP_FC_R2F_INVALID(rep_fc_r2f)      ((rep_fc_r2f) > 1)
 158
 159 /*
 160  * rep_fc_f2r == Flow control for the endpoint to representor direction
 161  */
 162 #define BNXT_DEVARG_REP_FC_F2R_INVALID(rep_fc_f2r)      ((rep_fc_f2r) > 1)
 163
 164 int bnxt_cfa_code_dynfield_offset = -1;
 165
 166 /*
 167  * max_num_kflows must be >= 32
 168  * and must be a power-of-2 supported value
 169  * return: 1 -> invalid
 170  *         0 -> valid
 171  */
 172 static int bnxt_devarg_max_num_kflow_invalid(uint16_t max_num_kflows)
 173 {
 174         if (max_num_kflows < 32 || !rte_is_power_of_2(max_num_kflows))
 175                 return 1;
 176         return 0;
 177 }
 178
 179 static int bnxt_vlan_offload_set_op(struct rte_eth_dev *dev, int mask);
 180 static int bnxt_dev_uninit(struct rte_eth_dev *eth_dev);
 181 static int bnxt_init_resources(struct bnxt *bp, bool reconfig_dev);
 182 static int bnxt_uninit_resources(struct bnxt *bp, bool reconfig_dev);
 183 static void bnxt_cancel_fw_health_check(struct bnxt *bp);
 184 static int bnxt_restore_vlan_filters(struct bnxt *bp);
 185 static void bnxt_dev_recover(void *arg);
 186 static void bnxt_free_error_recovery_info(struct bnxt *bp);
 187 static void bnxt_free_rep_info(struct bnxt *bp);
 188
 189 int is_bnxt_in_error(struct bnxt *bp)
 190 {
 191         if (bp->flags & BNXT_FLAG_FATAL_ERROR)
 192                 return -EIO;
 193         if (bp->flags & BNXT_FLAG_FW_RESET)
 194                 return -EBUSY;
 195
 196         return 0;
 197 }
 198
 199 /***********************/
 200
 201 /*
 202  * High level utility functions
 203  */
 204
 205 static uint16_t bnxt_rss_ctxts(const struct bnxt *bp)
 206 {
 207         unsigned int num_rss_rings = RTE_MIN(bp->rx_nr_rings,
 208                                              BNXT_RSS_TBL_SIZE_P5);
 209
 210         if (!BNXT_CHIP_P5(bp))
 211                 return 1;
 212
 213         return RTE_ALIGN_MUL_CEIL(num_rss_rings,
 214                                   BNXT_RSS_ENTRIES_PER_CTX_P5) /
 215                                   BNXT_RSS_ENTRIES_PER_CTX_P5;
 216 }
 217
 218 uint16_t bnxt_rss_hash_tbl_size(const struct bnxt *bp)
 219 {
 220         if (!BNXT_CHIP_P5(bp))
 221                 return HW_HASH_INDEX_SIZE;
 222
 223         return bnxt_rss_ctxts(bp) * BNXT_RSS_ENTRIES_PER_CTX_P5;
 224 }
 225
 226 static void bnxt_free_parent_info(struct bnxt *bp)
 227 {
 228         rte_free(bp->parent);
 229         bp->parent = NULL;
 230 }
 231
 232 static void bnxt_free_pf_info(struct bnxt *bp)
 233 {
 234         rte_free(bp->pf);
 235         bp->pf = NULL;
 236 }
 237
 238 static void bnxt_free_link_info(struct bnxt *bp)
 239 {
 240         rte_free(bp->link_info);
 241         bp->link_info = NULL;
 242 }
 243
 244 static void bnxt_free_leds_info(struct bnxt *bp)
 245 {
 246         if (BNXT_VF(bp))
 247                 return;
 248
 249         rte_free(bp->leds);
 250         bp->leds = NULL;
 251 }
 252
 253 static void bnxt_free_flow_stats_info(struct bnxt *bp)
 254 {
 255         rte_free(bp->flow_stat);
 256         bp->flow_stat = NULL;
 257 }
 258
 259 static void bnxt_free_cos_queues(struct bnxt *bp)
 260 {
 261         rte_free(bp->rx_cos_queue);
 262         bp->rx_cos_queue = NULL;
 263         rte_free(bp->tx_cos_queue);
 264         bp->tx_cos_queue = NULL;
 265 }
 266
 267 static void bnxt_free_mem(struct bnxt *bp, bool reconfig)
 268 {
 269         bnxt_free_filter_mem(bp);
 270         bnxt_free_vnic_attributes(bp);
 271         bnxt_free_vnic_mem(bp);
 272
 273         /* tx/rx rings are configured as part of *_queue_setup callbacks.
 274          * If the number of rings change across fw update,
 275          * we don't have much choice except to warn the user.
 276          */
 277         if (!reconfig) {
 278                 bnxt_free_stats(bp);
 279                 bnxt_free_tx_rings(bp);
 280                 bnxt_free_rx_rings(bp);
 281         }
 282         bnxt_free_async_cp_ring(bp);
 283         bnxt_free_rxtx_nq_ring(bp);
 284
 285         rte_free(bp->grp_info);
 286         bp->grp_info = NULL;
 287 }
 288
 289 static int bnxt_alloc_parent_info(struct bnxt *bp)
 290 {
 291         bp->parent = rte_zmalloc("bnxt_parent_info",
 292                                  sizeof(struct bnxt_parent_info), 0);
 293         if (bp->parent == NULL)
 294                 return -ENOMEM;
 295
 296         return 0;
 297 }
 298
 299 static int bnxt_alloc_pf_info(struct bnxt *bp)
 300 {
 301         bp->pf = rte_zmalloc("bnxt_pf_info", sizeof(struct bnxt_pf_info), 0);
 302         if (bp->pf == NULL)
 303                 return -ENOMEM;
 304
 305         return 0;
 306 }
 307
 308 static int bnxt_alloc_link_info(struct bnxt *bp)
 309 {
 310         bp->link_info =
 311                 rte_zmalloc("bnxt_link_info", sizeof(struct bnxt_link_info), 0);
 312         if (bp->link_info == NULL)
 313                 return -ENOMEM;
 314
 315         return 0;
 316 }
 317
 318 static int bnxt_alloc_leds_info(struct bnxt *bp)
 319 {
 320         if (BNXT_VF(bp))
 321                 return 0;
 322
 323         bp->leds = rte_zmalloc("bnxt_leds",
 324                                BNXT_MAX_LED * sizeof(struct bnxt_led_info),
 325                                0);
 326         if (bp->leds == NULL)
 327                 return -ENOMEM;
 328
 329         return 0;
 330 }
 331
 332 static int bnxt_alloc_cos_queues(struct bnxt *bp)
 333 {
 334         bp->rx_cos_queue =
 335                 rte_zmalloc("bnxt_rx_cosq",
 336                             BNXT_COS_QUEUE_COUNT *
 337                             sizeof(struct bnxt_cos_queue_info),
 338                             0);
 339         if (bp->rx_cos_queue == NULL)
 340                 return -ENOMEM;
 341
 342         bp->tx_cos_queue =
 343                 rte_zmalloc("bnxt_tx_cosq",
 344                             BNXT_COS_QUEUE_COUNT *
 345                             sizeof(struct bnxt_cos_queue_info),
 346                             0);
 347         if (bp->tx_cos_queue == NULL)
 348                 return -ENOMEM;
 349
 350         return 0;
 351 }
 352
 353 static int bnxt_alloc_flow_stats_info(struct bnxt *bp)
 354 {
 355         bp->flow_stat = rte_zmalloc("bnxt_flow_xstat",
 356                                     sizeof(struct bnxt_flow_stat_info), 0);
 357         if (bp->flow_stat == NULL)
 358                 return -ENOMEM;
 359
 360         return 0;
 361 }
 362
 363 static int bnxt_alloc_mem(struct bnxt *bp, bool reconfig)
 364 {
 365         int rc;
 366
 367         rc = bnxt_alloc_ring_grps(bp);
 368         if (rc)
 369                 goto alloc_mem_err;
 370
 371         rc = bnxt_alloc_async_ring_struct(bp);
 372         if (rc)
 373                 goto alloc_mem_err;
 374
 375         rc = bnxt_alloc_vnic_mem(bp);
 376         if (rc)
 377                 goto alloc_mem_err;
 378
 379         rc = bnxt_alloc_vnic_attributes(bp);
 380         if (rc)
 381                 goto alloc_mem_err;
 382
 383         rc = bnxt_alloc_filter_mem(bp);
 384         if (rc)
 385                 goto alloc_mem_err;
 386
 387         rc = bnxt_alloc_async_cp_ring(bp);
 388         if (rc)
 389                 goto alloc_mem_err;
 390
 391         rc = bnxt_alloc_rxtx_nq_ring(bp);
 392         if (rc)
 393                 goto alloc_mem_err;
 394
 395         if (BNXT_FLOW_XSTATS_EN(bp)) {
 396                 rc = bnxt_alloc_flow_stats_info(bp);
 397                 if (rc)
 398                         goto alloc_mem_err;
 399         }
 400
 401         return 0;
 402
 403 alloc_mem_err:
 404         bnxt_free_mem(bp, reconfig);
 405         return rc;
 406 }
 407
 408 static int bnxt_setup_one_vnic(struct bnxt *bp, uint16_t vnic_id)
 409 {
 410         struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
 411         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
 412         uint64_t rx_offloads = dev_conf->rxmode.offloads;
 413         struct bnxt_rx_queue *rxq;
 414         unsigned int j;
 415         int rc;
 416
 417         rc = bnxt_vnic_grp_alloc(bp, vnic);
 418         if (rc)
 419                 goto err_out;
 420
 421         PMD_DRV_LOG(DEBUG, "vnic[%d] = %p vnic->fw_grp_ids = %p\n",
 422                     vnic_id, vnic, vnic->fw_grp_ids);
 423
 424         rc = bnxt_hwrm_vnic_alloc(bp, vnic);
 425         if (rc)
 426                 goto err_out;
 427
 428         /* Alloc RSS context only if RSS mode is enabled */
 429         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS) {
 430                 int j, nr_ctxs = bnxt_rss_ctxts(bp);
 431
 432                 /* RSS table size in Thor is 512.
 433                  * Cap max Rx rings to same value
 434                  */
 435                 if (bp->rx_nr_rings > BNXT_RSS_TBL_SIZE_P5) {
 436                         PMD_DRV_LOG(ERR, "RxQ cnt %d > reta_size %d\n",
 437                                     bp->rx_nr_rings, BNXT_RSS_TBL_SIZE_P5);
 438                         goto err_out;
 439                 }
 440
 441                 rc = 0;
 442                 for (j = 0; j < nr_ctxs; j++) {
 443                         rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic, j);
 444                         if (rc)
 445                                 break;
 446                 }
 447                 if (rc) {
 448                         PMD_DRV_LOG(ERR,
 449                                     "HWRM vnic %d ctx %d alloc failure rc: %x\n",
 450                                     vnic_id, j, rc);
 451                         goto err_out;
 452                 }
 453                 vnic->num_lb_ctxts = nr_ctxs;
 454         }
 455
 456         /*
 457          * Firmware sets pf pair in default vnic cfg. If the VLAN strip
 458          * setting is not available at this time, it will not be
 459          * configured correctly in the CFA.
 460          */
 461         if (rx_offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
 462                 vnic->vlan_strip = true;
 463         else
 464                 vnic->vlan_strip = false;
 465
 466         rc = bnxt_hwrm_vnic_cfg(bp, vnic);
 467         if (rc)
 468                 goto err_out;
 469
 470         rc = bnxt_set_hwrm_vnic_filters(bp, vnic);
 471         if (rc)
 472                 goto err_out;
 473
 474         for (j = 0; j < bp->rx_num_qs_per_vnic; j++) {
 475                 rxq = bp->eth_dev->data->rx_queues[j];
 476
 477                 PMD_DRV_LOG(DEBUG,
 478                             "rxq[%d]->vnic=%p vnic->fw_grp_ids=%p\n",
 479                             j, rxq->vnic, rxq->vnic->fw_grp_ids);
 480
 481                 if (BNXT_HAS_RING_GRPS(bp) && rxq->rx_deferred_start)
 482                         rxq->vnic->fw_grp_ids[j] = INVALID_HW_RING_ID;
 483                 else
 484                         vnic->rx_queue_cnt++;
 485         }
 486
 487         PMD_DRV_LOG(DEBUG, "vnic->rx_queue_cnt = %d\n", vnic->rx_queue_cnt);
 488
 489         rc = bnxt_vnic_rss_configure(bp, vnic);
 490         if (rc)
 491                 goto err_out;
 492
 493         bnxt_hwrm_vnic_plcmode_cfg(bp, vnic);
 494
 495         rc = bnxt_hwrm_vnic_tpa_cfg(bp, vnic,
 496                                     (rx_offloads & DEV_RX_OFFLOAD_TCP_LRO) ?
 497                                     true : false);
 498         if (rc)
 499                 goto err_out;
 500
 501         return 0;
 502 err_out:
 503         PMD_DRV_LOG(ERR, "HWRM vnic %d cfg failure rc: %x\n",
 504                     vnic_id, rc);
 505         return rc;
 506 }
 507
 508 static int bnxt_register_fc_ctx_mem(struct bnxt *bp)
 509 {
 510         int rc = 0;
 511
 512         rc = bnxt_hwrm_ctx_rgtr(bp, bp->flow_stat->rx_fc_in_tbl.dma,
 513                                 &bp->flow_stat->rx_fc_in_tbl.ctx_id);
 514         if (rc)
 515                 return rc;
 516
 517         PMD_DRV_LOG(DEBUG,
 518                     "rx_fc_in_tbl.va = %p rx_fc_in_tbl.dma = %p"
 519                     " rx_fc_in_tbl.ctx_id = %d\n",
 520                     bp->flow_stat->rx_fc_in_tbl.va,
 521                     (void *)((uintptr_t)bp->flow_stat->rx_fc_in_tbl.dma),
 522                     bp->flow_stat->rx_fc_in_tbl.ctx_id);
 523
 524         rc = bnxt_hwrm_ctx_rgtr(bp, bp->flow_stat->rx_fc_out_tbl.dma,
 525                                 &bp->flow_stat->rx_fc_out_tbl.ctx_id);
 526         if (rc)
 527                 return rc;
 528
 529         PMD_DRV_LOG(DEBUG,
 530                     "rx_fc_out_tbl.va = %p rx_fc_out_tbl.dma = %p"
 531                     " rx_fc_out_tbl.ctx_id = %d\n",
 532                     bp->flow_stat->rx_fc_out_tbl.va,
 533                     (void *)((uintptr_t)bp->flow_stat->rx_fc_out_tbl.dma),
 534                     bp->flow_stat->rx_fc_out_tbl.ctx_id);
 535
 536         rc = bnxt_hwrm_ctx_rgtr(bp, bp->flow_stat->tx_fc_in_tbl.dma,
 537                                 &bp->flow_stat->tx_fc_in_tbl.ctx_id);
 538         if (rc)
 539                 return rc;
 540
 541         PMD_DRV_LOG(DEBUG,
 542                     "tx_fc_in_tbl.va = %p tx_fc_in_tbl.dma = %p"
 543                     " tx_fc_in_tbl.ctx_id = %d\n",
 544                     bp->flow_stat->tx_fc_in_tbl.va,
 545                     (void *)((uintptr_t)bp->flow_stat->tx_fc_in_tbl.dma),
 546                     bp->flow_stat->tx_fc_in_tbl.ctx_id);
 547
 548         rc = bnxt_hwrm_ctx_rgtr(bp, bp->flow_stat->tx_fc_out_tbl.dma,
 549                                 &bp->flow_stat->tx_fc_out_tbl.ctx_id);
 550         if (rc)
 551                 return rc;
 552
 553         PMD_DRV_LOG(DEBUG,
 554                     "tx_fc_out_tbl.va = %p tx_fc_out_tbl.dma = %p"
 555                     " tx_fc_out_tbl.ctx_id = %d\n",
 556                     bp->flow_stat->tx_fc_out_tbl.va,
 557                     (void *)((uintptr_t)bp->flow_stat->tx_fc_out_tbl.dma),
 558                     bp->flow_stat->tx_fc_out_tbl.ctx_id);
 559
 560         memset(bp->flow_stat->rx_fc_out_tbl.va,
 561                0,
 562                bp->flow_stat->rx_fc_out_tbl.size);
 563         rc = bnxt_hwrm_cfa_counter_cfg(bp, BNXT_DIR_RX,
 564                                        CFA_COUNTER_CFG_IN_COUNTER_TYPE_FC,
 565                                        bp->flow_stat->rx_fc_out_tbl.ctx_id,
 566                                        bp->flow_stat->max_fc,
 567                                        true);
 568         if (rc)
 569                 return rc;
 570
 571         memset(bp->flow_stat->tx_fc_out_tbl.va,
 572                0,
 573                bp->flow_stat->tx_fc_out_tbl.size);
 574         rc = bnxt_hwrm_cfa_counter_cfg(bp, BNXT_DIR_TX,
 575                                        CFA_COUNTER_CFG_IN_COUNTER_TYPE_FC,
 576                                        bp->flow_stat->tx_fc_out_tbl.ctx_id,
 577                                        bp->flow_stat->max_fc,
 578                                        true);
 579
 580         return rc;
 581 }
 582
 583 static int bnxt_alloc_ctx_mem_buf(struct bnxt *bp, char *type, size_t size,
 584                                   struct bnxt_ctx_mem_buf_info *ctx)
 585 {
 586         if (!ctx)
 587                 return -EINVAL;
 588
 589         ctx->va = rte_zmalloc_socket(type, size, 0,
 590                                      bp->eth_dev->device->numa_node);
 591         if (ctx->va == NULL)
 592                 return -ENOMEM;
 593         rte_mem_lock_page(ctx->va);
 594         ctx->size = size;
 595         ctx->dma = rte_mem_virt2iova(ctx->va);
 596         if (ctx->dma == RTE_BAD_IOVA)
 597                 return -ENOMEM;
 598
 599         return 0;
 600 }
 601
 602 static int bnxt_init_fc_ctx_mem(struct bnxt *bp)
 603 {
 604         struct rte_pci_device *pdev = bp->pdev;
 605         char type[RTE_MEMZONE_NAMESIZE];
 606         uint16_t max_fc;
 607         int rc = 0;
 608
 609         max_fc = bp->flow_stat->max_fc;
 610
 611         sprintf(type, "bnxt_rx_fc_in_" PCI_PRI_FMT, pdev->addr.domain,
 612                 pdev->addr.bus, pdev->addr.devid, pdev->addr.function);
 613         /* 4 bytes for each counter-id */
 614         rc = bnxt_alloc_ctx_mem_buf(bp, type,
 615                                     max_fc * 4,
 616                                     &bp->flow_stat->rx_fc_in_tbl);
 617         if (rc)
 618                 return rc;
 619
 620         sprintf(type, "bnxt_rx_fc_out_" PCI_PRI_FMT, pdev->addr.domain,
 621                 pdev->addr.bus, pdev->addr.devid, pdev->addr.function);
 622         /* 16 bytes for each counter - 8 bytes pkt_count, 8 bytes byte_count */
 623         rc = bnxt_alloc_ctx_mem_buf(bp, type,
 624                                     max_fc * 16,
 625                                     &bp->flow_stat->rx_fc_out_tbl);
 626         if (rc)
 627                 return rc;
 628
 629         sprintf(type, "bnxt_tx_fc_in_" PCI_PRI_FMT, pdev->addr.domain,
 630                 pdev->addr.bus, pdev->addr.devid, pdev->addr.function);
 631         /* 4 bytes for each counter-id */
 632         rc = bnxt_alloc_ctx_mem_buf(bp, type,
 633                                     max_fc * 4,
 634                                     &bp->flow_stat->tx_fc_in_tbl);
 635         if (rc)
 636                 return rc;
 637
 638         sprintf(type, "bnxt_tx_fc_out_" PCI_PRI_FMT, pdev->addr.domain,
 639                 pdev->addr.bus, pdev->addr.devid, pdev->addr.function);
 640         /* 16 bytes for each counter - 8 bytes pkt_count, 8 bytes byte_count */
 641         rc = bnxt_alloc_ctx_mem_buf(bp, type,
 642                                     max_fc * 16,
 643                                     &bp->flow_stat->tx_fc_out_tbl);
 644         if (rc)
 645                 return rc;
 646
 647         rc = bnxt_register_fc_ctx_mem(bp);
 648
 649         return rc;
 650 }
 651
 652 static int bnxt_init_ctx_mem(struct bnxt *bp)
 653 {
 654         int rc = 0;
 655
 656         if (!(bp->fw_cap & BNXT_FW_CAP_ADV_FLOW_COUNTERS) ||
 657             !(BNXT_PF(bp) || BNXT_VF_IS_TRUSTED(bp)) ||
 658             !BNXT_FLOW_XSTATS_EN(bp))
 659                 return 0;
 660
 661         rc = bnxt_hwrm_cfa_counter_qcaps(bp, &bp->flow_stat->max_fc);
 662         if (rc)
 663                 return rc;
 664
 665         rc = bnxt_init_fc_ctx_mem(bp);
 666
 667         return rc;
 668 }
 669
 670 static int bnxt_update_phy_setting(struct bnxt *bp)
 671 {
 672         struct rte_eth_link new;
 673         int rc;
 674
 675         rc = bnxt_get_hwrm_link_config(bp, &new);
 676         if (rc) {
 677                 PMD_DRV_LOG(ERR, "Failed to get link settings\n");
 678                 return rc;
 679         }
 680
 681         /*
 682          * On BCM957508-N2100 adapters, FW will not allow any user other
 683          * than BMC to shutdown the port. bnxt_get_hwrm_link_config() call
 684          * always returns link up. Force phy update always in that case.
 685          */
 686         if (!new.link_status || IS_BNXT_DEV_957508_N2100(bp)) {
 687                 rc = bnxt_set_hwrm_link_config(bp, true);
 688                 if (rc) {
 689                         PMD_DRV_LOG(ERR, "Failed to update PHY settings\n");
 690                         return rc;
 691                 }
 692         }
 693
 694         return rc;
 695 }
 696
 697 static void bnxt_free_prev_ring_stats(struct bnxt *bp)
 698 {
 699         rte_free(bp->prev_rx_ring_stats);
 700         rte_free(bp->prev_tx_ring_stats);
 701
 702         bp->prev_rx_ring_stats = NULL;
 703         bp->prev_tx_ring_stats = NULL;
 704 }
 705
 706 static int bnxt_alloc_prev_ring_stats(struct bnxt *bp)
 707 {
 708         bp->prev_rx_ring_stats =  rte_zmalloc("bnxt_prev_rx_ring_stats",
 709                                               sizeof(struct bnxt_ring_stats) *
 710                                               bp->rx_cp_nr_rings,
 711                                               0);
 712         if (bp->prev_rx_ring_stats == NULL)
 713                 return -ENOMEM;
 714
 715         bp->prev_tx_ring_stats = rte_zmalloc("bnxt_prev_tx_ring_stats",
 716                                              sizeof(struct bnxt_ring_stats) *
 717                                              bp->tx_cp_nr_rings,
 718                                              0);
 719         if (bp->prev_tx_ring_stats == NULL)
 720                 goto error;
 721
 722         return 0;
 723
 724 error:
 725         bnxt_free_prev_ring_stats(bp);
 726         return -ENOMEM;
 727 }
 728
 729 static int bnxt_start_nic(struct bnxt *bp)
 730 {
 731         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(bp->eth_dev);
 732         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 733         uint32_t intr_vector = 0;
 734         uint32_t queue_id, base = BNXT_MISC_VEC_ID;
 735         uint32_t vec = BNXT_MISC_VEC_ID;
 736         unsigned int i, j;
 737         int rc;
 738
 739         if (bp->eth_dev->data->mtu > RTE_ETHER_MTU) {
 740                 bp->eth_dev->data->dev_conf.rxmode.offloads |=
 741                         DEV_RX_OFFLOAD_JUMBO_FRAME;
 742                 bp->flags |= BNXT_FLAG_JUMBO;
 743         } else {
 744                 bp->eth_dev->data->dev_conf.rxmode.offloads &=
 745                         ~DEV_RX_OFFLOAD_JUMBO_FRAME;
 746                 bp->flags &= ~BNXT_FLAG_JUMBO;
 747         }
 748
 749         /* THOR does not support ring groups.
 750          * But we will use the array to save RSS context IDs.
 751          */
 752         if (BNXT_CHIP_P5(bp))
 753                 bp->max_ring_grps = BNXT_MAX_RSS_CTXTS_P5;
 754
 755         rc = bnxt_alloc_all_hwrm_stat_ctxs(bp);
 756         if (rc) {
 757                 PMD_DRV_LOG(ERR, "HWRM stat ctx alloc failure rc: %x\n", rc);
 758                 goto err_out;
 759         }
 760
 761         rc = bnxt_alloc_hwrm_rings(bp);
 762         if (rc) {
 763                 PMD_DRV_LOG(ERR, "HWRM ring alloc failure rc: %x\n", rc);
 764                 goto err_out;
 765         }
 766
 767         rc = bnxt_alloc_all_hwrm_ring_grps(bp);
 768         if (rc) {
 769                 PMD_DRV_LOG(ERR, "HWRM ring grp alloc failure: %x\n", rc);
 770                 goto err_out;
 771         }
 772
 773         if (!(bp->vnic_cap_flags & BNXT_VNIC_CAP_COS_CLASSIFY))
 774                 goto skip_cosq_cfg;
 775
 776         for (j = 0, i = 0; i < BNXT_COS_QUEUE_COUNT; i++) {
 777                 if (bp->rx_cos_queue[i].id != 0xff) {
 778                         struct bnxt_vnic_info *vnic = &bp->vnic_info[j++];
 779
 780                         if (!vnic) {
 781                                 PMD_DRV_LOG(ERR,
 782                                             "Num pools more than FW profile\n");
 783                                 rc = -EINVAL;
 784                                 goto err_out;
 785                         }
 786                         vnic->cos_queue_id = bp->rx_cos_queue[i].id;
 787                         bp->rx_cosq_cnt++;
 788                 }
 789         }
 790
 791 skip_cosq_cfg:
 792         rc = bnxt_mq_rx_configure(bp);
 793         if (rc) {
 794                 PMD_DRV_LOG(ERR, "MQ mode configure failure rc: %x\n", rc);
 795                 goto err_out;
 796         }
 797
 798         /* default vnic 0 */
 799         rc = bnxt_setup_one_vnic(bp, 0);
 800         if (rc)
 801                 goto err_out;
 802         /* VNIC configuration */
 803         if (BNXT_RFS_NEEDS_VNIC(bp)) {
 804                 for (i = 1; i < bp->nr_vnics; i++) {
 805                         rc = bnxt_setup_one_vnic(bp, i);
 806                         if (rc)
 807                                 goto err_out;
 808                 }
 809         }
 810
 811         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, &bp->vnic_info[0], 0, NULL);
 812         if (rc) {
 813                 PMD_DRV_LOG(ERR,
 814                         "HWRM cfa l2 rx mask failure rc: %x\n", rc);
 815                 goto err_out;
 816         }
 817
 818         /* check and configure queue intr-vector mapping */
 819         if ((rte_intr_cap_multiple(intr_handle) ||
 820              !RTE_ETH_DEV_SRIOV(bp->eth_dev).active) &&
 821             bp->eth_dev->data->dev_conf.intr_conf.rxq != 0) {
 822                 intr_vector = bp->eth_dev->data->nb_rx_queues;
 823                 PMD_DRV_LOG(DEBUG, "intr_vector = %d\n", intr_vector);
 824                 if (intr_vector > bp->rx_cp_nr_rings) {
 825                         PMD_DRV_LOG(ERR, "At most %d intr queues supported",
 826                                         bp->rx_cp_nr_rings);
 827                         return -ENOTSUP;
 828                 }
 829                 rc = rte_intr_efd_enable(intr_handle, intr_vector);
 830                 if (rc)
 831                         return rc;
 832         }
 833
 834         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
 835                 intr_handle->intr_vec =
 836                         rte_zmalloc("intr_vec",
 837                                     bp->eth_dev->data->nb_rx_queues *
 838                                     sizeof(int), 0);
 839                 if (intr_handle->intr_vec == NULL) {
 840                         PMD_DRV_LOG(ERR, "Failed to allocate %d rx_queues"
 841                                 " intr_vec", bp->eth_dev->data->nb_rx_queues);
 842                         rc = -ENOMEM;
 843                         goto err_out;
 844                 }
 845                 PMD_DRV_LOG(DEBUG, "intr_handle->intr_vec = %p "
 846                         "intr_handle->nb_efd = %d intr_handle->max_intr = %d\n",
 847                          intr_handle->intr_vec, intr_handle->nb_efd,
 848                         intr_handle->max_intr);
 849                 for (queue_id = 0; queue_id < bp->eth_dev->data->nb_rx_queues;
 850                      queue_id++) {
 851                         intr_handle->intr_vec[queue_id] =
 852                                                         vec + BNXT_RX_VEC_START;
 853                         if (vec < base + intr_handle->nb_efd - 1)
 854                                 vec++;
 855                 }
 856         }
 857
 858         /* enable uio/vfio intr/eventfd mapping */
 859         rc = rte_intr_enable(intr_handle);
 860 #ifndef RTE_EXEC_ENV_FREEBSD
 861         /* In FreeBSD OS, nic_uio driver does not support interrupts */
 862         if (rc)
 863                 goto err_out;
 864 #endif
 865
 866         rc = bnxt_update_phy_setting(bp);
 867         if (rc)
 868                 goto err_out;
 869
 870         bp->mark_table = rte_zmalloc("bnxt_mark_table", BNXT_MARK_TABLE_SZ, 0);
 871         if (!bp->mark_table)
 872                 PMD_DRV_LOG(ERR, "Allocation of mark table failed\n");
 873
 874         return 0;
 875
 876 err_out:
 877         /* Some of the error status returned by FW may not be from errno.h */
 878         if (rc > 0)
 879                 rc = -EIO;
 880
 881         return rc;
 882 }
 883
 884 static int bnxt_shutdown_nic(struct bnxt *bp)
 885 {
 886         bnxt_free_all_hwrm_resources(bp);
 887         bnxt_free_all_filters(bp);
 888         bnxt_free_all_vnics(bp);
 889         return 0;
 890 }
 891
 892 /*
 893  * Device configuration and status function
 894  */
 895
 896 uint32_t bnxt_get_speed_capabilities(struct bnxt *bp)
 897 {
 898         uint32_t link_speed = 0;
 899         uint32_t speed_capa = 0;
 900
 901         if (bp->link_info == NULL)
 902                 return 0;
 903
 904         link_speed = bp->link_info->support_speeds;
 905
 906         /* If PAM4 is configured, use PAM4 supported speed */
 907         if (link_speed == 0 && bp->link_info->support_pam4_speeds > 0)
 908                 link_speed = bp->link_info->support_pam4_speeds;
 909
 910         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_100MB)
 911                 speed_capa |= ETH_LINK_SPEED_100M;
 912         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100MBHD)
 913                 speed_capa |= ETH_LINK_SPEED_100M_HD;
 914         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_1GB)
 915                 speed_capa |= ETH_LINK_SPEED_1G;
 916         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_2_5GB)
 917                 speed_capa |= ETH_LINK_SPEED_2_5G;
 918         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10GB)
 919                 speed_capa |= ETH_LINK_SPEED_10G;
 920         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_20GB)
 921                 speed_capa |= ETH_LINK_SPEED_20G;
 922         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_25GB)
 923                 speed_capa |= ETH_LINK_SPEED_25G;
 924         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_40GB)
 925                 speed_capa |= ETH_LINK_SPEED_40G;
 926         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_50GB)
 927                 speed_capa |= ETH_LINK_SPEED_50G;
 928         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100GB)
 929                 speed_capa |= ETH_LINK_SPEED_100G;
 930         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_PAM4_SPEEDS_50G)
 931                 speed_capa |= ETH_LINK_SPEED_50G;
 932         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_PAM4_SPEEDS_100G)
 933                 speed_capa |= ETH_LINK_SPEED_100G;
 934         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_PAM4_SPEEDS_200G)
 935                 speed_capa |= ETH_LINK_SPEED_200G;
 936
 937         if (bp->link_info->auto_mode ==
 938             HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_MODE_NONE)
 939                 speed_capa |= ETH_LINK_SPEED_FIXED;
 940
 941         return speed_capa;
 942 }
 943
 944 static int bnxt_dev_info_get_op(struct rte_eth_dev *eth_dev,
 945                                 struct rte_eth_dev_info *dev_info)
 946 {
 947         struct rte_pci_device *pdev = RTE_DEV_TO_PCI(eth_dev->device);
 948         struct bnxt *bp = eth_dev->data->dev_private;
 949         uint16_t max_vnics, i, j, vpool, vrxq;
 950         unsigned int max_rx_rings;
 951         int rc;
 952
 953         rc = is_bnxt_in_error(bp);
 954         if (rc)
 955                 return rc;
 956
 957         /* MAC Specifics */
 958         dev_info->max_mac_addrs = bp->max_l2_ctx;
 959         dev_info->max_hash_mac_addrs = 0;
 960
 961         /* PF/VF specifics */
 962         if (BNXT_PF(bp))
 963                 dev_info->max_vfs = pdev->max_vfs;
 964
 965         max_rx_rings = bnxt_max_rings(bp);
 966         /* For the sake of symmetry, max_rx_queues = max_tx_queues */
 967         dev_info->max_rx_queues = max_rx_rings;
 968         dev_info->max_tx_queues = max_rx_rings;
 969         dev_info->reta_size = bnxt_rss_hash_tbl_size(bp);
 970         dev_info->hash_key_size = HW_HASH_KEY_SIZE;
 971         max_vnics = bp->max_vnics;
 972
 973         /* MTU specifics */
 974         dev_info->min_mtu = RTE_ETHER_MIN_MTU;
 975         dev_info->max_mtu = BNXT_MAX_MTU;
 976
 977         /* Fast path specifics */
 978         dev_info->min_rx_bufsize = 1;
 979         dev_info->max_rx_pktlen = BNXT_MAX_PKT_LEN;
 980
 981         dev_info->rx_offload_capa = BNXT_DEV_RX_OFFLOAD_SUPPORT;
 982         if (bp->flags & BNXT_FLAG_PTP_SUPPORTED)
 983                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TIMESTAMP;
 984         dev_info->tx_queue_offload_capa = DEV_TX_OFFLOAD_MBUF_FAST_FREE;
 985         dev_info->tx_offload_capa = BNXT_DEV_TX_OFFLOAD_SUPPORT |
 986                                     dev_info->tx_queue_offload_capa;
 987         dev_info->flow_type_rss_offloads = BNXT_ETH_RSS_SUPPORT;
 988
 989         dev_info->speed_capa = bnxt_get_speed_capabilities(bp);
 990         dev_info->dev_capa = RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP |
 991                              RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP;
 992
 993         dev_info->default_rxconf = (struct rte_eth_rxconf) {
 994                 .rx_thresh = {
 995                         .pthresh = 8,
 996                         .hthresh = 8,
 997                         .wthresh = 0,
 998                 },
 999                 .rx_free_thresh = 32,
1000                 .rx_drop_en = BNXT_DEFAULT_RX_DROP_EN,
1001         };
1002
1003         dev_info->default_txconf = (struct rte_eth_txconf) {
1004                 .tx_thresh = {
1005                         .pthresh = 32,
1006                         .hthresh = 0,
1007                         .wthresh = 0,
1008                 },
1009                 .tx_free_thresh = 32,
1010                 .tx_rs_thresh = 32,
1011         };
1012         eth_dev->data->dev_conf.intr_conf.lsc = 1;
1013
1014         eth_dev->data->dev_conf.intr_conf.rxq = 1;
1015         dev_info->rx_desc_lim.nb_min = BNXT_MIN_RING_DESC;
1016         dev_info->rx_desc_lim.nb_max = BNXT_MAX_RX_RING_DESC;
1017         dev_info->tx_desc_lim.nb_min = BNXT_MIN_RING_DESC;
1018         dev_info->tx_desc_lim.nb_max = BNXT_MAX_TX_RING_DESC;
1019
1020         if (BNXT_PF(bp) || BNXT_VF_IS_TRUSTED(bp)) {
1021                 dev_info->switch_info.name = eth_dev->device->name;
1022                 dev_info->switch_info.domain_id = bp->switch_domain_id;
1023                 dev_info->switch_info.port_id =
1024                                 BNXT_PF(bp) ? BNXT_SWITCH_PORT_ID_PF :
1025                                     BNXT_SWITCH_PORT_ID_TRUSTED_VF;
1026         }
1027
1028         /*
1029          * TODO: default_rxconf, default_txconf, rx_desc_lim, and tx_desc_lim
1030          *       need further investigation.
1031          */
1032
1033         /* VMDq resources */
1034         vpool = 64; /* ETH_64_POOLS */
1035         vrxq = 128; /* ETH_VMDQ_DCB_NUM_QUEUES */
1036         for (i = 0; i < 4; vpool >>= 1, i++) {
1037                 if (max_vnics > vpool) {
1038                         for (j = 0; j < 5; vrxq >>= 1, j++) {
1039                                 if (dev_info->max_rx_queues > vrxq) {
1040                                         if (vpool > vrxq)
1041                                                 vpool = vrxq;
1042                                         goto found;
1043                                 }
1044                         }
1045                         /* Not enough resources to support VMDq */
1046                         break;
1047                 }
1048         }
1049         /* Not enough resources to support VMDq */
1050         vpool = 0;
1051         vrxq = 0;
1052 found:
1053         dev_info->max_vmdq_pools = vpool;
1054         dev_info->vmdq_queue_num = vrxq;
1055
1056         dev_info->vmdq_pool_base = 0;
1057         dev_info->vmdq_queue_base = 0;
1058
1059         return 0;
1060 }
1061
1062 /* Configure the device based on the configuration provided */
1063 static int bnxt_dev_configure_op(struct rte_eth_dev *eth_dev)
1064 {
1065         struct bnxt *bp = eth_dev->data->dev_private;
1066         uint64_t rx_offloads = eth_dev->data->dev_conf.rxmode.offloads;
1067         int rc;
1068
1069         bp->rx_queues = (void *)eth_dev->data->rx_queues;
1070         bp->tx_queues = (void *)eth_dev->data->tx_queues;
1071         bp->tx_nr_rings = eth_dev->data->nb_tx_queues;
1072         bp->rx_nr_rings = eth_dev->data->nb_rx_queues;
1073
1074         rc = is_bnxt_in_error(bp);
1075         if (rc)
1076                 return rc;
1077
1078         if (BNXT_VF(bp) && (bp->flags & BNXT_FLAG_NEW_RM)) {
1079                 rc = bnxt_hwrm_check_vf_rings(bp);
1080                 if (rc) {
1081                         PMD_DRV_LOG(ERR, "HWRM insufficient resources\n");
1082                         return -ENOSPC;
1083                 }
1084
1085                 /* If a resource has already been allocated - in this case
1086                  * it is the async completion ring, free it. Reallocate it after
1087                  * resource reservation. This will ensure the resource counts
1088                  * are calculated correctly.
1089                  */
1090
1091                 pthread_mutex_lock(&bp->def_cp_lock);
1092
1093                 if (!BNXT_HAS_NQ(bp) && bp->async_cp_ring) {
1094                         bnxt_disable_int(bp);
1095                         bnxt_free_cp_ring(bp, bp->async_cp_ring);
1096                 }
1097
1098                 rc = bnxt_hwrm_func_reserve_vf_resc(bp, false);
1099                 if (rc) {
1100                         PMD_DRV_LOG(ERR, "HWRM resource alloc fail:%x\n", rc);
1101                         pthread_mutex_unlock(&bp->def_cp_lock);
1102                         return -ENOSPC;
1103                 }
1104
1105                 if (!BNXT_HAS_NQ(bp) && bp->async_cp_ring) {
1106                         rc = bnxt_alloc_async_cp_ring(bp);
1107                         if (rc) {
1108                                 pthread_mutex_unlock(&bp->def_cp_lock);
1109                                 return rc;
1110                         }
1111                         bnxt_enable_int(bp);
1112                 }
1113
1114                 pthread_mutex_unlock(&bp->def_cp_lock);
1115         }
1116
1117         /* Inherit new configurations */
1118         if (eth_dev->data->nb_rx_queues > bp->max_rx_rings ||
1119             eth_dev->data->nb_tx_queues > bp->max_tx_rings ||
1120             eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues
1121                 + BNXT_NUM_ASYNC_CPR(bp) > bp->max_cp_rings ||
1122             eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues >
1123             bp->max_stat_ctx)
1124                 goto resource_error;
1125
1126         if (BNXT_HAS_RING_GRPS(bp) &&
1127             (uint32_t)(eth_dev->data->nb_rx_queues) > bp->max_ring_grps)
1128                 goto resource_error;
1129
1130         if (!(eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS) &&
1131             bp->max_vnics < eth_dev->data->nb_rx_queues)
1132                 goto resource_error;
1133
1134         bp->rx_cp_nr_rings = bp->rx_nr_rings;
1135         bp->tx_cp_nr_rings = bp->tx_nr_rings;
1136
1137         if (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)
1138                 rx_offloads |= DEV_RX_OFFLOAD_RSS_HASH;
1139         eth_dev->data->dev_conf.rxmode.offloads = rx_offloads;
1140
1141         if (rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
1142                 eth_dev->data->mtu =
1143                         eth_dev->data->dev_conf.rxmode.max_rx_pkt_len -
1144                         RTE_ETHER_HDR_LEN - RTE_ETHER_CRC_LEN - VLAN_TAG_SIZE *
1145                         BNXT_NUM_VLANS;
1146                 bnxt_mtu_set_op(eth_dev, eth_dev->data->mtu);
1147         }
1148         return 0;
1149
1150 resource_error:
1151         PMD_DRV_LOG(ERR,
1152                     "Insufficient resources to support requested config\n");
1153         PMD_DRV_LOG(ERR,
1154                     "Num Queues Requested: Tx %d, Rx %d\n",
1155                     eth_dev->data->nb_tx_queues,
1156                     eth_dev->data->nb_rx_queues);
1157         PMD_DRV_LOG(ERR,
1158                     "MAX: TxQ %d, RxQ %d, CQ %d Stat %d, Grp %d, Vnic %d\n",
1159                     bp->max_tx_rings, bp->max_rx_rings, bp->max_cp_rings,
1160                     bp->max_stat_ctx, bp->max_ring_grps, bp->max_vnics);
1161         return -ENOSPC;
1162 }
1163
1164 void bnxt_print_link_info(struct rte_eth_dev *eth_dev)
1165 {
1166         struct rte_eth_link *link = &eth_dev->data->dev_link;
1167
1168         if (link->link_status)
1169                 PMD_DRV_LOG(INFO, "Port %d Link Up - speed %u Mbps - %s\n",
1170                         eth_dev->data->port_id,
1171                         (uint32_t)link->link_speed,
1172                         (link->link_duplex == ETH_LINK_FULL_DUPLEX) ?
1173                         ("full-duplex") : ("half-duplex\n"));
1174         else
1175                 PMD_DRV_LOG(INFO, "Port %d Link Down\n",
1176                         eth_dev->data->port_id);
1177 }
1178
1179 /*
1180  * Determine whether the current configuration requires support for scattered
1181  * receive; return 1 if scattered receive is required and 0 if not.
1182  */
1183 static int bnxt_scattered_rx(struct rte_eth_dev *eth_dev)
1184 {
1185         uint16_t buf_size;
1186         int i;
1187
1188         if (eth_dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_SCATTER)
1189                 return 1;
1190
1191         if (eth_dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_TCP_LRO)
1192                 return 1;
1193
1194         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
1195                 struct bnxt_rx_queue *rxq = eth_dev->data->rx_queues[i];
1196
1197                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
1198                                       RTE_PKTMBUF_HEADROOM);
1199                 if (eth_dev->data->dev_conf.rxmode.max_rx_pkt_len > buf_size)
1200                         return 1;
1201         }
1202         return 0;
1203 }
1204
1205 static eth_rx_burst_t
1206 bnxt_receive_function(struct rte_eth_dev *eth_dev)
1207 {
1208         struct bnxt *bp = eth_dev->data->dev_private;
1209
1210         /* Disable vector mode RX for Stingray2 for now */
1211         if (BNXT_CHIP_SR2(bp)) {
1212                 bp->flags &= ~BNXT_FLAG_RX_VECTOR_PKT_MODE;
1213                 return bnxt_recv_pkts;
1214         }
1215
1216 #if (defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)) && \
1217         !defined(RTE_LIBRTE_IEEE1588)
1218
1219         /* Vector mode receive cannot be enabled if scattered rx is in use. */
1220         if (eth_dev->data->scattered_rx)
1221                 goto use_scalar_rx;
1222
1223         /*
1224          * Vector mode receive cannot be enabled if Truflow is enabled or if
1225          * asynchronous completions and receive completions can be placed in
1226          * the same completion ring.
1227          */
1228         if (BNXT_TRUFLOW_EN(bp) || !BNXT_NUM_ASYNC_CPR(bp))
1229                 goto use_scalar_rx;
1230
1231         /*
1232          * Vector mode receive cannot be enabled if any receive offloads outside
1233          * a limited subset have been enabled.
1234          */
1235         if (eth_dev->data->dev_conf.rxmode.offloads &
1236                 ~(DEV_RX_OFFLOAD_VLAN_STRIP |
1237                   DEV_RX_OFFLOAD_KEEP_CRC |
1238                   DEV_RX_OFFLOAD_JUMBO_FRAME |
1239                   DEV_RX_OFFLOAD_IPV4_CKSUM |
1240                   DEV_RX_OFFLOAD_UDP_CKSUM |
1241                   DEV_RX_OFFLOAD_TCP_CKSUM |
1242                   DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
1243                   DEV_RX_OFFLOAD_OUTER_UDP_CKSUM |
1244                   DEV_RX_OFFLOAD_RSS_HASH |
1245                   DEV_RX_OFFLOAD_VLAN_FILTER))
1246                 goto use_scalar_rx;
1247
1248 #if defined(RTE_ARCH_X86) && defined(CC_AVX2_SUPPORT)
1249         if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256 &&
1250             rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1) {
1251                 PMD_DRV_LOG(INFO,
1252                             "Using AVX2 vector mode receive for port %d\n",
1253                             eth_dev->data->port_id);
1254                 bp->flags |= BNXT_FLAG_RX_VECTOR_PKT_MODE;
1255                 return bnxt_recv_pkts_vec_avx2;
1256         }
1257  #endif
1258         if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
1259                 PMD_DRV_LOG(INFO,
1260                             "Using SSE vector mode receive for port %d\n",
1261                             eth_dev->data->port_id);
1262                 bp->flags |= BNXT_FLAG_RX_VECTOR_PKT_MODE;
1263                 return bnxt_recv_pkts_vec;
1264         }
1265
1266 use_scalar_rx:
1267         PMD_DRV_LOG(INFO, "Vector mode receive disabled for port %d\n",
1268                     eth_dev->data->port_id);
1269         PMD_DRV_LOG(INFO,
1270                     "Port %d scatter: %d rx offload: %" PRIX64 "\n",
1271                     eth_dev->data->port_id,
1272                     eth_dev->data->scattered_rx,
1273                     eth_dev->data->dev_conf.rxmode.offloads);
1274 #endif
1275         bp->flags &= ~BNXT_FLAG_RX_VECTOR_PKT_MODE;
1276         return bnxt_recv_pkts;
1277 }
1278
1279 static eth_tx_burst_t
1280 bnxt_transmit_function(struct rte_eth_dev *eth_dev)
1281 {
1282         struct bnxt *bp = eth_dev->data->dev_private;
1283
1284         /* Disable vector mode TX for Stingray2 for now */
1285         if (BNXT_CHIP_SR2(bp))
1286                 return bnxt_xmit_pkts;
1287
1288 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64) && \
1289         !defined(RTE_LIBRTE_IEEE1588)
1290         uint64_t offloads = eth_dev->data->dev_conf.txmode.offloads;
1291
1292         /*
1293          * Vector mode transmit can be enabled only if not using scatter rx
1294          * or tx offloads.
1295          */
1296         if (eth_dev->data->scattered_rx ||
1297             (offloads & ~DEV_TX_OFFLOAD_MBUF_FAST_FREE) ||
1298             BNXT_TRUFLOW_EN(bp))
1299                 goto use_scalar_tx;
1300
1301 #if defined(RTE_ARCH_X86) && defined(CC_AVX2_SUPPORT)
1302         if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256 &&
1303             rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1) {
1304                 PMD_DRV_LOG(INFO,
1305                             "Using AVX2 vector mode transmit for port %d\n",
1306                             eth_dev->data->port_id);
1307                 return bnxt_xmit_pkts_vec_avx2;
1308         }
1309 #endif
1310         if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
1311                 PMD_DRV_LOG(INFO,
1312                             "Using SSE vector mode transmit for port %d\n",
1313                             eth_dev->data->port_id);
1314                 return bnxt_xmit_pkts_vec;
1315         }
1316
1317 use_scalar_tx:
1318         PMD_DRV_LOG(INFO, "Vector mode transmit disabled for port %d\n",
1319                     eth_dev->data->port_id);
1320         PMD_DRV_LOG(INFO,
1321                     "Port %d scatter: %d tx offload: %" PRIX64 "\n",
1322                     eth_dev->data->port_id,
1323                     eth_dev->data->scattered_rx,
1324                     offloads);
1325 #endif
1326         return bnxt_xmit_pkts;
1327 }
1328
1329 static int bnxt_handle_if_change_status(struct bnxt *bp)
1330 {
1331         int rc;
1332
1333         /* Since fw has undergone a reset and lost all contexts,
1334          * set fatal flag to not issue hwrm during cleanup
1335          */
1336         bp->flags |= BNXT_FLAG_FATAL_ERROR;
1337         bnxt_uninit_resources(bp, true);
1338
1339         /* clear fatal flag so that re-init happens */
1340         bp->flags &= ~BNXT_FLAG_FATAL_ERROR;
1341         rc = bnxt_init_resources(bp, true);
1342
1343         bp->flags &= ~BNXT_FLAG_IF_CHANGE_HOT_FW_RESET_DONE;
1344
1345         return rc;
1346 }
1347
1348 static int bnxt_dev_set_link_up_op(struct rte_eth_dev *eth_dev)
1349 {
1350         struct bnxt *bp = eth_dev->data->dev_private;
1351         int rc = 0;
1352
1353         if (!BNXT_SINGLE_PF(bp))
1354                 return -ENOTSUP;
1355
1356         if (!bp->link_info->link_up)
1357                 rc = bnxt_set_hwrm_link_config(bp, true);
1358         if (!rc)
1359                 eth_dev->data->dev_link.link_status = 1;
1360
1361         bnxt_print_link_info(eth_dev);
1362         return rc;
1363 }
1364
1365 static int bnxt_dev_set_link_down_op(struct rte_eth_dev *eth_dev)
1366 {
1367         struct bnxt *bp = eth_dev->data->dev_private;
1368
1369         if (!BNXT_SINGLE_PF(bp))
1370                 return -ENOTSUP;
1371
1372         eth_dev->data->dev_link.link_status = 0;
1373         bnxt_set_hwrm_link_config(bp, false);
1374         bp->link_info->link_up = 0;
1375
1376         return 0;
1377 }
1378
1379 static void bnxt_free_switch_domain(struct bnxt *bp)
1380 {
1381         int rc = 0;
1382
1383         if (!(BNXT_PF(bp) || BNXT_VF_IS_TRUSTED(bp)))
1384                 return;
1385
1386         rc = rte_eth_switch_domain_free(bp->switch_domain_id);
1387         if (rc)
1388                 PMD_DRV_LOG(ERR, "free switch domain:%d fail: %d\n",
1389                             bp->switch_domain_id, rc);
1390 }
1391
1392 static void bnxt_ptp_get_current_time(void *arg)
1393 {
1394         struct bnxt *bp = arg;
1395         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
1396         int rc;
1397
1398         rc = is_bnxt_in_error(bp);
1399         if (rc)
1400                 return;
1401
1402         if (!ptp)
1403                 return;
1404
1405         bnxt_hwrm_port_ts_query(bp, BNXT_PTP_FLAGS_CURRENT_TIME,
1406                                 &ptp->current_time);
1407
1408         rc = rte_eal_alarm_set(US_PER_S, bnxt_ptp_get_current_time, (void *)bp);
1409         if (rc != 0) {
1410                 PMD_DRV_LOG(ERR, "Failed to re-schedule PTP alarm\n");
1411                 bp->flags2 &= ~BNXT_FLAGS2_PTP_ALARM_SCHEDULED;
1412         }
1413 }
1414
1415 static int bnxt_schedule_ptp_alarm(struct bnxt *bp)
1416 {
1417         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
1418         int rc;
1419
1420         if (bp->flags2 & BNXT_FLAGS2_PTP_ALARM_SCHEDULED)
1421                 return 0;
1422
1423         bnxt_hwrm_port_ts_query(bp, BNXT_PTP_FLAGS_CURRENT_TIME,
1424                                 &ptp->current_time);
1425
1426         rc = rte_eal_alarm_set(US_PER_S, bnxt_ptp_get_current_time, (void *)bp);
1427         return rc;
1428 }
1429
1430 static void bnxt_cancel_ptp_alarm(struct bnxt *bp)
1431 {
1432         if (bp->flags2 & BNXT_FLAGS2_PTP_ALARM_SCHEDULED) {
1433                 rte_eal_alarm_cancel(bnxt_ptp_get_current_time, (void *)bp);
1434                 bp->flags2 &= ~BNXT_FLAGS2_PTP_ALARM_SCHEDULED;
1435         }
1436 }
1437
1438 static void bnxt_ptp_stop(struct bnxt *bp)
1439 {
1440         bnxt_cancel_ptp_alarm(bp);
1441         bp->flags2 &= ~BNXT_FLAGS2_PTP_TIMESYNC_ENABLED;
1442 }
1443
1444 static int bnxt_ptp_start(struct bnxt *bp)
1445 {
1446         int rc;
1447
1448         rc = bnxt_schedule_ptp_alarm(bp);
1449         if (rc != 0) {
1450                 PMD_DRV_LOG(ERR, "Failed to schedule PTP alarm\n");
1451         } else {
1452                 bp->flags2 |= BNXT_FLAGS2_PTP_TIMESYNC_ENABLED;
1453                 bp->flags2 |= BNXT_FLAGS2_PTP_ALARM_SCHEDULED;
1454         }
1455
1456         return rc;
1457 }
1458
1459 static int bnxt_dev_stop(struct rte_eth_dev *eth_dev)
1460 {
1461         struct bnxt *bp = eth_dev->data->dev_private;
1462         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
1463         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
1464         struct rte_eth_link link;
1465         int ret;
1466
1467         eth_dev->data->dev_started = 0;
1468         eth_dev->data->scattered_rx = 0;
1469
1470         /* Prevent crashes when queues are still in use */
1471         eth_dev->rx_pkt_burst = &bnxt_dummy_recv_pkts;
1472         eth_dev->tx_pkt_burst = &bnxt_dummy_xmit_pkts;
1473
1474         bnxt_disable_int(bp);
1475
1476         /* disable uio/vfio intr/eventfd mapping */
1477         rte_intr_disable(intr_handle);
1478
1479         /* Stop the child representors for this device */
1480         ret = bnxt_rep_stop_all(bp);
1481         if (ret != 0)
1482                 return ret;
1483
1484         /* delete the bnxt ULP port details */
1485         bnxt_ulp_port_deinit(bp);
1486
1487         bnxt_cancel_fw_health_check(bp);
1488
1489         if (BNXT_P5_PTP_TIMESYNC_ENABLED(bp))
1490                 bnxt_cancel_ptp_alarm(bp);
1491
1492         /* Do not bring link down during reset recovery */
1493         if (!is_bnxt_in_error(bp)) {
1494                 bnxt_dev_set_link_down_op(eth_dev);
1495                 /* Wait for link to be reset */
1496                 if (BNXT_SINGLE_PF(bp))
1497                         rte_delay_ms(500);
1498                 /* clear the recorded link status */
1499                 memset(&link, 0, sizeof(link));
1500                 rte_eth_linkstatus_set(eth_dev, &link);
1501         }
1502
1503         /* Clean queue intr-vector mapping */
1504         rte_intr_efd_disable(intr_handle);
1505         if (intr_handle->intr_vec != NULL) {
1506                 rte_free(intr_handle->intr_vec);
1507                 intr_handle->intr_vec = NULL;
1508         }
1509
1510         bnxt_hwrm_port_clr_stats(bp);
1511         bnxt_free_tx_mbufs(bp);
1512         bnxt_free_rx_mbufs(bp);
1513         /* Process any remaining notifications in default completion queue */
1514         bnxt_int_handler(eth_dev);
1515         bnxt_shutdown_nic(bp);
1516         bnxt_hwrm_if_change(bp, false);
1517
1518         bnxt_free_prev_ring_stats(bp);
1519         rte_free(bp->mark_table);
1520         bp->mark_table = NULL;
1521
1522         bp->flags &= ~BNXT_FLAG_RX_VECTOR_PKT_MODE;
1523         bp->rx_cosq_cnt = 0;
1524         /* All filters are deleted on a port stop. */
1525         if (BNXT_FLOW_XSTATS_EN(bp))
1526                 bp->flow_stat->flow_count = 0;
1527
1528         return 0;
1529 }
1530
1531 /* Unload the driver, release resources */
1532 static int bnxt_dev_stop_op(struct rte_eth_dev *eth_dev)
1533 {
1534         struct bnxt *bp = eth_dev->data->dev_private;
1535
1536         pthread_mutex_lock(&bp->err_recovery_lock);
1537         if (bp->flags & BNXT_FLAG_FW_RESET) {
1538                 PMD_DRV_LOG(ERR,
1539                             "Adapter recovering from error..Please retry\n");
1540                 pthread_mutex_unlock(&bp->err_recovery_lock);
1541                 return -EAGAIN;
1542         }
1543         pthread_mutex_unlock(&bp->err_recovery_lock);
1544
1545         return bnxt_dev_stop(eth_dev);
1546 }
1547
1548 static int bnxt_dev_start_op(struct rte_eth_dev *eth_dev)
1549 {
1550         struct bnxt *bp = eth_dev->data->dev_private;
1551         uint64_t rx_offloads = eth_dev->data->dev_conf.rxmode.offloads;
1552         int vlan_mask = 0;
1553         int rc, retry_cnt = BNXT_IF_CHANGE_RETRY_COUNT;
1554
1555         if (!eth_dev->data->nb_tx_queues || !eth_dev->data->nb_rx_queues) {
1556                 PMD_DRV_LOG(ERR, "Queues are not configured yet!\n");
1557                 return -EINVAL;
1558         }
1559
1560         if (bp->rx_cp_nr_rings > RTE_ETHDEV_QUEUE_STAT_CNTRS)
1561                 PMD_DRV_LOG(ERR,
1562                             "RxQ cnt %d > RTE_ETHDEV_QUEUE_STAT_CNTRS %d\n",
1563                             bp->rx_cp_nr_rings, RTE_ETHDEV_QUEUE_STAT_CNTRS);
1564
1565         do {
1566                 rc = bnxt_hwrm_if_change(bp, true);
1567                 if (rc == 0 || rc != -EAGAIN)
1568                         break;
1569
1570                 rte_delay_ms(BNXT_IF_CHANGE_RETRY_INTERVAL);
1571         } while (retry_cnt--);
1572
1573         if (rc)
1574                 return rc;
1575
1576         if (bp->flags & BNXT_FLAG_IF_CHANGE_HOT_FW_RESET_DONE) {
1577                 rc = bnxt_handle_if_change_status(bp);
1578                 if (rc)
1579                         return rc;
1580         }
1581
1582         bnxt_enable_int(bp);
1583
1584         eth_dev->data->scattered_rx = bnxt_scattered_rx(eth_dev);
1585
1586         rc = bnxt_start_nic(bp);
1587         if (rc)
1588                 goto error;
1589
1590         rc = bnxt_alloc_prev_ring_stats(bp);
1591         if (rc)
1592                 goto error;
1593
1594         eth_dev->data->dev_started = 1;
1595
1596         bnxt_link_update_op(eth_dev, 1);
1597
1598         if (rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
1599                 vlan_mask |= ETH_VLAN_FILTER_MASK;
1600         if (rx_offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
1601                 vlan_mask |= ETH_VLAN_STRIP_MASK;
1602         rc = bnxt_vlan_offload_set_op(eth_dev, vlan_mask);
1603         if (rc)
1604                 goto error;
1605
1606         /* Initialize bnxt ULP port details */
1607         rc = bnxt_ulp_port_init(bp);
1608         if (rc)
1609                 goto error;
1610
1611         eth_dev->rx_pkt_burst = bnxt_receive_function(eth_dev);
1612         eth_dev->tx_pkt_burst = bnxt_transmit_function(eth_dev);
1613
1614         bnxt_schedule_fw_health_check(bp);
1615
1616         if (BNXT_P5_PTP_TIMESYNC_ENABLED(bp))
1617                 bnxt_schedule_ptp_alarm(bp);
1618
1619         return 0;
1620
1621 error:
1622         bnxt_dev_stop(eth_dev);
1623         return rc;
1624 }
1625
1626 static void
1627 bnxt_uninit_locks(struct bnxt *bp)
1628 {
1629         pthread_mutex_destroy(&bp->flow_lock);
1630         pthread_mutex_destroy(&bp->def_cp_lock);
1631         pthread_mutex_destroy(&bp->health_check_lock);
1632         pthread_mutex_destroy(&bp->err_recovery_lock);
1633         if (bp->rep_info) {
1634                 pthread_mutex_destroy(&bp->rep_info->vfr_lock);
1635                 pthread_mutex_destroy(&bp->rep_info->vfr_start_lock);
1636         }
1637 }
1638
1639 static void bnxt_drv_uninit(struct bnxt *bp)
1640 {
1641         bnxt_free_leds_info(bp);
1642         bnxt_free_cos_queues(bp);
1643         bnxt_free_link_info(bp);
1644         bnxt_free_parent_info(bp);
1645         bnxt_uninit_locks(bp);
1646
1647         rte_memzone_free((const struct rte_memzone *)bp->tx_mem_zone);
1648         bp->tx_mem_zone = NULL;
1649         rte_memzone_free((const struct rte_memzone *)bp->rx_mem_zone);
1650         bp->rx_mem_zone = NULL;
1651
1652         bnxt_free_vf_info(bp);
1653         bnxt_free_pf_info(bp);
1654
1655         rte_free(bp->grp_info);
1656         bp->grp_info = NULL;
1657 }
1658
1659 static int bnxt_dev_close_op(struct rte_eth_dev *eth_dev)
1660 {
1661         struct bnxt *bp = eth_dev->data->dev_private;
1662         int ret = 0;
1663
1664         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1665                 return 0;
1666
1667         pthread_mutex_lock(&bp->err_recovery_lock);
1668         if (bp->flags & BNXT_FLAG_FW_RESET) {
1669                 PMD_DRV_LOG(ERR,
1670                             "Adapter recovering from error...Please retry\n");
1671                 pthread_mutex_unlock(&bp->err_recovery_lock);
1672                 return -EAGAIN;
1673         }
1674         pthread_mutex_unlock(&bp->err_recovery_lock);
1675
1676         /* cancel the recovery handler before remove dev */
1677         rte_eal_alarm_cancel(bnxt_dev_reset_and_resume, (void *)bp);
1678         rte_eal_alarm_cancel(bnxt_dev_recover, (void *)bp);
1679         bnxt_cancel_fc_thread(bp);
1680
1681         if (eth_dev->data->dev_started)
1682                 ret = bnxt_dev_stop(eth_dev);
1683
1684         bnxt_uninit_resources(bp, false);
1685
1686         bnxt_drv_uninit(bp);
1687
1688         return ret;
1689 }
1690
1691 static void bnxt_mac_addr_remove_op(struct rte_eth_dev *eth_dev,
1692                                     uint32_t index)
1693 {
1694         struct bnxt *bp = eth_dev->data->dev_private;
1695         uint64_t pool_mask = eth_dev->data->mac_pool_sel[index];
1696         struct bnxt_vnic_info *vnic;
1697         struct bnxt_filter_info *filter, *temp_filter;
1698         uint32_t i;
1699
1700         if (is_bnxt_in_error(bp))
1701                 return;
1702
1703         /*
1704          * Loop through all VNICs from the specified filter flow pools to
1705          * remove the corresponding MAC addr filter
1706          */
1707         for (i = 0; i < bp->nr_vnics; i++) {
1708                 if (!(pool_mask & (1ULL << i)))
1709                         continue;
1710
1711                 vnic = &bp->vnic_info[i];
1712                 filter = STAILQ_FIRST(&vnic->filter);
1713                 while (filter) {
1714                         temp_filter = STAILQ_NEXT(filter, next);
1715                         if (filter->mac_index == index) {
1716                                 STAILQ_REMOVE(&vnic->filter, filter,
1717                                                 bnxt_filter_info, next);
1718                                 bnxt_hwrm_clear_l2_filter(bp, filter);
1719                                 bnxt_free_filter(bp, filter);
1720                         }
1721                         filter = temp_filter;
1722                 }
1723         }
1724 }
1725
1726 static int bnxt_add_mac_filter(struct bnxt *bp, struct bnxt_vnic_info *vnic,
1727                                struct rte_ether_addr *mac_addr, uint32_t index,
1728                                uint32_t pool)
1729 {
1730         struct bnxt_filter_info *filter;
1731         int rc = 0;
1732
1733         /* Attach requested MAC address to the new l2_filter */
1734         STAILQ_FOREACH(filter, &vnic->filter, next) {
1735                 if (filter->mac_index == index) {
1736                         PMD_DRV_LOG(DEBUG,
1737                                     "MAC addr already existed for pool %d\n",
1738                                     pool);
1739                         return 0;
1740                 }
1741         }
1742
1743         filter = bnxt_alloc_filter(bp);
1744         if (!filter) {
1745                 PMD_DRV_LOG(ERR, "L2 filter alloc failed\n");
1746                 return -ENODEV;
1747         }
1748
1749         /* bnxt_alloc_filter copies default MAC to filter->l2_addr. So,
1750          * if the MAC that's been programmed now is a different one, then,
1751          * copy that addr to filter->l2_addr
1752          */
1753         if (mac_addr)
1754                 memcpy(filter->l2_addr, mac_addr, RTE_ETHER_ADDR_LEN);
1755         filter->flags |= HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_OUTERMOST;
1756
1757         rc = bnxt_hwrm_set_l2_filter(bp, vnic->fw_vnic_id, filter);
1758         if (!rc) {
1759                 filter->mac_index = index;
1760                 if (filter->mac_index == 0)
1761                         STAILQ_INSERT_HEAD(&vnic->filter, filter, next);
1762                 else
1763                         STAILQ_INSERT_TAIL(&vnic->filter, filter, next);
1764         } else {
1765                 bnxt_free_filter(bp, filter);
1766         }
1767
1768         return rc;
1769 }
1770
1771 static int bnxt_mac_addr_add_op(struct rte_eth_dev *eth_dev,
1772                                 struct rte_ether_addr *mac_addr,
1773                                 uint32_t index, uint32_t pool)
1774 {
1775         struct bnxt *bp = eth_dev->data->dev_private;
1776         struct bnxt_vnic_info *vnic = &bp->vnic_info[pool];
1777         int rc = 0;
1778
1779         rc = is_bnxt_in_error(bp);
1780         if (rc)
1781                 return rc;
1782
1783         if (BNXT_VF(bp) && !BNXT_VF_IS_TRUSTED(bp)) {
1784                 PMD_DRV_LOG(ERR, "Cannot add MAC address to a VF interface\n");
1785                 return -ENOTSUP;
1786         }
1787
1788         if (!vnic) {
1789                 PMD_DRV_LOG(ERR, "VNIC not found for pool %d!\n", pool);
1790                 return -EINVAL;
1791         }
1792
1793         /* Filter settings will get applied when port is started */
1794         if (!eth_dev->data->dev_started)
1795                 return 0;
1796
1797         rc = bnxt_add_mac_filter(bp, vnic, mac_addr, index, pool);
1798
1799         return rc;
1800 }
1801
1802 int bnxt_link_update_op(struct rte_eth_dev *eth_dev, int wait_to_complete)
1803 {
1804         int rc = 0;
1805         struct bnxt *bp = eth_dev->data->dev_private;
1806         struct rte_eth_link new;
1807         int cnt = wait_to_complete ? BNXT_MAX_LINK_WAIT_CNT :
1808                         BNXT_MIN_LINK_WAIT_CNT;
1809
1810         rc = is_bnxt_in_error(bp);
1811         if (rc)
1812                 return rc;
1813
1814         memset(&new, 0, sizeof(new));
1815
1816         if (bp->link_info == NULL)
1817                 goto out;
1818
1819         do {
1820                 /* Retrieve link info from hardware */
1821                 rc = bnxt_get_hwrm_link_config(bp, &new);
1822                 if (rc) {
1823                         new.link_speed = ETH_LINK_SPEED_100M;
1824                         new.link_duplex = ETH_LINK_FULL_DUPLEX;
1825                         PMD_DRV_LOG(ERR,
1826                                 "Failed to retrieve link rc = 0x%x!\n", rc);
1827                         goto out;
1828                 }
1829
1830                 if (!wait_to_complete || new.link_status)
1831                         break;
1832
1833                 rte_delay_ms(BNXT_LINK_WAIT_INTERVAL);
1834         } while (cnt--);
1835
1836         /* Only single function PF can bring phy down.
1837          * When port is stopped, report link down for VF/MH/NPAR functions.
1838          */
1839         if (!BNXT_SINGLE_PF(bp) && !eth_dev->data->dev_started)
1840                 memset(&new, 0, sizeof(new));
1841
1842 out:
1843         /* Timed out or success */
1844         if (new.link_status != eth_dev->data->dev_link.link_status ||
1845             new.link_speed != eth_dev->data->dev_link.link_speed) {
1846                 rte_eth_linkstatus_set(eth_dev, &new);
1847
1848                 rte_eth_dev_callback_process(eth_dev,
1849                                              RTE_ETH_EVENT_INTR_LSC,
1850                                              NULL);
1851
1852                 bnxt_print_link_info(eth_dev);
1853         }
1854
1855         return rc;
1856 }
1857
1858 static int bnxt_promiscuous_enable_op(struct rte_eth_dev *eth_dev)
1859 {
1860         struct bnxt *bp = eth_dev->data->dev_private;
1861         struct bnxt_vnic_info *vnic;
1862         uint32_t old_flags;
1863         int rc;
1864
1865         rc = is_bnxt_in_error(bp);
1866         if (rc)
1867                 return rc;
1868
1869         /* Filter settings will get applied when port is started */
1870         if (!eth_dev->data->dev_started)
1871                 return 0;
1872
1873         if (bp->vnic_info == NULL)
1874                 return 0;
1875
1876         vnic = BNXT_GET_DEFAULT_VNIC(bp);
1877
1878         old_flags = vnic->flags;
1879         vnic->flags |= BNXT_VNIC_INFO_PROMISC;
1880         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
1881         if (rc != 0)
1882                 vnic->flags = old_flags;
1883
1884         return rc;
1885 }
1886
1887 static int bnxt_promiscuous_disable_op(struct rte_eth_dev *eth_dev)
1888 {
1889         struct bnxt *bp = eth_dev->data->dev_private;
1890         struct bnxt_vnic_info *vnic;
1891         uint32_t old_flags;
1892         int rc;
1893
1894         rc = is_bnxt_in_error(bp);
1895         if (rc)
1896                 return rc;
1897
1898         /* Filter settings will get applied when port is started */
1899         if (!eth_dev->data->dev_started)
1900                 return 0;
1901
1902         if (bp->vnic_info == NULL)
1903                 return 0;
1904
1905         vnic = BNXT_GET_DEFAULT_VNIC(bp);
1906
1907         old_flags = vnic->flags;
1908         vnic->flags &= ~BNXT_VNIC_INFO_PROMISC;
1909         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
1910         if (rc != 0)
1911                 vnic->flags = old_flags;
1912
1913         return rc;
1914 }
1915
1916 static int bnxt_allmulticast_enable_op(struct rte_eth_dev *eth_dev)
1917 {
1918         struct bnxt *bp = eth_dev->data->dev_private;
1919         struct bnxt_vnic_info *vnic;
1920         uint32_t old_flags;
1921         int rc;
1922
1923         rc = is_bnxt_in_error(bp);
1924         if (rc)
1925                 return rc;
1926
1927         /* Filter settings will get applied when port is started */
1928         if (!eth_dev->data->dev_started)
1929                 return 0;
1930
1931         if (bp->vnic_info == NULL)
1932                 return 0;
1933
1934         vnic = BNXT_GET_DEFAULT_VNIC(bp);
1935
1936         old_flags = vnic->flags;
1937         vnic->flags |= BNXT_VNIC_INFO_ALLMULTI;
1938         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
1939         if (rc != 0)
1940                 vnic->flags = old_flags;
1941
1942         return rc;
1943 }
1944
1945 static int bnxt_allmulticast_disable_op(struct rte_eth_dev *eth_dev)
1946 {
1947         struct bnxt *bp = eth_dev->data->dev_private;
1948         struct bnxt_vnic_info *vnic;
1949         uint32_t old_flags;
1950         int rc;
1951
1952         rc = is_bnxt_in_error(bp);
1953         if (rc)
1954                 return rc;
1955
1956         /* Filter settings will get applied when port is started */
1957         if (!eth_dev->data->dev_started)
1958                 return 0;
1959
1960         if (bp->vnic_info == NULL)
1961                 return 0;
1962
1963         vnic = BNXT_GET_DEFAULT_VNIC(bp);
1964
1965         old_flags = vnic->flags;
1966         vnic->flags &= ~BNXT_VNIC_INFO_ALLMULTI;
1967         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
1968         if (rc != 0)
1969                 vnic->flags = old_flags;
1970
1971         return rc;
1972 }
1973
1974 /* Return bnxt_rx_queue pointer corresponding to a given rxq. */
1975 static struct bnxt_rx_queue *bnxt_qid_to_rxq(struct bnxt *bp, uint16_t qid)
1976 {
1977         if (qid >= bp->rx_nr_rings)
1978                 return NULL;
1979
1980         return bp->eth_dev->data->rx_queues[qid];
1981 }
1982
1983 /* Return rxq corresponding to a given rss table ring/group ID. */
1984 static uint16_t bnxt_rss_to_qid(struct bnxt *bp, uint16_t fwr)
1985 {
1986         struct bnxt_rx_queue *rxq;
1987         unsigned int i;
1988
1989         if (!BNXT_HAS_RING_GRPS(bp)) {
1990                 for (i = 0; i < bp->rx_nr_rings; i++) {
1991                         rxq = bp->eth_dev->data->rx_queues[i];
1992                         if (rxq->rx_ring->rx_ring_struct->fw_ring_id == fwr)
1993                                 return rxq->index;
1994                 }
1995         } else {
1996                 for (i = 0; i < bp->rx_nr_rings; i++) {
1997                         if (bp->grp_info[i].fw_grp_id == fwr)
1998                                 return i;
1999                 }
2000         }
2001
2002         return INVALID_HW_RING_ID;
2003 }
2004
2005 static int bnxt_reta_update_op(struct rte_eth_dev *eth_dev,
2006                             struct rte_eth_rss_reta_entry64 *reta_conf,
2007                             uint16_t reta_size)
2008 {
2009         struct bnxt *bp = eth_dev->data->dev_private;
2010         struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
2011         struct bnxt_vnic_info *vnic = BNXT_GET_DEFAULT_VNIC(bp);
2012         uint16_t tbl_size = bnxt_rss_hash_tbl_size(bp);
2013         uint16_t idx, sft;
2014         int i, rc;
2015
2016         rc = is_bnxt_in_error(bp);
2017         if (rc)
2018                 return rc;
2019
2020         if (!vnic->rss_table)
2021                 return -EINVAL;
2022
2023         if (!(dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
2024                 return -EINVAL;
2025
2026         if (reta_size != tbl_size) {
2027                 PMD_DRV_LOG(ERR, "The configured hash table lookup size "
2028                         "(%d) must equal the size supported by the hardware "
2029                         "(%d)\n", reta_size, tbl_size);
2030                 return -EINVAL;
2031         }
2032
2033         for (i = 0; i < reta_size; i++) {
2034                 struct bnxt_rx_queue *rxq;
2035
2036                 idx = i / RTE_RETA_GROUP_SIZE;
2037                 sft = i % RTE_RETA_GROUP_SIZE;
2038
2039                 if (!(reta_conf[idx].mask & (1ULL << sft)))
2040                         continue;
2041
2042                 rxq = bnxt_qid_to_rxq(bp, reta_conf[idx].reta[sft]);
2043                 if (!rxq) {
2044                         PMD_DRV_LOG(ERR, "Invalid ring in reta_conf.\n");
2045                         return -EINVAL;
2046                 }
2047
2048                 if (BNXT_CHIP_P5(bp)) {
2049                         vnic->rss_table[i * 2] =
2050                                 rxq->rx_ring->rx_ring_struct->fw_ring_id;
2051                         vnic->rss_table[i * 2 + 1] =
2052                                 rxq->cp_ring->cp_ring_struct->fw_ring_id;
2053                 } else {
2054                         vnic->rss_table[i] =
2055                             vnic->fw_grp_ids[reta_conf[idx].reta[sft]];
2056                 }
2057         }
2058
2059         rc = bnxt_hwrm_vnic_rss_cfg(bp, vnic);
2060         return rc;
2061 }
2062
2063 static int bnxt_reta_query_op(struct rte_eth_dev *eth_dev,
2064                               struct rte_eth_rss_reta_entry64 *reta_conf,
2065                               uint16_t reta_size)
2066 {
2067         struct bnxt *bp = eth_dev->data->dev_private;
2068         struct bnxt_vnic_info *vnic = BNXT_GET_DEFAULT_VNIC(bp);
2069         uint16_t tbl_size = bnxt_rss_hash_tbl_size(bp);
2070         uint16_t idx, sft, i;
2071         int rc;
2072
2073         rc = is_bnxt_in_error(bp);
2074         if (rc)
2075                 return rc;
2076
2077         if (!vnic)
2078                 return -EINVAL;
2079         if (!vnic->rss_table)
2080                 return -EINVAL;
2081
2082         if (reta_size != tbl_size) {
2083                 PMD_DRV_LOG(ERR, "The configured hash table lookup size "
2084                         "(%d) must equal the size supported by the hardware "
2085                         "(%d)\n", reta_size, tbl_size);
2086                 return -EINVAL;
2087         }
2088
2089         for (idx = 0, i = 0; i < reta_size; i++) {
2090                 idx = i / RTE_RETA_GROUP_SIZE;
2091                 sft = i % RTE_RETA_GROUP_SIZE;
2092
2093                 if (reta_conf[idx].mask & (1ULL << sft)) {
2094                         uint16_t qid;
2095
2096                         if (BNXT_CHIP_P5(bp))
2097                                 qid = bnxt_rss_to_qid(bp,
2098                                                       vnic->rss_table[i * 2]);
2099                         else
2100                                 qid = bnxt_rss_to_qid(bp, vnic->rss_table[i]);
2101
2102                         if (qid == INVALID_HW_RING_ID) {
2103                                 PMD_DRV_LOG(ERR, "Inv. entry in rss table.\n");
2104                                 return -EINVAL;
2105                         }
2106                         reta_conf[idx].reta[sft] = qid;
2107                 }
2108         }
2109
2110         return 0;
2111 }
2112
2113 static int bnxt_rss_hash_update_op(struct rte_eth_dev *eth_dev,
2114                                    struct rte_eth_rss_conf *rss_conf)
2115 {
2116         struct bnxt *bp = eth_dev->data->dev_private;
2117         struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
2118         struct bnxt_vnic_info *vnic;
2119         int rc;
2120
2121         rc = is_bnxt_in_error(bp);
2122         if (rc)
2123                 return rc;
2124
2125         /*
2126          * If RSS enablement were different than dev_configure,
2127          * then return -EINVAL
2128          */
2129         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) {
2130                 if (!rss_conf->rss_hf)
2131                         PMD_DRV_LOG(ERR, "Hash type NONE\n");
2132         } else {
2133                 if (rss_conf->rss_hf & BNXT_ETH_RSS_SUPPORT)
2134                         return -EINVAL;
2135         }
2136
2137         bp->flags |= BNXT_FLAG_UPDATE_HASH;
2138         memcpy(&eth_dev->data->dev_conf.rx_adv_conf.rss_conf,
2139                rss_conf,
2140                sizeof(*rss_conf));
2141
2142         /* Update the default RSS VNIC(s) */
2143         vnic = BNXT_GET_DEFAULT_VNIC(bp);
2144         vnic->hash_type = bnxt_rte_to_hwrm_hash_types(rss_conf->rss_hf);
2145         vnic->hash_mode =
2146                 bnxt_rte_to_hwrm_hash_level(bp, rss_conf->rss_hf,
2147                                             ETH_RSS_LEVEL(rss_conf->rss_hf));
2148
2149         /*
2150          * If hashkey is not specified, use the previously configured
2151          * hashkey
2152          */
2153         if (!rss_conf->rss_key)
2154                 goto rss_config;
2155
2156         if (rss_conf->rss_key_len != HW_HASH_KEY_SIZE) {
2157                 PMD_DRV_LOG(ERR,
2158                             "Invalid hashkey length, should be %d bytes\n",
2159                             HW_HASH_KEY_SIZE);
2160                 return -EINVAL;
2161         }
2162         memcpy(vnic->rss_hash_key, rss_conf->rss_key, rss_conf->rss_key_len);
2163
2164 rss_config:
2165         rc = bnxt_hwrm_vnic_rss_cfg(bp, vnic);
2166         return rc;
2167 }
2168
2169 static int bnxt_rss_hash_conf_get_op(struct rte_eth_dev *eth_dev,
2170                                      struct rte_eth_rss_conf *rss_conf)
2171 {
2172         struct bnxt *bp = eth_dev->data->dev_private;
2173         struct bnxt_vnic_info *vnic = BNXT_GET_DEFAULT_VNIC(bp);
2174         int len, rc;
2175         uint32_t hash_types;
2176
2177         rc = is_bnxt_in_error(bp);
2178         if (rc)
2179                 return rc;
2180
2181         /* RSS configuration is the same for all VNICs */
2182         if (vnic && vnic->rss_hash_key) {
2183                 if (rss_conf->rss_key) {
2184                         len = rss_conf->rss_key_len <= HW_HASH_KEY_SIZE ?
2185                               rss_conf->rss_key_len : HW_HASH_KEY_SIZE;
2186                         memcpy(rss_conf->rss_key, vnic->rss_hash_key, len);
2187                 }
2188
2189                 hash_types = vnic->hash_type;
2190                 rss_conf->rss_hf = 0;
2191                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV4) {
2192                         rss_conf->rss_hf |= ETH_RSS_IPV4;
2193                         hash_types &= ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV4;
2194                 }
2195                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV4) {
2196                         rss_conf->rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
2197                         hash_types &=
2198                                 ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV4;
2199                 }
2200                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV4) {
2201                         rss_conf->rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
2202                         hash_types &=
2203                                 ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV4;
2204                 }
2205                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV6) {
2206                         rss_conf->rss_hf |= ETH_RSS_IPV6;
2207                         hash_types &= ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV6;
2208                 }
2209                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV6) {
2210                         rss_conf->rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
2211                         hash_types &=
2212                                 ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV6;
2213                 }
2214                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV6) {
2215                         rss_conf->rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
2216                         hash_types &=
2217                                 ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV6;
2218                 }
2219
2220                 rss_conf->rss_hf |=
2221                         bnxt_hwrm_to_rte_rss_level(bp, vnic->hash_mode);
2222
2223                 if (hash_types) {
2224                         PMD_DRV_LOG(ERR,
2225                                 "Unknown RSS config from firmware (%08x), RSS disabled",
2226                                 vnic->hash_type);
2227                         return -ENOTSUP;
2228                 }
2229         } else {
2230                 rss_conf->rss_hf = 0;
2231         }
2232         return 0;
2233 }
2234
2235 static int bnxt_flow_ctrl_get_op(struct rte_eth_dev *dev,
2236                                struct rte_eth_fc_conf *fc_conf)
2237 {
2238         struct bnxt *bp = dev->data->dev_private;
2239         struct rte_eth_link link_info;
2240         int rc;
2241
2242         rc = is_bnxt_in_error(bp);
2243         if (rc)
2244                 return rc;
2245
2246         rc = bnxt_get_hwrm_link_config(bp, &link_info);
2247         if (rc)
2248                 return rc;
2249
2250         memset(fc_conf, 0, sizeof(*fc_conf));
2251         if (bp->link_info->auto_pause)
2252                 fc_conf->autoneg = 1;
2253         switch (bp->link_info->pause) {
2254         case 0:
2255                 fc_conf->mode = RTE_FC_NONE;
2256                 break;
2257         case HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX:
2258                 fc_conf->mode = RTE_FC_TX_PAUSE;
2259                 break;
2260         case HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX:
2261                 fc_conf->mode = RTE_FC_RX_PAUSE;
2262                 break;
2263         case (HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX |
2264                         HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX):
2265                 fc_conf->mode = RTE_FC_FULL;
2266                 break;
2267         }
2268         return 0;
2269 }
2270
2271 static int bnxt_flow_ctrl_set_op(struct rte_eth_dev *dev,
2272                                struct rte_eth_fc_conf *fc_conf)
2273 {
2274         struct bnxt *bp = dev->data->dev_private;
2275         int rc;
2276
2277         rc = is_bnxt_in_error(bp);
2278         if (rc)
2279                 return rc;
2280
2281         if (!BNXT_SINGLE_PF(bp)) {
2282                 PMD_DRV_LOG(ERR,
2283                             "Flow Control Settings cannot be modified on VF or on shared PF\n");
2284                 return -ENOTSUP;
2285         }
2286
2287         switch (fc_conf->mode) {
2288         case RTE_FC_NONE:
2289                 bp->link_info->auto_pause = 0;
2290                 bp->link_info->force_pause = 0;
2291                 break;
2292         case RTE_FC_RX_PAUSE:
2293                 if (fc_conf->autoneg) {
2294                         bp->link_info->auto_pause =
2295                                         HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_RX;
2296                         bp->link_info->force_pause = 0;
2297                 } else {
2298                         bp->link_info->auto_pause = 0;
2299                         bp->link_info->force_pause =
2300                                         HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_RX;
2301                 }
2302                 break;
2303         case RTE_FC_TX_PAUSE:
2304                 if (fc_conf->autoneg) {
2305                         bp->link_info->auto_pause =
2306                                         HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_TX;
2307                         bp->link_info->force_pause = 0;
2308                 } else {
2309                         bp->link_info->auto_pause = 0;
2310                         bp->link_info->force_pause =
2311                                         HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_TX;
2312                 }
2313                 break;
2314         case RTE_FC_FULL:
2315                 if (fc_conf->autoneg) {
2316                         bp->link_info->auto_pause =
2317                                         HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_TX |
2318                                         HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_RX;
2319                         bp->link_info->force_pause = 0;
2320                 } else {
2321                         bp->link_info->auto_pause = 0;
2322                         bp->link_info->force_pause =
2323                                         HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_TX |
2324                                         HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_RX;
2325                 }
2326                 break;
2327         }
2328         return bnxt_set_hwrm_link_config(bp, true);
2329 }
2330
2331 /* Add UDP tunneling port */
2332 static int
2333 bnxt_udp_tunnel_port_add_op(struct rte_eth_dev *eth_dev,
2334                          struct rte_eth_udp_tunnel *udp_tunnel)
2335 {
2336         struct bnxt *bp = eth_dev->data->dev_private;
2337         uint16_t tunnel_type = 0;
2338         int rc = 0;
2339
2340         rc = is_bnxt_in_error(bp);
2341         if (rc)
2342                 return rc;
2343
2344         switch (udp_tunnel->prot_type) {
2345         case RTE_TUNNEL_TYPE_VXLAN:
2346                 if (bp->vxlan_port_cnt) {
2347                         PMD_DRV_LOG(ERR, "Tunnel Port %d already programmed\n",
2348                                 udp_tunnel->udp_port);
2349                         if (bp->vxlan_port != udp_tunnel->udp_port) {
2350                                 PMD_DRV_LOG(ERR, "Only one port allowed\n");
2351                                 return -ENOSPC;
2352                         }
2353                         bp->vxlan_port_cnt++;
2354                         return 0;
2355                 }
2356                 tunnel_type =
2357                         HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_VXLAN;
2358                 bp->vxlan_port_cnt++;
2359                 break;
2360         case RTE_TUNNEL_TYPE_GENEVE:
2361                 if (bp->geneve_port_cnt) {
2362                         PMD_DRV_LOG(ERR, "Tunnel Port %d already programmed\n",
2363                                 udp_tunnel->udp_port);
2364                         if (bp->geneve_port != udp_tunnel->udp_port) {
2365                                 PMD_DRV_LOG(ERR, "Only one port allowed\n");
2366                                 return -ENOSPC;
2367                         }
2368                         bp->geneve_port_cnt++;
2369                         return 0;
2370                 }
2371                 tunnel_type =
2372                         HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_GENEVE;
2373                 bp->geneve_port_cnt++;
2374                 break;
2375         default:
2376                 PMD_DRV_LOG(ERR, "Tunnel type is not supported\n");
2377                 return -ENOTSUP;
2378         }
2379         rc = bnxt_hwrm_tunnel_dst_port_alloc(bp, udp_tunnel->udp_port,
2380                                              tunnel_type);
2381         return rc;
2382 }
2383
2384 static int
2385 bnxt_udp_tunnel_port_del_op(struct rte_eth_dev *eth_dev,
2386                          struct rte_eth_udp_tunnel *udp_tunnel)
2387 {
2388         struct bnxt *bp = eth_dev->data->dev_private;
2389         uint16_t tunnel_type = 0;
2390         uint16_t port = 0;
2391         int rc = 0;
2392
2393         rc = is_bnxt_in_error(bp);
2394         if (rc)
2395                 return rc;
2396
2397         switch (udp_tunnel->prot_type) {
2398         case RTE_TUNNEL_TYPE_VXLAN:
2399                 if (!bp->vxlan_port_cnt) {
2400                         PMD_DRV_LOG(ERR, "No Tunnel port configured yet\n");
2401                         return -EINVAL;
2402                 }
2403                 if (bp->vxlan_port != udp_tunnel->udp_port) {
2404                         PMD_DRV_LOG(ERR, "Req Port: %d. Configured port: %d\n",
2405                                 udp_tunnel->udp_port, bp->vxlan_port);
2406                         return -EINVAL;
2407                 }
2408                 if (--bp->vxlan_port_cnt)
2409                         return 0;
2410
2411                 tunnel_type =
2412                         HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_VXLAN;
2413                 port = bp->vxlan_fw_dst_port_id;
2414                 break;
2415         case RTE_TUNNEL_TYPE_GENEVE:
2416                 if (!bp->geneve_port_cnt) {
2417                         PMD_DRV_LOG(ERR, "No Tunnel port configured yet\n");
2418                         return -EINVAL;
2419                 }
2420                 if (bp->geneve_port != udp_tunnel->udp_port) {
2421                         PMD_DRV_LOG(ERR, "Req Port: %d. Configured port: %d\n",
2422                                 udp_tunnel->udp_port, bp->geneve_port);
2423                         return -EINVAL;
2424                 }
2425                 if (--bp->geneve_port_cnt)
2426                         return 0;
2427
2428                 tunnel_type =
2429                         HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_GENEVE;
2430                 port = bp->geneve_fw_dst_port_id;
2431                 break;
2432         default:
2433                 PMD_DRV_LOG(ERR, "Tunnel type is not supported\n");
2434                 return -ENOTSUP;
2435         }
2436
2437         rc = bnxt_hwrm_tunnel_dst_port_free(bp, port, tunnel_type);
2438         return rc;
2439 }
2440
2441 static int bnxt_del_vlan_filter(struct bnxt *bp, uint16_t vlan_id)
2442 {
2443         struct bnxt_filter_info *filter;
2444         struct bnxt_vnic_info *vnic;
2445         int rc = 0;
2446         uint32_t chk = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN;
2447
2448         vnic = BNXT_GET_DEFAULT_VNIC(bp);
2449         filter = STAILQ_FIRST(&vnic->filter);
2450         while (filter) {
2451                 /* Search for this matching MAC+VLAN filter */
2452                 if (bnxt_vlan_filter_exists(bp, filter, chk, vlan_id)) {
2453                         /* Delete the filter */
2454                         rc = bnxt_hwrm_clear_l2_filter(bp, filter);
2455                         if (rc)
2456                                 return rc;
2457                         STAILQ_REMOVE(&vnic->filter, filter,
2458                                       bnxt_filter_info, next);
2459                         bnxt_free_filter(bp, filter);
2460                         PMD_DRV_LOG(INFO,
2461                                     "Deleted vlan filter for %d\n",
2462                                     vlan_id);
2463                         return 0;
2464                 }
2465                 filter = STAILQ_NEXT(filter, next);
2466         }
2467         return -ENOENT;
2468 }
2469
2470 static int bnxt_add_vlan_filter(struct bnxt *bp, uint16_t vlan_id)
2471 {
2472         struct bnxt_filter_info *filter;
2473         struct bnxt_vnic_info *vnic;
2474         int rc = 0;
2475         uint32_t en = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN |
2476                 HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN_MASK;
2477         uint32_t chk = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN;
2478
2479         /* Implementation notes on the use of VNIC in this command:
2480          *
2481          * By default, these filters belong to default vnic for the function.
2482          * Once these filters are set up, only destination VNIC can be modified.
2483          * If the destination VNIC is not specified in this command,
2484          * then the HWRM shall only create an l2 context id.
2485          */
2486
2487         vnic = BNXT_GET_DEFAULT_VNIC(bp);
2488         filter = STAILQ_FIRST(&vnic->filter);
2489         /* Check if the VLAN has already been added */
2490         while (filter) {
2491                 if (bnxt_vlan_filter_exists(bp, filter, chk, vlan_id))
2492                         return -EEXIST;
2493
2494                 filter = STAILQ_NEXT(filter, next);
2495         }
2496
2497         /* No match found. Alloc a fresh filter and issue the L2_FILTER_ALLOC
2498          * command to create MAC+VLAN filter with the right flags, enables set.
2499          */
2500         filter = bnxt_alloc_filter(bp);
2501         if (!filter) {
2502                 PMD_DRV_LOG(ERR,
2503                             "MAC/VLAN filter alloc failed\n");
2504                 return -ENOMEM;
2505         }
2506         /* MAC + VLAN ID filter */
2507         /* If l2_ivlan == 0 and l2_ivlan_mask != 0, only
2508          * untagged packets are received
2509          *
2510          * If l2_ivlan != 0 and l2_ivlan_mask != 0, untagged
2511          * packets and only the programmed vlan's packets are received
2512          */
2513         filter->l2_ivlan = vlan_id;
2514         filter->l2_ivlan_mask = 0x0FFF;
2515         filter->enables |= en;
2516         filter->flags |= HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_OUTERMOST;
2517
2518         rc = bnxt_hwrm_set_l2_filter(bp, vnic->fw_vnic_id, filter);
2519         if (rc) {
2520                 /* Free the newly allocated filter as we were
2521                  * not able to create the filter in hardware.
2522                  */
2523                 bnxt_free_filter(bp, filter);
2524                 return rc;
2525         }
2526
2527         filter->mac_index = 0;
2528         /* Add this new filter to the list */
2529         if (vlan_id == 0)
2530                 STAILQ_INSERT_HEAD(&vnic->filter, filter, next);
2531         else
2532                 STAILQ_INSERT_TAIL(&vnic->filter, filter, next);
2533
2534         PMD_DRV_LOG(INFO,
2535                     "Added Vlan filter for %d\n", vlan_id);
2536         return rc;
2537 }
2538
2539 static int bnxt_vlan_filter_set_op(struct rte_eth_dev *eth_dev,
2540                 uint16_t vlan_id, int on)
2541 {
2542         struct bnxt *bp = eth_dev->data->dev_private;
2543         int rc;
2544
2545         rc = is_bnxt_in_error(bp);
2546         if (rc)
2547                 return rc;
2548
2549         if (!eth_dev->data->dev_started) {
2550                 PMD_DRV_LOG(ERR, "port must be started before setting vlan\n");
2551                 return -EINVAL;
2552         }
2553
2554         /* These operations apply to ALL existing MAC/VLAN filters */
2555         if (on)
2556                 return bnxt_add_vlan_filter(bp, vlan_id);
2557         else
2558                 return bnxt_del_vlan_filter(bp, vlan_id);
2559 }
2560
2561 static int bnxt_del_dflt_mac_filter(struct bnxt *bp,
2562                                     struct bnxt_vnic_info *vnic)
2563 {
2564         struct bnxt_filter_info *filter;
2565         int rc;
2566
2567         filter = STAILQ_FIRST(&vnic->filter);
2568         while (filter) {
2569                 if (filter->mac_index == 0 &&
2570                     !memcmp(filter->l2_addr, bp->mac_addr,
2571                             RTE_ETHER_ADDR_LEN)) {
2572                         rc = bnxt_hwrm_clear_l2_filter(bp, filter);
2573                         if (!rc) {
2574                                 STAILQ_REMOVE(&vnic->filter, filter,
2575                                               bnxt_filter_info, next);
2576                                 bnxt_free_filter(bp, filter);
2577                         }
2578                         return rc;
2579                 }
2580                 filter = STAILQ_NEXT(filter, next);
2581         }
2582         return 0;
2583 }
2584
2585 static int
2586 bnxt_config_vlan_hw_filter(struct bnxt *bp, uint64_t rx_offloads)
2587 {
2588         struct bnxt_vnic_info *vnic;
2589         unsigned int i;
2590         int rc;
2591
2592         vnic = BNXT_GET_DEFAULT_VNIC(bp);
2593         if (!(rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER)) {
2594                 /* Remove any VLAN filters programmed */
2595                 for (i = 0; i < RTE_ETHER_MAX_VLAN_ID; i++)
2596                         bnxt_del_vlan_filter(bp, i);
2597
2598                 rc = bnxt_add_mac_filter(bp, vnic, NULL, 0, 0);
2599                 if (rc)
2600                         return rc;
2601         } else {
2602                 /* Default filter will allow packets that match the
2603                  * dest mac. So, it has to be deleted, otherwise, we
2604                  * will endup receiving vlan packets for which the
2605                  * filter is not programmed, when hw-vlan-filter
2606                  * configuration is ON
2607                  */
2608                 bnxt_del_dflt_mac_filter(bp, vnic);
2609                 /* This filter will allow only untagged packets */
2610                 bnxt_add_vlan_filter(bp, 0);
2611         }
2612         PMD_DRV_LOG(DEBUG, "VLAN Filtering: %d\n",
2613                     !!(rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER));
2614
2615         return 0;
2616 }
2617
2618 static int bnxt_free_one_vnic(struct bnxt *bp, uint16_t vnic_id)
2619 {
2620         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
2621         unsigned int i;
2622         int rc;
2623
2624         /* Destroy vnic filters and vnic */
2625         if (bp->eth_dev->data->dev_conf.rxmode.offloads &
2626             DEV_RX_OFFLOAD_VLAN_FILTER) {
2627                 for (i = 0; i < RTE_ETHER_MAX_VLAN_ID; i++)
2628                         bnxt_del_vlan_filter(bp, i);
2629         }
2630         bnxt_del_dflt_mac_filter(bp, vnic);
2631
2632         rc = bnxt_hwrm_vnic_ctx_free(bp, vnic);
2633         if (rc)
2634                 return rc;
2635
2636         rc = bnxt_hwrm_vnic_free(bp, vnic);
2637         if (rc)
2638                 return rc;
2639
2640         rte_free(vnic->fw_grp_ids);
2641         vnic->fw_grp_ids = NULL;
2642
2643         vnic->rx_queue_cnt = 0;
2644
2645         return 0;
2646 }
2647
2648 static int
2649 bnxt_config_vlan_hw_stripping(struct bnxt *bp, uint64_t rx_offloads)
2650 {
2651         struct bnxt_vnic_info *vnic = BNXT_GET_DEFAULT_VNIC(bp);
2652         int rc;
2653
2654         /* Destroy, recreate and reconfigure the default vnic */
2655         rc = bnxt_free_one_vnic(bp, 0);
2656         if (rc)
2657                 return rc;
2658
2659         /* default vnic 0 */
2660         rc = bnxt_setup_one_vnic(bp, 0);
2661         if (rc)
2662                 return rc;
2663
2664         if (bp->eth_dev->data->dev_conf.rxmode.offloads &
2665             DEV_RX_OFFLOAD_VLAN_FILTER) {
2666                 rc = bnxt_add_vlan_filter(bp, 0);
2667                 if (rc)
2668                         return rc;
2669                 rc = bnxt_restore_vlan_filters(bp);
2670                 if (rc)
2671                         return rc;
2672         } else {
2673                 rc = bnxt_add_mac_filter(bp, vnic, NULL, 0, 0);
2674                 if (rc)
2675                         return rc;
2676         }
2677
2678         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
2679         if (rc)
2680                 return rc;
2681
2682         PMD_DRV_LOG(DEBUG, "VLAN Strip Offload: %d\n",
2683                     !!(rx_offloads & DEV_RX_OFFLOAD_VLAN_STRIP));
2684
2685         return rc;
2686 }
2687
2688 static int
2689 bnxt_vlan_offload_set_op(struct rte_eth_dev *dev, int mask)
2690 {
2691         uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads;
2692         struct bnxt *bp = dev->data->dev_private;
2693         int rc;
2694
2695         rc = is_bnxt_in_error(bp);
2696         if (rc)
2697                 return rc;
2698
2699         /* Filter settings will get applied when port is started */
2700         if (!dev->data->dev_started)
2701                 return 0;
2702
2703         if (mask & ETH_VLAN_FILTER_MASK) {
2704                 /* Enable or disable VLAN filtering */
2705                 rc = bnxt_config_vlan_hw_filter(bp, rx_offloads);
2706                 if (rc)
2707                         return rc;
2708         }
2709
2710         if (mask & ETH_VLAN_STRIP_MASK) {
2711                 /* Enable or disable VLAN stripping */
2712                 rc = bnxt_config_vlan_hw_stripping(bp, rx_offloads);
2713                 if (rc)
2714                         return rc;
2715         }
2716
2717         if (mask & ETH_VLAN_EXTEND_MASK) {
2718                 if (rx_offloads & DEV_RX_OFFLOAD_VLAN_EXTEND)
2719                         PMD_DRV_LOG(DEBUG, "Extend VLAN supported\n");
2720                 else
2721                         PMD_DRV_LOG(INFO, "Extend VLAN unsupported\n");
2722         }
2723
2724         return 0;
2725 }
2726
2727 static int
2728 bnxt_vlan_tpid_set_op(struct rte_eth_dev *dev, enum rte_vlan_type vlan_type,
2729                       uint16_t tpid)
2730 {
2731         struct bnxt *bp = dev->data->dev_private;
2732         int qinq = dev->data->dev_conf.rxmode.offloads &
2733                    DEV_RX_OFFLOAD_VLAN_EXTEND;
2734
2735         if (vlan_type != ETH_VLAN_TYPE_INNER &&
2736             vlan_type != ETH_VLAN_TYPE_OUTER) {
2737                 PMD_DRV_LOG(ERR,
2738                             "Unsupported vlan type.");
2739                 return -EINVAL;
2740         }
2741         if (!qinq) {
2742                 PMD_DRV_LOG(ERR,
2743                             "QinQ not enabled. Needs to be ON as we can "
2744                             "accelerate only outer vlan\n");
2745                 return -EINVAL;
2746         }
2747
2748         if (vlan_type == ETH_VLAN_TYPE_OUTER) {
2749                 switch (tpid) {
2750                 case RTE_ETHER_TYPE_QINQ:
2751                         bp->outer_tpid_bd =
2752                                 TX_BD_LONG_CFA_META_VLAN_TPID_TPID88A8;
2753                                 break;
2754                 case RTE_ETHER_TYPE_VLAN:
2755                         bp->outer_tpid_bd =
2756                                 TX_BD_LONG_CFA_META_VLAN_TPID_TPID8100;
2757                                 break;
2758                 case RTE_ETHER_TYPE_QINQ1:
2759                         bp->outer_tpid_bd =
2760                                 TX_BD_LONG_CFA_META_VLAN_TPID_TPID9100;
2761                                 break;
2762                 case RTE_ETHER_TYPE_QINQ2:
2763                         bp->outer_tpid_bd =
2764                                 TX_BD_LONG_CFA_META_VLAN_TPID_TPID9200;
2765                                 break;
2766                 case RTE_ETHER_TYPE_QINQ3:
2767                         bp->outer_tpid_bd =
2768                                  TX_BD_LONG_CFA_META_VLAN_TPID_TPID9300;
2769                                 break;
2770                 default:
2771                         PMD_DRV_LOG(ERR, "Invalid TPID: %x\n", tpid);
2772                         return -EINVAL;
2773                 }
2774                 bp->outer_tpid_bd |= tpid;
2775                 PMD_DRV_LOG(INFO, "outer_tpid_bd = %x\n", bp->outer_tpid_bd);
2776         } else if (vlan_type == ETH_VLAN_TYPE_INNER) {
2777                 PMD_DRV_LOG(ERR,
2778                             "Can accelerate only outer vlan in QinQ\n");
2779                 return -EINVAL;
2780         }
2781
2782         return 0;
2783 }
2784
2785 static int
2786 bnxt_set_default_mac_addr_op(struct rte_eth_dev *dev,
2787                              struct rte_ether_addr *addr)
2788 {
2789         struct bnxt *bp = dev->data->dev_private;
2790         /* Default Filter is tied to VNIC 0 */
2791         struct bnxt_vnic_info *vnic = BNXT_GET_DEFAULT_VNIC(bp);
2792         int rc;
2793
2794         rc = is_bnxt_in_error(bp);
2795         if (rc)
2796                 return rc;
2797
2798         if (BNXT_VF(bp) && !BNXT_VF_IS_TRUSTED(bp))
2799                 return -EPERM;
2800
2801         if (rte_is_zero_ether_addr(addr))
2802                 return -EINVAL;
2803
2804         /* Filter settings will get applied when port is started */
2805         if (!dev->data->dev_started)
2806                 return 0;
2807
2808         /* Check if the requested MAC is already added */
2809         if (memcmp(addr, bp->mac_addr, RTE_ETHER_ADDR_LEN) == 0)
2810                 return 0;
2811
2812         /* Destroy filter and re-create it */
2813         bnxt_del_dflt_mac_filter(bp, vnic);
2814
2815         memcpy(bp->mac_addr, addr, RTE_ETHER_ADDR_LEN);
2816         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_VLAN_FILTER) {
2817                 /* This filter will allow only untagged packets */
2818                 rc = bnxt_add_vlan_filter(bp, 0);
2819         } else {
2820                 rc = bnxt_add_mac_filter(bp, vnic, addr, 0, 0);
2821         }
2822
2823         PMD_DRV_LOG(DEBUG, "Set MAC addr\n");
2824         return rc;
2825 }
2826
2827 static int
2828 bnxt_dev_set_mc_addr_list_op(struct rte_eth_dev *eth_dev,
2829                           struct rte_ether_addr *mc_addr_set,
2830                           uint32_t nb_mc_addr)
2831 {
2832         struct bnxt *bp = eth_dev->data->dev_private;
2833         char *mc_addr_list = (char *)mc_addr_set;
2834         struct bnxt_vnic_info *vnic;
2835         uint32_t off = 0, i = 0;
2836         int rc;
2837
2838         rc = is_bnxt_in_error(bp);
2839         if (rc)
2840                 return rc;
2841
2842         vnic = BNXT_GET_DEFAULT_VNIC(bp);
2843
2844         if (nb_mc_addr > BNXT_MAX_MC_ADDRS) {
2845                 vnic->flags |= BNXT_VNIC_INFO_ALLMULTI;
2846                 goto allmulti;
2847         }
2848
2849         /* TODO Check for Duplicate mcast addresses */
2850         vnic->flags &= ~BNXT_VNIC_INFO_ALLMULTI;
2851         for (i = 0; i < nb_mc_addr; i++) {
2852                 memcpy(vnic->mc_list + off, &mc_addr_list[i],
2853                         RTE_ETHER_ADDR_LEN);
2854                 off += RTE_ETHER_ADDR_LEN;
2855         }
2856
2857         vnic->mc_addr_cnt = i;
2858         if (vnic->mc_addr_cnt)
2859                 vnic->flags |= BNXT_VNIC_INFO_MCAST;
2860         else
2861                 vnic->flags &= ~BNXT_VNIC_INFO_MCAST;
2862
2863 allmulti:
2864         return bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
2865 }
2866
2867 static int
2868 bnxt_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size)
2869 {
2870         struct bnxt *bp = dev->data->dev_private;
2871         uint8_t fw_major = (bp->fw_ver >> 24) & 0xff;
2872         uint8_t fw_minor = (bp->fw_ver >> 16) & 0xff;
2873         uint8_t fw_updt = (bp->fw_ver >> 8) & 0xff;
2874         uint8_t fw_rsvd = bp->fw_ver & 0xff;
2875         int ret;
2876
2877         ret = snprintf(fw_version, fw_size, "%d.%d.%d.%d",
2878                         fw_major, fw_minor, fw_updt, fw_rsvd);
2879         if (ret < 0)
2880                 return -EINVAL;
2881
2882         ret += 1; /* add the size of '\0' */
2883         if (fw_size < (size_t)ret)
2884                 return ret;
2885         else
2886                 return 0;
2887 }
2888
2889 static void
2890 bnxt_rxq_info_get_op(struct rte_eth_dev *dev, uint16_t queue_id,
2891         struct rte_eth_rxq_info *qinfo)
2892 {
2893         struct bnxt *bp = dev->data->dev_private;
2894         struct bnxt_rx_queue *rxq;
2895
2896         if (is_bnxt_in_error(bp))
2897                 return;
2898
2899         rxq = dev->data->rx_queues[queue_id];
2900
2901         qinfo->mp = rxq->mb_pool;
2902         qinfo->scattered_rx = dev->data->scattered_rx;
2903         qinfo->nb_desc = rxq->nb_rx_desc;
2904
2905         qinfo->conf.rx_free_thresh = rxq->rx_free_thresh;
2906         qinfo->conf.rx_drop_en = rxq->drop_en;
2907         qinfo->conf.rx_deferred_start = rxq->rx_deferred_start;
2908         qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads;
2909 }
2910
2911 static void
2912 bnxt_txq_info_get_op(struct rte_eth_dev *dev, uint16_t queue_id,
2913         struct rte_eth_txq_info *qinfo)
2914 {
2915         struct bnxt *bp = dev->data->dev_private;
2916         struct bnxt_tx_queue *txq;
2917
2918         if (is_bnxt_in_error(bp))
2919                 return;
2920
2921         txq = dev->data->tx_queues[queue_id];
2922
2923         qinfo->nb_desc = txq->nb_tx_desc;
2924
2925         qinfo->conf.tx_thresh.pthresh = txq->pthresh;
2926         qinfo->conf.tx_thresh.hthresh = txq->hthresh;
2927         qinfo->conf.tx_thresh.wthresh = txq->wthresh;
2928
2929         qinfo->conf.tx_free_thresh = txq->tx_free_thresh;
2930         qinfo->conf.tx_rs_thresh = 0;
2931         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
2932         qinfo->conf.offloads = txq->offloads;
2933 }
2934
2935 static const struct {
2936         eth_rx_burst_t pkt_burst;
2937         const char *info;
2938 } bnxt_rx_burst_info[] = {
2939         {bnxt_recv_pkts,                "Scalar"},
2940 #if defined(RTE_ARCH_X86)
2941         {bnxt_recv_pkts_vec,            "Vector SSE"},
2942 #endif
2943 #if defined(RTE_ARCH_X86) && defined(CC_AVX2_SUPPORT)
2944         {bnxt_recv_pkts_vec_avx2,       "Vector AVX2"},
2945 #endif
2946 #if defined(RTE_ARCH_ARM64)
2947         {bnxt_recv_pkts_vec,            "Vector Neon"},
2948 #endif
2949 };
2950
2951 static int
2952 bnxt_rx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id,
2953                        struct rte_eth_burst_mode *mode)
2954 {
2955         eth_rx_burst_t pkt_burst = dev->rx_pkt_burst;
2956         size_t i;
2957
2958         for (i = 0; i < RTE_DIM(bnxt_rx_burst_info); i++) {
2959                 if (pkt_burst == bnxt_rx_burst_info[i].pkt_burst) {
2960                         snprintf(mode->info, sizeof(mode->info), "%s",
2961                                  bnxt_rx_burst_info[i].info);
2962                         return 0;
2963                 }
2964         }
2965
2966         return -EINVAL;
2967 }
2968
2969 static const struct {
2970         eth_tx_burst_t pkt_burst;
2971         const char *info;
2972 } bnxt_tx_burst_info[] = {
2973         {bnxt_xmit_pkts,                "Scalar"},
2974 #if defined(RTE_ARCH_X86)
2975         {bnxt_xmit_pkts_vec,            "Vector SSE"},
2976 #endif
2977 #if defined(RTE_ARCH_X86) && defined(CC_AVX2_SUPPORT)
2978         {bnxt_xmit_pkts_vec_avx2,       "Vector AVX2"},
2979 #endif
2980 #if defined(RTE_ARCH_ARM64)
2981         {bnxt_xmit_pkts_vec,            "Vector Neon"},
2982 #endif
2983 };
2984
2985 static int
2986 bnxt_tx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id,
2987                        struct rte_eth_burst_mode *mode)
2988 {
2989         eth_tx_burst_t pkt_burst = dev->tx_pkt_burst;
2990         size_t i;
2991
2992         for (i = 0; i < RTE_DIM(bnxt_tx_burst_info); i++) {
2993                 if (pkt_burst == bnxt_tx_burst_info[i].pkt_burst) {
2994                         snprintf(mode->info, sizeof(mode->info), "%s",
2995                                  bnxt_tx_burst_info[i].info);
2996                         return 0;
2997                 }
2998         }
2999
3000         return -EINVAL;
3001 }
3002
3003 int bnxt_mtu_set_op(struct rte_eth_dev *eth_dev, uint16_t new_mtu)
3004 {
3005         struct bnxt *bp = eth_dev->data->dev_private;
3006         uint32_t new_pkt_size;
3007         uint32_t rc = 0;
3008         uint32_t i;
3009
3010         rc = is_bnxt_in_error(bp);
3011         if (rc)
3012                 return rc;
3013
3014         /* Exit if receive queues are not configured yet */
3015         if (!eth_dev->data->nb_rx_queues)
3016                 return rc;
3017
3018         new_pkt_size = new_mtu + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN +
3019                        VLAN_TAG_SIZE * BNXT_NUM_VLANS;
3020
3021         /*
3022          * Disallow any MTU change that would require scattered receive support
3023          * if it is not already enabled.
3024          */
3025         if (eth_dev->data->dev_started &&
3026             !eth_dev->data->scattered_rx &&
3027             (new_pkt_size >
3028              eth_dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)) {
3029                 PMD_DRV_LOG(ERR,
3030                             "MTU change would require scattered rx support. ");
3031                 PMD_DRV_LOG(ERR, "Stop port before changing MTU.\n");
3032                 return -EINVAL;
3033         }
3034
3035         if (new_mtu > RTE_ETHER_MTU) {
3036                 bp->flags |= BNXT_FLAG_JUMBO;
3037                 bp->eth_dev->data->dev_conf.rxmode.offloads |=
3038                         DEV_RX_OFFLOAD_JUMBO_FRAME;
3039         } else {
3040                 bp->eth_dev->data->dev_conf.rxmode.offloads &=
3041                         ~DEV_RX_OFFLOAD_JUMBO_FRAME;
3042                 bp->flags &= ~BNXT_FLAG_JUMBO;
3043         }
3044
3045         /* Is there a change in mtu setting? */
3046         if (eth_dev->data->dev_conf.rxmode.max_rx_pkt_len == new_pkt_size)
3047                 return rc;
3048
3049         for (i = 0; i < bp->nr_vnics; i++) {
3050                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3051                 uint16_t size = 0;
3052
3053                 vnic->mru = BNXT_VNIC_MRU(new_mtu);
3054                 rc = bnxt_hwrm_vnic_cfg(bp, vnic);
3055                 if (rc)
3056                         break;
3057
3058                 size = rte_pktmbuf_data_room_size(bp->rx_queues[0]->mb_pool);
3059                 size -= RTE_PKTMBUF_HEADROOM;
3060
3061                 if (size < new_mtu) {
3062                         rc = bnxt_hwrm_vnic_plcmode_cfg(bp, vnic);
3063                         if (rc)
3064                                 return rc;
3065                 }
3066         }
3067
3068         if (!rc)
3069                 eth_dev->data->dev_conf.rxmode.max_rx_pkt_len = new_pkt_size;
3070
3071         PMD_DRV_LOG(INFO, "New MTU is %d\n", new_mtu);
3072
3073         return rc;
3074 }
3075
3076 static int
3077 bnxt_vlan_pvid_set_op(struct rte_eth_dev *dev, uint16_t pvid, int on)
3078 {
3079         struct bnxt *bp = dev->data->dev_private;
3080         uint16_t vlan = bp->vlan;
3081         int rc;
3082
3083         rc = is_bnxt_in_error(bp);
3084         if (rc)
3085                 return rc;
3086
3087         if (!BNXT_SINGLE_PF(bp)) {
3088                 PMD_DRV_LOG(ERR, "PVID cannot be modified on VF or on shared PF\n");
3089                 return -ENOTSUP;
3090         }
3091         bp->vlan = on ? pvid : 0;
3092
3093         rc = bnxt_hwrm_set_default_vlan(bp, 0, 0);
3094         if (rc)
3095                 bp->vlan = vlan;
3096         return rc;
3097 }
3098
3099 static int
3100 bnxt_dev_led_on_op(struct rte_eth_dev *dev)
3101 {
3102         struct bnxt *bp = dev->data->dev_private;
3103         int rc;
3104
3105         rc = is_bnxt_in_error(bp);
3106         if (rc)
3107                 return rc;
3108
3109         return bnxt_hwrm_port_led_cfg(bp, true);
3110 }
3111
3112 static int
3113 bnxt_dev_led_off_op(struct rte_eth_dev *dev)
3114 {
3115         struct bnxt *bp = dev->data->dev_private;
3116         int rc;
3117
3118         rc = is_bnxt_in_error(bp);
3119         if (rc)
3120                 return rc;
3121
3122         return bnxt_hwrm_port_led_cfg(bp, false);
3123 }
3124
3125 static uint32_t
3126 bnxt_rx_queue_count_op(struct rte_eth_dev *dev, uint16_t rx_queue_id)
3127 {
3128         struct bnxt *bp = (struct bnxt *)dev->data->dev_private;
3129         struct bnxt_cp_ring_info *cpr;
3130         uint32_t desc = 0, raw_cons;
3131         struct bnxt_rx_queue *rxq;
3132         struct rx_pkt_cmpl *rxcmp;
3133         int rc;
3134
3135         rc = is_bnxt_in_error(bp);
3136         if (rc)
3137                 return rc;
3138
3139         rxq = dev->data->rx_queues[rx_queue_id];
3140         cpr = rxq->cp_ring;
3141         raw_cons = cpr->cp_raw_cons;
3142
3143         while (1) {
3144                 uint32_t agg_cnt, cons, cmpl_type;
3145
3146                 cons = RING_CMP(cpr->cp_ring_struct, raw_cons);
3147                 rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons];
3148
3149                 if (!CMP_VALID(rxcmp, raw_cons, cpr->cp_ring_struct))
3150                         break;
3151
3152                 cmpl_type = CMP_TYPE(rxcmp);
3153
3154                 switch (cmpl_type) {
3155                 case CMPL_BASE_TYPE_RX_L2:
3156                 case CMPL_BASE_TYPE_RX_L2_V2:
3157                         agg_cnt = BNXT_RX_L2_AGG_BUFS(rxcmp);
3158                         raw_cons = raw_cons + CMP_LEN(cmpl_type) + agg_cnt;
3159                         desc++;
3160                         break;
3161
3162                 case CMPL_BASE_TYPE_RX_TPA_END:
3163                         if (BNXT_CHIP_P5(rxq->bp)) {
3164                                 struct rx_tpa_v2_end_cmpl_hi *p5_tpa_end;
3165
3166                                 p5_tpa_end = (void *)rxcmp;
3167                                 agg_cnt = BNXT_TPA_END_AGG_BUFS_TH(p5_tpa_end);
3168                         } else {
3169                                 struct rx_tpa_end_cmpl *tpa_end;
3170
3171                                 tpa_end = (void *)rxcmp;
3172                                 agg_cnt = BNXT_TPA_END_AGG_BUFS(tpa_end);
3173                         }
3174
3175                         raw_cons = raw_cons + CMP_LEN(cmpl_type) + agg_cnt;
3176                         desc++;
3177                         break;
3178
3179                 default:
3180                         raw_cons += CMP_LEN(cmpl_type);
3181                 }
3182         }
3183
3184         return desc;
3185 }
3186
3187 static int
3188 bnxt_rx_descriptor_status_op(void *rx_queue, uint16_t offset)
3189 {
3190         struct bnxt_rx_queue *rxq = rx_queue;
3191         struct bnxt_cp_ring_info *cpr;
3192         struct bnxt_rx_ring_info *rxr;
3193         uint32_t desc, raw_cons;
3194         struct bnxt *bp = rxq->bp;
3195         struct rx_pkt_cmpl *rxcmp;
3196         int rc;
3197
3198         rc = is_bnxt_in_error(bp);
3199         if (rc)
3200                 return rc;
3201
3202         if (offset >= rxq->nb_rx_desc)
3203                 return -EINVAL;
3204
3205         rxr = rxq->rx_ring;
3206         cpr = rxq->cp_ring;
3207
3208         /*
3209          * For the vector receive case, the completion at the requested
3210          * offset can be indexed directly.
3211          */
3212 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
3213         if (bp->flags & BNXT_FLAG_RX_VECTOR_PKT_MODE) {
3214                 struct rx_pkt_cmpl *rxcmp;
3215                 uint32_t cons;
3216
3217                 /* Check status of completion descriptor. */
3218                 raw_cons = cpr->cp_raw_cons +
3219                            offset * CMP_LEN(CMPL_BASE_TYPE_RX_L2);
3220                 cons = RING_CMP(cpr->cp_ring_struct, raw_cons);
3221                 rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons];
3222
3223                 if (CMP_VALID(rxcmp, raw_cons, cpr->cp_ring_struct))
3224                         return RTE_ETH_RX_DESC_DONE;
3225
3226                 /* Check whether rx desc has an mbuf attached. */
3227                 cons = RING_CMP(rxr->rx_ring_struct, raw_cons / 2);
3228                 if (cons >= rxq->rxrearm_start &&
3229                     cons < rxq->rxrearm_start + rxq->rxrearm_nb) {
3230                         return RTE_ETH_RX_DESC_UNAVAIL;
3231                 }
3232
3233                 return RTE_ETH_RX_DESC_AVAIL;
3234         }
3235 #endif
3236
3237         /*
3238          * For the non-vector receive case, scan the completion ring to
3239          * locate the completion descriptor for the requested offset.
3240          */
3241         raw_cons = cpr->cp_raw_cons;
3242         desc = 0;
3243         while (1) {
3244                 uint32_t agg_cnt, cons, cmpl_type;
3245
3246                 cons = RING_CMP(cpr->cp_ring_struct, raw_cons);
3247                 rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons];
3248
3249                 if (!CMP_VALID(rxcmp, raw_cons, cpr->cp_ring_struct))
3250                         break;
3251
3252                 cmpl_type = CMP_TYPE(rxcmp);
3253
3254                 switch (cmpl_type) {
3255                 case CMPL_BASE_TYPE_RX_L2:
3256                 case CMPL_BASE_TYPE_RX_L2_V2:
3257                         if (desc == offset) {
3258                                 cons = rxcmp->opaque;
3259                                 if (rxr->rx_buf_ring[cons])
3260                                         return RTE_ETH_RX_DESC_DONE;
3261                                 else
3262                                         return RTE_ETH_RX_DESC_UNAVAIL;
3263                         }
3264                         agg_cnt = BNXT_RX_L2_AGG_BUFS(rxcmp);
3265                         raw_cons = raw_cons + CMP_LEN(cmpl_type) + agg_cnt;
3266                         desc++;
3267                         break;
3268
3269                 case CMPL_BASE_TYPE_RX_TPA_END:
3270                         if (desc == offset)
3271                                 return RTE_ETH_RX_DESC_DONE;
3272
3273                         if (BNXT_CHIP_P5(rxq->bp)) {
3274                                 struct rx_tpa_v2_end_cmpl_hi *p5_tpa_end;
3275
3276                                 p5_tpa_end = (void *)rxcmp;
3277                                 agg_cnt = BNXT_TPA_END_AGG_BUFS_TH(p5_tpa_end);
3278                         } else {
3279                                 struct rx_tpa_end_cmpl *tpa_end;
3280
3281                                 tpa_end = (void *)rxcmp;
3282                                 agg_cnt = BNXT_TPA_END_AGG_BUFS(tpa_end);
3283                         }
3284
3285                         raw_cons = raw_cons + CMP_LEN(cmpl_type) + agg_cnt;
3286                         desc++;
3287                         break;
3288
3289                 default:
3290                         raw_cons += CMP_LEN(cmpl_type);
3291                 }
3292         }
3293
3294         return RTE_ETH_RX_DESC_AVAIL;
3295 }
3296
3297 static int
3298 bnxt_tx_descriptor_status_op(void *tx_queue, uint16_t offset)
3299 {
3300         struct bnxt_tx_queue *txq = (struct bnxt_tx_queue *)tx_queue;
3301         struct bnxt_cp_ring_info *cpr = txq->cp_ring;
3302         uint32_t ring_mask, raw_cons, nb_tx_pkts = 0;
3303         struct bnxt_ring *cp_ring_struct;
3304         struct cmpl_base *cp_desc_ring;
3305         int rc;
3306
3307         rc = is_bnxt_in_error(txq->bp);
3308         if (rc)
3309                 return rc;
3310
3311         if (offset >= txq->nb_tx_desc)
3312                 return -EINVAL;
3313
3314         /* Return "desc done" if descriptor is available for use. */
3315         if (bnxt_tx_bds_in_hw(txq) <= offset)
3316                 return RTE_ETH_TX_DESC_DONE;
3317
3318         raw_cons = cpr->cp_raw_cons;
3319         cp_desc_ring = cpr->cp_desc_ring;
3320         cp_ring_struct = cpr->cp_ring_struct;
3321         ring_mask = cpr->cp_ring_struct->ring_mask;
3322
3323         /* Check to see if hw has posted a completion for the descriptor. */
3324         while (1) {
3325                 struct tx_cmpl *txcmp;
3326                 uint32_t cons;
3327
3328                 cons = RING_CMPL(ring_mask, raw_cons);
3329                 txcmp = (struct tx_cmpl *)&cp_desc_ring[cons];
3330
3331                 if (!CMP_VALID(txcmp, raw_cons, cp_ring_struct))
3332                         break;
3333
3334                 if (CMP_TYPE(txcmp) == TX_CMPL_TYPE_TX_L2)
3335                         nb_tx_pkts += rte_le_to_cpu_32(txcmp->opaque);
3336
3337                 if (nb_tx_pkts > offset)
3338                         return RTE_ETH_TX_DESC_DONE;
3339
3340                 raw_cons = NEXT_RAW_CMP(raw_cons);
3341         }
3342
3343         /* Descriptor is pending transmit, not yet completed by hardware. */
3344         return RTE_ETH_TX_DESC_FULL;
3345 }
3346
3347 int
3348 bnxt_flow_ops_get_op(struct rte_eth_dev *dev,
3349                      const struct rte_flow_ops **ops)
3350 {
3351         struct bnxt *bp = dev->data->dev_private;
3352         int ret = 0;
3353
3354         if (!bp)
3355                 return -EIO;
3356
3357         if (BNXT_ETH_DEV_IS_REPRESENTOR(dev)) {
3358                 struct bnxt_representor *vfr = dev->data->dev_private;
3359                 bp = vfr->parent_dev->data->dev_private;
3360                 /* parent is deleted while children are still valid */
3361                 if (!bp) {
3362                         PMD_DRV_LOG(DEBUG, "BNXT Port:%d VFR Error\n",
3363                                     dev->data->port_id);
3364                         return -EIO;
3365                 }
3366         }
3367
3368         ret = is_bnxt_in_error(bp);
3369         if (ret)
3370                 return ret;
3371
3372         /* PMD supports thread-safe flow operations.  rte_flow API
3373          * functions can avoid mutex for multi-thread safety.
3374          */
3375         dev->data->dev_flags |= RTE_ETH_DEV_FLOW_OPS_THREAD_SAFE;
3376
3377         if (BNXT_TRUFLOW_EN(bp))
3378                 *ops = &bnxt_ulp_rte_flow_ops;
3379         else
3380                 *ops = &bnxt_flow_ops;
3381
3382         return ret;
3383 }
3384
3385 static const uint32_t *
3386 bnxt_dev_supported_ptypes_get_op(struct rte_eth_dev *dev)
3387 {
3388         static const uint32_t ptypes[] = {
3389                 RTE_PTYPE_L2_ETHER_VLAN,
3390                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
3391                 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
3392                 RTE_PTYPE_L4_ICMP,
3393                 RTE_PTYPE_L4_TCP,
3394                 RTE_PTYPE_L4_UDP,
3395                 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
3396                 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
3397                 RTE_PTYPE_INNER_L4_ICMP,
3398                 RTE_PTYPE_INNER_L4_TCP,
3399                 RTE_PTYPE_INNER_L4_UDP,
3400                 RTE_PTYPE_UNKNOWN
3401         };
3402
3403         if (!dev->rx_pkt_burst)
3404                 return NULL;
3405
3406         return ptypes;
3407 }
3408
3409 static int bnxt_map_regs(struct bnxt *bp, uint32_t *reg_arr, int count,
3410                          int reg_win)
3411 {
3412         uint32_t reg_base = *reg_arr & 0xfffff000;
3413         uint32_t win_off;
3414         int i;
3415
3416         for (i = 0; i < count; i++) {
3417                 if ((reg_arr[i] & 0xfffff000) != reg_base)
3418                         return -ERANGE;
3419         }
3420         win_off = BNXT_GRCPF_REG_WINDOW_BASE_OUT + (reg_win - 1) * 4;
3421         rte_write32(reg_base, (uint8_t *)bp->bar0 + win_off);
3422         return 0;
3423 }
3424
3425 static int bnxt_map_ptp_regs(struct bnxt *bp)
3426 {
3427         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3428         uint32_t *reg_arr;
3429         int rc, i;
3430
3431         reg_arr = ptp->rx_regs;
3432         rc = bnxt_map_regs(bp, reg_arr, BNXT_PTP_RX_REGS, 5);
3433         if (rc)
3434                 return rc;
3435
3436         reg_arr = ptp->tx_regs;
3437         rc = bnxt_map_regs(bp, reg_arr, BNXT_PTP_TX_REGS, 6);
3438         if (rc)
3439                 return rc;
3440
3441         for (i = 0; i < BNXT_PTP_RX_REGS; i++)
3442                 ptp->rx_mapped_regs[i] = 0x5000 + (ptp->rx_regs[i] & 0xfff);
3443
3444         for (i = 0; i < BNXT_PTP_TX_REGS; i++)
3445                 ptp->tx_mapped_regs[i] = 0x6000 + (ptp->tx_regs[i] & 0xfff);
3446
3447         return 0;
3448 }
3449
3450 static void bnxt_unmap_ptp_regs(struct bnxt *bp)
3451 {
3452         rte_write32(0, (uint8_t *)bp->bar0 +
3453                          BNXT_GRCPF_REG_WINDOW_BASE_OUT + 16);
3454         rte_write32(0, (uint8_t *)bp->bar0 +
3455                          BNXT_GRCPF_REG_WINDOW_BASE_OUT + 20);
3456 }
3457
3458 static uint64_t bnxt_cc_read(struct bnxt *bp)
3459 {
3460         uint64_t ns;
3461
3462         ns = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3463                               BNXT_GRCPF_REG_SYNC_TIME));
3464         ns |= (uint64_t)(rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3465                                           BNXT_GRCPF_REG_SYNC_TIME + 4))) << 32;
3466         return ns;
3467 }
3468
3469 static int bnxt_get_tx_ts(struct bnxt *bp, uint64_t *ts)
3470 {
3471         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3472         uint32_t fifo;
3473
3474         fifo = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3475                                 ptp->tx_mapped_regs[BNXT_PTP_TX_FIFO]));
3476         if (fifo & BNXT_PTP_TX_FIFO_EMPTY)
3477                 return -EAGAIN;
3478
3479         fifo = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3480                                 ptp->tx_mapped_regs[BNXT_PTP_TX_FIFO]));
3481         *ts = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3482                                 ptp->tx_mapped_regs[BNXT_PTP_TX_TS_L]));
3483         *ts |= (uint64_t)rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3484                                 ptp->tx_mapped_regs[BNXT_PTP_TX_TS_H])) << 32;
3485         rte_read32((uint8_t *)bp->bar0 + ptp->tx_mapped_regs[BNXT_PTP_TX_SEQ]);
3486
3487         return 0;
3488 }
3489
3490 static int bnxt_clr_rx_ts(struct bnxt *bp, uint64_t *last_ts)
3491 {
3492         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3493         struct bnxt_pf_info *pf = bp->pf;
3494         uint16_t port_id;
3495         int i = 0;
3496         uint32_t fifo;
3497
3498         if (!ptp || (bp->flags & BNXT_FLAG_CHIP_P5))
3499                 return -EINVAL;
3500
3501         port_id = pf->port_id;
3502         fifo = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3503                                 ptp->rx_mapped_regs[BNXT_PTP_RX_FIFO]));
3504         while ((fifo & BNXT_PTP_RX_FIFO_PENDING) && (i < BNXT_PTP_RX_PND_CNT)) {
3505                 rte_write32(1 << port_id, (uint8_t *)bp->bar0 +
3506                             ptp->rx_mapped_regs[BNXT_PTP_RX_FIFO_ADV]);
3507                 fifo = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3508                                         ptp->rx_mapped_regs[BNXT_PTP_RX_FIFO]));
3509                 *last_ts = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3510                                         ptp->rx_mapped_regs[BNXT_PTP_RX_TS_L]));
3511                 *last_ts |= (uint64_t)rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3512                                         ptp->rx_mapped_regs[BNXT_PTP_RX_TS_H])) << 32;
3513                 i++;
3514         }
3515
3516         if (i >= BNXT_PTP_RX_PND_CNT)
3517                 return -EBUSY;
3518
3519         return 0;
3520 }
3521
3522 static int bnxt_get_rx_ts(struct bnxt *bp, uint64_t *ts)
3523 {
3524         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3525         struct bnxt_pf_info *pf = bp->pf;
3526         uint16_t port_id;
3527         uint32_t fifo;
3528
3529         fifo = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3530                                 ptp->rx_mapped_regs[BNXT_PTP_RX_FIFO]));
3531         if (!(fifo & BNXT_PTP_RX_FIFO_PENDING))
3532                 return -EAGAIN;
3533
3534         port_id = pf->port_id;
3535         rte_write32(1 << port_id, (uint8_t *)bp->bar0 +
3536                ptp->rx_mapped_regs[BNXT_PTP_RX_FIFO_ADV]);
3537
3538         fifo = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3539                                    ptp->rx_mapped_regs[BNXT_PTP_RX_FIFO]));
3540         if (fifo & BNXT_PTP_RX_FIFO_PENDING)
3541                 return bnxt_clr_rx_ts(bp, ts);
3542
3543         *ts = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3544                                 ptp->rx_mapped_regs[BNXT_PTP_RX_TS_L]));
3545         *ts |= (uint64_t)rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3546                                 ptp->rx_mapped_regs[BNXT_PTP_RX_TS_H])) << 32;
3547
3548         return 0;
3549 }
3550
3551 static int
3552 bnxt_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts)
3553 {
3554         uint64_t ns;
3555         struct bnxt *bp = dev->data->dev_private;
3556         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3557
3558         if (!ptp)
3559                 return -ENOTSUP;
3560
3561         ns = rte_timespec_to_ns(ts);
3562         /* Set the timecounters to a new value. */
3563         ptp->tc.nsec = ns;
3564         ptp->tx_tstamp_tc.nsec = ns;
3565         ptp->rx_tstamp_tc.nsec = ns;
3566
3567         return 0;
3568 }
3569
3570 static int
3571 bnxt_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts)
3572 {
3573         struct bnxt *bp = dev->data->dev_private;
3574         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3575         uint64_t ns, systime_cycles = 0;
3576         int rc = 0;
3577
3578         if (!ptp)
3579                 return -ENOTSUP;
3580
3581         if (BNXT_CHIP_P5(bp))
3582                 rc = bnxt_hwrm_port_ts_query(bp, BNXT_PTP_FLAGS_CURRENT_TIME,
3583                                              &systime_cycles);
3584         else
3585                 systime_cycles = bnxt_cc_read(bp);
3586
3587         ns = rte_timecounter_update(&ptp->tc, systime_cycles);
3588         *ts = rte_ns_to_timespec(ns);
3589
3590         return rc;
3591 }
3592 static int
3593 bnxt_timesync_enable(struct rte_eth_dev *dev)
3594 {
3595         struct bnxt *bp = dev->data->dev_private;
3596         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3597         uint32_t shift = 0;
3598         int rc;
3599
3600         if (!ptp)
3601                 return -ENOTSUP;
3602
3603         ptp->rx_filter = 1;
3604         ptp->tx_tstamp_en = 1;
3605         ptp->rxctl = BNXT_PTP_MSG_EVENTS;
3606
3607         rc = bnxt_hwrm_ptp_cfg(bp);
3608         if (rc)
3609                 return rc;
3610
3611         memset(&ptp->tc, 0, sizeof(struct rte_timecounter));
3612         memset(&ptp->rx_tstamp_tc, 0, sizeof(struct rte_timecounter));
3613         memset(&ptp->tx_tstamp_tc, 0, sizeof(struct rte_timecounter));
3614
3615         ptp->tc.cc_mask = BNXT_CYCLECOUNTER_MASK;
3616         ptp->tc.cc_shift = shift;
3617         ptp->tc.nsec_mask = (1ULL << shift) - 1;
3618
3619         ptp->rx_tstamp_tc.cc_mask = BNXT_CYCLECOUNTER_MASK;
3620         ptp->rx_tstamp_tc.cc_shift = shift;
3621         ptp->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
3622
3623         ptp->tx_tstamp_tc.cc_mask = BNXT_CYCLECOUNTER_MASK;
3624         ptp->tx_tstamp_tc.cc_shift = shift;
3625         ptp->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
3626
3627         if (!BNXT_CHIP_P5(bp))
3628                 bnxt_map_ptp_regs(bp);
3629         else
3630                 rc = bnxt_ptp_start(bp);
3631
3632         return rc;
3633 }
3634
3635 static int
3636 bnxt_timesync_disable(struct rte_eth_dev *dev)
3637 {
3638         struct bnxt *bp = dev->data->dev_private;
3639         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3640
3641         if (!ptp)
3642                 return -ENOTSUP;
3643
3644         ptp->rx_filter = 0;
3645         ptp->tx_tstamp_en = 0;
3646         ptp->rxctl = 0;
3647
3648         bnxt_hwrm_ptp_cfg(bp);
3649
3650         if (!BNXT_CHIP_P5(bp))
3651                 bnxt_unmap_ptp_regs(bp);
3652         else
3653                 bnxt_ptp_stop(bp);
3654
3655         return 0;
3656 }
3657
3658 static int
3659 bnxt_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
3660                                  struct timespec *timestamp,
3661                                  uint32_t flags __rte_unused)
3662 {
3663         struct bnxt *bp = dev->data->dev_private;
3664         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3665         uint64_t rx_tstamp_cycles = 0;
3666         uint64_t ns;
3667
3668         if (!ptp)
3669                 return -ENOTSUP;
3670
3671         if (BNXT_CHIP_P5(bp))
3672                 rx_tstamp_cycles = ptp->rx_timestamp;
3673         else
3674                 bnxt_get_rx_ts(bp, &rx_tstamp_cycles);
3675
3676         ns = rte_timecounter_update(&ptp->rx_tstamp_tc, rx_tstamp_cycles);
3677         *timestamp = rte_ns_to_timespec(ns);
3678         return  0;
3679 }
3680
3681 static int
3682 bnxt_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
3683                                  struct timespec *timestamp)
3684 {
3685         struct bnxt *bp = dev->data->dev_private;
3686         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3687         uint64_t tx_tstamp_cycles = 0;
3688         uint64_t ns;
3689         int rc = 0;
3690
3691         if (!ptp)
3692                 return -ENOTSUP;
3693
3694         if (BNXT_CHIP_P5(bp))
3695                 rc = bnxt_hwrm_port_ts_query(bp, BNXT_PTP_FLAGS_PATH_TX,
3696                                              &tx_tstamp_cycles);
3697         else
3698                 rc = bnxt_get_tx_ts(bp, &tx_tstamp_cycles);
3699
3700         ns = rte_timecounter_update(&ptp->tx_tstamp_tc, tx_tstamp_cycles);
3701         *timestamp = rte_ns_to_timespec(ns);
3702
3703         return rc;
3704 }
3705
3706 static int
3707 bnxt_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta)
3708 {
3709         struct bnxt *bp = dev->data->dev_private;
3710         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3711
3712         if (!ptp)
3713                 return -ENOTSUP;
3714
3715         ptp->tc.nsec += delta;
3716         ptp->tx_tstamp_tc.nsec += delta;
3717         ptp->rx_tstamp_tc.nsec += delta;
3718
3719         return 0;
3720 }
3721
3722 static int
3723 bnxt_get_eeprom_length_op(struct rte_eth_dev *dev)
3724 {
3725         struct bnxt *bp = dev->data->dev_private;
3726         int rc;
3727         uint32_t dir_entries;
3728         uint32_t entry_length;
3729
3730         rc = is_bnxt_in_error(bp);
3731         if (rc)
3732                 return rc;
3733
3734         PMD_DRV_LOG(INFO, PCI_PRI_FMT "\n",
3735                     bp->pdev->addr.domain, bp->pdev->addr.bus,
3736                     bp->pdev->addr.devid, bp->pdev->addr.function);
3737
3738         rc = bnxt_hwrm_nvm_get_dir_info(bp, &dir_entries, &entry_length);
3739         if (rc != 0)
3740                 return rc;
3741
3742         return dir_entries * entry_length;
3743 }
3744
3745 static int
3746 bnxt_get_eeprom_op(struct rte_eth_dev *dev,
3747                 struct rte_dev_eeprom_info *in_eeprom)
3748 {
3749         struct bnxt *bp = dev->data->dev_private;
3750         uint32_t index;
3751         uint32_t offset;
3752         int rc;
3753
3754         rc = is_bnxt_in_error(bp);
3755         if (rc)
3756                 return rc;
3757
3758         PMD_DRV_LOG(INFO, PCI_PRI_FMT " in_eeprom->offset = %d len = %d\n",
3759                     bp->pdev->addr.domain, bp->pdev->addr.bus,
3760                     bp->pdev->addr.devid, bp->pdev->addr.function,
3761                     in_eeprom->offset, in_eeprom->length);
3762
3763         if (in_eeprom->offset == 0) /* special offset value to get directory */
3764                 return bnxt_get_nvram_directory(bp, in_eeprom->length,
3765                                                 in_eeprom->data);
3766
3767         index = in_eeprom->offset >> 24;
3768         offset = in_eeprom->offset & 0xffffff;
3769
3770         if (index != 0)
3771                 return bnxt_hwrm_get_nvram_item(bp, index - 1, offset,
3772                                            in_eeprom->length, in_eeprom->data);
3773
3774         return 0;
3775 }
3776
3777 static bool bnxt_dir_type_is_ape_bin_format(uint16_t dir_type)
3778 {
3779         switch (dir_type) {
3780         case BNX_DIR_TYPE_CHIMP_PATCH:
3781         case BNX_DIR_TYPE_BOOTCODE:
3782         case BNX_DIR_TYPE_BOOTCODE_2:
3783         case BNX_DIR_TYPE_APE_FW:
3784         case BNX_DIR_TYPE_APE_PATCH:
3785         case BNX_DIR_TYPE_KONG_FW:
3786         case BNX_DIR_TYPE_KONG_PATCH:
3787         case BNX_DIR_TYPE_BONO_FW:
3788         case BNX_DIR_TYPE_BONO_PATCH:
3789                 /* FALLTHROUGH */
3790                 return true;
3791         }
3792
3793         return false;
3794 }
3795
3796 static bool bnxt_dir_type_is_other_exec_format(uint16_t dir_type)
3797 {
3798         switch (dir_type) {
3799         case BNX_DIR_TYPE_AVS:
3800         case BNX_DIR_TYPE_EXP_ROM_MBA:
3801         case BNX_DIR_TYPE_PCIE:
3802         case BNX_DIR_TYPE_TSCF_UCODE:
3803         case BNX_DIR_TYPE_EXT_PHY:
3804         case BNX_DIR_TYPE_CCM:
3805         case BNX_DIR_TYPE_ISCSI_BOOT:
3806         case BNX_DIR_TYPE_ISCSI_BOOT_IPV6:
3807         case BNX_DIR_TYPE_ISCSI_BOOT_IPV4N6:
3808                 /* FALLTHROUGH */
3809                 return true;
3810         }
3811
3812         return false;
3813 }
3814
3815 static bool bnxt_dir_type_is_executable(uint16_t dir_type)
3816 {
3817         return bnxt_dir_type_is_ape_bin_format(dir_type) ||
3818                 bnxt_dir_type_is_other_exec_format(dir_type);
3819 }
3820
3821 static int
3822 bnxt_set_eeprom_op(struct rte_eth_dev *dev,
3823                 struct rte_dev_eeprom_info *in_eeprom)
3824 {
3825         struct bnxt *bp = dev->data->dev_private;
3826         uint8_t index, dir_op;
3827         uint16_t type, ext, ordinal, attr;
3828         int rc;
3829
3830         rc = is_bnxt_in_error(bp);
3831         if (rc)
3832                 return rc;
3833
3834         PMD_DRV_LOG(INFO, PCI_PRI_FMT " in_eeprom->offset = %d len = %d\n",
3835                     bp->pdev->addr.domain, bp->pdev->addr.bus,
3836                     bp->pdev->addr.devid, bp->pdev->addr.function,
3837                     in_eeprom->offset, in_eeprom->length);
3838
3839         if (!BNXT_PF(bp)) {
3840                 PMD_DRV_LOG(ERR, "NVM write not supported from a VF\n");
3841                 return -EINVAL;
3842         }
3843
3844         type = in_eeprom->magic >> 16;
3845
3846         if (type == 0xffff) { /* special value for directory operations */
3847                 index = in_eeprom->magic & 0xff;
3848                 dir_op = in_eeprom->magic >> 8;
3849                 if (index == 0)
3850                         return -EINVAL;
3851                 switch (dir_op) {
3852                 case 0x0e: /* erase */
3853                         if (in_eeprom->offset != ~in_eeprom->magic)
3854                                 return -EINVAL;
3855                         return bnxt_hwrm_erase_nvram_directory(bp, index - 1);
3856                 default:
3857                         return -EINVAL;
3858                 }
3859         }
3860
3861         /* Create or re-write an NVM item: */
3862         if (bnxt_dir_type_is_executable(type) == true)
3863                 return -EOPNOTSUPP;
3864         ext = in_eeprom->magic & 0xffff;
3865         ordinal = in_eeprom->offset >> 16;
3866         attr = in_eeprom->offset & 0xffff;
3867
3868         return bnxt_hwrm_flash_nvram(bp, type, ordinal, ext, attr,
3869                                      in_eeprom->data, in_eeprom->length);
3870 }
3871
3872 static int bnxt_get_module_info(struct rte_eth_dev *dev,
3873                                 struct rte_eth_dev_module_info *modinfo)
3874 {
3875         uint8_t module_info[SFF_DIAG_SUPPORT_OFFSET + 1];
3876         struct bnxt *bp = dev->data->dev_private;
3877         int rc;
3878
3879         /* No point in going further if phy status indicates
3880          * module is not inserted or if it is powered down or
3881          * if it is of type 10GBase-T
3882          */
3883         if (bp->link_info->module_status >
3884             HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_WARNINGMSG) {
3885                 PMD_DRV_LOG(NOTICE, "Port %u : Module is not inserted or is powered down\n",
3886                             dev->data->port_id);
3887                 return -ENOTSUP;
3888         }
3889
3890         /* This feature is not supported in older firmware versions */
3891         if (bp->hwrm_spec_code < 0x10202) {
3892                 PMD_DRV_LOG(NOTICE, "Port %u : Feature is not supported in older firmware\n",
3893                             dev->data->port_id);
3894                 return -ENOTSUP;
3895         }
3896
3897         rc = bnxt_hwrm_read_sfp_module_eeprom_info(bp, I2C_DEV_ADDR_A0, 0, 0,
3898                                                    SFF_DIAG_SUPPORT_OFFSET + 1,
3899                                                    module_info);
3900
3901         if (rc)
3902                 return rc;
3903
3904         switch (module_info[0]) {
3905         case SFF_MODULE_ID_SFP:
3906                 modinfo->type = RTE_ETH_MODULE_SFF_8472;
3907                 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8472_LEN;
3908                 if (module_info[SFF_DIAG_SUPPORT_OFFSET] == 0)
3909                         modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8436_LEN;
3910                 break;
3911         case SFF_MODULE_ID_QSFP:
3912         case SFF_MODULE_ID_QSFP_PLUS:
3913                 modinfo->type = RTE_ETH_MODULE_SFF_8436;
3914                 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8436_LEN;
3915                 break;
3916         case SFF_MODULE_ID_QSFP28:
3917                 modinfo->type = RTE_ETH_MODULE_SFF_8636;
3918                 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8636_MAX_LEN;
3919                 if (module_info[SFF8636_FLATMEM_OFFSET] & SFF8636_FLATMEM_MASK)
3920                         modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8636_LEN;
3921                 break;
3922         default:
3923                 PMD_DRV_LOG(NOTICE, "Port %u : Unsupported module\n", dev->data->port_id);
3924                 return -ENOTSUP;
3925         }
3926
3927         PMD_DRV_LOG(INFO, "Port %u : modinfo->type = %d modinfo->eeprom_len = %d\n",
3928                     dev->data->port_id, modinfo->type, modinfo->eeprom_len);
3929
3930         return 0;
3931 }
3932
3933 static int bnxt_get_module_eeprom(struct rte_eth_dev *dev,
3934                                   struct rte_dev_eeprom_info *info)
3935 {
3936         uint8_t pg_addr[5] = { I2C_DEV_ADDR_A0, I2C_DEV_ADDR_A0 };
3937         uint32_t offset = info->offset, length = info->length;
3938         uint8_t module_info[SFF_DIAG_SUPPORT_OFFSET + 1];
3939         struct bnxt *bp = dev->data->dev_private;
3940         uint8_t *data = info->data;
3941         uint8_t page = offset >> 7;
3942         uint8_t max_pages = 2;
3943         uint8_t opt_pages;
3944         int rc;
3945
3946         rc = bnxt_hwrm_read_sfp_module_eeprom_info(bp, I2C_DEV_ADDR_A0, 0, 0,
3947                                                    SFF_DIAG_SUPPORT_OFFSET + 1,
3948                                                    module_info);
3949         if (rc)
3950                 return rc;
3951
3952         switch (module_info[0]) {
3953         case SFF_MODULE_ID_SFP:
3954                 module_info[SFF_DIAG_SUPPORT_OFFSET] = 0;
3955                 if (module_info[SFF_DIAG_SUPPORT_OFFSET]) {
3956                         pg_addr[2] = I2C_DEV_ADDR_A2;
3957                         pg_addr[3] = I2C_DEV_ADDR_A2;
3958                         max_pages = 4;
3959                 }
3960                 break;
3961         case SFF_MODULE_ID_QSFP28:
3962                 rc = bnxt_hwrm_read_sfp_module_eeprom_info(bp, I2C_DEV_ADDR_A0, 0,
3963                                                            SFF8636_OPT_PAGES_OFFSET,
3964                                                            1, &opt_pages);
3965                 if (rc)
3966                         return rc;
3967
3968                 if (opt_pages & SFF8636_PAGE1_MASK) {
3969                         pg_addr[2] = I2C_DEV_ADDR_A0;
3970                         max_pages = 3;
3971                 }
3972                 if (opt_pages & SFF8636_PAGE2_MASK) {
3973                         pg_addr[3] = I2C_DEV_ADDR_A0;
3974                         max_pages = 4;
3975                 }
3976                 if (~module_info[SFF8636_FLATMEM_OFFSET] & SFF8636_FLATMEM_MASK) {
3977                         pg_addr[4] = I2C_DEV_ADDR_A0;
3978                         max_pages = 5;
3979                 }
3980                 break;
3981         default:
3982                 break;
3983         }
3984
3985         memset(data, 0, length);
3986
3987         offset &= 0xff;
3988         while (length && page < max_pages) {
3989                 uint8_t raw_page = page ? page - 1 : 0;
3990                 uint16_t chunk;
3991
3992                 if (pg_addr[page] == I2C_DEV_ADDR_A2)
3993                         raw_page = 0;
3994                 else if (page)
3995                         offset |= 0x80;
3996                 chunk = RTE_MIN(length, 256 - offset);
3997
3998                 if (pg_addr[page]) {
3999                         rc = bnxt_hwrm_read_sfp_module_eeprom_info(bp, pg_addr[page],
4000                                                                    raw_page, offset,
4001                                                                    chunk, data);
4002                         if (rc)
4003                                 return rc;
4004                 }
4005
4006                 data += chunk;
4007                 length -= chunk;
4008                 offset = 0;
4009                 page += 1 + (chunk > 128);
4010         }
4011
4012         return length ? -EINVAL : 0;
4013 }
4014
4015 /*
4016  * Initialization
4017  */
4018
4019 static const struct eth_dev_ops bnxt_dev_ops = {
4020         .dev_infos_get = bnxt_dev_info_get_op,
4021         .dev_close = bnxt_dev_close_op,
4022         .dev_configure = bnxt_dev_configure_op,
4023         .dev_start = bnxt_dev_start_op,
4024         .dev_stop = bnxt_dev_stop_op,
4025         .dev_set_link_up = bnxt_dev_set_link_up_op,
4026         .dev_set_link_down = bnxt_dev_set_link_down_op,
4027         .stats_get = bnxt_stats_get_op,
4028         .stats_reset = bnxt_stats_reset_op,
4029         .rx_queue_setup = bnxt_rx_queue_setup_op,
4030         .rx_queue_release = bnxt_rx_queue_release_op,
4031         .tx_queue_setup = bnxt_tx_queue_setup_op,
4032         .tx_queue_release = bnxt_tx_queue_release_op,
4033         .rx_queue_intr_enable = bnxt_rx_queue_intr_enable_op,
4034         .rx_queue_intr_disable = bnxt_rx_queue_intr_disable_op,
4035         .reta_update = bnxt_reta_update_op,
4036         .reta_query = bnxt_reta_query_op,
4037         .rss_hash_update = bnxt_rss_hash_update_op,
4038         .rss_hash_conf_get = bnxt_rss_hash_conf_get_op,
4039         .link_update = bnxt_link_update_op,
4040         .promiscuous_enable = bnxt_promiscuous_enable_op,
4041         .promiscuous_disable = bnxt_promiscuous_disable_op,
4042         .allmulticast_enable = bnxt_allmulticast_enable_op,
4043         .allmulticast_disable = bnxt_allmulticast_disable_op,
4044         .mac_addr_add = bnxt_mac_addr_add_op,
4045         .mac_addr_remove = bnxt_mac_addr_remove_op,
4046         .flow_ctrl_get = bnxt_flow_ctrl_get_op,
4047         .flow_ctrl_set = bnxt_flow_ctrl_set_op,
4048         .udp_tunnel_port_add  = bnxt_udp_tunnel_port_add_op,
4049         .udp_tunnel_port_del  = bnxt_udp_tunnel_port_del_op,
4050         .vlan_filter_set = bnxt_vlan_filter_set_op,
4051         .vlan_offload_set = bnxt_vlan_offload_set_op,
4052         .vlan_tpid_set = bnxt_vlan_tpid_set_op,
4053         .vlan_pvid_set = bnxt_vlan_pvid_set_op,
4054         .mtu_set = bnxt_mtu_set_op,
4055         .mac_addr_set = bnxt_set_default_mac_addr_op,
4056         .xstats_get = bnxt_dev_xstats_get_op,
4057         .xstats_get_names = bnxt_dev_xstats_get_names_op,
4058         .xstats_reset = bnxt_dev_xstats_reset_op,
4059         .fw_version_get = bnxt_fw_version_get,
4060         .set_mc_addr_list = bnxt_dev_set_mc_addr_list_op,
4061         .rxq_info_get = bnxt_rxq_info_get_op,
4062         .txq_info_get = bnxt_txq_info_get_op,
4063         .rx_burst_mode_get = bnxt_rx_burst_mode_get,
4064         .tx_burst_mode_get = bnxt_tx_burst_mode_get,
4065         .dev_led_on = bnxt_dev_led_on_op,
4066         .dev_led_off = bnxt_dev_led_off_op,
4067         .rx_queue_start = bnxt_rx_queue_start,
4068         .rx_queue_stop = bnxt_rx_queue_stop,
4069         .tx_queue_start = bnxt_tx_queue_start,
4070         .tx_queue_stop = bnxt_tx_queue_stop,
4071         .flow_ops_get = bnxt_flow_ops_get_op,
4072         .dev_supported_ptypes_get = bnxt_dev_supported_ptypes_get_op,
4073         .get_eeprom_length    = bnxt_get_eeprom_length_op,
4074         .get_eeprom           = bnxt_get_eeprom_op,
4075         .set_eeprom           = bnxt_set_eeprom_op,
4076         .get_module_info = bnxt_get_module_info,
4077         .get_module_eeprom = bnxt_get_module_eeprom,
4078         .timesync_enable      = bnxt_timesync_enable,
4079         .timesync_disable     = bnxt_timesync_disable,
4080         .timesync_read_time   = bnxt_timesync_read_time,
4081         .timesync_write_time   = bnxt_timesync_write_time,
4082         .timesync_adjust_time = bnxt_timesync_adjust_time,
4083         .timesync_read_rx_timestamp = bnxt_timesync_read_rx_timestamp,
4084         .timesync_read_tx_timestamp = bnxt_timesync_read_tx_timestamp,
4085 };
4086
4087 static uint32_t bnxt_map_reset_regs(struct bnxt *bp, uint32_t reg)
4088 {
4089         uint32_t offset;
4090
4091         /* Only pre-map the reset GRC registers using window 3 */
4092         rte_write32(reg & 0xfffff000, (uint8_t *)bp->bar0 +
4093                     BNXT_GRCPF_REG_WINDOW_BASE_OUT + 8);
4094
4095         offset = BNXT_GRCP_WINDOW_3_BASE + (reg & 0xffc);
4096
4097         return offset;
4098 }
4099
4100 int bnxt_map_fw_health_status_regs(struct bnxt *bp)
4101 {
4102         struct bnxt_error_recovery_info *info = bp->recovery_info;
4103         uint32_t reg_base = 0xffffffff;
4104         int i;
4105
4106         /* Only pre-map the monitoring GRC registers using window 2 */
4107         for (i = 0; i < BNXT_FW_STATUS_REG_CNT; i++) {
4108                 uint32_t reg = info->status_regs[i];
4109
4110                 if (BNXT_FW_STATUS_REG_TYPE(reg) != BNXT_FW_STATUS_REG_TYPE_GRC)
4111                         continue;
4112
4113                 if (reg_base == 0xffffffff)
4114                         reg_base = reg & 0xfffff000;
4115                 if ((reg & 0xfffff000) != reg_base)
4116                         return -ERANGE;
4117
4118                 /* Use mask 0xffc as the Lower 2 bits indicates
4119                  * address space location
4120                  */
4121                 info->mapped_status_regs[i] = BNXT_GRCP_WINDOW_2_BASE +
4122                                                 (reg & 0xffc);
4123         }
4124
4125         if (reg_base == 0xffffffff)
4126                 return 0;
4127
4128         rte_write32(reg_base, (uint8_t *)bp->bar0 +
4129                     BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
4130
4131         return 0;
4132 }
4133
4134 static void bnxt_write_fw_reset_reg(struct bnxt *bp, uint32_t index)
4135 {
4136         struct bnxt_error_recovery_info *info = bp->recovery_info;
4137         uint32_t delay = info->delay_after_reset[index];
4138         uint32_t val = info->reset_reg_val[index];
4139         uint32_t reg = info->reset_reg[index];
4140         uint32_t type, offset;
4141         int ret;
4142
4143         type = BNXT_FW_STATUS_REG_TYPE(reg);
4144         offset = BNXT_FW_STATUS_REG_OFF(reg);
4145
4146         switch (type) {
4147         case BNXT_FW_STATUS_REG_TYPE_CFG:
4148                 ret = rte_pci_write_config(bp->pdev, &val, sizeof(val), offset);
4149                 if (ret < 0) {
4150                         PMD_DRV_LOG(ERR, "Failed to write %#x at PCI offset %#x",
4151                                     val, offset);
4152                         return;
4153                 }
4154                 break;
4155         case BNXT_FW_STATUS_REG_TYPE_GRC:
4156                 offset = bnxt_map_reset_regs(bp, offset);
4157                 rte_write32(val, (uint8_t *)bp->bar0 + offset);
4158                 break;
4159         case BNXT_FW_STATUS_REG_TYPE_BAR0:
4160                 rte_write32(val, (uint8_t *)bp->bar0 + offset);
4161                 break;
4162         }
4163         /* wait on a specific interval of time until core reset is complete */
4164         if (delay)
4165                 rte_delay_ms(delay);
4166 }
4167
4168 static void bnxt_dev_cleanup(struct bnxt *bp)
4169 {
4170         bp->eth_dev->data->dev_link.link_status = 0;
4171         bp->link_info->link_up = 0;
4172         if (bp->eth_dev->data->dev_started)
4173                 bnxt_dev_stop(bp->eth_dev);
4174
4175         bnxt_uninit_resources(bp, true);
4176 }
4177
4178 static int
4179 bnxt_check_fw_reset_done(struct bnxt *bp)
4180 {
4181         int timeout = bp->fw_reset_max_msecs;
4182         uint16_t val = 0;
4183         int rc;
4184
4185         do {
4186                 rc = rte_pci_read_config(bp->pdev, &val, sizeof(val), PCI_SUBSYSTEM_ID_OFFSET);
4187                 if (rc < 0) {
4188                         PMD_DRV_LOG(ERR, "Failed to read PCI offset 0x%x", PCI_SUBSYSTEM_ID_OFFSET);
4189                         return rc;
4190                 }
4191                 if (val != 0xffff)
4192                         break;
4193                 rte_delay_ms(1);
4194         } while (timeout--);
4195
4196         if (val == 0xffff) {
4197                 PMD_DRV_LOG(ERR, "Firmware reset aborted, PCI config space invalid\n");
4198                 return -1;
4199         }
4200
4201         return 0;
4202 }
4203
4204 static int bnxt_restore_vlan_filters(struct bnxt *bp)
4205 {
4206         struct rte_eth_dev *dev = bp->eth_dev;
4207         struct rte_vlan_filter_conf *vfc;
4208         int vidx, vbit, rc;
4209         uint16_t vlan_id;
4210
4211         for (vlan_id = 1; vlan_id <= RTE_ETHER_MAX_VLAN_ID; vlan_id++) {
4212                 vfc = &dev->data->vlan_filter_conf;
4213                 vidx = vlan_id / 64;
4214                 vbit = vlan_id % 64;
4215
4216                 /* Each bit corresponds to a VLAN id */
4217                 if (vfc->ids[vidx] & (UINT64_C(1) << vbit)) {
4218                         rc = bnxt_add_vlan_filter(bp, vlan_id);
4219                         if (rc)
4220                                 return rc;
4221                 }
4222         }
4223
4224         return 0;
4225 }
4226
4227 static int bnxt_restore_mac_filters(struct bnxt *bp)
4228 {
4229         struct rte_eth_dev *dev = bp->eth_dev;
4230         struct rte_eth_dev_info dev_info;
4231         struct rte_ether_addr *addr;
4232         uint64_t pool_mask;
4233         uint32_t pool = 0;
4234         uint32_t i;
4235         int rc;
4236
4237         if (BNXT_VF(bp) && !BNXT_VF_IS_TRUSTED(bp))
4238                 return 0;
4239
4240         rc = bnxt_dev_info_get_op(dev, &dev_info);
4241         if (rc)
4242                 return rc;
4243
4244         /* replay MAC address configuration */
4245         for (i = 1; i < dev_info.max_mac_addrs; i++) {
4246                 addr = &dev->data->mac_addrs[i];
4247
4248                 /* skip zero address */
4249                 if (rte_is_zero_ether_addr(addr))
4250                         continue;
4251
4252                 pool = 0;
4253                 pool_mask = dev->data->mac_pool_sel[i];
4254
4255                 do {
4256                         if (pool_mask & 1ULL) {
4257                                 rc = bnxt_mac_addr_add_op(dev, addr, i, pool);
4258                                 if (rc)
4259                                         return rc;
4260                         }
4261                         pool_mask >>= 1;
4262                         pool++;
4263                 } while (pool_mask);
4264         }
4265
4266         return 0;
4267 }
4268
4269 static int bnxt_restore_filters(struct bnxt *bp)
4270 {
4271         struct rte_eth_dev *dev = bp->eth_dev;
4272         int ret = 0;
4273
4274         if (dev->data->all_multicast) {
4275                 ret = bnxt_allmulticast_enable_op(dev);
4276                 if (ret)
4277                         return ret;
4278         }
4279         if (dev->data->promiscuous) {
4280                 ret = bnxt_promiscuous_enable_op(dev);
4281                 if (ret)
4282                         return ret;
4283         }
4284
4285         ret = bnxt_restore_mac_filters(bp);
4286         if (ret)
4287                 return ret;
4288
4289         ret = bnxt_restore_vlan_filters(bp);
4290         /* TODO restore other filters as well */
4291         return ret;
4292 }
4293
4294 static int bnxt_check_fw_ready(struct bnxt *bp)
4295 {
4296         int timeout = bp->fw_reset_max_msecs;
4297         int rc = 0;
4298
4299         do {
4300                 rc = bnxt_hwrm_poll_ver_get(bp);
4301                 if (rc == 0)
4302                         break;
4303                 rte_delay_ms(BNXT_FW_READY_WAIT_INTERVAL);
4304                 timeout -= BNXT_FW_READY_WAIT_INTERVAL;
4305         } while (rc && timeout > 0);
4306
4307         if (rc)
4308                 PMD_DRV_LOG(ERR, "FW is not Ready after reset\n");
4309
4310         return rc;
4311 }
4312
4313 static void bnxt_dev_recover(void *arg)
4314 {
4315         struct bnxt *bp = arg;
4316         int rc = 0;
4317
4318         pthread_mutex_lock(&bp->err_recovery_lock);
4319
4320         if (!bp->fw_reset_min_msecs) {
4321                 rc = bnxt_check_fw_reset_done(bp);
4322                 if (rc)
4323                         goto err;
4324         }
4325
4326         /* Clear Error flag so that device re-init should happen */
4327         bp->flags &= ~BNXT_FLAG_FATAL_ERROR;
4328
4329         rc = bnxt_check_fw_ready(bp);
4330         if (rc)
4331                 goto err;
4332
4333         rc = bnxt_init_resources(bp, true);
4334         if (rc) {
4335                 PMD_DRV_LOG(ERR,
4336                             "Failed to initialize resources after reset\n");
4337                 goto err;
4338         }
4339         /* clear reset flag as the device is initialized now */
4340         bp->flags &= ~BNXT_FLAG_FW_RESET;
4341
4342         rc = bnxt_dev_start_op(bp->eth_dev);
4343         if (rc) {
4344                 PMD_DRV_LOG(ERR, "Failed to start port after reset\n");
4345                 goto err_start;
4346         }
4347
4348         rc = bnxt_restore_filters(bp);
4349         if (rc)
4350                 goto err_start;
4351
4352         PMD_DRV_LOG(INFO, "Recovered from FW reset\n");
4353         pthread_mutex_unlock(&bp->err_recovery_lock);
4354
4355         return;
4356 err_start:
4357         bnxt_dev_stop(bp->eth_dev);
4358 err:
4359         bp->flags |= BNXT_FLAG_FATAL_ERROR;
4360         bnxt_uninit_resources(bp, false);
4361         if (bp->eth_dev->data->dev_conf.intr_conf.rmv)
4362                 rte_eth_dev_callback_process(bp->eth_dev,
4363                                              RTE_ETH_EVENT_INTR_RMV,
4364                                              NULL);
4365         pthread_mutex_unlock(&bp->err_recovery_lock);
4366         PMD_DRV_LOG(ERR, "Failed to recover from FW reset\n");
4367 }
4368
4369 void bnxt_dev_reset_and_resume(void *arg)
4370 {
4371         struct bnxt *bp = arg;
4372         uint32_t us = US_PER_MS * bp->fw_reset_min_msecs;
4373         uint16_t val = 0;
4374         int rc;
4375
4376         bnxt_dev_cleanup(bp);
4377
4378         bnxt_wait_for_device_shutdown(bp);
4379
4380         /* During some fatal firmware error conditions, the PCI config space
4381          * register 0x2e which normally contains the subsystem ID will become
4382          * 0xffff. This register will revert back to the normal value after
4383          * the chip has completed core reset. If we detect this condition,
4384          * we can poll this config register immediately for the value to revert.
4385          */
4386         if (bp->flags & BNXT_FLAG_FATAL_ERROR) {
4387                 rc = rte_pci_read_config(bp->pdev, &val, sizeof(val), PCI_SUBSYSTEM_ID_OFFSET);
4388                 if (rc < 0) {
4389                         PMD_DRV_LOG(ERR, "Failed to read PCI offset 0x%x", PCI_SUBSYSTEM_ID_OFFSET);
4390                         return;
4391                 }
4392                 if (val == 0xffff) {
4393                         bp->fw_reset_min_msecs = 0;
4394                         us = 1;
4395                 }
4396         }
4397
4398         rc = rte_eal_alarm_set(us, bnxt_dev_recover, (void *)bp);
4399         if (rc)
4400                 PMD_DRV_LOG(ERR, "Error setting recovery alarm");
4401 }
4402
4403 uint32_t bnxt_read_fw_status_reg(struct bnxt *bp, uint32_t index)
4404 {
4405         struct bnxt_error_recovery_info *info = bp->recovery_info;
4406         uint32_t reg = info->status_regs[index];
4407         uint32_t type, offset, val = 0;
4408         int ret = 0;
4409
4410         type = BNXT_FW_STATUS_REG_TYPE(reg);
4411         offset = BNXT_FW_STATUS_REG_OFF(reg);
4412
4413         switch (type) {
4414         case BNXT_FW_STATUS_REG_TYPE_CFG:
4415                 ret = rte_pci_read_config(bp->pdev, &val, sizeof(val), offset);
4416                 if (ret < 0)
4417                         PMD_DRV_LOG(ERR, "Failed to read PCI offset %#x",
4418                                     offset);
4419                 break;
4420         case BNXT_FW_STATUS_REG_TYPE_GRC:
4421                 offset = info->mapped_status_regs[index];
4422                 /* FALLTHROUGH */
4423         case BNXT_FW_STATUS_REG_TYPE_BAR0:
4424                 val = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
4425                                        offset));
4426                 break;
4427         }
4428
4429         return val;
4430 }
4431
4432 static int bnxt_fw_reset_all(struct bnxt *bp)
4433 {
4434         struct bnxt_error_recovery_info *info = bp->recovery_info;
4435         uint32_t i;
4436         int rc = 0;
4437
4438         if (info->flags & BNXT_FLAG_ERROR_RECOVERY_HOST) {
4439                 /* Reset through master function driver */
4440                 for (i = 0; i < info->reg_array_cnt; i++)
4441                         bnxt_write_fw_reset_reg(bp, i);
4442                 /* Wait for time specified by FW after triggering reset */
4443                 rte_delay_ms(info->master_func_wait_period_after_reset);
4444         } else if (info->flags & BNXT_FLAG_ERROR_RECOVERY_CO_CPU) {
4445                 /* Reset with the help of Kong processor */
4446                 rc = bnxt_hwrm_fw_reset(bp);
4447                 if (rc)
4448                         PMD_DRV_LOG(ERR, "Failed to reset FW\n");
4449         }
4450
4451         return rc;
4452 }
4453
4454 static void bnxt_fw_reset_cb(void *arg)
4455 {
4456         struct bnxt *bp = arg;
4457         struct bnxt_error_recovery_info *info = bp->recovery_info;
4458         int rc = 0;
4459
4460         /* Only Master function can do FW reset */
4461         if (bnxt_is_master_func(bp) &&
4462             bnxt_is_recovery_enabled(bp)) {
4463                 rc = bnxt_fw_reset_all(bp);
4464                 if (rc) {
4465                         PMD_DRV_LOG(ERR, "Adapter recovery failed\n");
4466                         return;
4467                 }
4468         }
4469
4470         /* if recovery method is ERROR_RECOVERY_CO_CPU, KONG will send
4471          * EXCEPTION_FATAL_ASYNC event to all the functions
4472          * (including MASTER FUNC). After receiving this Async, all the active
4473          * drivers should treat this case as FW initiated recovery
4474          */
4475         if (info->flags & BNXT_FLAG_ERROR_RECOVERY_HOST) {
4476                 bp->fw_reset_min_msecs = BNXT_MIN_FW_READY_TIMEOUT;
4477                 bp->fw_reset_max_msecs = BNXT_MAX_FW_RESET_TIMEOUT;
4478
4479                 /* To recover from error */
4480                 rte_eal_alarm_set(US_PER_MS, bnxt_dev_reset_and_resume,
4481                                   (void *)bp);
4482         }
4483 }
4484
4485 /* Driver should poll FW heartbeat, reset_counter with the frequency
4486  * advertised by FW in HWRM_ERROR_RECOVERY_QCFG.
4487  * When the driver detects heartbeat stop or change in reset_counter,
4488  * it has to trigger a reset to recover from the error condition.
4489  * A “master PF” is the function who will have the privilege to
4490  * initiate the chimp reset. The master PF will be elected by the
4491  * firmware and will be notified through async message.
4492  */
4493 static void bnxt_check_fw_health(void *arg)
4494 {
4495         struct bnxt *bp = arg;
4496         struct bnxt_error_recovery_info *info = bp->recovery_info;
4497         uint32_t val = 0, wait_msec;
4498
4499         if (!info || !bnxt_is_recovery_enabled(bp) ||
4500             is_bnxt_in_error(bp))
4501                 return;
4502
4503         val = bnxt_read_fw_status_reg(bp, BNXT_FW_HEARTBEAT_CNT_REG);
4504         if (val == info->last_heart_beat)
4505                 goto reset;
4506
4507         info->last_heart_beat = val;
4508
4509         val = bnxt_read_fw_status_reg(bp, BNXT_FW_RECOVERY_CNT_REG);
4510         if (val != info->last_reset_counter)
4511                 goto reset;
4512
4513         info->last_reset_counter = val;
4514
4515         rte_eal_alarm_set(US_PER_MS * info->driver_polling_freq,
4516                           bnxt_check_fw_health, (void *)bp);
4517
4518         return;
4519 reset:
4520         /* Stop DMA to/from device */
4521         bp->flags |= BNXT_FLAG_FATAL_ERROR;
4522         bp->flags |= BNXT_FLAG_FW_RESET;
4523
4524         bnxt_stop_rxtx(bp);
4525
4526         PMD_DRV_LOG(ERR, "Detected FW dead condition\n");
4527
4528         if (bnxt_is_master_func(bp))
4529                 wait_msec = info->master_func_wait_period;
4530         else
4531                 wait_msec = info->normal_func_wait_period;
4532
4533         rte_eal_alarm_set(US_PER_MS * wait_msec,
4534                           bnxt_fw_reset_cb, (void *)bp);
4535 }
4536
4537 void bnxt_schedule_fw_health_check(struct bnxt *bp)
4538 {
4539         uint32_t polling_freq;
4540
4541         pthread_mutex_lock(&bp->health_check_lock);
4542
4543         if (!bnxt_is_recovery_enabled(bp))
4544                 goto done;
4545
4546         if (bp->flags & BNXT_FLAG_FW_HEALTH_CHECK_SCHEDULED)
4547                 goto done;
4548
4549         polling_freq = bp->recovery_info->driver_polling_freq;
4550
4551         rte_eal_alarm_set(US_PER_MS * polling_freq,
4552                           bnxt_check_fw_health, (void *)bp);
4553         bp->flags |= BNXT_FLAG_FW_HEALTH_CHECK_SCHEDULED;
4554
4555 done:
4556         pthread_mutex_unlock(&bp->health_check_lock);
4557 }
4558
4559 static void bnxt_cancel_fw_health_check(struct bnxt *bp)
4560 {
4561         rte_eal_alarm_cancel(bnxt_check_fw_health, (void *)bp);
4562         bp->flags &= ~BNXT_FLAG_FW_HEALTH_CHECK_SCHEDULED;
4563 }
4564
4565 static bool bnxt_vf_pciid(uint16_t device_id)
4566 {
4567         switch (device_id) {
4568         case BROADCOM_DEV_ID_57304_VF:
4569         case BROADCOM_DEV_ID_57406_VF:
4570         case BROADCOM_DEV_ID_5731X_VF:
4571         case BROADCOM_DEV_ID_5741X_VF:
4572         case BROADCOM_DEV_ID_57414_VF:
4573         case BROADCOM_DEV_ID_STRATUS_NIC_VF1:
4574         case BROADCOM_DEV_ID_STRATUS_NIC_VF2:
4575         case BROADCOM_DEV_ID_58802_VF:
4576         case BROADCOM_DEV_ID_57500_VF1:
4577         case BROADCOM_DEV_ID_57500_VF2:
4578         case BROADCOM_DEV_ID_58818_VF:
4579                 /* FALLTHROUGH */
4580                 return true;
4581         default:
4582                 return false;
4583         }
4584 }
4585
4586 /* Phase 5 device */
4587 static bool bnxt_p5_device(uint16_t device_id)
4588 {
4589         switch (device_id) {
4590         case BROADCOM_DEV_ID_57508:
4591         case BROADCOM_DEV_ID_57504:
4592         case BROADCOM_DEV_ID_57502:
4593         case BROADCOM_DEV_ID_57508_MF1:
4594         case BROADCOM_DEV_ID_57504_MF1:
4595         case BROADCOM_DEV_ID_57502_MF1:
4596         case BROADCOM_DEV_ID_57508_MF2:
4597         case BROADCOM_DEV_ID_57504_MF2:
4598         case BROADCOM_DEV_ID_57502_MF2:
4599         case BROADCOM_DEV_ID_57500_VF1:
4600         case BROADCOM_DEV_ID_57500_VF2:
4601         case BROADCOM_DEV_ID_58812:
4602         case BROADCOM_DEV_ID_58814:
4603         case BROADCOM_DEV_ID_58818:
4604         case BROADCOM_DEV_ID_58818_VF:
4605                 /* FALLTHROUGH */
4606                 return true;
4607         default:
4608                 return false;
4609         }
4610 }
4611
4612 bool bnxt_stratus_device(struct bnxt *bp)
4613 {
4614         uint16_t device_id = bp->pdev->id.device_id;
4615
4616         switch (device_id) {
4617         case BROADCOM_DEV_ID_STRATUS_NIC:
4618         case BROADCOM_DEV_ID_STRATUS_NIC_VF1:
4619         case BROADCOM_DEV_ID_STRATUS_NIC_VF2:
4620                 /* FALLTHROUGH */
4621                 return true;
4622         default:
4623                 return false;
4624         }
4625 }
4626
4627 static int bnxt_map_pci_bars(struct rte_eth_dev *eth_dev)
4628 {
4629         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
4630         struct bnxt *bp = eth_dev->data->dev_private;
4631
4632         /* enable device (incl. PCI PM wakeup), and bus-mastering */
4633         bp->bar0 = (void *)pci_dev->mem_resource[0].addr;
4634         bp->doorbell_base = (void *)pci_dev->mem_resource[2].addr;
4635         if (!bp->bar0 || !bp->doorbell_base) {
4636                 PMD_DRV_LOG(ERR, "Unable to access Hardware\n");
4637                 return -ENODEV;
4638         }
4639
4640         bp->eth_dev = eth_dev;
4641         bp->pdev = pci_dev;
4642
4643         return 0;
4644 }
4645
4646 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
4647                                   struct bnxt_ctx_pg_info *ctx_pg,
4648                                   uint32_t mem_size,
4649                                   const char *suffix,
4650                                   uint16_t idx)
4651 {
4652         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
4653         const struct rte_memzone *mz = NULL;
4654         char mz_name[RTE_MEMZONE_NAMESIZE];
4655         rte_iova_t mz_phys_addr;
4656         uint64_t valid_bits = 0;
4657         uint32_t sz;
4658         int i;
4659
4660         if (!mem_size)
4661                 return 0;
4662
4663         rmem->nr_pages = RTE_ALIGN_MUL_CEIL(mem_size, BNXT_PAGE_SIZE) /
4664                          BNXT_PAGE_SIZE;
4665         rmem->page_size = BNXT_PAGE_SIZE;
4666         rmem->pg_arr = ctx_pg->ctx_pg_arr;
4667         rmem->dma_arr = ctx_pg->ctx_dma_arr;
4668         rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
4669
4670         valid_bits = PTU_PTE_VALID;
4671
4672         if (rmem->nr_pages > 1) {
4673                 snprintf(mz_name, RTE_MEMZONE_NAMESIZE,
4674                          "bnxt_ctx_pg_tbl%s_%x_%d",
4675                          suffix, idx, bp->eth_dev->data->port_id);
4676                 mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0;
4677                 mz = rte_memzone_lookup(mz_name);
4678                 if (!mz) {
4679                         mz = rte_memzone_reserve_aligned(mz_name,
4680                                                 rmem->nr_pages * 8,
4681                                                 bp->eth_dev->device->numa_node,
4682                                                 RTE_MEMZONE_2MB |
4683                                                 RTE_MEMZONE_SIZE_HINT_ONLY |
4684                                                 RTE_MEMZONE_IOVA_CONTIG,
4685                                                 BNXT_PAGE_SIZE);
4686                         if (mz == NULL)
4687                                 return -ENOMEM;
4688                 }
4689
4690                 memset(mz->addr, 0, mz->len);
4691                 mz_phys_addr = mz->iova;
4692
4693                 rmem->pg_tbl = mz->addr;
4694                 rmem->pg_tbl_map = mz_phys_addr;
4695                 rmem->pg_tbl_mz = mz;
4696         }
4697
4698         snprintf(mz_name, RTE_MEMZONE_NAMESIZE, "bnxt_ctx_%s_%x_%d",
4699                  suffix, idx, bp->eth_dev->data->port_id);
4700         mz = rte_memzone_lookup(mz_name);
4701         if (!mz) {
4702                 mz = rte_memzone_reserve_aligned(mz_name,
4703                                                  mem_size,
4704                                                  bp->eth_dev->device->numa_node,
4705                                                  RTE_MEMZONE_1GB |
4706                                                  RTE_MEMZONE_SIZE_HINT_ONLY |
4707                                                  RTE_MEMZONE_IOVA_CONTIG,
4708                                                  BNXT_PAGE_SIZE);
4709                 if (mz == NULL)
4710                         return -ENOMEM;
4711         }
4712
4713         memset(mz->addr, 0, mz->len);
4714         mz_phys_addr = mz->iova;
4715
4716         for (sz = 0, i = 0; sz < mem_size; sz += BNXT_PAGE_SIZE, i++) {
4717                 rmem->pg_arr[i] = ((char *)mz->addr) + sz;
4718                 rmem->dma_arr[i] = mz_phys_addr + sz;
4719
4720                 if (rmem->nr_pages > 1) {
4721                         if (i == rmem->nr_pages - 2 &&
4722                             (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
4723                                 valid_bits |= PTU_PTE_NEXT_TO_LAST;
4724                         else if (i == rmem->nr_pages - 1 &&
4725                                  (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
4726                                 valid_bits |= PTU_PTE_LAST;
4727
4728                         rmem->pg_tbl[i] = rte_cpu_to_le_64(rmem->dma_arr[i] |
4729                                                            valid_bits);
4730                 }
4731         }
4732
4733         rmem->mz = mz;
4734         if (rmem->vmem_size)
4735                 rmem->vmem = (void **)mz->addr;
4736         rmem->dma_arr[0] = mz_phys_addr;
4737         return 0;
4738 }
4739
4740 static void bnxt_free_ctx_mem(struct bnxt *bp)
4741 {
4742         int i;
4743
4744         if (!bp->ctx || !(bp->ctx->flags & BNXT_CTX_FLAG_INITED))
4745                 return;
4746
4747         bp->ctx->flags &= ~BNXT_CTX_FLAG_INITED;
4748         rte_memzone_free(bp->ctx->qp_mem.ring_mem.mz);
4749         rte_memzone_free(bp->ctx->srq_mem.ring_mem.mz);
4750         rte_memzone_free(bp->ctx->cq_mem.ring_mem.mz);
4751         rte_memzone_free(bp->ctx->vnic_mem.ring_mem.mz);
4752         rte_memzone_free(bp->ctx->stat_mem.ring_mem.mz);
4753         rte_memzone_free(bp->ctx->qp_mem.ring_mem.pg_tbl_mz);
4754         rte_memzone_free(bp->ctx->srq_mem.ring_mem.pg_tbl_mz);
4755         rte_memzone_free(bp->ctx->cq_mem.ring_mem.pg_tbl_mz);
4756         rte_memzone_free(bp->ctx->vnic_mem.ring_mem.pg_tbl_mz);
4757         rte_memzone_free(bp->ctx->stat_mem.ring_mem.pg_tbl_mz);
4758
4759         for (i = 0; i < bp->ctx->tqm_fp_rings_count + 1; i++) {
4760                 if (bp->ctx->tqm_mem[i])
4761                         rte_memzone_free(bp->ctx->tqm_mem[i]->ring_mem.mz);
4762         }
4763
4764         rte_free(bp->ctx);
4765         bp->ctx = NULL;
4766 }
4767
4768 #define bnxt_roundup(x, y)   ((((x) + ((y) - 1)) / (y)) * (y))
4769
4770 #define min_t(type, x, y) ({                    \
4771         type __min1 = (x);                      \
4772         type __min2 = (y);                      \
4773         __min1 < __min2 ? __min1 : __min2; })
4774
4775 #define max_t(type, x, y) ({                    \
4776         type __max1 = (x);                      \
4777         type __max2 = (y);                      \
4778         __max1 > __max2 ? __max1 : __max2; })
4779
4780 #define clamp_t(type, _x, min, max)     min_t(type, max_t(type, _x, min), max)
4781
4782 int bnxt_alloc_ctx_mem(struct bnxt *bp)
4783 {
4784         struct bnxt_ctx_pg_info *ctx_pg;
4785         struct bnxt_ctx_mem_info *ctx;
4786         uint32_t mem_size, ena, entries;
4787         uint32_t entries_sp, min;
4788         int i, rc;
4789
4790         rc = bnxt_hwrm_func_backing_store_qcaps(bp);
4791         if (rc) {
4792                 PMD_DRV_LOG(ERR, "Query context mem capability failed\n");
4793                 return rc;
4794         }
4795         ctx = bp->ctx;
4796         if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
4797                 return 0;
4798
4799         ctx_pg = &ctx->qp_mem;
4800         ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries;
4801         if (ctx->qp_entry_size) {
4802                 mem_size = ctx->qp_entry_size * ctx_pg->entries;
4803                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size, "qp_mem", 0);
4804                 if (rc)
4805                         return rc;
4806         }
4807
4808         ctx_pg = &ctx->srq_mem;
4809         ctx_pg->entries = ctx->srq_max_l2_entries;
4810         if (ctx->srq_entry_size) {
4811                 mem_size = ctx->srq_entry_size * ctx_pg->entries;
4812                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size, "srq_mem", 0);
4813                 if (rc)
4814                         return rc;
4815         }
4816
4817         ctx_pg = &ctx->cq_mem;
4818         ctx_pg->entries = ctx->cq_max_l2_entries;
4819         if (ctx->cq_entry_size) {
4820                 mem_size = ctx->cq_entry_size * ctx_pg->entries;
4821                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size, "cq_mem", 0);
4822                 if (rc)
4823                         return rc;
4824         }
4825
4826         ctx_pg = &ctx->vnic_mem;
4827         ctx_pg->entries = ctx->vnic_max_vnic_entries +
4828                 ctx->vnic_max_ring_table_entries;
4829         if (ctx->vnic_entry_size) {
4830                 mem_size = ctx->vnic_entry_size * ctx_pg->entries;
4831                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size, "vnic_mem", 0);
4832                 if (rc)
4833                         return rc;
4834         }
4835
4836         ctx_pg = &ctx->stat_mem;
4837         ctx_pg->entries = ctx->stat_max_entries;
4838         if (ctx->stat_entry_size) {
4839                 mem_size = ctx->stat_entry_size * ctx_pg->entries;
4840                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size, "stat_mem", 0);
4841                 if (rc)
4842                         return rc;
4843         }
4844
4845         min = ctx->tqm_min_entries_per_ring;
4846
4847         entries_sp = ctx->qp_max_l2_entries +
4848                      ctx->vnic_max_vnic_entries +
4849                      2 * ctx->qp_min_qp1_entries + min;
4850         entries_sp = bnxt_roundup(entries_sp, ctx->tqm_entries_multiple);
4851
4852         entries = ctx->qp_max_l2_entries + ctx->qp_min_qp1_entries;
4853         entries = bnxt_roundup(entries, ctx->tqm_entries_multiple);
4854         entries = clamp_t(uint32_t, entries, min,
4855                           ctx->tqm_max_entries_per_ring);
4856         for (i = 0, ena = 0; i < ctx->tqm_fp_rings_count + 1; i++) {
4857                 /* i=0 is for TQM_SP. i=1 to i=8 applies to RING0 to RING7.
4858                  * i > 8 is other ext rings.
4859                  */
4860                 ctx_pg = ctx->tqm_mem[i];
4861                 ctx_pg->entries = i ? entries : entries_sp;
4862                 if (ctx->tqm_entry_size) {
4863                         mem_size = ctx->tqm_entry_size * ctx_pg->entries;
4864                         rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size,
4865                                                     "tqm_mem", i);
4866                         if (rc)
4867                                 return rc;
4868                 }
4869                 if (i < BNXT_MAX_TQM_LEGACY_RINGS)
4870                         ena |= HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_TQM_SP << i;
4871                 else
4872                         ena |= HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_TQM_RING8;
4873         }
4874
4875         ena |= FUNC_BACKING_STORE_CFG_INPUT_DFLT_ENABLES;
4876         rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
4877         if (rc)
4878                 PMD_DRV_LOG(ERR,
4879                             "Failed to configure context mem: rc = %d\n", rc);
4880         else
4881                 ctx->flags |= BNXT_CTX_FLAG_INITED;
4882
4883         return rc;
4884 }
4885
4886 static int bnxt_alloc_stats_mem(struct bnxt *bp)
4887 {
4888         struct rte_pci_device *pci_dev = bp->pdev;
4889         char mz_name[RTE_MEMZONE_NAMESIZE];
4890         const struct rte_memzone *mz = NULL;
4891         uint32_t total_alloc_len;
4892         rte_iova_t mz_phys_addr;
4893
4894         if (pci_dev->id.device_id == BROADCOM_DEV_ID_NS2)
4895                 return 0;
4896
4897         snprintf(mz_name, RTE_MEMZONE_NAMESIZE,
4898                  "bnxt_" PCI_PRI_FMT "-%s", pci_dev->addr.domain,
4899                  pci_dev->addr.bus, pci_dev->addr.devid,
4900                  pci_dev->addr.function, "rx_port_stats");
4901         mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0;
4902         mz = rte_memzone_lookup(mz_name);
4903         total_alloc_len =
4904                 RTE_CACHE_LINE_ROUNDUP(sizeof(struct rx_port_stats) +
4905                                        sizeof(struct rx_port_stats_ext) + 512);
4906         if (!mz) {
4907                 mz = rte_memzone_reserve(mz_name, total_alloc_len,
4908                                          SOCKET_ID_ANY,
4909                                          RTE_MEMZONE_2MB |
4910                                          RTE_MEMZONE_SIZE_HINT_ONLY |
4911                                          RTE_MEMZONE_IOVA_CONTIG);
4912                 if (mz == NULL)
4913                         return -ENOMEM;
4914         }
4915         memset(mz->addr, 0, mz->len);
4916         mz_phys_addr = mz->iova;
4917
4918         bp->rx_mem_zone = (const void *)mz;
4919         bp->hw_rx_port_stats = mz->addr;
4920         bp->hw_rx_port_stats_map = mz_phys_addr;
4921
4922         snprintf(mz_name, RTE_MEMZONE_NAMESIZE,
4923                  "bnxt_" PCI_PRI_FMT "-%s", pci_dev->addr.domain,
4924                  pci_dev->addr.bus, pci_dev->addr.devid,
4925                  pci_dev->addr.function, "tx_port_stats");
4926         mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0;
4927         mz = rte_memzone_lookup(mz_name);
4928         total_alloc_len =
4929                 RTE_CACHE_LINE_ROUNDUP(sizeof(struct tx_port_stats) +
4930                                        sizeof(struct tx_port_stats_ext) + 512);
4931         if (!mz) {
4932                 mz = rte_memzone_reserve(mz_name,
4933                                          total_alloc_len,
4934                                          SOCKET_ID_ANY,
4935                                          RTE_MEMZONE_2MB |
4936                                          RTE_MEMZONE_SIZE_HINT_ONLY |
4937                                          RTE_MEMZONE_IOVA_CONTIG);
4938                 if (mz == NULL)
4939                         return -ENOMEM;
4940         }
4941         memset(mz->addr, 0, mz->len);
4942         mz_phys_addr = mz->iova;
4943
4944         bp->tx_mem_zone = (const void *)mz;
4945         bp->hw_tx_port_stats = mz->addr;
4946         bp->hw_tx_port_stats_map = mz_phys_addr;
4947         bp->flags |= BNXT_FLAG_PORT_STATS;
4948
4949         /* Display extended statistics if FW supports it */
4950         if (bp->hwrm_spec_code < HWRM_SPEC_CODE_1_8_4 ||
4951             bp->hwrm_spec_code == HWRM_SPEC_CODE_1_9_0 ||
4952             !(bp->flags & BNXT_FLAG_EXT_STATS_SUPPORTED))
4953                 return 0;
4954
4955         bp->hw_rx_port_stats_ext = (void *)
4956                 ((uint8_t *)bp->hw_rx_port_stats +
4957                  sizeof(struct rx_port_stats));
4958         bp->hw_rx_port_stats_ext_map = bp->hw_rx_port_stats_map +
4959                 sizeof(struct rx_port_stats);
4960         bp->flags |= BNXT_FLAG_EXT_RX_PORT_STATS;
4961
4962         if (bp->hwrm_spec_code < HWRM_SPEC_CODE_1_9_2 ||
4963             bp->flags & BNXT_FLAG_EXT_STATS_SUPPORTED) {
4964                 bp->hw_tx_port_stats_ext = (void *)
4965                         ((uint8_t *)bp->hw_tx_port_stats +
4966                          sizeof(struct tx_port_stats));
4967                 bp->hw_tx_port_stats_ext_map =
4968                         bp->hw_tx_port_stats_map +
4969                         sizeof(struct tx_port_stats);
4970                 bp->flags |= BNXT_FLAG_EXT_TX_PORT_STATS;
4971         }
4972
4973         return 0;
4974 }
4975
4976 static int bnxt_setup_mac_addr(struct rte_eth_dev *eth_dev)
4977 {
4978         struct bnxt *bp = eth_dev->data->dev_private;
4979         int rc = 0;
4980
4981         eth_dev->data->mac_addrs = rte_zmalloc("bnxt_mac_addr_tbl",
4982                                                RTE_ETHER_ADDR_LEN *
4983                                                bp->max_l2_ctx,
4984                                                0);
4985         if (eth_dev->data->mac_addrs == NULL) {
4986                 PMD_DRV_LOG(ERR, "Failed to alloc MAC addr tbl\n");
4987                 return -ENOMEM;
4988         }
4989
4990         if (!BNXT_HAS_DFLT_MAC_SET(bp)) {
4991                 if (BNXT_PF(bp))
4992                         return -EINVAL;
4993
4994                 /* Generate a random MAC address, if none was assigned by PF */
4995                 PMD_DRV_LOG(INFO, "VF MAC address not assigned by Host PF\n");
4996                 bnxt_eth_hw_addr_random(bp->mac_addr);
4997                 PMD_DRV_LOG(INFO,
4998                             "Assign random MAC:%02X:%02X:%02X:%02X:%02X:%02X\n",
4999                             bp->mac_addr[0], bp->mac_addr[1], bp->mac_addr[2],
5000                             bp->mac_addr[3], bp->mac_addr[4], bp->mac_addr[5]);
5001
5002                 rc = bnxt_hwrm_set_mac(bp);
5003                 if (rc)
5004                         return rc;
5005         }
5006
5007         /* Copy the permanent MAC from the FUNC_QCAPS response */
5008         memcpy(&eth_dev->data->mac_addrs[0], bp->mac_addr, RTE_ETHER_ADDR_LEN);
5009
5010         return rc;
5011 }
5012
5013 static int bnxt_restore_dflt_mac(struct bnxt *bp)
5014 {
5015         int rc = 0;
5016
5017         /* MAC is already configured in FW */
5018         if (BNXT_HAS_DFLT_MAC_SET(bp))
5019                 return 0;
5020
5021         /* Restore the old MAC configured */
5022         rc = bnxt_hwrm_set_mac(bp);
5023         if (rc)
5024                 PMD_DRV_LOG(ERR, "Failed to restore MAC address\n");
5025
5026         return rc;
5027 }
5028
5029 static void bnxt_config_vf_req_fwd(struct bnxt *bp)
5030 {
5031         if (!BNXT_PF(bp))
5032                 return;
5033
5034         memset(bp->pf->vf_req_fwd, 0, sizeof(bp->pf->vf_req_fwd));
5035
5036         if (!(bp->fw_cap & BNXT_FW_CAP_LINK_ADMIN))
5037                 BNXT_HWRM_CMD_TO_FORWARD(HWRM_PORT_PHY_QCFG);
5038         BNXT_HWRM_CMD_TO_FORWARD(HWRM_FUNC_CFG);
5039         BNXT_HWRM_CMD_TO_FORWARD(HWRM_FUNC_VF_CFG);
5040         BNXT_HWRM_CMD_TO_FORWARD(HWRM_CFA_L2_FILTER_ALLOC);
5041         BNXT_HWRM_CMD_TO_FORWARD(HWRM_OEM_CMD);
5042 }
5043
5044 struct bnxt *
5045 bnxt_get_bp(uint16_t port)
5046 {
5047         struct bnxt *bp;
5048         struct rte_eth_dev *dev;
5049
5050         if (!rte_eth_dev_is_valid_port(port)) {
5051                 PMD_DRV_LOG(ERR, "Invalid port %d\n", port);
5052                 return NULL;
5053         }
5054
5055         dev = &rte_eth_devices[port];
5056         if (!is_bnxt_supported(dev)) {
5057                 PMD_DRV_LOG(ERR, "Device %d not supported\n", port);
5058                 return NULL;
5059         }
5060
5061         bp = (struct bnxt *)dev->data->dev_private;
5062         if (!BNXT_TRUFLOW_EN(bp)) {
5063                 PMD_DRV_LOG(ERR, "TRUFLOW not enabled\n");
5064                 return NULL;
5065         }
5066
5067         return bp;
5068 }
5069
5070 uint16_t
5071 bnxt_get_svif(uint16_t port_id, bool func_svif,
5072               enum bnxt_ulp_intf_type type)
5073 {
5074         struct rte_eth_dev *eth_dev;
5075         struct bnxt *bp;
5076
5077         eth_dev = &rte_eth_devices[port_id];
5078         if (BNXT_ETH_DEV_IS_REPRESENTOR(eth_dev)) {
5079                 struct bnxt_representor *vfr = eth_dev->data->dev_private;
5080                 if (!vfr)
5081                         return 0;
5082
5083                 if (type == BNXT_ULP_INTF_TYPE_VF_REP)
5084                         return vfr->svif;
5085
5086                 eth_dev = vfr->parent_dev;
5087         }
5088
5089         bp = eth_dev->data->dev_private;
5090
5091         return func_svif ? bp->func_svif : bp->port_svif;
5092 }
5093
5094 void
5095 bnxt_get_iface_mac(uint16_t port, enum bnxt_ulp_intf_type type,
5096                    uint8_t *mac, uint8_t *parent_mac)
5097 {
5098         struct rte_eth_dev *eth_dev;
5099         struct bnxt *bp;
5100
5101         if (type != BNXT_ULP_INTF_TYPE_TRUSTED_VF &&
5102             type != BNXT_ULP_INTF_TYPE_PF)
5103                 return;
5104
5105         eth_dev = &rte_eth_devices[port];
5106         bp = eth_dev->data->dev_private;
5107         memcpy(mac, bp->mac_addr, RTE_ETHER_ADDR_LEN);
5108
5109         if (type == BNXT_ULP_INTF_TYPE_TRUSTED_VF)
5110                 memcpy(parent_mac, bp->parent->mac_addr, RTE_ETHER_ADDR_LEN);
5111 }
5112
5113 uint16_t
5114 bnxt_get_parent_vnic_id(uint16_t port, enum bnxt_ulp_intf_type type)
5115 {
5116         struct rte_eth_dev *eth_dev;
5117         struct bnxt *bp;
5118
5119         if (type != BNXT_ULP_INTF_TYPE_TRUSTED_VF)
5120                 return 0;
5121
5122         eth_dev = &rte_eth_devices[port];
5123         bp = eth_dev->data->dev_private;
5124
5125         return bp->parent->vnic;
5126 }
5127 uint16_t
5128 bnxt_get_vnic_id(uint16_t port, enum bnxt_ulp_intf_type type)
5129 {
5130         struct rte_eth_dev *eth_dev;
5131         struct bnxt_vnic_info *vnic;
5132         struct bnxt *bp;
5133
5134         eth_dev = &rte_eth_devices[port];
5135         if (BNXT_ETH_DEV_IS_REPRESENTOR(eth_dev)) {
5136                 struct bnxt_representor *vfr = eth_dev->data->dev_private;
5137                 if (!vfr)
5138                         return 0;
5139
5140                 if (type == BNXT_ULP_INTF_TYPE_VF_REP)
5141                         return vfr->dflt_vnic_id;
5142
5143                 eth_dev = vfr->parent_dev;
5144         }
5145
5146         bp = eth_dev->data->dev_private;
5147
5148         vnic = BNXT_GET_DEFAULT_VNIC(bp);
5149
5150         return vnic->fw_vnic_id;
5151 }
5152
5153 uint16_t
5154 bnxt_get_fw_func_id(uint16_t port, enum bnxt_ulp_intf_type type)
5155 {
5156         struct rte_eth_dev *eth_dev;
5157         struct bnxt *bp;
5158
5159         eth_dev = &rte_eth_devices[port];
5160         if (BNXT_ETH_DEV_IS_REPRESENTOR(eth_dev)) {
5161                 struct bnxt_representor *vfr = eth_dev->data->dev_private;
5162                 if (!vfr)
5163                         return 0;
5164
5165                 if (type == BNXT_ULP_INTF_TYPE_VF_REP)
5166                         return vfr->fw_fid;
5167
5168                 eth_dev = vfr->parent_dev;
5169         }
5170
5171         bp = eth_dev->data->dev_private;
5172
5173         return bp->fw_fid;
5174 }
5175
5176 enum bnxt_ulp_intf_type
5177 bnxt_get_interface_type(uint16_t port)
5178 {
5179         struct rte_eth_dev *eth_dev;
5180         struct bnxt *bp;
5181
5182         eth_dev = &rte_eth_devices[port];
5183         if (BNXT_ETH_DEV_IS_REPRESENTOR(eth_dev))
5184                 return BNXT_ULP_INTF_TYPE_VF_REP;
5185
5186         bp = eth_dev->data->dev_private;
5187         if (BNXT_PF(bp))
5188                 return BNXT_ULP_INTF_TYPE_PF;
5189         else if (BNXT_VF_IS_TRUSTED(bp))
5190                 return BNXT_ULP_INTF_TYPE_TRUSTED_VF;
5191         else if (BNXT_VF(bp))
5192                 return BNXT_ULP_INTF_TYPE_VF;
5193
5194         return BNXT_ULP_INTF_TYPE_INVALID;
5195 }
5196
5197 uint16_t
5198 bnxt_get_phy_port_id(uint16_t port_id)
5199 {
5200         struct bnxt_representor *vfr;
5201         struct rte_eth_dev *eth_dev;
5202         struct bnxt *bp;
5203
5204         eth_dev = &rte_eth_devices[port_id];
5205         if (BNXT_ETH_DEV_IS_REPRESENTOR(eth_dev)) {
5206                 vfr = eth_dev->data->dev_private;
5207                 if (!vfr)
5208                         return 0;
5209
5210                 eth_dev = vfr->parent_dev;
5211         }
5212
5213         bp = eth_dev->data->dev_private;
5214
5215         return BNXT_PF(bp) ? bp->pf->port_id : bp->parent->port_id;
5216 }
5217
5218 uint16_t
5219 bnxt_get_parif(uint16_t port_id, enum bnxt_ulp_intf_type type)
5220 {
5221         struct rte_eth_dev *eth_dev;
5222         struct bnxt *bp;
5223
5224         eth_dev = &rte_eth_devices[port_id];
5225         if (BNXT_ETH_DEV_IS_REPRESENTOR(eth_dev)) {
5226                 struct bnxt_representor *vfr = eth_dev->data->dev_private;
5227                 if (!vfr)
5228                         return 0;
5229
5230                 if (type == BNXT_ULP_INTF_TYPE_VF_REP)
5231                         return vfr->fw_fid - 1;
5232
5233                 eth_dev = vfr->parent_dev;
5234         }
5235
5236         bp = eth_dev->data->dev_private;
5237
5238         return BNXT_PF(bp) ? bp->fw_fid - 1 : bp->parent->fid - 1;
5239 }
5240
5241 uint16_t
5242 bnxt_get_vport(uint16_t port_id)
5243 {
5244         return (1 << bnxt_get_phy_port_id(port_id));
5245 }
5246
5247 static void bnxt_alloc_error_recovery_info(struct bnxt *bp)
5248 {
5249         struct bnxt_error_recovery_info *info = bp->recovery_info;
5250
5251         if (info) {
5252                 if (!(bp->fw_cap & BNXT_FW_CAP_HCOMM_FW_STATUS))
5253                         memset(info, 0, sizeof(*info));
5254                 return;
5255         }
5256
5257         if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
5258                 return;
5259
5260         info = rte_zmalloc("bnxt_hwrm_error_recovery_qcfg",
5261                            sizeof(*info), 0);
5262         if (!info)
5263                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
5264
5265         bp->recovery_info = info;
5266 }
5267
5268 static void bnxt_check_fw_status(struct bnxt *bp)
5269 {
5270         uint32_t fw_status;
5271
5272         if (!(bp->recovery_info &&
5273               (bp->fw_cap & BNXT_FW_CAP_HCOMM_FW_STATUS)))
5274                 return;
5275
5276         fw_status = bnxt_read_fw_status_reg(bp, BNXT_FW_STATUS_REG);
5277         if (fw_status != BNXT_FW_STATUS_HEALTHY)
5278                 PMD_DRV_LOG(ERR, "Firmware not responding, status: %#x\n",
5279                             fw_status);
5280 }
5281
5282 static int bnxt_map_hcomm_fw_status_reg(struct bnxt *bp)
5283 {
5284         struct bnxt_error_recovery_info *info = bp->recovery_info;
5285         uint32_t status_loc;
5286         uint32_t sig_ver;
5287
5288         rte_write32(HCOMM_STATUS_STRUCT_LOC, (uint8_t *)bp->bar0 +
5289                     BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
5290         sig_ver = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
5291                                    BNXT_GRCP_WINDOW_2_BASE +
5292                                    offsetof(struct hcomm_status,
5293                                             sig_ver)));
5294         /* If the signature is absent, then FW does not support this feature */
5295         if ((sig_ver & HCOMM_STATUS_SIGNATURE_MASK) !=
5296             HCOMM_STATUS_SIGNATURE_VAL)
5297                 return 0;
5298
5299         if (!info) {
5300                 info = rte_zmalloc("bnxt_hwrm_error_recovery_qcfg",
5301                                    sizeof(*info), 0);
5302                 if (!info)
5303                         return -ENOMEM;
5304                 bp->recovery_info = info;
5305         } else {
5306                 memset(info, 0, sizeof(*info));
5307         }
5308
5309         status_loc = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
5310                                       BNXT_GRCP_WINDOW_2_BASE +
5311                                       offsetof(struct hcomm_status,
5312                                                fw_status_loc)));
5313
5314         /* Only pre-map the FW health status GRC register */
5315         if (BNXT_FW_STATUS_REG_TYPE(status_loc) != BNXT_FW_STATUS_REG_TYPE_GRC)
5316                 return 0;
5317
5318         info->status_regs[BNXT_FW_STATUS_REG] = status_loc;
5319         info->mapped_status_regs[BNXT_FW_STATUS_REG] =
5320                 BNXT_GRCP_WINDOW_2_BASE + (status_loc & BNXT_GRCP_OFFSET_MASK);
5321
5322         rte_write32((status_loc & BNXT_GRCP_BASE_MASK), (uint8_t *)bp->bar0 +
5323                     BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
5324
5325         bp->fw_cap |= BNXT_FW_CAP_HCOMM_FW_STATUS;
5326
5327         return 0;
5328 }
5329
5330 /* This function gets the FW version along with the
5331  * capabilities(MAX and current) of the function, vnic,
5332  * error recovery, phy and other chip related info
5333  */
5334 static int bnxt_get_config(struct bnxt *bp)
5335 {
5336         uint16_t mtu;
5337         int rc = 0;
5338
5339         bp->fw_cap = 0;
5340
5341         rc = bnxt_map_hcomm_fw_status_reg(bp);
5342         if (rc)
5343                 return rc;
5344
5345         rc = bnxt_hwrm_ver_get(bp, DFLT_HWRM_CMD_TIMEOUT);
5346         if (rc) {
5347                 bnxt_check_fw_status(bp);
5348                 return rc;
5349         }
5350
5351         rc = bnxt_hwrm_func_reset(bp);
5352         if (rc)
5353                 return -EIO;
5354
5355         rc = bnxt_hwrm_vnic_qcaps(bp);
5356         if (rc)
5357                 return rc;
5358
5359         rc = bnxt_hwrm_queue_qportcfg(bp);
5360         if (rc)
5361                 return rc;
5362
5363         /* Get the MAX capabilities for this function.
5364          * This function also allocates context memory for TQM rings and
5365          * informs the firmware about this allocated backing store memory.
5366          */
5367         rc = bnxt_hwrm_func_qcaps(bp);
5368         if (rc)
5369                 return rc;
5370
5371         rc = bnxt_hwrm_func_qcfg(bp, &mtu);
5372         if (rc)
5373                 return rc;
5374
5375         rc = bnxt_hwrm_cfa_adv_flow_mgmt_qcaps(bp);
5376         if (rc)
5377                 return rc;
5378
5379         bnxt_hwrm_port_mac_qcfg(bp);
5380
5381         bnxt_hwrm_parent_pf_qcfg(bp);
5382
5383         bnxt_hwrm_port_phy_qcaps(bp);
5384
5385         bnxt_alloc_error_recovery_info(bp);
5386         /* Get the adapter error recovery support info */
5387         rc = bnxt_hwrm_error_recovery_qcfg(bp);
5388         if (rc)
5389                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
5390
5391         bnxt_hwrm_port_led_qcaps(bp);
5392
5393         return 0;
5394 }
5395
5396 static int
5397 bnxt_init_locks(struct bnxt *bp)
5398 {
5399         int err;
5400
5401         err = pthread_mutex_init(&bp->flow_lock, NULL);
5402         if (err) {
5403                 PMD_DRV_LOG(ERR, "Unable to initialize flow_lock\n");
5404                 return err;
5405         }
5406
5407         err = pthread_mutex_init(&bp->def_cp_lock, NULL);
5408         if (err) {
5409                 PMD_DRV_LOG(ERR, "Unable to initialize def_cp_lock\n");
5410                 return err;
5411         }
5412
5413         err = pthread_mutex_init(&bp->health_check_lock, NULL);
5414         if (err) {
5415                 PMD_DRV_LOG(ERR, "Unable to initialize health_check_lock\n");
5416                 return err;
5417         }
5418
5419         err = pthread_mutex_init(&bp->err_recovery_lock, NULL);
5420         if (err)
5421                 PMD_DRV_LOG(ERR, "Unable to initialize err_recovery_lock\n");
5422
5423         return err;
5424 }
5425
5426 static int bnxt_init_resources(struct bnxt *bp, bool reconfig_dev)
5427 {
5428         int rc = 0;
5429
5430         rc = bnxt_get_config(bp);
5431         if (rc)
5432                 return rc;
5433
5434         if (!reconfig_dev) {
5435                 rc = bnxt_setup_mac_addr(bp->eth_dev);
5436                 if (rc)
5437                         return rc;
5438         } else {
5439                 rc = bnxt_restore_dflt_mac(bp);
5440                 if (rc)
5441                         return rc;
5442         }
5443
5444         bnxt_config_vf_req_fwd(bp);
5445
5446         rc = bnxt_hwrm_func_driver_register(bp);
5447         if (rc) {
5448                 PMD_DRV_LOG(ERR, "Failed to register driver");
5449                 return -EBUSY;
5450         }
5451
5452         if (BNXT_PF(bp)) {
5453                 if (bp->pdev->max_vfs) {
5454                         rc = bnxt_hwrm_allocate_vfs(bp, bp->pdev->max_vfs);
5455                         if (rc) {
5456                                 PMD_DRV_LOG(ERR, "Failed to allocate VFs\n");
5457                                 return rc;
5458                         }
5459                 } else {
5460                         rc = bnxt_hwrm_allocate_pf_only(bp);
5461                         if (rc) {
5462                                 PMD_DRV_LOG(ERR,
5463                                             "Failed to allocate PF resources");
5464                                 return rc;
5465                         }
5466                 }
5467         }
5468
5469         rc = bnxt_alloc_mem(bp, reconfig_dev);
5470         if (rc)
5471                 return rc;
5472
5473         rc = bnxt_setup_int(bp);
5474         if (rc)
5475                 return rc;
5476
5477         rc = bnxt_request_int(bp);
5478         if (rc)
5479                 return rc;
5480
5481         rc = bnxt_init_ctx_mem(bp);
5482         if (rc) {
5483                 PMD_DRV_LOG(ERR, "Failed to init adv_flow_counters\n");
5484                 return rc;
5485         }
5486
5487         return 0;
5488 }
5489
5490 static int
5491 bnxt_parse_devarg_accum_stats(__rte_unused const char *key,
5492                               const char *value, void *opaque_arg)
5493 {
5494         struct bnxt *bp = opaque_arg;
5495         unsigned long accum_stats;
5496         char *end = NULL;
5497
5498         if (!value || !opaque_arg) {
5499                 PMD_DRV_LOG(ERR,
5500                             "Invalid parameter passed to accum-stats devargs.\n");
5501                 return -EINVAL;
5502         }
5503
5504         accum_stats = strtoul(value, &end, 10);
5505         if (end == NULL || *end != '\0' ||
5506             (accum_stats == ULONG_MAX && errno == ERANGE)) {
5507                 PMD_DRV_LOG(ERR,
5508                             "Invalid parameter passed to accum-stats devargs.\n");
5509                 return -EINVAL;
5510         }
5511
5512         if (BNXT_DEVARG_ACCUM_STATS_INVALID(accum_stats)) {
5513                 PMD_DRV_LOG(ERR,
5514                             "Invalid value passed to accum-stats devargs.\n");
5515                 return -EINVAL;
5516         }
5517
5518         if (accum_stats) {
5519                 bp->flags2 |= BNXT_FLAGS2_ACCUM_STATS_EN;
5520                 PMD_DRV_LOG(INFO, "Host-based accum-stats feature enabled.\n");
5521         } else {
5522                 bp->flags2 &= ~BNXT_FLAGS2_ACCUM_STATS_EN;
5523                 PMD_DRV_LOG(INFO, "Host-based accum-stats feature disabled.\n");
5524         }
5525
5526         return 0;
5527 }
5528
5529 static int
5530 bnxt_parse_devarg_flow_xstat(__rte_unused const char *key,
5531                              const char *value, void *opaque_arg)
5532 {
5533         struct bnxt *bp = opaque_arg;
5534         unsigned long flow_xstat;
5535         char *end = NULL;
5536
5537         if (!value || !opaque_arg) {
5538                 PMD_DRV_LOG(ERR,
5539                             "Invalid parameter passed to flow_xstat devarg.\n");
5540                 return -EINVAL;
5541         }
5542
5543         flow_xstat = strtoul(value, &end, 10);
5544         if (end == NULL || *end != '\0' ||
5545             (flow_xstat == ULONG_MAX && errno == ERANGE)) {
5546                 PMD_DRV_LOG(ERR,
5547                             "Invalid parameter passed to flow_xstat devarg.\n");
5548                 return -EINVAL;
5549         }
5550
5551         if (BNXT_DEVARG_FLOW_XSTAT_INVALID(flow_xstat)) {
5552                 PMD_DRV_LOG(ERR,
5553                             "Invalid value passed to flow_xstat devarg.\n");
5554                 return -EINVAL;
5555         }
5556
5557         bp->flags |= BNXT_FLAG_FLOW_XSTATS_EN;
5558         if (BNXT_FLOW_XSTATS_EN(bp))
5559                 PMD_DRV_LOG(INFO, "flow_xstat feature enabled.\n");
5560
5561         return 0;
5562 }
5563
5564 static int
5565 bnxt_parse_devarg_max_num_kflows(__rte_unused const char *key,
5566                                         const char *value, void *opaque_arg)
5567 {
5568         struct bnxt *bp = opaque_arg;
5569         unsigned long max_num_kflows;
5570         char *end = NULL;
5571
5572         if (!value || !opaque_arg) {
5573                 PMD_DRV_LOG(ERR,
5574                         "Invalid parameter passed to max_num_kflows devarg.\n");
5575                 return -EINVAL;
5576         }
5577
5578         max_num_kflows = strtoul(value, &end, 10);
5579         if (end == NULL || *end != '\0' ||
5580                 (max_num_kflows == ULONG_MAX && errno == ERANGE)) {
5581                 PMD_DRV_LOG(ERR,
5582                         "Invalid parameter passed to max_num_kflows devarg.\n");
5583                 return -EINVAL;
5584         }
5585
5586         if (bnxt_devarg_max_num_kflow_invalid(max_num_kflows)) {
5587                 PMD_DRV_LOG(ERR,
5588                         "Invalid value passed to max_num_kflows devarg.\n");
5589                 return -EINVAL;
5590         }
5591
5592         bp->max_num_kflows = max_num_kflows;
5593         if (bp->max_num_kflows)
5594                 PMD_DRV_LOG(INFO, "max_num_kflows set as %ldK.\n",
5595                                 max_num_kflows);
5596
5597         return 0;
5598 }
5599
5600 static int
5601 bnxt_parse_devarg_app_id(__rte_unused const char *key,
5602                                  const char *value, void *opaque_arg)
5603 {
5604         struct bnxt *bp = opaque_arg;
5605         unsigned long app_id;
5606         char *end = NULL;
5607
5608         if (!value || !opaque_arg) {
5609                 PMD_DRV_LOG(ERR,
5610                             "Invalid parameter passed to app-id "
5611                             "devargs.\n");
5612                 return -EINVAL;
5613         }
5614
5615         app_id = strtoul(value, &end, 10);
5616         if (end == NULL || *end != '\0' ||
5617             (app_id == ULONG_MAX && errno == ERANGE)) {
5618                 PMD_DRV_LOG(ERR,
5619                             "Invalid parameter passed to app_id "
5620                             "devargs.\n");
5621                 return -EINVAL;
5622         }
5623
5624         if (BNXT_DEVARG_APP_ID_INVALID(app_id)) {
5625                 PMD_DRV_LOG(ERR, "Invalid app-id(%d) devargs.\n",
5626                             (uint16_t)app_id);
5627                 return -EINVAL;
5628         }
5629
5630         bp->app_id = app_id;
5631         PMD_DRV_LOG(INFO, "app-id=%d feature enabled.\n", (uint16_t)app_id);
5632
5633         return 0;
5634 }
5635
5636 static int
5637 bnxt_parse_devarg_rep_is_pf(__rte_unused const char *key,
5638                             const char *value, void *opaque_arg)
5639 {
5640         struct bnxt_representor *vfr_bp = opaque_arg;
5641         unsigned long rep_is_pf;
5642         char *end = NULL;
5643
5644         if (!value || !opaque_arg) {
5645                 PMD_DRV_LOG(ERR,
5646                             "Invalid parameter passed to rep_is_pf devargs.\n");
5647                 return -EINVAL;
5648         }
5649
5650         rep_is_pf = strtoul(value, &end, 10);
5651         if (end == NULL || *end != '\0' ||
5652             (rep_is_pf == ULONG_MAX && errno == ERANGE)) {
5653                 PMD_DRV_LOG(ERR,
5654                             "Invalid parameter passed to rep_is_pf devargs.\n");
5655                 return -EINVAL;
5656         }
5657
5658         if (BNXT_DEVARG_REP_IS_PF_INVALID(rep_is_pf)) {
5659                 PMD_DRV_LOG(ERR,
5660                             "Invalid value passed to rep_is_pf devargs.\n");
5661                 return -EINVAL;
5662         }
5663
5664         vfr_bp->flags |= rep_is_pf;
5665         if (BNXT_REP_PF(vfr_bp))
5666                 PMD_DRV_LOG(INFO, "PF representor\n");
5667         else
5668                 PMD_DRV_LOG(INFO, "VF representor\n");
5669
5670         return 0;
5671 }
5672
5673 static int
5674 bnxt_parse_devarg_rep_based_pf(__rte_unused const char *key,
5675                                const char *value, void *opaque_arg)
5676 {
5677         struct bnxt_representor *vfr_bp = opaque_arg;
5678         unsigned long rep_based_pf;
5679         char *end = NULL;
5680
5681         if (!value || !opaque_arg) {
5682                 PMD_DRV_LOG(ERR,
5683                             "Invalid parameter passed to rep_based_pf "
5684                             "devargs.\n");
5685                 return -EINVAL;
5686         }
5687
5688         rep_based_pf = strtoul(value, &end, 10);
5689         if (end == NULL || *end != '\0' ||
5690             (rep_based_pf == ULONG_MAX && errno == ERANGE)) {
5691                 PMD_DRV_LOG(ERR,
5692                             "Invalid parameter passed to rep_based_pf "
5693                             "devargs.\n");
5694                 return -EINVAL;
5695         }
5696
5697         if (BNXT_DEVARG_REP_BASED_PF_INVALID(rep_based_pf)) {
5698                 PMD_DRV_LOG(ERR,
5699                             "Invalid value passed to rep_based_pf devargs.\n");
5700                 return -EINVAL;
5701         }
5702
5703         vfr_bp->rep_based_pf = rep_based_pf;
5704         vfr_bp->flags |= BNXT_REP_BASED_PF_VALID;
5705
5706         PMD_DRV_LOG(INFO, "rep-based-pf = %d\n", vfr_bp->rep_based_pf);
5707
5708         return 0;
5709 }
5710
5711 static int
5712 bnxt_parse_devarg_rep_q_r2f(__rte_unused const char *key,
5713                             const char *value, void *opaque_arg)
5714 {
5715         struct bnxt_representor *vfr_bp = opaque_arg;
5716         unsigned long rep_q_r2f;
5717         char *end = NULL;
5718
5719         if (!value || !opaque_arg) {
5720                 PMD_DRV_LOG(ERR,
5721                             "Invalid parameter passed to rep_q_r2f "
5722                             "devargs.\n");
5723                 return -EINVAL;
5724         }
5725
5726         rep_q_r2f = strtoul(value, &end, 10);
5727         if (end == NULL || *end != '\0' ||
5728             (rep_q_r2f == ULONG_MAX && errno == ERANGE)) {
5729                 PMD_DRV_LOG(ERR,
5730                             "Invalid parameter passed to rep_q_r2f "
5731                             "devargs.\n");
5732                 return -EINVAL;
5733         }
5734
5735         if (BNXT_DEVARG_REP_Q_R2F_INVALID(rep_q_r2f)) {
5736                 PMD_DRV_LOG(ERR,
5737                             "Invalid value passed to rep_q_r2f devargs.\n");
5738                 return -EINVAL;
5739         }
5740
5741         vfr_bp->rep_q_r2f = rep_q_r2f;
5742         vfr_bp->flags |= BNXT_REP_Q_R2F_VALID;
5743         PMD_DRV_LOG(INFO, "rep-q-r2f = %d\n", vfr_bp->rep_q_r2f);
5744
5745         return 0;
5746 }
5747
5748 static int
5749 bnxt_parse_devarg_rep_q_f2r(__rte_unused const char *key,
5750                             const char *value, void *opaque_arg)
5751 {
5752         struct bnxt_representor *vfr_bp = opaque_arg;
5753         unsigned long rep_q_f2r;
5754         char *end = NULL;
5755
5756         if (!value || !opaque_arg) {
5757                 PMD_DRV_LOG(ERR,
5758                             "Invalid parameter passed to rep_q_f2r "
5759                             "devargs.\n");
5760                 return -EINVAL;
5761         }
5762
5763         rep_q_f2r = strtoul(value, &end, 10);
5764         if (end == NULL || *end != '\0' ||
5765             (rep_q_f2r == ULONG_MAX && errno == ERANGE)) {
5766                 PMD_DRV_LOG(ERR,
5767                             "Invalid parameter passed to rep_q_f2r "
5768                             "devargs.\n");
5769                 return -EINVAL;
5770         }
5771
5772         if (BNXT_DEVARG_REP_Q_F2R_INVALID(rep_q_f2r)) {
5773                 PMD_DRV_LOG(ERR,
5774                             "Invalid value passed to rep_q_f2r devargs.\n");
5775                 return -EINVAL;
5776         }
5777
5778         vfr_bp->rep_q_f2r = rep_q_f2r;
5779         vfr_bp->flags |= BNXT_REP_Q_F2R_VALID;
5780         PMD_DRV_LOG(INFO, "rep-q-f2r = %d\n", vfr_bp->rep_q_f2r);
5781
5782         return 0;
5783 }
5784
5785 static int
5786 bnxt_parse_devarg_rep_fc_r2f(__rte_unused const char *key,
5787                              const char *value, void *opaque_arg)
5788 {
5789         struct bnxt_representor *vfr_bp = opaque_arg;
5790         unsigned long rep_fc_r2f;
5791         char *end = NULL;
5792
5793         if (!value || !opaque_arg) {
5794                 PMD_DRV_LOG(ERR,
5795                             "Invalid parameter passed to rep_fc_r2f "
5796                             "devargs.\n");
5797                 return -EINVAL;
5798         }
5799
5800         rep_fc_r2f = strtoul(value, &end, 10);
5801         if (end == NULL || *end != '\0' ||
5802             (rep_fc_r2f == ULONG_MAX && errno == ERANGE)) {
5803                 PMD_DRV_LOG(ERR,
5804                             "Invalid parameter passed to rep_fc_r2f "
5805                             "devargs.\n");
5806                 return -EINVAL;
5807         }
5808
5809         if (BNXT_DEVARG_REP_FC_R2F_INVALID(rep_fc_r2f)) {
5810                 PMD_DRV_LOG(ERR,
5811                             "Invalid value passed to rep_fc_r2f devargs.\n");
5812                 return -EINVAL;
5813         }
5814
5815         vfr_bp->flags |= BNXT_REP_FC_R2F_VALID;
5816         vfr_bp->rep_fc_r2f = rep_fc_r2f;
5817         PMD_DRV_LOG(INFO, "rep-fc-r2f = %lu\n", rep_fc_r2f);
5818
5819         return 0;
5820 }
5821
5822 static int
5823 bnxt_parse_devarg_rep_fc_f2r(__rte_unused const char *key,
5824                              const char *value, void *opaque_arg)
5825 {
5826         struct bnxt_representor *vfr_bp = opaque_arg;
5827         unsigned long rep_fc_f2r;
5828         char *end = NULL;
5829
5830         if (!value || !opaque_arg) {
5831                 PMD_DRV_LOG(ERR,
5832                             "Invalid parameter passed to rep_fc_f2r "
5833                             "devargs.\n");
5834                 return -EINVAL;
5835         }
5836
5837         rep_fc_f2r = strtoul(value, &end, 10);
5838         if (end == NULL || *end != '\0' ||
5839             (rep_fc_f2r == ULONG_MAX && errno == ERANGE)) {
5840                 PMD_DRV_LOG(ERR,
5841                             "Invalid parameter passed to rep_fc_f2r "
5842                             "devargs.\n");
5843                 return -EINVAL;
5844         }
5845
5846         if (BNXT_DEVARG_REP_FC_F2R_INVALID(rep_fc_f2r)) {
5847                 PMD_DRV_LOG(ERR,
5848                             "Invalid value passed to rep_fc_f2r devargs.\n");
5849                 return -EINVAL;
5850         }
5851
5852         vfr_bp->flags |= BNXT_REP_FC_F2R_VALID;
5853         vfr_bp->rep_fc_f2r = rep_fc_f2r;
5854         PMD_DRV_LOG(INFO, "rep-fc-f2r = %lu\n", rep_fc_f2r);
5855
5856         return 0;
5857 }
5858
5859 static int
5860 bnxt_parse_dev_args(struct bnxt *bp, struct rte_devargs *devargs)
5861 {
5862         struct rte_kvargs *kvlist;
5863         int ret;
5864
5865         if (devargs == NULL)
5866                 return 0;
5867
5868         kvlist = rte_kvargs_parse(devargs->args, bnxt_dev_args);
5869         if (kvlist == NULL)
5870                 return -EINVAL;
5871
5872         /*
5873          * Handler for "flow_xstat" devarg.
5874          * Invoked as for ex: "-a 0000:00:0d.0,flow_xstat=1"
5875          */
5876         ret = rte_kvargs_process(kvlist, BNXT_DEVARG_FLOW_XSTAT,
5877                                  bnxt_parse_devarg_flow_xstat, bp);
5878         if (ret)
5879                 goto err;
5880
5881         /*
5882          * Handler for "accum-stats" devarg.
5883          * Invoked as for ex: "-a 0000:00:0d.0,accum-stats=1"
5884          */
5885         rte_kvargs_process(kvlist, BNXT_DEVARG_ACCUM_STATS,
5886                            bnxt_parse_devarg_accum_stats, bp);
5887         /*
5888          * Handler for "max_num_kflows" devarg.
5889          * Invoked as for ex: "-a 000:00:0d.0,max_num_kflows=32"
5890          */
5891         ret = rte_kvargs_process(kvlist, BNXT_DEVARG_MAX_NUM_KFLOWS,
5892                                  bnxt_parse_devarg_max_num_kflows, bp);
5893         if (ret)
5894                 goto err;
5895
5896 err:
5897         /*
5898          * Handler for "app-id" devarg.
5899          * Invoked as for ex: "-a 000:00:0d.0,app-id=1"
5900          */
5901         rte_kvargs_process(kvlist, BNXT_DEVARG_APP_ID,
5902                            bnxt_parse_devarg_app_id, bp);
5903
5904         rte_kvargs_free(kvlist);
5905         return ret;
5906 }
5907
5908 static int bnxt_alloc_switch_domain(struct bnxt *bp)
5909 {
5910         int rc = 0;
5911
5912         if (BNXT_PF(bp) || BNXT_VF_IS_TRUSTED(bp)) {
5913                 rc = rte_eth_switch_domain_alloc(&bp->switch_domain_id);
5914                 if (rc)
5915                         PMD_DRV_LOG(ERR,
5916                                     "Failed to alloc switch domain: %d\n", rc);
5917                 else
5918                         PMD_DRV_LOG(INFO,
5919                                     "Switch domain allocated %d\n",
5920                                     bp->switch_domain_id);
5921         }
5922
5923         return rc;
5924 }
5925
5926 /* Allocate and initialize various fields in bnxt struct that
5927  * need to be allocated/destroyed only once in the lifetime of the driver
5928  */
5929 static int bnxt_drv_init(struct rte_eth_dev *eth_dev)
5930 {
5931         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
5932         struct bnxt *bp = eth_dev->data->dev_private;
5933         int rc = 0;
5934
5935         bp->flags &= ~BNXT_FLAG_RX_VECTOR_PKT_MODE;
5936
5937         if (bnxt_vf_pciid(pci_dev->id.device_id))
5938                 bp->flags |= BNXT_FLAG_VF;
5939
5940         if (bnxt_p5_device(pci_dev->id.device_id))
5941                 bp->flags |= BNXT_FLAG_CHIP_P5;
5942
5943         if (pci_dev->id.device_id == BROADCOM_DEV_ID_58802 ||
5944             pci_dev->id.device_id == BROADCOM_DEV_ID_58804 ||
5945             pci_dev->id.device_id == BROADCOM_DEV_ID_58808 ||
5946             pci_dev->id.device_id == BROADCOM_DEV_ID_58802_VF)
5947                 bp->flags |= BNXT_FLAG_STINGRAY;
5948
5949         if (BNXT_TRUFLOW_EN(bp)) {
5950                 /* extra mbuf field is required to store CFA code from mark */
5951                 static const struct rte_mbuf_dynfield bnxt_cfa_code_dynfield_desc = {
5952                         .name = RTE_PMD_BNXT_CFA_CODE_DYNFIELD_NAME,
5953                         .size = sizeof(bnxt_cfa_code_dynfield_t),
5954                         .align = __alignof__(bnxt_cfa_code_dynfield_t),
5955                 };
5956                 bnxt_cfa_code_dynfield_offset =
5957                         rte_mbuf_dynfield_register(&bnxt_cfa_code_dynfield_desc);
5958                 if (bnxt_cfa_code_dynfield_offset < 0) {
5959                         PMD_DRV_LOG(ERR,
5960                             "Failed to register mbuf field for TruFlow mark\n");
5961                         return -rte_errno;
5962                 }
5963         }
5964
5965         rc = bnxt_map_pci_bars(eth_dev);
5966         if (rc) {
5967                 PMD_DRV_LOG(ERR,
5968                             "Failed to initialize board rc: %x\n", rc);
5969                 return rc;
5970         }
5971
5972         rc = bnxt_alloc_pf_info(bp);
5973         if (rc)
5974                 return rc;
5975
5976         rc = bnxt_alloc_link_info(bp);
5977         if (rc)
5978                 return rc;
5979
5980         rc = bnxt_alloc_parent_info(bp);
5981         if (rc)
5982                 return rc;
5983
5984         rc = bnxt_alloc_hwrm_resources(bp);
5985         if (rc) {
5986                 PMD_DRV_LOG(ERR,
5987                             "Failed to allocate response buffer rc: %x\n", rc);
5988                 return rc;
5989         }
5990         rc = bnxt_alloc_leds_info(bp);
5991         if (rc)
5992                 return rc;
5993
5994         rc = bnxt_alloc_cos_queues(bp);
5995         if (rc)
5996                 return rc;
5997
5998         rc = bnxt_init_locks(bp);
5999         if (rc)
6000                 return rc;
6001
6002         rc = bnxt_alloc_switch_domain(bp);
6003         if (rc)
6004                 return rc;
6005
6006         return rc;
6007 }
6008
6009 static int
6010 bnxt_dev_init(struct rte_eth_dev *eth_dev, void *params __rte_unused)
6011 {
6012         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
6013         static int version_printed;
6014         struct bnxt *bp;
6015         int rc;
6016
6017         if (version_printed++ == 0)
6018                 PMD_DRV_LOG(INFO, "%s\n", bnxt_version);
6019
6020         eth_dev->dev_ops = &bnxt_dev_ops;
6021         eth_dev->rx_queue_count = bnxt_rx_queue_count_op;
6022         eth_dev->rx_descriptor_status = bnxt_rx_descriptor_status_op;
6023         eth_dev->tx_descriptor_status = bnxt_tx_descriptor_status_op;
6024         eth_dev->rx_pkt_burst = &bnxt_recv_pkts;
6025         eth_dev->tx_pkt_burst = &bnxt_xmit_pkts;
6026
6027         /*
6028          * For secondary processes, we don't initialise any further
6029          * as primary has already done this work.
6030          */
6031         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
6032                 return 0;
6033
6034         rte_eth_copy_pci_info(eth_dev, pci_dev);
6035         eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
6036
6037         bp = eth_dev->data->dev_private;
6038
6039         /* Parse dev arguments passed on when starting the DPDK application. */
6040         rc = bnxt_parse_dev_args(bp, pci_dev->device.devargs);
6041         if (rc)
6042                 goto error_free;
6043
6044         rc = bnxt_drv_init(eth_dev);
6045         if (rc)
6046                 goto error_free;
6047
6048         rc = bnxt_init_resources(bp, false);
6049         if (rc)
6050                 goto error_free;
6051
6052         rc = bnxt_alloc_stats_mem(bp);
6053         if (rc)
6054                 goto error_free;
6055
6056         PMD_DRV_LOG(INFO,
6057                     "Found %s device at mem %" PRIX64 ", node addr %pM\n",
6058                     DRV_MODULE_NAME,
6059                     pci_dev->mem_resource[0].phys_addr,
6060                     pci_dev->mem_resource[0].addr);
6061
6062         return 0;
6063
6064 error_free:
6065         bnxt_dev_uninit(eth_dev);
6066         return rc;
6067 }
6068
6069
6070 static void bnxt_free_ctx_mem_buf(struct bnxt_ctx_mem_buf_info *ctx)
6071 {
6072         if (!ctx)
6073                 return;
6074
6075         if (ctx->va)
6076                 rte_free(ctx->va);
6077
6078         ctx->va = NULL;
6079         ctx->dma = RTE_BAD_IOVA;
6080         ctx->ctx_id = BNXT_CTX_VAL_INVAL;
6081 }
6082
6083 static void bnxt_unregister_fc_ctx_mem(struct bnxt *bp)
6084 {
6085         bnxt_hwrm_cfa_counter_cfg(bp, BNXT_DIR_RX,
6086                                   CFA_COUNTER_CFG_IN_COUNTER_TYPE_FC,
6087                                   bp->flow_stat->rx_fc_out_tbl.ctx_id,
6088                                   bp->flow_stat->max_fc,
6089                                   false);
6090
6091         bnxt_hwrm_cfa_counter_cfg(bp, BNXT_DIR_TX,
6092                                   CFA_COUNTER_CFG_IN_COUNTER_TYPE_FC,
6093                                   bp->flow_stat->tx_fc_out_tbl.ctx_id,
6094                                   bp->flow_stat->max_fc,
6095                                   false);
6096
6097         if (bp->flow_stat->rx_fc_in_tbl.ctx_id != BNXT_CTX_VAL_INVAL)
6098                 bnxt_hwrm_ctx_unrgtr(bp, bp->flow_stat->rx_fc_in_tbl.ctx_id);
6099         bp->flow_stat->rx_fc_in_tbl.ctx_id = BNXT_CTX_VAL_INVAL;
6100
6101         if (bp->flow_stat->rx_fc_out_tbl.ctx_id != BNXT_CTX_VAL_INVAL)
6102                 bnxt_hwrm_ctx_unrgtr(bp, bp->flow_stat->rx_fc_out_tbl.ctx_id);
6103         bp->flow_stat->rx_fc_out_tbl.ctx_id = BNXT_CTX_VAL_INVAL;
6104
6105         if (bp->flow_stat->tx_fc_in_tbl.ctx_id != BNXT_CTX_VAL_INVAL)
6106                 bnxt_hwrm_ctx_unrgtr(bp, bp->flow_stat->tx_fc_in_tbl.ctx_id);
6107         bp->flow_stat->tx_fc_in_tbl.ctx_id = BNXT_CTX_VAL_INVAL;
6108
6109         if (bp->flow_stat->tx_fc_out_tbl.ctx_id != BNXT_CTX_VAL_INVAL)
6110                 bnxt_hwrm_ctx_unrgtr(bp, bp->flow_stat->tx_fc_out_tbl.ctx_id);
6111         bp->flow_stat->tx_fc_out_tbl.ctx_id = BNXT_CTX_VAL_INVAL;
6112 }
6113
6114 static void bnxt_uninit_fc_ctx_mem(struct bnxt *bp)
6115 {
6116         bnxt_unregister_fc_ctx_mem(bp);
6117
6118         bnxt_free_ctx_mem_buf(&bp->flow_stat->rx_fc_in_tbl);
6119         bnxt_free_ctx_mem_buf(&bp->flow_stat->rx_fc_out_tbl);
6120         bnxt_free_ctx_mem_buf(&bp->flow_stat->tx_fc_in_tbl);
6121         bnxt_free_ctx_mem_buf(&bp->flow_stat->tx_fc_out_tbl);
6122 }
6123
6124 static void bnxt_uninit_ctx_mem(struct bnxt *bp)
6125 {
6126         if (BNXT_FLOW_XSTATS_EN(bp))
6127                 bnxt_uninit_fc_ctx_mem(bp);
6128 }
6129
6130 static void
6131 bnxt_free_error_recovery_info(struct bnxt *bp)
6132 {
6133         rte_free(bp->recovery_info);
6134         bp->recovery_info = NULL;
6135         bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
6136 }
6137
6138 static int
6139 bnxt_uninit_resources(struct bnxt *bp, bool reconfig_dev)
6140 {
6141         int rc;
6142
6143         bnxt_free_int(bp);
6144         bnxt_free_mem(bp, reconfig_dev);
6145
6146         bnxt_hwrm_func_buf_unrgtr(bp);
6147         if (bp->pf != NULL) {
6148                 rte_free(bp->pf->vf_req_buf);
6149                 bp->pf->vf_req_buf = NULL;
6150         }
6151
6152         rc = bnxt_hwrm_func_driver_unregister(bp, 0);
6153         bp->flags &= ~BNXT_FLAG_REGISTERED;
6154         bnxt_free_ctx_mem(bp);
6155         if (!reconfig_dev) {
6156                 bnxt_free_hwrm_resources(bp);
6157                 bnxt_free_error_recovery_info(bp);
6158         }
6159
6160         bnxt_uninit_ctx_mem(bp);
6161
6162         bnxt_free_flow_stats_info(bp);
6163         if (bp->rep_info != NULL)
6164                 bnxt_free_switch_domain(bp);
6165         bnxt_free_rep_info(bp);
6166         rte_free(bp->ptp_cfg);
6167         bp->ptp_cfg = NULL;
6168         return rc;
6169 }
6170
6171 static int
6172 bnxt_dev_uninit(struct rte_eth_dev *eth_dev)
6173 {
6174         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
6175                 return -EPERM;
6176
6177         PMD_DRV_LOG(DEBUG, "Calling Device uninit\n");
6178
6179         if (eth_dev->state != RTE_ETH_DEV_UNUSED)
6180                 bnxt_dev_close_op(eth_dev);
6181
6182         return 0;
6183 }
6184
6185 static int bnxt_pci_remove_dev_with_reps(struct rte_eth_dev *eth_dev)
6186 {
6187         struct bnxt *bp = eth_dev->data->dev_private;
6188         struct rte_eth_dev *vf_rep_eth_dev;
6189         int ret = 0, i;
6190
6191         if (!bp)
6192                 return -EINVAL;
6193
6194         for (i = 0; i < bp->num_reps; i++) {
6195                 vf_rep_eth_dev = bp->rep_info[i].vfr_eth_dev;
6196                 if (!vf_rep_eth_dev)
6197                         continue;
6198                 PMD_DRV_LOG(DEBUG, "BNXT Port:%d VFR pci remove\n",
6199                             vf_rep_eth_dev->data->port_id);
6200                 rte_eth_dev_destroy(vf_rep_eth_dev, bnxt_representor_uninit);
6201         }
6202         PMD_DRV_LOG(DEBUG, "BNXT Port:%d pci remove\n",
6203                     eth_dev->data->port_id);
6204         ret = rte_eth_dev_destroy(eth_dev, bnxt_dev_uninit);
6205
6206         return ret;
6207 }
6208
6209 static void bnxt_free_rep_info(struct bnxt *bp)
6210 {
6211         rte_free(bp->rep_info);
6212         bp->rep_info = NULL;
6213         rte_free(bp->cfa_code_map);
6214         bp->cfa_code_map = NULL;
6215 }
6216
6217 static int bnxt_init_rep_info(struct bnxt *bp)
6218 {
6219         int i = 0, rc;
6220
6221         if (bp->rep_info)
6222                 return 0;
6223
6224         bp->rep_info = rte_zmalloc("bnxt_rep_info",
6225                                    sizeof(bp->rep_info[0]) * BNXT_MAX_VF_REPS,
6226                                    0);
6227         if (!bp->rep_info) {
6228                 PMD_DRV_LOG(ERR, "Failed to alloc memory for rep info\n");
6229                 return -ENOMEM;
6230         }
6231         bp->cfa_code_map = rte_zmalloc("bnxt_cfa_code_map",
6232                                        sizeof(*bp->cfa_code_map) *
6233                                        BNXT_MAX_CFA_CODE, 0);
6234         if (!bp->cfa_code_map) {
6235                 PMD_DRV_LOG(ERR, "Failed to alloc memory for cfa_code_map\n");
6236                 bnxt_free_rep_info(bp);
6237                 return -ENOMEM;
6238         }
6239
6240         for (i = 0; i < BNXT_MAX_CFA_CODE; i++)
6241                 bp->cfa_code_map[i] = BNXT_VF_IDX_INVALID;
6242
6243         rc = pthread_mutex_init(&bp->rep_info->vfr_lock, NULL);
6244         if (rc) {
6245                 PMD_DRV_LOG(ERR, "Unable to initialize vfr_lock\n");
6246                 bnxt_free_rep_info(bp);
6247                 return rc;
6248         }
6249
6250         rc = pthread_mutex_init(&bp->rep_info->vfr_start_lock, NULL);
6251         if (rc) {
6252                 PMD_DRV_LOG(ERR, "Unable to initialize vfr_start_lock\n");
6253                 bnxt_free_rep_info(bp);
6254                 return rc;
6255         }
6256
6257         return rc;
6258 }
6259
6260 static int bnxt_rep_port_probe(struct rte_pci_device *pci_dev,
6261                                struct rte_eth_devargs *eth_da,
6262                                struct rte_eth_dev *backing_eth_dev,
6263                                const char *dev_args)
6264 {
6265         struct rte_eth_dev *vf_rep_eth_dev;
6266         char name[RTE_ETH_NAME_MAX_LEN];
6267         struct bnxt *backing_bp;
6268         uint16_t num_rep;
6269         int i, ret = 0;
6270         struct rte_kvargs *kvlist = NULL;
6271
6272         if (eth_da->type == RTE_ETH_REPRESENTOR_NONE)
6273                 return 0;
6274         if (eth_da->type != RTE_ETH_REPRESENTOR_VF) {
6275                 PMD_DRV_LOG(ERR, "unsupported representor type %d\n",
6276                             eth_da->type);
6277                 return -ENOTSUP;
6278         }
6279         num_rep = eth_da->nb_representor_ports;
6280         if (num_rep > BNXT_MAX_VF_REPS) {
6281                 PMD_DRV_LOG(ERR, "nb_representor_ports = %d > %d MAX VF REPS\n",
6282                             num_rep, BNXT_MAX_VF_REPS);
6283                 return -EINVAL;
6284         }
6285
6286         if (num_rep >= RTE_MAX_ETHPORTS) {
6287                 PMD_DRV_LOG(ERR,
6288                             "nb_representor_ports = %d > %d MAX ETHPORTS\n",
6289                             num_rep, RTE_MAX_ETHPORTS);
6290                 return -EINVAL;
6291         }
6292
6293         backing_bp = backing_eth_dev->data->dev_private;
6294
6295         if (!(BNXT_PF(backing_bp) || BNXT_VF_IS_TRUSTED(backing_bp))) {
6296                 PMD_DRV_LOG(ERR,
6297                             "Not a PF or trusted VF. No Representor support\n");
6298                 /* Returning an error is not an option.
6299                  * Applications are not handling this correctly
6300                  */
6301                 return 0;
6302         }
6303
6304         if (bnxt_init_rep_info(backing_bp))
6305                 return 0;
6306
6307         for (i = 0; i < num_rep; i++) {
6308                 struct bnxt_representor representor = {
6309                         .vf_id = eth_da->representor_ports[i],
6310                         .switch_domain_id = backing_bp->switch_domain_id,
6311                         .parent_dev = backing_eth_dev
6312                 };
6313
6314                 if (representor.vf_id >= BNXT_MAX_VF_REPS) {
6315                         PMD_DRV_LOG(ERR, "VF-Rep id %d >= %d MAX VF ID\n",
6316                                     representor.vf_id, BNXT_MAX_VF_REPS);
6317                         continue;
6318                 }
6319
6320                 /* representor port net_bdf_port */
6321                 snprintf(name, sizeof(name), "net_%s_representor_%d",
6322                          pci_dev->device.name, eth_da->representor_ports[i]);
6323
6324                 kvlist = rte_kvargs_parse(dev_args, bnxt_dev_args);
6325                 if (kvlist) {
6326                         /*
6327                          * Handler for "rep_is_pf" devarg.
6328                          * Invoked as for ex: "-a 000:00:0d.0,
6329                          * rep-based-pf=<pf index> rep-is-pf=<VF=0 or PF=1>"
6330                          */
6331                         ret = rte_kvargs_process(kvlist, BNXT_DEVARG_REP_IS_PF,
6332                                                  bnxt_parse_devarg_rep_is_pf,
6333                                                  (void *)&representor);
6334                         if (ret) {
6335                                 ret = -EINVAL;
6336                                 goto err;
6337                         }
6338                         /*
6339                          * Handler for "rep_based_pf" devarg.
6340                          * Invoked as for ex: "-a 000:00:0d.0,
6341                          * rep-based-pf=<pf index> rep-is-pf=<VF=0 or PF=1>"
6342                          */
6343                         ret = rte_kvargs_process(kvlist,
6344                                                  BNXT_DEVARG_REP_BASED_PF,
6345                                                  bnxt_parse_devarg_rep_based_pf,
6346                                                  (void *)&representor);
6347                         if (ret) {
6348                                 ret = -EINVAL;
6349                                 goto err;
6350                         }
6351                         /*
6352                          * Handler for "rep_based_pf" devarg.
6353                          * Invoked as for ex: "-a 000:00:0d.0,
6354                          * rep-based-pf=<pf index> rep-is-pf=<VF=0 or PF=1>"
6355                          */
6356                         ret = rte_kvargs_process(kvlist, BNXT_DEVARG_REP_Q_R2F,
6357                                                  bnxt_parse_devarg_rep_q_r2f,
6358                                                  (void *)&representor);
6359                         if (ret) {
6360                                 ret = -EINVAL;
6361                                 goto err;
6362                         }
6363                         /*
6364                          * Handler for "rep_based_pf" devarg.
6365                          * Invoked as for ex: "-a 000:00:0d.0,
6366                          * rep-based-pf=<pf index> rep-is-pf=<VF=0 or PF=1>"
6367                          */
6368                         ret = rte_kvargs_process(kvlist, BNXT_DEVARG_REP_Q_F2R,
6369                                                  bnxt_parse_devarg_rep_q_f2r,
6370                                                  (void *)&representor);
6371                         if (ret) {
6372                                 ret = -EINVAL;
6373                                 goto err;
6374                         }
6375                         /*
6376                          * Handler for "rep_based_pf" devarg.
6377                          * Invoked as for ex: "-a 000:00:0d.0,
6378                          * rep-based-pf=<pf index> rep-is-pf=<VF=0 or PF=1>"
6379                          */
6380                         ret = rte_kvargs_process(kvlist, BNXT_DEVARG_REP_FC_R2F,
6381                                                  bnxt_parse_devarg_rep_fc_r2f,
6382                                                  (void *)&representor);
6383                         if (ret) {
6384                                 ret = -EINVAL;
6385                                 goto err;
6386                         }
6387                         /*
6388                          * Handler for "rep_based_pf" devarg.
6389                          * Invoked as for ex: "-a 000:00:0d.0,
6390                          * rep-based-pf=<pf index> rep-is-pf=<VF=0 or PF=1>"
6391                          */
6392                         ret = rte_kvargs_process(kvlist, BNXT_DEVARG_REP_FC_F2R,
6393                                                  bnxt_parse_devarg_rep_fc_f2r,
6394                                                  (void *)&representor);
6395                         if (ret) {
6396                                 ret = -EINVAL;
6397                                 goto err;
6398                         }
6399                 }
6400
6401                 ret = rte_eth_dev_create(&pci_dev->device, name,
6402                                          sizeof(struct bnxt_representor),
6403                                          NULL, NULL,
6404                                          bnxt_representor_init,
6405                                          &representor);
6406                 if (ret) {
6407                         PMD_DRV_LOG(ERR, "failed to create bnxt vf "
6408                                     "representor %s.", name);
6409                         goto err;
6410                 }
6411
6412                 vf_rep_eth_dev = rte_eth_dev_allocated(name);
6413                 if (!vf_rep_eth_dev) {
6414                         PMD_DRV_LOG(ERR, "Failed to find the eth_dev"
6415                                     " for VF-Rep: %s.", name);
6416                         ret = -ENODEV;
6417                         goto err;
6418                 }
6419
6420                 PMD_DRV_LOG(DEBUG, "BNXT Port:%d VFR pci probe\n",
6421                             backing_eth_dev->data->port_id);
6422                 backing_bp->rep_info[representor.vf_id].vfr_eth_dev =
6423                                                          vf_rep_eth_dev;
6424                 backing_bp->num_reps++;
6425
6426         }
6427
6428         rte_kvargs_free(kvlist);
6429         return 0;
6430
6431 err:
6432         /* If num_rep > 1, then rollback already created
6433          * ports, since we'll be failing the probe anyway
6434          */
6435         if (num_rep > 1)
6436                 bnxt_pci_remove_dev_with_reps(backing_eth_dev);
6437         rte_errno = -ret;
6438         rte_kvargs_free(kvlist);
6439
6440         return ret;
6441 }
6442
6443 static int bnxt_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
6444                           struct rte_pci_device *pci_dev)
6445 {
6446         struct rte_eth_devargs eth_da = { .nb_representor_ports = 0 };
6447         struct rte_eth_dev *backing_eth_dev;
6448         uint16_t num_rep;
6449         int ret = 0;
6450
6451         if (pci_dev->device.devargs) {
6452                 ret = rte_eth_devargs_parse(pci_dev->device.devargs->args,
6453                                             &eth_da);
6454                 if (ret)
6455                         return ret;
6456         }
6457
6458         num_rep = eth_da.nb_representor_ports;
6459         PMD_DRV_LOG(DEBUG, "nb_representor_ports = %d\n",
6460                     num_rep);
6461
6462         /* We could come here after first level of probe is already invoked
6463          * as part of an application bringup(OVS-DPDK vswitchd), so first check
6464          * for already allocated eth_dev for the backing device (PF/Trusted VF)
6465          */
6466         backing_eth_dev = rte_eth_dev_allocated(pci_dev->device.name);
6467         if (backing_eth_dev == NULL) {
6468                 ret = rte_eth_dev_create(&pci_dev->device, pci_dev->device.name,
6469                                          sizeof(struct bnxt),
6470                                          eth_dev_pci_specific_init, pci_dev,
6471                                          bnxt_dev_init, NULL);
6472
6473                 if (ret || !num_rep)
6474                         return ret;
6475
6476                 backing_eth_dev = rte_eth_dev_allocated(pci_dev->device.name);
6477         }
6478         PMD_DRV_LOG(DEBUG, "BNXT Port:%d pci probe\n",
6479                     backing_eth_dev->data->port_id);
6480
6481         if (!num_rep)
6482                 return ret;
6483
6484         /* probe representor ports now */
6485         ret = bnxt_rep_port_probe(pci_dev, &eth_da, backing_eth_dev,
6486                                   pci_dev->device.devargs->args);
6487
6488         return ret;
6489 }
6490
6491 static int bnxt_pci_remove(struct rte_pci_device *pci_dev)
6492 {
6493         struct rte_eth_dev *eth_dev;
6494
6495         eth_dev = rte_eth_dev_allocated(pci_dev->device.name);
6496         if (!eth_dev)
6497                 return 0; /* Invoked typically only by OVS-DPDK, by the
6498                            * time it comes here the eth_dev is already
6499                            * deleted by rte_eth_dev_close(), so returning
6500                            * +ve value will at least help in proper cleanup
6501                            */
6502
6503         PMD_DRV_LOG(DEBUG, "BNXT Port:%d pci remove\n", eth_dev->data->port_id);
6504         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
6505                 if (eth_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)
6506                         return rte_eth_dev_destroy(eth_dev,
6507                                                    bnxt_representor_uninit);
6508                 else
6509                         return rte_eth_dev_destroy(eth_dev,
6510                                                    bnxt_dev_uninit);
6511         } else {
6512                 return rte_eth_dev_pci_generic_remove(pci_dev, NULL);
6513         }
6514 }
6515
6516 static struct rte_pci_driver bnxt_rte_pmd = {
6517         .id_table = bnxt_pci_id_map,
6518         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
6519                         RTE_PCI_DRV_INTR_RMV |
6520                         RTE_PCI_DRV_PROBE_AGAIN, /* Needed in case of VF-REPs
6521                                                   * and OVS-DPDK
6522                                                   */
6523         .probe = bnxt_pci_probe,
6524         .remove = bnxt_pci_remove,
6525 };
6526
6527 static bool
6528 is_device_supported(struct rte_eth_dev *dev, struct rte_pci_driver *drv)
6529 {
6530         if (strcmp(dev->device->driver->name, drv->driver.name))
6531                 return false;
6532
6533         return true;
6534 }
6535
6536 bool is_bnxt_supported(struct rte_eth_dev *dev)
6537 {
6538         return is_device_supported(dev, &bnxt_rte_pmd);
6539 }
6540
6541 RTE_LOG_REGISTER_SUFFIX(bnxt_logtype_driver, driver, NOTICE);
6542 RTE_PMD_REGISTER_PCI(net_bnxt, bnxt_rte_pmd);
6543 RTE_PMD_REGISTER_PCI_TABLE(net_bnxt, bnxt_pci_id_map);
6544 RTE_PMD_REGISTER_KMOD_DEP(net_bnxt, "* igb_uio | uio_pci_generic | vfio-pci");