drivers/net/bnxt/bnxt_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2014-2021 Broadcom
   3  * All rights reserved.
   4  */
   5
   6 #include <inttypes.h>
   7 #include <stdbool.h>
   8
   9 #include <rte_dev.h>
  10 #include <ethdev_driver.h>
  11 #include <ethdev_pci.h>
  12 #include <rte_malloc.h>
  13 #include <rte_cycles.h>
  14 #include <rte_alarm.h>
  15 #include <rte_kvargs.h>
  16 #include <rte_vect.h>
  17
  18 #include "bnxt.h"
  19 #include "bnxt_filter.h"
  20 #include "bnxt_hwrm.h"
  21 #include "bnxt_irq.h"
  22 #include "bnxt_reps.h"
  23 #include "bnxt_ring.h"
  24 #include "bnxt_rxq.h"
  25 #include "bnxt_rxr.h"
  26 #include "bnxt_stats.h"
  27 #include "bnxt_txq.h"
  28 #include "bnxt_txr.h"
  29 #include "bnxt_vnic.h"
  30 #include "hsi_struct_def_dpdk.h"
  31 #include "bnxt_nvm_defs.h"
  32 #include "bnxt_tf_common.h"
  33 #include "ulp_flow_db.h"
  34 #include "rte_pmd_bnxt.h"
  35
  36 #define DRV_MODULE_NAME         "bnxt"
  37 static const char bnxt_version[] =
  38         "Broadcom NetXtreme driver " DRV_MODULE_NAME;
  39
  40 /*
  41  * The set of PCI devices this driver supports
  42  */
  43 static const struct rte_pci_id bnxt_pci_id_map[] = {
  44         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM,
  45                          BROADCOM_DEV_ID_STRATUS_NIC_VF1) },
  46         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM,
  47                          BROADCOM_DEV_ID_STRATUS_NIC_VF2) },
  48         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_STRATUS_NIC) },
  49         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57414_VF) },
  50         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57304_VF) },
  51         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_NS2) },
  52         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57406_VF) },
  53         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57407_MF) },
  54         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_5741X_VF) },
  55         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_5731X_VF) },
  56         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57417_MF) },
  57         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57412) },
  58         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57414) },
  59         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57416_RJ45) },
  60         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57417_RJ45) },
  61         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57412_MF) },
  62         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57317_RJ45) },
  63         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57417_SFP) },
  64         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57416_SFP) },
  65         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57317_SFP) },
  66         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57414_MF) },
  67         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57416_MF) },
  68         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58802) },
  69         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58804) },
  70         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58808) },
  71         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58802_VF) },
  72         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57508) },
  73         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57504) },
  74         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57502) },
  75         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57500_VF1) },
  76         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57500_VF2) },
  77         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57508_MF1) },
  78         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57504_MF1) },
  79         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57502_MF1) },
  80         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57508_MF2) },
  81         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57504_MF2) },
  82         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57502_MF2) },
  83         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58812) },
  84         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58814) },
  85         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58818) },
  86         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58818_VF) },
  87         { .vendor_id = 0, /* sentinel */ },
  88 };
  89
  90 #define BNXT_DEVARG_FLOW_XSTAT  "flow-xstat"
  91 #define BNXT_DEVARG_MAX_NUM_KFLOWS  "max-num-kflows"
  92 #define BNXT_DEVARG_REPRESENTOR "representor"
  93 #define BNXT_DEVARG_REP_BASED_PF  "rep-based-pf"
  94 #define BNXT_DEVARG_REP_IS_PF  "rep-is-pf"
  95 #define BNXT_DEVARG_REP_Q_R2F  "rep-q-r2f"
  96 #define BNXT_DEVARG_REP_Q_F2R  "rep-q-f2r"
  97 #define BNXT_DEVARG_REP_FC_R2F  "rep-fc-r2f"
  98 #define BNXT_DEVARG_REP_FC_F2R  "rep-fc-f2r"
  99 #define BNXT_DEVARG_APP_ID      "app-id"
 100
 101 static const char *const bnxt_dev_args[] = {
 102         BNXT_DEVARG_REPRESENTOR,
 103         BNXT_DEVARG_FLOW_XSTAT,
 104         BNXT_DEVARG_MAX_NUM_KFLOWS,
 105         BNXT_DEVARG_REP_BASED_PF,
 106         BNXT_DEVARG_REP_IS_PF,
 107         BNXT_DEVARG_REP_Q_R2F,
 108         BNXT_DEVARG_REP_Q_F2R,
 109         BNXT_DEVARG_REP_FC_R2F,
 110         BNXT_DEVARG_REP_FC_F2R,
 111         BNXT_DEVARG_APP_ID,
 112         NULL
 113 };
 114
 115 /*
 116  * app-id = an non-negative 8-bit number
 117  */
 118 #define BNXT_DEVARG_APP_ID_INVALID(val)                 ((val) > 255)
 119
 120 /*
 121  * flow_xstat == false to disable the feature
 122  * flow_xstat == true to enable the feature
 123  */
 124 #define BNXT_DEVARG_FLOW_XSTAT_INVALID(flow_xstat)      ((flow_xstat) > 1)
 125
 126 /*
 127  * rep_is_pf == false to indicate VF representor
 128  * rep_is_pf == true to indicate PF representor
 129  */
 130 #define BNXT_DEVARG_REP_IS_PF_INVALID(rep_is_pf)        ((rep_is_pf) > 1)
 131
 132 /*
 133  * rep_based_pf == Physical index of the PF
 134  */
 135 #define BNXT_DEVARG_REP_BASED_PF_INVALID(rep_based_pf)  ((rep_based_pf) > 15)
 136 /*
 137  * rep_q_r2f == Logical COS Queue index for the rep to endpoint direction
 138  */
 139 #define BNXT_DEVARG_REP_Q_R2F_INVALID(rep_q_r2f)        ((rep_q_r2f) > 3)
 140
 141 /*
 142  * rep_q_f2r == Logical COS Queue index for the endpoint to rep direction
 143  */
 144 #define BNXT_DEVARG_REP_Q_F2R_INVALID(rep_q_f2r)        ((rep_q_f2r) > 3)
 145
 146 /*
 147  * rep_fc_r2f == Flow control for the representor to endpoint direction
 148  */
 149 #define BNXT_DEVARG_REP_FC_R2F_INVALID(rep_fc_r2f)      ((rep_fc_r2f) > 1)
 150
 151 /*
 152  * rep_fc_f2r == Flow control for the endpoint to representor direction
 153  */
 154 #define BNXT_DEVARG_REP_FC_F2R_INVALID(rep_fc_f2r)      ((rep_fc_f2r) > 1)
 155
 156 int bnxt_cfa_code_dynfield_offset = -1;
 157
 158 /*
 159  * max_num_kflows must be >= 32
 160  * and must be a power-of-2 supported value
 161  * return: 1 -> invalid
 162  *         0 -> valid
 163  */
 164 static int bnxt_devarg_max_num_kflow_invalid(uint16_t max_num_kflows)
 165 {
 166         if (max_num_kflows < 32 || !rte_is_power_of_2(max_num_kflows))
 167                 return 1;
 168         return 0;
 169 }
 170
 171 static int bnxt_vlan_offload_set_op(struct rte_eth_dev *dev, int mask);
 172 static int bnxt_dev_uninit(struct rte_eth_dev *eth_dev);
 173 static int bnxt_init_resources(struct bnxt *bp, bool reconfig_dev);
 174 static int bnxt_uninit_resources(struct bnxt *bp, bool reconfig_dev);
 175 static void bnxt_cancel_fw_health_check(struct bnxt *bp);
 176 static int bnxt_restore_vlan_filters(struct bnxt *bp);
 177 static void bnxt_dev_recover(void *arg);
 178 static void bnxt_free_error_recovery_info(struct bnxt *bp);
 179 static void bnxt_free_rep_info(struct bnxt *bp);
 180
 181 int is_bnxt_in_error(struct bnxt *bp)
 182 {
 183         if (bp->flags & BNXT_FLAG_FATAL_ERROR)
 184                 return -EIO;
 185         if (bp->flags & BNXT_FLAG_FW_RESET)
 186                 return -EBUSY;
 187
 188         return 0;
 189 }
 190
 191 /***********************/
 192
 193 /*
 194  * High level utility functions
 195  */
 196
 197 static uint16_t bnxt_rss_ctxts(const struct bnxt *bp)
 198 {
 199         unsigned int num_rss_rings = RTE_MIN(bp->rx_nr_rings,
 200                                              BNXT_RSS_TBL_SIZE_P5);
 201
 202         if (!BNXT_CHIP_P5(bp))
 203                 return 1;
 204
 205         return RTE_ALIGN_MUL_CEIL(num_rss_rings,
 206                                   BNXT_RSS_ENTRIES_PER_CTX_P5) /
 207                                   BNXT_RSS_ENTRIES_PER_CTX_P5;
 208 }
 209
 210 uint16_t bnxt_rss_hash_tbl_size(const struct bnxt *bp)
 211 {
 212         if (!BNXT_CHIP_P5(bp))
 213                 return HW_HASH_INDEX_SIZE;
 214
 215         return bnxt_rss_ctxts(bp) * BNXT_RSS_ENTRIES_PER_CTX_P5;
 216 }
 217
 218 static void bnxt_free_parent_info(struct bnxt *bp)
 219 {
 220         rte_free(bp->parent);
 221         bp->parent = NULL;
 222 }
 223
 224 static void bnxt_free_pf_info(struct bnxt *bp)
 225 {
 226         rte_free(bp->pf);
 227         bp->pf = NULL;
 228 }
 229
 230 static void bnxt_free_link_info(struct bnxt *bp)
 231 {
 232         rte_free(bp->link_info);
 233         bp->link_info = NULL;
 234 }
 235
 236 static void bnxt_free_leds_info(struct bnxt *bp)
 237 {
 238         if (BNXT_VF(bp))
 239                 return;
 240
 241         rte_free(bp->leds);
 242         bp->leds = NULL;
 243 }
 244
 245 static void bnxt_free_flow_stats_info(struct bnxt *bp)
 246 {
 247         rte_free(bp->flow_stat);
 248         bp->flow_stat = NULL;
 249 }
 250
 251 static void bnxt_free_cos_queues(struct bnxt *bp)
 252 {
 253         rte_free(bp->rx_cos_queue);
 254         bp->rx_cos_queue = NULL;
 255         rte_free(bp->tx_cos_queue);
 256         bp->tx_cos_queue = NULL;
 257 }
 258
 259 static void bnxt_free_mem(struct bnxt *bp, bool reconfig)
 260 {
 261         bnxt_free_filter_mem(bp);
 262         bnxt_free_vnic_attributes(bp);
 263         bnxt_free_vnic_mem(bp);
 264
 265         /* tx/rx rings are configured as part of *_queue_setup callbacks.
 266          * If the number of rings change across fw update,
 267          * we don't have much choice except to warn the user.
 268          */
 269         if (!reconfig) {
 270                 bnxt_free_stats(bp);
 271                 bnxt_free_tx_rings(bp);
 272                 bnxt_free_rx_rings(bp);
 273         }
 274         bnxt_free_async_cp_ring(bp);
 275         bnxt_free_rxtx_nq_ring(bp);
 276
 277         rte_free(bp->grp_info);
 278         bp->grp_info = NULL;
 279 }
 280
 281 static int bnxt_alloc_parent_info(struct bnxt *bp)
 282 {
 283         bp->parent = rte_zmalloc("bnxt_parent_info",
 284                                  sizeof(struct bnxt_parent_info), 0);
 285         if (bp->parent == NULL)
 286                 return -ENOMEM;
 287
 288         return 0;
 289 }
 290
 291 static int bnxt_alloc_pf_info(struct bnxt *bp)
 292 {
 293         bp->pf = rte_zmalloc("bnxt_pf_info", sizeof(struct bnxt_pf_info), 0);
 294         if (bp->pf == NULL)
 295                 return -ENOMEM;
 296
 297         return 0;
 298 }
 299
 300 static int bnxt_alloc_link_info(struct bnxt *bp)
 301 {
 302         bp->link_info =
 303                 rte_zmalloc("bnxt_link_info", sizeof(struct bnxt_link_info), 0);
 304         if (bp->link_info == NULL)
 305                 return -ENOMEM;
 306
 307         return 0;
 308 }
 309
 310 static int bnxt_alloc_leds_info(struct bnxt *bp)
 311 {
 312         if (BNXT_VF(bp))
 313                 return 0;
 314
 315         bp->leds = rte_zmalloc("bnxt_leds",
 316                                BNXT_MAX_LED * sizeof(struct bnxt_led_info),
 317                                0);
 318         if (bp->leds == NULL)
 319                 return -ENOMEM;
 320
 321         return 0;
 322 }
 323
 324 static int bnxt_alloc_cos_queues(struct bnxt *bp)
 325 {
 326         bp->rx_cos_queue =
 327                 rte_zmalloc("bnxt_rx_cosq",
 328                             BNXT_COS_QUEUE_COUNT *
 329                             sizeof(struct bnxt_cos_queue_info),
 330                             0);
 331         if (bp->rx_cos_queue == NULL)
 332                 return -ENOMEM;
 333
 334         bp->tx_cos_queue =
 335                 rte_zmalloc("bnxt_tx_cosq",
 336                             BNXT_COS_QUEUE_COUNT *
 337                             sizeof(struct bnxt_cos_queue_info),
 338                             0);
 339         if (bp->tx_cos_queue == NULL)
 340                 return -ENOMEM;
 341
 342         return 0;
 343 }
 344
 345 static int bnxt_alloc_flow_stats_info(struct bnxt *bp)
 346 {
 347         bp->flow_stat = rte_zmalloc("bnxt_flow_xstat",
 348                                     sizeof(struct bnxt_flow_stat_info), 0);
 349         if (bp->flow_stat == NULL)
 350                 return -ENOMEM;
 351
 352         return 0;
 353 }
 354
 355 static int bnxt_alloc_mem(struct bnxt *bp, bool reconfig)
 356 {
 357         int rc;
 358
 359         rc = bnxt_alloc_ring_grps(bp);
 360         if (rc)
 361                 goto alloc_mem_err;
 362
 363         rc = bnxt_alloc_async_ring_struct(bp);
 364         if (rc)
 365                 goto alloc_mem_err;
 366
 367         rc = bnxt_alloc_vnic_mem(bp);
 368         if (rc)
 369                 goto alloc_mem_err;
 370
 371         rc = bnxt_alloc_vnic_attributes(bp, reconfig);
 372         if (rc)
 373                 goto alloc_mem_err;
 374
 375         rc = bnxt_alloc_filter_mem(bp);
 376         if (rc)
 377                 goto alloc_mem_err;
 378
 379         rc = bnxt_alloc_async_cp_ring(bp);
 380         if (rc)
 381                 goto alloc_mem_err;
 382
 383         rc = bnxt_alloc_rxtx_nq_ring(bp);
 384         if (rc)
 385                 goto alloc_mem_err;
 386
 387         if (BNXT_FLOW_XSTATS_EN(bp)) {
 388                 rc = bnxt_alloc_flow_stats_info(bp);
 389                 if (rc)
 390                         goto alloc_mem_err;
 391         }
 392
 393         return 0;
 394
 395 alloc_mem_err:
 396         bnxt_free_mem(bp, reconfig);
 397         return rc;
 398 }
 399
 400 static int bnxt_setup_one_vnic(struct bnxt *bp, uint16_t vnic_id)
 401 {
 402         struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
 403         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
 404         uint64_t rx_offloads = dev_conf->rxmode.offloads;
 405         struct bnxt_rx_queue *rxq;
 406         unsigned int j;
 407         int rc;
 408
 409         rc = bnxt_vnic_grp_alloc(bp, vnic);
 410         if (rc)
 411                 goto err_out;
 412
 413         PMD_DRV_LOG(DEBUG, "vnic[%d] = %p vnic->fw_grp_ids = %p\n",
 414                     vnic_id, vnic, vnic->fw_grp_ids);
 415
 416         rc = bnxt_hwrm_vnic_alloc(bp, vnic);
 417         if (rc)
 418                 goto err_out;
 419
 420         /* Alloc RSS context only if RSS mode is enabled */
 421         if (dev_conf->rxmode.mq_mode & RTE_ETH_MQ_RX_RSS) {
 422                 int j, nr_ctxs = bnxt_rss_ctxts(bp);
 423
 424                 /* RSS table size in Thor is 512.
 425                  * Cap max Rx rings to same value
 426                  */
 427                 if (bp->rx_nr_rings > BNXT_RSS_TBL_SIZE_P5) {
 428                         PMD_DRV_LOG(ERR, "RxQ cnt %d > reta_size %d\n",
 429                                     bp->rx_nr_rings, BNXT_RSS_TBL_SIZE_P5);
 430                         goto err_out;
 431                 }
 432
 433                 rc = 0;
 434                 for (j = 0; j < nr_ctxs; j++) {
 435                         rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic, j);
 436                         if (rc)
 437                                 break;
 438                 }
 439                 if (rc) {
 440                         PMD_DRV_LOG(ERR,
 441                                     "HWRM vnic %d ctx %d alloc failure rc: %x\n",
 442                                     vnic_id, j, rc);
 443                         goto err_out;
 444                 }
 445                 vnic->num_lb_ctxts = nr_ctxs;
 446         }
 447
 448         /*
 449          * Firmware sets pf pair in default vnic cfg. If the VLAN strip
 450          * setting is not available at this time, it will not be
 451          * configured correctly in the CFA.
 452          */
 453         if (rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)
 454                 vnic->vlan_strip = true;
 455         else
 456                 vnic->vlan_strip = false;
 457
 458         rc = bnxt_hwrm_vnic_cfg(bp, vnic);
 459         if (rc)
 460                 goto err_out;
 461
 462         rc = bnxt_set_hwrm_vnic_filters(bp, vnic);
 463         if (rc)
 464                 goto err_out;
 465
 466         for (j = 0; j < bp->rx_num_qs_per_vnic; j++) {
 467                 rxq = bp->eth_dev->data->rx_queues[j];
 468
 469                 PMD_DRV_LOG(DEBUG,
 470                             "rxq[%d]->vnic=%p vnic->fw_grp_ids=%p\n",
 471                             j, rxq->vnic, rxq->vnic->fw_grp_ids);
 472
 473                 if (BNXT_HAS_RING_GRPS(bp) && rxq->rx_deferred_start)
 474                         rxq->vnic->fw_grp_ids[j] = INVALID_HW_RING_ID;
 475                 else
 476                         vnic->rx_queue_cnt++;
 477         }
 478
 479         PMD_DRV_LOG(DEBUG, "vnic->rx_queue_cnt = %d\n", vnic->rx_queue_cnt);
 480
 481         rc = bnxt_vnic_rss_configure(bp, vnic);
 482         if (rc)
 483                 goto err_out;
 484
 485         bnxt_hwrm_vnic_plcmode_cfg(bp, vnic);
 486
 487         rc = bnxt_hwrm_vnic_tpa_cfg(bp, vnic,
 488                                     (rx_offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO) ?
 489                                     true : false);
 490         if (rc)
 491                 goto err_out;
 492
 493         return 0;
 494 err_out:
 495         PMD_DRV_LOG(ERR, "HWRM vnic %d cfg failure rc: %x\n",
 496                     vnic_id, rc);
 497         return rc;
 498 }
 499
 500 static int bnxt_register_fc_ctx_mem(struct bnxt *bp)
 501 {
 502         int rc = 0;
 503
 504         rc = bnxt_hwrm_ctx_rgtr(bp, bp->flow_stat->rx_fc_in_tbl.dma,
 505                                 &bp->flow_stat->rx_fc_in_tbl.ctx_id);
 506         if (rc)
 507                 return rc;
 508
 509         PMD_DRV_LOG(DEBUG,
 510                     "rx_fc_in_tbl.va = %p rx_fc_in_tbl.dma = %p"
 511                     " rx_fc_in_tbl.ctx_id = %d\n",
 512                     bp->flow_stat->rx_fc_in_tbl.va,
 513                     (void *)((uintptr_t)bp->flow_stat->rx_fc_in_tbl.dma),
 514                     bp->flow_stat->rx_fc_in_tbl.ctx_id);
 515
 516         rc = bnxt_hwrm_ctx_rgtr(bp, bp->flow_stat->rx_fc_out_tbl.dma,
 517                                 &bp->flow_stat->rx_fc_out_tbl.ctx_id);
 518         if (rc)
 519                 return rc;
 520
 521         PMD_DRV_LOG(DEBUG,
 522                     "rx_fc_out_tbl.va = %p rx_fc_out_tbl.dma = %p"
 523                     " rx_fc_out_tbl.ctx_id = %d\n",
 524                     bp->flow_stat->rx_fc_out_tbl.va,
 525                     (void *)((uintptr_t)bp->flow_stat->rx_fc_out_tbl.dma),
 526                     bp->flow_stat->rx_fc_out_tbl.ctx_id);
 527
 528         rc = bnxt_hwrm_ctx_rgtr(bp, bp->flow_stat->tx_fc_in_tbl.dma,
 529                                 &bp->flow_stat->tx_fc_in_tbl.ctx_id);
 530         if (rc)
 531                 return rc;
 532
 533         PMD_DRV_LOG(DEBUG,
 534                     "tx_fc_in_tbl.va = %p tx_fc_in_tbl.dma = %p"
 535                     " tx_fc_in_tbl.ctx_id = %d\n",
 536                     bp->flow_stat->tx_fc_in_tbl.va,
 537                     (void *)((uintptr_t)bp->flow_stat->tx_fc_in_tbl.dma),
 538                     bp->flow_stat->tx_fc_in_tbl.ctx_id);
 539
 540         rc = bnxt_hwrm_ctx_rgtr(bp, bp->flow_stat->tx_fc_out_tbl.dma,
 541                                 &bp->flow_stat->tx_fc_out_tbl.ctx_id);
 542         if (rc)
 543                 return rc;
 544
 545         PMD_DRV_LOG(DEBUG,
 546                     "tx_fc_out_tbl.va = %p tx_fc_out_tbl.dma = %p"
 547                     " tx_fc_out_tbl.ctx_id = %d\n",
 548                     bp->flow_stat->tx_fc_out_tbl.va,
 549                     (void *)((uintptr_t)bp->flow_stat->tx_fc_out_tbl.dma),
 550                     bp->flow_stat->tx_fc_out_tbl.ctx_id);
 551
 552         memset(bp->flow_stat->rx_fc_out_tbl.va,
 553                0,
 554                bp->flow_stat->rx_fc_out_tbl.size);
 555         rc = bnxt_hwrm_cfa_counter_cfg(bp, BNXT_DIR_RX,
 556                                        CFA_COUNTER_CFG_IN_COUNTER_TYPE_FC,
 557                                        bp->flow_stat->rx_fc_out_tbl.ctx_id,
 558                                        bp->flow_stat->max_fc,
 559                                        true);
 560         if (rc)
 561                 return rc;
 562
 563         memset(bp->flow_stat->tx_fc_out_tbl.va,
 564                0,
 565                bp->flow_stat->tx_fc_out_tbl.size);
 566         rc = bnxt_hwrm_cfa_counter_cfg(bp, BNXT_DIR_TX,
 567                                        CFA_COUNTER_CFG_IN_COUNTER_TYPE_FC,
 568                                        bp->flow_stat->tx_fc_out_tbl.ctx_id,
 569                                        bp->flow_stat->max_fc,
 570                                        true);
 571
 572         return rc;
 573 }
 574
 575 static int bnxt_alloc_ctx_mem_buf(struct bnxt *bp, char *type, size_t size,
 576                                   struct bnxt_ctx_mem_buf_info *ctx)
 577 {
 578         if (!ctx)
 579                 return -EINVAL;
 580
 581         ctx->va = rte_zmalloc_socket(type, size, 0,
 582                                      bp->eth_dev->device->numa_node);
 583         if (ctx->va == NULL)
 584                 return -ENOMEM;
 585         rte_mem_lock_page(ctx->va);
 586         ctx->size = size;
 587         ctx->dma = rte_mem_virt2iova(ctx->va);
 588         if (ctx->dma == RTE_BAD_IOVA)
 589                 return -ENOMEM;
 590
 591         return 0;
 592 }
 593
 594 static int bnxt_init_fc_ctx_mem(struct bnxt *bp)
 595 {
 596         struct rte_pci_device *pdev = bp->pdev;
 597         char type[RTE_MEMZONE_NAMESIZE];
 598         uint16_t max_fc;
 599         int rc = 0;
 600
 601         max_fc = bp->flow_stat->max_fc;
 602
 603         sprintf(type, "bnxt_rx_fc_in_" PCI_PRI_FMT, pdev->addr.domain,
 604                 pdev->addr.bus, pdev->addr.devid, pdev->addr.function);
 605         /* 4 bytes for each counter-id */
 606         rc = bnxt_alloc_ctx_mem_buf(bp, type,
 607                                     max_fc * 4,
 608                                     &bp->flow_stat->rx_fc_in_tbl);
 609         if (rc)
 610                 return rc;
 611
 612         sprintf(type, "bnxt_rx_fc_out_" PCI_PRI_FMT, pdev->addr.domain,
 613                 pdev->addr.bus, pdev->addr.devid, pdev->addr.function);
 614         /* 16 bytes for each counter - 8 bytes pkt_count, 8 bytes byte_count */
 615         rc = bnxt_alloc_ctx_mem_buf(bp, type,
 616                                     max_fc * 16,
 617                                     &bp->flow_stat->rx_fc_out_tbl);
 618         if (rc)
 619                 return rc;
 620
 621         sprintf(type, "bnxt_tx_fc_in_" PCI_PRI_FMT, pdev->addr.domain,
 622                 pdev->addr.bus, pdev->addr.devid, pdev->addr.function);
 623         /* 4 bytes for each counter-id */
 624         rc = bnxt_alloc_ctx_mem_buf(bp, type,
 625                                     max_fc * 4,
 626                                     &bp->flow_stat->tx_fc_in_tbl);
 627         if (rc)
 628                 return rc;
 629
 630         sprintf(type, "bnxt_tx_fc_out_" PCI_PRI_FMT, pdev->addr.domain,
 631                 pdev->addr.bus, pdev->addr.devid, pdev->addr.function);
 632         /* 16 bytes for each counter - 8 bytes pkt_count, 8 bytes byte_count */
 633         rc = bnxt_alloc_ctx_mem_buf(bp, type,
 634                                     max_fc * 16,
 635                                     &bp->flow_stat->tx_fc_out_tbl);
 636         if (rc)
 637                 return rc;
 638
 639         rc = bnxt_register_fc_ctx_mem(bp);
 640
 641         return rc;
 642 }
 643
 644 static int bnxt_init_ctx_mem(struct bnxt *bp)
 645 {
 646         int rc = 0;
 647
 648         if (!(bp->fw_cap & BNXT_FW_CAP_ADV_FLOW_COUNTERS) ||
 649             !(BNXT_PF(bp) || BNXT_VF_IS_TRUSTED(bp)) ||
 650             !BNXT_FLOW_XSTATS_EN(bp))
 651                 return 0;
 652
 653         rc = bnxt_hwrm_cfa_counter_qcaps(bp, &bp->flow_stat->max_fc);
 654         if (rc)
 655                 return rc;
 656
 657         rc = bnxt_init_fc_ctx_mem(bp);
 658
 659         return rc;
 660 }
 661
 662 static int bnxt_update_phy_setting(struct bnxt *bp)
 663 {
 664         struct rte_eth_link new;
 665         int rc;
 666
 667         rc = bnxt_get_hwrm_link_config(bp, &new);
 668         if (rc) {
 669                 PMD_DRV_LOG(ERR, "Failed to get link settings\n");
 670                 return rc;
 671         }
 672
 673         /*
 674          * On BCM957508-N2100 adapters, FW will not allow any user other
 675          * than BMC to shutdown the port. bnxt_get_hwrm_link_config() call
 676          * always returns link up. Force phy update always in that case.
 677          */
 678         if (!new.link_status || IS_BNXT_DEV_957508_N2100(bp)) {
 679                 rc = bnxt_set_hwrm_link_config(bp, true);
 680                 if (rc) {
 681                         PMD_DRV_LOG(ERR, "Failed to update PHY settings\n");
 682                         return rc;
 683                 }
 684         }
 685
 686         return rc;
 687 }
 688
 689 static void bnxt_free_prev_ring_stats(struct bnxt *bp)
 690 {
 691         rte_free(bp->prev_rx_ring_stats);
 692         rte_free(bp->prev_tx_ring_stats);
 693
 694         bp->prev_rx_ring_stats = NULL;
 695         bp->prev_tx_ring_stats = NULL;
 696 }
 697
 698 static int bnxt_alloc_prev_ring_stats(struct bnxt *bp)
 699 {
 700         bp->prev_rx_ring_stats =  rte_zmalloc("bnxt_prev_rx_ring_stats",
 701                                               sizeof(struct bnxt_ring_stats) *
 702                                               bp->rx_cp_nr_rings,
 703                                               0);
 704         if (bp->prev_rx_ring_stats == NULL)
 705                 return -ENOMEM;
 706
 707         bp->prev_tx_ring_stats = rte_zmalloc("bnxt_prev_tx_ring_stats",
 708                                              sizeof(struct bnxt_ring_stats) *
 709                                              bp->tx_cp_nr_rings,
 710                                              0);
 711         if (bp->prev_tx_ring_stats == NULL)
 712                 goto error;
 713
 714         return 0;
 715
 716 error:
 717         bnxt_free_prev_ring_stats(bp);
 718         return -ENOMEM;
 719 }
 720
 721 static int bnxt_start_nic(struct bnxt *bp)
 722 {
 723         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(bp->eth_dev);
 724         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
 725         uint32_t intr_vector = 0;
 726         uint32_t queue_id, base = BNXT_MISC_VEC_ID;
 727         uint32_t vec = BNXT_MISC_VEC_ID;
 728         unsigned int i, j;
 729         int rc;
 730
 731         if (bp->eth_dev->data->mtu > RTE_ETHER_MTU)
 732                 bp->flags |= BNXT_FLAG_JUMBO;
 733         else
 734                 bp->flags &= ~BNXT_FLAG_JUMBO;
 735
 736         /* THOR does not support ring groups.
 737          * But we will use the array to save RSS context IDs.
 738          */
 739         if (BNXT_CHIP_P5(bp))
 740                 bp->max_ring_grps = BNXT_MAX_RSS_CTXTS_P5;
 741
 742         rc = bnxt_alloc_hwrm_rings(bp);
 743         if (rc) {
 744                 PMD_DRV_LOG(ERR, "HWRM ring alloc failure rc: %x\n", rc);
 745                 goto err_out;
 746         }
 747
 748         rc = bnxt_alloc_all_hwrm_ring_grps(bp);
 749         if (rc) {
 750                 PMD_DRV_LOG(ERR, "HWRM ring grp alloc failure: %x\n", rc);
 751                 goto err_out;
 752         }
 753
 754         if (!(bp->vnic_cap_flags & BNXT_VNIC_CAP_COS_CLASSIFY))
 755                 goto skip_cosq_cfg;
 756
 757         for (j = 0, i = 0; i < BNXT_COS_QUEUE_COUNT; i++) {
 758                 if (bp->rx_cos_queue[i].id != 0xff) {
 759                         struct bnxt_vnic_info *vnic = &bp->vnic_info[j++];
 760
 761                         if (!vnic) {
 762                                 PMD_DRV_LOG(ERR,
 763                                             "Num pools more than FW profile\n");
 764                                 rc = -EINVAL;
 765                                 goto err_out;
 766                         }
 767                         vnic->cos_queue_id = bp->rx_cos_queue[i].id;
 768                         bp->rx_cosq_cnt++;
 769                 }
 770         }
 771
 772 skip_cosq_cfg:
 773         rc = bnxt_mq_rx_configure(bp);
 774         if (rc) {
 775                 PMD_DRV_LOG(ERR, "MQ mode configure failure rc: %x\n", rc);
 776                 goto err_out;
 777         }
 778
 779         for (j = 0; j < bp->rx_nr_rings; j++) {
 780                 struct bnxt_rx_queue *rxq = bp->rx_queues[j];
 781
 782                 if (!rxq->rx_deferred_start) {
 783                         bp->eth_dev->data->rx_queue_state[j] =
 784                                 RTE_ETH_QUEUE_STATE_STARTED;
 785                         rxq->rx_started = true;
 786                 }
 787         }
 788
 789         /* VNIC configuration */
 790         for (i = 0; i < bp->nr_vnics; i++) {
 791                 rc = bnxt_setup_one_vnic(bp, i);
 792                 if (rc)
 793                         goto err_out;
 794         }
 795
 796         for (j = 0; j < bp->tx_nr_rings; j++) {
 797                 struct bnxt_tx_queue *txq = bp->tx_queues[j];
 798
 799                 if (!txq->tx_deferred_start) {
 800                         bp->eth_dev->data->tx_queue_state[j] =
 801                                 RTE_ETH_QUEUE_STATE_STARTED;
 802                         txq->tx_started = true;
 803                 }
 804         }
 805
 806         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, &bp->vnic_info[0], 0, NULL);
 807         if (rc) {
 808                 PMD_DRV_LOG(ERR,
 809                         "HWRM cfa l2 rx mask failure rc: %x\n", rc);
 810                 goto err_out;
 811         }
 812
 813         /* check and configure queue intr-vector mapping */
 814         if ((rte_intr_cap_multiple(intr_handle) ||
 815              !RTE_ETH_DEV_SRIOV(bp->eth_dev).active) &&
 816             bp->eth_dev->data->dev_conf.intr_conf.rxq != 0) {
 817                 intr_vector = bp->eth_dev->data->nb_rx_queues;
 818                 PMD_DRV_LOG(DEBUG, "intr_vector = %d\n", intr_vector);
 819                 if (intr_vector > bp->rx_cp_nr_rings) {
 820                         PMD_DRV_LOG(ERR, "At most %d intr queues supported",
 821                                         bp->rx_cp_nr_rings);
 822                         return -ENOTSUP;
 823                 }
 824                 rc = rte_intr_efd_enable(intr_handle, intr_vector);
 825                 if (rc)
 826                         return rc;
 827         }
 828
 829         if (rte_intr_dp_is_en(intr_handle)) {
 830                 if (rte_intr_vec_list_alloc(intr_handle, "intr_vec",
 831                                         bp->eth_dev->data->nb_rx_queues)) {
 832                         PMD_DRV_LOG(ERR, "Failed to allocate %d rx_queues"
 833                                 " intr_vec", bp->eth_dev->data->nb_rx_queues);
 834                         rc = -ENOMEM;
 835                         goto err_out;
 836                 }
 837                 PMD_DRV_LOG(DEBUG, "intr_handle->nb_efd = %d "
 838                             "intr_handle->max_intr = %d\n",
 839                             rte_intr_nb_efd_get(intr_handle),
 840                             rte_intr_max_intr_get(intr_handle));
 841                 for (queue_id = 0; queue_id < bp->eth_dev->data->nb_rx_queues;
 842                      queue_id++) {
 843                         rte_intr_vec_list_index_set(intr_handle,
 844                                         queue_id, vec + BNXT_RX_VEC_START);
 845                         if (vec < base + rte_intr_nb_efd_get(intr_handle)
 846                             - 1)
 847                                 vec++;
 848                 }
 849         }
 850
 851         /* enable uio/vfio intr/eventfd mapping */
 852         rc = rte_intr_enable(intr_handle);
 853 #ifndef RTE_EXEC_ENV_FREEBSD
 854         /* In FreeBSD OS, nic_uio driver does not support interrupts */
 855         if (rc)
 856                 goto err_out;
 857 #endif
 858
 859         rc = bnxt_update_phy_setting(bp);
 860         if (rc)
 861                 goto err_out;
 862
 863         bp->mark_table = rte_zmalloc("bnxt_mark_table", BNXT_MARK_TABLE_SZ, 0);
 864         if (!bp->mark_table)
 865                 PMD_DRV_LOG(ERR, "Allocation of mark table failed\n");
 866
 867         return 0;
 868
 869 err_out:
 870         /* Some of the error status returned by FW may not be from errno.h */
 871         if (rc > 0)
 872                 rc = -EIO;
 873
 874         return rc;
 875 }
 876
 877 static int bnxt_shutdown_nic(struct bnxt *bp)
 878 {
 879         bnxt_free_all_hwrm_resources(bp);
 880         bnxt_free_all_filters(bp);
 881         bnxt_free_all_vnics(bp);
 882         return 0;
 883 }
 884
 885 /*
 886  * Device configuration and status function
 887  */
 888
 889 uint32_t bnxt_get_speed_capabilities(struct bnxt *bp)
 890 {
 891         uint32_t link_speed = 0;
 892         uint32_t speed_capa = 0;
 893
 894         if (bp->link_info == NULL)
 895                 return 0;
 896
 897         link_speed = bp->link_info->support_speeds;
 898
 899         /* If PAM4 is configured, use PAM4 supported speed */
 900         if (link_speed == 0 && bp->link_info->support_pam4_speeds > 0)
 901                 link_speed = bp->link_info->support_pam4_speeds;
 902
 903         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_100MB)
 904                 speed_capa |= RTE_ETH_LINK_SPEED_100M;
 905         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100MBHD)
 906                 speed_capa |= RTE_ETH_LINK_SPEED_100M_HD;
 907         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_1GB)
 908                 speed_capa |= RTE_ETH_LINK_SPEED_1G;
 909         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_2_5GB)
 910                 speed_capa |= RTE_ETH_LINK_SPEED_2_5G;
 911         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10GB)
 912                 speed_capa |= RTE_ETH_LINK_SPEED_10G;
 913         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_20GB)
 914                 speed_capa |= RTE_ETH_LINK_SPEED_20G;
 915         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_25GB)
 916                 speed_capa |= RTE_ETH_LINK_SPEED_25G;
 917         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_40GB)
 918                 speed_capa |= RTE_ETH_LINK_SPEED_40G;
 919         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_50GB)
 920                 speed_capa |= RTE_ETH_LINK_SPEED_50G;
 921         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100GB)
 922                 speed_capa |= RTE_ETH_LINK_SPEED_100G;
 923         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_PAM4_SPEEDS_50G)
 924                 speed_capa |= RTE_ETH_LINK_SPEED_50G;
 925         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_PAM4_SPEEDS_100G)
 926                 speed_capa |= RTE_ETH_LINK_SPEED_100G;
 927         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_PAM4_SPEEDS_200G)
 928                 speed_capa |= RTE_ETH_LINK_SPEED_200G;
 929
 930         if (bp->link_info->auto_mode ==
 931             HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_MODE_NONE)
 932                 speed_capa |= RTE_ETH_LINK_SPEED_FIXED;
 933
 934         return speed_capa;
 935 }
 936
 937 static int bnxt_dev_info_get_op(struct rte_eth_dev *eth_dev,
 938                                 struct rte_eth_dev_info *dev_info)
 939 {
 940         struct rte_pci_device *pdev = RTE_DEV_TO_PCI(eth_dev->device);
 941         struct bnxt *bp = eth_dev->data->dev_private;
 942         uint16_t max_vnics, i, j, vpool, vrxq;
 943         unsigned int max_rx_rings;
 944         int rc;
 945
 946         rc = is_bnxt_in_error(bp);
 947         if (rc)
 948                 return rc;
 949
 950         /* MAC Specifics */
 951         dev_info->max_mac_addrs = RTE_MIN(bp->max_l2_ctx, RTE_ETH_NUM_RECEIVE_MAC_ADDR);
 952         dev_info->max_hash_mac_addrs = 0;
 953
 954         /* PF/VF specifics */
 955         if (BNXT_PF(bp))
 956                 dev_info->max_vfs = pdev->max_vfs;
 957
 958         max_rx_rings = bnxt_max_rings(bp);
 959         /* For the sake of symmetry, max_rx_queues = max_tx_queues */
 960         dev_info->max_rx_queues = max_rx_rings;
 961         dev_info->max_tx_queues = max_rx_rings;
 962         dev_info->reta_size = bnxt_rss_hash_tbl_size(bp);
 963         dev_info->hash_key_size = HW_HASH_KEY_SIZE;
 964         max_vnics = bp->max_vnics;
 965
 966         /* MTU specifics */
 967         dev_info->min_mtu = RTE_ETHER_MIN_MTU;
 968         dev_info->max_mtu = BNXT_MAX_MTU;
 969
 970         /* Fast path specifics */
 971         dev_info->min_rx_bufsize = 1;
 972         dev_info->max_rx_pktlen = BNXT_MAX_PKT_LEN;
 973
 974         dev_info->rx_offload_capa = BNXT_DEV_RX_OFFLOAD_SUPPORT;
 975         if (bp->flags & BNXT_FLAG_PTP_SUPPORTED)
 976                 dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_TIMESTAMP;
 977         if (bp->vnic_cap_flags & BNXT_VNIC_CAP_VLAN_RX_STRIP)
 978                 dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_VLAN_STRIP;
 979         dev_info->tx_queue_offload_capa = RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
 980         dev_info->tx_offload_capa = BNXT_DEV_TX_OFFLOAD_SUPPORT |
 981                                     dev_info->tx_queue_offload_capa;
 982         if (bp->fw_cap & BNXT_FW_CAP_VLAN_TX_INSERT)
 983                 dev_info->tx_offload_capa |= RTE_ETH_TX_OFFLOAD_VLAN_INSERT;
 984         dev_info->flow_type_rss_offloads = BNXT_ETH_RSS_SUPPORT;
 985
 986         dev_info->speed_capa = bnxt_get_speed_capabilities(bp);
 987         dev_info->dev_capa = RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP |
 988                              RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP;
 989         dev_info->dev_capa &= ~RTE_ETH_DEV_CAPA_FLOW_RULE_KEEP;
 990
 991         dev_info->default_rxconf = (struct rte_eth_rxconf) {
 992                 .rx_thresh = {
 993                         .pthresh = 8,
 994                         .hthresh = 8,
 995                         .wthresh = 0,
 996                 },
 997                 .rx_free_thresh = 32,
 998                 .rx_drop_en = BNXT_DEFAULT_RX_DROP_EN,
 999         };
1000
1001         dev_info->default_txconf = (struct rte_eth_txconf) {
1002                 .tx_thresh = {
1003                         .pthresh = 32,
1004                         .hthresh = 0,
1005                         .wthresh = 0,
1006                 },
1007                 .tx_free_thresh = 32,
1008                 .tx_rs_thresh = 32,
1009         };
1010         eth_dev->data->dev_conf.intr_conf.lsc = 1;
1011
1012         dev_info->rx_desc_lim.nb_min = BNXT_MIN_RING_DESC;
1013         dev_info->rx_desc_lim.nb_max = BNXT_MAX_RX_RING_DESC;
1014         dev_info->tx_desc_lim.nb_min = BNXT_MIN_RING_DESC;
1015         dev_info->tx_desc_lim.nb_max = BNXT_MAX_TX_RING_DESC;
1016
1017         if (BNXT_PF(bp) || BNXT_VF_IS_TRUSTED(bp)) {
1018                 dev_info->switch_info.name = eth_dev->device->name;
1019                 dev_info->switch_info.domain_id = bp->switch_domain_id;
1020                 dev_info->switch_info.port_id =
1021                                 BNXT_PF(bp) ? BNXT_SWITCH_PORT_ID_PF :
1022                                     BNXT_SWITCH_PORT_ID_TRUSTED_VF;
1023         }
1024
1025         /*
1026          * TODO: default_rxconf, default_txconf, rx_desc_lim, and tx_desc_lim
1027          *       need further investigation.
1028          */
1029
1030         /* VMDq resources */
1031         vpool = 64; /* RTE_ETH_64_POOLS */
1032         vrxq = 128; /* RTE_ETH_VMDQ_DCB_NUM_QUEUES */
1033         for (i = 0; i < 4; vpool >>= 1, i++) {
1034                 if (max_vnics > vpool) {
1035                         for (j = 0; j < 5; vrxq >>= 1, j++) {
1036                                 if (dev_info->max_rx_queues > vrxq) {
1037                                         if (vpool > vrxq)
1038                                                 vpool = vrxq;
1039                                         goto found;
1040                                 }
1041                         }
1042                         /* Not enough resources to support VMDq */
1043                         break;
1044                 }
1045         }
1046         /* Not enough resources to support VMDq */
1047         vpool = 0;
1048         vrxq = 0;
1049 found:
1050         dev_info->max_vmdq_pools = vpool;
1051         dev_info->vmdq_queue_num = vrxq;
1052
1053         dev_info->vmdq_pool_base = 0;
1054         dev_info->vmdq_queue_base = 0;
1055
1056         return 0;
1057 }
1058
1059 /* Configure the device based on the configuration provided */
1060 static int bnxt_dev_configure_op(struct rte_eth_dev *eth_dev)
1061 {
1062         struct bnxt *bp = eth_dev->data->dev_private;
1063         uint64_t rx_offloads = eth_dev->data->dev_conf.rxmode.offloads;
1064         struct rte_eth_rss_conf *rss_conf = &eth_dev->data->dev_conf.rx_adv_conf.rss_conf;
1065         int rc;
1066
1067         bp->rx_queues = (void *)eth_dev->data->rx_queues;
1068         bp->tx_queues = (void *)eth_dev->data->tx_queues;
1069         bp->tx_nr_rings = eth_dev->data->nb_tx_queues;
1070         bp->rx_nr_rings = eth_dev->data->nb_rx_queues;
1071
1072         rc = is_bnxt_in_error(bp);
1073         if (rc)
1074                 return rc;
1075
1076         if (BNXT_VF(bp) && (bp->flags & BNXT_FLAG_NEW_RM)) {
1077                 rc = bnxt_hwrm_check_vf_rings(bp);
1078                 if (rc) {
1079                         PMD_DRV_LOG(ERR, "HWRM insufficient resources\n");
1080                         return -ENOSPC;
1081                 }
1082
1083                 /* If a resource has already been allocated - in this case
1084                  * it is the async completion ring, free it. Reallocate it after
1085                  * resource reservation. This will ensure the resource counts
1086                  * are calculated correctly.
1087                  */
1088
1089                 pthread_mutex_lock(&bp->def_cp_lock);
1090
1091                 if (!BNXT_HAS_NQ(bp) && bp->async_cp_ring) {
1092                         bnxt_disable_int(bp);
1093                         bnxt_free_cp_ring(bp, bp->async_cp_ring);
1094                 }
1095
1096                 rc = bnxt_hwrm_func_reserve_vf_resc(bp, false);
1097                 if (rc) {
1098                         PMD_DRV_LOG(ERR, "HWRM resource alloc fail:%x\n", rc);
1099                         pthread_mutex_unlock(&bp->def_cp_lock);
1100                         return -ENOSPC;
1101                 }
1102
1103                 if (!BNXT_HAS_NQ(bp) && bp->async_cp_ring) {
1104                         rc = bnxt_alloc_async_cp_ring(bp);
1105                         if (rc) {
1106                                 pthread_mutex_unlock(&bp->def_cp_lock);
1107                                 return rc;
1108                         }
1109                         bnxt_enable_int(bp);
1110                 }
1111
1112                 pthread_mutex_unlock(&bp->def_cp_lock);
1113         }
1114
1115         /* Inherit new configurations */
1116         if (eth_dev->data->nb_rx_queues > bp->max_rx_rings ||
1117             eth_dev->data->nb_tx_queues > bp->max_tx_rings ||
1118             eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues
1119                 + BNXT_NUM_ASYNC_CPR(bp) > bp->max_cp_rings ||
1120             eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues >
1121             bp->max_stat_ctx)
1122                 goto resource_error;
1123
1124         if (BNXT_HAS_RING_GRPS(bp) &&
1125             (uint32_t)(eth_dev->data->nb_rx_queues) > bp->max_ring_grps)
1126                 goto resource_error;
1127
1128         if (!(eth_dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS) &&
1129             bp->max_vnics < eth_dev->data->nb_rx_queues)
1130                 goto resource_error;
1131
1132         bp->rx_cp_nr_rings = bp->rx_nr_rings;
1133         bp->tx_cp_nr_rings = bp->tx_nr_rings;
1134
1135         if (eth_dev->data->dev_conf.rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG)
1136                 rx_offloads |= RTE_ETH_RX_OFFLOAD_RSS_HASH;
1137         eth_dev->data->dev_conf.rxmode.offloads = rx_offloads;
1138
1139         /* application provides the hash key to program */
1140         if (rss_conf->rss_key != NULL) {
1141                 if (rss_conf->rss_key_len != HW_HASH_KEY_SIZE)
1142                         PMD_DRV_LOG(WARNING, "port %u RSS key len must be %d bytes long",
1143                                     eth_dev->data->port_id, HW_HASH_KEY_SIZE);
1144                 else
1145                         memcpy(bp->rss_conf.rss_key, rss_conf->rss_key, HW_HASH_KEY_SIZE);
1146         }
1147         bp->rss_conf.rss_key_len = HW_HASH_KEY_SIZE;
1148         bp->rss_conf.rss_hf = rss_conf->rss_hf;
1149
1150         bnxt_mtu_set_op(eth_dev, eth_dev->data->mtu);
1151
1152         return 0;
1153
1154 resource_error:
1155         PMD_DRV_LOG(ERR,
1156                     "Insufficient resources to support requested config\n");
1157         PMD_DRV_LOG(ERR,
1158                     "Num Queues Requested: Tx %d, Rx %d\n",
1159                     eth_dev->data->nb_tx_queues,
1160                     eth_dev->data->nb_rx_queues);
1161         PMD_DRV_LOG(ERR,
1162                     "MAX: TxQ %d, RxQ %d, CQ %d Stat %d, Grp %d, Vnic %d\n",
1163                     bp->max_tx_rings, bp->max_rx_rings, bp->max_cp_rings,
1164                     bp->max_stat_ctx, bp->max_ring_grps, bp->max_vnics);
1165         return -ENOSPC;
1166 }
1167
1168 void bnxt_print_link_info(struct rte_eth_dev *eth_dev)
1169 {
1170         struct rte_eth_link *link = &eth_dev->data->dev_link;
1171
1172         if (link->link_status)
1173                 PMD_DRV_LOG(DEBUG, "Port %d Link Up - speed %u Mbps - %s\n",
1174                         eth_dev->data->port_id,
1175                         (uint32_t)link->link_speed,
1176                         (link->link_duplex == RTE_ETH_LINK_FULL_DUPLEX) ?
1177                         ("full-duplex") : ("half-duplex\n"));
1178         else
1179                 PMD_DRV_LOG(INFO, "Port %d Link Down\n",
1180                         eth_dev->data->port_id);
1181 }
1182
1183 /*
1184  * Determine whether the current configuration requires support for scattered
1185  * receive; return 1 if scattered receive is required and 0 if not.
1186  */
1187 static int bnxt_scattered_rx(struct rte_eth_dev *eth_dev)
1188 {
1189         uint32_t overhead = BNXT_MAX_PKT_LEN - BNXT_MAX_MTU;
1190         uint16_t buf_size;
1191         int i;
1192
1193         if (eth_dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_SCATTER)
1194                 return 1;
1195
1196         if (eth_dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO)
1197                 return 1;
1198
1199         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
1200                 struct bnxt_rx_queue *rxq = eth_dev->data->rx_queues[i];
1201
1202                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
1203                                       RTE_PKTMBUF_HEADROOM);
1204                 if (eth_dev->data->mtu + overhead > buf_size)
1205                         return 1;
1206         }
1207         return 0;
1208 }
1209
1210 static eth_rx_burst_t
1211 bnxt_receive_function(struct rte_eth_dev *eth_dev)
1212 {
1213         struct bnxt *bp = eth_dev->data->dev_private;
1214
1215         /* Disable vector mode RX for Stingray2 for now */
1216         if (BNXT_CHIP_SR2(bp)) {
1217                 bp->flags &= ~BNXT_FLAG_RX_VECTOR_PKT_MODE;
1218                 return bnxt_recv_pkts;
1219         }
1220
1221 #if (defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)) && \
1222         !defined(RTE_LIBRTE_IEEE1588)
1223
1224         /* Vector mode receive cannot be enabled if scattered rx is in use. */
1225         if (eth_dev->data->scattered_rx)
1226                 goto use_scalar_rx;
1227
1228         /*
1229          * Vector mode receive cannot be enabled if Truflow is enabled or if
1230          * asynchronous completions and receive completions can be placed in
1231          * the same completion ring.
1232          */
1233         if (BNXT_TRUFLOW_EN(bp) || !BNXT_NUM_ASYNC_CPR(bp))
1234                 goto use_scalar_rx;
1235
1236         /*
1237          * Vector mode receive cannot be enabled if any receive offloads outside
1238          * a limited subset have been enabled.
1239          */
1240         if (eth_dev->data->dev_conf.rxmode.offloads &
1241                 ~(RTE_ETH_RX_OFFLOAD_VLAN_STRIP |
1242                   RTE_ETH_RX_OFFLOAD_KEEP_CRC |
1243                   RTE_ETH_RX_OFFLOAD_IPV4_CKSUM |
1244                   RTE_ETH_RX_OFFLOAD_UDP_CKSUM |
1245                   RTE_ETH_RX_OFFLOAD_TCP_CKSUM |
1246                   RTE_ETH_RX_OFFLOAD_OUTER_IPV4_CKSUM |
1247                   RTE_ETH_RX_OFFLOAD_OUTER_UDP_CKSUM |
1248                   RTE_ETH_RX_OFFLOAD_RSS_HASH |
1249                   RTE_ETH_RX_OFFLOAD_VLAN_FILTER))
1250                 goto use_scalar_rx;
1251
1252 #if defined(RTE_ARCH_X86) && defined(CC_AVX2_SUPPORT)
1253         if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256 &&
1254             rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1) {
1255                 PMD_DRV_LOG(INFO,
1256                             "Using AVX2 vector mode receive for port %d\n",
1257                             eth_dev->data->port_id);
1258                 bp->flags |= BNXT_FLAG_RX_VECTOR_PKT_MODE;
1259                 return bnxt_recv_pkts_vec_avx2;
1260         }
1261  #endif
1262         if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
1263                 PMD_DRV_LOG(INFO,
1264                             "Using SSE vector mode receive for port %d\n",
1265                             eth_dev->data->port_id);
1266                 bp->flags |= BNXT_FLAG_RX_VECTOR_PKT_MODE;
1267                 return bnxt_recv_pkts_vec;
1268         }
1269
1270 use_scalar_rx:
1271         PMD_DRV_LOG(INFO, "Vector mode receive disabled for port %d\n",
1272                     eth_dev->data->port_id);
1273         PMD_DRV_LOG(INFO,
1274                     "Port %d scatter: %d rx offload: %" PRIX64 "\n",
1275                     eth_dev->data->port_id,
1276                     eth_dev->data->scattered_rx,
1277                     eth_dev->data->dev_conf.rxmode.offloads);
1278 #endif
1279         bp->flags &= ~BNXT_FLAG_RX_VECTOR_PKT_MODE;
1280         return bnxt_recv_pkts;
1281 }
1282
1283 static eth_tx_burst_t
1284 bnxt_transmit_function(struct rte_eth_dev *eth_dev)
1285 {
1286         struct bnxt *bp = eth_dev->data->dev_private;
1287
1288         /* Disable vector mode TX for Stingray2 for now */
1289         if (BNXT_CHIP_SR2(bp))
1290                 return bnxt_xmit_pkts;
1291
1292 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64) && \
1293         !defined(RTE_LIBRTE_IEEE1588)
1294         uint64_t offloads = eth_dev->data->dev_conf.txmode.offloads;
1295
1296         /*
1297          * Vector mode transmit can be enabled only if not using scatter rx
1298          * or tx offloads.
1299          */
1300         if (eth_dev->data->scattered_rx ||
1301             (offloads & ~RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) ||
1302             BNXT_TRUFLOW_EN(bp))
1303                 goto use_scalar_tx;
1304
1305 #if defined(RTE_ARCH_X86) && defined(CC_AVX2_SUPPORT)
1306         if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256 &&
1307             rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) == 1) {
1308                 PMD_DRV_LOG(INFO,
1309                             "Using AVX2 vector mode transmit for port %d\n",
1310                             eth_dev->data->port_id);
1311                 return bnxt_xmit_pkts_vec_avx2;
1312         }
1313 #endif
1314         if (rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) {
1315                 PMD_DRV_LOG(INFO,
1316                             "Using SSE vector mode transmit for port %d\n",
1317                             eth_dev->data->port_id);
1318                 return bnxt_xmit_pkts_vec;
1319         }
1320
1321 use_scalar_tx:
1322         PMD_DRV_LOG(INFO, "Vector mode transmit disabled for port %d\n",
1323                     eth_dev->data->port_id);
1324         PMD_DRV_LOG(INFO,
1325                     "Port %d scatter: %d tx offload: %" PRIX64 "\n",
1326                     eth_dev->data->port_id,
1327                     eth_dev->data->scattered_rx,
1328                     offloads);
1329 #endif
1330         return bnxt_xmit_pkts;
1331 }
1332
1333 static int bnxt_handle_if_change_status(struct bnxt *bp)
1334 {
1335         int rc;
1336
1337         /* Since fw has undergone a reset and lost all contexts,
1338          * set fatal flag to not issue hwrm during cleanup
1339          */
1340         bp->flags |= BNXT_FLAG_FATAL_ERROR;
1341         bnxt_uninit_resources(bp, true);
1342
1343         /* clear fatal flag so that re-init happens */
1344         bp->flags &= ~BNXT_FLAG_FATAL_ERROR;
1345         rc = bnxt_init_resources(bp, true);
1346
1347         bp->flags &= ~BNXT_FLAG_IF_CHANGE_HOT_FW_RESET_DONE;
1348
1349         return rc;
1350 }
1351
1352 static int bnxt_dev_set_link_up_op(struct rte_eth_dev *eth_dev)
1353 {
1354         struct bnxt *bp = eth_dev->data->dev_private;
1355         int rc = 0;
1356
1357         if (!BNXT_SINGLE_PF(bp))
1358                 return -ENOTSUP;
1359
1360         if (!bp->link_info->link_up)
1361                 rc = bnxt_set_hwrm_link_config(bp, true);
1362         if (!rc)
1363                 eth_dev->data->dev_link.link_status = 1;
1364
1365         bnxt_print_link_info(eth_dev);
1366         return rc;
1367 }
1368
1369 static int bnxt_dev_set_link_down_op(struct rte_eth_dev *eth_dev)
1370 {
1371         struct bnxt *bp = eth_dev->data->dev_private;
1372
1373         if (!BNXT_SINGLE_PF(bp))
1374                 return -ENOTSUP;
1375
1376         eth_dev->data->dev_link.link_status = 0;
1377         bnxt_set_hwrm_link_config(bp, false);
1378         bp->link_info->link_up = 0;
1379
1380         return 0;
1381 }
1382
1383 static void bnxt_free_switch_domain(struct bnxt *bp)
1384 {
1385         int rc = 0;
1386
1387         if (!(BNXT_PF(bp) || BNXT_VF_IS_TRUSTED(bp)))
1388                 return;
1389
1390         rc = rte_eth_switch_domain_free(bp->switch_domain_id);
1391         if (rc)
1392                 PMD_DRV_LOG(ERR, "free switch domain:%d fail: %d\n",
1393                             bp->switch_domain_id, rc);
1394 }
1395
1396 static void bnxt_ptp_get_current_time(void *arg)
1397 {
1398         struct bnxt *bp = arg;
1399         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
1400         int rc;
1401
1402         rc = is_bnxt_in_error(bp);
1403         if (rc)
1404                 return;
1405
1406         if (!ptp)
1407                 return;
1408
1409         bnxt_hwrm_port_ts_query(bp, BNXT_PTP_FLAGS_CURRENT_TIME,
1410                                 &ptp->current_time);
1411
1412         rc = rte_eal_alarm_set(US_PER_S, bnxt_ptp_get_current_time, (void *)bp);
1413         if (rc != 0) {
1414                 PMD_DRV_LOG(ERR, "Failed to re-schedule PTP alarm\n");
1415                 bp->flags2 &= ~BNXT_FLAGS2_PTP_ALARM_SCHEDULED;
1416         }
1417 }
1418
1419 static int bnxt_schedule_ptp_alarm(struct bnxt *bp)
1420 {
1421         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
1422         int rc;
1423
1424         if (bp->flags2 & BNXT_FLAGS2_PTP_ALARM_SCHEDULED)
1425                 return 0;
1426
1427         bnxt_hwrm_port_ts_query(bp, BNXT_PTP_FLAGS_CURRENT_TIME,
1428                                 &ptp->current_time);
1429
1430         rc = rte_eal_alarm_set(US_PER_S, bnxt_ptp_get_current_time, (void *)bp);
1431         return rc;
1432 }
1433
1434 static void bnxt_cancel_ptp_alarm(struct bnxt *bp)
1435 {
1436         if (bp->flags2 & BNXT_FLAGS2_PTP_ALARM_SCHEDULED) {
1437                 rte_eal_alarm_cancel(bnxt_ptp_get_current_time, (void *)bp);
1438                 bp->flags2 &= ~BNXT_FLAGS2_PTP_ALARM_SCHEDULED;
1439         }
1440 }
1441
1442 static void bnxt_ptp_stop(struct bnxt *bp)
1443 {
1444         bnxt_cancel_ptp_alarm(bp);
1445         bp->flags2 &= ~BNXT_FLAGS2_PTP_TIMESYNC_ENABLED;
1446 }
1447
1448 static int bnxt_ptp_start(struct bnxt *bp)
1449 {
1450         int rc;
1451
1452         rc = bnxt_schedule_ptp_alarm(bp);
1453         if (rc != 0) {
1454                 PMD_DRV_LOG(ERR, "Failed to schedule PTP alarm\n");
1455         } else {
1456                 bp->flags2 |= BNXT_FLAGS2_PTP_TIMESYNC_ENABLED;
1457                 bp->flags2 |= BNXT_FLAGS2_PTP_ALARM_SCHEDULED;
1458         }
1459
1460         return rc;
1461 }
1462
1463 static int bnxt_dev_stop(struct rte_eth_dev *eth_dev)
1464 {
1465         struct bnxt *bp = eth_dev->data->dev_private;
1466         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
1467         struct rte_intr_handle *intr_handle = pci_dev->intr_handle;
1468         struct rte_eth_link link;
1469         int ret;
1470
1471         eth_dev->data->dev_started = 0;
1472
1473         /* Prevent crashes when queues are still in use */
1474         bnxt_stop_rxtx(eth_dev);
1475
1476         bnxt_disable_int(bp);
1477
1478         /* disable uio/vfio intr/eventfd mapping */
1479         rte_intr_disable(intr_handle);
1480
1481         /* Stop the child representors for this device */
1482         ret = bnxt_rep_stop_all(bp);
1483         if (ret != 0)
1484                 return ret;
1485
1486         /* delete the bnxt ULP port details */
1487         bnxt_ulp_port_deinit(bp);
1488
1489         bnxt_cancel_fw_health_check(bp);
1490
1491         if (BNXT_P5_PTP_TIMESYNC_ENABLED(bp))
1492                 bnxt_cancel_ptp_alarm(bp);
1493
1494         /* Do not bring link down during reset recovery */
1495         if (!is_bnxt_in_error(bp)) {
1496                 bnxt_dev_set_link_down_op(eth_dev);
1497                 /* Wait for link to be reset */
1498                 if (BNXT_SINGLE_PF(bp))
1499                         rte_delay_ms(500);
1500                 /* clear the recorded link status */
1501                 memset(&link, 0, sizeof(link));
1502                 rte_eth_linkstatus_set(eth_dev, &link);
1503         }
1504
1505         /* Clean queue intr-vector mapping */
1506         rte_intr_efd_disable(intr_handle);
1507         rte_intr_vec_list_free(intr_handle);
1508
1509         bnxt_hwrm_port_clr_stats(bp);
1510         bnxt_free_tx_mbufs(bp);
1511         bnxt_free_rx_mbufs(bp);
1512         /* Process any remaining notifications in default completion queue */
1513         bnxt_int_handler(eth_dev);
1514         bnxt_shutdown_nic(bp);
1515         bnxt_hwrm_if_change(bp, false);
1516
1517         bnxt_free_prev_ring_stats(bp);
1518         rte_free(bp->mark_table);
1519         bp->mark_table = NULL;
1520
1521         bp->flags &= ~BNXT_FLAG_RX_VECTOR_PKT_MODE;
1522         bp->rx_cosq_cnt = 0;
1523         /* All filters are deleted on a port stop. */
1524         if (BNXT_FLOW_XSTATS_EN(bp))
1525                 bp->flow_stat->flow_count = 0;
1526
1527         eth_dev->data->scattered_rx = 0;
1528
1529         return 0;
1530 }
1531
1532 /* Unload the driver, release resources */
1533 int bnxt_dev_stop_op(struct rte_eth_dev *eth_dev)
1534 {
1535         struct bnxt *bp = eth_dev->data->dev_private;
1536
1537         pthread_mutex_lock(&bp->err_recovery_lock);
1538         if (bp->flags & BNXT_FLAG_FW_RESET) {
1539                 PMD_DRV_LOG(ERR,
1540                             "Adapter recovering from error..Please retry\n");
1541                 pthread_mutex_unlock(&bp->err_recovery_lock);
1542                 return -EAGAIN;
1543         }
1544         pthread_mutex_unlock(&bp->err_recovery_lock);
1545
1546         return bnxt_dev_stop(eth_dev);
1547 }
1548
1549 int bnxt_dev_start_op(struct rte_eth_dev *eth_dev)
1550 {
1551         struct bnxt *bp = eth_dev->data->dev_private;
1552         uint64_t rx_offloads = eth_dev->data->dev_conf.rxmode.offloads;
1553         int vlan_mask = 0;
1554         int rc, retry_cnt = BNXT_IF_CHANGE_RETRY_COUNT;
1555
1556         if (!eth_dev->data->nb_tx_queues || !eth_dev->data->nb_rx_queues) {
1557                 PMD_DRV_LOG(ERR, "Queues are not configured yet!\n");
1558                 return -EINVAL;
1559         }
1560
1561         if (bp->rx_cp_nr_rings > RTE_ETHDEV_QUEUE_STAT_CNTRS)
1562                 PMD_DRV_LOG(ERR,
1563                             "RxQ cnt %d > RTE_ETHDEV_QUEUE_STAT_CNTRS %d\n",
1564                             bp->rx_cp_nr_rings, RTE_ETHDEV_QUEUE_STAT_CNTRS);
1565
1566         do {
1567                 rc = bnxt_hwrm_if_change(bp, true);
1568                 if (rc == 0 || rc != -EAGAIN)
1569                         break;
1570
1571                 rte_delay_ms(BNXT_IF_CHANGE_RETRY_INTERVAL);
1572         } while (retry_cnt--);
1573
1574         if (rc)
1575                 return rc;
1576
1577         if (bp->flags & BNXT_FLAG_IF_CHANGE_HOT_FW_RESET_DONE) {
1578                 rc = bnxt_handle_if_change_status(bp);
1579                 if (rc)
1580                         return rc;
1581         }
1582
1583         bnxt_enable_int(bp);
1584
1585         eth_dev->data->scattered_rx = bnxt_scattered_rx(eth_dev);
1586
1587         rc = bnxt_start_nic(bp);
1588         if (rc)
1589                 goto error;
1590
1591         rc = bnxt_alloc_prev_ring_stats(bp);
1592         if (rc)
1593                 goto error;
1594
1595         eth_dev->data->dev_started = 1;
1596
1597         bnxt_link_update_op(eth_dev, 1);
1598
1599         if (rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER)
1600                 vlan_mask |= RTE_ETH_VLAN_FILTER_MASK;
1601         if (rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP)
1602                 vlan_mask |= RTE_ETH_VLAN_STRIP_MASK;
1603         rc = bnxt_vlan_offload_set_op(eth_dev, vlan_mask);
1604         if (rc)
1605                 goto error;
1606
1607         /* Initialize bnxt ULP port details */
1608         rc = bnxt_ulp_port_init(bp);
1609         if (rc)
1610                 goto error;
1611
1612         eth_dev->rx_pkt_burst = bnxt_receive_function(eth_dev);
1613         eth_dev->tx_pkt_burst = bnxt_transmit_function(eth_dev);
1614
1615         bnxt_schedule_fw_health_check(bp);
1616
1617         if (BNXT_P5_PTP_TIMESYNC_ENABLED(bp))
1618                 bnxt_schedule_ptp_alarm(bp);
1619
1620         return 0;
1621
1622 error:
1623         bnxt_dev_stop(eth_dev);
1624         return rc;
1625 }
1626
1627 static void
1628 bnxt_uninit_locks(struct bnxt *bp)
1629 {
1630         pthread_mutex_destroy(&bp->flow_lock);
1631         pthread_mutex_destroy(&bp->def_cp_lock);
1632         pthread_mutex_destroy(&bp->health_check_lock);
1633         pthread_mutex_destroy(&bp->err_recovery_lock);
1634         if (bp->rep_info) {
1635                 pthread_mutex_destroy(&bp->rep_info->vfr_lock);
1636                 pthread_mutex_destroy(&bp->rep_info->vfr_start_lock);
1637         }
1638 }
1639
1640 static void bnxt_drv_uninit(struct bnxt *bp)
1641 {
1642         bnxt_free_leds_info(bp);
1643         bnxt_free_cos_queues(bp);
1644         bnxt_free_link_info(bp);
1645         bnxt_free_parent_info(bp);
1646         bnxt_uninit_locks(bp);
1647
1648         rte_memzone_free((const struct rte_memzone *)bp->tx_mem_zone);
1649         bp->tx_mem_zone = NULL;
1650         rte_memzone_free((const struct rte_memzone *)bp->rx_mem_zone);
1651         bp->rx_mem_zone = NULL;
1652
1653         bnxt_free_vf_info(bp);
1654         bnxt_free_pf_info(bp);
1655
1656         rte_free(bp->grp_info);
1657         bp->grp_info = NULL;
1658 }
1659
1660 static int bnxt_dev_close_op(struct rte_eth_dev *eth_dev)
1661 {
1662         struct bnxt *bp = eth_dev->data->dev_private;
1663         int ret = 0;
1664
1665         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1666                 return 0;
1667
1668         pthread_mutex_lock(&bp->err_recovery_lock);
1669         if (bp->flags & BNXT_FLAG_FW_RESET) {
1670                 PMD_DRV_LOG(ERR,
1671                             "Adapter recovering from error...Please retry\n");
1672                 pthread_mutex_unlock(&bp->err_recovery_lock);
1673                 return -EAGAIN;
1674         }
1675         pthread_mutex_unlock(&bp->err_recovery_lock);
1676
1677         /* cancel the recovery handler before remove dev */
1678         rte_eal_alarm_cancel(bnxt_dev_reset_and_resume, (void *)bp);
1679         rte_eal_alarm_cancel(bnxt_dev_recover, (void *)bp);
1680         bnxt_cancel_fc_thread(bp);
1681         rte_eal_alarm_cancel(bnxt_handle_vf_cfg_change, (void *)bp);
1682
1683         if (eth_dev->data->dev_started)
1684                 ret = bnxt_dev_stop(eth_dev);
1685
1686         bnxt_uninit_resources(bp, false);
1687
1688         bnxt_drv_uninit(bp);
1689
1690         return ret;
1691 }
1692
1693 static void bnxt_mac_addr_remove_op(struct rte_eth_dev *eth_dev,
1694                                     uint32_t index)
1695 {
1696         struct bnxt *bp = eth_dev->data->dev_private;
1697         uint64_t pool_mask = eth_dev->data->mac_pool_sel[index];
1698         struct bnxt_vnic_info *vnic;
1699         struct bnxt_filter_info *filter, *temp_filter;
1700         uint32_t i;
1701
1702         if (is_bnxt_in_error(bp))
1703                 return;
1704
1705         /*
1706          * Loop through all VNICs from the specified filter flow pools to
1707          * remove the corresponding MAC addr filter
1708          */
1709         for (i = 0; i < bp->nr_vnics; i++) {
1710                 if (!(pool_mask & (1ULL << i)))
1711                         continue;
1712
1713                 vnic = &bp->vnic_info[i];
1714                 filter = STAILQ_FIRST(&vnic->filter);
1715                 while (filter) {
1716                         temp_filter = STAILQ_NEXT(filter, next);
1717                         if (filter->mac_index == index) {
1718                                 STAILQ_REMOVE(&vnic->filter, filter,
1719                                                 bnxt_filter_info, next);
1720                                 bnxt_hwrm_clear_l2_filter(bp, filter);
1721                                 bnxt_free_filter(bp, filter);
1722                         }
1723                         filter = temp_filter;
1724                 }
1725         }
1726 }
1727
1728 static int bnxt_add_mac_filter(struct bnxt *bp, struct bnxt_vnic_info *vnic,
1729                                struct rte_ether_addr *mac_addr, uint32_t index,
1730                                uint32_t pool)
1731 {
1732         struct bnxt_filter_info *filter;
1733         int rc = 0;
1734
1735         /* Attach requested MAC address to the new l2_filter */
1736         STAILQ_FOREACH(filter, &vnic->filter, next) {
1737                 if (filter->mac_index == index) {
1738                         PMD_DRV_LOG(DEBUG,
1739                                     "MAC addr already existed for pool %d\n",
1740                                     pool);
1741                         return 0;
1742                 }
1743         }
1744
1745         filter = bnxt_alloc_filter(bp);
1746         if (!filter) {
1747                 PMD_DRV_LOG(ERR, "L2 filter alloc failed\n");
1748                 return -ENODEV;
1749         }
1750
1751         /* bnxt_alloc_filter copies default MAC to filter->l2_addr. So,
1752          * if the MAC that's been programmed now is a different one, then,
1753          * copy that addr to filter->l2_addr
1754          */
1755         if (mac_addr)
1756                 memcpy(filter->l2_addr, mac_addr, RTE_ETHER_ADDR_LEN);
1757         filter->flags |= HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_OUTERMOST;
1758
1759         rc = bnxt_hwrm_set_l2_filter(bp, vnic->fw_vnic_id, filter);
1760         if (!rc) {
1761                 filter->mac_index = index;
1762                 if (filter->mac_index == 0)
1763                         STAILQ_INSERT_HEAD(&vnic->filter, filter, next);
1764                 else
1765                         STAILQ_INSERT_TAIL(&vnic->filter, filter, next);
1766         } else {
1767                 bnxt_free_filter(bp, filter);
1768         }
1769
1770         return rc;
1771 }
1772
1773 static int bnxt_mac_addr_add_op(struct rte_eth_dev *eth_dev,
1774                                 struct rte_ether_addr *mac_addr,
1775                                 uint32_t index, uint32_t pool)
1776 {
1777         struct bnxt *bp = eth_dev->data->dev_private;
1778         struct bnxt_vnic_info *vnic = &bp->vnic_info[pool];
1779         int rc = 0;
1780
1781         rc = is_bnxt_in_error(bp);
1782         if (rc)
1783                 return rc;
1784
1785         if (BNXT_VF(bp) && !BNXT_VF_IS_TRUSTED(bp)) {
1786                 PMD_DRV_LOG(ERR, "Cannot add MAC address to a VF interface\n");
1787                 return -ENOTSUP;
1788         }
1789
1790         if (!vnic) {
1791                 PMD_DRV_LOG(ERR, "VNIC not found for pool %d!\n", pool);
1792                 return -EINVAL;
1793         }
1794
1795         /* Filter settings will get applied when port is started */
1796         if (!eth_dev->data->dev_started)
1797                 return 0;
1798
1799         rc = bnxt_add_mac_filter(bp, vnic, mac_addr, index, pool);
1800
1801         return rc;
1802 }
1803
1804 int bnxt_link_update_op(struct rte_eth_dev *eth_dev, int wait_to_complete)
1805 {
1806         int rc = 0;
1807         struct bnxt *bp = eth_dev->data->dev_private;
1808         struct rte_eth_link new;
1809         int cnt = wait_to_complete ? BNXT_MAX_LINK_WAIT_CNT :
1810                         BNXT_MIN_LINK_WAIT_CNT;
1811
1812         rc = is_bnxt_in_error(bp);
1813         if (rc)
1814                 return rc;
1815
1816         memset(&new, 0, sizeof(new));
1817
1818         if (bp->link_info == NULL)
1819                 goto out;
1820
1821         do {
1822                 /* Retrieve link info from hardware */
1823                 rc = bnxt_get_hwrm_link_config(bp, &new);
1824                 if (rc) {
1825                         new.link_speed = RTE_ETH_LINK_SPEED_100M;
1826                         new.link_duplex = RTE_ETH_LINK_FULL_DUPLEX;
1827                         PMD_DRV_LOG(ERR,
1828                                 "Failed to retrieve link rc = 0x%x!\n", rc);
1829                         goto out;
1830                 }
1831
1832                 if (!wait_to_complete || new.link_status)
1833                         break;
1834
1835                 rte_delay_ms(BNXT_LINK_WAIT_INTERVAL);
1836         } while (cnt--);
1837
1838         /* Only single function PF can bring phy down.
1839          * When port is stopped, report link down for VF/MH/NPAR functions.
1840          */
1841         if (!BNXT_SINGLE_PF(bp) && !eth_dev->data->dev_started)
1842                 memset(&new, 0, sizeof(new));
1843
1844 out:
1845         /* Timed out or success */
1846         if (new.link_status != eth_dev->data->dev_link.link_status ||
1847             new.link_speed != eth_dev->data->dev_link.link_speed) {
1848                 rte_eth_linkstatus_set(eth_dev, &new);
1849                 bnxt_print_link_info(eth_dev);
1850         }
1851
1852         return rc;
1853 }
1854
1855 static int bnxt_promiscuous_enable_op(struct rte_eth_dev *eth_dev)
1856 {
1857         struct bnxt *bp = eth_dev->data->dev_private;
1858         struct bnxt_vnic_info *vnic;
1859         uint32_t old_flags;
1860         int rc;
1861
1862         rc = is_bnxt_in_error(bp);
1863         if (rc)
1864                 return rc;
1865
1866         /* Filter settings will get applied when port is started */
1867         if (!eth_dev->data->dev_started)
1868                 return 0;
1869
1870         if (bp->vnic_info == NULL)
1871                 return 0;
1872
1873         vnic = BNXT_GET_DEFAULT_VNIC(bp);
1874
1875         old_flags = vnic->flags;
1876         vnic->flags |= BNXT_VNIC_INFO_PROMISC;
1877         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
1878         if (rc != 0)
1879                 vnic->flags = old_flags;
1880
1881         return rc;
1882 }
1883
1884 static int bnxt_promiscuous_disable_op(struct rte_eth_dev *eth_dev)
1885 {
1886         struct bnxt *bp = eth_dev->data->dev_private;
1887         struct bnxt_vnic_info *vnic;
1888         uint32_t old_flags;
1889         int rc;
1890
1891         rc = is_bnxt_in_error(bp);
1892         if (rc)
1893                 return rc;
1894
1895         /* Filter settings will get applied when port is started */
1896         if (!eth_dev->data->dev_started)
1897                 return 0;
1898
1899         if (bp->vnic_info == NULL)
1900                 return 0;
1901
1902         vnic = BNXT_GET_DEFAULT_VNIC(bp);
1903
1904         old_flags = vnic->flags;
1905         vnic->flags &= ~BNXT_VNIC_INFO_PROMISC;
1906         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
1907         if (rc != 0)
1908                 vnic->flags = old_flags;
1909
1910         return rc;
1911 }
1912
1913 static int bnxt_allmulticast_enable_op(struct rte_eth_dev *eth_dev)
1914 {
1915         struct bnxt *bp = eth_dev->data->dev_private;
1916         struct bnxt_vnic_info *vnic;
1917         uint32_t old_flags;
1918         int rc;
1919
1920         rc = is_bnxt_in_error(bp);
1921         if (rc)
1922                 return rc;
1923
1924         /* Filter settings will get applied when port is started */
1925         if (!eth_dev->data->dev_started)
1926                 return 0;
1927
1928         if (bp->vnic_info == NULL)
1929                 return 0;
1930
1931         vnic = BNXT_GET_DEFAULT_VNIC(bp);
1932
1933         old_flags = vnic->flags;
1934         vnic->flags |= BNXT_VNIC_INFO_ALLMULTI;
1935         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
1936         if (rc != 0)
1937                 vnic->flags = old_flags;
1938
1939         return rc;
1940 }
1941
1942 static int bnxt_allmulticast_disable_op(struct rte_eth_dev *eth_dev)
1943 {
1944         struct bnxt *bp = eth_dev->data->dev_private;
1945         struct bnxt_vnic_info *vnic;
1946         uint32_t old_flags;
1947         int rc;
1948
1949         rc = is_bnxt_in_error(bp);
1950         if (rc)
1951                 return rc;
1952
1953         /* Filter settings will get applied when port is started */
1954         if (!eth_dev->data->dev_started)
1955                 return 0;
1956
1957         if (bp->vnic_info == NULL)
1958                 return 0;
1959
1960         vnic = BNXT_GET_DEFAULT_VNIC(bp);
1961
1962         old_flags = vnic->flags;
1963         vnic->flags &= ~BNXT_VNIC_INFO_ALLMULTI;
1964         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
1965         if (rc != 0)
1966                 vnic->flags = old_flags;
1967
1968         return rc;
1969 }
1970
1971 /* Return bnxt_rx_queue pointer corresponding to a given rxq. */
1972 static struct bnxt_rx_queue *bnxt_qid_to_rxq(struct bnxt *bp, uint16_t qid)
1973 {
1974         if (qid >= bp->rx_nr_rings)
1975                 return NULL;
1976
1977         return bp->eth_dev->data->rx_queues[qid];
1978 }
1979
1980 /* Return rxq corresponding to a given rss table ring/group ID. */
1981 static uint16_t bnxt_rss_to_qid(struct bnxt *bp, uint16_t fwr)
1982 {
1983         struct bnxt_rx_queue *rxq;
1984         unsigned int i;
1985
1986         if (!BNXT_HAS_RING_GRPS(bp)) {
1987                 for (i = 0; i < bp->rx_nr_rings; i++) {
1988                         rxq = bp->eth_dev->data->rx_queues[i];
1989                         if (rxq->rx_ring->rx_ring_struct->fw_ring_id == fwr)
1990                                 return rxq->index;
1991                 }
1992         } else {
1993                 for (i = 0; i < bp->rx_nr_rings; i++) {
1994                         if (bp->grp_info[i].fw_grp_id == fwr)
1995                                 return i;
1996                 }
1997         }
1998
1999         return INVALID_HW_RING_ID;
2000 }
2001
2002 static int bnxt_reta_update_op(struct rte_eth_dev *eth_dev,
2003                             struct rte_eth_rss_reta_entry64 *reta_conf,
2004                             uint16_t reta_size)
2005 {
2006         struct bnxt *bp = eth_dev->data->dev_private;
2007         struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
2008         struct bnxt_vnic_info *vnic = BNXT_GET_DEFAULT_VNIC(bp);
2009         uint16_t tbl_size = bnxt_rss_hash_tbl_size(bp);
2010         uint16_t idx, sft;
2011         int i, rc;
2012
2013         rc = is_bnxt_in_error(bp);
2014         if (rc)
2015                 return rc;
2016
2017         if (!vnic->rss_table)
2018                 return -EINVAL;
2019
2020         if (!(dev_conf->rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG))
2021                 return -EINVAL;
2022
2023         if (reta_size != tbl_size) {
2024                 PMD_DRV_LOG(ERR, "The configured hash table lookup size "
2025                         "(%d) must equal the size supported by the hardware "
2026                         "(%d)\n", reta_size, tbl_size);
2027                 return -EINVAL;
2028         }
2029
2030         for (i = 0; i < reta_size; i++) {
2031                 struct bnxt_rx_queue *rxq;
2032
2033                 idx = i / RTE_ETH_RETA_GROUP_SIZE;
2034                 sft = i % RTE_ETH_RETA_GROUP_SIZE;
2035
2036                 if (!(reta_conf[idx].mask & (1ULL << sft)))
2037                         continue;
2038
2039                 rxq = bnxt_qid_to_rxq(bp, reta_conf[idx].reta[sft]);
2040                 if (!rxq) {
2041                         PMD_DRV_LOG(ERR, "Invalid ring in reta_conf.\n");
2042                         return -EINVAL;
2043                 }
2044
2045                 if (BNXT_CHIP_P5(bp)) {
2046                         vnic->rss_table[i * 2] =
2047                                 rxq->rx_ring->rx_ring_struct->fw_ring_id;
2048                         vnic->rss_table[i * 2 + 1] =
2049                                 rxq->cp_ring->cp_ring_struct->fw_ring_id;
2050                 } else {
2051                         vnic->rss_table[i] =
2052                             vnic->fw_grp_ids[reta_conf[idx].reta[sft]];
2053                 }
2054         }
2055
2056         rc = bnxt_hwrm_vnic_rss_cfg(bp, vnic);
2057         return rc;
2058 }
2059
2060 static int bnxt_reta_query_op(struct rte_eth_dev *eth_dev,
2061                               struct rte_eth_rss_reta_entry64 *reta_conf,
2062                               uint16_t reta_size)
2063 {
2064         struct bnxt *bp = eth_dev->data->dev_private;
2065         struct bnxt_vnic_info *vnic = BNXT_GET_DEFAULT_VNIC(bp);
2066         uint16_t tbl_size = bnxt_rss_hash_tbl_size(bp);
2067         uint16_t idx, sft, i;
2068         int rc;
2069
2070         rc = is_bnxt_in_error(bp);
2071         if (rc)
2072                 return rc;
2073
2074         if (!vnic)
2075                 return -EINVAL;
2076         if (!vnic->rss_table)
2077                 return -EINVAL;
2078
2079         if (reta_size != tbl_size) {
2080                 PMD_DRV_LOG(ERR, "The configured hash table lookup size "
2081                         "(%d) must equal the size supported by the hardware "
2082                         "(%d)\n", reta_size, tbl_size);
2083                 return -EINVAL;
2084         }
2085
2086         for (idx = 0, i = 0; i < reta_size; i++) {
2087                 idx = i / RTE_ETH_RETA_GROUP_SIZE;
2088                 sft = i % RTE_ETH_RETA_GROUP_SIZE;
2089
2090                 if (reta_conf[idx].mask & (1ULL << sft)) {
2091                         uint16_t qid;
2092
2093                         if (BNXT_CHIP_P5(bp))
2094                                 qid = bnxt_rss_to_qid(bp,
2095                                                       vnic->rss_table[i * 2]);
2096                         else
2097                                 qid = bnxt_rss_to_qid(bp, vnic->rss_table[i]);
2098
2099                         if (qid == INVALID_HW_RING_ID) {
2100                                 PMD_DRV_LOG(ERR, "Inv. entry in rss table.\n");
2101                                 return -EINVAL;
2102                         }
2103                         reta_conf[idx].reta[sft] = qid;
2104                 }
2105         }
2106
2107         return 0;
2108 }
2109
2110 static int bnxt_rss_hash_update_op(struct rte_eth_dev *eth_dev,
2111                                    struct rte_eth_rss_conf *rss_conf)
2112 {
2113         struct bnxt *bp = eth_dev->data->dev_private;
2114         struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
2115         struct bnxt_vnic_info *vnic;
2116         int rc;
2117
2118         rc = is_bnxt_in_error(bp);
2119         if (rc)
2120                 return rc;
2121
2122         /*
2123          * If RSS enablement were different than dev_configure,
2124          * then return -EINVAL
2125          */
2126         if (dev_conf->rxmode.mq_mode & RTE_ETH_MQ_RX_RSS_FLAG) {
2127                 if (!rss_conf->rss_hf)
2128                         PMD_DRV_LOG(ERR, "Hash type NONE\n");
2129         } else {
2130                 if (rss_conf->rss_hf & BNXT_ETH_RSS_SUPPORT)
2131                         return -EINVAL;
2132         }
2133
2134         bp->flags |= BNXT_FLAG_UPDATE_HASH;
2135
2136         /* Update the default RSS VNIC(s) */
2137         vnic = BNXT_GET_DEFAULT_VNIC(bp);
2138         vnic->hash_type = bnxt_rte_to_hwrm_hash_types(rss_conf->rss_hf);
2139         vnic->hash_mode =
2140                 bnxt_rte_to_hwrm_hash_level(bp, rss_conf->rss_hf,
2141                                             RTE_ETH_RSS_LEVEL(rss_conf->rss_hf));
2142
2143         /* Cache the hash function */
2144         bp->rss_conf.rss_hf = rss_conf->rss_hf;
2145
2146         /*
2147          * If hashkey is not specified, use the previously configured
2148          * hashkey
2149          */
2150         if (!rss_conf->rss_key)
2151                 goto rss_config;
2152
2153         if (rss_conf->rss_key_len != HW_HASH_KEY_SIZE) {
2154                 PMD_DRV_LOG(ERR,
2155                             "Invalid hashkey length, should be %d bytes\n",
2156                             HW_HASH_KEY_SIZE);
2157                 return -EINVAL;
2158         }
2159         memcpy(vnic->rss_hash_key, rss_conf->rss_key, rss_conf->rss_key_len);
2160
2161         /* Cache the hash key */
2162         memcpy(bp->rss_conf.rss_key, rss_conf->rss_key, HW_HASH_KEY_SIZE);
2163
2164 rss_config:
2165         rc = bnxt_hwrm_vnic_rss_cfg(bp, vnic);
2166         return rc;
2167 }
2168
2169 static int bnxt_rss_hash_conf_get_op(struct rte_eth_dev *eth_dev,
2170                                      struct rte_eth_rss_conf *rss_conf)
2171 {
2172         struct bnxt *bp = eth_dev->data->dev_private;
2173         struct bnxt_vnic_info *vnic = BNXT_GET_DEFAULT_VNIC(bp);
2174         int len, rc;
2175         uint32_t hash_types;
2176
2177         rc = is_bnxt_in_error(bp);
2178         if (rc)
2179                 return rc;
2180
2181         /* RSS configuration is the same for all VNICs */
2182         if (vnic && vnic->rss_hash_key) {
2183                 if (rss_conf->rss_key) {
2184                         len = rss_conf->rss_key_len <= HW_HASH_KEY_SIZE ?
2185                               rss_conf->rss_key_len : HW_HASH_KEY_SIZE;
2186                         memcpy(rss_conf->rss_key, vnic->rss_hash_key, len);
2187                 }
2188
2189                 hash_types = vnic->hash_type;
2190                 rss_conf->rss_hf = 0;
2191                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV4) {
2192                         rss_conf->rss_hf |= RTE_ETH_RSS_IPV4;
2193                         hash_types &= ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV4;
2194                 }
2195                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV4) {
2196                         rss_conf->rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_TCP;
2197                         hash_types &=
2198                                 ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV4;
2199                 }
2200                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV4) {
2201                         rss_conf->rss_hf |= RTE_ETH_RSS_NONFRAG_IPV4_UDP;
2202                         hash_types &=
2203                                 ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV4;
2204                 }
2205                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV6) {
2206                         rss_conf->rss_hf |= RTE_ETH_RSS_IPV6;
2207                         hash_types &= ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV6;
2208                 }
2209                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV6) {
2210                         rss_conf->rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_TCP;
2211                         hash_types &=
2212                                 ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV6;
2213                 }
2214                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV6) {
2215                         rss_conf->rss_hf |= RTE_ETH_RSS_NONFRAG_IPV6_UDP;
2216                         hash_types &=
2217                                 ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV6;
2218                 }
2219
2220                 rss_conf->rss_hf |=
2221                         bnxt_hwrm_to_rte_rss_level(bp, vnic->hash_mode);
2222
2223                 if (hash_types) {
2224                         PMD_DRV_LOG(ERR,
2225                                 "Unknown RSS config from firmware (%08x), RSS disabled",
2226                                 vnic->hash_type);
2227                         return -ENOTSUP;
2228                 }
2229         } else {
2230                 rss_conf->rss_hf = 0;
2231         }
2232         return 0;
2233 }
2234
2235 static int bnxt_flow_ctrl_get_op(struct rte_eth_dev *dev,
2236                                struct rte_eth_fc_conf *fc_conf)
2237 {
2238         struct bnxt *bp = dev->data->dev_private;
2239         struct rte_eth_link link_info;
2240         int rc;
2241
2242         rc = is_bnxt_in_error(bp);
2243         if (rc)
2244                 return rc;
2245
2246         rc = bnxt_get_hwrm_link_config(bp, &link_info);
2247         if (rc)
2248                 return rc;
2249
2250         memset(fc_conf, 0, sizeof(*fc_conf));
2251         if (bp->link_info->auto_pause)
2252                 fc_conf->autoneg = 1;
2253         switch (bp->link_info->pause) {
2254         case 0:
2255                 fc_conf->mode = RTE_ETH_FC_NONE;
2256                 break;
2257         case HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX:
2258                 fc_conf->mode = RTE_ETH_FC_TX_PAUSE;
2259                 break;
2260         case HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX:
2261                 fc_conf->mode = RTE_ETH_FC_RX_PAUSE;
2262                 break;
2263         case (HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX |
2264                         HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX):
2265                 fc_conf->mode = RTE_ETH_FC_FULL;
2266                 break;
2267         }
2268         return 0;
2269 }
2270
2271 static int bnxt_flow_ctrl_set_op(struct rte_eth_dev *dev,
2272                                struct rte_eth_fc_conf *fc_conf)
2273 {
2274         struct bnxt *bp = dev->data->dev_private;
2275         int rc;
2276
2277         rc = is_bnxt_in_error(bp);
2278         if (rc)
2279                 return rc;
2280
2281         if (!BNXT_SINGLE_PF(bp)) {
2282                 PMD_DRV_LOG(ERR,
2283                             "Flow Control Settings cannot be modified on VF or on shared PF\n");
2284                 return -ENOTSUP;
2285         }
2286
2287         switch (fc_conf->mode) {
2288         case RTE_ETH_FC_NONE:
2289                 bp->link_info->auto_pause = 0;
2290                 bp->link_info->force_pause = 0;
2291                 break;
2292         case RTE_ETH_FC_RX_PAUSE:
2293                 if (fc_conf->autoneg) {
2294                         bp->link_info->auto_pause =
2295                                         HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_RX;
2296                         bp->link_info->force_pause = 0;
2297                 } else {
2298                         bp->link_info->auto_pause = 0;
2299                         bp->link_info->force_pause =
2300                                         HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_RX;
2301                 }
2302                 break;
2303         case RTE_ETH_FC_TX_PAUSE:
2304                 if (fc_conf->autoneg) {
2305                         bp->link_info->auto_pause =
2306                                         HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_TX;
2307                         bp->link_info->force_pause = 0;
2308                 } else {
2309                         bp->link_info->auto_pause = 0;
2310                         bp->link_info->force_pause =
2311                                         HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_TX;
2312                 }
2313                 break;
2314         case RTE_ETH_FC_FULL:
2315                 if (fc_conf->autoneg) {
2316                         bp->link_info->auto_pause =
2317                                         HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_TX |
2318                                         HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_RX;
2319                         bp->link_info->force_pause = 0;
2320                 } else {
2321                         bp->link_info->auto_pause = 0;
2322                         bp->link_info->force_pause =
2323                                         HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_TX |
2324                                         HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_RX;
2325                 }
2326                 break;
2327         }
2328         return bnxt_set_hwrm_link_config(bp, true);
2329 }
2330
2331 /* Add UDP tunneling port */
2332 static int
2333 bnxt_udp_tunnel_port_add_op(struct rte_eth_dev *eth_dev,
2334                          struct rte_eth_udp_tunnel *udp_tunnel)
2335 {
2336         struct bnxt *bp = eth_dev->data->dev_private;
2337         uint16_t tunnel_type = 0;
2338         int rc = 0;
2339
2340         rc = is_bnxt_in_error(bp);
2341         if (rc)
2342                 return rc;
2343
2344         switch (udp_tunnel->prot_type) {
2345         case RTE_ETH_TUNNEL_TYPE_VXLAN:
2346                 if (bp->vxlan_port_cnt) {
2347                         PMD_DRV_LOG(ERR, "Tunnel Port %d already programmed\n",
2348                                 udp_tunnel->udp_port);
2349                         if (bp->vxlan_port != udp_tunnel->udp_port) {
2350                                 PMD_DRV_LOG(ERR, "Only one port allowed\n");
2351                                 return -ENOSPC;
2352                         }
2353                         bp->vxlan_port_cnt++;
2354                         return 0;
2355                 }
2356                 tunnel_type =
2357                         HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_VXLAN;
2358                 break;
2359         case RTE_ETH_TUNNEL_TYPE_GENEVE:
2360                 if (bp->geneve_port_cnt) {
2361                         PMD_DRV_LOG(ERR, "Tunnel Port %d already programmed\n",
2362                                 udp_tunnel->udp_port);
2363                         if (bp->geneve_port != udp_tunnel->udp_port) {
2364                                 PMD_DRV_LOG(ERR, "Only one port allowed\n");
2365                                 return -ENOSPC;
2366                         }
2367                         bp->geneve_port_cnt++;
2368                         return 0;
2369                 }
2370                 tunnel_type =
2371                         HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_GENEVE;
2372                 break;
2373         default:
2374                 PMD_DRV_LOG(ERR, "Tunnel type is not supported\n");
2375                 return -ENOTSUP;
2376         }
2377         rc = bnxt_hwrm_tunnel_dst_port_alloc(bp, udp_tunnel->udp_port,
2378                                              tunnel_type);
2379
2380         if (rc != 0)
2381                 return rc;
2382
2383         if (tunnel_type ==
2384             HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_VXLAN)
2385                 bp->vxlan_port_cnt++;
2386
2387         if (tunnel_type ==
2388             HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_GENEVE)
2389                 bp->geneve_port_cnt++;
2390
2391         return rc;
2392 }
2393
2394 static int
2395 bnxt_udp_tunnel_port_del_op(struct rte_eth_dev *eth_dev,
2396                          struct rte_eth_udp_tunnel *udp_tunnel)
2397 {
2398         struct bnxt *bp = eth_dev->data->dev_private;
2399         uint16_t tunnel_type = 0;
2400         uint16_t port = 0;
2401         int rc = 0;
2402
2403         rc = is_bnxt_in_error(bp);
2404         if (rc)
2405                 return rc;
2406
2407         switch (udp_tunnel->prot_type) {
2408         case RTE_ETH_TUNNEL_TYPE_VXLAN:
2409                 if (!bp->vxlan_port_cnt) {
2410                         PMD_DRV_LOG(ERR, "No Tunnel port configured yet\n");
2411                         return -EINVAL;
2412                 }
2413                 if (bp->vxlan_port != udp_tunnel->udp_port) {
2414                         PMD_DRV_LOG(ERR, "Req Port: %d. Configured port: %d\n",
2415                                 udp_tunnel->udp_port, bp->vxlan_port);
2416                         return -EINVAL;
2417                 }
2418                 if (--bp->vxlan_port_cnt)
2419                         return 0;
2420
2421                 tunnel_type =
2422                         HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_VXLAN;
2423                 port = bp->vxlan_fw_dst_port_id;
2424                 break;
2425         case RTE_ETH_TUNNEL_TYPE_GENEVE:
2426                 if (!bp->geneve_port_cnt) {
2427                         PMD_DRV_LOG(ERR, "No Tunnel port configured yet\n");
2428                         return -EINVAL;
2429                 }
2430                 if (bp->geneve_port != udp_tunnel->udp_port) {
2431                         PMD_DRV_LOG(ERR, "Req Port: %d. Configured port: %d\n",
2432                                 udp_tunnel->udp_port, bp->geneve_port);
2433                         return -EINVAL;
2434                 }
2435                 if (--bp->geneve_port_cnt)
2436                         return 0;
2437
2438                 tunnel_type =
2439                         HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_GENEVE;
2440                 port = bp->geneve_fw_dst_port_id;
2441                 break;
2442         default:
2443                 PMD_DRV_LOG(ERR, "Tunnel type is not supported\n");
2444                 return -ENOTSUP;
2445         }
2446
2447         rc = bnxt_hwrm_tunnel_dst_port_free(bp, port, tunnel_type);
2448         return rc;
2449 }
2450
2451 static int bnxt_del_vlan_filter(struct bnxt *bp, uint16_t vlan_id)
2452 {
2453         struct bnxt_filter_info *filter;
2454         struct bnxt_vnic_info *vnic;
2455         int rc = 0;
2456         uint32_t chk = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN;
2457
2458         vnic = BNXT_GET_DEFAULT_VNIC(bp);
2459         filter = STAILQ_FIRST(&vnic->filter);
2460         while (filter) {
2461                 /* Search for this matching MAC+VLAN filter */
2462                 if (bnxt_vlan_filter_exists(bp, filter, chk, vlan_id)) {
2463                         /* Delete the filter */
2464                         rc = bnxt_hwrm_clear_l2_filter(bp, filter);
2465                         if (rc)
2466                                 return rc;
2467                         STAILQ_REMOVE(&vnic->filter, filter,
2468                                       bnxt_filter_info, next);
2469                         bnxt_free_filter(bp, filter);
2470                         PMD_DRV_LOG(INFO,
2471                                     "Deleted vlan filter for %d\n",
2472                                     vlan_id);
2473                         return 0;
2474                 }
2475                 filter = STAILQ_NEXT(filter, next);
2476         }
2477         return -ENOENT;
2478 }
2479
2480 static int bnxt_add_vlan_filter(struct bnxt *bp, uint16_t vlan_id)
2481 {
2482         struct bnxt_filter_info *filter;
2483         struct bnxt_vnic_info *vnic;
2484         int rc = 0;
2485         uint32_t en = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN |
2486                 HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN_MASK;
2487         uint32_t chk = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN;
2488
2489         /* Implementation notes on the use of VNIC in this command:
2490          *
2491          * By default, these filters belong to default vnic for the function.
2492          * Once these filters are set up, only destination VNIC can be modified.
2493          * If the destination VNIC is not specified in this command,
2494          * then the HWRM shall only create an l2 context id.
2495          */
2496
2497         vnic = BNXT_GET_DEFAULT_VNIC(bp);
2498         filter = STAILQ_FIRST(&vnic->filter);
2499         /* Check if the VLAN has already been added */
2500         while (filter) {
2501                 if (bnxt_vlan_filter_exists(bp, filter, chk, vlan_id))
2502                         return -EEXIST;
2503
2504                 filter = STAILQ_NEXT(filter, next);
2505         }
2506
2507         /* No match found. Alloc a fresh filter and issue the L2_FILTER_ALLOC
2508          * command to create MAC+VLAN filter with the right flags, enables set.
2509          */
2510         filter = bnxt_alloc_filter(bp);
2511         if (!filter) {
2512                 PMD_DRV_LOG(ERR,
2513                             "MAC/VLAN filter alloc failed\n");
2514                 return -ENOMEM;
2515         }
2516         /* MAC + VLAN ID filter */
2517         /* If l2_ivlan == 0 and l2_ivlan_mask != 0, only
2518          * untagged packets are received
2519          *
2520          * If l2_ivlan != 0 and l2_ivlan_mask != 0, untagged
2521          * packets and only the programmed vlan's packets are received
2522          */
2523         filter->l2_ivlan = vlan_id;
2524         filter->l2_ivlan_mask = 0x0FFF;
2525         filter->enables |= en;
2526         filter->flags |= HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_OUTERMOST;
2527
2528         rc = bnxt_hwrm_set_l2_filter(bp, vnic->fw_vnic_id, filter);
2529         if (rc) {
2530                 /* Free the newly allocated filter as we were
2531                  * not able to create the filter in hardware.
2532                  */
2533                 bnxt_free_filter(bp, filter);
2534                 return rc;
2535         }
2536
2537         filter->mac_index = 0;
2538         /* Add this new filter to the list */
2539         if (vlan_id == 0)
2540                 STAILQ_INSERT_HEAD(&vnic->filter, filter, next);
2541         else
2542                 STAILQ_INSERT_TAIL(&vnic->filter, filter, next);
2543
2544         PMD_DRV_LOG(INFO,
2545                     "Added Vlan filter for %d\n", vlan_id);
2546         return rc;
2547 }
2548
2549 static int bnxt_vlan_filter_set_op(struct rte_eth_dev *eth_dev,
2550                 uint16_t vlan_id, int on)
2551 {
2552         struct bnxt *bp = eth_dev->data->dev_private;
2553         int rc;
2554
2555         rc = is_bnxt_in_error(bp);
2556         if (rc)
2557                 return rc;
2558
2559         if (!eth_dev->data->dev_started) {
2560                 PMD_DRV_LOG(ERR, "port must be started before setting vlan\n");
2561                 return -EINVAL;
2562         }
2563
2564         /* These operations apply to ALL existing MAC/VLAN filters */
2565         if (on)
2566                 return bnxt_add_vlan_filter(bp, vlan_id);
2567         else
2568                 return bnxt_del_vlan_filter(bp, vlan_id);
2569 }
2570
2571 static int bnxt_del_dflt_mac_filter(struct bnxt *bp,
2572                                     struct bnxt_vnic_info *vnic)
2573 {
2574         struct bnxt_filter_info *filter;
2575         int rc;
2576
2577         filter = STAILQ_FIRST(&vnic->filter);
2578         while (filter) {
2579                 if (filter->mac_index == 0 &&
2580                     !memcmp(filter->l2_addr, bp->mac_addr,
2581                             RTE_ETHER_ADDR_LEN)) {
2582                         rc = bnxt_hwrm_clear_l2_filter(bp, filter);
2583                         if (!rc) {
2584                                 STAILQ_REMOVE(&vnic->filter, filter,
2585                                               bnxt_filter_info, next);
2586                                 bnxt_free_filter(bp, filter);
2587                         }
2588                         return rc;
2589                 }
2590                 filter = STAILQ_NEXT(filter, next);
2591         }
2592         return 0;
2593 }
2594
2595 static int
2596 bnxt_config_vlan_hw_filter(struct bnxt *bp, uint64_t rx_offloads)
2597 {
2598         struct bnxt_vnic_info *vnic;
2599         unsigned int i;
2600         int rc;
2601
2602         vnic = BNXT_GET_DEFAULT_VNIC(bp);
2603         if (!(rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER)) {
2604                 /* Remove any VLAN filters programmed */
2605                 for (i = 0; i < RTE_ETHER_MAX_VLAN_ID; i++)
2606                         bnxt_del_vlan_filter(bp, i);
2607
2608                 rc = bnxt_add_mac_filter(bp, vnic, NULL, 0, 0);
2609                 if (rc)
2610                         return rc;
2611         } else {
2612                 /* Default filter will allow packets that match the
2613                  * dest mac. So, it has to be deleted, otherwise, we
2614                  * will endup receiving vlan packets for which the
2615                  * filter is not programmed, when hw-vlan-filter
2616                  * configuration is ON
2617                  */
2618                 bnxt_del_dflt_mac_filter(bp, vnic);
2619                 /* This filter will allow only untagged packets */
2620                 bnxt_add_vlan_filter(bp, 0);
2621         }
2622         PMD_DRV_LOG(DEBUG, "VLAN Filtering: %d\n",
2623                     !!(rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER));
2624
2625         return 0;
2626 }
2627
2628 static int bnxt_free_one_vnic(struct bnxt *bp, uint16_t vnic_id)
2629 {
2630         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
2631         unsigned int i;
2632         int rc;
2633
2634         /* Destroy vnic filters and vnic */
2635         if (bp->eth_dev->data->dev_conf.rxmode.offloads &
2636             RTE_ETH_RX_OFFLOAD_VLAN_FILTER) {
2637                 for (i = 0; i < RTE_ETHER_MAX_VLAN_ID; i++)
2638                         bnxt_del_vlan_filter(bp, i);
2639         }
2640         bnxt_del_dflt_mac_filter(bp, vnic);
2641
2642         rc = bnxt_hwrm_vnic_ctx_free(bp, vnic);
2643         if (rc)
2644                 return rc;
2645
2646         rc = bnxt_hwrm_vnic_free(bp, vnic);
2647         if (rc)
2648                 return rc;
2649
2650         rte_free(vnic->fw_grp_ids);
2651         vnic->fw_grp_ids = NULL;
2652
2653         vnic->rx_queue_cnt = 0;
2654
2655         return 0;
2656 }
2657
2658 static int
2659 bnxt_config_vlan_hw_stripping(struct bnxt *bp, uint64_t rx_offloads)
2660 {
2661         struct bnxt_vnic_info *vnic = BNXT_GET_DEFAULT_VNIC(bp);
2662         int rc;
2663
2664         /* Destroy, recreate and reconfigure the default vnic */
2665         rc = bnxt_free_one_vnic(bp, 0);
2666         if (rc)
2667                 return rc;
2668
2669         /* default vnic 0 */
2670         rc = bnxt_setup_one_vnic(bp, 0);
2671         if (rc)
2672                 return rc;
2673
2674         if (bp->eth_dev->data->dev_conf.rxmode.offloads &
2675             RTE_ETH_RX_OFFLOAD_VLAN_FILTER) {
2676                 rc = bnxt_add_vlan_filter(bp, 0);
2677                 if (rc)
2678                         return rc;
2679                 rc = bnxt_restore_vlan_filters(bp);
2680                 if (rc)
2681                         return rc;
2682         } else {
2683                 rc = bnxt_add_mac_filter(bp, vnic, NULL, 0, 0);
2684                 if (rc)
2685                         return rc;
2686         }
2687
2688         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
2689         if (rc)
2690                 return rc;
2691
2692         PMD_DRV_LOG(DEBUG, "VLAN Strip Offload: %d\n",
2693                     !!(rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_STRIP));
2694
2695         return rc;
2696 }
2697
2698 static int
2699 bnxt_vlan_offload_set_op(struct rte_eth_dev *dev, int mask)
2700 {
2701         uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads;
2702         struct bnxt *bp = dev->data->dev_private;
2703         int rc;
2704
2705         rc = is_bnxt_in_error(bp);
2706         if (rc)
2707                 return rc;
2708
2709         /* Filter settings will get applied when port is started */
2710         if (!dev->data->dev_started)
2711                 return 0;
2712
2713         if (mask & RTE_ETH_VLAN_FILTER_MASK) {
2714                 /* Enable or disable VLAN filtering */
2715                 rc = bnxt_config_vlan_hw_filter(bp, rx_offloads);
2716                 if (rc)
2717                         return rc;
2718         }
2719
2720         if (mask & RTE_ETH_VLAN_STRIP_MASK) {
2721                 /* Enable or disable VLAN stripping */
2722                 rc = bnxt_config_vlan_hw_stripping(bp, rx_offloads);
2723                 if (rc)
2724                         return rc;
2725         }
2726
2727         if (mask & RTE_ETH_VLAN_EXTEND_MASK) {
2728                 if (rx_offloads & RTE_ETH_RX_OFFLOAD_VLAN_EXTEND)
2729                         PMD_DRV_LOG(DEBUG, "Extend VLAN supported\n");
2730                 else
2731                         PMD_DRV_LOG(INFO, "Extend VLAN unsupported\n");
2732         }
2733
2734         return 0;
2735 }
2736
2737 static int
2738 bnxt_vlan_tpid_set_op(struct rte_eth_dev *dev, enum rte_vlan_type vlan_type,
2739                       uint16_t tpid)
2740 {
2741         struct bnxt *bp = dev->data->dev_private;
2742         int qinq = dev->data->dev_conf.rxmode.offloads &
2743                    RTE_ETH_RX_OFFLOAD_VLAN_EXTEND;
2744
2745         if (vlan_type != RTE_ETH_VLAN_TYPE_INNER &&
2746             vlan_type != RTE_ETH_VLAN_TYPE_OUTER) {
2747                 PMD_DRV_LOG(ERR,
2748                             "Unsupported vlan type.");
2749                 return -EINVAL;
2750         }
2751         if (!qinq) {
2752                 PMD_DRV_LOG(ERR,
2753                             "QinQ not enabled. Needs to be ON as we can "
2754                             "accelerate only outer vlan\n");
2755                 return -EINVAL;
2756         }
2757
2758         if (vlan_type == RTE_ETH_VLAN_TYPE_OUTER) {
2759                 switch (tpid) {
2760                 case RTE_ETHER_TYPE_QINQ:
2761                         bp->outer_tpid_bd =
2762                                 TX_BD_LONG_CFA_META_VLAN_TPID_TPID88A8;
2763                                 break;
2764                 case RTE_ETHER_TYPE_VLAN:
2765                         bp->outer_tpid_bd =
2766                                 TX_BD_LONG_CFA_META_VLAN_TPID_TPID8100;
2767                                 break;
2768                 case RTE_ETHER_TYPE_QINQ1:
2769                         bp->outer_tpid_bd =
2770                                 TX_BD_LONG_CFA_META_VLAN_TPID_TPID9100;
2771                                 break;
2772                 case RTE_ETHER_TYPE_QINQ2:
2773                         bp->outer_tpid_bd =
2774                                 TX_BD_LONG_CFA_META_VLAN_TPID_TPID9200;
2775                                 break;
2776                 case RTE_ETHER_TYPE_QINQ3:
2777                         bp->outer_tpid_bd =
2778                                  TX_BD_LONG_CFA_META_VLAN_TPID_TPID9300;
2779                                 break;
2780                 default:
2781                         PMD_DRV_LOG(ERR, "Invalid TPID: %x\n", tpid);
2782                         return -EINVAL;
2783                 }
2784                 bp->outer_tpid_bd |= tpid;
2785                 PMD_DRV_LOG(INFO, "outer_tpid_bd = %x\n", bp->outer_tpid_bd);
2786         } else if (vlan_type == RTE_ETH_VLAN_TYPE_INNER) {
2787                 PMD_DRV_LOG(ERR,
2788                             "Can accelerate only outer vlan in QinQ\n");
2789                 return -EINVAL;
2790         }
2791
2792         return 0;
2793 }
2794
2795 static int
2796 bnxt_set_default_mac_addr_op(struct rte_eth_dev *dev,
2797                              struct rte_ether_addr *addr)
2798 {
2799         struct bnxt *bp = dev->data->dev_private;
2800         /* Default Filter is tied to VNIC 0 */
2801         struct bnxt_vnic_info *vnic = BNXT_GET_DEFAULT_VNIC(bp);
2802         int rc;
2803
2804         rc = is_bnxt_in_error(bp);
2805         if (rc)
2806                 return rc;
2807
2808         if (BNXT_VF(bp) && !BNXT_VF_IS_TRUSTED(bp))
2809                 return -EPERM;
2810
2811         if (rte_is_zero_ether_addr(addr))
2812                 return -EINVAL;
2813
2814         /* Filter settings will get applied when port is started */
2815         if (!dev->data->dev_started)
2816                 return 0;
2817
2818         /* Check if the requested MAC is already added */
2819         if (memcmp(addr, bp->mac_addr, RTE_ETHER_ADDR_LEN) == 0)
2820                 return 0;
2821
2822         /* Destroy filter and re-create it */
2823         bnxt_del_dflt_mac_filter(bp, vnic);
2824
2825         memcpy(bp->mac_addr, addr, RTE_ETHER_ADDR_LEN);
2826         if (dev->data->dev_conf.rxmode.offloads & RTE_ETH_RX_OFFLOAD_VLAN_FILTER) {
2827                 /* This filter will allow only untagged packets */
2828                 rc = bnxt_add_vlan_filter(bp, 0);
2829         } else {
2830                 rc = bnxt_add_mac_filter(bp, vnic, addr, 0, 0);
2831         }
2832
2833         PMD_DRV_LOG(DEBUG, "Set MAC addr\n");
2834         return rc;
2835 }
2836
2837 static int
2838 bnxt_dev_set_mc_addr_list_op(struct rte_eth_dev *eth_dev,
2839                           struct rte_ether_addr *mc_addr_set,
2840                           uint32_t nb_mc_addr)
2841 {
2842         struct bnxt *bp = eth_dev->data->dev_private;
2843         struct bnxt_vnic_info *vnic;
2844         uint32_t i = 0;
2845         int rc;
2846
2847         rc = is_bnxt_in_error(bp);
2848         if (rc)
2849                 return rc;
2850
2851         vnic = BNXT_GET_DEFAULT_VNIC(bp);
2852
2853         bp->nb_mc_addr = nb_mc_addr;
2854
2855         if (nb_mc_addr > BNXT_MAX_MC_ADDRS) {
2856                 vnic->flags |= BNXT_VNIC_INFO_ALLMULTI;
2857                 goto allmulti;
2858         }
2859
2860         /* TODO Check for Duplicate mcast addresses */
2861         vnic->flags &= ~BNXT_VNIC_INFO_ALLMULTI;
2862         for (i = 0; i < nb_mc_addr; i++)
2863                 rte_ether_addr_copy(&mc_addr_set[i], &bp->mcast_addr_list[i]);
2864
2865         if (bp->nb_mc_addr)
2866                 vnic->flags |= BNXT_VNIC_INFO_MCAST;
2867         else
2868                 vnic->flags &= ~BNXT_VNIC_INFO_MCAST;
2869
2870 allmulti:
2871         return bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
2872 }
2873
2874 static int
2875 bnxt_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size)
2876 {
2877         struct bnxt *bp = dev->data->dev_private;
2878         uint8_t fw_major = (bp->fw_ver >> 24) & 0xff;
2879         uint8_t fw_minor = (bp->fw_ver >> 16) & 0xff;
2880         uint8_t fw_updt = (bp->fw_ver >> 8) & 0xff;
2881         uint8_t fw_rsvd = bp->fw_ver & 0xff;
2882         int ret;
2883
2884         ret = snprintf(fw_version, fw_size, "%d.%d.%d.%d",
2885                         fw_major, fw_minor, fw_updt, fw_rsvd);
2886         if (ret < 0)
2887                 return -EINVAL;
2888
2889         ret += 1; /* add the size of '\0' */
2890         if (fw_size < (size_t)ret)
2891                 return ret;
2892         else
2893                 return 0;
2894 }
2895
2896 static void
2897 bnxt_rxq_info_get_op(struct rte_eth_dev *dev, uint16_t queue_id,
2898         struct rte_eth_rxq_info *qinfo)
2899 {
2900         struct bnxt *bp = dev->data->dev_private;
2901         struct bnxt_rx_queue *rxq;
2902
2903         if (is_bnxt_in_error(bp))
2904                 return;
2905
2906         rxq = dev->data->rx_queues[queue_id];
2907
2908         qinfo->mp = rxq->mb_pool;
2909         qinfo->scattered_rx = dev->data->scattered_rx;
2910         qinfo->nb_desc = rxq->nb_rx_desc;
2911
2912         qinfo->conf.rx_free_thresh = rxq->rx_free_thresh;
2913         qinfo->conf.rx_drop_en = rxq->drop_en;
2914         qinfo->conf.rx_deferred_start = rxq->rx_deferred_start;
2915         qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads;
2916 }
2917
2918 static void
2919 bnxt_txq_info_get_op(struct rte_eth_dev *dev, uint16_t queue_id,
2920         struct rte_eth_txq_info *qinfo)
2921 {
2922         struct bnxt *bp = dev->data->dev_private;
2923         struct bnxt_tx_queue *txq;
2924
2925         if (is_bnxt_in_error(bp))
2926                 return;
2927
2928         txq = dev->data->tx_queues[queue_id];
2929
2930         qinfo->nb_desc = txq->nb_tx_desc;
2931
2932         qinfo->conf.tx_thresh.pthresh = txq->pthresh;
2933         qinfo->conf.tx_thresh.hthresh = txq->hthresh;
2934         qinfo->conf.tx_thresh.wthresh = txq->wthresh;
2935
2936         qinfo->conf.tx_free_thresh = txq->tx_free_thresh;
2937         qinfo->conf.tx_rs_thresh = 0;
2938         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
2939         qinfo->conf.offloads = txq->offloads;
2940 }
2941
2942 static const struct {
2943         eth_rx_burst_t pkt_burst;
2944         const char *info;
2945 } bnxt_rx_burst_info[] = {
2946         {bnxt_recv_pkts,                "Scalar"},
2947 #if defined(RTE_ARCH_X86)
2948         {bnxt_recv_pkts_vec,            "Vector SSE"},
2949 #endif
2950 #if defined(RTE_ARCH_X86) && defined(CC_AVX2_SUPPORT)
2951         {bnxt_recv_pkts_vec_avx2,       "Vector AVX2"},
2952 #endif
2953 #if defined(RTE_ARCH_ARM64)
2954         {bnxt_recv_pkts_vec,            "Vector Neon"},
2955 #endif
2956 };
2957
2958 static int
2959 bnxt_rx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id,
2960                        struct rte_eth_burst_mode *mode)
2961 {
2962         eth_rx_burst_t pkt_burst = dev->rx_pkt_burst;
2963         size_t i;
2964
2965         for (i = 0; i < RTE_DIM(bnxt_rx_burst_info); i++) {
2966                 if (pkt_burst == bnxt_rx_burst_info[i].pkt_burst) {
2967                         snprintf(mode->info, sizeof(mode->info), "%s",
2968                                  bnxt_rx_burst_info[i].info);
2969                         return 0;
2970                 }
2971         }
2972
2973         return -EINVAL;
2974 }
2975
2976 static const struct {
2977         eth_tx_burst_t pkt_burst;
2978         const char *info;
2979 } bnxt_tx_burst_info[] = {
2980         {bnxt_xmit_pkts,                "Scalar"},
2981 #if defined(RTE_ARCH_X86)
2982         {bnxt_xmit_pkts_vec,            "Vector SSE"},
2983 #endif
2984 #if defined(RTE_ARCH_X86) && defined(CC_AVX2_SUPPORT)
2985         {bnxt_xmit_pkts_vec_avx2,       "Vector AVX2"},
2986 #endif
2987 #if defined(RTE_ARCH_ARM64)
2988         {bnxt_xmit_pkts_vec,            "Vector Neon"},
2989 #endif
2990 };
2991
2992 static int
2993 bnxt_tx_burst_mode_get(struct rte_eth_dev *dev, __rte_unused uint16_t queue_id,
2994                        struct rte_eth_burst_mode *mode)
2995 {
2996         eth_tx_burst_t pkt_burst = dev->tx_pkt_burst;
2997         size_t i;
2998
2999         for (i = 0; i < RTE_DIM(bnxt_tx_burst_info); i++) {
3000                 if (pkt_burst == bnxt_tx_burst_info[i].pkt_burst) {
3001                         snprintf(mode->info, sizeof(mode->info), "%s",
3002                                  bnxt_tx_burst_info[i].info);
3003                         return 0;
3004                 }
3005         }
3006
3007         return -EINVAL;
3008 }
3009
3010 int bnxt_mtu_set_op(struct rte_eth_dev *eth_dev, uint16_t new_mtu)
3011 {
3012         uint32_t overhead = BNXT_MAX_PKT_LEN - BNXT_MAX_MTU;
3013         struct bnxt *bp = eth_dev->data->dev_private;
3014         uint32_t new_pkt_size;
3015         uint32_t rc;
3016         uint32_t i;
3017
3018         rc = is_bnxt_in_error(bp);
3019         if (rc)
3020                 return rc;
3021
3022         /* Exit if receive queues are not configured yet */
3023         if (!eth_dev->data->nb_rx_queues)
3024                 return rc;
3025
3026         new_pkt_size = new_mtu + overhead;
3027
3028         /*
3029          * Disallow any MTU change that would require scattered receive support
3030          * if it is not already enabled.
3031          */
3032         if (eth_dev->data->dev_started &&
3033             !eth_dev->data->scattered_rx &&
3034             (new_pkt_size >
3035              eth_dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)) {
3036                 PMD_DRV_LOG(ERR,
3037                             "MTU change would require scattered rx support. ");
3038                 PMD_DRV_LOG(ERR, "Stop port before changing MTU.\n");
3039                 return -EINVAL;
3040         }
3041
3042         if (new_mtu > RTE_ETHER_MTU)
3043                 bp->flags |= BNXT_FLAG_JUMBO;
3044         else
3045                 bp->flags &= ~BNXT_FLAG_JUMBO;
3046
3047         /* Is there a change in mtu setting? */
3048         if (eth_dev->data->mtu == new_mtu)
3049                 return rc;
3050
3051         for (i = 0; i < bp->nr_vnics; i++) {
3052                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3053                 uint16_t size = 0;
3054
3055                 vnic->mru = BNXT_VNIC_MRU(new_mtu);
3056                 rc = bnxt_hwrm_vnic_cfg(bp, vnic);
3057                 if (rc)
3058                         break;
3059
3060                 size = rte_pktmbuf_data_room_size(bp->rx_queues[0]->mb_pool);
3061                 size -= RTE_PKTMBUF_HEADROOM;
3062
3063                 if (size < new_mtu) {
3064                         rc = bnxt_hwrm_vnic_plcmode_cfg(bp, vnic);
3065                         if (rc)
3066                                 return rc;
3067                 }
3068         }
3069
3070         if (bnxt_hwrm_config_host_mtu(bp))
3071                 PMD_DRV_LOG(WARNING, "Failed to configure host MTU\n");
3072
3073         PMD_DRV_LOG(INFO, "New MTU is %d\n", new_mtu);
3074
3075         return rc;
3076 }
3077
3078 static int
3079 bnxt_vlan_pvid_set_op(struct rte_eth_dev *dev, uint16_t pvid, int on)
3080 {
3081         struct bnxt *bp = dev->data->dev_private;
3082         uint16_t vlan = bp->vlan;
3083         int rc;
3084
3085         rc = is_bnxt_in_error(bp);
3086         if (rc)
3087                 return rc;
3088
3089         if (!BNXT_SINGLE_PF(bp)) {
3090                 PMD_DRV_LOG(ERR, "PVID cannot be modified on VF or on shared PF\n");
3091                 return -ENOTSUP;
3092         }
3093         bp->vlan = on ? pvid : 0;
3094
3095         rc = bnxt_hwrm_set_default_vlan(bp, 0, 0);
3096         if (rc)
3097                 bp->vlan = vlan;
3098         return rc;
3099 }
3100
3101 static int
3102 bnxt_dev_led_on_op(struct rte_eth_dev *dev)
3103 {
3104         struct bnxt *bp = dev->data->dev_private;
3105         int rc;
3106
3107         rc = is_bnxt_in_error(bp);
3108         if (rc)
3109                 return rc;
3110
3111         return bnxt_hwrm_port_led_cfg(bp, true);
3112 }
3113
3114 static int
3115 bnxt_dev_led_off_op(struct rte_eth_dev *dev)
3116 {
3117         struct bnxt *bp = dev->data->dev_private;
3118         int rc;
3119
3120         rc = is_bnxt_in_error(bp);
3121         if (rc)
3122                 return rc;
3123
3124         return bnxt_hwrm_port_led_cfg(bp, false);
3125 }
3126
3127 static uint32_t
3128 bnxt_rx_queue_count_op(void *rx_queue)
3129 {
3130         struct bnxt *bp;
3131         struct bnxt_cp_ring_info *cpr;
3132         uint32_t desc = 0, raw_cons, cp_ring_size;
3133         struct bnxt_rx_queue *rxq;
3134         struct rx_pkt_cmpl *rxcmp;
3135         int rc;
3136
3137         rxq = rx_queue;
3138         bp = rxq->bp;
3139
3140         rc = is_bnxt_in_error(bp);
3141         if (rc)
3142                 return rc;
3143
3144         cpr = rxq->cp_ring;
3145         raw_cons = cpr->cp_raw_cons;
3146         cp_ring_size = cpr->cp_ring_struct->ring_size;
3147
3148         while (1) {
3149                 uint32_t agg_cnt, cons, cmpl_type;
3150
3151                 cons = RING_CMP(cpr->cp_ring_struct, raw_cons);
3152                 rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons];
3153
3154                 if (!bnxt_cpr_cmp_valid(rxcmp, raw_cons, cp_ring_size))
3155                         break;
3156
3157                 cmpl_type = CMP_TYPE(rxcmp);
3158
3159                 switch (cmpl_type) {
3160                 case CMPL_BASE_TYPE_RX_L2:
3161                 case CMPL_BASE_TYPE_RX_L2_V2:
3162                         agg_cnt = BNXT_RX_L2_AGG_BUFS(rxcmp);
3163                         raw_cons = raw_cons + CMP_LEN(cmpl_type) + agg_cnt;
3164                         desc++;
3165                         break;
3166
3167                 case CMPL_BASE_TYPE_RX_TPA_END:
3168                         if (BNXT_CHIP_P5(rxq->bp)) {
3169                                 struct rx_tpa_v2_end_cmpl_hi *p5_tpa_end;
3170
3171                                 p5_tpa_end = (void *)rxcmp;
3172                                 agg_cnt = BNXT_TPA_END_AGG_BUFS_TH(p5_tpa_end);
3173                         } else {
3174                                 struct rx_tpa_end_cmpl *tpa_end;
3175
3176                                 tpa_end = (void *)rxcmp;
3177                                 agg_cnt = BNXT_TPA_END_AGG_BUFS(tpa_end);
3178                         }
3179
3180                         raw_cons = raw_cons + CMP_LEN(cmpl_type) + agg_cnt;
3181                         desc++;
3182                         break;
3183
3184                 default:
3185                         raw_cons += CMP_LEN(cmpl_type);
3186                 }
3187         }
3188
3189         return desc;
3190 }
3191
3192 static int
3193 bnxt_rx_descriptor_status_op(void *rx_queue, uint16_t offset)
3194 {
3195         struct bnxt_rx_queue *rxq = rx_queue;
3196         struct bnxt_cp_ring_info *cpr;
3197         struct bnxt_rx_ring_info *rxr;
3198         uint32_t desc, raw_cons, cp_ring_size;
3199         struct bnxt *bp = rxq->bp;
3200         struct rx_pkt_cmpl *rxcmp;
3201         int rc;
3202
3203         rc = is_bnxt_in_error(bp);
3204         if (rc)
3205                 return rc;
3206
3207         if (offset >= rxq->nb_rx_desc)
3208                 return -EINVAL;
3209
3210         rxr = rxq->rx_ring;
3211         cpr = rxq->cp_ring;
3212         cp_ring_size = cpr->cp_ring_struct->ring_size;
3213
3214         /*
3215          * For the vector receive case, the completion at the requested
3216          * offset can be indexed directly.
3217          */
3218 #if defined(RTE_ARCH_X86) || defined(RTE_ARCH_ARM64)
3219         if (bp->flags & BNXT_FLAG_RX_VECTOR_PKT_MODE) {
3220                 struct rx_pkt_cmpl *rxcmp;
3221                 uint32_t cons;
3222
3223                 /* Check status of completion descriptor. */
3224                 raw_cons = cpr->cp_raw_cons +
3225                            offset * CMP_LEN(CMPL_BASE_TYPE_RX_L2);
3226                 cons = RING_CMP(cpr->cp_ring_struct, raw_cons);
3227                 rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons];
3228
3229                 if (bnxt_cpr_cmp_valid(rxcmp, raw_cons, cp_ring_size))
3230                         return RTE_ETH_RX_DESC_DONE;
3231
3232                 /* Check whether rx desc has an mbuf attached. */
3233                 cons = RING_CMP(rxr->rx_ring_struct, raw_cons / 2);
3234                 if (cons >= rxq->rxrearm_start &&
3235                     cons < rxq->rxrearm_start + rxq->rxrearm_nb) {
3236                         return RTE_ETH_RX_DESC_UNAVAIL;
3237                 }
3238
3239                 return RTE_ETH_RX_DESC_AVAIL;
3240         }
3241 #endif
3242
3243         /*
3244          * For the non-vector receive case, scan the completion ring to
3245          * locate the completion descriptor for the requested offset.
3246          */
3247         raw_cons = cpr->cp_raw_cons;
3248         desc = 0;
3249         while (1) {
3250                 uint32_t agg_cnt, cons, cmpl_type;
3251
3252                 cons = RING_CMP(cpr->cp_ring_struct, raw_cons);
3253                 rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons];
3254
3255                 if (!bnxt_cpr_cmp_valid(rxcmp, raw_cons, cp_ring_size))
3256                         break;
3257
3258                 cmpl_type = CMP_TYPE(rxcmp);
3259
3260                 switch (cmpl_type) {
3261                 case CMPL_BASE_TYPE_RX_L2:
3262                 case CMPL_BASE_TYPE_RX_L2_V2:
3263                         if (desc == offset) {
3264                                 cons = rxcmp->opaque;
3265                                 if (rxr->rx_buf_ring[cons])
3266                                         return RTE_ETH_RX_DESC_DONE;
3267                                 else
3268                                         return RTE_ETH_RX_DESC_UNAVAIL;
3269                         }
3270                         agg_cnt = BNXT_RX_L2_AGG_BUFS(rxcmp);
3271                         raw_cons = raw_cons + CMP_LEN(cmpl_type) + agg_cnt;
3272                         desc++;
3273                         break;
3274
3275                 case CMPL_BASE_TYPE_RX_TPA_END:
3276                         if (desc == offset)
3277                                 return RTE_ETH_RX_DESC_DONE;
3278
3279                         if (BNXT_CHIP_P5(rxq->bp)) {
3280                                 struct rx_tpa_v2_end_cmpl_hi *p5_tpa_end;
3281
3282                                 p5_tpa_end = (void *)rxcmp;
3283                                 agg_cnt = BNXT_TPA_END_AGG_BUFS_TH(p5_tpa_end);
3284                         } else {
3285                                 struct rx_tpa_end_cmpl *tpa_end;
3286
3287                                 tpa_end = (void *)rxcmp;
3288                                 agg_cnt = BNXT_TPA_END_AGG_BUFS(tpa_end);
3289                         }
3290
3291                         raw_cons = raw_cons + CMP_LEN(cmpl_type) + agg_cnt;
3292                         desc++;
3293                         break;
3294
3295                 default:
3296                         raw_cons += CMP_LEN(cmpl_type);
3297                 }
3298         }
3299
3300         return RTE_ETH_RX_DESC_AVAIL;
3301 }
3302
3303 static int
3304 bnxt_tx_descriptor_status_op(void *tx_queue, uint16_t offset)
3305 {
3306         struct bnxt_tx_queue *txq = (struct bnxt_tx_queue *)tx_queue;
3307         struct bnxt_cp_ring_info *cpr = txq->cp_ring;
3308         uint32_t ring_mask, raw_cons, nb_tx_pkts = 0;
3309         struct cmpl_base *cp_desc_ring;
3310         int rc;
3311
3312         rc = is_bnxt_in_error(txq->bp);
3313         if (rc)
3314                 return rc;
3315
3316         if (offset >= txq->nb_tx_desc)
3317                 return -EINVAL;
3318
3319         /* Return "desc done" if descriptor is available for use. */
3320         if (bnxt_tx_bds_in_hw(txq) <= offset)
3321                 return RTE_ETH_TX_DESC_DONE;
3322
3323         raw_cons = cpr->cp_raw_cons;
3324         cp_desc_ring = cpr->cp_desc_ring;
3325         ring_mask = cpr->cp_ring_struct->ring_mask;
3326
3327         /* Check to see if hw has posted a completion for the descriptor. */
3328         while (1) {
3329                 struct tx_cmpl *txcmp;
3330                 uint32_t cons;
3331
3332                 cons = RING_CMPL(ring_mask, raw_cons);
3333                 txcmp = (struct tx_cmpl *)&cp_desc_ring[cons];
3334
3335                 if (!bnxt_cpr_cmp_valid(txcmp, raw_cons, ring_mask + 1))
3336                         break;
3337
3338                 if (CMP_TYPE(txcmp) == TX_CMPL_TYPE_TX_L2)
3339                         nb_tx_pkts += rte_le_to_cpu_32(txcmp->opaque);
3340
3341                 if (nb_tx_pkts > offset)
3342                         return RTE_ETH_TX_DESC_DONE;
3343
3344                 raw_cons = NEXT_RAW_CMP(raw_cons);
3345         }
3346
3347         /* Descriptor is pending transmit, not yet completed by hardware. */
3348         return RTE_ETH_TX_DESC_FULL;
3349 }
3350
3351 int
3352 bnxt_flow_ops_get_op(struct rte_eth_dev *dev,
3353                      const struct rte_flow_ops **ops)
3354 {
3355         struct bnxt *bp = dev->data->dev_private;
3356         int ret = 0;
3357
3358         if (!bp)
3359                 return -EIO;
3360
3361         if (BNXT_ETH_DEV_IS_REPRESENTOR(dev)) {
3362                 struct bnxt_representor *vfr = dev->data->dev_private;
3363                 bp = vfr->parent_dev->data->dev_private;
3364                 /* parent is deleted while children are still valid */
3365                 if (!bp) {
3366                         PMD_DRV_LOG(DEBUG, "BNXT Port:%d VFR Error\n",
3367                                     dev->data->port_id);
3368                         return -EIO;
3369                 }
3370         }
3371
3372         ret = is_bnxt_in_error(bp);
3373         if (ret)
3374                 return ret;
3375
3376         /* PMD supports thread-safe flow operations.  rte_flow API
3377          * functions can avoid mutex for multi-thread safety.
3378          */
3379         dev->data->dev_flags |= RTE_ETH_DEV_FLOW_OPS_THREAD_SAFE;
3380
3381         if (BNXT_TRUFLOW_EN(bp))
3382                 *ops = &bnxt_ulp_rte_flow_ops;
3383         else
3384                 *ops = &bnxt_flow_ops;
3385
3386         return ret;
3387 }
3388
3389 static const uint32_t *
3390 bnxt_dev_supported_ptypes_get_op(struct rte_eth_dev *dev)
3391 {
3392         static const uint32_t ptypes[] = {
3393                 RTE_PTYPE_L2_ETHER_VLAN,
3394                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
3395                 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
3396                 RTE_PTYPE_L4_ICMP,
3397                 RTE_PTYPE_L4_TCP,
3398                 RTE_PTYPE_L4_UDP,
3399                 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
3400                 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
3401                 RTE_PTYPE_INNER_L4_ICMP,
3402                 RTE_PTYPE_INNER_L4_TCP,
3403                 RTE_PTYPE_INNER_L4_UDP,
3404                 RTE_PTYPE_UNKNOWN
3405         };
3406
3407         if (!dev->rx_pkt_burst)
3408                 return NULL;
3409
3410         return ptypes;
3411 }
3412
3413 static int bnxt_map_regs(struct bnxt *bp, uint32_t *reg_arr, int count,
3414                          int reg_win)
3415 {
3416         uint32_t reg_base = *reg_arr & 0xfffff000;
3417         uint32_t win_off;
3418         int i;
3419
3420         for (i = 0; i < count; i++) {
3421                 if ((reg_arr[i] & 0xfffff000) != reg_base)
3422                         return -ERANGE;
3423         }
3424         win_off = BNXT_GRCPF_REG_WINDOW_BASE_OUT + (reg_win - 1) * 4;
3425         rte_write32(reg_base, (uint8_t *)bp->bar0 + win_off);
3426         return 0;
3427 }
3428
3429 static int bnxt_map_ptp_regs(struct bnxt *bp)
3430 {
3431         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3432         uint32_t *reg_arr;
3433         int rc, i;
3434
3435         reg_arr = ptp->rx_regs;
3436         rc = bnxt_map_regs(bp, reg_arr, BNXT_PTP_RX_REGS, 5);
3437         if (rc)
3438                 return rc;
3439
3440         reg_arr = ptp->tx_regs;
3441         rc = bnxt_map_regs(bp, reg_arr, BNXT_PTP_TX_REGS, 6);
3442         if (rc)
3443                 return rc;
3444
3445         for (i = 0; i < BNXT_PTP_RX_REGS; i++)
3446                 ptp->rx_mapped_regs[i] = 0x5000 + (ptp->rx_regs[i] & 0xfff);
3447
3448         for (i = 0; i < BNXT_PTP_TX_REGS; i++)
3449                 ptp->tx_mapped_regs[i] = 0x6000 + (ptp->tx_regs[i] & 0xfff);
3450
3451         return 0;
3452 }
3453
3454 static void bnxt_unmap_ptp_regs(struct bnxt *bp)
3455 {
3456         rte_write32(0, (uint8_t *)bp->bar0 +
3457                          BNXT_GRCPF_REG_WINDOW_BASE_OUT + 16);
3458         rte_write32(0, (uint8_t *)bp->bar0 +
3459                          BNXT_GRCPF_REG_WINDOW_BASE_OUT + 20);
3460 }
3461
3462 static uint64_t bnxt_cc_read(struct bnxt *bp)
3463 {
3464         uint64_t ns;
3465
3466         ns = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3467                               BNXT_GRCPF_REG_SYNC_TIME));
3468         ns |= (uint64_t)(rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3469                                           BNXT_GRCPF_REG_SYNC_TIME + 4))) << 32;
3470         return ns;
3471 }
3472
3473 static int bnxt_get_tx_ts(struct bnxt *bp, uint64_t *ts)
3474 {
3475         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3476         uint32_t fifo;
3477
3478         fifo = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3479                                 ptp->tx_mapped_regs[BNXT_PTP_TX_FIFO]));
3480         if (fifo & BNXT_PTP_TX_FIFO_EMPTY)
3481                 return -EAGAIN;
3482
3483         fifo = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3484                                 ptp->tx_mapped_regs[BNXT_PTP_TX_FIFO]));
3485         *ts = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3486                                 ptp->tx_mapped_regs[BNXT_PTP_TX_TS_L]));
3487         *ts |= (uint64_t)rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3488                                 ptp->tx_mapped_regs[BNXT_PTP_TX_TS_H])) << 32;
3489         rte_read32((uint8_t *)bp->bar0 + ptp->tx_mapped_regs[BNXT_PTP_TX_SEQ]);
3490
3491         return 0;
3492 }
3493
3494 static int bnxt_clr_rx_ts(struct bnxt *bp, uint64_t *last_ts)
3495 {
3496         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3497         struct bnxt_pf_info *pf = bp->pf;
3498         uint16_t port_id;
3499         int i = 0;
3500         uint32_t fifo;
3501
3502         if (!ptp || (bp->flags & BNXT_FLAG_CHIP_P5))
3503                 return -EINVAL;
3504
3505         port_id = pf->port_id;
3506         fifo = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3507                                 ptp->rx_mapped_regs[BNXT_PTP_RX_FIFO]));
3508         while ((fifo & BNXT_PTP_RX_FIFO_PENDING) && (i < BNXT_PTP_RX_PND_CNT)) {
3509                 rte_write32(1 << port_id, (uint8_t *)bp->bar0 +
3510                             ptp->rx_mapped_regs[BNXT_PTP_RX_FIFO_ADV]);
3511                 fifo = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3512                                         ptp->rx_mapped_regs[BNXT_PTP_RX_FIFO]));
3513                 *last_ts = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3514                                         ptp->rx_mapped_regs[BNXT_PTP_RX_TS_L]));
3515                 *last_ts |= (uint64_t)rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3516                                         ptp->rx_mapped_regs[BNXT_PTP_RX_TS_H])) << 32;
3517                 i++;
3518         }
3519
3520         if (i >= BNXT_PTP_RX_PND_CNT)
3521                 return -EBUSY;
3522
3523         return 0;
3524 }
3525
3526 static int bnxt_get_rx_ts(struct bnxt *bp, uint64_t *ts)
3527 {
3528         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3529         struct bnxt_pf_info *pf = bp->pf;
3530         uint16_t port_id;
3531         uint32_t fifo;
3532
3533         fifo = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3534                                 ptp->rx_mapped_regs[BNXT_PTP_RX_FIFO]));
3535         if (!(fifo & BNXT_PTP_RX_FIFO_PENDING))
3536                 return -EAGAIN;
3537
3538         port_id = pf->port_id;
3539         rte_write32(1 << port_id, (uint8_t *)bp->bar0 +
3540                ptp->rx_mapped_regs[BNXT_PTP_RX_FIFO_ADV]);
3541
3542         fifo = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3543                                    ptp->rx_mapped_regs[BNXT_PTP_RX_FIFO]));
3544         if (fifo & BNXT_PTP_RX_FIFO_PENDING)
3545                 return bnxt_clr_rx_ts(bp, ts);
3546
3547         *ts = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3548                                 ptp->rx_mapped_regs[BNXT_PTP_RX_TS_L]));
3549         *ts |= (uint64_t)rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3550                                 ptp->rx_mapped_regs[BNXT_PTP_RX_TS_H])) << 32;
3551
3552         return 0;
3553 }
3554
3555 static int
3556 bnxt_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts)
3557 {
3558         uint64_t ns;
3559         struct bnxt *bp = dev->data->dev_private;
3560         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3561
3562         if (!ptp)
3563                 return -ENOTSUP;
3564
3565         ns = rte_timespec_to_ns(ts);
3566         /* Set the timecounters to a new value. */
3567         ptp->tc.nsec = ns;
3568         ptp->tx_tstamp_tc.nsec = ns;
3569         ptp->rx_tstamp_tc.nsec = ns;
3570
3571         return 0;
3572 }
3573
3574 static int
3575 bnxt_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts)
3576 {
3577         struct bnxt *bp = dev->data->dev_private;
3578         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3579         uint64_t ns, systime_cycles = 0;
3580         int rc = 0;
3581
3582         if (!ptp)
3583                 return -ENOTSUP;
3584
3585         if (BNXT_CHIP_P5(bp))
3586                 rc = bnxt_hwrm_port_ts_query(bp, BNXT_PTP_FLAGS_CURRENT_TIME,
3587                                              &systime_cycles);
3588         else
3589                 systime_cycles = bnxt_cc_read(bp);
3590
3591         ns = rte_timecounter_update(&ptp->tc, systime_cycles);
3592         *ts = rte_ns_to_timespec(ns);
3593
3594         return rc;
3595 }
3596 static int
3597 bnxt_timesync_enable(struct rte_eth_dev *dev)
3598 {
3599         struct bnxt *bp = dev->data->dev_private;
3600         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3601         uint32_t shift = 0;
3602         int rc;
3603
3604         if (!ptp)
3605                 return -ENOTSUP;
3606
3607         ptp->rx_filter = 1;
3608         ptp->tx_tstamp_en = 1;
3609         ptp->rxctl = BNXT_PTP_MSG_EVENTS;
3610
3611         rc = bnxt_hwrm_ptp_cfg(bp);
3612         if (rc)
3613                 return rc;
3614
3615         memset(&ptp->tc, 0, sizeof(struct rte_timecounter));
3616         memset(&ptp->rx_tstamp_tc, 0, sizeof(struct rte_timecounter));
3617         memset(&ptp->tx_tstamp_tc, 0, sizeof(struct rte_timecounter));
3618
3619         ptp->tc.cc_mask = BNXT_CYCLECOUNTER_MASK;
3620         ptp->tc.cc_shift = shift;
3621         ptp->tc.nsec_mask = (1ULL << shift) - 1;
3622
3623         ptp->rx_tstamp_tc.cc_mask = BNXT_CYCLECOUNTER_MASK;
3624         ptp->rx_tstamp_tc.cc_shift = shift;
3625         ptp->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
3626
3627         ptp->tx_tstamp_tc.cc_mask = BNXT_CYCLECOUNTER_MASK;
3628         ptp->tx_tstamp_tc.cc_shift = shift;
3629         ptp->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
3630
3631         if (!BNXT_CHIP_P5(bp))
3632                 bnxt_map_ptp_regs(bp);
3633         else
3634                 rc = bnxt_ptp_start(bp);
3635
3636         return rc;
3637 }
3638
3639 static int
3640 bnxt_timesync_disable(struct rte_eth_dev *dev)
3641 {
3642         struct bnxt *bp = dev->data->dev_private;
3643         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3644
3645         if (!ptp)
3646                 return -ENOTSUP;
3647
3648         ptp->rx_filter = 0;
3649         ptp->tx_tstamp_en = 0;
3650         ptp->rxctl = 0;
3651
3652         bnxt_hwrm_ptp_cfg(bp);
3653
3654         if (!BNXT_CHIP_P5(bp))
3655                 bnxt_unmap_ptp_regs(bp);
3656         else
3657                 bnxt_ptp_stop(bp);
3658
3659         return 0;
3660 }
3661
3662 static int
3663 bnxt_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
3664                                  struct timespec *timestamp,
3665                                  uint32_t flags __rte_unused)
3666 {
3667         struct bnxt *bp = dev->data->dev_private;
3668         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3669         uint64_t rx_tstamp_cycles = 0;
3670         uint64_t ns;
3671
3672         if (!ptp)
3673                 return -ENOTSUP;
3674
3675         if (BNXT_CHIP_P5(bp))
3676                 rx_tstamp_cycles = ptp->rx_timestamp;
3677         else
3678                 bnxt_get_rx_ts(bp, &rx_tstamp_cycles);
3679
3680         ns = rte_timecounter_update(&ptp->rx_tstamp_tc, rx_tstamp_cycles);
3681         *timestamp = rte_ns_to_timespec(ns);
3682         return  0;
3683 }
3684
3685 static int
3686 bnxt_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
3687                                  struct timespec *timestamp)
3688 {
3689         struct bnxt *bp = dev->data->dev_private;
3690         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3691         uint64_t tx_tstamp_cycles = 0;
3692         uint64_t ns;
3693         int rc = 0;
3694
3695         if (!ptp)
3696                 return -ENOTSUP;
3697
3698         if (BNXT_CHIP_P5(bp))
3699                 rc = bnxt_hwrm_port_ts_query(bp, BNXT_PTP_FLAGS_PATH_TX,
3700                                              &tx_tstamp_cycles);
3701         else
3702                 rc = bnxt_get_tx_ts(bp, &tx_tstamp_cycles);
3703
3704         ns = rte_timecounter_update(&ptp->tx_tstamp_tc, tx_tstamp_cycles);
3705         *timestamp = rte_ns_to_timespec(ns);
3706
3707         return rc;
3708 }
3709
3710 static int
3711 bnxt_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta)
3712 {
3713         struct bnxt *bp = dev->data->dev_private;
3714         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3715
3716         if (!ptp)
3717                 return -ENOTSUP;
3718
3719         ptp->tc.nsec += delta;
3720         ptp->tx_tstamp_tc.nsec += delta;
3721         ptp->rx_tstamp_tc.nsec += delta;
3722
3723         return 0;
3724 }
3725
3726 static int
3727 bnxt_get_eeprom_length_op(struct rte_eth_dev *dev)
3728 {
3729         struct bnxt *bp = dev->data->dev_private;
3730         int rc;
3731         uint32_t dir_entries;
3732         uint32_t entry_length;
3733
3734         rc = is_bnxt_in_error(bp);
3735         if (rc)
3736                 return rc;
3737
3738         PMD_DRV_LOG(INFO, PCI_PRI_FMT "\n",
3739                     bp->pdev->addr.domain, bp->pdev->addr.bus,
3740                     bp->pdev->addr.devid, bp->pdev->addr.function);
3741
3742         rc = bnxt_hwrm_nvm_get_dir_info(bp, &dir_entries, &entry_length);
3743         if (rc != 0)
3744                 return rc;
3745
3746         return dir_entries * entry_length;
3747 }
3748
3749 static int
3750 bnxt_get_eeprom_op(struct rte_eth_dev *dev,
3751                 struct rte_dev_eeprom_info *in_eeprom)
3752 {
3753         struct bnxt *bp = dev->data->dev_private;
3754         uint32_t index;
3755         uint32_t offset;
3756         int rc;
3757
3758         rc = is_bnxt_in_error(bp);
3759         if (rc)
3760                 return rc;
3761
3762         PMD_DRV_LOG(INFO, PCI_PRI_FMT " in_eeprom->offset = %d len = %d\n",
3763                     bp->pdev->addr.domain, bp->pdev->addr.bus,
3764                     bp->pdev->addr.devid, bp->pdev->addr.function,
3765                     in_eeprom->offset, in_eeprom->length);
3766
3767         if (in_eeprom->offset == 0) /* special offset value to get directory */
3768                 return bnxt_get_nvram_directory(bp, in_eeprom->length,
3769                                                 in_eeprom->data);
3770
3771         index = in_eeprom->offset >> 24;
3772         offset = in_eeprom->offset & 0xffffff;
3773
3774         if (index != 0)
3775                 return bnxt_hwrm_get_nvram_item(bp, index - 1, offset,
3776                                            in_eeprom->length, in_eeprom->data);
3777
3778         return 0;
3779 }
3780
3781 static bool bnxt_dir_type_is_ape_bin_format(uint16_t dir_type)
3782 {
3783         switch (dir_type) {
3784         case BNX_DIR_TYPE_CHIMP_PATCH:
3785         case BNX_DIR_TYPE_BOOTCODE:
3786         case BNX_DIR_TYPE_BOOTCODE_2:
3787         case BNX_DIR_TYPE_APE_FW:
3788         case BNX_DIR_TYPE_APE_PATCH:
3789         case BNX_DIR_TYPE_KONG_FW:
3790         case BNX_DIR_TYPE_KONG_PATCH:
3791         case BNX_DIR_TYPE_BONO_FW:
3792         case BNX_DIR_TYPE_BONO_PATCH:
3793                 /* FALLTHROUGH */
3794                 return true;
3795         }
3796
3797         return false;
3798 }
3799
3800 static bool bnxt_dir_type_is_other_exec_format(uint16_t dir_type)
3801 {
3802         switch (dir_type) {
3803         case BNX_DIR_TYPE_AVS:
3804         case BNX_DIR_TYPE_EXP_ROM_MBA:
3805         case BNX_DIR_TYPE_PCIE:
3806         case BNX_DIR_TYPE_TSCF_UCODE:
3807         case BNX_DIR_TYPE_EXT_PHY:
3808         case BNX_DIR_TYPE_CCM:
3809         case BNX_DIR_TYPE_ISCSI_BOOT:
3810         case BNX_DIR_TYPE_ISCSI_BOOT_IPV6:
3811         case BNX_DIR_TYPE_ISCSI_BOOT_IPV4N6:
3812                 /* FALLTHROUGH */
3813                 return true;
3814         }
3815
3816         return false;
3817 }
3818
3819 static bool bnxt_dir_type_is_executable(uint16_t dir_type)
3820 {
3821         return bnxt_dir_type_is_ape_bin_format(dir_type) ||
3822                 bnxt_dir_type_is_other_exec_format(dir_type);
3823 }
3824
3825 static int
3826 bnxt_set_eeprom_op(struct rte_eth_dev *dev,
3827                 struct rte_dev_eeprom_info *in_eeprom)
3828 {
3829         struct bnxt *bp = dev->data->dev_private;
3830         uint8_t index, dir_op;
3831         uint16_t type, ext, ordinal, attr;
3832         int rc;
3833
3834         rc = is_bnxt_in_error(bp);
3835         if (rc)
3836                 return rc;
3837
3838         PMD_DRV_LOG(INFO, PCI_PRI_FMT " in_eeprom->offset = %d len = %d\n",
3839                     bp->pdev->addr.domain, bp->pdev->addr.bus,
3840                     bp->pdev->addr.devid, bp->pdev->addr.function,
3841                     in_eeprom->offset, in_eeprom->length);
3842
3843         if (!BNXT_PF(bp)) {
3844                 PMD_DRV_LOG(ERR, "NVM write not supported from a VF\n");
3845                 return -EINVAL;
3846         }
3847
3848         type = in_eeprom->magic >> 16;
3849
3850         if (type == 0xffff) { /* special value for directory operations */
3851                 index = in_eeprom->magic & 0xff;
3852                 dir_op = in_eeprom->magic >> 8;
3853                 if (index == 0)
3854                         return -EINVAL;
3855                 switch (dir_op) {
3856                 case 0x0e: /* erase */
3857                         if (in_eeprom->offset != ~in_eeprom->magic)
3858                                 return -EINVAL;
3859                         return bnxt_hwrm_erase_nvram_directory(bp, index - 1);
3860                 default:
3861                         return -EINVAL;
3862                 }
3863         }
3864
3865         /* Create or re-write an NVM item: */
3866         if (bnxt_dir_type_is_executable(type) == true)
3867                 return -EOPNOTSUPP;
3868         ext = in_eeprom->magic & 0xffff;
3869         ordinal = in_eeprom->offset >> 16;
3870         attr = in_eeprom->offset & 0xffff;
3871
3872         return bnxt_hwrm_flash_nvram(bp, type, ordinal, ext, attr,
3873                                      in_eeprom->data, in_eeprom->length);
3874 }
3875
3876 static int bnxt_get_module_info(struct rte_eth_dev *dev,
3877                                 struct rte_eth_dev_module_info *modinfo)
3878 {
3879         uint8_t module_info[SFF_DIAG_SUPPORT_OFFSET + 1];
3880         struct bnxt *bp = dev->data->dev_private;
3881         int rc;
3882
3883         /* No point in going further if phy status indicates
3884          * module is not inserted or if it is powered down or
3885          * if it is of type 10GBase-T
3886          */
3887         if (bp->link_info->module_status >
3888             HWRM_PORT_PHY_QCFG_OUTPUT_MODULE_STATUS_WARNINGMSG) {
3889                 PMD_DRV_LOG(NOTICE, "Port %u : Module is not inserted or is powered down\n",
3890                             dev->data->port_id);
3891                 return -ENOTSUP;
3892         }
3893
3894         /* This feature is not supported in older firmware versions */
3895         if (bp->hwrm_spec_code < 0x10202) {
3896                 PMD_DRV_LOG(NOTICE, "Port %u : Feature is not supported in older firmware\n",
3897                             dev->data->port_id);
3898                 return -ENOTSUP;
3899         }
3900
3901         rc = bnxt_hwrm_read_sfp_module_eeprom_info(bp, I2C_DEV_ADDR_A0, 0, 0,
3902                                                    SFF_DIAG_SUPPORT_OFFSET + 1,
3903                                                    module_info);
3904
3905         if (rc)
3906                 return rc;
3907
3908         switch (module_info[0]) {
3909         case SFF_MODULE_ID_SFP:
3910                 modinfo->type = RTE_ETH_MODULE_SFF_8472;
3911                 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8472_LEN;
3912                 if (module_info[SFF_DIAG_SUPPORT_OFFSET] == 0)
3913                         modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8436_LEN;
3914                 break;
3915         case SFF_MODULE_ID_QSFP:
3916         case SFF_MODULE_ID_QSFP_PLUS:
3917                 modinfo->type = RTE_ETH_MODULE_SFF_8436;
3918                 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8436_LEN;
3919                 break;
3920         case SFF_MODULE_ID_QSFP28:
3921                 modinfo->type = RTE_ETH_MODULE_SFF_8636;
3922                 modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8636_MAX_LEN;
3923                 if (module_info[SFF8636_FLATMEM_OFFSET] & SFF8636_FLATMEM_MASK)
3924                         modinfo->eeprom_len = RTE_ETH_MODULE_SFF_8636_LEN;
3925                 break;
3926         default:
3927                 PMD_DRV_LOG(NOTICE, "Port %u : Unsupported module\n", dev->data->port_id);
3928                 return -ENOTSUP;
3929         }
3930
3931         PMD_DRV_LOG(INFO, "Port %u : modinfo->type = %d modinfo->eeprom_len = %d\n",
3932                     dev->data->port_id, modinfo->type, modinfo->eeprom_len);
3933
3934         return 0;
3935 }
3936
3937 static int bnxt_get_module_eeprom(struct rte_eth_dev *dev,
3938                                   struct rte_dev_eeprom_info *info)
3939 {
3940         uint8_t pg_addr[5] = { I2C_DEV_ADDR_A0, I2C_DEV_ADDR_A0 };
3941         uint32_t offset = info->offset, length = info->length;
3942         uint8_t module_info[SFF_DIAG_SUPPORT_OFFSET + 1];
3943         struct bnxt *bp = dev->data->dev_private;
3944         uint8_t *data = info->data;
3945         uint8_t page = offset >> 7;
3946         uint8_t max_pages = 2;
3947         uint8_t opt_pages;
3948         int rc;
3949
3950         rc = bnxt_hwrm_read_sfp_module_eeprom_info(bp, I2C_DEV_ADDR_A0, 0, 0,
3951                                                    SFF_DIAG_SUPPORT_OFFSET + 1,
3952                                                    module_info);
3953         if (rc)
3954                 return rc;
3955
3956         switch (module_info[0]) {
3957         case SFF_MODULE_ID_SFP:
3958                 module_info[SFF_DIAG_SUPPORT_OFFSET] = 0;
3959                 if (module_info[SFF_DIAG_SUPPORT_OFFSET]) {
3960                         pg_addr[2] = I2C_DEV_ADDR_A2;
3961                         pg_addr[3] = I2C_DEV_ADDR_A2;
3962                         max_pages = 4;
3963                 }
3964                 break;
3965         case SFF_MODULE_ID_QSFP28:
3966                 rc = bnxt_hwrm_read_sfp_module_eeprom_info(bp, I2C_DEV_ADDR_A0, 0,
3967                                                            SFF8636_OPT_PAGES_OFFSET,
3968                                                            1, &opt_pages);
3969                 if (rc)
3970                         return rc;
3971
3972                 if (opt_pages & SFF8636_PAGE1_MASK) {
3973                         pg_addr[2] = I2C_DEV_ADDR_A0;
3974                         max_pages = 3;
3975                 }
3976                 if (opt_pages & SFF8636_PAGE2_MASK) {
3977                         pg_addr[3] = I2C_DEV_ADDR_A0;
3978                         max_pages = 4;
3979                 }
3980                 if (~module_info[SFF8636_FLATMEM_OFFSET] & SFF8636_FLATMEM_MASK) {
3981                         pg_addr[4] = I2C_DEV_ADDR_A0;
3982                         max_pages = 5;
3983                 }
3984                 break;
3985         default:
3986                 break;
3987         }
3988
3989         memset(data, 0, length);
3990
3991         offset &= 0xff;
3992         while (length && page < max_pages) {
3993                 uint8_t raw_page = page ? page - 1 : 0;
3994                 uint16_t chunk;
3995
3996                 if (pg_addr[page] == I2C_DEV_ADDR_A2)
3997                         raw_page = 0;
3998                 else if (page)
3999                         offset |= 0x80;
4000                 chunk = RTE_MIN(length, 256 - offset);
4001
4002                 if (pg_addr[page]) {
4003                         rc = bnxt_hwrm_read_sfp_module_eeprom_info(bp, pg_addr[page],
4004                                                                    raw_page, offset,
4005                                                                    chunk, data);
4006                         if (rc)
4007                                 return rc;
4008                 }
4009
4010                 data += chunk;
4011                 length -= chunk;
4012                 offset = 0;
4013                 page += 1 + (chunk > 128);
4014         }
4015
4016         return length ? -EINVAL : 0;
4017 }
4018
4019 /*
4020  * Initialization
4021  */
4022
4023 static const struct eth_dev_ops bnxt_dev_ops = {
4024         .dev_infos_get = bnxt_dev_info_get_op,
4025         .dev_close = bnxt_dev_close_op,
4026         .dev_configure = bnxt_dev_configure_op,
4027         .dev_start = bnxt_dev_start_op,
4028         .dev_stop = bnxt_dev_stop_op,
4029         .dev_set_link_up = bnxt_dev_set_link_up_op,
4030         .dev_set_link_down = bnxt_dev_set_link_down_op,
4031         .stats_get = bnxt_stats_get_op,
4032         .stats_reset = bnxt_stats_reset_op,
4033         .rx_queue_setup = bnxt_rx_queue_setup_op,
4034         .rx_queue_release = bnxt_rx_queue_release_op,
4035         .tx_queue_setup = bnxt_tx_queue_setup_op,
4036         .tx_queue_release = bnxt_tx_queue_release_op,
4037         .rx_queue_intr_enable = bnxt_rx_queue_intr_enable_op,
4038         .rx_queue_intr_disable = bnxt_rx_queue_intr_disable_op,
4039         .reta_update = bnxt_reta_update_op,
4040         .reta_query = bnxt_reta_query_op,
4041         .rss_hash_update = bnxt_rss_hash_update_op,
4042         .rss_hash_conf_get = bnxt_rss_hash_conf_get_op,
4043         .link_update = bnxt_link_update_op,
4044         .promiscuous_enable = bnxt_promiscuous_enable_op,
4045         .promiscuous_disable = bnxt_promiscuous_disable_op,
4046         .allmulticast_enable = bnxt_allmulticast_enable_op,
4047         .allmulticast_disable = bnxt_allmulticast_disable_op,
4048         .mac_addr_add = bnxt_mac_addr_add_op,
4049         .mac_addr_remove = bnxt_mac_addr_remove_op,
4050         .flow_ctrl_get = bnxt_flow_ctrl_get_op,
4051         .flow_ctrl_set = bnxt_flow_ctrl_set_op,
4052         .udp_tunnel_port_add  = bnxt_udp_tunnel_port_add_op,
4053         .udp_tunnel_port_del  = bnxt_udp_tunnel_port_del_op,
4054         .vlan_filter_set = bnxt_vlan_filter_set_op,
4055         .vlan_offload_set = bnxt_vlan_offload_set_op,
4056         .vlan_tpid_set = bnxt_vlan_tpid_set_op,
4057         .vlan_pvid_set = bnxt_vlan_pvid_set_op,
4058         .mtu_set = bnxt_mtu_set_op,
4059         .mac_addr_set = bnxt_set_default_mac_addr_op,
4060         .xstats_get = bnxt_dev_xstats_get_op,
4061         .xstats_get_names = bnxt_dev_xstats_get_names_op,
4062         .xstats_reset = bnxt_dev_xstats_reset_op,
4063         .fw_version_get = bnxt_fw_version_get,
4064         .set_mc_addr_list = bnxt_dev_set_mc_addr_list_op,
4065         .rxq_info_get = bnxt_rxq_info_get_op,
4066         .txq_info_get = bnxt_txq_info_get_op,
4067         .rx_burst_mode_get = bnxt_rx_burst_mode_get,
4068         .tx_burst_mode_get = bnxt_tx_burst_mode_get,
4069         .dev_led_on = bnxt_dev_led_on_op,
4070         .dev_led_off = bnxt_dev_led_off_op,
4071         .rx_queue_start = bnxt_rx_queue_start,
4072         .rx_queue_stop = bnxt_rx_queue_stop,
4073         .tx_queue_start = bnxt_tx_queue_start,
4074         .tx_queue_stop = bnxt_tx_queue_stop,
4075         .flow_ops_get = bnxt_flow_ops_get_op,
4076         .dev_supported_ptypes_get = bnxt_dev_supported_ptypes_get_op,
4077         .get_eeprom_length    = bnxt_get_eeprom_length_op,
4078         .get_eeprom           = bnxt_get_eeprom_op,
4079         .set_eeprom           = bnxt_set_eeprom_op,
4080         .get_module_info = bnxt_get_module_info,
4081         .get_module_eeprom = bnxt_get_module_eeprom,
4082         .timesync_enable      = bnxt_timesync_enable,
4083         .timesync_disable     = bnxt_timesync_disable,
4084         .timesync_read_time   = bnxt_timesync_read_time,
4085         .timesync_write_time   = bnxt_timesync_write_time,
4086         .timesync_adjust_time = bnxt_timesync_adjust_time,
4087         .timesync_read_rx_timestamp = bnxt_timesync_read_rx_timestamp,
4088         .timesync_read_tx_timestamp = bnxt_timesync_read_tx_timestamp,
4089 };
4090
4091 static uint32_t bnxt_map_reset_regs(struct bnxt *bp, uint32_t reg)
4092 {
4093         uint32_t offset;
4094
4095         /* Only pre-map the reset GRC registers using window 3 */
4096         rte_write32(reg & 0xfffff000, (uint8_t *)bp->bar0 +
4097                     BNXT_GRCPF_REG_WINDOW_BASE_OUT + 8);
4098
4099         offset = BNXT_GRCP_WINDOW_3_BASE + (reg & 0xffc);
4100
4101         return offset;
4102 }
4103
4104 int bnxt_map_fw_health_status_regs(struct bnxt *bp)
4105 {
4106         struct bnxt_error_recovery_info *info = bp->recovery_info;
4107         uint32_t reg_base = 0xffffffff;
4108         int i;
4109
4110         /* Only pre-map the monitoring GRC registers using window 2 */
4111         for (i = 0; i < BNXT_FW_STATUS_REG_CNT; i++) {
4112                 uint32_t reg = info->status_regs[i];
4113
4114                 if (BNXT_FW_STATUS_REG_TYPE(reg) != BNXT_FW_STATUS_REG_TYPE_GRC)
4115                         continue;
4116
4117                 if (reg_base == 0xffffffff)
4118                         reg_base = reg & 0xfffff000;
4119                 if ((reg & 0xfffff000) != reg_base)
4120                         return -ERANGE;
4121
4122                 /* Use mask 0xffc as the Lower 2 bits indicates
4123                  * address space location
4124                  */
4125                 info->mapped_status_regs[i] = BNXT_GRCP_WINDOW_2_BASE +
4126                                                 (reg & 0xffc);
4127         }
4128
4129         if (reg_base == 0xffffffff)
4130                 return 0;
4131
4132         rte_write32(reg_base, (uint8_t *)bp->bar0 +
4133                     BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
4134
4135         return 0;
4136 }
4137
4138 static void bnxt_write_fw_reset_reg(struct bnxt *bp, uint32_t index)
4139 {
4140         struct bnxt_error_recovery_info *info = bp->recovery_info;
4141         uint32_t delay = info->delay_after_reset[index];
4142         uint32_t val = info->reset_reg_val[index];
4143         uint32_t reg = info->reset_reg[index];
4144         uint32_t type, offset;
4145         int ret;
4146
4147         type = BNXT_FW_STATUS_REG_TYPE(reg);
4148         offset = BNXT_FW_STATUS_REG_OFF(reg);
4149
4150         switch (type) {
4151         case BNXT_FW_STATUS_REG_TYPE_CFG:
4152                 ret = rte_pci_write_config(bp->pdev, &val, sizeof(val), offset);
4153                 if (ret < 0) {
4154                         PMD_DRV_LOG(ERR, "Failed to write %#x at PCI offset %#x",
4155                                     val, offset);
4156                         return;
4157                 }
4158                 break;
4159         case BNXT_FW_STATUS_REG_TYPE_GRC:
4160                 offset = bnxt_map_reset_regs(bp, offset);
4161                 rte_write32(val, (uint8_t *)bp->bar0 + offset);
4162                 break;
4163         case BNXT_FW_STATUS_REG_TYPE_BAR0:
4164                 rte_write32(val, (uint8_t *)bp->bar0 + offset);
4165                 break;
4166         }
4167         /* wait on a specific interval of time until core reset is complete */
4168         if (delay)
4169                 rte_delay_ms(delay);
4170 }
4171
4172 static void bnxt_dev_cleanup(struct bnxt *bp)
4173 {
4174         bp->eth_dev->data->dev_link.link_status = 0;
4175         bp->link_info->link_up = 0;
4176         if (bp->eth_dev->data->dev_started)
4177                 bnxt_dev_stop(bp->eth_dev);
4178
4179         bnxt_uninit_resources(bp, true);
4180 }
4181
4182 static int
4183 bnxt_check_fw_reset_done(struct bnxt *bp)
4184 {
4185         int timeout = bp->fw_reset_max_msecs;
4186         uint16_t val = 0;
4187         int rc;
4188
4189         do {
4190                 rc = rte_pci_read_config(bp->pdev, &val, sizeof(val), PCI_SUBSYSTEM_ID_OFFSET);
4191                 if (rc < 0) {
4192                         PMD_DRV_LOG(ERR, "Failed to read PCI offset 0x%x", PCI_SUBSYSTEM_ID_OFFSET);
4193                         return rc;
4194                 }
4195                 if (val != 0xffff)
4196                         break;
4197                 rte_delay_ms(1);
4198         } while (timeout--);
4199
4200         if (val == 0xffff) {
4201                 PMD_DRV_LOG(ERR, "Firmware reset aborted, PCI config space invalid\n");
4202                 return -1;
4203         }
4204
4205         return 0;
4206 }
4207
4208 static int bnxt_restore_vlan_filters(struct bnxt *bp)
4209 {
4210         struct rte_eth_dev *dev = bp->eth_dev;
4211         struct rte_vlan_filter_conf *vfc;
4212         int vidx, vbit, rc;
4213         uint16_t vlan_id;
4214
4215         for (vlan_id = 1; vlan_id <= RTE_ETHER_MAX_VLAN_ID; vlan_id++) {
4216                 vfc = &dev->data->vlan_filter_conf;
4217                 vidx = vlan_id / 64;
4218                 vbit = vlan_id % 64;
4219
4220                 /* Each bit corresponds to a VLAN id */
4221                 if (vfc->ids[vidx] & (UINT64_C(1) << vbit)) {
4222                         rc = bnxt_add_vlan_filter(bp, vlan_id);
4223                         if (rc)
4224                                 return rc;
4225                 }
4226         }
4227
4228         return 0;
4229 }
4230
4231 static int bnxt_restore_mac_filters(struct bnxt *bp)
4232 {
4233         struct rte_eth_dev *dev = bp->eth_dev;
4234         struct rte_eth_dev_info dev_info;
4235         struct rte_ether_addr *addr;
4236         uint64_t pool_mask;
4237         uint32_t pool = 0;
4238         uint32_t i;
4239         int rc;
4240
4241         if (BNXT_VF(bp) && !BNXT_VF_IS_TRUSTED(bp))
4242                 return 0;
4243
4244         rc = bnxt_dev_info_get_op(dev, &dev_info);
4245         if (rc)
4246                 return rc;
4247
4248         /* replay MAC address configuration */
4249         for (i = 1; i < dev_info.max_mac_addrs; i++) {
4250                 addr = &dev->data->mac_addrs[i];
4251
4252                 /* skip zero address */
4253                 if (rte_is_zero_ether_addr(addr))
4254                         continue;
4255
4256                 pool = 0;
4257                 pool_mask = dev->data->mac_pool_sel[i];
4258
4259                 do {
4260                         if (pool_mask & 1ULL) {
4261                                 rc = bnxt_mac_addr_add_op(dev, addr, i, pool);
4262                                 if (rc)
4263                                         return rc;
4264                         }
4265                         pool_mask >>= 1;
4266                         pool++;
4267                 } while (pool_mask);
4268         }
4269
4270         return 0;
4271 }
4272
4273 static int bnxt_restore_mcast_mac_filters(struct bnxt *bp)
4274 {
4275         int ret = 0;
4276
4277         ret = bnxt_dev_set_mc_addr_list_op(bp->eth_dev, bp->mcast_addr_list,
4278                                            bp->nb_mc_addr);
4279         if (ret)
4280                 PMD_DRV_LOG(ERR, "Failed to restore multicast MAC addreeses\n");
4281
4282         return ret;
4283 }
4284
4285 static int bnxt_restore_filters(struct bnxt *bp)
4286 {
4287         struct rte_eth_dev *dev = bp->eth_dev;
4288         int ret = 0;
4289
4290         if (dev->data->all_multicast) {
4291                 ret = bnxt_allmulticast_enable_op(dev);
4292                 if (ret)
4293                         return ret;
4294         }
4295         if (dev->data->promiscuous) {
4296                 ret = bnxt_promiscuous_enable_op(dev);
4297                 if (ret)
4298                         return ret;
4299         }
4300
4301         ret = bnxt_restore_mac_filters(bp);
4302         if (ret)
4303                 return ret;
4304
4305         /* if vlans are already programmed, this can fail with -EEXIST */
4306         ret = bnxt_restore_vlan_filters(bp);
4307         if (ret && ret != -EEXIST)
4308                 return ret;
4309
4310         ret = bnxt_restore_mcast_mac_filters(bp);
4311         if (ret)
4312                 return ret;
4313
4314         return ret;
4315 }
4316
4317 static int bnxt_check_fw_ready(struct bnxt *bp)
4318 {
4319         int timeout = bp->fw_reset_max_msecs;
4320         int rc = 0;
4321
4322         do {
4323                 rc = bnxt_hwrm_poll_ver_get(bp);
4324                 if (rc == 0)
4325                         break;
4326                 rte_delay_ms(BNXT_FW_READY_WAIT_INTERVAL);
4327                 timeout -= BNXT_FW_READY_WAIT_INTERVAL;
4328         } while (rc && timeout > 0);
4329
4330         if (rc)
4331                 PMD_DRV_LOG(ERR, "FW is not Ready after reset\n");
4332
4333         return rc;
4334 }
4335
4336 static void bnxt_dev_recover(void *arg)
4337 {
4338         struct bnxt *bp = arg;
4339         int rc = 0;
4340
4341         pthread_mutex_lock(&bp->err_recovery_lock);
4342
4343         if (!bp->fw_reset_min_msecs) {
4344                 rc = bnxt_check_fw_reset_done(bp);
4345                 if (rc)
4346                         goto err;
4347         }
4348
4349         /* Clear Error flag so that device re-init should happen */
4350         bp->flags &= ~BNXT_FLAG_FATAL_ERROR;
4351         PMD_DRV_LOG(INFO, "Port: %u Starting recovery...\n",
4352                     bp->eth_dev->data->port_id);
4353
4354         rc = bnxt_check_fw_ready(bp);
4355         if (rc)
4356                 goto err;
4357
4358         rc = bnxt_init_resources(bp, true);
4359         if (rc) {
4360                 PMD_DRV_LOG(ERR,
4361                             "Failed to initialize resources after reset\n");
4362                 goto err;
4363         }
4364         /* clear reset flag as the device is initialized now */
4365         bp->flags &= ~BNXT_FLAG_FW_RESET;
4366
4367         rc = bnxt_dev_start_op(bp->eth_dev);
4368         if (rc) {
4369                 PMD_DRV_LOG(ERR, "Failed to start port after reset\n");
4370                 goto err_start;
4371         }
4372
4373         rc = bnxt_restore_filters(bp);
4374         if (rc)
4375                 goto err_start;
4376
4377         rte_eth_fp_ops[bp->eth_dev->data->port_id].rx_pkt_burst =
4378                 bp->eth_dev->rx_pkt_burst;
4379         rte_eth_fp_ops[bp->eth_dev->data->port_id].tx_pkt_burst =
4380                 bp->eth_dev->tx_pkt_burst;
4381         rte_mb();
4382
4383         PMD_DRV_LOG(INFO, "Port: %u Recovered from FW reset\n",
4384                     bp->eth_dev->data->port_id);
4385         pthread_mutex_unlock(&bp->err_recovery_lock);
4386
4387         return;
4388 err_start:
4389         bnxt_dev_stop(bp->eth_dev);
4390 err:
4391         bp->flags |= BNXT_FLAG_FATAL_ERROR;
4392         bnxt_uninit_resources(bp, false);
4393         if (bp->eth_dev->data->dev_conf.intr_conf.rmv)
4394                 rte_eth_dev_callback_process(bp->eth_dev,
4395                                              RTE_ETH_EVENT_INTR_RMV,
4396                                              NULL);
4397         pthread_mutex_unlock(&bp->err_recovery_lock);
4398         PMD_DRV_LOG(ERR, "Port %u: Failed to recover from FW reset\n",
4399                     bp->eth_dev->data->port_id);
4400 }
4401
4402 void bnxt_dev_reset_and_resume(void *arg)
4403 {
4404         struct bnxt *bp = arg;
4405         uint32_t us = US_PER_MS * bp->fw_reset_min_msecs;
4406         uint16_t val = 0;
4407         int rc;
4408
4409         bnxt_dev_cleanup(bp);
4410         PMD_DRV_LOG(INFO, "Port: %u Finished bnxt_dev_cleanup\n",
4411                     bp->eth_dev->data->port_id);
4412
4413         bnxt_wait_for_device_shutdown(bp);
4414
4415         /* During some fatal firmware error conditions, the PCI config space
4416          * register 0x2e which normally contains the subsystem ID will become
4417          * 0xffff. This register will revert back to the normal value after
4418          * the chip has completed core reset. If we detect this condition,
4419          * we can poll this config register immediately for the value to revert.
4420          */
4421         if (bp->flags & BNXT_FLAG_FATAL_ERROR) {
4422                 rc = rte_pci_read_config(bp->pdev, &val, sizeof(val), PCI_SUBSYSTEM_ID_OFFSET);
4423                 if (rc < 0) {
4424                         PMD_DRV_LOG(ERR, "Failed to read PCI offset 0x%x", PCI_SUBSYSTEM_ID_OFFSET);
4425                         return;
4426                 }
4427                 if (val == 0xffff) {
4428                         bp->fw_reset_min_msecs = 0;
4429                         us = 1;
4430                 }
4431         }
4432
4433         rc = rte_eal_alarm_set(us, bnxt_dev_recover, (void *)bp);
4434         if (rc)
4435                 PMD_DRV_LOG(ERR, "Port %u: Error setting recovery alarm",
4436                             bp->eth_dev->data->port_id);
4437 }
4438
4439 uint32_t bnxt_read_fw_status_reg(struct bnxt *bp, uint32_t index)
4440 {
4441         struct bnxt_error_recovery_info *info = bp->recovery_info;
4442         uint32_t reg = info->status_regs[index];
4443         uint32_t type, offset, val = 0;
4444         int ret = 0;
4445
4446         type = BNXT_FW_STATUS_REG_TYPE(reg);
4447         offset = BNXT_FW_STATUS_REG_OFF(reg);
4448
4449         switch (type) {
4450         case BNXT_FW_STATUS_REG_TYPE_CFG:
4451                 ret = rte_pci_read_config(bp->pdev, &val, sizeof(val), offset);
4452                 if (ret < 0)
4453                         PMD_DRV_LOG(ERR, "Failed to read PCI offset %#x",
4454                                     offset);
4455                 break;
4456         case BNXT_FW_STATUS_REG_TYPE_GRC:
4457                 offset = info->mapped_status_regs[index];
4458                 /* FALLTHROUGH */
4459         case BNXT_FW_STATUS_REG_TYPE_BAR0:
4460                 val = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
4461                                        offset));
4462                 break;
4463         }
4464
4465         return val;
4466 }
4467
4468 static int bnxt_fw_reset_all(struct bnxt *bp)
4469 {
4470         struct bnxt_error_recovery_info *info = bp->recovery_info;
4471         uint32_t i;
4472         int rc = 0;
4473
4474         if (info->flags & BNXT_FLAG_ERROR_RECOVERY_HOST) {
4475                 /* Reset through primary function driver */
4476                 for (i = 0; i < info->reg_array_cnt; i++)
4477                         bnxt_write_fw_reset_reg(bp, i);
4478                 /* Wait for time specified by FW after triggering reset */
4479                 rte_delay_ms(info->primary_func_wait_period_after_reset);
4480         } else if (info->flags & BNXT_FLAG_ERROR_RECOVERY_CO_CPU) {
4481                 /* Reset with the help of Kong processor */
4482                 rc = bnxt_hwrm_fw_reset(bp);
4483                 if (rc)
4484                         PMD_DRV_LOG(ERR, "Failed to reset FW\n");
4485         }
4486
4487         return rc;
4488 }
4489
4490 static void bnxt_fw_reset_cb(void *arg)
4491 {
4492         struct bnxt *bp = arg;
4493         struct bnxt_error_recovery_info *info = bp->recovery_info;
4494         int rc = 0;
4495
4496         /* Only Primary function can do FW reset */
4497         if (bnxt_is_primary_func(bp) &&
4498             bnxt_is_recovery_enabled(bp)) {
4499                 rc = bnxt_fw_reset_all(bp);
4500                 if (rc) {
4501                         PMD_DRV_LOG(ERR, "Adapter recovery failed\n");
4502                         return;
4503                 }
4504         }
4505
4506         /* if recovery method is ERROR_RECOVERY_CO_CPU, KONG will send
4507          * EXCEPTION_FATAL_ASYNC event to all the functions
4508          * (including MASTER FUNC). After receiving this Async, all the active
4509          * drivers should treat this case as FW initiated recovery
4510          */
4511         if (info->flags & BNXT_FLAG_ERROR_RECOVERY_HOST) {
4512                 bp->fw_reset_min_msecs = BNXT_MIN_FW_READY_TIMEOUT;
4513                 bp->fw_reset_max_msecs = BNXT_MAX_FW_RESET_TIMEOUT;
4514
4515                 /* To recover from error */
4516                 rte_eal_alarm_set(US_PER_MS, bnxt_dev_reset_and_resume,
4517                                   (void *)bp);
4518         }
4519 }
4520
4521 /* Driver should poll FW heartbeat, reset_counter with the frequency
4522  * advertised by FW in HWRM_ERROR_RECOVERY_QCFG.
4523  * When the driver detects heartbeat stop or change in reset_counter,
4524  * it has to trigger a reset to recover from the error condition.
4525  * A “primary function” is the function who will have the privilege to
4526  * initiate the chimp reset. The primary function will be elected by the
4527  * firmware and will be notified through async message.
4528  */
4529 static void bnxt_check_fw_health(void *arg)
4530 {
4531         struct bnxt *bp = arg;
4532         struct bnxt_error_recovery_info *info = bp->recovery_info;
4533         uint32_t val = 0, wait_msec;
4534
4535         if (!info || !bnxt_is_recovery_enabled(bp) ||
4536             is_bnxt_in_error(bp))
4537                 return;
4538
4539         val = bnxt_read_fw_status_reg(bp, BNXT_FW_HEARTBEAT_CNT_REG);
4540         if (val == info->last_heart_beat)
4541                 goto reset;
4542
4543         info->last_heart_beat = val;
4544
4545         val = bnxt_read_fw_status_reg(bp, BNXT_FW_RECOVERY_CNT_REG);
4546         if (val != info->last_reset_counter)
4547                 goto reset;
4548
4549         info->last_reset_counter = val;
4550
4551         rte_eal_alarm_set(US_PER_MS * info->driver_polling_freq,
4552                           bnxt_check_fw_health, (void *)bp);
4553
4554         return;
4555 reset:
4556         /* Stop DMA to/from device */
4557         bp->flags |= BNXT_FLAG_FATAL_ERROR;
4558         bp->flags |= BNXT_FLAG_FW_RESET;
4559
4560         bnxt_stop_rxtx(bp->eth_dev);
4561
4562         PMD_DRV_LOG(ERR, "Detected FW dead condition\n");
4563
4564         if (bnxt_is_primary_func(bp))
4565                 wait_msec = info->primary_func_wait_period;
4566         else
4567                 wait_msec = info->normal_func_wait_period;
4568
4569         rte_eal_alarm_set(US_PER_MS * wait_msec,
4570                           bnxt_fw_reset_cb, (void *)bp);
4571 }
4572
4573 void bnxt_schedule_fw_health_check(struct bnxt *bp)
4574 {
4575         uint32_t polling_freq;
4576
4577         pthread_mutex_lock(&bp->health_check_lock);
4578
4579         if (!bnxt_is_recovery_enabled(bp))
4580                 goto done;
4581
4582         if (bp->flags & BNXT_FLAG_FW_HEALTH_CHECK_SCHEDULED)
4583                 goto done;
4584
4585         polling_freq = bp->recovery_info->driver_polling_freq;
4586
4587         rte_eal_alarm_set(US_PER_MS * polling_freq,
4588                           bnxt_check_fw_health, (void *)bp);
4589         bp->flags |= BNXT_FLAG_FW_HEALTH_CHECK_SCHEDULED;
4590
4591 done:
4592         pthread_mutex_unlock(&bp->health_check_lock);
4593 }
4594
4595 static void bnxt_cancel_fw_health_check(struct bnxt *bp)
4596 {
4597         rte_eal_alarm_cancel(bnxt_check_fw_health, (void *)bp);
4598         bp->flags &= ~BNXT_FLAG_FW_HEALTH_CHECK_SCHEDULED;
4599 }
4600
4601 static bool bnxt_vf_pciid(uint16_t device_id)
4602 {
4603         switch (device_id) {
4604         case BROADCOM_DEV_ID_57304_VF:
4605         case BROADCOM_DEV_ID_57406_VF:
4606         case BROADCOM_DEV_ID_5731X_VF:
4607         case BROADCOM_DEV_ID_5741X_VF:
4608         case BROADCOM_DEV_ID_57414_VF:
4609         case BROADCOM_DEV_ID_STRATUS_NIC_VF1:
4610         case BROADCOM_DEV_ID_STRATUS_NIC_VF2:
4611         case BROADCOM_DEV_ID_58802_VF:
4612         case BROADCOM_DEV_ID_57500_VF1:
4613         case BROADCOM_DEV_ID_57500_VF2:
4614         case BROADCOM_DEV_ID_58818_VF:
4615                 /* FALLTHROUGH */
4616                 return true;
4617         default:
4618                 return false;
4619         }
4620 }
4621
4622 /* Phase 5 device */
4623 static bool bnxt_p5_device(uint16_t device_id)
4624 {
4625         switch (device_id) {
4626         case BROADCOM_DEV_ID_57508:
4627         case BROADCOM_DEV_ID_57504:
4628         case BROADCOM_DEV_ID_57502:
4629         case BROADCOM_DEV_ID_57508_MF1:
4630         case BROADCOM_DEV_ID_57504_MF1:
4631         case BROADCOM_DEV_ID_57502_MF1:
4632         case BROADCOM_DEV_ID_57508_MF2:
4633         case BROADCOM_DEV_ID_57504_MF2:
4634         case BROADCOM_DEV_ID_57502_MF2:
4635         case BROADCOM_DEV_ID_57500_VF1:
4636         case BROADCOM_DEV_ID_57500_VF2:
4637         case BROADCOM_DEV_ID_58812:
4638         case BROADCOM_DEV_ID_58814:
4639         case BROADCOM_DEV_ID_58818:
4640         case BROADCOM_DEV_ID_58818_VF:
4641                 /* FALLTHROUGH */
4642                 return true;
4643         default:
4644                 return false;
4645         }
4646 }
4647
4648 bool bnxt_stratus_device(struct bnxt *bp)
4649 {
4650         uint16_t device_id = bp->pdev->id.device_id;
4651
4652         switch (device_id) {
4653         case BROADCOM_DEV_ID_STRATUS_NIC:
4654         case BROADCOM_DEV_ID_STRATUS_NIC_VF1:
4655         case BROADCOM_DEV_ID_STRATUS_NIC_VF2:
4656                 /* FALLTHROUGH */
4657                 return true;
4658         default:
4659                 return false;
4660         }
4661 }
4662
4663 static int bnxt_map_pci_bars(struct rte_eth_dev *eth_dev)
4664 {
4665         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
4666         struct bnxt *bp = eth_dev->data->dev_private;
4667
4668         /* enable device (incl. PCI PM wakeup), and bus-mastering */
4669         bp->bar0 = (void *)pci_dev->mem_resource[0].addr;
4670         bp->doorbell_base = (void *)pci_dev->mem_resource[2].addr;
4671         if (!bp->bar0 || !bp->doorbell_base) {
4672                 PMD_DRV_LOG(ERR, "Unable to access Hardware\n");
4673                 return -ENODEV;
4674         }
4675
4676         bp->eth_dev = eth_dev;
4677         bp->pdev = pci_dev;
4678
4679         return 0;
4680 }
4681
4682 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
4683                                   struct bnxt_ctx_pg_info *ctx_pg,
4684                                   uint32_t mem_size,
4685                                   const char *suffix,
4686                                   uint16_t idx)
4687 {
4688         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
4689         const struct rte_memzone *mz = NULL;
4690         char mz_name[RTE_MEMZONE_NAMESIZE];
4691         rte_iova_t mz_phys_addr;
4692         uint64_t valid_bits = 0;
4693         uint32_t sz;
4694         int i;
4695
4696         if (!mem_size)
4697                 return 0;
4698
4699         rmem->nr_pages = RTE_ALIGN_MUL_CEIL(mem_size, BNXT_PAGE_SIZE) /
4700                          BNXT_PAGE_SIZE;
4701         rmem->page_size = BNXT_PAGE_SIZE;
4702         rmem->pg_arr = ctx_pg->ctx_pg_arr;
4703         rmem->dma_arr = ctx_pg->ctx_dma_arr;
4704         rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
4705
4706         valid_bits = PTU_PTE_VALID;
4707
4708         if (rmem->nr_pages > 1) {
4709                 snprintf(mz_name, RTE_MEMZONE_NAMESIZE,
4710                          "bnxt_ctx_pg_tbl%s_%x_%d",
4711                          suffix, idx, bp->eth_dev->data->port_id);
4712                 mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0;
4713                 mz = rte_memzone_lookup(mz_name);
4714                 if (!mz) {
4715                         mz = rte_memzone_reserve_aligned(mz_name,
4716                                                 rmem->nr_pages * 8,
4717                                                 bp->eth_dev->device->numa_node,
4718                                                 RTE_MEMZONE_2MB |
4719                                                 RTE_MEMZONE_SIZE_HINT_ONLY |
4720                                                 RTE_MEMZONE_IOVA_CONTIG,
4721                                                 BNXT_PAGE_SIZE);
4722                         if (mz == NULL)
4723                                 return -ENOMEM;
4724                 }
4725
4726                 memset(mz->addr, 0, mz->len);
4727                 mz_phys_addr = mz->iova;
4728
4729                 rmem->pg_tbl = mz->addr;
4730                 rmem->pg_tbl_map = mz_phys_addr;
4731                 rmem->pg_tbl_mz = mz;
4732         }
4733
4734         snprintf(mz_name, RTE_MEMZONE_NAMESIZE, "bnxt_ctx_%s_%x_%d",
4735                  suffix, idx, bp->eth_dev->data->port_id);
4736         mz = rte_memzone_lookup(mz_name);
4737         if (!mz) {
4738                 mz = rte_memzone_reserve_aligned(mz_name,
4739                                                  mem_size,
4740                                                  bp->eth_dev->device->numa_node,
4741                                                  RTE_MEMZONE_1GB |
4742                                                  RTE_MEMZONE_SIZE_HINT_ONLY |
4743                                                  RTE_MEMZONE_IOVA_CONTIG,
4744                                                  BNXT_PAGE_SIZE);
4745                 if (mz == NULL)
4746                         return -ENOMEM;
4747         }
4748
4749         memset(mz->addr, 0, mz->len);
4750         mz_phys_addr = mz->iova;
4751
4752         for (sz = 0, i = 0; sz < mem_size; sz += BNXT_PAGE_SIZE, i++) {
4753                 rmem->pg_arr[i] = ((char *)mz->addr) + sz;
4754                 rmem->dma_arr[i] = mz_phys_addr + sz;
4755
4756                 if (rmem->nr_pages > 1) {
4757                         if (i == rmem->nr_pages - 2 &&
4758                             (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
4759                                 valid_bits |= PTU_PTE_NEXT_TO_LAST;
4760                         else if (i == rmem->nr_pages - 1 &&
4761                                  (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
4762                                 valid_bits |= PTU_PTE_LAST;
4763
4764                         rmem->pg_tbl[i] = rte_cpu_to_le_64(rmem->dma_arr[i] |
4765                                                            valid_bits);
4766                 }
4767         }
4768
4769         rmem->mz = mz;
4770         if (rmem->vmem_size)
4771                 rmem->vmem = (void **)mz->addr;
4772         rmem->dma_arr[0] = mz_phys_addr;
4773         return 0;
4774 }
4775
4776 static void bnxt_free_ctx_mem(struct bnxt *bp)
4777 {
4778         int i;
4779
4780         if (!bp->ctx || !(bp->ctx->flags & BNXT_CTX_FLAG_INITED))
4781                 return;
4782
4783         bp->ctx->flags &= ~BNXT_CTX_FLAG_INITED;
4784         rte_memzone_free(bp->ctx->qp_mem.ring_mem.mz);
4785         rte_memzone_free(bp->ctx->srq_mem.ring_mem.mz);
4786         rte_memzone_free(bp->ctx->cq_mem.ring_mem.mz);
4787         rte_memzone_free(bp->ctx->vnic_mem.ring_mem.mz);
4788         rte_memzone_free(bp->ctx->stat_mem.ring_mem.mz);
4789         rte_memzone_free(bp->ctx->qp_mem.ring_mem.pg_tbl_mz);
4790         rte_memzone_free(bp->ctx->srq_mem.ring_mem.pg_tbl_mz);
4791         rte_memzone_free(bp->ctx->cq_mem.ring_mem.pg_tbl_mz);
4792         rte_memzone_free(bp->ctx->vnic_mem.ring_mem.pg_tbl_mz);
4793         rte_memzone_free(bp->ctx->stat_mem.ring_mem.pg_tbl_mz);
4794
4795         for (i = 0; i < bp->ctx->tqm_fp_rings_count + 1; i++) {
4796                 if (bp->ctx->tqm_mem[i])
4797                         rte_memzone_free(bp->ctx->tqm_mem[i]->ring_mem.mz);
4798         }
4799
4800         rte_free(bp->ctx);
4801         bp->ctx = NULL;
4802 }
4803
4804 #define bnxt_roundup(x, y)   ((((x) + ((y) - 1)) / (y)) * (y))
4805
4806 #define min_t(type, x, y) ({                    \
4807         type __min1 = (x);                      \
4808         type __min2 = (y);                      \
4809         __min1 < __min2 ? __min1 : __min2; })
4810
4811 #define max_t(type, x, y) ({                    \
4812         type __max1 = (x);                      \
4813         type __max2 = (y);                      \
4814         __max1 > __max2 ? __max1 : __max2; })
4815
4816 #define clamp_t(type, _x, min, max)     min_t(type, max_t(type, _x, min), max)
4817
4818 int bnxt_alloc_ctx_mem(struct bnxt *bp)
4819 {
4820         struct bnxt_ctx_pg_info *ctx_pg;
4821         struct bnxt_ctx_mem_info *ctx;
4822         uint32_t mem_size, ena, entries;
4823         uint32_t entries_sp, min;
4824         int i, rc;
4825
4826         rc = bnxt_hwrm_func_backing_store_qcaps(bp);
4827         if (rc) {
4828                 PMD_DRV_LOG(ERR, "Query context mem capability failed\n");
4829                 return rc;
4830         }
4831         ctx = bp->ctx;
4832         if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
4833                 return 0;
4834
4835         ctx_pg = &ctx->qp_mem;
4836         ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries;
4837         if (ctx->qp_entry_size) {
4838                 mem_size = ctx->qp_entry_size * ctx_pg->entries;
4839                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size, "qp_mem", 0);
4840                 if (rc)
4841                         return rc;
4842         }
4843
4844         ctx_pg = &ctx->srq_mem;
4845         ctx_pg->entries = ctx->srq_max_l2_entries;
4846         if (ctx->srq_entry_size) {
4847                 mem_size = ctx->srq_entry_size * ctx_pg->entries;
4848                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size, "srq_mem", 0);
4849                 if (rc)
4850                         return rc;
4851         }
4852
4853         ctx_pg = &ctx->cq_mem;
4854         ctx_pg->entries = ctx->cq_max_l2_entries;
4855         if (ctx->cq_entry_size) {
4856                 mem_size = ctx->cq_entry_size * ctx_pg->entries;
4857                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size, "cq_mem", 0);
4858                 if (rc)
4859                         return rc;
4860         }
4861
4862         ctx_pg = &ctx->vnic_mem;
4863         ctx_pg->entries = ctx->vnic_max_vnic_entries +
4864                 ctx->vnic_max_ring_table_entries;
4865         if (ctx->vnic_entry_size) {
4866                 mem_size = ctx->vnic_entry_size * ctx_pg->entries;
4867                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size, "vnic_mem", 0);
4868                 if (rc)
4869                         return rc;
4870         }
4871
4872         ctx_pg = &ctx->stat_mem;
4873         ctx_pg->entries = ctx->stat_max_entries;
4874         if (ctx->stat_entry_size) {
4875                 mem_size = ctx->stat_entry_size * ctx_pg->entries;
4876                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size, "stat_mem", 0);
4877                 if (rc)
4878                         return rc;
4879         }
4880
4881         min = ctx->tqm_min_entries_per_ring;
4882
4883         entries_sp = ctx->qp_max_l2_entries +
4884                      ctx->vnic_max_vnic_entries +
4885                      2 * ctx->qp_min_qp1_entries + min;
4886         entries_sp = bnxt_roundup(entries_sp, ctx->tqm_entries_multiple);
4887
4888         entries = ctx->qp_max_l2_entries + ctx->qp_min_qp1_entries;
4889         entries = bnxt_roundup(entries, ctx->tqm_entries_multiple);
4890         entries = clamp_t(uint32_t, entries, min,
4891                           ctx->tqm_max_entries_per_ring);
4892         for (i = 0, ena = 0; i < ctx->tqm_fp_rings_count + 1; i++) {
4893                 /* i=0 is for TQM_SP. i=1 to i=8 applies to RING0 to RING7.
4894                  * i > 8 is other ext rings.
4895                  */
4896                 ctx_pg = ctx->tqm_mem[i];
4897                 ctx_pg->entries = i ? entries : entries_sp;
4898                 if (ctx->tqm_entry_size) {
4899                         mem_size = ctx->tqm_entry_size * ctx_pg->entries;
4900                         rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size,
4901                                                     "tqm_mem", i);
4902                         if (rc)
4903                                 return rc;
4904                 }
4905                 if (i < BNXT_MAX_TQM_LEGACY_RINGS)
4906                         ena |= HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_TQM_SP << i;
4907                 else
4908                         ena |= HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_TQM_RING8;
4909         }
4910
4911         ena |= FUNC_BACKING_STORE_CFG_INPUT_DFLT_ENABLES;
4912         rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
4913         if (rc)
4914                 PMD_DRV_LOG(ERR,
4915                             "Failed to configure context mem: rc = %d\n", rc);
4916         else
4917                 ctx->flags |= BNXT_CTX_FLAG_INITED;
4918
4919         return rc;
4920 }
4921
4922 static int bnxt_alloc_stats_mem(struct bnxt *bp)
4923 {
4924         struct rte_pci_device *pci_dev = bp->pdev;
4925         char mz_name[RTE_MEMZONE_NAMESIZE];
4926         const struct rte_memzone *mz = NULL;
4927         uint32_t total_alloc_len;
4928         rte_iova_t mz_phys_addr;
4929
4930         if (pci_dev->id.device_id == BROADCOM_DEV_ID_NS2)
4931                 return 0;
4932
4933         snprintf(mz_name, RTE_MEMZONE_NAMESIZE,
4934                  "bnxt_" PCI_PRI_FMT "-%s", pci_dev->addr.domain,
4935                  pci_dev->addr.bus, pci_dev->addr.devid,
4936                  pci_dev->addr.function, "rx_port_stats");
4937         mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0;
4938         mz = rte_memzone_lookup(mz_name);
4939         total_alloc_len =
4940                 RTE_CACHE_LINE_ROUNDUP(sizeof(struct rx_port_stats) +
4941                                        sizeof(struct rx_port_stats_ext) + 512);
4942         if (!mz) {
4943                 mz = rte_memzone_reserve(mz_name, total_alloc_len,
4944                                          SOCKET_ID_ANY,
4945                                          RTE_MEMZONE_2MB |
4946                                          RTE_MEMZONE_SIZE_HINT_ONLY |
4947                                          RTE_MEMZONE_IOVA_CONTIG);
4948                 if (mz == NULL)
4949                         return -ENOMEM;
4950         }
4951         memset(mz->addr, 0, mz->len);
4952         mz_phys_addr = mz->iova;
4953
4954         bp->rx_mem_zone = (const void *)mz;
4955         bp->hw_rx_port_stats = mz->addr;
4956         bp->hw_rx_port_stats_map = mz_phys_addr;
4957
4958         snprintf(mz_name, RTE_MEMZONE_NAMESIZE,
4959                  "bnxt_" PCI_PRI_FMT "-%s", pci_dev->addr.domain,
4960                  pci_dev->addr.bus, pci_dev->addr.devid,
4961                  pci_dev->addr.function, "tx_port_stats");
4962         mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0;
4963         mz = rte_memzone_lookup(mz_name);
4964         total_alloc_len =
4965                 RTE_CACHE_LINE_ROUNDUP(sizeof(struct tx_port_stats) +
4966                                        sizeof(struct tx_port_stats_ext) + 512);
4967         if (!mz) {
4968                 mz = rte_memzone_reserve(mz_name,
4969                                          total_alloc_len,
4970                                          SOCKET_ID_ANY,
4971                                          RTE_MEMZONE_2MB |
4972                                          RTE_MEMZONE_SIZE_HINT_ONLY |
4973                                          RTE_MEMZONE_IOVA_CONTIG);
4974                 if (mz == NULL)
4975                         return -ENOMEM;
4976         }
4977         memset(mz->addr, 0, mz->len);
4978         mz_phys_addr = mz->iova;
4979
4980         bp->tx_mem_zone = (const void *)mz;
4981         bp->hw_tx_port_stats = mz->addr;
4982         bp->hw_tx_port_stats_map = mz_phys_addr;
4983         bp->flags |= BNXT_FLAG_PORT_STATS;
4984
4985         /* Display extended statistics if FW supports it */
4986         if (bp->hwrm_spec_code < HWRM_SPEC_CODE_1_8_4 ||
4987             bp->hwrm_spec_code == HWRM_SPEC_CODE_1_9_0 ||
4988             !(bp->flags & BNXT_FLAG_EXT_STATS_SUPPORTED))
4989                 return 0;
4990
4991         bp->hw_rx_port_stats_ext = (void *)
4992                 ((uint8_t *)bp->hw_rx_port_stats +
4993                  sizeof(struct rx_port_stats));
4994         bp->hw_rx_port_stats_ext_map = bp->hw_rx_port_stats_map +
4995                 sizeof(struct rx_port_stats);
4996         bp->flags |= BNXT_FLAG_EXT_RX_PORT_STATS;
4997
4998         if (bp->hwrm_spec_code < HWRM_SPEC_CODE_1_9_2 ||
4999             bp->flags & BNXT_FLAG_EXT_STATS_SUPPORTED) {
5000                 bp->hw_tx_port_stats_ext = (void *)
5001                         ((uint8_t *)bp->hw_tx_port_stats +
5002                          sizeof(struct tx_port_stats));
5003                 bp->hw_tx_port_stats_ext_map =
5004                         bp->hw_tx_port_stats_map +
5005                         sizeof(struct tx_port_stats);
5006                 bp->flags |= BNXT_FLAG_EXT_TX_PORT_STATS;
5007         }
5008
5009         return 0;
5010 }
5011
5012 static int bnxt_setup_mac_addr(struct rte_eth_dev *eth_dev)
5013 {
5014         struct bnxt *bp = eth_dev->data->dev_private;
5015         size_t max_mac_addr = RTE_MIN(bp->max_l2_ctx, RTE_ETH_NUM_RECEIVE_MAC_ADDR);
5016         int rc = 0;
5017
5018         if (bp->max_l2_ctx > RTE_ETH_NUM_RECEIVE_MAC_ADDR)
5019                 PMD_DRV_LOG(INFO, "Max number of MAC addrs supported is %d, but will be limited to %d\n",
5020                             bp->max_l2_ctx, RTE_ETH_NUM_RECEIVE_MAC_ADDR);
5021
5022         eth_dev->data->mac_addrs = rte_zmalloc("bnxt_mac_addr_tbl",
5023                                                RTE_ETHER_ADDR_LEN * max_mac_addr,
5024                                                0);
5025         if (eth_dev->data->mac_addrs == NULL) {
5026                 PMD_DRV_LOG(ERR, "Failed to alloc MAC addr tbl\n");
5027                 return -ENOMEM;
5028         }
5029
5030         if (!BNXT_HAS_DFLT_MAC_SET(bp)) {
5031                 if (BNXT_PF(bp))
5032                         return -EINVAL;
5033
5034                 /* Generate a random MAC address, if none was assigned by PF */
5035                 PMD_DRV_LOG(INFO, "VF MAC address not assigned by Host PF\n");
5036                 bnxt_eth_hw_addr_random(bp->mac_addr);
5037                 PMD_DRV_LOG(INFO,
5038                             "Assign random MAC:" RTE_ETHER_ADDR_PRT_FMT "\n",
5039                             bp->mac_addr[0], bp->mac_addr[1], bp->mac_addr[2],
5040                             bp->mac_addr[3], bp->mac_addr[4], bp->mac_addr[5]);
5041
5042                 rc = bnxt_hwrm_set_mac(bp);
5043                 if (rc)
5044                         return rc;
5045         }
5046
5047         /* Copy the permanent MAC from the FUNC_QCAPS response */
5048         memcpy(&eth_dev->data->mac_addrs[0], bp->mac_addr, RTE_ETHER_ADDR_LEN);
5049
5050         /*
5051          *  Allocate memory to hold multicast mac addresses added.
5052          *  Used to restore them during reset recovery
5053          */
5054         bp->mcast_addr_list = rte_zmalloc("bnxt_mcast_addr_tbl",
5055                                           sizeof(struct rte_ether_addr) *
5056                                           BNXT_MAX_MC_ADDRS, 0);
5057         if (bp->mcast_addr_list == NULL) {
5058                 PMD_DRV_LOG(ERR, "Failed to allocate multicast addr table\n");
5059                 return -ENOMEM;
5060         }
5061         bp->mc_list_dma_addr = rte_malloc_virt2iova(bp->mcast_addr_list);
5062         if (bp->mc_list_dma_addr == RTE_BAD_IOVA) {
5063                 PMD_DRV_LOG(ERR, "Fail to map mcast_addr_list to physical memory\n");
5064                 return -ENOMEM;
5065         }
5066
5067         return rc;
5068 }
5069
5070 static int bnxt_restore_dflt_mac(struct bnxt *bp)
5071 {
5072         int rc = 0;
5073
5074         /* MAC is already configured in FW */
5075         if (BNXT_HAS_DFLT_MAC_SET(bp))
5076                 return 0;
5077
5078         /* Restore the old MAC configured */
5079         rc = bnxt_hwrm_set_mac(bp);
5080         if (rc)
5081                 PMD_DRV_LOG(ERR, "Failed to restore MAC address\n");
5082
5083         return rc;
5084 }
5085
5086 static void bnxt_config_vf_req_fwd(struct bnxt *bp)
5087 {
5088         if (!BNXT_PF(bp))
5089                 return;
5090
5091         memset(bp->pf->vf_req_fwd, 0, sizeof(bp->pf->vf_req_fwd));
5092
5093         if (!(bp->fw_cap & BNXT_FW_CAP_LINK_ADMIN))
5094                 BNXT_HWRM_CMD_TO_FORWARD(HWRM_PORT_PHY_QCFG);
5095         BNXT_HWRM_CMD_TO_FORWARD(HWRM_FUNC_CFG);
5096         BNXT_HWRM_CMD_TO_FORWARD(HWRM_FUNC_VF_CFG);
5097         BNXT_HWRM_CMD_TO_FORWARD(HWRM_CFA_L2_FILTER_ALLOC);
5098         BNXT_HWRM_CMD_TO_FORWARD(HWRM_OEM_CMD);
5099 }
5100
5101 static void bnxt_alloc_error_recovery_info(struct bnxt *bp)
5102 {
5103         struct bnxt_error_recovery_info *info = bp->recovery_info;
5104
5105         if (info) {
5106                 if (!(bp->fw_cap & BNXT_FW_CAP_HCOMM_FW_STATUS))
5107                         memset(info, 0, sizeof(*info));
5108                 return;
5109         }
5110
5111         if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
5112                 return;
5113
5114         info = rte_zmalloc("bnxt_hwrm_error_recovery_qcfg",
5115                            sizeof(*info), 0);
5116         if (!info)
5117                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
5118
5119         bp->recovery_info = info;
5120 }
5121
5122 static void bnxt_check_fw_status(struct bnxt *bp)
5123 {
5124         uint32_t fw_status;
5125
5126         if (!(bp->recovery_info &&
5127               (bp->fw_cap & BNXT_FW_CAP_HCOMM_FW_STATUS)))
5128                 return;
5129
5130         fw_status = bnxt_read_fw_status_reg(bp, BNXT_FW_STATUS_REG);
5131         if (fw_status != BNXT_FW_STATUS_HEALTHY)
5132                 PMD_DRV_LOG(ERR, "Firmware not responding, status: %#x\n",
5133                             fw_status);
5134 }
5135
5136 static int bnxt_map_hcomm_fw_status_reg(struct bnxt *bp)
5137 {
5138         struct bnxt_error_recovery_info *info = bp->recovery_info;
5139         uint32_t status_loc;
5140         uint32_t sig_ver;
5141
5142         rte_write32(HCOMM_STATUS_STRUCT_LOC, (uint8_t *)bp->bar0 +
5143                     BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
5144         sig_ver = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
5145                                    BNXT_GRCP_WINDOW_2_BASE +
5146                                    offsetof(struct hcomm_status,
5147                                             sig_ver)));
5148         /* If the signature is absent, then FW does not support this feature */
5149         if ((sig_ver & HCOMM_STATUS_SIGNATURE_MASK) !=
5150             HCOMM_STATUS_SIGNATURE_VAL)
5151                 return 0;
5152
5153         if (!info) {
5154                 info = rte_zmalloc("bnxt_hwrm_error_recovery_qcfg",
5155                                    sizeof(*info), 0);
5156                 if (!info)
5157                         return -ENOMEM;
5158                 bp->recovery_info = info;
5159         } else {
5160                 memset(info, 0, sizeof(*info));
5161         }
5162
5163         status_loc = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
5164                                       BNXT_GRCP_WINDOW_2_BASE +
5165                                       offsetof(struct hcomm_status,
5166                                                fw_status_loc)));
5167
5168         /* Only pre-map the FW health status GRC register */
5169         if (BNXT_FW_STATUS_REG_TYPE(status_loc) != BNXT_FW_STATUS_REG_TYPE_GRC)
5170                 return 0;
5171
5172         info->status_regs[BNXT_FW_STATUS_REG] = status_loc;
5173         info->mapped_status_regs[BNXT_FW_STATUS_REG] =
5174                 BNXT_GRCP_WINDOW_2_BASE + (status_loc & BNXT_GRCP_OFFSET_MASK);
5175
5176         rte_write32((status_loc & BNXT_GRCP_BASE_MASK), (uint8_t *)bp->bar0 +
5177                     BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
5178
5179         bp->fw_cap |= BNXT_FW_CAP_HCOMM_FW_STATUS;
5180
5181         return 0;
5182 }
5183
5184 /* This function gets the FW version along with the
5185  * capabilities(MAX and current) of the function, vnic,
5186  * error recovery, phy and other chip related info
5187  */
5188 static int bnxt_get_config(struct bnxt *bp)
5189 {
5190         uint16_t mtu;
5191         int rc = 0;
5192
5193         bp->fw_cap = 0;
5194
5195         rc = bnxt_map_hcomm_fw_status_reg(bp);
5196         if (rc)
5197                 return rc;
5198
5199         rc = bnxt_hwrm_ver_get(bp, DFLT_HWRM_CMD_TIMEOUT);
5200         if (rc) {
5201                 bnxt_check_fw_status(bp);
5202                 return rc;
5203         }
5204
5205         rc = bnxt_hwrm_func_reset(bp);
5206         if (rc)
5207                 return -EIO;
5208
5209         rc = bnxt_hwrm_vnic_qcaps(bp);
5210         if (rc)
5211                 return rc;
5212
5213         rc = bnxt_hwrm_queue_qportcfg(bp);
5214         if (rc)
5215                 return rc;
5216
5217         /* Get the MAX capabilities for this function.
5218          * This function also allocates context memory for TQM rings and
5219          * informs the firmware about this allocated backing store memory.
5220          */
5221         rc = bnxt_hwrm_func_qcaps(bp);
5222         if (rc)
5223                 return rc;
5224
5225         rc = bnxt_hwrm_func_qcfg(bp, &mtu);
5226         if (rc)
5227                 return rc;
5228
5229         bnxt_hwrm_port_mac_qcfg(bp);
5230
5231         bnxt_hwrm_parent_pf_qcfg(bp);
5232
5233         bnxt_hwrm_port_phy_qcaps(bp);
5234
5235         bnxt_alloc_error_recovery_info(bp);
5236         /* Get the adapter error recovery support info */
5237         rc = bnxt_hwrm_error_recovery_qcfg(bp);
5238         if (rc)
5239                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
5240
5241         bnxt_hwrm_port_led_qcaps(bp);
5242
5243         return 0;
5244 }
5245
5246 static int
5247 bnxt_init_locks(struct bnxt *bp)
5248 {
5249         int err;
5250
5251         err = pthread_mutex_init(&bp->flow_lock, NULL);
5252         if (err) {
5253                 PMD_DRV_LOG(ERR, "Unable to initialize flow_lock\n");
5254                 return err;
5255         }
5256
5257         err = pthread_mutex_init(&bp->def_cp_lock, NULL);
5258         if (err) {
5259                 PMD_DRV_LOG(ERR, "Unable to initialize def_cp_lock\n");
5260                 return err;
5261         }
5262
5263         err = pthread_mutex_init(&bp->health_check_lock, NULL);
5264         if (err) {
5265                 PMD_DRV_LOG(ERR, "Unable to initialize health_check_lock\n");
5266                 return err;
5267         }
5268
5269         err = pthread_mutex_init(&bp->err_recovery_lock, NULL);
5270         if (err)
5271                 PMD_DRV_LOG(ERR, "Unable to initialize err_recovery_lock\n");
5272
5273         return err;
5274 }
5275
5276 static int bnxt_init_resources(struct bnxt *bp, bool reconfig_dev)
5277 {
5278         int rc = 0;
5279
5280         rc = bnxt_get_config(bp);
5281         if (rc)
5282                 return rc;
5283
5284         if (!reconfig_dev) {
5285                 rc = bnxt_setup_mac_addr(bp->eth_dev);
5286                 if (rc)
5287                         return rc;
5288         } else {
5289                 rc = bnxt_restore_dflt_mac(bp);
5290                 if (rc)
5291                         return rc;
5292         }
5293
5294         bnxt_config_vf_req_fwd(bp);
5295
5296         rc = bnxt_hwrm_func_driver_register(bp);
5297         if (rc) {
5298                 PMD_DRV_LOG(ERR, "Failed to register driver");
5299                 return -EBUSY;
5300         }
5301
5302         if (BNXT_PF(bp)) {
5303                 if (bp->pdev->max_vfs) {
5304                         rc = bnxt_hwrm_allocate_vfs(bp, bp->pdev->max_vfs);
5305                         if (rc) {
5306                                 PMD_DRV_LOG(ERR, "Failed to allocate VFs\n");
5307                                 return rc;
5308                         }
5309                 } else {
5310                         rc = bnxt_hwrm_allocate_pf_only(bp);
5311                         if (rc) {
5312                                 PMD_DRV_LOG(ERR,
5313                                             "Failed to allocate PF resources");
5314                                 return rc;
5315                         }
5316                 }
5317         }
5318
5319         if (!reconfig_dev) {
5320                 bp->rss_conf.rss_key = rte_zmalloc("bnxt_rss_key",
5321                                                    HW_HASH_KEY_SIZE, 0);
5322                 if (bp->rss_conf.rss_key == NULL) {
5323                         PMD_DRV_LOG(ERR, "port %u cannot allocate RSS hash key memory",
5324                                     bp->eth_dev->data->port_id);
5325                         return -ENOMEM;
5326                 }
5327         }
5328
5329         rc = bnxt_alloc_mem(bp, reconfig_dev);
5330         if (rc)
5331                 return rc;
5332
5333         rc = bnxt_setup_int(bp);
5334         if (rc)
5335                 return rc;
5336
5337         rc = bnxt_request_int(bp);
5338         if (rc)
5339                 return rc;
5340
5341         rc = bnxt_init_ctx_mem(bp);
5342         if (rc) {
5343                 PMD_DRV_LOG(ERR, "Failed to init adv_flow_counters\n");
5344                 return rc;
5345         }
5346
5347         return 0;
5348 }
5349
5350 static int
5351 bnxt_parse_devarg_flow_xstat(__rte_unused const char *key,
5352                              const char *value, void *opaque_arg)
5353 {
5354         struct bnxt *bp = opaque_arg;
5355         unsigned long flow_xstat;
5356         char *end = NULL;
5357
5358         if (!value || !opaque_arg) {
5359                 PMD_DRV_LOG(ERR,
5360                             "Invalid parameter passed to flow_xstat devarg.\n");
5361                 return -EINVAL;
5362         }
5363
5364         flow_xstat = strtoul(value, &end, 10);
5365         if (end == NULL || *end != '\0' ||
5366             (flow_xstat == ULONG_MAX && errno == ERANGE)) {
5367                 PMD_DRV_LOG(ERR,
5368                             "Invalid parameter passed to flow_xstat devarg.\n");
5369                 return -EINVAL;
5370         }
5371
5372         if (BNXT_DEVARG_FLOW_XSTAT_INVALID(flow_xstat)) {
5373                 PMD_DRV_LOG(ERR,
5374                             "Invalid value passed to flow_xstat devarg.\n");
5375                 return -EINVAL;
5376         }
5377
5378         bp->flags |= BNXT_FLAG_FLOW_XSTATS_EN;
5379         if (BNXT_FLOW_XSTATS_EN(bp))
5380                 PMD_DRV_LOG(INFO, "flow_xstat feature enabled.\n");
5381
5382         return 0;
5383 }
5384
5385 static int
5386 bnxt_parse_devarg_max_num_kflows(__rte_unused const char *key,
5387                                         const char *value, void *opaque_arg)
5388 {
5389         struct bnxt *bp = opaque_arg;
5390         unsigned long max_num_kflows;
5391         char *end = NULL;
5392
5393         if (!value || !opaque_arg) {
5394                 PMD_DRV_LOG(ERR,
5395                         "Invalid parameter passed to max_num_kflows devarg.\n");
5396                 return -EINVAL;
5397         }
5398
5399         max_num_kflows = strtoul(value, &end, 10);
5400         if (end == NULL || *end != '\0' ||
5401                 (max_num_kflows == ULONG_MAX && errno == ERANGE)) {
5402                 PMD_DRV_LOG(ERR,
5403                         "Invalid parameter passed to max_num_kflows devarg.\n");
5404                 return -EINVAL;
5405         }
5406
5407         if (bnxt_devarg_max_num_kflow_invalid(max_num_kflows)) {
5408                 PMD_DRV_LOG(ERR,
5409                         "Invalid value passed to max_num_kflows devarg.\n");
5410                 return -EINVAL;
5411         }
5412
5413         bp->max_num_kflows = max_num_kflows;
5414         if (bp->max_num_kflows)
5415                 PMD_DRV_LOG(INFO, "max_num_kflows set as %ldK.\n",
5416                                 max_num_kflows);
5417
5418         return 0;
5419 }
5420
5421 static int
5422 bnxt_parse_devarg_app_id(__rte_unused const char *key,
5423                                  const char *value, void *opaque_arg)
5424 {
5425         struct bnxt *bp = opaque_arg;
5426         unsigned long app_id;
5427         char *end = NULL;
5428
5429         if (!value || !opaque_arg) {
5430                 PMD_DRV_LOG(ERR,
5431                             "Invalid parameter passed to app-id "
5432                             "devargs.\n");
5433                 return -EINVAL;
5434         }
5435
5436         app_id = strtoul(value, &end, 10);
5437         if (end == NULL || *end != '\0' ||
5438             (app_id == ULONG_MAX && errno == ERANGE)) {
5439                 PMD_DRV_LOG(ERR,
5440                             "Invalid parameter passed to app_id "
5441                             "devargs.\n");
5442                 return -EINVAL;
5443         }
5444
5445         if (BNXT_DEVARG_APP_ID_INVALID(app_id)) {
5446                 PMD_DRV_LOG(ERR, "Invalid app-id(%d) devargs.\n",
5447                             (uint16_t)app_id);
5448                 return -EINVAL;
5449         }
5450
5451         bp->app_id = app_id;
5452         PMD_DRV_LOG(INFO, "app-id=%d feature enabled.\n", (uint16_t)app_id);
5453
5454         return 0;
5455 }
5456
5457 static int
5458 bnxt_parse_devarg_rep_is_pf(__rte_unused const char *key,
5459                             const char *value, void *opaque_arg)
5460 {
5461         struct bnxt_representor *vfr_bp = opaque_arg;
5462         unsigned long rep_is_pf;
5463         char *end = NULL;
5464
5465         if (!value || !opaque_arg) {
5466                 PMD_DRV_LOG(ERR,
5467                             "Invalid parameter passed to rep_is_pf devargs.\n");
5468                 return -EINVAL;
5469         }
5470
5471         rep_is_pf = strtoul(value, &end, 10);
5472         if (end == NULL || *end != '\0' ||
5473             (rep_is_pf == ULONG_MAX && errno == ERANGE)) {
5474                 PMD_DRV_LOG(ERR,
5475                             "Invalid parameter passed to rep_is_pf devargs.\n");
5476                 return -EINVAL;
5477         }
5478
5479         if (BNXT_DEVARG_REP_IS_PF_INVALID(rep_is_pf)) {
5480                 PMD_DRV_LOG(ERR,
5481                             "Invalid value passed to rep_is_pf devargs.\n");
5482                 return -EINVAL;
5483         }
5484
5485         vfr_bp->flags |= rep_is_pf;
5486         if (BNXT_REP_PF(vfr_bp))
5487                 PMD_DRV_LOG(INFO, "PF representor\n");
5488         else
5489                 PMD_DRV_LOG(INFO, "VF representor\n");
5490
5491         return 0;
5492 }
5493
5494 static int
5495 bnxt_parse_devarg_rep_based_pf(__rte_unused const char *key,
5496                                const char *value, void *opaque_arg)
5497 {
5498         struct bnxt_representor *vfr_bp = opaque_arg;
5499         unsigned long rep_based_pf;
5500         char *end = NULL;
5501
5502         if (!value || !opaque_arg) {
5503                 PMD_DRV_LOG(ERR,
5504                             "Invalid parameter passed to rep_based_pf "
5505                             "devargs.\n");
5506                 return -EINVAL;
5507         }
5508
5509         rep_based_pf = strtoul(value, &end, 10);
5510         if (end == NULL || *end != '\0' ||
5511             (rep_based_pf == ULONG_MAX && errno == ERANGE)) {
5512                 PMD_DRV_LOG(ERR,
5513                             "Invalid parameter passed to rep_based_pf "
5514                             "devargs.\n");
5515                 return -EINVAL;
5516         }
5517
5518         if (BNXT_DEVARG_REP_BASED_PF_INVALID(rep_based_pf)) {
5519                 PMD_DRV_LOG(ERR,
5520                             "Invalid value passed to rep_based_pf devargs.\n");
5521                 return -EINVAL;
5522         }
5523
5524         vfr_bp->rep_based_pf = rep_based_pf;
5525         vfr_bp->flags |= BNXT_REP_BASED_PF_VALID;
5526
5527         PMD_DRV_LOG(INFO, "rep-based-pf = %d\n", vfr_bp->rep_based_pf);
5528
5529         return 0;
5530 }
5531
5532 static int
5533 bnxt_parse_devarg_rep_q_r2f(__rte_unused const char *key,
5534                             const char *value, void *opaque_arg)
5535 {
5536         struct bnxt_representor *vfr_bp = opaque_arg;
5537         unsigned long rep_q_r2f;
5538         char *end = NULL;
5539
5540         if (!value || !opaque_arg) {
5541                 PMD_DRV_LOG(ERR,
5542                             "Invalid parameter passed to rep_q_r2f "
5543                             "devargs.\n");
5544                 return -EINVAL;
5545         }
5546
5547         rep_q_r2f = strtoul(value, &end, 10);
5548         if (end == NULL || *end != '\0' ||
5549             (rep_q_r2f == ULONG_MAX && errno == ERANGE)) {
5550                 PMD_DRV_LOG(ERR,
5551                             "Invalid parameter passed to rep_q_r2f "
5552                             "devargs.\n");
5553                 return -EINVAL;
5554         }
5555
5556         if (BNXT_DEVARG_REP_Q_R2F_INVALID(rep_q_r2f)) {
5557                 PMD_DRV_LOG(ERR,
5558                             "Invalid value passed to rep_q_r2f devargs.\n");
5559                 return -EINVAL;
5560         }
5561
5562         vfr_bp->rep_q_r2f = rep_q_r2f;
5563         vfr_bp->flags |= BNXT_REP_Q_R2F_VALID;
5564         PMD_DRV_LOG(INFO, "rep-q-r2f = %d\n", vfr_bp->rep_q_r2f);
5565
5566         return 0;
5567 }
5568
5569 static int
5570 bnxt_parse_devarg_rep_q_f2r(__rte_unused const char *key,
5571                             const char *value, void *opaque_arg)
5572 {
5573         struct bnxt_representor *vfr_bp = opaque_arg;
5574         unsigned long rep_q_f2r;
5575         char *end = NULL;
5576
5577         if (!value || !opaque_arg) {
5578                 PMD_DRV_LOG(ERR,
5579                             "Invalid parameter passed to rep_q_f2r "
5580                             "devargs.\n");
5581                 return -EINVAL;
5582         }
5583
5584         rep_q_f2r = strtoul(value, &end, 10);
5585         if (end == NULL || *end != '\0' ||
5586             (rep_q_f2r == ULONG_MAX && errno == ERANGE)) {
5587                 PMD_DRV_LOG(ERR,
5588                             "Invalid parameter passed to rep_q_f2r "
5589                             "devargs.\n");
5590                 return -EINVAL;
5591         }
5592
5593         if (BNXT_DEVARG_REP_Q_F2R_INVALID(rep_q_f2r)) {
5594                 PMD_DRV_LOG(ERR,
5595                             "Invalid value passed to rep_q_f2r devargs.\n");
5596                 return -EINVAL;
5597         }
5598
5599         vfr_bp->rep_q_f2r = rep_q_f2r;
5600         vfr_bp->flags |= BNXT_REP_Q_F2R_VALID;
5601         PMD_DRV_LOG(INFO, "rep-q-f2r = %d\n", vfr_bp->rep_q_f2r);
5602
5603         return 0;
5604 }
5605
5606 static int
5607 bnxt_parse_devarg_rep_fc_r2f(__rte_unused const char *key,
5608                              const char *value, void *opaque_arg)
5609 {
5610         struct bnxt_representor *vfr_bp = opaque_arg;
5611         unsigned long rep_fc_r2f;
5612         char *end = NULL;
5613
5614         if (!value || !opaque_arg) {
5615                 PMD_DRV_LOG(ERR,
5616                             "Invalid parameter passed to rep_fc_r2f "
5617                             "devargs.\n");
5618                 return -EINVAL;
5619         }
5620
5621         rep_fc_r2f = strtoul(value, &end, 10);
5622         if (end == NULL || *end != '\0' ||
5623             (rep_fc_r2f == ULONG_MAX && errno == ERANGE)) {
5624                 PMD_DRV_LOG(ERR,
5625                             "Invalid parameter passed to rep_fc_r2f "
5626                             "devargs.\n");
5627                 return -EINVAL;
5628         }
5629
5630         if (BNXT_DEVARG_REP_FC_R2F_INVALID(rep_fc_r2f)) {
5631                 PMD_DRV_LOG(ERR,
5632                             "Invalid value passed to rep_fc_r2f devargs.\n");
5633                 return -EINVAL;
5634         }
5635
5636         vfr_bp->flags |= BNXT_REP_FC_R2F_VALID;
5637         vfr_bp->rep_fc_r2f = rep_fc_r2f;
5638         PMD_DRV_LOG(INFO, "rep-fc-r2f = %lu\n", rep_fc_r2f);
5639
5640         return 0;
5641 }
5642
5643 static int
5644 bnxt_parse_devarg_rep_fc_f2r(__rte_unused const char *key,
5645                              const char *value, void *opaque_arg)
5646 {
5647         struct bnxt_representor *vfr_bp = opaque_arg;
5648         unsigned long rep_fc_f2r;
5649         char *end = NULL;
5650
5651         if (!value || !opaque_arg) {
5652                 PMD_DRV_LOG(ERR,
5653                             "Invalid parameter passed to rep_fc_f2r "
5654                             "devargs.\n");
5655                 return -EINVAL;
5656         }
5657
5658         rep_fc_f2r = strtoul(value, &end, 10);
5659         if (end == NULL || *end != '\0' ||
5660             (rep_fc_f2r == ULONG_MAX && errno == ERANGE)) {
5661                 PMD_DRV_LOG(ERR,
5662                             "Invalid parameter passed to rep_fc_f2r "
5663                             "devargs.\n");
5664                 return -EINVAL;
5665         }
5666
5667         if (BNXT_DEVARG_REP_FC_F2R_INVALID(rep_fc_f2r)) {
5668                 PMD_DRV_LOG(ERR,
5669                             "Invalid value passed to rep_fc_f2r devargs.\n");
5670                 return -EINVAL;
5671         }
5672
5673         vfr_bp->flags |= BNXT_REP_FC_F2R_VALID;
5674         vfr_bp->rep_fc_f2r = rep_fc_f2r;
5675         PMD_DRV_LOG(INFO, "rep-fc-f2r = %lu\n", rep_fc_f2r);
5676
5677         return 0;
5678 }
5679
5680 static int
5681 bnxt_parse_dev_args(struct bnxt *bp, struct rte_devargs *devargs)
5682 {
5683         struct rte_kvargs *kvlist;
5684         int ret;
5685
5686         if (devargs == NULL)
5687                 return 0;
5688
5689         kvlist = rte_kvargs_parse(devargs->args, bnxt_dev_args);
5690         if (kvlist == NULL)
5691                 return -EINVAL;
5692
5693         /*
5694          * Handler for "flow_xstat" devarg.
5695          * Invoked as for ex: "-a 0000:00:0d.0,flow_xstat=1"
5696          */
5697         ret = rte_kvargs_process(kvlist, BNXT_DEVARG_FLOW_XSTAT,
5698                                  bnxt_parse_devarg_flow_xstat, bp);
5699         if (ret)
5700                 goto err;
5701
5702         /*
5703          * Handler for "max_num_kflows" devarg.
5704          * Invoked as for ex: "-a 000:00:0d.0,max_num_kflows=32"
5705          */
5706         ret = rte_kvargs_process(kvlist, BNXT_DEVARG_MAX_NUM_KFLOWS,
5707                                  bnxt_parse_devarg_max_num_kflows, bp);
5708         if (ret)
5709                 goto err;
5710
5711 err:
5712         /*
5713          * Handler for "app-id" devarg.
5714          * Invoked as for ex: "-a 000:00:0d.0,app-id=1"
5715          */
5716         rte_kvargs_process(kvlist, BNXT_DEVARG_APP_ID,
5717                            bnxt_parse_devarg_app_id, bp);
5718
5719         rte_kvargs_free(kvlist);
5720         return ret;
5721 }
5722
5723 static int bnxt_alloc_switch_domain(struct bnxt *bp)
5724 {
5725         int rc = 0;
5726
5727         if (BNXT_PF(bp) || BNXT_VF_IS_TRUSTED(bp)) {
5728                 rc = rte_eth_switch_domain_alloc(&bp->switch_domain_id);
5729                 if (rc)
5730                         PMD_DRV_LOG(ERR,
5731                                     "Failed to alloc switch domain: %d\n", rc);
5732                 else
5733                         PMD_DRV_LOG(INFO,
5734                                     "Switch domain allocated %d\n",
5735                                     bp->switch_domain_id);
5736         }
5737
5738         return rc;
5739 }
5740
5741 /* Allocate and initialize various fields in bnxt struct that
5742  * need to be allocated/destroyed only once in the lifetime of the driver
5743  */
5744 static int bnxt_drv_init(struct rte_eth_dev *eth_dev)
5745 {
5746         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
5747         struct bnxt *bp = eth_dev->data->dev_private;
5748         int rc = 0;
5749
5750         bp->flags &= ~BNXT_FLAG_RX_VECTOR_PKT_MODE;
5751
5752         if (bnxt_vf_pciid(pci_dev->id.device_id))
5753                 bp->flags |= BNXT_FLAG_VF;
5754
5755         if (bnxt_p5_device(pci_dev->id.device_id))
5756                 bp->flags |= BNXT_FLAG_CHIP_P5;
5757
5758         if (pci_dev->id.device_id == BROADCOM_DEV_ID_58802 ||
5759             pci_dev->id.device_id == BROADCOM_DEV_ID_58804 ||
5760             pci_dev->id.device_id == BROADCOM_DEV_ID_58808 ||
5761             pci_dev->id.device_id == BROADCOM_DEV_ID_58802_VF)
5762                 bp->flags |= BNXT_FLAG_STINGRAY;
5763
5764         if (BNXT_TRUFLOW_EN(bp)) {
5765                 /* extra mbuf field is required to store CFA code from mark */
5766                 static const struct rte_mbuf_dynfield bnxt_cfa_code_dynfield_desc = {
5767                         .name = RTE_PMD_BNXT_CFA_CODE_DYNFIELD_NAME,
5768                         .size = sizeof(bnxt_cfa_code_dynfield_t),
5769                         .align = __alignof__(bnxt_cfa_code_dynfield_t),
5770                 };
5771                 bnxt_cfa_code_dynfield_offset =
5772                         rte_mbuf_dynfield_register(&bnxt_cfa_code_dynfield_desc);
5773                 if (bnxt_cfa_code_dynfield_offset < 0) {
5774                         PMD_DRV_LOG(ERR,
5775                             "Failed to register mbuf field for TruFlow mark\n");
5776                         return -rte_errno;
5777                 }
5778         }
5779
5780         rc = bnxt_map_pci_bars(eth_dev);
5781         if (rc) {
5782                 PMD_DRV_LOG(ERR,
5783                             "Failed to initialize board rc: %x\n", rc);
5784                 return rc;
5785         }
5786
5787         rc = bnxt_alloc_pf_info(bp);
5788         if (rc)
5789                 return rc;
5790
5791         rc = bnxt_alloc_link_info(bp);
5792         if (rc)
5793                 return rc;
5794
5795         rc = bnxt_alloc_parent_info(bp);
5796         if (rc)
5797                 return rc;
5798
5799         rc = bnxt_alloc_hwrm_resources(bp);
5800         if (rc) {
5801                 PMD_DRV_LOG(ERR,
5802                             "Failed to allocate response buffer rc: %x\n", rc);
5803                 return rc;
5804         }
5805         rc = bnxt_alloc_leds_info(bp);
5806         if (rc)
5807                 return rc;
5808
5809         rc = bnxt_alloc_cos_queues(bp);
5810         if (rc)
5811                 return rc;
5812
5813         rc = bnxt_init_locks(bp);
5814         if (rc)
5815                 return rc;
5816
5817         rc = bnxt_alloc_switch_domain(bp);
5818         if (rc)
5819                 return rc;
5820
5821         return rc;
5822 }
5823
5824 static int
5825 bnxt_dev_init(struct rte_eth_dev *eth_dev, void *params __rte_unused)
5826 {
5827         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
5828         static int version_printed;
5829         struct bnxt *bp;
5830         int rc;
5831
5832         if (version_printed++ == 0)
5833                 PMD_DRV_LOG(INFO, "%s\n", bnxt_version);
5834
5835         eth_dev->dev_ops = &bnxt_dev_ops;
5836         eth_dev->rx_queue_count = bnxt_rx_queue_count_op;
5837         eth_dev->rx_descriptor_status = bnxt_rx_descriptor_status_op;
5838         eth_dev->tx_descriptor_status = bnxt_tx_descriptor_status_op;
5839         eth_dev->rx_pkt_burst = &bnxt_recv_pkts;
5840         eth_dev->tx_pkt_burst = &bnxt_xmit_pkts;
5841
5842         /*
5843          * For secondary processes, we don't initialise any further
5844          * as primary has already done this work.
5845          */
5846         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
5847                 return 0;
5848
5849         rte_eth_copy_pci_info(eth_dev, pci_dev);
5850         eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
5851
5852         bp = eth_dev->data->dev_private;
5853
5854         /* Parse dev arguments passed on when starting the DPDK application. */
5855         rc = bnxt_parse_dev_args(bp, pci_dev->device.devargs);
5856         if (rc)
5857                 goto error_free;
5858
5859         rc = bnxt_drv_init(eth_dev);
5860         if (rc)
5861                 goto error_free;
5862
5863         rc = bnxt_init_resources(bp, false);
5864         if (rc)
5865                 goto error_free;
5866
5867         rc = bnxt_alloc_stats_mem(bp);
5868         if (rc)
5869                 goto error_free;
5870
5871         PMD_DRV_LOG(INFO,
5872                     "Found %s device at mem %" PRIX64 ", node addr %pM\n",
5873                     DRV_MODULE_NAME,
5874                     pci_dev->mem_resource[0].phys_addr,
5875                     pci_dev->mem_resource[0].addr);
5876
5877         return 0;
5878
5879 error_free:
5880         bnxt_dev_uninit(eth_dev);
5881         return rc;
5882 }
5883
5884
5885 static void bnxt_free_ctx_mem_buf(struct bnxt_ctx_mem_buf_info *ctx)
5886 {
5887         if (!ctx)
5888                 return;
5889
5890         if (ctx->va)
5891                 rte_free(ctx->va);
5892
5893         ctx->va = NULL;
5894         ctx->dma = RTE_BAD_IOVA;
5895         ctx->ctx_id = BNXT_CTX_VAL_INVAL;
5896 }
5897
5898 static void bnxt_unregister_fc_ctx_mem(struct bnxt *bp)
5899 {
5900         bnxt_hwrm_cfa_counter_cfg(bp, BNXT_DIR_RX,
5901                                   CFA_COUNTER_CFG_IN_COUNTER_TYPE_FC,
5902                                   bp->flow_stat->rx_fc_out_tbl.ctx_id,
5903                                   bp->flow_stat->max_fc,
5904                                   false);
5905
5906         bnxt_hwrm_cfa_counter_cfg(bp, BNXT_DIR_TX,
5907                                   CFA_COUNTER_CFG_IN_COUNTER_TYPE_FC,
5908                                   bp->flow_stat->tx_fc_out_tbl.ctx_id,
5909                                   bp->flow_stat->max_fc,
5910                                   false);
5911
5912         if (bp->flow_stat->rx_fc_in_tbl.ctx_id != BNXT_CTX_VAL_INVAL)
5913                 bnxt_hwrm_ctx_unrgtr(bp, bp->flow_stat->rx_fc_in_tbl.ctx_id);
5914         bp->flow_stat->rx_fc_in_tbl.ctx_id = BNXT_CTX_VAL_INVAL;
5915
5916         if (bp->flow_stat->rx_fc_out_tbl.ctx_id != BNXT_CTX_VAL_INVAL)
5917                 bnxt_hwrm_ctx_unrgtr(bp, bp->flow_stat->rx_fc_out_tbl.ctx_id);
5918         bp->flow_stat->rx_fc_out_tbl.ctx_id = BNXT_CTX_VAL_INVAL;
5919
5920         if (bp->flow_stat->tx_fc_in_tbl.ctx_id != BNXT_CTX_VAL_INVAL)
5921                 bnxt_hwrm_ctx_unrgtr(bp, bp->flow_stat->tx_fc_in_tbl.ctx_id);
5922         bp->flow_stat->tx_fc_in_tbl.ctx_id = BNXT_CTX_VAL_INVAL;
5923
5924         if (bp->flow_stat->tx_fc_out_tbl.ctx_id != BNXT_CTX_VAL_INVAL)
5925                 bnxt_hwrm_ctx_unrgtr(bp, bp->flow_stat->tx_fc_out_tbl.ctx_id);
5926         bp->flow_stat->tx_fc_out_tbl.ctx_id = BNXT_CTX_VAL_INVAL;
5927 }
5928
5929 static void bnxt_uninit_fc_ctx_mem(struct bnxt *bp)
5930 {
5931         bnxt_unregister_fc_ctx_mem(bp);
5932
5933         bnxt_free_ctx_mem_buf(&bp->flow_stat->rx_fc_in_tbl);
5934         bnxt_free_ctx_mem_buf(&bp->flow_stat->rx_fc_out_tbl);
5935         bnxt_free_ctx_mem_buf(&bp->flow_stat->tx_fc_in_tbl);
5936         bnxt_free_ctx_mem_buf(&bp->flow_stat->tx_fc_out_tbl);
5937 }
5938
5939 static void bnxt_uninit_ctx_mem(struct bnxt *bp)
5940 {
5941         if (BNXT_FLOW_XSTATS_EN(bp))
5942                 bnxt_uninit_fc_ctx_mem(bp);
5943 }
5944
5945 static void
5946 bnxt_free_error_recovery_info(struct bnxt *bp)
5947 {
5948         rte_free(bp->recovery_info);
5949         bp->recovery_info = NULL;
5950         bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
5951 }
5952
5953 static int
5954 bnxt_uninit_resources(struct bnxt *bp, bool reconfig_dev)
5955 {
5956         int rc;
5957
5958         bnxt_free_int(bp);
5959         bnxt_free_mem(bp, reconfig_dev);
5960
5961         bnxt_hwrm_func_buf_unrgtr(bp);
5962         if (bp->pf != NULL) {
5963                 rte_free(bp->pf->vf_req_buf);
5964                 bp->pf->vf_req_buf = NULL;
5965         }
5966
5967         rc = bnxt_hwrm_func_driver_unregister(bp);
5968         bp->flags &= ~BNXT_FLAG_REGISTERED;
5969         bnxt_free_ctx_mem(bp);
5970         if (!reconfig_dev) {
5971                 bnxt_free_hwrm_resources(bp);
5972                 bnxt_free_error_recovery_info(bp);
5973                 rte_free(bp->mcast_addr_list);
5974                 bp->mcast_addr_list = NULL;
5975                 rte_free(bp->rss_conf.rss_key);
5976                 bp->rss_conf.rss_key = NULL;
5977         }
5978
5979         bnxt_uninit_ctx_mem(bp);
5980
5981         bnxt_free_flow_stats_info(bp);
5982         if (bp->rep_info != NULL)
5983                 bnxt_free_switch_domain(bp);
5984         bnxt_free_rep_info(bp);
5985         rte_free(bp->ptp_cfg);
5986         bp->ptp_cfg = NULL;
5987         return rc;
5988 }
5989
5990 static int
5991 bnxt_dev_uninit(struct rte_eth_dev *eth_dev)
5992 {
5993         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
5994                 return -EPERM;
5995
5996         PMD_DRV_LOG(DEBUG, "Calling Device uninit\n");
5997
5998         if (eth_dev->state != RTE_ETH_DEV_UNUSED)
5999                 bnxt_dev_close_op(eth_dev);
6000
6001         return 0;
6002 }
6003
6004 static int bnxt_pci_remove_dev_with_reps(struct rte_eth_dev *eth_dev)
6005 {
6006         struct bnxt *bp = eth_dev->data->dev_private;
6007         struct rte_eth_dev *vf_rep_eth_dev;
6008         int ret = 0, i;
6009
6010         if (!bp)
6011                 return -EINVAL;
6012
6013         for (i = 0; i < bp->num_reps; i++) {
6014                 vf_rep_eth_dev = bp->rep_info[i].vfr_eth_dev;
6015                 if (!vf_rep_eth_dev)
6016                         continue;
6017                 PMD_DRV_LOG(DEBUG, "BNXT Port:%d VFR pci remove\n",
6018                             vf_rep_eth_dev->data->port_id);
6019                 rte_eth_dev_destroy(vf_rep_eth_dev, bnxt_representor_uninit);
6020         }
6021         PMD_DRV_LOG(DEBUG, "BNXT Port:%d pci remove\n",
6022                     eth_dev->data->port_id);
6023         ret = rte_eth_dev_destroy(eth_dev, bnxt_dev_uninit);
6024
6025         return ret;
6026 }
6027
6028 static void bnxt_free_rep_info(struct bnxt *bp)
6029 {
6030         rte_free(bp->rep_info);
6031         bp->rep_info = NULL;
6032         rte_free(bp->cfa_code_map);
6033         bp->cfa_code_map = NULL;
6034 }
6035
6036 static int bnxt_init_rep_info(struct bnxt *bp)
6037 {
6038         int i = 0, rc;
6039
6040         if (bp->rep_info)
6041                 return 0;
6042
6043         bp->rep_info = rte_zmalloc("bnxt_rep_info",
6044                                    sizeof(bp->rep_info[0]) * BNXT_MAX_VF_REPS(bp),
6045                                    0);
6046         if (!bp->rep_info) {
6047                 PMD_DRV_LOG(ERR, "Failed to alloc memory for rep info\n");
6048                 return -ENOMEM;
6049         }
6050         bp->cfa_code_map = rte_zmalloc("bnxt_cfa_code_map",
6051                                        sizeof(*bp->cfa_code_map) *
6052                                        BNXT_MAX_CFA_CODE, 0);
6053         if (!bp->cfa_code_map) {
6054                 PMD_DRV_LOG(ERR, "Failed to alloc memory for cfa_code_map\n");
6055                 bnxt_free_rep_info(bp);
6056                 return -ENOMEM;
6057         }
6058
6059         for (i = 0; i < BNXT_MAX_CFA_CODE; i++)
6060                 bp->cfa_code_map[i] = BNXT_VF_IDX_INVALID;
6061
6062         rc = pthread_mutex_init(&bp->rep_info->vfr_lock, NULL);
6063         if (rc) {
6064                 PMD_DRV_LOG(ERR, "Unable to initialize vfr_lock\n");
6065                 bnxt_free_rep_info(bp);
6066                 return rc;
6067         }
6068
6069         rc = pthread_mutex_init(&bp->rep_info->vfr_start_lock, NULL);
6070         if (rc) {
6071                 PMD_DRV_LOG(ERR, "Unable to initialize vfr_start_lock\n");
6072                 bnxt_free_rep_info(bp);
6073                 return rc;
6074         }
6075
6076         return rc;
6077 }
6078
6079 static int bnxt_rep_port_probe(struct rte_pci_device *pci_dev,
6080                                struct rte_eth_devargs *eth_da,
6081                                struct rte_eth_dev *backing_eth_dev,
6082                                const char *dev_args)
6083 {
6084         struct rte_eth_dev *vf_rep_eth_dev;
6085         char name[RTE_ETH_NAME_MAX_LEN];
6086         struct bnxt *backing_bp = backing_eth_dev->data->dev_private;
6087         uint16_t max_vf_reps = BNXT_MAX_VF_REPS(backing_bp);
6088
6089         uint16_t num_rep;
6090         int i, ret = 0;
6091         struct rte_kvargs *kvlist = NULL;
6092
6093         if (eth_da->type == RTE_ETH_REPRESENTOR_NONE)
6094                 return 0;
6095         if (eth_da->type != RTE_ETH_REPRESENTOR_VF) {
6096                 PMD_DRV_LOG(ERR, "unsupported representor type %d\n",
6097                             eth_da->type);
6098                 return -ENOTSUP;
6099         }
6100         num_rep = eth_da->nb_representor_ports;
6101         if (num_rep > max_vf_reps) {
6102                 PMD_DRV_LOG(ERR, "nb_representor_ports = %d > %d MAX VF REPS\n",
6103                             num_rep, max_vf_reps);
6104                 return -EINVAL;
6105         }
6106
6107         if (num_rep >= RTE_MAX_ETHPORTS) {
6108                 PMD_DRV_LOG(ERR,
6109                             "nb_representor_ports = %d > %d MAX ETHPORTS\n",
6110                             num_rep, RTE_MAX_ETHPORTS);
6111                 return -EINVAL;
6112         }
6113
6114         if (!(BNXT_PF(backing_bp) || BNXT_VF_IS_TRUSTED(backing_bp))) {
6115                 PMD_DRV_LOG(ERR,
6116                             "Not a PF or trusted VF. No Representor support\n");
6117                 /* Returning an error is not an option.
6118                  * Applications are not handling this correctly
6119                  */
6120                 return 0;
6121         }
6122
6123         if (bnxt_init_rep_info(backing_bp))
6124                 return 0;
6125
6126         for (i = 0; i < num_rep; i++) {
6127                 struct bnxt_representor representor = {
6128                         .vf_id = eth_da->representor_ports[i],
6129                         .switch_domain_id = backing_bp->switch_domain_id,
6130                         .parent_dev = backing_eth_dev
6131                 };
6132
6133                 if (representor.vf_id >= max_vf_reps) {
6134                         PMD_DRV_LOG(ERR, "VF-Rep id %d >= %d MAX VF ID\n",
6135                                     representor.vf_id, max_vf_reps);
6136                         continue;
6137                 }
6138
6139                 /* representor port net_bdf_port */
6140                 snprintf(name, sizeof(name), "net_%s_representor_%d",
6141                          pci_dev->device.name, eth_da->representor_ports[i]);
6142
6143                 kvlist = rte_kvargs_parse(dev_args, bnxt_dev_args);
6144                 if (kvlist) {
6145                         /*
6146                          * Handler for "rep_is_pf" devarg.
6147                          * Invoked as for ex: "-a 000:00:0d.0,
6148                          * rep-based-pf=<pf index> rep-is-pf=<VF=0 or PF=1>"
6149                          */
6150                         ret = rte_kvargs_process(kvlist, BNXT_DEVARG_REP_IS_PF,
6151                                                  bnxt_parse_devarg_rep_is_pf,
6152                                                  (void *)&representor);
6153                         if (ret) {
6154                                 ret = -EINVAL;
6155                                 goto err;
6156                         }
6157                         /*
6158                          * Handler for "rep_based_pf" devarg.
6159                          * Invoked as for ex: "-a 000:00:0d.0,
6160                          * rep-based-pf=<pf index> rep-is-pf=<VF=0 or PF=1>"
6161                          */
6162                         ret = rte_kvargs_process(kvlist,
6163                                                  BNXT_DEVARG_REP_BASED_PF,
6164                                                  bnxt_parse_devarg_rep_based_pf,
6165                                                  (void *)&representor);
6166                         if (ret) {
6167                                 ret = -EINVAL;
6168                                 goto err;
6169                         }
6170                         /*
6171                          * Handler for "rep_based_pf" devarg.
6172                          * Invoked as for ex: "-a 000:00:0d.0,
6173                          * rep-based-pf=<pf index> rep-is-pf=<VF=0 or PF=1>"
6174                          */
6175                         ret = rte_kvargs_process(kvlist, BNXT_DEVARG_REP_Q_R2F,
6176                                                  bnxt_parse_devarg_rep_q_r2f,
6177                                                  (void *)&representor);
6178                         if (ret) {
6179                                 ret = -EINVAL;
6180                                 goto err;
6181                         }
6182                         /*
6183                          * Handler for "rep_based_pf" devarg.
6184                          * Invoked as for ex: "-a 000:00:0d.0,
6185                          * rep-based-pf=<pf index> rep-is-pf=<VF=0 or PF=1>"
6186                          */
6187                         ret = rte_kvargs_process(kvlist, BNXT_DEVARG_REP_Q_F2R,
6188                                                  bnxt_parse_devarg_rep_q_f2r,
6189                                                  (void *)&representor);
6190                         if (ret) {
6191                                 ret = -EINVAL;
6192                                 goto err;
6193                         }
6194                         /*
6195                          * Handler for "rep_based_pf" devarg.
6196                          * Invoked as for ex: "-a 000:00:0d.0,
6197                          * rep-based-pf=<pf index> rep-is-pf=<VF=0 or PF=1>"
6198                          */
6199                         ret = rte_kvargs_process(kvlist, BNXT_DEVARG_REP_FC_R2F,
6200                                                  bnxt_parse_devarg_rep_fc_r2f,
6201                                                  (void *)&representor);
6202                         if (ret) {
6203                                 ret = -EINVAL;
6204                                 goto err;
6205                         }
6206                         /*
6207                          * Handler for "rep_based_pf" devarg.
6208                          * Invoked as for ex: "-a 000:00:0d.0,
6209                          * rep-based-pf=<pf index> rep-is-pf=<VF=0 or PF=1>"
6210                          */
6211                         ret = rte_kvargs_process(kvlist, BNXT_DEVARG_REP_FC_F2R,
6212                                                  bnxt_parse_devarg_rep_fc_f2r,
6213                                                  (void *)&representor);
6214                         if (ret) {
6215                                 ret = -EINVAL;
6216                                 goto err;
6217                         }
6218                 }
6219
6220                 ret = rte_eth_dev_create(&pci_dev->device, name,
6221                                          sizeof(struct bnxt_representor),
6222                                          NULL, NULL,
6223                                          bnxt_representor_init,
6224                                          &representor);
6225                 if (ret) {
6226                         PMD_DRV_LOG(ERR, "failed to create bnxt vf "
6227                                     "representor %s.", name);
6228                         goto err;
6229                 }
6230
6231                 vf_rep_eth_dev = rte_eth_dev_allocated(name);
6232                 if (!vf_rep_eth_dev) {
6233                         PMD_DRV_LOG(ERR, "Failed to find the eth_dev"
6234                                     " for VF-Rep: %s.", name);
6235                         ret = -ENODEV;
6236                         goto err;
6237                 }
6238
6239                 PMD_DRV_LOG(DEBUG, "BNXT Port:%d VFR pci probe\n",
6240                             backing_eth_dev->data->port_id);
6241                 backing_bp->rep_info[representor.vf_id].vfr_eth_dev =
6242                                                          vf_rep_eth_dev;
6243                 backing_bp->num_reps++;
6244
6245         }
6246
6247         rte_kvargs_free(kvlist);
6248         return 0;
6249
6250 err:
6251         /* If num_rep > 1, then rollback already created
6252          * ports, since we'll be failing the probe anyway
6253          */
6254         if (num_rep > 1)
6255                 bnxt_pci_remove_dev_with_reps(backing_eth_dev);
6256         rte_errno = -ret;
6257         rte_kvargs_free(kvlist);
6258
6259         return ret;
6260 }
6261
6262 static int bnxt_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
6263                           struct rte_pci_device *pci_dev)
6264 {
6265         struct rte_eth_devargs eth_da = { .nb_representor_ports = 0 };
6266         struct rte_eth_dev *backing_eth_dev;
6267         uint16_t num_rep;
6268         int ret = 0;
6269
6270         if (pci_dev->device.devargs) {
6271                 ret = rte_eth_devargs_parse(pci_dev->device.devargs->args,
6272                                             &eth_da);
6273                 if (ret)
6274                         return ret;
6275         }
6276
6277         num_rep = eth_da.nb_representor_ports;
6278         PMD_DRV_LOG(DEBUG, "nb_representor_ports = %d\n",
6279                     num_rep);
6280
6281         /* We could come here after first level of probe is already invoked
6282          * as part of an application bringup(OVS-DPDK vswitchd), so first check
6283          * for already allocated eth_dev for the backing device (PF/Trusted VF)
6284          */
6285         backing_eth_dev = rte_eth_dev_allocated(pci_dev->device.name);
6286         if (backing_eth_dev == NULL) {
6287                 ret = rte_eth_dev_create(&pci_dev->device, pci_dev->device.name,
6288                                          sizeof(struct bnxt),
6289                                          eth_dev_pci_specific_init, pci_dev,
6290                                          bnxt_dev_init, NULL);
6291
6292                 if (ret || !num_rep)
6293                         return ret;
6294
6295                 backing_eth_dev = rte_eth_dev_allocated(pci_dev->device.name);
6296         }
6297         PMD_DRV_LOG(DEBUG, "BNXT Port:%d pci probe\n",
6298                     backing_eth_dev->data->port_id);
6299
6300         if (!num_rep)
6301                 return ret;
6302
6303         /* probe representor ports now */
6304         ret = bnxt_rep_port_probe(pci_dev, &eth_da, backing_eth_dev,
6305                                   pci_dev->device.devargs->args);
6306
6307         return ret;
6308 }
6309
6310 static int bnxt_pci_remove(struct rte_pci_device *pci_dev)
6311 {
6312         struct rte_eth_dev *eth_dev;
6313
6314         eth_dev = rte_eth_dev_allocated(pci_dev->device.name);
6315         if (!eth_dev)
6316                 return 0; /* Invoked typically only by OVS-DPDK, by the
6317                            * time it comes here the eth_dev is already
6318                            * deleted by rte_eth_dev_close(), so returning
6319                            * +ve value will at least help in proper cleanup
6320                            */
6321
6322         PMD_DRV_LOG(DEBUG, "BNXT Port:%d pci remove\n", eth_dev->data->port_id);
6323         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
6324                 if (eth_dev->data->dev_flags & RTE_ETH_DEV_REPRESENTOR)
6325                         return rte_eth_dev_destroy(eth_dev,
6326                                                    bnxt_representor_uninit);
6327                 else
6328                         return rte_eth_dev_destroy(eth_dev,
6329                                                    bnxt_dev_uninit);
6330         } else {
6331                 return rte_eth_dev_pci_generic_remove(pci_dev, NULL);
6332         }
6333 }
6334
6335 static struct rte_pci_driver bnxt_rte_pmd = {
6336         .id_table = bnxt_pci_id_map,
6337         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC |
6338                         RTE_PCI_DRV_INTR_RMV |
6339                         RTE_PCI_DRV_PROBE_AGAIN, /* Needed in case of VF-REPs
6340                                                   * and OVS-DPDK
6341                                                   */
6342         .probe = bnxt_pci_probe,
6343         .remove = bnxt_pci_remove,
6344 };
6345
6346 static bool
6347 is_device_supported(struct rte_eth_dev *dev, struct rte_pci_driver *drv)
6348 {
6349         if (strcmp(dev->device->driver->name, drv->driver.name))
6350                 return false;
6351
6352         return true;
6353 }
6354
6355 bool is_bnxt_supported(struct rte_eth_dev *dev)
6356 {
6357         return is_device_supported(dev, &bnxt_rte_pmd);
6358 }
6359
6360 RTE_LOG_REGISTER_SUFFIX(bnxt_logtype_driver, driver, NOTICE);
6361 RTE_PMD_REGISTER_PCI(net_bnxt, bnxt_rte_pmd);
6362 RTE_PMD_REGISTER_PCI_TABLE(net_bnxt, bnxt_pci_id_map);
6363 RTE_PMD_REGISTER_KMOD_DEP(net_bnxt, "* igb_uio | uio_pci_generic | vfio-pci");