drivers/net/bnxt/bnxt_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2014-2018 Broadcom
   3  * All rights reserved.
   4  */
   5
   6 #include <inttypes.h>
   7 #include <stdbool.h>
   8
   9 #include <rte_dev.h>
  10 #include <rte_ethdev_driver.h>
  11 #include <rte_ethdev_pci.h>
  12 #include <rte_malloc.h>
  13 #include <rte_cycles.h>
  14 #include <rte_alarm.h>
  15 #include <rte_kvargs.h>
  16
  17 #include "bnxt.h"
  18 #include "bnxt_filter.h"
  19 #include "bnxt_hwrm.h"
  20 #include "bnxt_irq.h"
  21 #include "bnxt_ring.h"
  22 #include "bnxt_rxq.h"
  23 #include "bnxt_rxr.h"
  24 #include "bnxt_stats.h"
  25 #include "bnxt_txq.h"
  26 #include "bnxt_txr.h"
  27 #include "bnxt_vnic.h"
  28 #include "hsi_struct_def_dpdk.h"
  29 #include "bnxt_nvm_defs.h"
  30
  31 #define DRV_MODULE_NAME         "bnxt"
  32 static const char bnxt_version[] =
  33         "Broadcom NetXtreme driver " DRV_MODULE_NAME;
  34 int bnxt_logtype_driver;
  35
  36 /*
  37  * The set of PCI devices this driver supports
  38  */
  39 static const struct rte_pci_id bnxt_pci_id_map[] = {
  40         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM,
  41                          BROADCOM_DEV_ID_STRATUS_NIC_VF1) },
  42         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM,
  43                          BROADCOM_DEV_ID_STRATUS_NIC_VF2) },
  44         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_STRATUS_NIC) },
  45         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57414_VF) },
  46         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57301) },
  47         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57302) },
  48         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57304_PF) },
  49         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57304_VF) },
  50         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_NS2) },
  51         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57402) },
  52         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57404) },
  53         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57406_PF) },
  54         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57406_VF) },
  55         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57402_MF) },
  56         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57407_RJ45) },
  57         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57404_MF) },
  58         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57406_MF) },
  59         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57407_SFP) },
  60         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57407_MF) },
  61         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_5741X_VF) },
  62         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_5731X_VF) },
  63         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57314) },
  64         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57417_MF) },
  65         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57311) },
  66         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57312) },
  67         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57412) },
  68         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57414) },
  69         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57416_RJ45) },
  70         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57417_RJ45) },
  71         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57412_MF) },
  72         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57317_RJ45) },
  73         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57417_SFP) },
  74         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57416_SFP) },
  75         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57317_SFP) },
  76         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57414_MF) },
  77         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57416_MF) },
  78         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58802) },
  79         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58804) },
  80         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58808) },
  81         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58802_VF) },
  82         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57508) },
  83         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57504) },
  84         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57502) },
  85         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57500_VF1) },
  86         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57500_VF2) },
  87         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57508_MF1) },
  88         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57504_MF1) },
  89         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57502_MF1) },
  90         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57508_MF2) },
  91         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57504_MF2) },
  92         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57502_MF2) },
  93         { .vendor_id = 0, /* sentinel */ },
  94 };
  95
  96 #define BNXT_ETH_RSS_SUPPORT (  \
  97         ETH_RSS_IPV4 |          \
  98         ETH_RSS_NONFRAG_IPV4_TCP |      \
  99         ETH_RSS_NONFRAG_IPV4_UDP |      \
 100         ETH_RSS_IPV6 |          \
 101         ETH_RSS_NONFRAG_IPV6_TCP |      \
 102         ETH_RSS_NONFRAG_IPV6_UDP)
 103
 104 #define BNXT_DEV_TX_OFFLOAD_SUPPORT (DEV_TX_OFFLOAD_VLAN_INSERT | \
 105                                      DEV_TX_OFFLOAD_IPV4_CKSUM | \
 106                                      DEV_TX_OFFLOAD_TCP_CKSUM | \
 107                                      DEV_TX_OFFLOAD_UDP_CKSUM | \
 108                                      DEV_TX_OFFLOAD_TCP_TSO | \
 109                                      DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | \
 110                                      DEV_TX_OFFLOAD_VXLAN_TNL_TSO | \
 111                                      DEV_TX_OFFLOAD_GRE_TNL_TSO | \
 112                                      DEV_TX_OFFLOAD_IPIP_TNL_TSO | \
 113                                      DEV_TX_OFFLOAD_GENEVE_TNL_TSO | \
 114                                      DEV_TX_OFFLOAD_QINQ_INSERT | \
 115                                      DEV_TX_OFFLOAD_MULTI_SEGS)
 116
 117 #define BNXT_DEV_RX_OFFLOAD_SUPPORT (DEV_RX_OFFLOAD_VLAN_FILTER | \
 118                                      DEV_RX_OFFLOAD_VLAN_STRIP | \
 119                                      DEV_RX_OFFLOAD_IPV4_CKSUM | \
 120                                      DEV_RX_OFFLOAD_UDP_CKSUM | \
 121                                      DEV_RX_OFFLOAD_TCP_CKSUM | \
 122                                      DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM | \
 123                                      DEV_RX_OFFLOAD_JUMBO_FRAME | \
 124                                      DEV_RX_OFFLOAD_KEEP_CRC | \
 125                                      DEV_RX_OFFLOAD_VLAN_EXTEND | \
 126                                      DEV_RX_OFFLOAD_TCP_LRO | \
 127                                      DEV_RX_OFFLOAD_SCATTER | \
 128                                      DEV_RX_OFFLOAD_RSS_HASH)
 129
 130 #define BNXT_DEVARG_TRUFLOW     "host-based-truflow"
 131 #define BNXT_DEVARG_FLOW_XSTAT  "flow-xstat"
 132 static const char *const bnxt_dev_args[] = {
 133         BNXT_DEVARG_TRUFLOW,
 134         BNXT_DEVARG_FLOW_XSTAT,
 135         NULL
 136 };
 137
 138 /*
 139  * truflow == false to disable the feature
 140  * truflow == true to enable the feature
 141  */
 142 #define BNXT_DEVARG_TRUFLOW_INVALID(truflow)    ((truflow) > 1)
 143
 144 /*
 145  * flow_xstat == false to disable the feature
 146  * flow_xstat == true to enable the feature
 147  */
 148 #define BNXT_DEVARG_FLOW_XSTAT_INVALID(flow_xstat)      ((flow_xstat) > 1)
 149
 150 static int bnxt_vlan_offload_set_op(struct rte_eth_dev *dev, int mask);
 151 static void bnxt_print_link_info(struct rte_eth_dev *eth_dev);
 152 static int bnxt_dev_uninit(struct rte_eth_dev *eth_dev);
 153 static int bnxt_init_resources(struct bnxt *bp, bool reconfig_dev);
 154 static int bnxt_uninit_resources(struct bnxt *bp, bool reconfig_dev);
 155 static void bnxt_cancel_fw_health_check(struct bnxt *bp);
 156 static int bnxt_restore_vlan_filters(struct bnxt *bp);
 157 static void bnxt_dev_recover(void *arg);
 158 static void bnxt_free_error_recovery_info(struct bnxt *bp);
 159
 160 int is_bnxt_in_error(struct bnxt *bp)
 161 {
 162         if (bp->flags & BNXT_FLAG_FATAL_ERROR)
 163                 return -EIO;
 164         if (bp->flags & BNXT_FLAG_FW_RESET)
 165                 return -EBUSY;
 166
 167         return 0;
 168 }
 169
 170 /***********************/
 171
 172 /*
 173  * High level utility functions
 174  */
 175
 176 static uint16_t bnxt_rss_ctxts(const struct bnxt *bp)
 177 {
 178         if (!BNXT_CHIP_THOR(bp))
 179                 return 1;
 180
 181         return RTE_ALIGN_MUL_CEIL(bp->rx_nr_rings,
 182                                   BNXT_RSS_ENTRIES_PER_CTX_THOR) /
 183                                     BNXT_RSS_ENTRIES_PER_CTX_THOR;
 184 }
 185
 186 static uint16_t  bnxt_rss_hash_tbl_size(const struct bnxt *bp)
 187 {
 188         if (!BNXT_CHIP_THOR(bp))
 189                 return HW_HASH_INDEX_SIZE;
 190
 191         return bnxt_rss_ctxts(bp) * BNXT_RSS_ENTRIES_PER_CTX_THOR;
 192 }
 193
 194 static void bnxt_free_leds_info(struct bnxt *bp)
 195 {
 196         rte_free(bp->leds);
 197         bp->leds = NULL;
 198 }
 199
 200 static void bnxt_free_flow_stats_info(struct bnxt *bp)
 201 {
 202         rte_free(bp->flow_stat);
 203         bp->flow_stat = NULL;
 204 }
 205
 206 static void bnxt_free_cos_queues(struct bnxt *bp)
 207 {
 208         rte_free(bp->rx_cos_queue);
 209         rte_free(bp->tx_cos_queue);
 210 }
 211
 212 static void bnxt_free_mem(struct bnxt *bp, bool reconfig)
 213 {
 214         bnxt_free_flow_stats_info(bp);
 215
 216         bnxt_free_filter_mem(bp);
 217         bnxt_free_vnic_attributes(bp);
 218         bnxt_free_vnic_mem(bp);
 219
 220         /* tx/rx rings are configured as part of *_queue_setup callbacks.
 221          * If the number of rings change across fw update,
 222          * we don't have much choice except to warn the user.
 223          */
 224         if (!reconfig) {
 225                 bnxt_free_stats(bp);
 226                 bnxt_free_tx_rings(bp);
 227                 bnxt_free_rx_rings(bp);
 228         }
 229         bnxt_free_async_cp_ring(bp);
 230         bnxt_free_rxtx_nq_ring(bp);
 231
 232         rte_free(bp->grp_info);
 233         bp->grp_info = NULL;
 234 }
 235
 236 static int bnxt_alloc_leds_info(struct bnxt *bp)
 237 {
 238         bp->leds = rte_zmalloc("bnxt_leds",
 239                                BNXT_MAX_LED * sizeof(struct bnxt_led_info),
 240                                0);
 241         if (bp->leds == NULL)
 242                 return -ENOMEM;
 243
 244         return 0;
 245 }
 246
 247 static int bnxt_alloc_cos_queues(struct bnxt *bp)
 248 {
 249         bp->rx_cos_queue =
 250                 rte_zmalloc("bnxt_rx_cosq",
 251                             BNXT_COS_QUEUE_COUNT *
 252                             sizeof(struct bnxt_cos_queue_info),
 253                             0);
 254         if (bp->rx_cos_queue == NULL)
 255                 return -ENOMEM;
 256
 257         bp->tx_cos_queue =
 258                 rte_zmalloc("bnxt_tx_cosq",
 259                             BNXT_COS_QUEUE_COUNT *
 260                             sizeof(struct bnxt_cos_queue_info),
 261                             0);
 262         if (bp->tx_cos_queue == NULL)
 263                 return -ENOMEM;
 264
 265         return 0;
 266 }
 267
 268 static int bnxt_alloc_flow_stats_info(struct bnxt *bp)
 269 {
 270         bp->flow_stat = rte_zmalloc("bnxt_flow_xstat",
 271                                     sizeof(struct bnxt_flow_stat_info), 0);
 272         if (bp->flow_stat == NULL)
 273                 return -ENOMEM;
 274
 275         return 0;
 276 }
 277
 278 static int bnxt_alloc_mem(struct bnxt *bp, bool reconfig)
 279 {
 280         int rc;
 281
 282         rc = bnxt_alloc_ring_grps(bp);
 283         if (rc)
 284                 goto alloc_mem_err;
 285
 286         rc = bnxt_alloc_async_ring_struct(bp);
 287         if (rc)
 288                 goto alloc_mem_err;
 289
 290         rc = bnxt_alloc_vnic_mem(bp);
 291         if (rc)
 292                 goto alloc_mem_err;
 293
 294         rc = bnxt_alloc_vnic_attributes(bp);
 295         if (rc)
 296                 goto alloc_mem_err;
 297
 298         rc = bnxt_alloc_filter_mem(bp);
 299         if (rc)
 300                 goto alloc_mem_err;
 301
 302         rc = bnxt_alloc_async_cp_ring(bp);
 303         if (rc)
 304                 goto alloc_mem_err;
 305
 306         rc = bnxt_alloc_rxtx_nq_ring(bp);
 307         if (rc)
 308                 goto alloc_mem_err;
 309
 310         if (BNXT_FLOW_XSTATS_EN(bp)) {
 311                 rc = bnxt_alloc_flow_stats_info(bp);
 312                 if (rc)
 313                         goto alloc_mem_err;
 314         }
 315
 316         return 0;
 317
 318 alloc_mem_err:
 319         bnxt_free_mem(bp, reconfig);
 320         return rc;
 321 }
 322
 323 static int bnxt_setup_one_vnic(struct bnxt *bp, uint16_t vnic_id)
 324 {
 325         struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
 326         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
 327         uint64_t rx_offloads = dev_conf->rxmode.offloads;
 328         struct bnxt_rx_queue *rxq;
 329         unsigned int j;
 330         int rc;
 331
 332         rc = bnxt_vnic_grp_alloc(bp, vnic);
 333         if (rc)
 334                 goto err_out;
 335
 336         PMD_DRV_LOG(DEBUG, "vnic[%d] = %p vnic->fw_grp_ids = %p\n",
 337                     vnic_id, vnic, vnic->fw_grp_ids);
 338
 339         rc = bnxt_hwrm_vnic_alloc(bp, vnic);
 340         if (rc)
 341                 goto err_out;
 342
 343         /* Alloc RSS context only if RSS mode is enabled */
 344         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS) {
 345                 int j, nr_ctxs = bnxt_rss_ctxts(bp);
 346
 347                 rc = 0;
 348                 for (j = 0; j < nr_ctxs; j++) {
 349                         rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic, j);
 350                         if (rc)
 351                                 break;
 352                 }
 353                 if (rc) {
 354                         PMD_DRV_LOG(ERR,
 355                                     "HWRM vnic %d ctx %d alloc failure rc: %x\n",
 356                                     vnic_id, j, rc);
 357                         goto err_out;
 358                 }
 359                 vnic->num_lb_ctxts = nr_ctxs;
 360         }
 361
 362         /*
 363          * Firmware sets pf pair in default vnic cfg. If the VLAN strip
 364          * setting is not available at this time, it will not be
 365          * configured correctly in the CFA.
 366          */
 367         if (rx_offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
 368                 vnic->vlan_strip = true;
 369         else
 370                 vnic->vlan_strip = false;
 371
 372         rc = bnxt_hwrm_vnic_cfg(bp, vnic);
 373         if (rc)
 374                 goto err_out;
 375
 376         rc = bnxt_set_hwrm_vnic_filters(bp, vnic);
 377         if (rc)
 378                 goto err_out;
 379
 380         for (j = 0; j < bp->rx_num_qs_per_vnic; j++) {
 381                 rxq = bp->eth_dev->data->rx_queues[j];
 382
 383                 PMD_DRV_LOG(DEBUG,
 384                             "rxq[%d]->vnic=%p vnic->fw_grp_ids=%p\n",
 385                             j, rxq->vnic, rxq->vnic->fw_grp_ids);
 386
 387                 if (BNXT_HAS_RING_GRPS(bp) && rxq->rx_deferred_start)
 388                         rxq->vnic->fw_grp_ids[j] = INVALID_HW_RING_ID;
 389                 else
 390                         vnic->rx_queue_cnt++;
 391         }
 392
 393         PMD_DRV_LOG(DEBUG, "vnic->rx_queue_cnt = %d\n", vnic->rx_queue_cnt);
 394
 395         rc = bnxt_vnic_rss_configure(bp, vnic);
 396         if (rc)
 397                 goto err_out;
 398
 399         bnxt_hwrm_vnic_plcmode_cfg(bp, vnic);
 400
 401         if (rx_offloads & DEV_RX_OFFLOAD_TCP_LRO)
 402                 bnxt_hwrm_vnic_tpa_cfg(bp, vnic, 1);
 403         else
 404                 bnxt_hwrm_vnic_tpa_cfg(bp, vnic, 0);
 405
 406         return 0;
 407 err_out:
 408         PMD_DRV_LOG(ERR, "HWRM vnic %d cfg failure rc: %x\n",
 409                     vnic_id, rc);
 410         return rc;
 411 }
 412
 413 static int bnxt_register_fc_ctx_mem(struct bnxt *bp)
 414 {
 415         int rc = 0;
 416
 417         rc = bnxt_hwrm_ctx_rgtr(bp, bp->flow_stat->rx_fc_in_tbl.dma,
 418                                 &bp->flow_stat->rx_fc_in_tbl.ctx_id);
 419         if (rc)
 420                 return rc;
 421
 422         PMD_DRV_LOG(DEBUG,
 423                     "rx_fc_in_tbl.va = %p rx_fc_in_tbl.dma = %p"
 424                     " rx_fc_in_tbl.ctx_id = %d\n",
 425                     bp->flow_stat->rx_fc_in_tbl.va,
 426                     (void *)((uintptr_t)bp->flow_stat->rx_fc_in_tbl.dma),
 427                     bp->flow_stat->rx_fc_in_tbl.ctx_id);
 428
 429         rc = bnxt_hwrm_ctx_rgtr(bp, bp->flow_stat->rx_fc_out_tbl.dma,
 430                                 &bp->flow_stat->rx_fc_out_tbl.ctx_id);
 431         if (rc)
 432                 return rc;
 433
 434         PMD_DRV_LOG(DEBUG,
 435                     "rx_fc_out_tbl.va = %p rx_fc_out_tbl.dma = %p"
 436                     " rx_fc_out_tbl.ctx_id = %d\n",
 437                     bp->flow_stat->rx_fc_out_tbl.va,
 438                     (void *)((uintptr_t)bp->flow_stat->rx_fc_out_tbl.dma),
 439                     bp->flow_stat->rx_fc_out_tbl.ctx_id);
 440
 441         rc = bnxt_hwrm_ctx_rgtr(bp, bp->flow_stat->tx_fc_in_tbl.dma,
 442                                 &bp->flow_stat->tx_fc_in_tbl.ctx_id);
 443         if (rc)
 444                 return rc;
 445
 446         PMD_DRV_LOG(DEBUG,
 447                     "tx_fc_in_tbl.va = %p tx_fc_in_tbl.dma = %p"
 448                     " tx_fc_in_tbl.ctx_id = %d\n",
 449                     bp->flow_stat->tx_fc_in_tbl.va,
 450                     (void *)((uintptr_t)bp->flow_stat->tx_fc_in_tbl.dma),
 451                     bp->flow_stat->tx_fc_in_tbl.ctx_id);
 452
 453         rc = bnxt_hwrm_ctx_rgtr(bp, bp->flow_stat->tx_fc_out_tbl.dma,
 454                                 &bp->flow_stat->tx_fc_out_tbl.ctx_id);
 455         if (rc)
 456                 return rc;
 457
 458         PMD_DRV_LOG(DEBUG,
 459                     "tx_fc_out_tbl.va = %p tx_fc_out_tbl.dma = %p"
 460                     " tx_fc_out_tbl.ctx_id = %d\n",
 461                     bp->flow_stat->tx_fc_out_tbl.va,
 462                     (void *)((uintptr_t)bp->flow_stat->tx_fc_out_tbl.dma),
 463                     bp->flow_stat->tx_fc_out_tbl.ctx_id);
 464
 465         memset(bp->flow_stat->rx_fc_out_tbl.va,
 466                0,
 467                bp->flow_stat->rx_fc_out_tbl.size);
 468         rc = bnxt_hwrm_cfa_counter_cfg(bp, BNXT_DIR_RX,
 469                                        CFA_COUNTER_CFG_IN_COUNTER_TYPE_FC,
 470                                        bp->flow_stat->rx_fc_out_tbl.ctx_id,
 471                                        bp->flow_stat->max_fc,
 472                                        true);
 473         if (rc)
 474                 return rc;
 475
 476         memset(bp->flow_stat->tx_fc_out_tbl.va,
 477                0,
 478                bp->flow_stat->tx_fc_out_tbl.size);
 479         rc = bnxt_hwrm_cfa_counter_cfg(bp, BNXT_DIR_TX,
 480                                        CFA_COUNTER_CFG_IN_COUNTER_TYPE_FC,
 481                                        bp->flow_stat->tx_fc_out_tbl.ctx_id,
 482                                        bp->flow_stat->max_fc,
 483                                        true);
 484
 485         return rc;
 486 }
 487
 488 static int bnxt_alloc_ctx_mem_buf(char *type, size_t size,
 489                                   struct bnxt_ctx_mem_buf_info *ctx)
 490 {
 491         if (!ctx)
 492                 return -EINVAL;
 493
 494         ctx->va = rte_zmalloc(type, size, 0);
 495         if (ctx->va == NULL)
 496                 return -ENOMEM;
 497         rte_mem_lock_page(ctx->va);
 498         ctx->size = size;
 499         ctx->dma = rte_mem_virt2iova(ctx->va);
 500         if (ctx->dma == RTE_BAD_IOVA)
 501                 return -ENOMEM;
 502
 503         return 0;
 504 }
 505
 506 static int bnxt_init_fc_ctx_mem(struct bnxt *bp)
 507 {
 508         struct rte_pci_device *pdev = bp->pdev;
 509         char type[RTE_MEMZONE_NAMESIZE];
 510         uint16_t max_fc;
 511         int rc = 0;
 512
 513         max_fc = bp->flow_stat->max_fc;
 514
 515         sprintf(type, "bnxt_rx_fc_in_" PCI_PRI_FMT, pdev->addr.domain,
 516                 pdev->addr.bus, pdev->addr.devid, pdev->addr.function);
 517         /* 4 bytes for each counter-id */
 518         rc = bnxt_alloc_ctx_mem_buf(type,
 519                                     max_fc * 4,
 520                                     &bp->flow_stat->rx_fc_in_tbl);
 521         if (rc)
 522                 return rc;
 523
 524         sprintf(type, "bnxt_rx_fc_out_" PCI_PRI_FMT, pdev->addr.domain,
 525                 pdev->addr.bus, pdev->addr.devid, pdev->addr.function);
 526         /* 16 bytes for each counter - 8 bytes pkt_count, 8 bytes byte_count */
 527         rc = bnxt_alloc_ctx_mem_buf(type,
 528                                     max_fc * 16,
 529                                     &bp->flow_stat->rx_fc_out_tbl);
 530         if (rc)
 531                 return rc;
 532
 533         sprintf(type, "bnxt_tx_fc_in_" PCI_PRI_FMT, pdev->addr.domain,
 534                 pdev->addr.bus, pdev->addr.devid, pdev->addr.function);
 535         /* 4 bytes for each counter-id */
 536         rc = bnxt_alloc_ctx_mem_buf(type,
 537                                     max_fc * 4,
 538                                     &bp->flow_stat->tx_fc_in_tbl);
 539         if (rc)
 540                 return rc;
 541
 542         sprintf(type, "bnxt_tx_fc_out_" PCI_PRI_FMT, pdev->addr.domain,
 543                 pdev->addr.bus, pdev->addr.devid, pdev->addr.function);
 544         /* 16 bytes for each counter - 8 bytes pkt_count, 8 bytes byte_count */
 545         rc = bnxt_alloc_ctx_mem_buf(type,
 546                                     max_fc * 16,
 547                                     &bp->flow_stat->tx_fc_out_tbl);
 548         if (rc)
 549                 return rc;
 550
 551         rc = bnxt_register_fc_ctx_mem(bp);
 552
 553         return rc;
 554 }
 555
 556 static int bnxt_init_ctx_mem(struct bnxt *bp)
 557 {
 558         int rc = 0;
 559
 560         if (!(bp->fw_cap & BNXT_FW_CAP_ADV_FLOW_COUNTERS) ||
 561             !(BNXT_PF(bp) || BNXT_VF_IS_TRUSTED(bp)) ||
 562             !BNXT_FLOW_XSTATS_EN(bp))
 563                 return 0;
 564
 565         rc = bnxt_hwrm_cfa_counter_qcaps(bp, &bp->flow_stat->max_fc);
 566         if (rc)
 567                 return rc;
 568
 569         rc = bnxt_init_fc_ctx_mem(bp);
 570
 571         return rc;
 572 }
 573
 574 static int bnxt_init_chip(struct bnxt *bp)
 575 {
 576         struct rte_eth_link new;
 577         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(bp->eth_dev);
 578         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 579         uint32_t intr_vector = 0;
 580         uint32_t queue_id, base = BNXT_MISC_VEC_ID;
 581         uint32_t vec = BNXT_MISC_VEC_ID;
 582         unsigned int i, j;
 583         int rc;
 584
 585         if (bp->eth_dev->data->mtu > RTE_ETHER_MTU) {
 586                 bp->eth_dev->data->dev_conf.rxmode.offloads |=
 587                         DEV_RX_OFFLOAD_JUMBO_FRAME;
 588                 bp->flags |= BNXT_FLAG_JUMBO;
 589         } else {
 590                 bp->eth_dev->data->dev_conf.rxmode.offloads &=
 591                         ~DEV_RX_OFFLOAD_JUMBO_FRAME;
 592                 bp->flags &= ~BNXT_FLAG_JUMBO;
 593         }
 594
 595         /* THOR does not support ring groups.
 596          * But we will use the array to save RSS context IDs.
 597          */
 598         if (BNXT_CHIP_THOR(bp))
 599                 bp->max_ring_grps = BNXT_MAX_RSS_CTXTS_THOR;
 600
 601         rc = bnxt_alloc_all_hwrm_stat_ctxs(bp);
 602         if (rc) {
 603                 PMD_DRV_LOG(ERR, "HWRM stat ctx alloc failure rc: %x\n", rc);
 604                 goto err_out;
 605         }
 606
 607         rc = bnxt_alloc_hwrm_rings(bp);
 608         if (rc) {
 609                 PMD_DRV_LOG(ERR, "HWRM ring alloc failure rc: %x\n", rc);
 610                 goto err_out;
 611         }
 612
 613         rc = bnxt_alloc_all_hwrm_ring_grps(bp);
 614         if (rc) {
 615                 PMD_DRV_LOG(ERR, "HWRM ring grp alloc failure: %x\n", rc);
 616                 goto err_out;
 617         }
 618
 619         if (!(bp->vnic_cap_flags & BNXT_VNIC_CAP_COS_CLASSIFY))
 620                 goto skip_cosq_cfg;
 621
 622         for (j = 0, i = 0; i < BNXT_COS_QUEUE_COUNT; i++) {
 623                 if (bp->rx_cos_queue[i].id != 0xff) {
 624                         struct bnxt_vnic_info *vnic = &bp->vnic_info[j++];
 625
 626                         if (!vnic) {
 627                                 PMD_DRV_LOG(ERR,
 628                                             "Num pools more than FW profile\n");
 629                                 rc = -EINVAL;
 630                                 goto err_out;
 631                         }
 632                         vnic->cos_queue_id = bp->rx_cos_queue[i].id;
 633                         bp->rx_cosq_cnt++;
 634                 }
 635         }
 636
 637 skip_cosq_cfg:
 638         rc = bnxt_mq_rx_configure(bp);
 639         if (rc) {
 640                 PMD_DRV_LOG(ERR, "MQ mode configure failure rc: %x\n", rc);
 641                 goto err_out;
 642         }
 643
 644         /* VNIC configuration */
 645         for (i = 0; i < bp->nr_vnics; i++) {
 646                 rc = bnxt_setup_one_vnic(bp, i);
 647                 if (rc)
 648                         goto err_out;
 649         }
 650
 651         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, &bp->vnic_info[0], 0, NULL);
 652         if (rc) {
 653                 PMD_DRV_LOG(ERR,
 654                         "HWRM cfa l2 rx mask failure rc: %x\n", rc);
 655                 goto err_out;
 656         }
 657
 658         /* check and configure queue intr-vector mapping */
 659         if ((rte_intr_cap_multiple(intr_handle) ||
 660              !RTE_ETH_DEV_SRIOV(bp->eth_dev).active) &&
 661             bp->eth_dev->data->dev_conf.intr_conf.rxq != 0) {
 662                 intr_vector = bp->eth_dev->data->nb_rx_queues;
 663                 PMD_DRV_LOG(DEBUG, "intr_vector = %d\n", intr_vector);
 664                 if (intr_vector > bp->rx_cp_nr_rings) {
 665                         PMD_DRV_LOG(ERR, "At most %d intr queues supported",
 666                                         bp->rx_cp_nr_rings);
 667                         return -ENOTSUP;
 668                 }
 669                 rc = rte_intr_efd_enable(intr_handle, intr_vector);
 670                 if (rc)
 671                         return rc;
 672         }
 673
 674         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
 675                 intr_handle->intr_vec =
 676                         rte_zmalloc("intr_vec",
 677                                     bp->eth_dev->data->nb_rx_queues *
 678                                     sizeof(int), 0);
 679                 if (intr_handle->intr_vec == NULL) {
 680                         PMD_DRV_LOG(ERR, "Failed to allocate %d rx_queues"
 681                                 " intr_vec", bp->eth_dev->data->nb_rx_queues);
 682                         rc = -ENOMEM;
 683                         goto err_disable;
 684                 }
 685                 PMD_DRV_LOG(DEBUG, "intr_handle->intr_vec = %p "
 686                         "intr_handle->nb_efd = %d intr_handle->max_intr = %d\n",
 687                          intr_handle->intr_vec, intr_handle->nb_efd,
 688                         intr_handle->max_intr);
 689                 for (queue_id = 0; queue_id < bp->eth_dev->data->nb_rx_queues;
 690                      queue_id++) {
 691                         intr_handle->intr_vec[queue_id] =
 692                                                         vec + BNXT_RX_VEC_START;
 693                         if (vec < base + intr_handle->nb_efd - 1)
 694                                 vec++;
 695                 }
 696         }
 697
 698         /* enable uio/vfio intr/eventfd mapping */
 699         rc = rte_intr_enable(intr_handle);
 700 #ifndef RTE_EXEC_ENV_FREEBSD
 701         /* In FreeBSD OS, nic_uio driver does not support interrupts */
 702         if (rc)
 703                 goto err_free;
 704 #endif
 705
 706         rc = bnxt_get_hwrm_link_config(bp, &new);
 707         if (rc) {
 708                 PMD_DRV_LOG(ERR, "HWRM Get link config failure rc: %x\n", rc);
 709                 goto err_free;
 710         }
 711
 712         if (!bp->link_info.link_up) {
 713                 rc = bnxt_set_hwrm_link_config(bp, true);
 714                 if (rc) {
 715                         PMD_DRV_LOG(ERR,
 716                                 "HWRM link config failure rc: %x\n", rc);
 717                         goto err_free;
 718                 }
 719         }
 720         bnxt_print_link_info(bp->eth_dev);
 721
 722         bp->mark_table = rte_zmalloc("bnxt_mark_table", BNXT_MARK_TABLE_SZ, 0);
 723         if (!bp->mark_table)
 724                 PMD_DRV_LOG(ERR, "Allocation of mark table failed\n");
 725
 726         return 0;
 727
 728 err_free:
 729         rte_free(intr_handle->intr_vec);
 730 err_disable:
 731         rte_intr_efd_disable(intr_handle);
 732 err_out:
 733         /* Some of the error status returned by FW may not be from errno.h */
 734         if (rc > 0)
 735                 rc = -EIO;
 736
 737         return rc;
 738 }
 739
 740 static int bnxt_shutdown_nic(struct bnxt *bp)
 741 {
 742         bnxt_free_all_hwrm_resources(bp);
 743         bnxt_free_all_filters(bp);
 744         bnxt_free_all_vnics(bp);
 745         return 0;
 746 }
 747
 748 /*
 749  * Device configuration and status function
 750  */
 751
 752 static uint32_t bnxt_get_speed_capabilities(struct bnxt *bp)
 753 {
 754         uint32_t link_speed = bp->link_info.support_speeds;
 755         uint32_t speed_capa = 0;
 756
 757         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_LINK_SPEED_100MB)
 758                 speed_capa |= ETH_LINK_SPEED_100M;
 759         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100MBHD)
 760                 speed_capa |= ETH_LINK_SPEED_100M_HD;
 761         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_1GB)
 762                 speed_capa |= ETH_LINK_SPEED_1G;
 763         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_2_5GB)
 764                 speed_capa |= ETH_LINK_SPEED_2_5G;
 765         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_10GB)
 766                 speed_capa |= ETH_LINK_SPEED_10G;
 767         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_20GB)
 768                 speed_capa |= ETH_LINK_SPEED_20G;
 769         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_25GB)
 770                 speed_capa |= ETH_LINK_SPEED_25G;
 771         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_40GB)
 772                 speed_capa |= ETH_LINK_SPEED_40G;
 773         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_50GB)
 774                 speed_capa |= ETH_LINK_SPEED_50G;
 775         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_100GB)
 776                 speed_capa |= ETH_LINK_SPEED_100G;
 777         if (link_speed & HWRM_PORT_PHY_QCFG_OUTPUT_SUPPORT_SPEEDS_200GB)
 778                 speed_capa |= ETH_LINK_SPEED_200G;
 779
 780         if (bp->link_info.auto_mode == HWRM_PORT_PHY_QCFG_OUTPUT_AUTO_MODE_NONE)
 781                 speed_capa |= ETH_LINK_SPEED_FIXED;
 782         else
 783                 speed_capa |= ETH_LINK_SPEED_AUTONEG;
 784
 785         return speed_capa;
 786 }
 787
 788 static int bnxt_dev_info_get_op(struct rte_eth_dev *eth_dev,
 789                                 struct rte_eth_dev_info *dev_info)
 790 {
 791         struct rte_pci_device *pdev = RTE_DEV_TO_PCI(eth_dev->device);
 792         struct bnxt *bp = eth_dev->data->dev_private;
 793         uint16_t max_vnics, i, j, vpool, vrxq;
 794         unsigned int max_rx_rings;
 795         int rc;
 796
 797         rc = is_bnxt_in_error(bp);
 798         if (rc)
 799                 return rc;
 800
 801         /* MAC Specifics */
 802         dev_info->max_mac_addrs = bp->max_l2_ctx;
 803         dev_info->max_hash_mac_addrs = 0;
 804
 805         /* PF/VF specifics */
 806         if (BNXT_PF(bp))
 807                 dev_info->max_vfs = pdev->max_vfs;
 808
 809         max_rx_rings = BNXT_MAX_RINGS(bp);
 810         /* For the sake of symmetry, max_rx_queues = max_tx_queues */
 811         dev_info->max_rx_queues = max_rx_rings;
 812         dev_info->max_tx_queues = max_rx_rings;
 813         dev_info->reta_size = bnxt_rss_hash_tbl_size(bp);
 814         dev_info->hash_key_size = 40;
 815         max_vnics = bp->max_vnics;
 816
 817         /* MTU specifics */
 818         dev_info->min_mtu = RTE_ETHER_MIN_MTU;
 819         dev_info->max_mtu = BNXT_MAX_MTU;
 820
 821         /* Fast path specifics */
 822         dev_info->min_rx_bufsize = 1;
 823         dev_info->max_rx_pktlen = BNXT_MAX_PKT_LEN;
 824
 825         dev_info->rx_offload_capa = BNXT_DEV_RX_OFFLOAD_SUPPORT;
 826         if (bp->flags & BNXT_FLAG_PTP_SUPPORTED)
 827                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TIMESTAMP;
 828         dev_info->tx_offload_capa = BNXT_DEV_TX_OFFLOAD_SUPPORT;
 829         dev_info->flow_type_rss_offloads = BNXT_ETH_RSS_SUPPORT;
 830
 831         dev_info->speed_capa = bnxt_get_speed_capabilities(bp);
 832
 833         /* *INDENT-OFF* */
 834         dev_info->default_rxconf = (struct rte_eth_rxconf) {
 835                 .rx_thresh = {
 836                         .pthresh = 8,
 837                         .hthresh = 8,
 838                         .wthresh = 0,
 839                 },
 840                 .rx_free_thresh = 32,
 841                 /* If no descriptors available, pkts are dropped by default */
 842                 .rx_drop_en = 1,
 843         };
 844
 845         dev_info->default_txconf = (struct rte_eth_txconf) {
 846                 .tx_thresh = {
 847                         .pthresh = 32,
 848                         .hthresh = 0,
 849                         .wthresh = 0,
 850                 },
 851                 .tx_free_thresh = 32,
 852                 .tx_rs_thresh = 32,
 853         };
 854         eth_dev->data->dev_conf.intr_conf.lsc = 1;
 855
 856         eth_dev->data->dev_conf.intr_conf.rxq = 1;
 857         dev_info->rx_desc_lim.nb_min = BNXT_MIN_RING_DESC;
 858         dev_info->rx_desc_lim.nb_max = BNXT_MAX_RX_RING_DESC;
 859         dev_info->tx_desc_lim.nb_min = BNXT_MIN_RING_DESC;
 860         dev_info->tx_desc_lim.nb_max = BNXT_MAX_TX_RING_DESC;
 861
 862         /* *INDENT-ON* */
 863
 864         /*
 865          * TODO: default_rxconf, default_txconf, rx_desc_lim, and tx_desc_lim
 866          *       need further investigation.
 867          */
 868
 869         /* VMDq resources */
 870         vpool = 64; /* ETH_64_POOLS */
 871         vrxq = 128; /* ETH_VMDQ_DCB_NUM_QUEUES */
 872         for (i = 0; i < 4; vpool >>= 1, i++) {
 873                 if (max_vnics > vpool) {
 874                         for (j = 0; j < 5; vrxq >>= 1, j++) {
 875                                 if (dev_info->max_rx_queues > vrxq) {
 876                                         if (vpool > vrxq)
 877                                                 vpool = vrxq;
 878                                         goto found;
 879                                 }
 880                         }
 881                         /* Not enough resources to support VMDq */
 882                         break;
 883                 }
 884         }
 885         /* Not enough resources to support VMDq */
 886         vpool = 0;
 887         vrxq = 0;
 888 found:
 889         dev_info->max_vmdq_pools = vpool;
 890         dev_info->vmdq_queue_num = vrxq;
 891
 892         dev_info->vmdq_pool_base = 0;
 893         dev_info->vmdq_queue_base = 0;
 894
 895         return 0;
 896 }
 897
 898 /* Configure the device based on the configuration provided */
 899 static int bnxt_dev_configure_op(struct rte_eth_dev *eth_dev)
 900 {
 901         struct bnxt *bp = eth_dev->data->dev_private;
 902         uint64_t rx_offloads = eth_dev->data->dev_conf.rxmode.offloads;
 903         int rc;
 904
 905         bp->rx_queues = (void *)eth_dev->data->rx_queues;
 906         bp->tx_queues = (void *)eth_dev->data->tx_queues;
 907         bp->tx_nr_rings = eth_dev->data->nb_tx_queues;
 908         bp->rx_nr_rings = eth_dev->data->nb_rx_queues;
 909
 910         rc = is_bnxt_in_error(bp);
 911         if (rc)
 912                 return rc;
 913
 914         if (BNXT_VF(bp) && (bp->flags & BNXT_FLAG_NEW_RM)) {
 915                 rc = bnxt_hwrm_check_vf_rings(bp);
 916                 if (rc) {
 917                         PMD_DRV_LOG(ERR, "HWRM insufficient resources\n");
 918                         return -ENOSPC;
 919                 }
 920
 921                 /* If a resource has already been allocated - in this case
 922                  * it is the async completion ring, free it. Reallocate it after
 923                  * resource reservation. This will ensure the resource counts
 924                  * are calculated correctly.
 925                  */
 926
 927                 pthread_mutex_lock(&bp->def_cp_lock);
 928
 929                 if (!BNXT_HAS_NQ(bp) && bp->async_cp_ring) {
 930                         bnxt_disable_int(bp);
 931                         bnxt_free_cp_ring(bp, bp->async_cp_ring);
 932                 }
 933
 934                 rc = bnxt_hwrm_func_reserve_vf_resc(bp, false);
 935                 if (rc) {
 936                         PMD_DRV_LOG(ERR, "HWRM resource alloc fail:%x\n", rc);
 937                         pthread_mutex_unlock(&bp->def_cp_lock);
 938                         return -ENOSPC;
 939                 }
 940
 941                 if (!BNXT_HAS_NQ(bp) && bp->async_cp_ring) {
 942                         rc = bnxt_alloc_async_cp_ring(bp);
 943                         if (rc) {
 944                                 pthread_mutex_unlock(&bp->def_cp_lock);
 945                                 return rc;
 946                         }
 947                         bnxt_enable_int(bp);
 948                 }
 949
 950                 pthread_mutex_unlock(&bp->def_cp_lock);
 951         } else {
 952                 /* legacy driver needs to get updated values */
 953                 rc = bnxt_hwrm_func_qcaps(bp);
 954                 if (rc) {
 955                         PMD_DRV_LOG(ERR, "hwrm func qcaps fail:%d\n", rc);
 956                         return rc;
 957                 }
 958         }
 959
 960         /* Inherit new configurations */
 961         if (eth_dev->data->nb_rx_queues > bp->max_rx_rings ||
 962             eth_dev->data->nb_tx_queues > bp->max_tx_rings ||
 963             eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues
 964                 + BNXT_NUM_ASYNC_CPR(bp) > bp->max_cp_rings ||
 965             eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues >
 966             bp->max_stat_ctx)
 967                 goto resource_error;
 968
 969         if (BNXT_HAS_RING_GRPS(bp) &&
 970             (uint32_t)(eth_dev->data->nb_rx_queues) > bp->max_ring_grps)
 971                 goto resource_error;
 972
 973         if (!(eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS) &&
 974             bp->max_vnics < eth_dev->data->nb_rx_queues)
 975                 goto resource_error;
 976
 977         bp->rx_cp_nr_rings = bp->rx_nr_rings;
 978         bp->tx_cp_nr_rings = bp->tx_nr_rings;
 979
 980         if (eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)
 981                 rx_offloads |= DEV_RX_OFFLOAD_RSS_HASH;
 982         eth_dev->data->dev_conf.rxmode.offloads = rx_offloads;
 983
 984         if (rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
 985                 eth_dev->data->mtu =
 986                         eth_dev->data->dev_conf.rxmode.max_rx_pkt_len -
 987                         RTE_ETHER_HDR_LEN - RTE_ETHER_CRC_LEN - VLAN_TAG_SIZE *
 988                         BNXT_NUM_VLANS;
 989                 bnxt_mtu_set_op(eth_dev, eth_dev->data->mtu);
 990         }
 991         return 0;
 992
 993 resource_error:
 994         PMD_DRV_LOG(ERR,
 995                     "Insufficient resources to support requested config\n");
 996         PMD_DRV_LOG(ERR,
 997                     "Num Queues Requested: Tx %d, Rx %d\n",
 998                     eth_dev->data->nb_tx_queues,
 999                     eth_dev->data->nb_rx_queues);
1000         PMD_DRV_LOG(ERR,
1001                     "MAX: TxQ %d, RxQ %d, CQ %d Stat %d, Grp %d, Vnic %d\n",
1002                     bp->max_tx_rings, bp->max_rx_rings, bp->max_cp_rings,
1003                     bp->max_stat_ctx, bp->max_ring_grps, bp->max_vnics);
1004         return -ENOSPC;
1005 }
1006
1007 static void bnxt_print_link_info(struct rte_eth_dev *eth_dev)
1008 {
1009         struct rte_eth_link *link = &eth_dev->data->dev_link;
1010
1011         if (link->link_status)
1012                 PMD_DRV_LOG(INFO, "Port %d Link Up - speed %u Mbps - %s\n",
1013                         eth_dev->data->port_id,
1014                         (uint32_t)link->link_speed,
1015                         (link->link_duplex == ETH_LINK_FULL_DUPLEX) ?
1016                         ("full-duplex") : ("half-duplex\n"));
1017         else
1018                 PMD_DRV_LOG(INFO, "Port %d Link Down\n",
1019                         eth_dev->data->port_id);
1020 }
1021
1022 /*
1023  * Determine whether the current configuration requires support for scattered
1024  * receive; return 1 if scattered receive is required and 0 if not.
1025  */
1026 static int bnxt_scattered_rx(struct rte_eth_dev *eth_dev)
1027 {
1028         uint16_t buf_size;
1029         int i;
1030
1031         if (eth_dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_SCATTER)
1032                 return 1;
1033
1034         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
1035                 struct bnxt_rx_queue *rxq = eth_dev->data->rx_queues[i];
1036
1037                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
1038                                       RTE_PKTMBUF_HEADROOM);
1039                 if (eth_dev->data->dev_conf.rxmode.max_rx_pkt_len > buf_size)
1040                         return 1;
1041         }
1042         return 0;
1043 }
1044
1045 static eth_rx_burst_t
1046 bnxt_receive_function(struct rte_eth_dev *eth_dev)
1047 {
1048         struct bnxt *bp = eth_dev->data->dev_private;
1049
1050 #ifdef RTE_ARCH_X86
1051 #ifndef RTE_LIBRTE_IEEE1588
1052         /*
1053          * Vector mode receive can be enabled only if scatter rx is not
1054          * in use and rx offloads are limited to VLAN stripping and
1055          * CRC stripping.
1056          */
1057         if (!eth_dev->data->scattered_rx &&
1058             !(eth_dev->data->dev_conf.rxmode.offloads &
1059               ~(DEV_RX_OFFLOAD_VLAN_STRIP |
1060                 DEV_RX_OFFLOAD_KEEP_CRC |
1061                 DEV_RX_OFFLOAD_JUMBO_FRAME |
1062                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1063                 DEV_RX_OFFLOAD_UDP_CKSUM |
1064                 DEV_RX_OFFLOAD_TCP_CKSUM |
1065                 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
1066                 DEV_RX_OFFLOAD_RSS_HASH |
1067                 DEV_RX_OFFLOAD_VLAN_FILTER)) &&
1068             !bp->truflow) {
1069                 PMD_DRV_LOG(INFO, "Using vector mode receive for port %d\n",
1070                             eth_dev->data->port_id);
1071                 bp->flags |= BNXT_FLAG_RX_VECTOR_PKT_MODE;
1072                 return bnxt_recv_pkts_vec;
1073         }
1074         PMD_DRV_LOG(INFO, "Vector mode receive disabled for port %d\n",
1075                     eth_dev->data->port_id);
1076         PMD_DRV_LOG(INFO,
1077                     "Port %d scatter: %d rx offload: %" PRIX64 "\n",
1078                     eth_dev->data->port_id,
1079                     eth_dev->data->scattered_rx,
1080                     eth_dev->data->dev_conf.rxmode.offloads);
1081 #endif
1082 #endif
1083         bp->flags &= ~BNXT_FLAG_RX_VECTOR_PKT_MODE;
1084         return bnxt_recv_pkts;
1085 }
1086
1087 static eth_tx_burst_t
1088 bnxt_transmit_function(__rte_unused struct rte_eth_dev *eth_dev)
1089 {
1090 #ifdef RTE_ARCH_X86
1091 #ifndef RTE_LIBRTE_IEEE1588
1092         /*
1093          * Vector mode transmit can be enabled only if not using scatter rx
1094          * or tx offloads.
1095          */
1096         if (!eth_dev->data->scattered_rx &&
1097             !eth_dev->data->dev_conf.txmode.offloads) {
1098                 PMD_DRV_LOG(INFO, "Using vector mode transmit for port %d\n",
1099                             eth_dev->data->port_id);
1100                 return bnxt_xmit_pkts_vec;
1101         }
1102         PMD_DRV_LOG(INFO, "Vector mode transmit disabled for port %d\n",
1103                     eth_dev->data->port_id);
1104         PMD_DRV_LOG(INFO,
1105                     "Port %d scatter: %d tx offload: %" PRIX64 "\n",
1106                     eth_dev->data->port_id,
1107                     eth_dev->data->scattered_rx,
1108                     eth_dev->data->dev_conf.txmode.offloads);
1109 #endif
1110 #endif
1111         return bnxt_xmit_pkts;
1112 }
1113
1114 static int bnxt_handle_if_change_status(struct bnxt *bp)
1115 {
1116         int rc;
1117
1118         /* Since fw has undergone a reset and lost all contexts,
1119          * set fatal flag to not issue hwrm during cleanup
1120          */
1121         bp->flags |= BNXT_FLAG_FATAL_ERROR;
1122         bnxt_uninit_resources(bp, true);
1123
1124         /* clear fatal flag so that re-init happens */
1125         bp->flags &= ~BNXT_FLAG_FATAL_ERROR;
1126         rc = bnxt_init_resources(bp, true);
1127
1128         bp->flags &= ~BNXT_FLAG_IF_CHANGE_HOT_FW_RESET_DONE;
1129
1130         return rc;
1131 }
1132
1133 static int bnxt_dev_start_op(struct rte_eth_dev *eth_dev)
1134 {
1135         struct bnxt *bp = eth_dev->data->dev_private;
1136         uint64_t rx_offloads = eth_dev->data->dev_conf.rxmode.offloads;
1137         int vlan_mask = 0;
1138         int rc, retry_cnt = BNXT_IF_CHANGE_RETRY_COUNT;
1139
1140         if (!eth_dev->data->nb_tx_queues || !eth_dev->data->nb_rx_queues) {
1141                 PMD_DRV_LOG(ERR, "Queues are not configured yet!\n");
1142                 return -EINVAL;
1143         }
1144
1145         if (bp->rx_cp_nr_rings > RTE_ETHDEV_QUEUE_STAT_CNTRS) {
1146                 PMD_DRV_LOG(ERR,
1147                         "RxQ cnt %d > CONFIG_RTE_ETHDEV_QUEUE_STAT_CNTRS %d\n",
1148                         bp->rx_cp_nr_rings, RTE_ETHDEV_QUEUE_STAT_CNTRS);
1149         }
1150
1151         do {
1152                 rc = bnxt_hwrm_if_change(bp, true);
1153                 if (rc == 0 || rc != -EAGAIN)
1154                         break;
1155
1156                 rte_delay_ms(BNXT_IF_CHANGE_RETRY_INTERVAL);
1157         } while (retry_cnt--);
1158
1159         if (rc)
1160                 return rc;
1161
1162         if (bp->flags & BNXT_FLAG_IF_CHANGE_HOT_FW_RESET_DONE) {
1163                 rc = bnxt_handle_if_change_status(bp);
1164                 if (rc)
1165                         return rc;
1166         }
1167
1168         bnxt_enable_int(bp);
1169
1170         rc = bnxt_init_chip(bp);
1171         if (rc)
1172                 goto error;
1173
1174         eth_dev->data->scattered_rx = bnxt_scattered_rx(eth_dev);
1175         eth_dev->data->dev_started = 1;
1176
1177         bnxt_link_update(eth_dev, 1, ETH_LINK_UP);
1178
1179         if (rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
1180                 vlan_mask |= ETH_VLAN_FILTER_MASK;
1181         if (rx_offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
1182                 vlan_mask |= ETH_VLAN_STRIP_MASK;
1183         rc = bnxt_vlan_offload_set_op(eth_dev, vlan_mask);
1184         if (rc)
1185                 goto error;
1186
1187         eth_dev->rx_pkt_burst = bnxt_receive_function(eth_dev);
1188         eth_dev->tx_pkt_burst = bnxt_transmit_function(eth_dev);
1189
1190         pthread_mutex_lock(&bp->def_cp_lock);
1191         bnxt_schedule_fw_health_check(bp);
1192         pthread_mutex_unlock(&bp->def_cp_lock);
1193
1194         if (bp->truflow)
1195                 bnxt_ulp_init(bp);
1196
1197         return 0;
1198
1199 error:
1200         bnxt_shutdown_nic(bp);
1201         bnxt_free_tx_mbufs(bp);
1202         bnxt_free_rx_mbufs(bp);
1203         bnxt_hwrm_if_change(bp, false);
1204         eth_dev->data->dev_started = 0;
1205         return rc;
1206 }
1207
1208 static int bnxt_dev_set_link_up_op(struct rte_eth_dev *eth_dev)
1209 {
1210         struct bnxt *bp = eth_dev->data->dev_private;
1211         int rc = 0;
1212
1213         if (!bp->link_info.link_up)
1214                 rc = bnxt_set_hwrm_link_config(bp, true);
1215         if (!rc)
1216                 eth_dev->data->dev_link.link_status = 1;
1217
1218         bnxt_print_link_info(eth_dev);
1219         return rc;
1220 }
1221
1222 static int bnxt_dev_set_link_down_op(struct rte_eth_dev *eth_dev)
1223 {
1224         struct bnxt *bp = eth_dev->data->dev_private;
1225
1226         eth_dev->data->dev_link.link_status = 0;
1227         bnxt_set_hwrm_link_config(bp, false);
1228         bp->link_info.link_up = 0;
1229
1230         return 0;
1231 }
1232
1233 /* Unload the driver, release resources */
1234 static void bnxt_dev_stop_op(struct rte_eth_dev *eth_dev)
1235 {
1236         struct bnxt *bp = eth_dev->data->dev_private;
1237         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
1238         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
1239
1240         if (bp->truflow)
1241                 bnxt_ulp_deinit(bp);
1242
1243         eth_dev->data->dev_started = 0;
1244         /* Prevent crashes when queues are still in use */
1245         eth_dev->rx_pkt_burst = &bnxt_dummy_recv_pkts;
1246         eth_dev->tx_pkt_burst = &bnxt_dummy_xmit_pkts;
1247
1248         bnxt_disable_int(bp);
1249
1250         /* disable uio/vfio intr/eventfd mapping */
1251         rte_intr_disable(intr_handle);
1252
1253         bnxt_cancel_fw_health_check(bp);
1254
1255         bnxt_dev_set_link_down_op(eth_dev);
1256
1257         /* Wait for link to be reset and the async notification to process.
1258          * During reset recovery, there is no need to wait and
1259          * VF/NPAR functions do not have privilege to change PHY config.
1260          */
1261         if (!is_bnxt_in_error(bp) && BNXT_SINGLE_PF(bp))
1262                 bnxt_link_update(eth_dev, 1, ETH_LINK_DOWN);
1263
1264         /* Clean queue intr-vector mapping */
1265         rte_intr_efd_disable(intr_handle);
1266         if (intr_handle->intr_vec != NULL) {
1267                 rte_free(intr_handle->intr_vec);
1268                 intr_handle->intr_vec = NULL;
1269         }
1270
1271         bnxt_hwrm_port_clr_stats(bp);
1272         bnxt_free_tx_mbufs(bp);
1273         bnxt_free_rx_mbufs(bp);
1274         /* Process any remaining notifications in default completion queue */
1275         bnxt_int_handler(eth_dev);
1276         bnxt_shutdown_nic(bp);
1277         bnxt_hwrm_if_change(bp, false);
1278
1279         rte_free(bp->mark_table);
1280         bp->mark_table = NULL;
1281
1282         bp->flags &= ~BNXT_FLAG_RX_VECTOR_PKT_MODE;
1283         bp->rx_cosq_cnt = 0;
1284         /* All filters are deleted on a port stop. */
1285         if (BNXT_FLOW_XSTATS_EN(bp))
1286                 bp->flow_stat->flow_count = 0;
1287 }
1288
1289 static void bnxt_dev_close_op(struct rte_eth_dev *eth_dev)
1290 {
1291         struct bnxt *bp = eth_dev->data->dev_private;
1292
1293         /* cancel the recovery handler before remove dev */
1294         rte_eal_alarm_cancel(bnxt_dev_reset_and_resume, (void *)bp);
1295         rte_eal_alarm_cancel(bnxt_dev_recover, (void *)bp);
1296         bnxt_cancel_fc_thread(bp);
1297
1298         if (eth_dev->data->dev_started)
1299                 bnxt_dev_stop_op(eth_dev);
1300
1301         bnxt_uninit_resources(bp, false);
1302
1303         bnxt_free_leds_info(bp);
1304         bnxt_free_cos_queues(bp);
1305
1306         eth_dev->dev_ops = NULL;
1307         eth_dev->rx_pkt_burst = NULL;
1308         eth_dev->tx_pkt_burst = NULL;
1309
1310         rte_memzone_free((const struct rte_memzone *)bp->tx_mem_zone);
1311         bp->tx_mem_zone = NULL;
1312         rte_memzone_free((const struct rte_memzone *)bp->rx_mem_zone);
1313         bp->rx_mem_zone = NULL;
1314
1315         rte_free(bp->pf.vf_info);
1316         bp->pf.vf_info = NULL;
1317
1318         rte_free(bp->grp_info);
1319         bp->grp_info = NULL;
1320 }
1321
1322 static void bnxt_mac_addr_remove_op(struct rte_eth_dev *eth_dev,
1323                                     uint32_t index)
1324 {
1325         struct bnxt *bp = eth_dev->data->dev_private;
1326         uint64_t pool_mask = eth_dev->data->mac_pool_sel[index];
1327         struct bnxt_vnic_info *vnic;
1328         struct bnxt_filter_info *filter, *temp_filter;
1329         uint32_t i;
1330
1331         if (is_bnxt_in_error(bp))
1332                 return;
1333
1334         /*
1335          * Loop through all VNICs from the specified filter flow pools to
1336          * remove the corresponding MAC addr filter
1337          */
1338         for (i = 0; i < bp->nr_vnics; i++) {
1339                 if (!(pool_mask & (1ULL << i)))
1340                         continue;
1341
1342                 vnic = &bp->vnic_info[i];
1343                 filter = STAILQ_FIRST(&vnic->filter);
1344                 while (filter) {
1345                         temp_filter = STAILQ_NEXT(filter, next);
1346                         if (filter->mac_index == index) {
1347                                 STAILQ_REMOVE(&vnic->filter, filter,
1348                                                 bnxt_filter_info, next);
1349                                 bnxt_hwrm_clear_l2_filter(bp, filter);
1350                                 bnxt_free_filter(bp, filter);
1351                         }
1352                         filter = temp_filter;
1353                 }
1354         }
1355 }
1356
1357 static int bnxt_add_mac_filter(struct bnxt *bp, struct bnxt_vnic_info *vnic,
1358                                struct rte_ether_addr *mac_addr, uint32_t index,
1359                                uint32_t pool)
1360 {
1361         struct bnxt_filter_info *filter;
1362         int rc = 0;
1363
1364         /* Attach requested MAC address to the new l2_filter */
1365         STAILQ_FOREACH(filter, &vnic->filter, next) {
1366                 if (filter->mac_index == index) {
1367                         PMD_DRV_LOG(DEBUG,
1368                                     "MAC addr already existed for pool %d\n",
1369                                     pool);
1370                         return 0;
1371                 }
1372         }
1373
1374         filter = bnxt_alloc_filter(bp);
1375         if (!filter) {
1376                 PMD_DRV_LOG(ERR, "L2 filter alloc failed\n");
1377                 return -ENODEV;
1378         }
1379
1380         /* bnxt_alloc_filter copies default MAC to filter->l2_addr. So,
1381          * if the MAC that's been programmed now is a different one, then,
1382          * copy that addr to filter->l2_addr
1383          */
1384         if (mac_addr)
1385                 memcpy(filter->l2_addr, mac_addr, RTE_ETHER_ADDR_LEN);
1386         filter->flags |= HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_OUTERMOST;
1387
1388         rc = bnxt_hwrm_set_l2_filter(bp, vnic->fw_vnic_id, filter);
1389         if (!rc) {
1390                 filter->mac_index = index;
1391                 if (filter->mac_index == 0)
1392                         STAILQ_INSERT_HEAD(&vnic->filter, filter, next);
1393                 else
1394                         STAILQ_INSERT_TAIL(&vnic->filter, filter, next);
1395         } else {
1396                 bnxt_free_filter(bp, filter);
1397         }
1398
1399         return rc;
1400 }
1401
1402 static int bnxt_mac_addr_add_op(struct rte_eth_dev *eth_dev,
1403                                 struct rte_ether_addr *mac_addr,
1404                                 uint32_t index, uint32_t pool)
1405 {
1406         struct bnxt *bp = eth_dev->data->dev_private;
1407         struct bnxt_vnic_info *vnic = &bp->vnic_info[pool];
1408         int rc = 0;
1409
1410         rc = is_bnxt_in_error(bp);
1411         if (rc)
1412                 return rc;
1413
1414         if (BNXT_VF(bp) & !BNXT_VF_IS_TRUSTED(bp)) {
1415                 PMD_DRV_LOG(ERR, "Cannot add MAC address to a VF interface\n");
1416                 return -ENOTSUP;
1417         }
1418
1419         if (!vnic) {
1420                 PMD_DRV_LOG(ERR, "VNIC not found for pool %d!\n", pool);
1421                 return -EINVAL;
1422         }
1423
1424         /* Filter settings will get applied when port is started */
1425         if (!eth_dev->data->dev_started)
1426                 return 0;
1427
1428         rc = bnxt_add_mac_filter(bp, vnic, mac_addr, index, pool);
1429
1430         return rc;
1431 }
1432
1433 int bnxt_link_update(struct rte_eth_dev *eth_dev, int wait_to_complete,
1434                      bool exp_link_status)
1435 {
1436         int rc = 0;
1437         struct bnxt *bp = eth_dev->data->dev_private;
1438         struct rte_eth_link new;
1439         int cnt = exp_link_status ? BNXT_LINK_UP_WAIT_CNT :
1440                   BNXT_LINK_DOWN_WAIT_CNT;
1441
1442         rc = is_bnxt_in_error(bp);
1443         if (rc)
1444                 return rc;
1445
1446         memset(&new, 0, sizeof(new));
1447         do {
1448                 /* Retrieve link info from hardware */
1449                 rc = bnxt_get_hwrm_link_config(bp, &new);
1450                 if (rc) {
1451                         new.link_speed = ETH_LINK_SPEED_100M;
1452                         new.link_duplex = ETH_LINK_FULL_DUPLEX;
1453                         PMD_DRV_LOG(ERR,
1454                                 "Failed to retrieve link rc = 0x%x!\n", rc);
1455                         goto out;
1456                 }
1457
1458                 if (!wait_to_complete || new.link_status == exp_link_status)
1459                         break;
1460
1461                 rte_delay_ms(BNXT_LINK_WAIT_INTERVAL);
1462         } while (cnt--);
1463
1464 out:
1465         /* Timed out or success */
1466         if (new.link_status != eth_dev->data->dev_link.link_status ||
1467         new.link_speed != eth_dev->data->dev_link.link_speed) {
1468                 rte_eth_linkstatus_set(eth_dev, &new);
1469
1470                 _rte_eth_dev_callback_process(eth_dev,
1471                                               RTE_ETH_EVENT_INTR_LSC,
1472                                               NULL);
1473
1474                 bnxt_print_link_info(eth_dev);
1475         }
1476
1477         return rc;
1478 }
1479
1480 static int bnxt_link_update_op(struct rte_eth_dev *eth_dev,
1481                                int wait_to_complete)
1482 {
1483         return bnxt_link_update(eth_dev, wait_to_complete, ETH_LINK_UP);
1484 }
1485
1486 static int bnxt_promiscuous_enable_op(struct rte_eth_dev *eth_dev)
1487 {
1488         struct bnxt *bp = eth_dev->data->dev_private;
1489         struct bnxt_vnic_info *vnic;
1490         uint32_t old_flags;
1491         int rc;
1492
1493         rc = is_bnxt_in_error(bp);
1494         if (rc)
1495                 return rc;
1496
1497         /* Filter settings will get applied when port is started */
1498         if (!eth_dev->data->dev_started)
1499                 return 0;
1500
1501         if (bp->vnic_info == NULL)
1502                 return 0;
1503
1504         vnic = BNXT_GET_DEFAULT_VNIC(bp);
1505
1506         old_flags = vnic->flags;
1507         vnic->flags |= BNXT_VNIC_INFO_PROMISC;
1508         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
1509         if (rc != 0)
1510                 vnic->flags = old_flags;
1511
1512         return rc;
1513 }
1514
1515 static int bnxt_promiscuous_disable_op(struct rte_eth_dev *eth_dev)
1516 {
1517         struct bnxt *bp = eth_dev->data->dev_private;
1518         struct bnxt_vnic_info *vnic;
1519         uint32_t old_flags;
1520         int rc;
1521
1522         rc = is_bnxt_in_error(bp);
1523         if (rc)
1524                 return rc;
1525
1526         /* Filter settings will get applied when port is started */
1527         if (!eth_dev->data->dev_started)
1528                 return 0;
1529
1530         if (bp->vnic_info == NULL)
1531                 return 0;
1532
1533         vnic = BNXT_GET_DEFAULT_VNIC(bp);
1534
1535         old_flags = vnic->flags;
1536         vnic->flags &= ~BNXT_VNIC_INFO_PROMISC;
1537         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
1538         if (rc != 0)
1539                 vnic->flags = old_flags;
1540
1541         return rc;
1542 }
1543
1544 static int bnxt_allmulticast_enable_op(struct rte_eth_dev *eth_dev)
1545 {
1546         struct bnxt *bp = eth_dev->data->dev_private;
1547         struct bnxt_vnic_info *vnic;
1548         uint32_t old_flags;
1549         int rc;
1550
1551         rc = is_bnxt_in_error(bp);
1552         if (rc)
1553                 return rc;
1554
1555         /* Filter settings will get applied when port is started */
1556         if (!eth_dev->data->dev_started)
1557                 return 0;
1558
1559         if (bp->vnic_info == NULL)
1560                 return 0;
1561
1562         vnic = BNXT_GET_DEFAULT_VNIC(bp);
1563
1564         old_flags = vnic->flags;
1565         vnic->flags |= BNXT_VNIC_INFO_ALLMULTI;
1566         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
1567         if (rc != 0)
1568                 vnic->flags = old_flags;
1569
1570         return rc;
1571 }
1572
1573 static int bnxt_allmulticast_disable_op(struct rte_eth_dev *eth_dev)
1574 {
1575         struct bnxt *bp = eth_dev->data->dev_private;
1576         struct bnxt_vnic_info *vnic;
1577         uint32_t old_flags;
1578         int rc;
1579
1580         rc = is_bnxt_in_error(bp);
1581         if (rc)
1582                 return rc;
1583
1584         /* Filter settings will get applied when port is started */
1585         if (!eth_dev->data->dev_started)
1586                 return 0;
1587
1588         if (bp->vnic_info == NULL)
1589                 return 0;
1590
1591         vnic = BNXT_GET_DEFAULT_VNIC(bp);
1592
1593         old_flags = vnic->flags;
1594         vnic->flags &= ~BNXT_VNIC_INFO_ALLMULTI;
1595         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
1596         if (rc != 0)
1597                 vnic->flags = old_flags;
1598
1599         return rc;
1600 }
1601
1602 /* Return bnxt_rx_queue pointer corresponding to a given rxq. */
1603 static struct bnxt_rx_queue *bnxt_qid_to_rxq(struct bnxt *bp, uint16_t qid)
1604 {
1605         if (qid >= bp->rx_nr_rings)
1606                 return NULL;
1607
1608         return bp->eth_dev->data->rx_queues[qid];
1609 }
1610
1611 /* Return rxq corresponding to a given rss table ring/group ID. */
1612 static uint16_t bnxt_rss_to_qid(struct bnxt *bp, uint16_t fwr)
1613 {
1614         struct bnxt_rx_queue *rxq;
1615         unsigned int i;
1616
1617         if (!BNXT_HAS_RING_GRPS(bp)) {
1618                 for (i = 0; i < bp->rx_nr_rings; i++) {
1619                         rxq = bp->eth_dev->data->rx_queues[i];
1620                         if (rxq->rx_ring->rx_ring_struct->fw_ring_id == fwr)
1621                                 return rxq->index;
1622                 }
1623         } else {
1624                 for (i = 0; i < bp->rx_nr_rings; i++) {
1625                         if (bp->grp_info[i].fw_grp_id == fwr)
1626                                 return i;
1627                 }
1628         }
1629
1630         return INVALID_HW_RING_ID;
1631 }
1632
1633 static int bnxt_reta_update_op(struct rte_eth_dev *eth_dev,
1634                             struct rte_eth_rss_reta_entry64 *reta_conf,
1635                             uint16_t reta_size)
1636 {
1637         struct bnxt *bp = eth_dev->data->dev_private;
1638         struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
1639         struct bnxt_vnic_info *vnic = BNXT_GET_DEFAULT_VNIC(bp);
1640         uint16_t tbl_size = bnxt_rss_hash_tbl_size(bp);
1641         uint16_t idx, sft;
1642         int i, rc;
1643
1644         rc = is_bnxt_in_error(bp);
1645         if (rc)
1646                 return rc;
1647
1648         if (!vnic->rss_table)
1649                 return -EINVAL;
1650
1651         if (!(dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
1652                 return -EINVAL;
1653
1654         if (reta_size != tbl_size) {
1655                 PMD_DRV_LOG(ERR, "The configured hash table lookup size "
1656                         "(%d) must equal the size supported by the hardware "
1657                         "(%d)\n", reta_size, tbl_size);
1658                 return -EINVAL;
1659         }
1660
1661         for (i = 0; i < reta_size; i++) {
1662                 struct bnxt_rx_queue *rxq;
1663
1664                 idx = i / RTE_RETA_GROUP_SIZE;
1665                 sft = i % RTE_RETA_GROUP_SIZE;
1666
1667                 if (!(reta_conf[idx].mask & (1ULL << sft)))
1668                         continue;
1669
1670                 rxq = bnxt_qid_to_rxq(bp, reta_conf[idx].reta[sft]);
1671                 if (!rxq) {
1672                         PMD_DRV_LOG(ERR, "Invalid ring in reta_conf.\n");
1673                         return -EINVAL;
1674                 }
1675
1676                 if (BNXT_CHIP_THOR(bp)) {
1677                         vnic->rss_table[i * 2] =
1678                                 rxq->rx_ring->rx_ring_struct->fw_ring_id;
1679                         vnic->rss_table[i * 2 + 1] =
1680                                 rxq->cp_ring->cp_ring_struct->fw_ring_id;
1681                 } else {
1682                         vnic->rss_table[i] =
1683                             vnic->fw_grp_ids[reta_conf[idx].reta[sft]];
1684                 }
1685         }
1686
1687         bnxt_hwrm_vnic_rss_cfg(bp, vnic);
1688         return 0;
1689 }
1690
1691 static int bnxt_reta_query_op(struct rte_eth_dev *eth_dev,
1692                               struct rte_eth_rss_reta_entry64 *reta_conf,
1693                               uint16_t reta_size)
1694 {
1695         struct bnxt *bp = eth_dev->data->dev_private;
1696         struct bnxt_vnic_info *vnic = BNXT_GET_DEFAULT_VNIC(bp);
1697         uint16_t tbl_size = bnxt_rss_hash_tbl_size(bp);
1698         uint16_t idx, sft, i;
1699         int rc;
1700
1701         rc = is_bnxt_in_error(bp);
1702         if (rc)
1703                 return rc;
1704
1705         /* Retrieve from the default VNIC */
1706         if (!vnic)
1707                 return -EINVAL;
1708         if (!vnic->rss_table)
1709                 return -EINVAL;
1710
1711         if (reta_size != tbl_size) {
1712                 PMD_DRV_LOG(ERR, "The configured hash table lookup size "
1713                         "(%d) must equal the size supported by the hardware "
1714                         "(%d)\n", reta_size, tbl_size);
1715                 return -EINVAL;
1716         }
1717
1718         for (idx = 0, i = 0; i < reta_size; i++) {
1719                 idx = i / RTE_RETA_GROUP_SIZE;
1720                 sft = i % RTE_RETA_GROUP_SIZE;
1721
1722                 if (reta_conf[idx].mask & (1ULL << sft)) {
1723                         uint16_t qid;
1724
1725                         if (BNXT_CHIP_THOR(bp))
1726                                 qid = bnxt_rss_to_qid(bp,
1727                                                       vnic->rss_table[i * 2]);
1728                         else
1729                                 qid = bnxt_rss_to_qid(bp, vnic->rss_table[i]);
1730
1731                         if (qid == INVALID_HW_RING_ID) {
1732                                 PMD_DRV_LOG(ERR, "Inv. entry in rss table.\n");
1733                                 return -EINVAL;
1734                         }
1735                         reta_conf[idx].reta[sft] = qid;
1736                 }
1737         }
1738
1739         return 0;
1740 }
1741
1742 static int bnxt_rss_hash_update_op(struct rte_eth_dev *eth_dev,
1743                                    struct rte_eth_rss_conf *rss_conf)
1744 {
1745         struct bnxt *bp = eth_dev->data->dev_private;
1746         struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
1747         struct bnxt_vnic_info *vnic;
1748         int rc;
1749
1750         rc = is_bnxt_in_error(bp);
1751         if (rc)
1752                 return rc;
1753
1754         /*
1755          * If RSS enablement were different than dev_configure,
1756          * then return -EINVAL
1757          */
1758         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) {
1759                 if (!rss_conf->rss_hf)
1760                         PMD_DRV_LOG(ERR, "Hash type NONE\n");
1761         } else {
1762                 if (rss_conf->rss_hf & BNXT_ETH_RSS_SUPPORT)
1763                         return -EINVAL;
1764         }
1765
1766         bp->flags |= BNXT_FLAG_UPDATE_HASH;
1767         memcpy(&bp->rss_conf, rss_conf, sizeof(*rss_conf));
1768
1769         /* Update the default RSS VNIC(s) */
1770         vnic = BNXT_GET_DEFAULT_VNIC(bp);
1771         vnic->hash_type = bnxt_rte_to_hwrm_hash_types(rss_conf->rss_hf);
1772
1773         /*
1774          * If hashkey is not specified, use the previously configured
1775          * hashkey
1776          */
1777         if (!rss_conf->rss_key)
1778                 goto rss_config;
1779
1780         if (rss_conf->rss_key_len != HW_HASH_KEY_SIZE) {
1781                 PMD_DRV_LOG(ERR,
1782                             "Invalid hashkey length, should be 16 bytes\n");
1783                 return -EINVAL;
1784         }
1785         memcpy(vnic->rss_hash_key, rss_conf->rss_key, rss_conf->rss_key_len);
1786
1787 rss_config:
1788         bnxt_hwrm_vnic_rss_cfg(bp, vnic);
1789         return 0;
1790 }
1791
1792 static int bnxt_rss_hash_conf_get_op(struct rte_eth_dev *eth_dev,
1793                                      struct rte_eth_rss_conf *rss_conf)
1794 {
1795         struct bnxt *bp = eth_dev->data->dev_private;
1796         struct bnxt_vnic_info *vnic = BNXT_GET_DEFAULT_VNIC(bp);
1797         int len, rc;
1798         uint32_t hash_types;
1799
1800         rc = is_bnxt_in_error(bp);
1801         if (rc)
1802                 return rc;
1803
1804         /* RSS configuration is the same for all VNICs */
1805         if (vnic && vnic->rss_hash_key) {
1806                 if (rss_conf->rss_key) {
1807                         len = rss_conf->rss_key_len <= HW_HASH_KEY_SIZE ?
1808                               rss_conf->rss_key_len : HW_HASH_KEY_SIZE;
1809                         memcpy(rss_conf->rss_key, vnic->rss_hash_key, len);
1810                 }
1811
1812                 hash_types = vnic->hash_type;
1813                 rss_conf->rss_hf = 0;
1814                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV4) {
1815                         rss_conf->rss_hf |= ETH_RSS_IPV4;
1816                         hash_types &= ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV4;
1817                 }
1818                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV4) {
1819                         rss_conf->rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
1820                         hash_types &=
1821                                 ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV4;
1822                 }
1823                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV4) {
1824                         rss_conf->rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
1825                         hash_types &=
1826                                 ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV4;
1827                 }
1828                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV6) {
1829                         rss_conf->rss_hf |= ETH_RSS_IPV6;
1830                         hash_types &= ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV6;
1831                 }
1832                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV6) {
1833                         rss_conf->rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
1834                         hash_types &=
1835                                 ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV6;
1836                 }
1837                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV6) {
1838                         rss_conf->rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
1839                         hash_types &=
1840                                 ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV6;
1841                 }
1842                 if (hash_types) {
1843                         PMD_DRV_LOG(ERR,
1844                                 "Unknown RSS config from firmware (%08x), RSS disabled",
1845                                 vnic->hash_type);
1846                         return -ENOTSUP;
1847                 }
1848         } else {
1849                 rss_conf->rss_hf = 0;
1850         }
1851         return 0;
1852 }
1853
1854 static int bnxt_flow_ctrl_get_op(struct rte_eth_dev *dev,
1855                                struct rte_eth_fc_conf *fc_conf)
1856 {
1857         struct bnxt *bp = dev->data->dev_private;
1858         struct rte_eth_link link_info;
1859         int rc;
1860
1861         rc = is_bnxt_in_error(bp);
1862         if (rc)
1863                 return rc;
1864
1865         rc = bnxt_get_hwrm_link_config(bp, &link_info);
1866         if (rc)
1867                 return rc;
1868
1869         memset(fc_conf, 0, sizeof(*fc_conf));
1870         if (bp->link_info.auto_pause)
1871                 fc_conf->autoneg = 1;
1872         switch (bp->link_info.pause) {
1873         case 0:
1874                 fc_conf->mode = RTE_FC_NONE;
1875                 break;
1876         case HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX:
1877                 fc_conf->mode = RTE_FC_TX_PAUSE;
1878                 break;
1879         case HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX:
1880                 fc_conf->mode = RTE_FC_RX_PAUSE;
1881                 break;
1882         case (HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX |
1883                         HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX):
1884                 fc_conf->mode = RTE_FC_FULL;
1885                 break;
1886         }
1887         return 0;
1888 }
1889
1890 static int bnxt_flow_ctrl_set_op(struct rte_eth_dev *dev,
1891                                struct rte_eth_fc_conf *fc_conf)
1892 {
1893         struct bnxt *bp = dev->data->dev_private;
1894         int rc;
1895
1896         rc = is_bnxt_in_error(bp);
1897         if (rc)
1898                 return rc;
1899
1900         if (!BNXT_SINGLE_PF(bp) || BNXT_VF(bp)) {
1901                 PMD_DRV_LOG(ERR, "Flow Control Settings cannot be modified\n");
1902                 return -ENOTSUP;
1903         }
1904
1905         switch (fc_conf->mode) {
1906         case RTE_FC_NONE:
1907                 bp->link_info.auto_pause = 0;
1908                 bp->link_info.force_pause = 0;
1909                 break;
1910         case RTE_FC_RX_PAUSE:
1911                 if (fc_conf->autoneg) {
1912                         bp->link_info.auto_pause =
1913                                         HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_RX;
1914                         bp->link_info.force_pause = 0;
1915                 } else {
1916                         bp->link_info.auto_pause = 0;
1917                         bp->link_info.force_pause =
1918                                         HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_RX;
1919                 }
1920                 break;
1921         case RTE_FC_TX_PAUSE:
1922                 if (fc_conf->autoneg) {
1923                         bp->link_info.auto_pause =
1924                                         HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_TX;
1925                         bp->link_info.force_pause = 0;
1926                 } else {
1927                         bp->link_info.auto_pause = 0;
1928                         bp->link_info.force_pause =
1929                                         HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_TX;
1930                 }
1931                 break;
1932         case RTE_FC_FULL:
1933                 if (fc_conf->autoneg) {
1934                         bp->link_info.auto_pause =
1935                                         HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_TX |
1936                                         HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_RX;
1937                         bp->link_info.force_pause = 0;
1938                 } else {
1939                         bp->link_info.auto_pause = 0;
1940                         bp->link_info.force_pause =
1941                                         HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_TX |
1942                                         HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_RX;
1943                 }
1944                 break;
1945         }
1946         return bnxt_set_hwrm_link_config(bp, true);
1947 }
1948
1949 /* Add UDP tunneling port */
1950 static int
1951 bnxt_udp_tunnel_port_add_op(struct rte_eth_dev *eth_dev,
1952                          struct rte_eth_udp_tunnel *udp_tunnel)
1953 {
1954         struct bnxt *bp = eth_dev->data->dev_private;
1955         uint16_t tunnel_type = 0;
1956         int rc = 0;
1957
1958         rc = is_bnxt_in_error(bp);
1959         if (rc)
1960                 return rc;
1961
1962         switch (udp_tunnel->prot_type) {
1963         case RTE_TUNNEL_TYPE_VXLAN:
1964                 if (bp->vxlan_port_cnt) {
1965                         PMD_DRV_LOG(ERR, "Tunnel Port %d already programmed\n",
1966                                 udp_tunnel->udp_port);
1967                         if (bp->vxlan_port != udp_tunnel->udp_port) {
1968                                 PMD_DRV_LOG(ERR, "Only one port allowed\n");
1969                                 return -ENOSPC;
1970                         }
1971                         bp->vxlan_port_cnt++;
1972                         return 0;
1973                 }
1974                 tunnel_type =
1975                         HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_VXLAN;
1976                 bp->vxlan_port_cnt++;
1977                 break;
1978         case RTE_TUNNEL_TYPE_GENEVE:
1979                 if (bp->geneve_port_cnt) {
1980                         PMD_DRV_LOG(ERR, "Tunnel Port %d already programmed\n",
1981                                 udp_tunnel->udp_port);
1982                         if (bp->geneve_port != udp_tunnel->udp_port) {
1983                                 PMD_DRV_LOG(ERR, "Only one port allowed\n");
1984                                 return -ENOSPC;
1985                         }
1986                         bp->geneve_port_cnt++;
1987                         return 0;
1988                 }
1989                 tunnel_type =
1990                         HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_GENEVE;
1991                 bp->geneve_port_cnt++;
1992                 break;
1993         default:
1994                 PMD_DRV_LOG(ERR, "Tunnel type is not supported\n");
1995                 return -ENOTSUP;
1996         }
1997         rc = bnxt_hwrm_tunnel_dst_port_alloc(bp, udp_tunnel->udp_port,
1998                                              tunnel_type);
1999         return rc;
2000 }
2001
2002 static int
2003 bnxt_udp_tunnel_port_del_op(struct rte_eth_dev *eth_dev,
2004                          struct rte_eth_udp_tunnel *udp_tunnel)
2005 {
2006         struct bnxt *bp = eth_dev->data->dev_private;
2007         uint16_t tunnel_type = 0;
2008         uint16_t port = 0;
2009         int rc = 0;
2010
2011         rc = is_bnxt_in_error(bp);
2012         if (rc)
2013                 return rc;
2014
2015         switch (udp_tunnel->prot_type) {
2016         case RTE_TUNNEL_TYPE_VXLAN:
2017                 if (!bp->vxlan_port_cnt) {
2018                         PMD_DRV_LOG(ERR, "No Tunnel port configured yet\n");
2019                         return -EINVAL;
2020                 }
2021                 if (bp->vxlan_port != udp_tunnel->udp_port) {
2022                         PMD_DRV_LOG(ERR, "Req Port: %d. Configured port: %d\n",
2023                                 udp_tunnel->udp_port, bp->vxlan_port);
2024                         return -EINVAL;
2025                 }
2026                 if (--bp->vxlan_port_cnt)
2027                         return 0;
2028
2029                 tunnel_type =
2030                         HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_VXLAN;
2031                 port = bp->vxlan_fw_dst_port_id;
2032                 break;
2033         case RTE_TUNNEL_TYPE_GENEVE:
2034                 if (!bp->geneve_port_cnt) {
2035                         PMD_DRV_LOG(ERR, "No Tunnel port configured yet\n");
2036                         return -EINVAL;
2037                 }
2038                 if (bp->geneve_port != udp_tunnel->udp_port) {
2039                         PMD_DRV_LOG(ERR, "Req Port: %d. Configured port: %d\n",
2040                                 udp_tunnel->udp_port, bp->geneve_port);
2041                         return -EINVAL;
2042                 }
2043                 if (--bp->geneve_port_cnt)
2044                         return 0;
2045
2046                 tunnel_type =
2047                         HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_GENEVE;
2048                 port = bp->geneve_fw_dst_port_id;
2049                 break;
2050         default:
2051                 PMD_DRV_LOG(ERR, "Tunnel type is not supported\n");
2052                 return -ENOTSUP;
2053         }
2054
2055         rc = bnxt_hwrm_tunnel_dst_port_free(bp, port, tunnel_type);
2056         if (!rc) {
2057                 if (tunnel_type ==
2058                     HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_VXLAN)
2059                         bp->vxlan_port = 0;
2060                 if (tunnel_type ==
2061                     HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_GENEVE)
2062                         bp->geneve_port = 0;
2063         }
2064         return rc;
2065 }
2066
2067 static int bnxt_del_vlan_filter(struct bnxt *bp, uint16_t vlan_id)
2068 {
2069         struct bnxt_filter_info *filter;
2070         struct bnxt_vnic_info *vnic;
2071         int rc = 0;
2072         uint32_t chk = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN;
2073
2074         vnic = BNXT_GET_DEFAULT_VNIC(bp);
2075         filter = STAILQ_FIRST(&vnic->filter);
2076         while (filter) {
2077                 /* Search for this matching MAC+VLAN filter */
2078                 if (bnxt_vlan_filter_exists(bp, filter, chk, vlan_id)) {
2079                         /* Delete the filter */
2080                         rc = bnxt_hwrm_clear_l2_filter(bp, filter);
2081                         if (rc)
2082                                 return rc;
2083                         STAILQ_REMOVE(&vnic->filter, filter,
2084                                       bnxt_filter_info, next);
2085                         bnxt_free_filter(bp, filter);
2086                         PMD_DRV_LOG(INFO,
2087                                     "Deleted vlan filter for %d\n",
2088                                     vlan_id);
2089                         return 0;
2090                 }
2091                 filter = STAILQ_NEXT(filter, next);
2092         }
2093         return -ENOENT;
2094 }
2095
2096 static int bnxt_add_vlan_filter(struct bnxt *bp, uint16_t vlan_id)
2097 {
2098         struct bnxt_filter_info *filter;
2099         struct bnxt_vnic_info *vnic;
2100         int rc = 0;
2101         uint32_t en = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN |
2102                 HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN_MASK;
2103         uint32_t chk = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN;
2104
2105         /* Implementation notes on the use of VNIC in this command:
2106          *
2107          * By default, these filters belong to default vnic for the function.
2108          * Once these filters are set up, only destination VNIC can be modified.
2109          * If the destination VNIC is not specified in this command,
2110          * then the HWRM shall only create an l2 context id.
2111          */
2112
2113         vnic = BNXT_GET_DEFAULT_VNIC(bp);
2114         filter = STAILQ_FIRST(&vnic->filter);
2115         /* Check if the VLAN has already been added */
2116         while (filter) {
2117                 if (bnxt_vlan_filter_exists(bp, filter, chk, vlan_id))
2118                         return -EEXIST;
2119
2120                 filter = STAILQ_NEXT(filter, next);
2121         }
2122
2123         /* No match found. Alloc a fresh filter and issue the L2_FILTER_ALLOC
2124          * command to create MAC+VLAN filter with the right flags, enables set.
2125          */
2126         filter = bnxt_alloc_filter(bp);
2127         if (!filter) {
2128                 PMD_DRV_LOG(ERR,
2129                             "MAC/VLAN filter alloc failed\n");
2130                 return -ENOMEM;
2131         }
2132         /* MAC + VLAN ID filter */
2133         /* If l2_ivlan == 0 and l2_ivlan_mask != 0, only
2134          * untagged packets are received
2135          *
2136          * If l2_ivlan != 0 and l2_ivlan_mask != 0, untagged
2137          * packets and only the programmed vlan's packets are received
2138          */
2139         filter->l2_ivlan = vlan_id;
2140         filter->l2_ivlan_mask = 0x0FFF;
2141         filter->enables |= en;
2142         filter->flags |= HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_OUTERMOST;
2143
2144         rc = bnxt_hwrm_set_l2_filter(bp, vnic->fw_vnic_id, filter);
2145         if (rc) {
2146                 /* Free the newly allocated filter as we were
2147                  * not able to create the filter in hardware.
2148                  */
2149                 bnxt_free_filter(bp, filter);
2150                 return rc;
2151         }
2152
2153         filter->mac_index = 0;
2154         /* Add this new filter to the list */
2155         if (vlan_id == 0)
2156                 STAILQ_INSERT_HEAD(&vnic->filter, filter, next);
2157         else
2158                 STAILQ_INSERT_TAIL(&vnic->filter, filter, next);
2159
2160         PMD_DRV_LOG(INFO,
2161                     "Added Vlan filter for %d\n", vlan_id);
2162         return rc;
2163 }
2164
2165 static int bnxt_vlan_filter_set_op(struct rte_eth_dev *eth_dev,
2166                 uint16_t vlan_id, int on)
2167 {
2168         struct bnxt *bp = eth_dev->data->dev_private;
2169         int rc;
2170
2171         rc = is_bnxt_in_error(bp);
2172         if (rc)
2173                 return rc;
2174
2175         if (!eth_dev->data->dev_started) {
2176                 PMD_DRV_LOG(ERR, "port must be started before setting vlan\n");
2177                 return -EINVAL;
2178         }
2179
2180         /* These operations apply to ALL existing MAC/VLAN filters */
2181         if (on)
2182                 return bnxt_add_vlan_filter(bp, vlan_id);
2183         else
2184                 return bnxt_del_vlan_filter(bp, vlan_id);
2185 }
2186
2187 static int bnxt_del_dflt_mac_filter(struct bnxt *bp,
2188                                     struct bnxt_vnic_info *vnic)
2189 {
2190         struct bnxt_filter_info *filter;
2191         int rc;
2192
2193         filter = STAILQ_FIRST(&vnic->filter);
2194         while (filter) {
2195                 if (filter->mac_index == 0 &&
2196                     !memcmp(filter->l2_addr, bp->mac_addr,
2197                             RTE_ETHER_ADDR_LEN)) {
2198                         rc = bnxt_hwrm_clear_l2_filter(bp, filter);
2199                         if (!rc) {
2200                                 STAILQ_REMOVE(&vnic->filter, filter,
2201                                               bnxt_filter_info, next);
2202                                 bnxt_free_filter(bp, filter);
2203                         }
2204                         return rc;
2205                 }
2206                 filter = STAILQ_NEXT(filter, next);
2207         }
2208         return 0;
2209 }
2210
2211 static int
2212 bnxt_config_vlan_hw_filter(struct bnxt *bp, uint64_t rx_offloads)
2213 {
2214         struct bnxt_vnic_info *vnic;
2215         unsigned int i;
2216         int rc;
2217
2218         vnic = BNXT_GET_DEFAULT_VNIC(bp);
2219         if (!(rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER)) {
2220                 /* Remove any VLAN filters programmed */
2221                 for (i = 0; i < RTE_ETHER_MAX_VLAN_ID; i++)
2222                         bnxt_del_vlan_filter(bp, i);
2223
2224                 rc = bnxt_add_mac_filter(bp, vnic, NULL, 0, 0);
2225                 if (rc)
2226                         return rc;
2227         } else {
2228                 /* Default filter will allow packets that match the
2229                  * dest mac. So, it has to be deleted, otherwise, we
2230                  * will endup receiving vlan packets for which the
2231                  * filter is not programmed, when hw-vlan-filter
2232                  * configuration is ON
2233                  */
2234                 bnxt_del_dflt_mac_filter(bp, vnic);
2235                 /* This filter will allow only untagged packets */
2236                 bnxt_add_vlan_filter(bp, 0);
2237         }
2238         PMD_DRV_LOG(DEBUG, "VLAN Filtering: %d\n",
2239                     !!(rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER));
2240
2241         return 0;
2242 }
2243
2244 static int bnxt_free_one_vnic(struct bnxt *bp, uint16_t vnic_id)
2245 {
2246         struct bnxt_vnic_info *vnic = &bp->vnic_info[vnic_id];
2247         unsigned int i;
2248         int rc;
2249
2250         /* Destroy vnic filters and vnic */
2251         if (bp->eth_dev->data->dev_conf.rxmode.offloads &
2252             DEV_RX_OFFLOAD_VLAN_FILTER) {
2253                 for (i = 0; i < RTE_ETHER_MAX_VLAN_ID; i++)
2254                         bnxt_del_vlan_filter(bp, i);
2255         }
2256         bnxt_del_dflt_mac_filter(bp, vnic);
2257
2258         rc = bnxt_hwrm_vnic_free(bp, vnic);
2259         if (rc)
2260                 return rc;
2261
2262         rte_free(vnic->fw_grp_ids);
2263         vnic->fw_grp_ids = NULL;
2264
2265         vnic->rx_queue_cnt = 0;
2266
2267         return 0;
2268 }
2269
2270 static int
2271 bnxt_config_vlan_hw_stripping(struct bnxt *bp, uint64_t rx_offloads)
2272 {
2273         struct bnxt_vnic_info *vnic = BNXT_GET_DEFAULT_VNIC(bp);
2274         int rc;
2275
2276         /* Destroy, recreate and reconfigure the default vnic */
2277         rc = bnxt_free_one_vnic(bp, 0);
2278         if (rc)
2279                 return rc;
2280
2281         /* default vnic 0 */
2282         rc = bnxt_setup_one_vnic(bp, 0);
2283         if (rc)
2284                 return rc;
2285
2286         if (bp->eth_dev->data->dev_conf.rxmode.offloads &
2287             DEV_RX_OFFLOAD_VLAN_FILTER) {
2288                 rc = bnxt_add_vlan_filter(bp, 0);
2289                 if (rc)
2290                         return rc;
2291                 rc = bnxt_restore_vlan_filters(bp);
2292                 if (rc)
2293                         return rc;
2294         } else {
2295                 rc = bnxt_add_mac_filter(bp, vnic, NULL, 0, 0);
2296                 if (rc)
2297                         return rc;
2298         }
2299
2300         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
2301         if (rc)
2302                 return rc;
2303
2304         PMD_DRV_LOG(DEBUG, "VLAN Strip Offload: %d\n",
2305                     !!(rx_offloads & DEV_RX_OFFLOAD_VLAN_STRIP));
2306
2307         return rc;
2308 }
2309
2310 static int
2311 bnxt_vlan_offload_set_op(struct rte_eth_dev *dev, int mask)
2312 {
2313         uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads;
2314         struct bnxt *bp = dev->data->dev_private;
2315         int rc;
2316
2317         rc = is_bnxt_in_error(bp);
2318         if (rc)
2319                 return rc;
2320
2321         /* Filter settings will get applied when port is started */
2322         if (!dev->data->dev_started)
2323                 return 0;
2324
2325         if (mask & ETH_VLAN_FILTER_MASK) {
2326                 /* Enable or disable VLAN filtering */
2327                 rc = bnxt_config_vlan_hw_filter(bp, rx_offloads);
2328                 if (rc)
2329                         return rc;
2330         }
2331
2332         if (mask & ETH_VLAN_STRIP_MASK) {
2333                 /* Enable or disable VLAN stripping */
2334                 rc = bnxt_config_vlan_hw_stripping(bp, rx_offloads);
2335                 if (rc)
2336                         return rc;
2337         }
2338
2339         if (mask & ETH_VLAN_EXTEND_MASK) {
2340                 if (rx_offloads & DEV_RX_OFFLOAD_VLAN_EXTEND)
2341                         PMD_DRV_LOG(DEBUG, "Extend VLAN supported\n");
2342                 else
2343                         PMD_DRV_LOG(INFO, "Extend VLAN unsupported\n");
2344         }
2345
2346         return 0;
2347 }
2348
2349 static int
2350 bnxt_vlan_tpid_set_op(struct rte_eth_dev *dev, enum rte_vlan_type vlan_type,
2351                       uint16_t tpid)
2352 {
2353         struct bnxt *bp = dev->data->dev_private;
2354         int qinq = dev->data->dev_conf.rxmode.offloads &
2355                    DEV_RX_OFFLOAD_VLAN_EXTEND;
2356
2357         if (vlan_type != ETH_VLAN_TYPE_INNER &&
2358             vlan_type != ETH_VLAN_TYPE_OUTER) {
2359                 PMD_DRV_LOG(ERR,
2360                             "Unsupported vlan type.");
2361                 return -EINVAL;
2362         }
2363         if (!qinq) {
2364                 PMD_DRV_LOG(ERR,
2365                             "QinQ not enabled. Needs to be ON as we can "
2366                             "accelerate only outer vlan\n");
2367                 return -EINVAL;
2368         }
2369
2370         if (vlan_type == ETH_VLAN_TYPE_OUTER) {
2371                 switch (tpid) {
2372                 case RTE_ETHER_TYPE_QINQ:
2373                         bp->outer_tpid_bd =
2374                                 TX_BD_LONG_CFA_META_VLAN_TPID_TPID88A8;
2375                                 break;
2376                 case RTE_ETHER_TYPE_VLAN:
2377                         bp->outer_tpid_bd =
2378                                 TX_BD_LONG_CFA_META_VLAN_TPID_TPID8100;
2379                                 break;
2380                 case 0x9100:
2381                         bp->outer_tpid_bd =
2382                                 TX_BD_LONG_CFA_META_VLAN_TPID_TPID9100;
2383                                 break;
2384                 case 0x9200:
2385                         bp->outer_tpid_bd =
2386                                 TX_BD_LONG_CFA_META_VLAN_TPID_TPID9200;
2387                                 break;
2388                 case 0x9300:
2389                         bp->outer_tpid_bd =
2390                                  TX_BD_LONG_CFA_META_VLAN_TPID_TPID9300;
2391                                 break;
2392                 default:
2393                         PMD_DRV_LOG(ERR, "Invalid TPID: %x\n", tpid);
2394                         return -EINVAL;
2395                 }
2396                 bp->outer_tpid_bd |= tpid;
2397                 PMD_DRV_LOG(INFO, "outer_tpid_bd = %x\n", bp->outer_tpid_bd);
2398         } else if (vlan_type == ETH_VLAN_TYPE_INNER) {
2399                 PMD_DRV_LOG(ERR,
2400                             "Can accelerate only outer vlan in QinQ\n");
2401                 return -EINVAL;
2402         }
2403
2404         return 0;
2405 }
2406
2407 static int
2408 bnxt_set_default_mac_addr_op(struct rte_eth_dev *dev,
2409                              struct rte_ether_addr *addr)
2410 {
2411         struct bnxt *bp = dev->data->dev_private;
2412         /* Default Filter is tied to VNIC 0 */
2413         struct bnxt_vnic_info *vnic = BNXT_GET_DEFAULT_VNIC(bp);
2414         int rc;
2415
2416         rc = is_bnxt_in_error(bp);
2417         if (rc)
2418                 return rc;
2419
2420         if (BNXT_VF(bp) && !BNXT_VF_IS_TRUSTED(bp))
2421                 return -EPERM;
2422
2423         if (rte_is_zero_ether_addr(addr))
2424                 return -EINVAL;
2425
2426         /* Filter settings will get applied when port is started */
2427         if (!dev->data->dev_started)
2428                 return 0;
2429
2430         /* Check if the requested MAC is already added */
2431         if (memcmp(addr, bp->mac_addr, RTE_ETHER_ADDR_LEN) == 0)
2432                 return 0;
2433
2434         /* Destroy filter and re-create it */
2435         bnxt_del_dflt_mac_filter(bp, vnic);
2436
2437         memcpy(bp->mac_addr, addr, RTE_ETHER_ADDR_LEN);
2438         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_VLAN_FILTER) {
2439                 /* This filter will allow only untagged packets */
2440                 rc = bnxt_add_vlan_filter(bp, 0);
2441         } else {
2442                 rc = bnxt_add_mac_filter(bp, vnic, addr, 0, 0);
2443         }
2444
2445         PMD_DRV_LOG(DEBUG, "Set MAC addr\n");
2446         return rc;
2447 }
2448
2449 static int
2450 bnxt_dev_set_mc_addr_list_op(struct rte_eth_dev *eth_dev,
2451                           struct rte_ether_addr *mc_addr_set,
2452                           uint32_t nb_mc_addr)
2453 {
2454         struct bnxt *bp = eth_dev->data->dev_private;
2455         char *mc_addr_list = (char *)mc_addr_set;
2456         struct bnxt_vnic_info *vnic;
2457         uint32_t off = 0, i = 0;
2458         int rc;
2459
2460         rc = is_bnxt_in_error(bp);
2461         if (rc)
2462                 return rc;
2463
2464         vnic = BNXT_GET_DEFAULT_VNIC(bp);
2465
2466         if (nb_mc_addr > BNXT_MAX_MC_ADDRS) {
2467                 vnic->flags |= BNXT_VNIC_INFO_ALLMULTI;
2468                 goto allmulti;
2469         }
2470
2471         /* TODO Check for Duplicate mcast addresses */
2472         vnic->flags &= ~BNXT_VNIC_INFO_ALLMULTI;
2473         for (i = 0; i < nb_mc_addr; i++) {
2474                 memcpy(vnic->mc_list + off, &mc_addr_list[i],
2475                         RTE_ETHER_ADDR_LEN);
2476                 off += RTE_ETHER_ADDR_LEN;
2477         }
2478
2479         vnic->mc_addr_cnt = i;
2480         if (vnic->mc_addr_cnt)
2481                 vnic->flags |= BNXT_VNIC_INFO_MCAST;
2482         else
2483                 vnic->flags &= ~BNXT_VNIC_INFO_MCAST;
2484
2485 allmulti:
2486         return bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
2487 }
2488
2489 static int
2490 bnxt_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size)
2491 {
2492         struct bnxt *bp = dev->data->dev_private;
2493         uint8_t fw_major = (bp->fw_ver >> 24) & 0xff;
2494         uint8_t fw_minor = (bp->fw_ver >> 16) & 0xff;
2495         uint8_t fw_updt = (bp->fw_ver >> 8) & 0xff;
2496         uint8_t fw_rsvd = bp->fw_ver & 0xff;
2497         int ret;
2498
2499         ret = snprintf(fw_version, fw_size, "%d.%d.%d.%d",
2500                         fw_major, fw_minor, fw_updt, fw_rsvd);
2501
2502         ret += 1; /* add the size of '\0' */
2503         if (fw_size < (uint32_t)ret)
2504                 return ret;
2505         else
2506                 return 0;
2507 }
2508
2509 static void
2510 bnxt_rxq_info_get_op(struct rte_eth_dev *dev, uint16_t queue_id,
2511         struct rte_eth_rxq_info *qinfo)
2512 {
2513         struct bnxt *bp = dev->data->dev_private;
2514         struct bnxt_rx_queue *rxq;
2515
2516         if (is_bnxt_in_error(bp))
2517                 return;
2518
2519         rxq = dev->data->rx_queues[queue_id];
2520
2521         qinfo->mp = rxq->mb_pool;
2522         qinfo->scattered_rx = dev->data->scattered_rx;
2523         qinfo->nb_desc = rxq->nb_rx_desc;
2524
2525         qinfo->conf.rx_free_thresh = rxq->rx_free_thresh;
2526         qinfo->conf.rx_drop_en = 0;
2527         qinfo->conf.rx_deferred_start = rxq->rx_deferred_start;
2528 }
2529
2530 static void
2531 bnxt_txq_info_get_op(struct rte_eth_dev *dev, uint16_t queue_id,
2532         struct rte_eth_txq_info *qinfo)
2533 {
2534         struct bnxt *bp = dev->data->dev_private;
2535         struct bnxt_tx_queue *txq;
2536
2537         if (is_bnxt_in_error(bp))
2538                 return;
2539
2540         txq = dev->data->tx_queues[queue_id];
2541
2542         qinfo->nb_desc = txq->nb_tx_desc;
2543
2544         qinfo->conf.tx_thresh.pthresh = txq->pthresh;
2545         qinfo->conf.tx_thresh.hthresh = txq->hthresh;
2546         qinfo->conf.tx_thresh.wthresh = txq->wthresh;
2547
2548         qinfo->conf.tx_free_thresh = txq->tx_free_thresh;
2549         qinfo->conf.tx_rs_thresh = 0;
2550         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
2551 }
2552
2553 int bnxt_mtu_set_op(struct rte_eth_dev *eth_dev, uint16_t new_mtu)
2554 {
2555         struct bnxt *bp = eth_dev->data->dev_private;
2556         uint32_t new_pkt_size;
2557         uint32_t rc = 0;
2558         uint32_t i;
2559
2560         rc = is_bnxt_in_error(bp);
2561         if (rc)
2562                 return rc;
2563
2564         /* Exit if receive queues are not configured yet */
2565         if (!eth_dev->data->nb_rx_queues)
2566                 return rc;
2567
2568         new_pkt_size = new_mtu + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN +
2569                        VLAN_TAG_SIZE * BNXT_NUM_VLANS;
2570
2571 #ifdef RTE_ARCH_X86
2572         /*
2573          * If vector-mode tx/rx is active, disallow any MTU change that would
2574          * require scattered receive support.
2575          */
2576         if (eth_dev->data->dev_started &&
2577             (eth_dev->rx_pkt_burst == bnxt_recv_pkts_vec ||
2578              eth_dev->tx_pkt_burst == bnxt_xmit_pkts_vec) &&
2579             (new_pkt_size >
2580              eth_dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)) {
2581                 PMD_DRV_LOG(ERR,
2582                             "MTU change would require scattered rx support. ");
2583                 PMD_DRV_LOG(ERR, "Stop port before changing MTU.\n");
2584                 return -EINVAL;
2585         }
2586 #endif
2587
2588         if (new_mtu > RTE_ETHER_MTU) {
2589                 bp->flags |= BNXT_FLAG_JUMBO;
2590                 bp->eth_dev->data->dev_conf.rxmode.offloads |=
2591                         DEV_RX_OFFLOAD_JUMBO_FRAME;
2592         } else {
2593                 bp->eth_dev->data->dev_conf.rxmode.offloads &=
2594                         ~DEV_RX_OFFLOAD_JUMBO_FRAME;
2595                 bp->flags &= ~BNXT_FLAG_JUMBO;
2596         }
2597
2598         /* Is there a change in mtu setting? */
2599         if (eth_dev->data->dev_conf.rxmode.max_rx_pkt_len == new_pkt_size)
2600                 return rc;
2601
2602         for (i = 0; i < bp->nr_vnics; i++) {
2603                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
2604                 uint16_t size = 0;
2605
2606                 vnic->mru = BNXT_VNIC_MRU(new_mtu);
2607                 rc = bnxt_hwrm_vnic_cfg(bp, vnic);
2608                 if (rc)
2609                         break;
2610
2611                 size = rte_pktmbuf_data_room_size(bp->rx_queues[0]->mb_pool);
2612                 size -= RTE_PKTMBUF_HEADROOM;
2613
2614                 if (size < new_mtu) {
2615                         rc = bnxt_hwrm_vnic_plcmode_cfg(bp, vnic);
2616                         if (rc)
2617                                 return rc;
2618                 }
2619         }
2620
2621         if (!rc)
2622                 eth_dev->data->dev_conf.rxmode.max_rx_pkt_len = new_pkt_size;
2623
2624         PMD_DRV_LOG(INFO, "New MTU is %d\n", new_mtu);
2625
2626         return rc;
2627 }
2628
2629 static int
2630 bnxt_vlan_pvid_set_op(struct rte_eth_dev *dev, uint16_t pvid, int on)
2631 {
2632         struct bnxt *bp = dev->data->dev_private;
2633         uint16_t vlan = bp->vlan;
2634         int rc;
2635
2636         rc = is_bnxt_in_error(bp);
2637         if (rc)
2638                 return rc;
2639
2640         if (!BNXT_SINGLE_PF(bp) || BNXT_VF(bp)) {
2641                 PMD_DRV_LOG(ERR,
2642                         "PVID cannot be modified for this function\n");
2643                 return -ENOTSUP;
2644         }
2645         bp->vlan = on ? pvid : 0;
2646
2647         rc = bnxt_hwrm_set_default_vlan(bp, 0, 0);
2648         if (rc)
2649                 bp->vlan = vlan;
2650         return rc;
2651 }
2652
2653 static int
2654 bnxt_dev_led_on_op(struct rte_eth_dev *dev)
2655 {
2656         struct bnxt *bp = dev->data->dev_private;
2657         int rc;
2658
2659         rc = is_bnxt_in_error(bp);
2660         if (rc)
2661                 return rc;
2662
2663         return bnxt_hwrm_port_led_cfg(bp, true);
2664 }
2665
2666 static int
2667 bnxt_dev_led_off_op(struct rte_eth_dev *dev)
2668 {
2669         struct bnxt *bp = dev->data->dev_private;
2670         int rc;
2671
2672         rc = is_bnxt_in_error(bp);
2673         if (rc)
2674                 return rc;
2675
2676         return bnxt_hwrm_port_led_cfg(bp, false);
2677 }
2678
2679 static uint32_t
2680 bnxt_rx_queue_count_op(struct rte_eth_dev *dev, uint16_t rx_queue_id)
2681 {
2682         struct bnxt *bp = (struct bnxt *)dev->data->dev_private;
2683         uint32_t desc = 0, raw_cons = 0, cons;
2684         struct bnxt_cp_ring_info *cpr;
2685         struct bnxt_rx_queue *rxq;
2686         struct rx_pkt_cmpl *rxcmp;
2687         int rc;
2688
2689         rc = is_bnxt_in_error(bp);
2690         if (rc)
2691                 return rc;
2692
2693         rxq = dev->data->rx_queues[rx_queue_id];
2694         cpr = rxq->cp_ring;
2695         raw_cons = cpr->cp_raw_cons;
2696
2697         while (1) {
2698                 cons = RING_CMP(cpr->cp_ring_struct, raw_cons);
2699                 rte_prefetch0(&cpr->cp_desc_ring[cons]);
2700                 rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons];
2701
2702                 if (!CMP_VALID(rxcmp, raw_cons, cpr->cp_ring_struct)) {
2703                         break;
2704                 } else {
2705                         raw_cons++;
2706                         desc++;
2707                 }
2708         }
2709
2710         return desc;
2711 }
2712
2713 static int
2714 bnxt_rx_descriptor_status_op(void *rx_queue, uint16_t offset)
2715 {
2716         struct bnxt_rx_queue *rxq = (struct bnxt_rx_queue *)rx_queue;
2717         struct bnxt_rx_ring_info *rxr;
2718         struct bnxt_cp_ring_info *cpr;
2719         struct bnxt_sw_rx_bd *rx_buf;
2720         struct rx_pkt_cmpl *rxcmp;
2721         uint32_t cons, cp_cons;
2722         int rc;
2723
2724         if (!rxq)
2725                 return -EINVAL;
2726
2727         rc = is_bnxt_in_error(rxq->bp);
2728         if (rc)
2729                 return rc;
2730
2731         cpr = rxq->cp_ring;
2732         rxr = rxq->rx_ring;
2733
2734         if (offset >= rxq->nb_rx_desc)
2735                 return -EINVAL;
2736
2737         cons = RING_CMP(cpr->cp_ring_struct, offset);
2738         cp_cons = cpr->cp_raw_cons;
2739         rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons];
2740
2741         if (cons > cp_cons) {
2742                 if (CMPL_VALID(rxcmp, cpr->valid))
2743                         return RTE_ETH_RX_DESC_DONE;
2744         } else {
2745                 if (CMPL_VALID(rxcmp, !cpr->valid))
2746                         return RTE_ETH_RX_DESC_DONE;
2747         }
2748         rx_buf = &rxr->rx_buf_ring[cons];
2749         if (rx_buf->mbuf == NULL)
2750                 return RTE_ETH_RX_DESC_UNAVAIL;
2751
2752
2753         return RTE_ETH_RX_DESC_AVAIL;
2754 }
2755
2756 static int
2757 bnxt_tx_descriptor_status_op(void *tx_queue, uint16_t offset)
2758 {
2759         struct bnxt_tx_queue *txq = (struct bnxt_tx_queue *)tx_queue;
2760         struct bnxt_tx_ring_info *txr;
2761         struct bnxt_cp_ring_info *cpr;
2762         struct bnxt_sw_tx_bd *tx_buf;
2763         struct tx_pkt_cmpl *txcmp;
2764         uint32_t cons, cp_cons;
2765         int rc;
2766
2767         if (!txq)
2768                 return -EINVAL;
2769
2770         rc = is_bnxt_in_error(txq->bp);
2771         if (rc)
2772                 return rc;
2773
2774         cpr = txq->cp_ring;
2775         txr = txq->tx_ring;
2776
2777         if (offset >= txq->nb_tx_desc)
2778                 return -EINVAL;
2779
2780         cons = RING_CMP(cpr->cp_ring_struct, offset);
2781         txcmp = (struct tx_pkt_cmpl *)&cpr->cp_desc_ring[cons];
2782         cp_cons = cpr->cp_raw_cons;
2783
2784         if (cons > cp_cons) {
2785                 if (CMPL_VALID(txcmp, cpr->valid))
2786                         return RTE_ETH_TX_DESC_UNAVAIL;
2787         } else {
2788                 if (CMPL_VALID(txcmp, !cpr->valid))
2789                         return RTE_ETH_TX_DESC_UNAVAIL;
2790         }
2791         tx_buf = &txr->tx_buf_ring[cons];
2792         if (tx_buf->mbuf == NULL)
2793                 return RTE_ETH_TX_DESC_DONE;
2794
2795         return RTE_ETH_TX_DESC_FULL;
2796 }
2797
2798 static struct bnxt_filter_info *
2799 bnxt_match_and_validate_ether_filter(struct bnxt *bp,
2800                                 struct rte_eth_ethertype_filter *efilter,
2801                                 struct bnxt_vnic_info *vnic0,
2802                                 struct bnxt_vnic_info *vnic,
2803                                 int *ret)
2804 {
2805         struct bnxt_filter_info *mfilter = NULL;
2806         int match = 0;
2807         *ret = 0;
2808
2809         if (efilter->ether_type == RTE_ETHER_TYPE_IPV4 ||
2810                 efilter->ether_type == RTE_ETHER_TYPE_IPV6) {
2811                 PMD_DRV_LOG(ERR, "invalid ether_type(0x%04x) in"
2812                         " ethertype filter.", efilter->ether_type);
2813                 *ret = -EINVAL;
2814                 goto exit;
2815         }
2816         if (efilter->queue >= bp->rx_nr_rings) {
2817                 PMD_DRV_LOG(ERR, "Invalid queue %d\n", efilter->queue);
2818                 *ret = -EINVAL;
2819                 goto exit;
2820         }
2821
2822         vnic0 = BNXT_GET_DEFAULT_VNIC(bp);
2823         vnic = &bp->vnic_info[efilter->queue];
2824         if (vnic == NULL) {
2825                 PMD_DRV_LOG(ERR, "Invalid queue %d\n", efilter->queue);
2826                 *ret = -EINVAL;
2827                 goto exit;
2828         }
2829
2830         if (efilter->flags & RTE_ETHTYPE_FLAGS_DROP) {
2831                 STAILQ_FOREACH(mfilter, &vnic0->filter, next) {
2832                         if ((!memcmp(efilter->mac_addr.addr_bytes,
2833                                      mfilter->l2_addr, RTE_ETHER_ADDR_LEN) &&
2834                              mfilter->flags ==
2835                              HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_DROP &&
2836                              mfilter->ethertype == efilter->ether_type)) {
2837                                 match = 1;
2838                                 break;
2839                         }
2840                 }
2841         } else {
2842                 STAILQ_FOREACH(mfilter, &vnic->filter, next)
2843                         if ((!memcmp(efilter->mac_addr.addr_bytes,
2844                                      mfilter->l2_addr, RTE_ETHER_ADDR_LEN) &&
2845                              mfilter->ethertype == efilter->ether_type &&
2846                              mfilter->flags ==
2847                              HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH_RX)) {
2848                                 match = 1;
2849                                 break;
2850                         }
2851         }
2852
2853         if (match)
2854                 *ret = -EEXIST;
2855
2856 exit:
2857         return mfilter;
2858 }
2859
2860 static int
2861 bnxt_ethertype_filter(struct rte_eth_dev *dev,
2862                         enum rte_filter_op filter_op,
2863                         void *arg)
2864 {
2865         struct bnxt *bp = dev->data->dev_private;
2866         struct rte_eth_ethertype_filter *efilter =
2867                         (struct rte_eth_ethertype_filter *)arg;
2868         struct bnxt_filter_info *bfilter, *filter1;
2869         struct bnxt_vnic_info *vnic, *vnic0;
2870         int ret;
2871
2872         if (filter_op == RTE_ETH_FILTER_NOP)
2873                 return 0;
2874
2875         if (arg == NULL) {
2876                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
2877                             filter_op);
2878                 return -EINVAL;
2879         }
2880
2881         vnic0 = BNXT_GET_DEFAULT_VNIC(bp);
2882         vnic = &bp->vnic_info[efilter->queue];
2883
2884         switch (filter_op) {
2885         case RTE_ETH_FILTER_ADD:
2886                 bnxt_match_and_validate_ether_filter(bp, efilter,
2887                                                         vnic0, vnic, &ret);
2888                 if (ret < 0)
2889                         return ret;
2890
2891                 bfilter = bnxt_get_unused_filter(bp);
2892                 if (bfilter == NULL) {
2893                         PMD_DRV_LOG(ERR,
2894                                 "Not enough resources for a new filter.\n");
2895                         return -ENOMEM;
2896                 }
2897                 bfilter->filter_type = HWRM_CFA_NTUPLE_FILTER;
2898                 memcpy(bfilter->l2_addr, efilter->mac_addr.addr_bytes,
2899                        RTE_ETHER_ADDR_LEN);
2900                 memcpy(bfilter->dst_macaddr, efilter->mac_addr.addr_bytes,
2901                        RTE_ETHER_ADDR_LEN);
2902                 bfilter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_MACADDR;
2903                 bfilter->ethertype = efilter->ether_type;
2904                 bfilter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE;
2905
2906                 filter1 = bnxt_get_l2_filter(bp, bfilter, vnic0);
2907                 if (filter1 == NULL) {
2908                         ret = -EINVAL;
2909                         goto cleanup;
2910                 }
2911                 bfilter->enables |=
2912                         HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_L2_FILTER_ID;
2913                 bfilter->fw_l2_filter_id = filter1->fw_l2_filter_id;
2914
2915                 bfilter->dst_id = vnic->fw_vnic_id;
2916
2917                 if (efilter->flags & RTE_ETHTYPE_FLAGS_DROP) {
2918                         bfilter->flags =
2919                                 HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_DROP;
2920                 }
2921
2922                 ret = bnxt_hwrm_set_ntuple_filter(bp, bfilter->dst_id, bfilter);
2923                 if (ret)
2924                         goto cleanup;
2925                 STAILQ_INSERT_TAIL(&vnic->filter, bfilter, next);
2926                 break;
2927         case RTE_ETH_FILTER_DELETE:
2928                 filter1 = bnxt_match_and_validate_ether_filter(bp, efilter,
2929                                                         vnic0, vnic, &ret);
2930                 if (ret == -EEXIST) {
2931                         ret = bnxt_hwrm_clear_ntuple_filter(bp, filter1);
2932
2933                         STAILQ_REMOVE(&vnic->filter, filter1, bnxt_filter_info,
2934                                       next);
2935                         bnxt_free_filter(bp, filter1);
2936                 } else if (ret == 0) {
2937                         PMD_DRV_LOG(ERR, "No matching filter found\n");
2938                 }
2939                 break;
2940         default:
2941                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
2942                 ret = -EINVAL;
2943                 goto error;
2944         }
2945         return ret;
2946 cleanup:
2947         bnxt_free_filter(bp, bfilter);
2948 error:
2949         return ret;
2950 }
2951
2952 static inline int
2953 parse_ntuple_filter(struct bnxt *bp,
2954                     struct rte_eth_ntuple_filter *nfilter,
2955                     struct bnxt_filter_info *bfilter)
2956 {
2957         uint32_t en = 0;
2958
2959         if (nfilter->queue >= bp->rx_nr_rings) {
2960                 PMD_DRV_LOG(ERR, "Invalid queue %d\n", nfilter->queue);
2961                 return -EINVAL;
2962         }
2963
2964         switch (nfilter->dst_port_mask) {
2965         case UINT16_MAX:
2966                 bfilter->dst_port_mask = -1;
2967                 bfilter->dst_port = nfilter->dst_port;
2968                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT |
2969                         NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK;
2970                 break;
2971         default:
2972                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
2973                 return -EINVAL;
2974         }
2975
2976         bfilter->ip_addr_type = NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV4;
2977         en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO;
2978
2979         switch (nfilter->proto_mask) {
2980         case UINT8_MAX:
2981                 if (nfilter->proto == 17) /* IPPROTO_UDP */
2982                         bfilter->ip_protocol = 17;
2983                 else if (nfilter->proto == 6) /* IPPROTO_TCP */
2984                         bfilter->ip_protocol = 6;
2985                 else
2986                         return -EINVAL;
2987                 en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO;
2988                 break;
2989         default:
2990                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
2991                 return -EINVAL;
2992         }
2993
2994         switch (nfilter->dst_ip_mask) {
2995         case UINT32_MAX:
2996                 bfilter->dst_ipaddr_mask[0] = -1;
2997                 bfilter->dst_ipaddr[0] = nfilter->dst_ip;
2998                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR |
2999                         NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK;
3000                 break;
3001         default:
3002                 PMD_DRV_LOG(ERR, "invalid dst_ip mask.");
3003                 return -EINVAL;
3004         }
3005
3006         switch (nfilter->src_ip_mask) {
3007         case UINT32_MAX:
3008                 bfilter->src_ipaddr_mask[0] = -1;
3009                 bfilter->src_ipaddr[0] = nfilter->src_ip;
3010                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR |
3011                         NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK;
3012                 break;
3013         default:
3014                 PMD_DRV_LOG(ERR, "invalid src_ip mask.");
3015                 return -EINVAL;
3016         }
3017
3018         switch (nfilter->src_port_mask) {
3019         case UINT16_MAX:
3020                 bfilter->src_port_mask = -1;
3021                 bfilter->src_port = nfilter->src_port;
3022                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT |
3023                         NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK;
3024                 break;
3025         default:
3026                 PMD_DRV_LOG(ERR, "invalid src_port mask.");
3027                 return -EINVAL;
3028         }
3029
3030         bfilter->enables = en;
3031         return 0;
3032 }
3033
3034 static struct bnxt_filter_info*
3035 bnxt_match_ntuple_filter(struct bnxt *bp,
3036                          struct bnxt_filter_info *bfilter,
3037                          struct bnxt_vnic_info **mvnic)
3038 {
3039         struct bnxt_filter_info *mfilter = NULL;
3040         int i;
3041
3042         for (i = bp->nr_vnics - 1; i >= 0; i--) {
3043                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3044                 STAILQ_FOREACH(mfilter, &vnic->filter, next) {
3045                         if (bfilter->src_ipaddr[0] == mfilter->src_ipaddr[0] &&
3046                             bfilter->src_ipaddr_mask[0] ==
3047                             mfilter->src_ipaddr_mask[0] &&
3048                             bfilter->src_port == mfilter->src_port &&
3049                             bfilter->src_port_mask == mfilter->src_port_mask &&
3050                             bfilter->dst_ipaddr[0] == mfilter->dst_ipaddr[0] &&
3051                             bfilter->dst_ipaddr_mask[0] ==
3052                             mfilter->dst_ipaddr_mask[0] &&
3053                             bfilter->dst_port == mfilter->dst_port &&
3054                             bfilter->dst_port_mask == mfilter->dst_port_mask &&
3055                             bfilter->flags == mfilter->flags &&
3056                             bfilter->enables == mfilter->enables) {
3057                                 if (mvnic)
3058                                         *mvnic = vnic;
3059                                 return mfilter;
3060                         }
3061                 }
3062         }
3063         return NULL;
3064 }
3065
3066 static int
3067 bnxt_cfg_ntuple_filter(struct bnxt *bp,
3068                        struct rte_eth_ntuple_filter *nfilter,
3069                        enum rte_filter_op filter_op)
3070 {
3071         struct bnxt_filter_info *bfilter, *mfilter, *filter1;
3072         struct bnxt_vnic_info *vnic, *vnic0, *mvnic;
3073         int ret;
3074
3075         if (nfilter->flags != RTE_5TUPLE_FLAGS) {
3076                 PMD_DRV_LOG(ERR, "only 5tuple is supported.");
3077                 return -EINVAL;
3078         }
3079
3080         if (nfilter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) {
3081                 PMD_DRV_LOG(ERR, "Ntuple filter: TCP flags not supported\n");
3082                 return -EINVAL;
3083         }
3084
3085         bfilter = bnxt_get_unused_filter(bp);
3086         if (bfilter == NULL) {
3087                 PMD_DRV_LOG(ERR,
3088                         "Not enough resources for a new filter.\n");
3089                 return -ENOMEM;
3090         }
3091         ret = parse_ntuple_filter(bp, nfilter, bfilter);
3092         if (ret < 0)
3093                 goto free_filter;
3094
3095         vnic = &bp->vnic_info[nfilter->queue];
3096         vnic0 = BNXT_GET_DEFAULT_VNIC(bp);
3097         filter1 = STAILQ_FIRST(&vnic0->filter);
3098         if (filter1 == NULL) {
3099                 ret = -EINVAL;
3100                 goto free_filter;
3101         }
3102
3103         bfilter->dst_id = vnic->fw_vnic_id;
3104         bfilter->fw_l2_filter_id = filter1->fw_l2_filter_id;
3105         bfilter->enables |=
3106                 HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_L2_FILTER_ID;
3107         bfilter->ethertype = 0x800;
3108         bfilter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE;
3109
3110         mfilter = bnxt_match_ntuple_filter(bp, bfilter, &mvnic);
3111
3112         if (mfilter != NULL && filter_op == RTE_ETH_FILTER_ADD &&
3113             bfilter->dst_id == mfilter->dst_id) {
3114                 PMD_DRV_LOG(ERR, "filter exists.\n");
3115                 ret = -EEXIST;
3116                 goto free_filter;
3117         } else if (mfilter != NULL && filter_op == RTE_ETH_FILTER_ADD &&
3118                    bfilter->dst_id != mfilter->dst_id) {
3119                 mfilter->dst_id = vnic->fw_vnic_id;
3120                 ret = bnxt_hwrm_set_ntuple_filter(bp, mfilter->dst_id, mfilter);
3121                 STAILQ_REMOVE(&mvnic->filter, mfilter, bnxt_filter_info, next);
3122                 STAILQ_INSERT_TAIL(&vnic->filter, mfilter, next);
3123                 PMD_DRV_LOG(ERR, "filter with matching pattern exists.\n");
3124                 PMD_DRV_LOG(ERR, " Updated it to the new destination queue\n");
3125                 goto free_filter;
3126         }
3127         if (mfilter == NULL && filter_op == RTE_ETH_FILTER_DELETE) {
3128                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
3129                 ret = -ENOENT;
3130                 goto free_filter;
3131         }
3132
3133         if (filter_op == RTE_ETH_FILTER_ADD) {
3134                 bfilter->filter_type = HWRM_CFA_NTUPLE_FILTER;
3135                 ret = bnxt_hwrm_set_ntuple_filter(bp, bfilter->dst_id, bfilter);
3136                 if (ret)
3137                         goto free_filter;
3138                 STAILQ_INSERT_TAIL(&vnic->filter, bfilter, next);
3139         } else {
3140                 if (mfilter == NULL) {
3141                         /* This should not happen. But for Coverity! */
3142                         ret = -ENOENT;
3143                         goto free_filter;
3144                 }
3145                 ret = bnxt_hwrm_clear_ntuple_filter(bp, mfilter);
3146
3147                 STAILQ_REMOVE(&vnic->filter, mfilter, bnxt_filter_info, next);
3148                 bnxt_free_filter(bp, mfilter);
3149                 bnxt_free_filter(bp, bfilter);
3150         }
3151
3152         return 0;
3153 free_filter:
3154         bnxt_free_filter(bp, bfilter);
3155         return ret;
3156 }
3157
3158 static int
3159 bnxt_ntuple_filter(struct rte_eth_dev *dev,
3160                         enum rte_filter_op filter_op,
3161                         void *arg)
3162 {
3163         struct bnxt *bp = dev->data->dev_private;
3164         int ret;
3165
3166         if (filter_op == RTE_ETH_FILTER_NOP)
3167                 return 0;
3168
3169         if (arg == NULL) {
3170                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
3171                             filter_op);
3172                 return -EINVAL;
3173         }
3174
3175         switch (filter_op) {
3176         case RTE_ETH_FILTER_ADD:
3177                 ret = bnxt_cfg_ntuple_filter(bp,
3178                         (struct rte_eth_ntuple_filter *)arg,
3179                         filter_op);
3180                 break;
3181         case RTE_ETH_FILTER_DELETE:
3182                 ret = bnxt_cfg_ntuple_filter(bp,
3183                         (struct rte_eth_ntuple_filter *)arg,
3184                         filter_op);
3185                 break;
3186         default:
3187                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
3188                 ret = -EINVAL;
3189                 break;
3190         }
3191         return ret;
3192 }
3193
3194 static int
3195 bnxt_parse_fdir_filter(struct bnxt *bp,
3196                        struct rte_eth_fdir_filter *fdir,
3197                        struct bnxt_filter_info *filter)
3198 {
3199         enum rte_fdir_mode fdir_mode =
3200                 bp->eth_dev->data->dev_conf.fdir_conf.mode;
3201         struct bnxt_vnic_info *vnic0, *vnic;
3202         struct bnxt_filter_info *filter1;
3203         uint32_t en = 0;
3204         int i;
3205
3206         if (fdir_mode == RTE_FDIR_MODE_PERFECT_TUNNEL)
3207                 return -EINVAL;
3208
3209         filter->l2_ovlan = fdir->input.flow_ext.vlan_tci;
3210         en |= EM_FLOW_ALLOC_INPUT_EN_OVLAN_VID;
3211
3212         switch (fdir->input.flow_type) {
3213         case RTE_ETH_FLOW_IPV4:
3214         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
3215                 /* FALLTHROUGH */
3216                 filter->src_ipaddr[0] = fdir->input.flow.ip4_flow.src_ip;
3217                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR;
3218                 filter->dst_ipaddr[0] = fdir->input.flow.ip4_flow.dst_ip;
3219                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR;
3220                 filter->ip_protocol = fdir->input.flow.ip4_flow.proto;
3221                 en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO;
3222                 filter->ip_addr_type =
3223                         NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV4;
3224                 filter->src_ipaddr_mask[0] = 0xffffffff;
3225                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK;
3226                 filter->dst_ipaddr_mask[0] = 0xffffffff;
3227                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK;
3228                 filter->ethertype = 0x800;
3229                 filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE;
3230                 break;
3231         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
3232                 filter->src_port = fdir->input.flow.tcp4_flow.src_port;
3233                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT;
3234                 filter->dst_port = fdir->input.flow.tcp4_flow.dst_port;
3235                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT;
3236                 filter->dst_port_mask = 0xffff;
3237                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK;
3238                 filter->src_port_mask = 0xffff;
3239                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK;
3240                 filter->src_ipaddr[0] = fdir->input.flow.tcp4_flow.ip.src_ip;
3241                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR;
3242                 filter->dst_ipaddr[0] = fdir->input.flow.tcp4_flow.ip.dst_ip;
3243                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR;
3244                 filter->ip_protocol = 6;
3245                 en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO;
3246                 filter->ip_addr_type =
3247                         NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV4;
3248                 filter->src_ipaddr_mask[0] = 0xffffffff;
3249                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK;
3250                 filter->dst_ipaddr_mask[0] = 0xffffffff;
3251                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK;
3252                 filter->ethertype = 0x800;
3253                 filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE;
3254                 break;
3255         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
3256                 filter->src_port = fdir->input.flow.udp4_flow.src_port;
3257                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT;
3258                 filter->dst_port = fdir->input.flow.udp4_flow.dst_port;
3259                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT;
3260                 filter->dst_port_mask = 0xffff;
3261                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK;
3262                 filter->src_port_mask = 0xffff;
3263                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK;
3264                 filter->src_ipaddr[0] = fdir->input.flow.udp4_flow.ip.src_ip;
3265                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR;
3266                 filter->dst_ipaddr[0] = fdir->input.flow.udp4_flow.ip.dst_ip;
3267                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR;
3268                 filter->ip_protocol = 17;
3269                 en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO;
3270                 filter->ip_addr_type =
3271                         NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV4;
3272                 filter->src_ipaddr_mask[0] = 0xffffffff;
3273                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK;
3274                 filter->dst_ipaddr_mask[0] = 0xffffffff;
3275                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK;
3276                 filter->ethertype = 0x800;
3277                 filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE;
3278                 break;
3279         case RTE_ETH_FLOW_IPV6:
3280         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
3281                 /* FALLTHROUGH */
3282                 filter->ip_addr_type =
3283                         NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV6;
3284                 filter->ip_protocol = fdir->input.flow.ipv6_flow.proto;
3285                 en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO;
3286                 rte_memcpy(filter->src_ipaddr,
3287                            fdir->input.flow.ipv6_flow.src_ip, 16);
3288                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR;
3289                 rte_memcpy(filter->dst_ipaddr,
3290                            fdir->input.flow.ipv6_flow.dst_ip, 16);
3291                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR;
3292                 memset(filter->dst_ipaddr_mask, 0xff, 16);
3293                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK;
3294                 memset(filter->src_ipaddr_mask, 0xff, 16);
3295                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK;
3296                 filter->ethertype = 0x86dd;
3297                 filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE;
3298                 break;
3299         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
3300                 filter->src_port = fdir->input.flow.tcp6_flow.src_port;
3301                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT;
3302                 filter->dst_port = fdir->input.flow.tcp6_flow.dst_port;
3303                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT;
3304                 filter->dst_port_mask = 0xffff;
3305                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK;
3306                 filter->src_port_mask = 0xffff;
3307                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK;
3308                 filter->ip_addr_type =
3309                         NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV6;
3310                 filter->ip_protocol = fdir->input.flow.tcp6_flow.ip.proto;
3311                 en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO;
3312                 rte_memcpy(filter->src_ipaddr,
3313                            fdir->input.flow.tcp6_flow.ip.src_ip, 16);
3314                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR;
3315                 rte_memcpy(filter->dst_ipaddr,
3316                            fdir->input.flow.tcp6_flow.ip.dst_ip, 16);
3317                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR;
3318                 memset(filter->dst_ipaddr_mask, 0xff, 16);
3319                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK;
3320                 memset(filter->src_ipaddr_mask, 0xff, 16);
3321                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK;
3322                 filter->ethertype = 0x86dd;
3323                 filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE;
3324                 break;
3325         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
3326                 filter->src_port = fdir->input.flow.udp6_flow.src_port;
3327                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT;
3328                 filter->dst_port = fdir->input.flow.udp6_flow.dst_port;
3329                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT;
3330                 filter->dst_port_mask = 0xffff;
3331                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK;
3332                 filter->src_port_mask = 0xffff;
3333                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK;
3334                 filter->ip_addr_type =
3335                         NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV6;
3336                 filter->ip_protocol = fdir->input.flow.udp6_flow.ip.proto;
3337                 en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO;
3338                 rte_memcpy(filter->src_ipaddr,
3339                            fdir->input.flow.udp6_flow.ip.src_ip, 16);
3340                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR;
3341                 rte_memcpy(filter->dst_ipaddr,
3342                            fdir->input.flow.udp6_flow.ip.dst_ip, 16);
3343                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR;
3344                 memset(filter->dst_ipaddr_mask, 0xff, 16);
3345                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK;
3346                 memset(filter->src_ipaddr_mask, 0xff, 16);
3347                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK;
3348                 filter->ethertype = 0x86dd;
3349                 filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE;
3350                 break;
3351         case RTE_ETH_FLOW_L2_PAYLOAD:
3352                 filter->ethertype = fdir->input.flow.l2_flow.ether_type;
3353                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE;
3354                 break;
3355         case RTE_ETH_FLOW_VXLAN:
3356                 if (fdir->action.behavior == RTE_ETH_FDIR_REJECT)
3357                         return -EINVAL;
3358                 filter->vni = fdir->input.flow.tunnel_flow.tunnel_id;
3359                 filter->tunnel_type =
3360                         CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN;
3361                 en |= HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_TUNNEL_TYPE;
3362                 break;
3363         case RTE_ETH_FLOW_NVGRE:
3364                 if (fdir->action.behavior == RTE_ETH_FDIR_REJECT)
3365                         return -EINVAL;
3366                 filter->vni = fdir->input.flow.tunnel_flow.tunnel_id;
3367                 filter->tunnel_type =
3368                         CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE;
3369                 en |= HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_TUNNEL_TYPE;
3370                 break;
3371         case RTE_ETH_FLOW_UNKNOWN:
3372         case RTE_ETH_FLOW_RAW:
3373         case RTE_ETH_FLOW_FRAG_IPV4:
3374         case RTE_ETH_FLOW_NONFRAG_IPV4_SCTP:
3375         case RTE_ETH_FLOW_FRAG_IPV6:
3376         case RTE_ETH_FLOW_NONFRAG_IPV6_SCTP:
3377         case RTE_ETH_FLOW_IPV6_EX:
3378         case RTE_ETH_FLOW_IPV6_TCP_EX:
3379         case RTE_ETH_FLOW_IPV6_UDP_EX:
3380         case RTE_ETH_FLOW_GENEVE:
3381                 /* FALLTHROUGH */
3382         default:
3383                 return -EINVAL;
3384         }
3385
3386         vnic0 = BNXT_GET_DEFAULT_VNIC(bp);
3387         vnic = &bp->vnic_info[fdir->action.rx_queue];
3388         if (vnic == NULL) {
3389                 PMD_DRV_LOG(ERR, "Invalid queue %d\n", fdir->action.rx_queue);
3390                 return -EINVAL;
3391         }
3392
3393         if (fdir_mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
3394                 rte_memcpy(filter->dst_macaddr,
3395                         fdir->input.flow.mac_vlan_flow.mac_addr.addr_bytes, 6);
3396                         en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_MACADDR;
3397         }
3398
3399         if (fdir->action.behavior == RTE_ETH_FDIR_REJECT) {
3400                 filter->flags = HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_DROP;
3401                 filter1 = STAILQ_FIRST(&vnic0->filter);
3402                 //filter1 = bnxt_get_l2_filter(bp, filter, vnic0);
3403         } else {
3404                 filter->dst_id = vnic->fw_vnic_id;
3405                 for (i = 0; i < RTE_ETHER_ADDR_LEN; i++)
3406                         if (filter->dst_macaddr[i] == 0x00)
3407                                 filter1 = STAILQ_FIRST(&vnic0->filter);
3408                         else
3409                                 filter1 = bnxt_get_l2_filter(bp, filter, vnic);
3410         }
3411
3412         if (filter1 == NULL)
3413                 return -EINVAL;
3414
3415         en |= HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_L2_FILTER_ID;
3416         filter->fw_l2_filter_id = filter1->fw_l2_filter_id;
3417
3418         filter->enables = en;
3419
3420         return 0;
3421 }
3422
3423 static struct bnxt_filter_info *
3424 bnxt_match_fdir(struct bnxt *bp, struct bnxt_filter_info *nf,
3425                 struct bnxt_vnic_info **mvnic)
3426 {
3427         struct bnxt_filter_info *mf = NULL;
3428         int i;
3429
3430         for (i = bp->nr_vnics - 1; i >= 0; i--) {
3431                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3432
3433                 STAILQ_FOREACH(mf, &vnic->filter, next) {
3434                         if (mf->filter_type == nf->filter_type &&
3435                             mf->flags == nf->flags &&
3436                             mf->src_port == nf->src_port &&
3437                             mf->src_port_mask == nf->src_port_mask &&
3438                             mf->dst_port == nf->dst_port &&
3439                             mf->dst_port_mask == nf->dst_port_mask &&
3440                             mf->ip_protocol == nf->ip_protocol &&
3441                             mf->ip_addr_type == nf->ip_addr_type &&
3442                             mf->ethertype == nf->ethertype &&
3443                             mf->vni == nf->vni &&
3444                             mf->tunnel_type == nf->tunnel_type &&
3445                             mf->l2_ovlan == nf->l2_ovlan &&
3446                             mf->l2_ovlan_mask == nf->l2_ovlan_mask &&
3447                             mf->l2_ivlan == nf->l2_ivlan &&
3448                             mf->l2_ivlan_mask == nf->l2_ivlan_mask &&
3449                             !memcmp(mf->l2_addr, nf->l2_addr,
3450                                     RTE_ETHER_ADDR_LEN) &&
3451                             !memcmp(mf->l2_addr_mask, nf->l2_addr_mask,
3452                                     RTE_ETHER_ADDR_LEN) &&
3453                             !memcmp(mf->src_macaddr, nf->src_macaddr,
3454                                     RTE_ETHER_ADDR_LEN) &&
3455                             !memcmp(mf->dst_macaddr, nf->dst_macaddr,
3456                                     RTE_ETHER_ADDR_LEN) &&
3457                             !memcmp(mf->src_ipaddr, nf->src_ipaddr,
3458                                     sizeof(nf->src_ipaddr)) &&
3459                             !memcmp(mf->src_ipaddr_mask, nf->src_ipaddr_mask,
3460                                     sizeof(nf->src_ipaddr_mask)) &&
3461                             !memcmp(mf->dst_ipaddr, nf->dst_ipaddr,
3462                                     sizeof(nf->dst_ipaddr)) &&
3463                             !memcmp(mf->dst_ipaddr_mask, nf->dst_ipaddr_mask,
3464                                     sizeof(nf->dst_ipaddr_mask))) {
3465                                 if (mvnic)
3466                                         *mvnic = vnic;
3467                                 return mf;
3468                         }
3469                 }
3470         }
3471         return NULL;
3472 }
3473
3474 static int
3475 bnxt_fdir_filter(struct rte_eth_dev *dev,
3476                  enum rte_filter_op filter_op,
3477                  void *arg)
3478 {
3479         struct bnxt *bp = dev->data->dev_private;
3480         struct rte_eth_fdir_filter *fdir  = (struct rte_eth_fdir_filter *)arg;
3481         struct bnxt_filter_info *filter, *match;
3482         struct bnxt_vnic_info *vnic, *mvnic;
3483         int ret = 0, i;
3484
3485         if (filter_op == RTE_ETH_FILTER_NOP)
3486                 return 0;
3487
3488         if (arg == NULL && filter_op != RTE_ETH_FILTER_FLUSH)
3489                 return -EINVAL;
3490
3491         switch (filter_op) {
3492         case RTE_ETH_FILTER_ADD:
3493         case RTE_ETH_FILTER_DELETE:
3494                 /* FALLTHROUGH */
3495                 filter = bnxt_get_unused_filter(bp);
3496                 if (filter == NULL) {
3497                         PMD_DRV_LOG(ERR,
3498                                 "Not enough resources for a new flow.\n");
3499                         return -ENOMEM;
3500                 }
3501
3502                 ret = bnxt_parse_fdir_filter(bp, fdir, filter);
3503                 if (ret != 0)
3504                         goto free_filter;
3505                 filter->filter_type = HWRM_CFA_NTUPLE_FILTER;
3506
3507                 if (fdir->action.behavior == RTE_ETH_FDIR_REJECT)
3508                         vnic = &bp->vnic_info[0];
3509                 else
3510                         vnic = &bp->vnic_info[fdir->action.rx_queue];
3511
3512                 match = bnxt_match_fdir(bp, filter, &mvnic);
3513                 if (match != NULL && filter_op == RTE_ETH_FILTER_ADD) {
3514                         if (match->dst_id == vnic->fw_vnic_id) {
3515                                 PMD_DRV_LOG(ERR, "Flow already exists.\n");
3516                                 ret = -EEXIST;
3517                                 goto free_filter;
3518                         } else {
3519                                 match->dst_id = vnic->fw_vnic_id;
3520                                 ret = bnxt_hwrm_set_ntuple_filter(bp,
3521                                                                   match->dst_id,
3522                                                                   match);
3523                                 STAILQ_REMOVE(&mvnic->filter, match,
3524                                               bnxt_filter_info, next);
3525                                 STAILQ_INSERT_TAIL(&vnic->filter, match, next);
3526                                 PMD_DRV_LOG(ERR,
3527                                         "Filter with matching pattern exist\n");
3528                                 PMD_DRV_LOG(ERR,
3529                                         "Updated it to new destination q\n");
3530                                 goto free_filter;
3531                         }
3532                 }
3533                 if (match == NULL && filter_op == RTE_ETH_FILTER_DELETE) {
3534                         PMD_DRV_LOG(ERR, "Flow does not exist.\n");
3535                         ret = -ENOENT;
3536                         goto free_filter;
3537                 }
3538
3539                 if (filter_op == RTE_ETH_FILTER_ADD) {
3540                         ret = bnxt_hwrm_set_ntuple_filter(bp,
3541                                                           filter->dst_id,
3542                                                           filter);
3543                         if (ret)
3544                                 goto free_filter;
3545                         STAILQ_INSERT_TAIL(&vnic->filter, filter, next);
3546                 } else {
3547                         ret = bnxt_hwrm_clear_ntuple_filter(bp, match);
3548                         STAILQ_REMOVE(&vnic->filter, match,
3549                                       bnxt_filter_info, next);
3550                         bnxt_free_filter(bp, match);
3551                         bnxt_free_filter(bp, filter);
3552                 }
3553                 break;
3554         case RTE_ETH_FILTER_FLUSH:
3555                 for (i = bp->nr_vnics - 1; i >= 0; i--) {
3556                         struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3557
3558                         STAILQ_FOREACH(filter, &vnic->filter, next) {
3559                                 if (filter->filter_type ==
3560                                     HWRM_CFA_NTUPLE_FILTER) {
3561                                         ret =
3562                                         bnxt_hwrm_clear_ntuple_filter(bp,
3563                                                                       filter);
3564                                         STAILQ_REMOVE(&vnic->filter, filter,
3565                                                       bnxt_filter_info, next);
3566                                 }
3567                         }
3568                 }
3569                 return ret;
3570         case RTE_ETH_FILTER_UPDATE:
3571         case RTE_ETH_FILTER_STATS:
3572         case RTE_ETH_FILTER_INFO:
3573                 PMD_DRV_LOG(ERR, "operation %u not implemented", filter_op);
3574                 break;
3575         default:
3576                 PMD_DRV_LOG(ERR, "unknown operation %u", filter_op);
3577                 ret = -EINVAL;
3578                 break;
3579         }
3580         return ret;
3581
3582 free_filter:
3583         bnxt_free_filter(bp, filter);
3584         return ret;
3585 }
3586
3587 static int
3588 bnxt_filter_ctrl_op(struct rte_eth_dev *dev,
3589                     enum rte_filter_type filter_type,
3590                     enum rte_filter_op filter_op, void *arg)
3591 {
3592         struct bnxt *bp = dev->data->dev_private;
3593         int ret = 0;
3594
3595         ret = is_bnxt_in_error(dev->data->dev_private);
3596         if (ret)
3597                 return ret;
3598
3599         switch (filter_type) {
3600         case RTE_ETH_FILTER_TUNNEL:
3601                 PMD_DRV_LOG(ERR,
3602                         "filter type: %d: To be implemented\n", filter_type);
3603                 break;
3604         case RTE_ETH_FILTER_FDIR:
3605                 ret = bnxt_fdir_filter(dev, filter_op, arg);
3606                 break;
3607         case RTE_ETH_FILTER_NTUPLE:
3608                 ret = bnxt_ntuple_filter(dev, filter_op, arg);
3609                 break;
3610         case RTE_ETH_FILTER_ETHERTYPE:
3611                 ret = bnxt_ethertype_filter(dev, filter_op, arg);
3612                 break;
3613         case RTE_ETH_FILTER_GENERIC:
3614                 if (filter_op != RTE_ETH_FILTER_GET)
3615                         return -EINVAL;
3616                 if (bp->truflow)
3617                         *(const void **)arg = &bnxt_ulp_rte_flow_ops;
3618                 else
3619                         *(const void **)arg = &bnxt_flow_ops;
3620                 break;
3621         default:
3622                 PMD_DRV_LOG(ERR,
3623                         "Filter type (%d) not supported", filter_type);
3624                 ret = -EINVAL;
3625                 break;
3626         }
3627         return ret;
3628 }
3629
3630 static const uint32_t *
3631 bnxt_dev_supported_ptypes_get_op(struct rte_eth_dev *dev)
3632 {
3633         static const uint32_t ptypes[] = {
3634                 RTE_PTYPE_L2_ETHER_VLAN,
3635                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
3636                 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
3637                 RTE_PTYPE_L4_ICMP,
3638                 RTE_PTYPE_L4_TCP,
3639                 RTE_PTYPE_L4_UDP,
3640                 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
3641                 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
3642                 RTE_PTYPE_INNER_L4_ICMP,
3643                 RTE_PTYPE_INNER_L4_TCP,
3644                 RTE_PTYPE_INNER_L4_UDP,
3645                 RTE_PTYPE_UNKNOWN
3646         };
3647
3648         if (!dev->rx_pkt_burst)
3649                 return NULL;
3650
3651         return ptypes;
3652 }
3653
3654 static int bnxt_map_regs(struct bnxt *bp, uint32_t *reg_arr, int count,
3655                          int reg_win)
3656 {
3657         uint32_t reg_base = *reg_arr & 0xfffff000;
3658         uint32_t win_off;
3659         int i;
3660
3661         for (i = 0; i < count; i++) {
3662                 if ((reg_arr[i] & 0xfffff000) != reg_base)
3663                         return -ERANGE;
3664         }
3665         win_off = BNXT_GRCPF_REG_WINDOW_BASE_OUT + (reg_win - 1) * 4;
3666         rte_write32(reg_base, (uint8_t *)bp->bar0 + win_off);
3667         return 0;
3668 }
3669
3670 static int bnxt_map_ptp_regs(struct bnxt *bp)
3671 {
3672         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3673         uint32_t *reg_arr;
3674         int rc, i;
3675
3676         reg_arr = ptp->rx_regs;
3677         rc = bnxt_map_regs(bp, reg_arr, BNXT_PTP_RX_REGS, 5);
3678         if (rc)
3679                 return rc;
3680
3681         reg_arr = ptp->tx_regs;
3682         rc = bnxt_map_regs(bp, reg_arr, BNXT_PTP_TX_REGS, 6);
3683         if (rc)
3684                 return rc;
3685
3686         for (i = 0; i < BNXT_PTP_RX_REGS; i++)
3687                 ptp->rx_mapped_regs[i] = 0x5000 + (ptp->rx_regs[i] & 0xfff);
3688
3689         for (i = 0; i < BNXT_PTP_TX_REGS; i++)
3690                 ptp->tx_mapped_regs[i] = 0x6000 + (ptp->tx_regs[i] & 0xfff);
3691
3692         return 0;
3693 }
3694
3695 static void bnxt_unmap_ptp_regs(struct bnxt *bp)
3696 {
3697         rte_write32(0, (uint8_t *)bp->bar0 +
3698                          BNXT_GRCPF_REG_WINDOW_BASE_OUT + 16);
3699         rte_write32(0, (uint8_t *)bp->bar0 +
3700                          BNXT_GRCPF_REG_WINDOW_BASE_OUT + 20);
3701 }
3702
3703 static uint64_t bnxt_cc_read(struct bnxt *bp)
3704 {
3705         uint64_t ns;
3706
3707         ns = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3708                               BNXT_GRCPF_REG_SYNC_TIME));
3709         ns |= (uint64_t)(rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3710                                           BNXT_GRCPF_REG_SYNC_TIME + 4))) << 32;
3711         return ns;
3712 }
3713
3714 static int bnxt_get_tx_ts(struct bnxt *bp, uint64_t *ts)
3715 {
3716         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3717         uint32_t fifo;
3718
3719         fifo = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3720                                 ptp->tx_mapped_regs[BNXT_PTP_TX_FIFO]));
3721         if (fifo & BNXT_PTP_TX_FIFO_EMPTY)
3722                 return -EAGAIN;
3723
3724         fifo = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3725                                 ptp->tx_mapped_regs[BNXT_PTP_TX_FIFO]));
3726         *ts = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3727                                 ptp->tx_mapped_regs[BNXT_PTP_TX_TS_L]));
3728         *ts |= (uint64_t)rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3729                                 ptp->tx_mapped_regs[BNXT_PTP_TX_TS_H])) << 32;
3730
3731         return 0;
3732 }
3733
3734 static int bnxt_get_rx_ts(struct bnxt *bp, uint64_t *ts)
3735 {
3736         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3737         struct bnxt_pf_info *pf = &bp->pf;
3738         uint16_t port_id;
3739         uint32_t fifo;
3740
3741         if (!ptp)
3742                 return -ENODEV;
3743
3744         fifo = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3745                                 ptp->rx_mapped_regs[BNXT_PTP_RX_FIFO]));
3746         if (!(fifo & BNXT_PTP_RX_FIFO_PENDING))
3747                 return -EAGAIN;
3748
3749         port_id = pf->port_id;
3750         rte_write32(1 << port_id, (uint8_t *)bp->bar0 +
3751                ptp->rx_mapped_regs[BNXT_PTP_RX_FIFO_ADV]);
3752
3753         fifo = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3754                                    ptp->rx_mapped_regs[BNXT_PTP_RX_FIFO]));
3755         if (fifo & BNXT_PTP_RX_FIFO_PENDING) {
3756 /*              bnxt_clr_rx_ts(bp);       TBD  */
3757                 return -EBUSY;
3758         }
3759
3760         *ts = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3761                                 ptp->rx_mapped_regs[BNXT_PTP_RX_TS_L]));
3762         *ts |= (uint64_t)rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3763                                 ptp->rx_mapped_regs[BNXT_PTP_RX_TS_H])) << 32;
3764
3765         return 0;
3766 }
3767
3768 static int
3769 bnxt_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts)
3770 {
3771         uint64_t ns;
3772         struct bnxt *bp = dev->data->dev_private;
3773         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3774
3775         if (!ptp)
3776                 return 0;
3777
3778         ns = rte_timespec_to_ns(ts);
3779         /* Set the timecounters to a new value. */
3780         ptp->tc.nsec = ns;
3781
3782         return 0;
3783 }
3784
3785 static int
3786 bnxt_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts)
3787 {
3788         struct bnxt *bp = dev->data->dev_private;
3789         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3790         uint64_t ns, systime_cycles = 0;
3791         int rc = 0;
3792
3793         if (!ptp)
3794                 return 0;
3795
3796         if (BNXT_CHIP_THOR(bp))
3797                 rc = bnxt_hwrm_port_ts_query(bp, BNXT_PTP_FLAGS_CURRENT_TIME,
3798                                              &systime_cycles);
3799         else
3800                 systime_cycles = bnxt_cc_read(bp);
3801
3802         ns = rte_timecounter_update(&ptp->tc, systime_cycles);
3803         *ts = rte_ns_to_timespec(ns);
3804
3805         return rc;
3806 }
3807 static int
3808 bnxt_timesync_enable(struct rte_eth_dev *dev)
3809 {
3810         struct bnxt *bp = dev->data->dev_private;
3811         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3812         uint32_t shift = 0;
3813         int rc;
3814
3815         if (!ptp)
3816                 return 0;
3817
3818         ptp->rx_filter = 1;
3819         ptp->tx_tstamp_en = 1;
3820         ptp->rxctl = BNXT_PTP_MSG_EVENTS;
3821
3822         rc = bnxt_hwrm_ptp_cfg(bp);
3823         if (rc)
3824                 return rc;
3825
3826         memset(&ptp->tc, 0, sizeof(struct rte_timecounter));
3827         memset(&ptp->rx_tstamp_tc, 0, sizeof(struct rte_timecounter));
3828         memset(&ptp->tx_tstamp_tc, 0, sizeof(struct rte_timecounter));
3829
3830         ptp->tc.cc_mask = BNXT_CYCLECOUNTER_MASK;
3831         ptp->tc.cc_shift = shift;
3832         ptp->tc.nsec_mask = (1ULL << shift) - 1;
3833
3834         ptp->rx_tstamp_tc.cc_mask = BNXT_CYCLECOUNTER_MASK;
3835         ptp->rx_tstamp_tc.cc_shift = shift;
3836         ptp->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
3837
3838         ptp->tx_tstamp_tc.cc_mask = BNXT_CYCLECOUNTER_MASK;
3839         ptp->tx_tstamp_tc.cc_shift = shift;
3840         ptp->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
3841
3842         if (!BNXT_CHIP_THOR(bp))
3843                 bnxt_map_ptp_regs(bp);
3844
3845         return 0;
3846 }
3847
3848 static int
3849 bnxt_timesync_disable(struct rte_eth_dev *dev)
3850 {
3851         struct bnxt *bp = dev->data->dev_private;
3852         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3853
3854         if (!ptp)
3855                 return 0;
3856
3857         ptp->rx_filter = 0;
3858         ptp->tx_tstamp_en = 0;
3859         ptp->rxctl = 0;
3860
3861         bnxt_hwrm_ptp_cfg(bp);
3862
3863         if (!BNXT_CHIP_THOR(bp))
3864                 bnxt_unmap_ptp_regs(bp);
3865
3866         return 0;
3867 }
3868
3869 static int
3870 bnxt_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
3871                                  struct timespec *timestamp,
3872                                  uint32_t flags __rte_unused)
3873 {
3874         struct bnxt *bp = dev->data->dev_private;
3875         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3876         uint64_t rx_tstamp_cycles = 0;
3877         uint64_t ns;
3878
3879         if (!ptp)
3880                 return 0;
3881
3882         if (BNXT_CHIP_THOR(bp))
3883                 rx_tstamp_cycles = ptp->rx_timestamp;
3884         else
3885                 bnxt_get_rx_ts(bp, &rx_tstamp_cycles);
3886
3887         ns = rte_timecounter_update(&ptp->rx_tstamp_tc, rx_tstamp_cycles);
3888         *timestamp = rte_ns_to_timespec(ns);
3889         return  0;
3890 }
3891
3892 static int
3893 bnxt_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
3894                                  struct timespec *timestamp)
3895 {
3896         struct bnxt *bp = dev->data->dev_private;
3897         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3898         uint64_t tx_tstamp_cycles = 0;
3899         uint64_t ns;
3900         int rc = 0;
3901
3902         if (!ptp)
3903                 return 0;
3904
3905         if (BNXT_CHIP_THOR(bp))
3906                 rc = bnxt_hwrm_port_ts_query(bp, BNXT_PTP_FLAGS_PATH_TX,
3907                                              &tx_tstamp_cycles);
3908         else
3909                 rc = bnxt_get_tx_ts(bp, &tx_tstamp_cycles);
3910
3911         ns = rte_timecounter_update(&ptp->tx_tstamp_tc, tx_tstamp_cycles);
3912         *timestamp = rte_ns_to_timespec(ns);
3913
3914         return rc;
3915 }
3916
3917 static int
3918 bnxt_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta)
3919 {
3920         struct bnxt *bp = dev->data->dev_private;
3921         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3922
3923         if (!ptp)
3924                 return 0;
3925
3926         ptp->tc.nsec += delta;
3927
3928         return 0;
3929 }
3930
3931 static int
3932 bnxt_get_eeprom_length_op(struct rte_eth_dev *dev)
3933 {
3934         struct bnxt *bp = dev->data->dev_private;
3935         int rc;
3936         uint32_t dir_entries;
3937         uint32_t entry_length;
3938
3939         rc = is_bnxt_in_error(bp);
3940         if (rc)
3941                 return rc;
3942
3943         PMD_DRV_LOG(INFO, PCI_PRI_FMT "\n",
3944                     bp->pdev->addr.domain, bp->pdev->addr.bus,
3945                     bp->pdev->addr.devid, bp->pdev->addr.function);
3946
3947         rc = bnxt_hwrm_nvm_get_dir_info(bp, &dir_entries, &entry_length);
3948         if (rc != 0)
3949                 return rc;
3950
3951         return dir_entries * entry_length;
3952 }
3953
3954 static int
3955 bnxt_get_eeprom_op(struct rte_eth_dev *dev,
3956                 struct rte_dev_eeprom_info *in_eeprom)
3957 {
3958         struct bnxt *bp = dev->data->dev_private;
3959         uint32_t index;
3960         uint32_t offset;
3961         int rc;
3962
3963         rc = is_bnxt_in_error(bp);
3964         if (rc)
3965                 return rc;
3966
3967         PMD_DRV_LOG(INFO, PCI_PRI_FMT " in_eeprom->offset = %d len = %d\n",
3968                     bp->pdev->addr.domain, bp->pdev->addr.bus,
3969                     bp->pdev->addr.devid, bp->pdev->addr.function,
3970                     in_eeprom->offset, in_eeprom->length);
3971
3972         if (in_eeprom->offset == 0) /* special offset value to get directory */
3973                 return bnxt_get_nvram_directory(bp, in_eeprom->length,
3974                                                 in_eeprom->data);
3975
3976         index = in_eeprom->offset >> 24;
3977         offset = in_eeprom->offset & 0xffffff;
3978
3979         if (index != 0)
3980                 return bnxt_hwrm_get_nvram_item(bp, index - 1, offset,
3981                                            in_eeprom->length, in_eeprom->data);
3982
3983         return 0;
3984 }
3985
3986 static bool bnxt_dir_type_is_ape_bin_format(uint16_t dir_type)
3987 {
3988         switch (dir_type) {
3989         case BNX_DIR_TYPE_CHIMP_PATCH:
3990         case BNX_DIR_TYPE_BOOTCODE:
3991         case BNX_DIR_TYPE_BOOTCODE_2:
3992         case BNX_DIR_TYPE_APE_FW:
3993         case BNX_DIR_TYPE_APE_PATCH:
3994         case BNX_DIR_TYPE_KONG_FW:
3995         case BNX_DIR_TYPE_KONG_PATCH:
3996         case BNX_DIR_TYPE_BONO_FW:
3997         case BNX_DIR_TYPE_BONO_PATCH:
3998                 /* FALLTHROUGH */
3999                 return true;
4000         }
4001
4002         return false;
4003 }
4004
4005 static bool bnxt_dir_type_is_other_exec_format(uint16_t dir_type)
4006 {
4007         switch (dir_type) {
4008         case BNX_DIR_TYPE_AVS:
4009         case BNX_DIR_TYPE_EXP_ROM_MBA:
4010         case BNX_DIR_TYPE_PCIE:
4011         case BNX_DIR_TYPE_TSCF_UCODE:
4012         case BNX_DIR_TYPE_EXT_PHY:
4013         case BNX_DIR_TYPE_CCM:
4014         case BNX_DIR_TYPE_ISCSI_BOOT:
4015         case BNX_DIR_TYPE_ISCSI_BOOT_IPV6:
4016         case BNX_DIR_TYPE_ISCSI_BOOT_IPV4N6:
4017                 /* FALLTHROUGH */
4018                 return true;
4019         }
4020
4021         return false;
4022 }
4023
4024 static bool bnxt_dir_type_is_executable(uint16_t dir_type)
4025 {
4026         return bnxt_dir_type_is_ape_bin_format(dir_type) ||
4027                 bnxt_dir_type_is_other_exec_format(dir_type);
4028 }
4029
4030 static int
4031 bnxt_set_eeprom_op(struct rte_eth_dev *dev,
4032                 struct rte_dev_eeprom_info *in_eeprom)
4033 {
4034         struct bnxt *bp = dev->data->dev_private;
4035         uint8_t index, dir_op;
4036         uint16_t type, ext, ordinal, attr;
4037         int rc;
4038
4039         rc = is_bnxt_in_error(bp);
4040         if (rc)
4041                 return rc;
4042
4043         PMD_DRV_LOG(INFO, PCI_PRI_FMT " in_eeprom->offset = %d len = %d\n",
4044                     bp->pdev->addr.domain, bp->pdev->addr.bus,
4045                     bp->pdev->addr.devid, bp->pdev->addr.function,
4046                     in_eeprom->offset, in_eeprom->length);
4047
4048         if (!BNXT_PF(bp)) {
4049                 PMD_DRV_LOG(ERR, "NVM write not supported from a VF\n");
4050                 return -EINVAL;
4051         }
4052
4053         type = in_eeprom->magic >> 16;
4054
4055         if (type == 0xffff) { /* special value for directory operations */
4056                 index = in_eeprom->magic & 0xff;
4057                 dir_op = in_eeprom->magic >> 8;
4058                 if (index == 0)
4059                         return -EINVAL;
4060                 switch (dir_op) {
4061                 case 0x0e: /* erase */
4062                         if (in_eeprom->offset != ~in_eeprom->magic)
4063                                 return -EINVAL;
4064                         return bnxt_hwrm_erase_nvram_directory(bp, index - 1);
4065                 default:
4066                         return -EINVAL;
4067                 }
4068         }
4069
4070         /* Create or re-write an NVM item: */
4071         if (bnxt_dir_type_is_executable(type) == true)
4072                 return -EOPNOTSUPP;
4073         ext = in_eeprom->magic & 0xffff;
4074         ordinal = in_eeprom->offset >> 16;
4075         attr = in_eeprom->offset & 0xffff;
4076
4077         return bnxt_hwrm_flash_nvram(bp, type, ordinal, ext, attr,
4078                                      in_eeprom->data, in_eeprom->length);
4079 }
4080
4081 /*
4082  * Initialization
4083  */
4084
4085 static const struct eth_dev_ops bnxt_dev_ops = {
4086         .dev_infos_get = bnxt_dev_info_get_op,
4087         .dev_close = bnxt_dev_close_op,
4088         .dev_configure = bnxt_dev_configure_op,
4089         .dev_start = bnxt_dev_start_op,
4090         .dev_stop = bnxt_dev_stop_op,
4091         .dev_set_link_up = bnxt_dev_set_link_up_op,
4092         .dev_set_link_down = bnxt_dev_set_link_down_op,
4093         .stats_get = bnxt_stats_get_op,
4094         .stats_reset = bnxt_stats_reset_op,
4095         .rx_queue_setup = bnxt_rx_queue_setup_op,
4096         .rx_queue_release = bnxt_rx_queue_release_op,
4097         .tx_queue_setup = bnxt_tx_queue_setup_op,
4098         .tx_queue_release = bnxt_tx_queue_release_op,
4099         .rx_queue_intr_enable = bnxt_rx_queue_intr_enable_op,
4100         .rx_queue_intr_disable = bnxt_rx_queue_intr_disable_op,
4101         .reta_update = bnxt_reta_update_op,
4102         .reta_query = bnxt_reta_query_op,
4103         .rss_hash_update = bnxt_rss_hash_update_op,
4104         .rss_hash_conf_get = bnxt_rss_hash_conf_get_op,
4105         .link_update = bnxt_link_update_op,
4106         .promiscuous_enable = bnxt_promiscuous_enable_op,
4107         .promiscuous_disable = bnxt_promiscuous_disable_op,
4108         .allmulticast_enable = bnxt_allmulticast_enable_op,
4109         .allmulticast_disable = bnxt_allmulticast_disable_op,
4110         .mac_addr_add = bnxt_mac_addr_add_op,
4111         .mac_addr_remove = bnxt_mac_addr_remove_op,
4112         .flow_ctrl_get = bnxt_flow_ctrl_get_op,
4113         .flow_ctrl_set = bnxt_flow_ctrl_set_op,
4114         .udp_tunnel_port_add  = bnxt_udp_tunnel_port_add_op,
4115         .udp_tunnel_port_del  = bnxt_udp_tunnel_port_del_op,
4116         .vlan_filter_set = bnxt_vlan_filter_set_op,
4117         .vlan_offload_set = bnxt_vlan_offload_set_op,
4118         .vlan_tpid_set = bnxt_vlan_tpid_set_op,
4119         .vlan_pvid_set = bnxt_vlan_pvid_set_op,
4120         .mtu_set = bnxt_mtu_set_op,
4121         .mac_addr_set = bnxt_set_default_mac_addr_op,
4122         .xstats_get = bnxt_dev_xstats_get_op,
4123         .xstats_get_names = bnxt_dev_xstats_get_names_op,
4124         .xstats_reset = bnxt_dev_xstats_reset_op,
4125         .fw_version_get = bnxt_fw_version_get,
4126         .set_mc_addr_list = bnxt_dev_set_mc_addr_list_op,
4127         .rxq_info_get = bnxt_rxq_info_get_op,
4128         .txq_info_get = bnxt_txq_info_get_op,
4129         .dev_led_on = bnxt_dev_led_on_op,
4130         .dev_led_off = bnxt_dev_led_off_op,
4131         .xstats_get_by_id = bnxt_dev_xstats_get_by_id_op,
4132         .xstats_get_names_by_id = bnxt_dev_xstats_get_names_by_id_op,
4133         .rx_queue_count = bnxt_rx_queue_count_op,
4134         .rx_descriptor_status = bnxt_rx_descriptor_status_op,
4135         .tx_descriptor_status = bnxt_tx_descriptor_status_op,
4136         .rx_queue_start = bnxt_rx_queue_start,
4137         .rx_queue_stop = bnxt_rx_queue_stop,
4138         .tx_queue_start = bnxt_tx_queue_start,
4139         .tx_queue_stop = bnxt_tx_queue_stop,
4140         .filter_ctrl = bnxt_filter_ctrl_op,
4141         .dev_supported_ptypes_get = bnxt_dev_supported_ptypes_get_op,
4142         .get_eeprom_length    = bnxt_get_eeprom_length_op,
4143         .get_eeprom           = bnxt_get_eeprom_op,
4144         .set_eeprom           = bnxt_set_eeprom_op,
4145         .timesync_enable      = bnxt_timesync_enable,
4146         .timesync_disable     = bnxt_timesync_disable,
4147         .timesync_read_time   = bnxt_timesync_read_time,
4148         .timesync_write_time   = bnxt_timesync_write_time,
4149         .timesync_adjust_time = bnxt_timesync_adjust_time,
4150         .timesync_read_rx_timestamp = bnxt_timesync_read_rx_timestamp,
4151         .timesync_read_tx_timestamp = bnxt_timesync_read_tx_timestamp,
4152 };
4153
4154 static uint32_t bnxt_map_reset_regs(struct bnxt *bp, uint32_t reg)
4155 {
4156         uint32_t offset;
4157
4158         /* Only pre-map the reset GRC registers using window 3 */
4159         rte_write32(reg & 0xfffff000, (uint8_t *)bp->bar0 +
4160                     BNXT_GRCPF_REG_WINDOW_BASE_OUT + 8);
4161
4162         offset = BNXT_GRCP_WINDOW_3_BASE + (reg & 0xffc);
4163
4164         return offset;
4165 }
4166
4167 int bnxt_map_fw_health_status_regs(struct bnxt *bp)
4168 {
4169         struct bnxt_error_recovery_info *info = bp->recovery_info;
4170         uint32_t reg_base = 0xffffffff;
4171         int i;
4172
4173         /* Only pre-map the monitoring GRC registers using window 2 */
4174         for (i = 0; i < BNXT_FW_STATUS_REG_CNT; i++) {
4175                 uint32_t reg = info->status_regs[i];
4176
4177                 if (BNXT_FW_STATUS_REG_TYPE(reg) != BNXT_FW_STATUS_REG_TYPE_GRC)
4178                         continue;
4179
4180                 if (reg_base == 0xffffffff)
4181                         reg_base = reg & 0xfffff000;
4182                 if ((reg & 0xfffff000) != reg_base)
4183                         return -ERANGE;
4184
4185                 /* Use mask 0xffc as the Lower 2 bits indicates
4186                  * address space location
4187                  */
4188                 info->mapped_status_regs[i] = BNXT_GRCP_WINDOW_2_BASE +
4189                                                 (reg & 0xffc);
4190         }
4191
4192         if (reg_base == 0xffffffff)
4193                 return 0;
4194
4195         rte_write32(reg_base, (uint8_t *)bp->bar0 +
4196                     BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
4197
4198         return 0;
4199 }
4200
4201 static void bnxt_write_fw_reset_reg(struct bnxt *bp, uint32_t index)
4202 {
4203         struct bnxt_error_recovery_info *info = bp->recovery_info;
4204         uint32_t delay = info->delay_after_reset[index];
4205         uint32_t val = info->reset_reg_val[index];
4206         uint32_t reg = info->reset_reg[index];
4207         uint32_t type, offset;
4208
4209         type = BNXT_FW_STATUS_REG_TYPE(reg);
4210         offset = BNXT_FW_STATUS_REG_OFF(reg);
4211
4212         switch (type) {
4213         case BNXT_FW_STATUS_REG_TYPE_CFG:
4214                 rte_pci_write_config(bp->pdev, &val, sizeof(val), offset);
4215                 break;
4216         case BNXT_FW_STATUS_REG_TYPE_GRC:
4217                 offset = bnxt_map_reset_regs(bp, offset);
4218                 rte_write32(val, (uint8_t *)bp->bar0 + offset);
4219                 break;
4220         case BNXT_FW_STATUS_REG_TYPE_BAR0:
4221                 rte_write32(val, (uint8_t *)bp->bar0 + offset);
4222                 break;
4223         }
4224         /* wait on a specific interval of time until core reset is complete */
4225         if (delay)
4226                 rte_delay_ms(delay);
4227 }
4228
4229 static void bnxt_dev_cleanup(struct bnxt *bp)
4230 {
4231         bnxt_set_hwrm_link_config(bp, false);
4232         bp->link_info.link_up = 0;
4233         if (bp->eth_dev->data->dev_started)
4234                 bnxt_dev_stop_op(bp->eth_dev);
4235
4236         bnxt_uninit_resources(bp, true);
4237 }
4238
4239 static int bnxt_restore_vlan_filters(struct bnxt *bp)
4240 {
4241         struct rte_eth_dev *dev = bp->eth_dev;
4242         struct rte_vlan_filter_conf *vfc;
4243         int vidx, vbit, rc;
4244         uint16_t vlan_id;
4245
4246         for (vlan_id = 1; vlan_id <= RTE_ETHER_MAX_VLAN_ID; vlan_id++) {
4247                 vfc = &dev->data->vlan_filter_conf;
4248                 vidx = vlan_id / 64;
4249                 vbit = vlan_id % 64;
4250
4251                 /* Each bit corresponds to a VLAN id */
4252                 if (vfc->ids[vidx] & (UINT64_C(1) << vbit)) {
4253                         rc = bnxt_add_vlan_filter(bp, vlan_id);
4254                         if (rc)
4255                                 return rc;
4256                 }
4257         }
4258
4259         return 0;
4260 }
4261
4262 static int bnxt_restore_mac_filters(struct bnxt *bp)
4263 {
4264         struct rte_eth_dev *dev = bp->eth_dev;
4265         struct rte_eth_dev_info dev_info;
4266         struct rte_ether_addr *addr;
4267         uint64_t pool_mask;
4268         uint32_t pool = 0;
4269         uint16_t i;
4270         int rc;
4271
4272         if (BNXT_VF(bp) & !BNXT_VF_IS_TRUSTED(bp))
4273                 return 0;
4274
4275         rc = bnxt_dev_info_get_op(dev, &dev_info);
4276         if (rc)
4277                 return rc;
4278
4279         /* replay MAC address configuration */
4280         for (i = 1; i < dev_info.max_mac_addrs; i++) {
4281                 addr = &dev->data->mac_addrs[i];
4282
4283                 /* skip zero address */
4284                 if (rte_is_zero_ether_addr(addr))
4285                         continue;
4286
4287                 pool = 0;
4288                 pool_mask = dev->data->mac_pool_sel[i];
4289
4290                 do {
4291                         if (pool_mask & 1ULL) {
4292                                 rc = bnxt_mac_addr_add_op(dev, addr, i, pool);
4293                                 if (rc)
4294                                         return rc;
4295                         }
4296                         pool_mask >>= 1;
4297                         pool++;
4298                 } while (pool_mask);
4299         }
4300
4301         return 0;
4302 }
4303
4304 static int bnxt_restore_filters(struct bnxt *bp)
4305 {
4306         struct rte_eth_dev *dev = bp->eth_dev;
4307         int ret = 0;
4308
4309         if (dev->data->all_multicast) {
4310                 ret = bnxt_allmulticast_enable_op(dev);
4311                 if (ret)
4312                         return ret;
4313         }
4314         if (dev->data->promiscuous) {
4315                 ret = bnxt_promiscuous_enable_op(dev);
4316                 if (ret)
4317                         return ret;
4318         }
4319
4320         ret = bnxt_restore_mac_filters(bp);
4321         if (ret)
4322                 return ret;
4323
4324         ret = bnxt_restore_vlan_filters(bp);
4325         /* TODO restore other filters as well */
4326         return ret;
4327 }
4328
4329 static void bnxt_dev_recover(void *arg)
4330 {
4331         struct bnxt *bp = arg;
4332         int timeout = bp->fw_reset_max_msecs;
4333         int rc = 0;
4334
4335         /* Clear Error flag so that device re-init should happen */
4336         bp->flags &= ~BNXT_FLAG_FATAL_ERROR;
4337
4338         do {
4339                 rc = bnxt_hwrm_ver_get(bp, SHORT_HWRM_CMD_TIMEOUT);
4340                 if (rc == 0)
4341                         break;
4342                 rte_delay_ms(BNXT_FW_READY_WAIT_INTERVAL);
4343                 timeout -= BNXT_FW_READY_WAIT_INTERVAL;
4344         } while (rc && timeout);
4345
4346         if (rc) {
4347                 PMD_DRV_LOG(ERR, "FW is not Ready after reset\n");
4348                 goto err;
4349         }
4350
4351         rc = bnxt_init_resources(bp, true);
4352         if (rc) {
4353                 PMD_DRV_LOG(ERR,
4354                             "Failed to initialize resources after reset\n");
4355                 goto err;
4356         }
4357         /* clear reset flag as the device is initialized now */
4358         bp->flags &= ~BNXT_FLAG_FW_RESET;
4359
4360         rc = bnxt_dev_start_op(bp->eth_dev);
4361         if (rc) {
4362                 PMD_DRV_LOG(ERR, "Failed to start port after reset\n");
4363                 goto err_start;
4364         }
4365
4366         rc = bnxt_restore_filters(bp);
4367         if (rc)
4368                 goto err_start;
4369
4370         PMD_DRV_LOG(INFO, "Recovered from FW reset\n");
4371         return;
4372 err_start:
4373         bnxt_dev_stop_op(bp->eth_dev);
4374 err:
4375         bp->flags |= BNXT_FLAG_FATAL_ERROR;
4376         bnxt_uninit_resources(bp, false);
4377         PMD_DRV_LOG(ERR, "Failed to recover from FW reset\n");
4378 }
4379
4380 void bnxt_dev_reset_and_resume(void *arg)
4381 {
4382         struct bnxt *bp = arg;
4383         int rc;
4384
4385         bnxt_dev_cleanup(bp);
4386
4387         bnxt_wait_for_device_shutdown(bp);
4388
4389         rc = rte_eal_alarm_set(US_PER_MS * bp->fw_reset_min_msecs,
4390                                bnxt_dev_recover, (void *)bp);
4391         if (rc)
4392                 PMD_DRV_LOG(ERR, "Error setting recovery alarm");
4393 }
4394
4395 uint32_t bnxt_read_fw_status_reg(struct bnxt *bp, uint32_t index)
4396 {
4397         struct bnxt_error_recovery_info *info = bp->recovery_info;
4398         uint32_t reg = info->status_regs[index];
4399         uint32_t type, offset, val = 0;
4400
4401         type = BNXT_FW_STATUS_REG_TYPE(reg);
4402         offset = BNXT_FW_STATUS_REG_OFF(reg);
4403
4404         switch (type) {
4405         case BNXT_FW_STATUS_REG_TYPE_CFG:
4406                 rte_pci_read_config(bp->pdev, &val, sizeof(val), offset);
4407                 break;
4408         case BNXT_FW_STATUS_REG_TYPE_GRC:
4409                 offset = info->mapped_status_regs[index];
4410                 /* FALLTHROUGH */
4411         case BNXT_FW_STATUS_REG_TYPE_BAR0:
4412                 val = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
4413                                        offset));
4414                 break;
4415         }
4416
4417         return val;
4418 }
4419
4420 static int bnxt_fw_reset_all(struct bnxt *bp)
4421 {
4422         struct bnxt_error_recovery_info *info = bp->recovery_info;
4423         uint32_t i;
4424         int rc = 0;
4425
4426         if (info->flags & BNXT_FLAG_ERROR_RECOVERY_HOST) {
4427                 /* Reset through master function driver */
4428                 for (i = 0; i < info->reg_array_cnt; i++)
4429                         bnxt_write_fw_reset_reg(bp, i);
4430                 /* Wait for time specified by FW after triggering reset */
4431                 rte_delay_ms(info->master_func_wait_period_after_reset);
4432         } else if (info->flags & BNXT_FLAG_ERROR_RECOVERY_CO_CPU) {
4433                 /* Reset with the help of Kong processor */
4434                 rc = bnxt_hwrm_fw_reset(bp);
4435                 if (rc)
4436                         PMD_DRV_LOG(ERR, "Failed to reset FW\n");
4437         }
4438
4439         return rc;
4440 }
4441
4442 static void bnxt_fw_reset_cb(void *arg)
4443 {
4444         struct bnxt *bp = arg;
4445         struct bnxt_error_recovery_info *info = bp->recovery_info;
4446         int rc = 0;
4447
4448         /* Only Master function can do FW reset */
4449         if (bnxt_is_master_func(bp) &&
4450             bnxt_is_recovery_enabled(bp)) {
4451                 rc = bnxt_fw_reset_all(bp);
4452                 if (rc) {
4453                         PMD_DRV_LOG(ERR, "Adapter recovery failed\n");
4454                         return;
4455                 }
4456         }
4457
4458         /* if recovery method is ERROR_RECOVERY_CO_CPU, KONG will send
4459          * EXCEPTION_FATAL_ASYNC event to all the functions
4460          * (including MASTER FUNC). After receiving this Async, all the active
4461          * drivers should treat this case as FW initiated recovery
4462          */
4463         if (info->flags & BNXT_FLAG_ERROR_RECOVERY_HOST) {
4464                 bp->fw_reset_min_msecs = BNXT_MIN_FW_READY_TIMEOUT;
4465                 bp->fw_reset_max_msecs = BNXT_MAX_FW_RESET_TIMEOUT;
4466
4467                 /* To recover from error */
4468                 rte_eal_alarm_set(US_PER_MS, bnxt_dev_reset_and_resume,
4469                                   (void *)bp);
4470         }
4471 }
4472
4473 /* Driver should poll FW heartbeat, reset_counter with the frequency
4474  * advertised by FW in HWRM_ERROR_RECOVERY_QCFG.
4475  * When the driver detects heartbeat stop or change in reset_counter,
4476  * it has to trigger a reset to recover from the error condition.
4477  * A “master PF” is the function who will have the privilege to
4478  * initiate the chimp reset. The master PF will be elected by the
4479  * firmware and will be notified through async message.
4480  */
4481 static void bnxt_check_fw_health(void *arg)
4482 {
4483         struct bnxt *bp = arg;
4484         struct bnxt_error_recovery_info *info = bp->recovery_info;
4485         uint32_t val = 0, wait_msec;
4486
4487         if (!info || !bnxt_is_recovery_enabled(bp) ||
4488             is_bnxt_in_error(bp))
4489                 return;
4490
4491         val = bnxt_read_fw_status_reg(bp, BNXT_FW_HEARTBEAT_CNT_REG);
4492         if (val == info->last_heart_beat)
4493                 goto reset;
4494
4495         info->last_heart_beat = val;
4496
4497         val = bnxt_read_fw_status_reg(bp, BNXT_FW_RECOVERY_CNT_REG);
4498         if (val != info->last_reset_counter)
4499                 goto reset;
4500
4501         info->last_reset_counter = val;
4502
4503         rte_eal_alarm_set(US_PER_MS * info->driver_polling_freq,
4504                           bnxt_check_fw_health, (void *)bp);
4505
4506         return;
4507 reset:
4508         /* Stop DMA to/from device */
4509         bp->flags |= BNXT_FLAG_FATAL_ERROR;
4510         bp->flags |= BNXT_FLAG_FW_RESET;
4511
4512         PMD_DRV_LOG(ERR, "Detected FW dead condition\n");
4513
4514         if (bnxt_is_master_func(bp))
4515                 wait_msec = info->master_func_wait_period;
4516         else
4517                 wait_msec = info->normal_func_wait_period;
4518
4519         rte_eal_alarm_set(US_PER_MS * wait_msec,
4520                           bnxt_fw_reset_cb, (void *)bp);
4521 }
4522
4523 void bnxt_schedule_fw_health_check(struct bnxt *bp)
4524 {
4525         uint32_t polling_freq;
4526
4527         if (!bnxt_is_recovery_enabled(bp))
4528                 return;
4529
4530         if (bp->flags & BNXT_FLAG_FW_HEALTH_CHECK_SCHEDULED)
4531                 return;
4532
4533         polling_freq = bp->recovery_info->driver_polling_freq;
4534
4535         rte_eal_alarm_set(US_PER_MS * polling_freq,
4536                           bnxt_check_fw_health, (void *)bp);
4537         bp->flags |= BNXT_FLAG_FW_HEALTH_CHECK_SCHEDULED;
4538 }
4539
4540 static void bnxt_cancel_fw_health_check(struct bnxt *bp)
4541 {
4542         if (!bnxt_is_recovery_enabled(bp))
4543                 return;
4544
4545         rte_eal_alarm_cancel(bnxt_check_fw_health, (void *)bp);
4546         bp->flags &= ~BNXT_FLAG_FW_HEALTH_CHECK_SCHEDULED;
4547 }
4548
4549 static bool bnxt_vf_pciid(uint16_t device_id)
4550 {
4551         switch (device_id) {
4552         case BROADCOM_DEV_ID_57304_VF:
4553         case BROADCOM_DEV_ID_57406_VF:
4554         case BROADCOM_DEV_ID_5731X_VF:
4555         case BROADCOM_DEV_ID_5741X_VF:
4556         case BROADCOM_DEV_ID_57414_VF:
4557         case BROADCOM_DEV_ID_STRATUS_NIC_VF1:
4558         case BROADCOM_DEV_ID_STRATUS_NIC_VF2:
4559         case BROADCOM_DEV_ID_58802_VF:
4560         case BROADCOM_DEV_ID_57500_VF1:
4561         case BROADCOM_DEV_ID_57500_VF2:
4562                 /* FALLTHROUGH */
4563                 return true;
4564         default:
4565                 return false;
4566         }
4567 }
4568
4569 static bool bnxt_thor_device(uint16_t device_id)
4570 {
4571         switch (device_id) {
4572         case BROADCOM_DEV_ID_57508:
4573         case BROADCOM_DEV_ID_57504:
4574         case BROADCOM_DEV_ID_57502:
4575         case BROADCOM_DEV_ID_57508_MF1:
4576         case BROADCOM_DEV_ID_57504_MF1:
4577         case BROADCOM_DEV_ID_57502_MF1:
4578         case BROADCOM_DEV_ID_57508_MF2:
4579         case BROADCOM_DEV_ID_57504_MF2:
4580         case BROADCOM_DEV_ID_57502_MF2:
4581         case BROADCOM_DEV_ID_57500_VF1:
4582         case BROADCOM_DEV_ID_57500_VF2:
4583                 /* FALLTHROUGH */
4584                 return true;
4585         default:
4586                 return false;
4587         }
4588 }
4589
4590 bool bnxt_stratus_device(struct bnxt *bp)
4591 {
4592         uint16_t device_id = bp->pdev->id.device_id;
4593
4594         switch (device_id) {
4595         case BROADCOM_DEV_ID_STRATUS_NIC:
4596         case BROADCOM_DEV_ID_STRATUS_NIC_VF1:
4597         case BROADCOM_DEV_ID_STRATUS_NIC_VF2:
4598                 /* FALLTHROUGH */
4599                 return true;
4600         default:
4601                 return false;
4602         }
4603 }
4604
4605 static int bnxt_init_board(struct rte_eth_dev *eth_dev)
4606 {
4607         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
4608         struct bnxt *bp = eth_dev->data->dev_private;
4609
4610         /* enable device (incl. PCI PM wakeup), and bus-mastering */
4611         bp->bar0 = (void *)pci_dev->mem_resource[0].addr;
4612         bp->doorbell_base = (void *)pci_dev->mem_resource[2].addr;
4613         if (!bp->bar0 || !bp->doorbell_base) {
4614                 PMD_DRV_LOG(ERR, "Unable to access Hardware\n");
4615                 return -ENODEV;
4616         }
4617
4618         bp->eth_dev = eth_dev;
4619         bp->pdev = pci_dev;
4620
4621         return 0;
4622 }
4623
4624 static int bnxt_alloc_ctx_mem_blk(struct bnxt *bp,
4625                                   struct bnxt_ctx_pg_info *ctx_pg,
4626                                   uint32_t mem_size,
4627                                   const char *suffix,
4628                                   uint16_t idx)
4629 {
4630         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
4631         const struct rte_memzone *mz = NULL;
4632         char mz_name[RTE_MEMZONE_NAMESIZE];
4633         rte_iova_t mz_phys_addr;
4634         uint64_t valid_bits = 0;
4635         uint32_t sz;
4636         int i;
4637
4638         if (!mem_size)
4639                 return 0;
4640
4641         rmem->nr_pages = RTE_ALIGN_MUL_CEIL(mem_size, BNXT_PAGE_SIZE) /
4642                          BNXT_PAGE_SIZE;
4643         rmem->page_size = BNXT_PAGE_SIZE;
4644         rmem->pg_arr = ctx_pg->ctx_pg_arr;
4645         rmem->dma_arr = ctx_pg->ctx_dma_arr;
4646         rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
4647
4648         valid_bits = PTU_PTE_VALID;
4649
4650         if (rmem->nr_pages > 1) {
4651                 snprintf(mz_name, RTE_MEMZONE_NAMESIZE,
4652                          "bnxt_ctx_pg_tbl%s_%x_%d",
4653                          suffix, idx, bp->eth_dev->data->port_id);
4654                 mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0;
4655                 mz = rte_memzone_lookup(mz_name);
4656                 if (!mz) {
4657                         mz = rte_memzone_reserve_aligned(mz_name,
4658                                                 rmem->nr_pages * 8,
4659                                                 SOCKET_ID_ANY,
4660                                                 RTE_MEMZONE_2MB |
4661                                                 RTE_MEMZONE_SIZE_HINT_ONLY |
4662                                                 RTE_MEMZONE_IOVA_CONTIG,
4663                                                 BNXT_PAGE_SIZE);
4664                         if (mz == NULL)
4665                                 return -ENOMEM;
4666                 }
4667
4668                 memset(mz->addr, 0, mz->len);
4669                 mz_phys_addr = mz->iova;
4670
4671                 rmem->pg_tbl = mz->addr;
4672                 rmem->pg_tbl_map = mz_phys_addr;
4673                 rmem->pg_tbl_mz = mz;
4674         }
4675
4676         snprintf(mz_name, RTE_MEMZONE_NAMESIZE, "bnxt_ctx_%s_%x_%d",
4677                  suffix, idx, bp->eth_dev->data->port_id);
4678         mz = rte_memzone_lookup(mz_name);
4679         if (!mz) {
4680                 mz = rte_memzone_reserve_aligned(mz_name,
4681                                                  mem_size,
4682                                                  SOCKET_ID_ANY,
4683                                                  RTE_MEMZONE_1GB |
4684                                                  RTE_MEMZONE_SIZE_HINT_ONLY |
4685                                                  RTE_MEMZONE_IOVA_CONTIG,
4686                                                  BNXT_PAGE_SIZE);
4687                 if (mz == NULL)
4688                         return -ENOMEM;
4689         }
4690
4691         memset(mz->addr, 0, mz->len);
4692         mz_phys_addr = mz->iova;
4693
4694         for (sz = 0, i = 0; sz < mem_size; sz += BNXT_PAGE_SIZE, i++) {
4695                 rmem->pg_arr[i] = ((char *)mz->addr) + sz;
4696                 rmem->dma_arr[i] = mz_phys_addr + sz;
4697
4698                 if (rmem->nr_pages > 1) {
4699                         if (i == rmem->nr_pages - 2 &&
4700                             (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
4701                                 valid_bits |= PTU_PTE_NEXT_TO_LAST;
4702                         else if (i == rmem->nr_pages - 1 &&
4703                                  (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
4704                                 valid_bits |= PTU_PTE_LAST;
4705
4706                         rmem->pg_tbl[i] = rte_cpu_to_le_64(rmem->dma_arr[i] |
4707                                                            valid_bits);
4708                 }
4709         }
4710
4711         rmem->mz = mz;
4712         if (rmem->vmem_size)
4713                 rmem->vmem = (void **)mz->addr;
4714         rmem->dma_arr[0] = mz_phys_addr;
4715         return 0;
4716 }
4717
4718 static void bnxt_free_ctx_mem(struct bnxt *bp)
4719 {
4720         int i;
4721
4722         if (!bp->ctx || !(bp->ctx->flags & BNXT_CTX_FLAG_INITED))
4723                 return;
4724
4725         bp->ctx->flags &= ~BNXT_CTX_FLAG_INITED;
4726         rte_memzone_free(bp->ctx->qp_mem.ring_mem.mz);
4727         rte_memzone_free(bp->ctx->srq_mem.ring_mem.mz);
4728         rte_memzone_free(bp->ctx->cq_mem.ring_mem.mz);
4729         rte_memzone_free(bp->ctx->vnic_mem.ring_mem.mz);
4730         rte_memzone_free(bp->ctx->stat_mem.ring_mem.mz);
4731         rte_memzone_free(bp->ctx->qp_mem.ring_mem.pg_tbl_mz);
4732         rte_memzone_free(bp->ctx->srq_mem.ring_mem.pg_tbl_mz);
4733         rte_memzone_free(bp->ctx->cq_mem.ring_mem.pg_tbl_mz);
4734         rte_memzone_free(bp->ctx->vnic_mem.ring_mem.pg_tbl_mz);
4735         rte_memzone_free(bp->ctx->stat_mem.ring_mem.pg_tbl_mz);
4736
4737         for (i = 0; i < bp->ctx->tqm_fp_rings_count + 1; i++) {
4738                 if (bp->ctx->tqm_mem[i])
4739                         rte_memzone_free(bp->ctx->tqm_mem[i]->ring_mem.mz);
4740         }
4741
4742         rte_free(bp->ctx);
4743         bp->ctx = NULL;
4744 }
4745
4746 #define bnxt_roundup(x, y)   ((((x) + ((y) - 1)) / (y)) * (y))
4747
4748 #define min_t(type, x, y) ({                    \
4749         type __min1 = (x);                      \
4750         type __min2 = (y);                      \
4751         __min1 < __min2 ? __min1 : __min2; })
4752
4753 #define max_t(type, x, y) ({                    \
4754         type __max1 = (x);                      \
4755         type __max2 = (y);                      \
4756         __max1 > __max2 ? __max1 : __max2; })
4757
4758 #define clamp_t(type, _x, min, max)     min_t(type, max_t(type, _x, min), max)
4759
4760 int bnxt_alloc_ctx_mem(struct bnxt *bp)
4761 {
4762         struct bnxt_ctx_pg_info *ctx_pg;
4763         struct bnxt_ctx_mem_info *ctx;
4764         uint32_t mem_size, ena, entries;
4765         uint32_t entries_sp, min;
4766         int i, rc;
4767
4768         rc = bnxt_hwrm_func_backing_store_qcaps(bp);
4769         if (rc) {
4770                 PMD_DRV_LOG(ERR, "Query context mem capability failed\n");
4771                 return rc;
4772         }
4773         ctx = bp->ctx;
4774         if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
4775                 return 0;
4776
4777         ctx_pg = &ctx->qp_mem;
4778         ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries;
4779         mem_size = ctx->qp_entry_size * ctx_pg->entries;
4780         rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size, "qp_mem", 0);
4781         if (rc)
4782                 return rc;
4783
4784         ctx_pg = &ctx->srq_mem;
4785         ctx_pg->entries = ctx->srq_max_l2_entries;
4786         mem_size = ctx->srq_entry_size * ctx_pg->entries;
4787         rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size, "srq_mem", 0);
4788         if (rc)
4789                 return rc;
4790
4791         ctx_pg = &ctx->cq_mem;
4792         ctx_pg->entries = ctx->cq_max_l2_entries;
4793         mem_size = ctx->cq_entry_size * ctx_pg->entries;
4794         rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size, "cq_mem", 0);
4795         if (rc)
4796                 return rc;
4797
4798         ctx_pg = &ctx->vnic_mem;
4799         ctx_pg->entries = ctx->vnic_max_vnic_entries +
4800                 ctx->vnic_max_ring_table_entries;
4801         mem_size = ctx->vnic_entry_size * ctx_pg->entries;
4802         rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size, "vnic_mem", 0);
4803         if (rc)
4804                 return rc;
4805
4806         ctx_pg = &ctx->stat_mem;
4807         ctx_pg->entries = ctx->stat_max_entries;
4808         mem_size = ctx->stat_entry_size * ctx_pg->entries;
4809         rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size, "stat_mem", 0);
4810         if (rc)
4811                 return rc;
4812
4813         min = ctx->tqm_min_entries_per_ring;
4814
4815         entries_sp = ctx->qp_max_l2_entries +
4816                      ctx->vnic_max_vnic_entries +
4817                      2 * ctx->qp_min_qp1_entries + min;
4818         entries_sp = bnxt_roundup(entries_sp, ctx->tqm_entries_multiple);
4819
4820         entries = ctx->qp_max_l2_entries + ctx->qp_min_qp1_entries;
4821         entries = bnxt_roundup(entries, ctx->tqm_entries_multiple);
4822         entries = clamp_t(uint32_t, entries, min,
4823                           ctx->tqm_max_entries_per_ring);
4824         for (i = 0, ena = 0; i < ctx->tqm_fp_rings_count + 1; i++) {
4825                 ctx_pg = ctx->tqm_mem[i];
4826                 ctx_pg->entries = i ? entries : entries_sp;
4827                 mem_size = ctx->tqm_entry_size * ctx_pg->entries;
4828                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size, "tqm_mem", i);
4829                 if (rc)
4830                         return rc;
4831                 ena |= HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_TQM_SP << i;
4832         }
4833
4834         ena |= FUNC_BACKING_STORE_CFG_INPUT_DFLT_ENABLES;
4835         rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
4836         if (rc)
4837                 PMD_DRV_LOG(ERR,
4838                             "Failed to configure context mem: rc = %d\n", rc);
4839         else
4840                 ctx->flags |= BNXT_CTX_FLAG_INITED;
4841
4842         return rc;
4843 }
4844
4845 static int bnxt_alloc_stats_mem(struct bnxt *bp)
4846 {
4847         struct rte_pci_device *pci_dev = bp->pdev;
4848         char mz_name[RTE_MEMZONE_NAMESIZE];
4849         const struct rte_memzone *mz = NULL;
4850         uint32_t total_alloc_len;
4851         rte_iova_t mz_phys_addr;
4852
4853         if (pci_dev->id.device_id == BROADCOM_DEV_ID_NS2)
4854                 return 0;
4855
4856         snprintf(mz_name, RTE_MEMZONE_NAMESIZE,
4857                  "bnxt_" PCI_PRI_FMT "-%s", pci_dev->addr.domain,
4858                  pci_dev->addr.bus, pci_dev->addr.devid,
4859                  pci_dev->addr.function, "rx_port_stats");
4860         mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0;
4861         mz = rte_memzone_lookup(mz_name);
4862         total_alloc_len =
4863                 RTE_CACHE_LINE_ROUNDUP(sizeof(struct rx_port_stats) +
4864                                        sizeof(struct rx_port_stats_ext) + 512);
4865         if (!mz) {
4866                 mz = rte_memzone_reserve(mz_name, total_alloc_len,
4867                                          SOCKET_ID_ANY,
4868                                          RTE_MEMZONE_2MB |
4869                                          RTE_MEMZONE_SIZE_HINT_ONLY |
4870                                          RTE_MEMZONE_IOVA_CONTIG);
4871                 if (mz == NULL)
4872                         return -ENOMEM;
4873         }
4874         memset(mz->addr, 0, mz->len);
4875         mz_phys_addr = mz->iova;
4876
4877         bp->rx_mem_zone = (const void *)mz;
4878         bp->hw_rx_port_stats = mz->addr;
4879         bp->hw_rx_port_stats_map = mz_phys_addr;
4880
4881         snprintf(mz_name, RTE_MEMZONE_NAMESIZE,
4882                  "bnxt_" PCI_PRI_FMT "-%s", pci_dev->addr.domain,
4883                  pci_dev->addr.bus, pci_dev->addr.devid,
4884                  pci_dev->addr.function, "tx_port_stats");
4885         mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0;
4886         mz = rte_memzone_lookup(mz_name);
4887         total_alloc_len =
4888                 RTE_CACHE_LINE_ROUNDUP(sizeof(struct tx_port_stats) +
4889                                        sizeof(struct tx_port_stats_ext) + 512);
4890         if (!mz) {
4891                 mz = rte_memzone_reserve(mz_name,
4892                                          total_alloc_len,
4893                                          SOCKET_ID_ANY,
4894                                          RTE_MEMZONE_2MB |
4895                                          RTE_MEMZONE_SIZE_HINT_ONLY |
4896                                          RTE_MEMZONE_IOVA_CONTIG);
4897                 if (mz == NULL)
4898                         return -ENOMEM;
4899         }
4900         memset(mz->addr, 0, mz->len);
4901         mz_phys_addr = mz->iova;
4902
4903         bp->tx_mem_zone = (const void *)mz;
4904         bp->hw_tx_port_stats = mz->addr;
4905         bp->hw_tx_port_stats_map = mz_phys_addr;
4906         bp->flags |= BNXT_FLAG_PORT_STATS;
4907
4908         /* Display extended statistics if FW supports it */
4909         if (bp->hwrm_spec_code < HWRM_SPEC_CODE_1_8_4 ||
4910             bp->hwrm_spec_code == HWRM_SPEC_CODE_1_9_0 ||
4911             !(bp->flags & BNXT_FLAG_EXT_STATS_SUPPORTED))
4912                 return 0;
4913
4914         bp->hw_rx_port_stats_ext = (void *)
4915                 ((uint8_t *)bp->hw_rx_port_stats +
4916                  sizeof(struct rx_port_stats));
4917         bp->hw_rx_port_stats_ext_map = bp->hw_rx_port_stats_map +
4918                 sizeof(struct rx_port_stats);
4919         bp->flags |= BNXT_FLAG_EXT_RX_PORT_STATS;
4920
4921         if (bp->hwrm_spec_code < HWRM_SPEC_CODE_1_9_2 ||
4922             bp->flags & BNXT_FLAG_EXT_STATS_SUPPORTED) {
4923                 bp->hw_tx_port_stats_ext = (void *)
4924                         ((uint8_t *)bp->hw_tx_port_stats +
4925                          sizeof(struct tx_port_stats));
4926                 bp->hw_tx_port_stats_ext_map =
4927                         bp->hw_tx_port_stats_map +
4928                         sizeof(struct tx_port_stats);
4929                 bp->flags |= BNXT_FLAG_EXT_TX_PORT_STATS;
4930         }
4931
4932         return 0;
4933 }
4934
4935 static int bnxt_setup_mac_addr(struct rte_eth_dev *eth_dev)
4936 {
4937         struct bnxt *bp = eth_dev->data->dev_private;
4938         int rc = 0;
4939
4940         eth_dev->data->mac_addrs = rte_zmalloc("bnxt_mac_addr_tbl",
4941                                                RTE_ETHER_ADDR_LEN *
4942                                                bp->max_l2_ctx,
4943                                                0);
4944         if (eth_dev->data->mac_addrs == NULL) {
4945                 PMD_DRV_LOG(ERR, "Failed to alloc MAC addr tbl\n");
4946                 return -ENOMEM;
4947         }
4948
4949         if (bnxt_check_zero_bytes(bp->dflt_mac_addr, RTE_ETHER_ADDR_LEN)) {
4950                 if (BNXT_PF(bp))
4951                         return -EINVAL;
4952
4953                 /* Generate a random MAC address, if none was assigned by PF */
4954                 PMD_DRV_LOG(INFO, "VF MAC address not assigned by Host PF\n");
4955                 bnxt_eth_hw_addr_random(bp->mac_addr);
4956                 PMD_DRV_LOG(INFO,
4957                             "Assign random MAC:%02X:%02X:%02X:%02X:%02X:%02X\n",
4958                             bp->mac_addr[0], bp->mac_addr[1], bp->mac_addr[2],
4959                             bp->mac_addr[3], bp->mac_addr[4], bp->mac_addr[5]);
4960
4961                 rc = bnxt_hwrm_set_mac(bp);
4962                 if (!rc)
4963                         memcpy(&bp->eth_dev->data->mac_addrs[0], bp->mac_addr,
4964                                RTE_ETHER_ADDR_LEN);
4965                 return rc;
4966         }
4967
4968         /* Copy the permanent MAC from the FUNC_QCAPS response */
4969         memcpy(bp->mac_addr, bp->dflt_mac_addr, RTE_ETHER_ADDR_LEN);
4970         memcpy(&eth_dev->data->mac_addrs[0], bp->mac_addr, RTE_ETHER_ADDR_LEN);
4971
4972         return rc;
4973 }
4974
4975 static int bnxt_restore_dflt_mac(struct bnxt *bp)
4976 {
4977         int rc = 0;
4978
4979         /* MAC is already configured in FW */
4980         if (!bnxt_check_zero_bytes(bp->dflt_mac_addr, RTE_ETHER_ADDR_LEN))
4981                 return 0;
4982
4983         /* Restore the old MAC configured */
4984         rc = bnxt_hwrm_set_mac(bp);
4985         if (rc)
4986                 PMD_DRV_LOG(ERR, "Failed to restore MAC address\n");
4987
4988         return rc;
4989 }
4990
4991 static void bnxt_config_vf_req_fwd(struct bnxt *bp)
4992 {
4993         if (!BNXT_PF(bp))
4994                 return;
4995
4996 #define ALLOW_FUNC(x)   \
4997         { \
4998                 uint32_t arg = (x); \
4999                 bp->pf.vf_req_fwd[((arg) >> 5)] &= \
5000                 ~rte_cpu_to_le_32(1 << ((arg) & 0x1f)); \
5001         }
5002
5003         /* Forward all requests if firmware is new enough */
5004         if (((bp->fw_ver >= ((20 << 24) | (6 << 16) | (100 << 8))) &&
5005              (bp->fw_ver < ((20 << 24) | (7 << 16)))) ||
5006             ((bp->fw_ver >= ((20 << 24) | (8 << 16))))) {
5007                 memset(bp->pf.vf_req_fwd, 0xff, sizeof(bp->pf.vf_req_fwd));
5008         } else {
5009                 PMD_DRV_LOG(WARNING,
5010                             "Firmware too old for VF mailbox functionality\n");
5011                 memset(bp->pf.vf_req_fwd, 0, sizeof(bp->pf.vf_req_fwd));
5012         }
5013
5014         /*
5015          * The following are used for driver cleanup. If we disallow these,
5016          * VF drivers can't clean up cleanly.
5017          */
5018         ALLOW_FUNC(HWRM_FUNC_DRV_UNRGTR);
5019         ALLOW_FUNC(HWRM_VNIC_FREE);
5020         ALLOW_FUNC(HWRM_RING_FREE);
5021         ALLOW_FUNC(HWRM_RING_GRP_FREE);
5022         ALLOW_FUNC(HWRM_VNIC_RSS_COS_LB_CTX_FREE);
5023         ALLOW_FUNC(HWRM_CFA_L2_FILTER_FREE);
5024         ALLOW_FUNC(HWRM_STAT_CTX_FREE);
5025         ALLOW_FUNC(HWRM_PORT_PHY_QCFG);
5026         ALLOW_FUNC(HWRM_VNIC_TPA_CFG);
5027 }
5028
5029 uint16_t
5030 bnxt_get_svif(uint16_t port_id, bool func_svif)
5031 {
5032         struct rte_eth_dev *eth_dev;
5033         struct bnxt *bp;
5034
5035         eth_dev = &rte_eth_devices[port_id];
5036         bp = eth_dev->data->dev_private;
5037
5038         return func_svif ? bp->func_svif : bp->port_svif;
5039 }
5040
5041 uint16_t
5042 bnxt_get_vnic_id(uint16_t port)
5043 {
5044         struct rte_eth_dev *eth_dev;
5045         struct bnxt_vnic_info *vnic;
5046         struct bnxt *bp;
5047
5048         eth_dev = &rte_eth_devices[port];
5049         bp = eth_dev->data->dev_private;
5050
5051         vnic = BNXT_GET_DEFAULT_VNIC(bp);
5052
5053         return vnic->fw_vnic_id;
5054 }
5055
5056 uint16_t
5057 bnxt_get_fw_func_id(uint16_t port)
5058 {
5059         struct rte_eth_dev *eth_dev;
5060         struct bnxt *bp;
5061
5062         eth_dev = &rte_eth_devices[port];
5063         bp = eth_dev->data->dev_private;
5064
5065         return bp->fw_fid;
5066 }
5067
5068 static void bnxt_alloc_error_recovery_info(struct bnxt *bp)
5069 {
5070         struct bnxt_error_recovery_info *info = bp->recovery_info;
5071
5072         if (info) {
5073                 if (!(bp->fw_cap & BNXT_FW_CAP_HCOMM_FW_STATUS))
5074                         memset(info, 0, sizeof(*info));
5075                 return;
5076         }
5077
5078         if (!(bp->fw_cap & BNXT_FW_CAP_ERROR_RECOVERY))
5079                 return;
5080
5081         info = rte_zmalloc("bnxt_hwrm_error_recovery_qcfg",
5082                            sizeof(*info), 0);
5083         if (!info)
5084                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
5085
5086         bp->recovery_info = info;
5087 }
5088
5089 static void bnxt_check_fw_status(struct bnxt *bp)
5090 {
5091         uint32_t fw_status;
5092
5093         if (!(bp->recovery_info &&
5094               (bp->fw_cap & BNXT_FW_CAP_HCOMM_FW_STATUS)))
5095                 return;
5096
5097         fw_status = bnxt_read_fw_status_reg(bp, BNXT_FW_STATUS_REG);
5098         if (fw_status != BNXT_FW_STATUS_HEALTHY)
5099                 PMD_DRV_LOG(ERR, "Firmware not responding, status: %#x\n",
5100                             fw_status);
5101 }
5102
5103 static int bnxt_map_hcomm_fw_status_reg(struct bnxt *bp)
5104 {
5105         struct bnxt_error_recovery_info *info = bp->recovery_info;
5106         uint32_t status_loc;
5107         uint32_t sig_ver;
5108
5109         rte_write32(HCOMM_STATUS_STRUCT_LOC, (uint8_t *)bp->bar0 +
5110                     BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
5111         sig_ver = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
5112                                    BNXT_GRCP_WINDOW_2_BASE +
5113                                    offsetof(struct hcomm_status,
5114                                             sig_ver)));
5115         /* If the signature is absent, then FW does not support this feature */
5116         if ((sig_ver & HCOMM_STATUS_SIGNATURE_MASK) !=
5117             HCOMM_STATUS_SIGNATURE_VAL)
5118                 return 0;
5119
5120         if (!info) {
5121                 info = rte_zmalloc("bnxt_hwrm_error_recovery_qcfg",
5122                                    sizeof(*info), 0);
5123                 if (!info)
5124                         return -ENOMEM;
5125                 bp->recovery_info = info;
5126         } else {
5127                 memset(info, 0, sizeof(*info));
5128         }
5129
5130         status_loc = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
5131                                       BNXT_GRCP_WINDOW_2_BASE +
5132                                       offsetof(struct hcomm_status,
5133                                                fw_status_loc)));
5134
5135         /* Only pre-map the FW health status GRC register */
5136         if (BNXT_FW_STATUS_REG_TYPE(status_loc) != BNXT_FW_STATUS_REG_TYPE_GRC)
5137                 return 0;
5138
5139         info->status_regs[BNXT_FW_STATUS_REG] = status_loc;
5140         info->mapped_status_regs[BNXT_FW_STATUS_REG] =
5141                 BNXT_GRCP_WINDOW_2_BASE + (status_loc & BNXT_GRCP_OFFSET_MASK);
5142
5143         rte_write32((status_loc & BNXT_GRCP_BASE_MASK), (uint8_t *)bp->bar0 +
5144                     BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
5145
5146         bp->fw_cap |= BNXT_FW_CAP_HCOMM_FW_STATUS;
5147
5148         return 0;
5149 }
5150
5151 static int bnxt_init_fw(struct bnxt *bp)
5152 {
5153         uint16_t mtu;
5154         int rc = 0;
5155
5156         bp->fw_cap = 0;
5157
5158         rc = bnxt_map_hcomm_fw_status_reg(bp);
5159         if (rc)
5160                 return rc;
5161
5162         rc = bnxt_hwrm_ver_get(bp, DFLT_HWRM_CMD_TIMEOUT);
5163         if (rc) {
5164                 bnxt_check_fw_status(bp);
5165                 return rc;
5166         }
5167
5168         rc = bnxt_hwrm_func_reset(bp);
5169         if (rc)
5170                 return -EIO;
5171
5172         rc = bnxt_hwrm_vnic_qcaps(bp);
5173         if (rc)
5174                 return rc;
5175
5176         rc = bnxt_hwrm_queue_qportcfg(bp);
5177         if (rc)
5178                 return rc;
5179
5180         /* Get the MAX capabilities for this function.
5181          * This function also allocates context memory for TQM rings and
5182          * informs the firmware about this allocated backing store memory.
5183          */
5184         rc = bnxt_hwrm_func_qcaps(bp);
5185         if (rc)
5186                 return rc;
5187
5188         rc = bnxt_hwrm_func_qcfg(bp, &mtu);
5189         if (rc)
5190                 return rc;
5191
5192         bnxt_hwrm_port_mac_qcfg(bp);
5193
5194         rc = bnxt_hwrm_cfa_adv_flow_mgmt_qcaps(bp);
5195         if (rc)
5196                 return rc;
5197
5198         bnxt_alloc_error_recovery_info(bp);
5199         /* Get the adapter error recovery support info */
5200         rc = bnxt_hwrm_error_recovery_qcfg(bp);
5201         if (rc)
5202                 bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
5203
5204         bnxt_hwrm_port_led_qcaps(bp);
5205
5206         return 0;
5207 }
5208
5209 static int
5210 bnxt_init_locks(struct bnxt *bp)
5211 {
5212         int err;
5213
5214         err = pthread_mutex_init(&bp->flow_lock, NULL);
5215         if (err) {
5216                 PMD_DRV_LOG(ERR, "Unable to initialize flow_lock\n");
5217                 return err;
5218         }
5219
5220         err = pthread_mutex_init(&bp->def_cp_lock, NULL);
5221         if (err)
5222                 PMD_DRV_LOG(ERR, "Unable to initialize def_cp_lock\n");
5223         return err;
5224 }
5225
5226 static int bnxt_init_resources(struct bnxt *bp, bool reconfig_dev)
5227 {
5228         int rc;
5229
5230         rc = bnxt_init_fw(bp);
5231         if (rc)
5232                 return rc;
5233
5234         if (!reconfig_dev) {
5235                 rc = bnxt_setup_mac_addr(bp->eth_dev);
5236                 if (rc)
5237                         return rc;
5238         } else {
5239                 rc = bnxt_restore_dflt_mac(bp);
5240                 if (rc)
5241                         return rc;
5242         }
5243
5244         bnxt_config_vf_req_fwd(bp);
5245
5246         rc = bnxt_hwrm_func_driver_register(bp);
5247         if (rc) {
5248                 PMD_DRV_LOG(ERR, "Failed to register driver");
5249                 return -EBUSY;
5250         }
5251
5252         if (BNXT_PF(bp)) {
5253                 if (bp->pdev->max_vfs) {
5254                         rc = bnxt_hwrm_allocate_vfs(bp, bp->pdev->max_vfs);
5255                         if (rc) {
5256                                 PMD_DRV_LOG(ERR, "Failed to allocate VFs\n");
5257                                 return rc;
5258                         }
5259                 } else {
5260                         rc = bnxt_hwrm_allocate_pf_only(bp);
5261                         if (rc) {
5262                                 PMD_DRV_LOG(ERR,
5263                                             "Failed to allocate PF resources");
5264                                 return rc;
5265                         }
5266                 }
5267         }
5268
5269         rc = bnxt_alloc_mem(bp, reconfig_dev);
5270         if (rc)
5271                 return rc;
5272
5273         rc = bnxt_setup_int(bp);
5274         if (rc)
5275                 return rc;
5276
5277         rc = bnxt_request_int(bp);
5278         if (rc)
5279                 return rc;
5280
5281         rc = bnxt_init_ctx_mem(bp);
5282         if (rc) {
5283                 PMD_DRV_LOG(ERR, "Failed to init adv_flow_counters\n");
5284                 return rc;
5285         }
5286
5287         rc = bnxt_init_locks(bp);
5288         if (rc)
5289                 return rc;
5290
5291         return 0;
5292 }
5293
5294 static int
5295 bnxt_parse_devarg_truflow(__rte_unused const char *key,
5296                           const char *value, void *opaque_arg)
5297 {
5298         struct bnxt *bp = opaque_arg;
5299         unsigned long truflow;
5300         char *end = NULL;
5301
5302         if (!value || !opaque_arg) {
5303                 PMD_DRV_LOG(ERR,
5304                             "Invalid parameter passed to truflow devargs.\n");
5305                 return -EINVAL;
5306         }
5307
5308         truflow = strtoul(value, &end, 10);
5309         if (end == NULL || *end != '\0' ||
5310             (truflow == ULONG_MAX && errno == ERANGE)) {
5311                 PMD_DRV_LOG(ERR,
5312                             "Invalid parameter passed to truflow devargs.\n");
5313                 return -EINVAL;
5314         }
5315
5316         if (BNXT_DEVARG_TRUFLOW_INVALID(truflow)) {
5317                 PMD_DRV_LOG(ERR,
5318                             "Invalid value passed to truflow devargs.\n");
5319                 return -EINVAL;
5320         }
5321
5322         bp->truflow = truflow;
5323         if (bp->truflow)
5324                 PMD_DRV_LOG(INFO, "Host-based truflow feature enabled.\n");
5325
5326         return 0;
5327 }
5328
5329 static int
5330 bnxt_parse_devarg_flow_xstat(__rte_unused const char *key,
5331                              const char *value, void *opaque_arg)
5332 {
5333         struct bnxt *bp = opaque_arg;
5334         unsigned long flow_xstat;
5335         char *end = NULL;
5336
5337         if (!value || !opaque_arg) {
5338                 PMD_DRV_LOG(ERR,
5339                             "Invalid parameter passed to flow_xstat devarg.\n");
5340                 return -EINVAL;
5341         }
5342
5343         flow_xstat = strtoul(value, &end, 10);
5344         if (end == NULL || *end != '\0' ||
5345             (flow_xstat == ULONG_MAX && errno == ERANGE)) {
5346                 PMD_DRV_LOG(ERR,
5347                             "Invalid parameter passed to flow_xstat devarg.\n");
5348                 return -EINVAL;
5349         }
5350
5351         if (BNXT_DEVARG_FLOW_XSTAT_INVALID(flow_xstat)) {
5352                 PMD_DRV_LOG(ERR,
5353                             "Invalid value passed to flow_xstat devarg.\n");
5354                 return -EINVAL;
5355         }
5356
5357         bp->flags |= BNXT_FLAG_FLOW_XSTATS_EN;
5358         if (BNXT_FLOW_XSTATS_EN(bp))
5359                 PMD_DRV_LOG(INFO, "flow_xstat feature enabled.\n");
5360
5361         return 0;
5362 }
5363
5364 static void
5365 bnxt_parse_dev_args(struct bnxt *bp, struct rte_devargs *devargs)
5366 {
5367         struct rte_kvargs *kvlist;
5368
5369         if (devargs == NULL)
5370                 return;
5371
5372         kvlist = rte_kvargs_parse(devargs->args, bnxt_dev_args);
5373         if (kvlist == NULL)
5374                 return;
5375
5376         /*
5377          * Handler for "truflow" devarg.
5378          * Invoked as for ex: "-w 0000:00:0d.0,host-based-truflow=1”
5379          */
5380         rte_kvargs_process(kvlist, BNXT_DEVARG_TRUFLOW,
5381                            bnxt_parse_devarg_truflow, bp);
5382
5383         /*
5384          * Handler for "flow_xstat" devarg.
5385          * Invoked as for ex: "-w 0000:00:0d.0,flow_xstat=1”
5386          */
5387         rte_kvargs_process(kvlist, BNXT_DEVARG_FLOW_XSTAT,
5388                            bnxt_parse_devarg_flow_xstat, bp);
5389
5390         rte_kvargs_free(kvlist);
5391 }
5392
5393 static int
5394 bnxt_dev_init(struct rte_eth_dev *eth_dev)
5395 {
5396         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
5397         static int version_printed;
5398         struct bnxt *bp;
5399         int rc;
5400
5401         if (version_printed++ == 0)
5402                 PMD_DRV_LOG(INFO, "%s\n", bnxt_version);
5403
5404         eth_dev->dev_ops = &bnxt_dev_ops;
5405         eth_dev->rx_pkt_burst = &bnxt_recv_pkts;
5406         eth_dev->tx_pkt_burst = &bnxt_xmit_pkts;
5407
5408         /*
5409          * For secondary processes, we don't initialise any further
5410          * as primary has already done this work.
5411          */
5412         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
5413                 return 0;
5414
5415         rte_eth_copy_pci_info(eth_dev, pci_dev);
5416
5417         bp = eth_dev->data->dev_private;
5418
5419         /* Parse dev arguments passed on when starting the DPDK application. */
5420         bnxt_parse_dev_args(bp, pci_dev->device.devargs);
5421
5422         bp->flags &= ~BNXT_FLAG_RX_VECTOR_PKT_MODE;
5423
5424         if (bnxt_vf_pciid(pci_dev->id.device_id))
5425                 bp->flags |= BNXT_FLAG_VF;
5426
5427         if (bnxt_thor_device(pci_dev->id.device_id))
5428                 bp->flags |= BNXT_FLAG_THOR_CHIP;
5429
5430         if (pci_dev->id.device_id == BROADCOM_DEV_ID_58802 ||
5431             pci_dev->id.device_id == BROADCOM_DEV_ID_58804 ||
5432             pci_dev->id.device_id == BROADCOM_DEV_ID_58808 ||
5433             pci_dev->id.device_id == BROADCOM_DEV_ID_58802_VF)
5434                 bp->flags |= BNXT_FLAG_STINGRAY;
5435
5436         rc = bnxt_init_board(eth_dev);
5437         if (rc) {
5438                 PMD_DRV_LOG(ERR,
5439                             "Failed to initialize board rc: %x\n", rc);
5440                 return rc;
5441         }
5442
5443         rc = bnxt_alloc_hwrm_resources(bp);
5444         if (rc) {
5445                 PMD_DRV_LOG(ERR,
5446                             "Failed to allocate hwrm resource rc: %x\n", rc);
5447                 goto error_free;
5448         }
5449         rc = bnxt_alloc_leds_info(bp);
5450         if (rc)
5451                 goto error_free;
5452
5453         rc = bnxt_alloc_cos_queues(bp);
5454         if (rc)
5455                 goto error_free;
5456
5457         rc = bnxt_init_resources(bp, false);
5458         if (rc)
5459                 goto error_free;
5460
5461         rc = bnxt_alloc_stats_mem(bp);
5462         if (rc)
5463                 goto error_free;
5464
5465         /* Pass the information to the rte_eth_dev_close() that it should also
5466          * release the private port resources.
5467          */
5468         eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
5469
5470         PMD_DRV_LOG(INFO,
5471                     DRV_MODULE_NAME "found at mem %" PRIX64 ", node addr %pM\n",
5472                     pci_dev->mem_resource[0].phys_addr,
5473                     pci_dev->mem_resource[0].addr);
5474
5475         return 0;
5476
5477 error_free:
5478         bnxt_dev_uninit(eth_dev);
5479         return rc;
5480 }
5481
5482
5483 static void bnxt_free_ctx_mem_buf(struct bnxt_ctx_mem_buf_info *ctx)
5484 {
5485         if (!ctx)
5486                 return;
5487
5488         if (ctx->va)
5489                 rte_free(ctx->va);
5490
5491         ctx->va = NULL;
5492         ctx->dma = RTE_BAD_IOVA;
5493         ctx->ctx_id = BNXT_CTX_VAL_INVAL;
5494 }
5495
5496 static void bnxt_unregister_fc_ctx_mem(struct bnxt *bp)
5497 {
5498         bnxt_hwrm_cfa_counter_cfg(bp, BNXT_DIR_RX,
5499                                   CFA_COUNTER_CFG_IN_COUNTER_TYPE_FC,
5500                                   bp->flow_stat->rx_fc_out_tbl.ctx_id,
5501                                   bp->flow_stat->max_fc,
5502                                   false);
5503
5504         bnxt_hwrm_cfa_counter_cfg(bp, BNXT_DIR_TX,
5505                                   CFA_COUNTER_CFG_IN_COUNTER_TYPE_FC,
5506                                   bp->flow_stat->tx_fc_out_tbl.ctx_id,
5507                                   bp->flow_stat->max_fc,
5508                                   false);
5509
5510         if (bp->flow_stat->rx_fc_in_tbl.ctx_id != BNXT_CTX_VAL_INVAL)
5511                 bnxt_hwrm_ctx_unrgtr(bp, bp->flow_stat->rx_fc_in_tbl.ctx_id);
5512         bp->flow_stat->rx_fc_in_tbl.ctx_id = BNXT_CTX_VAL_INVAL;
5513
5514         if (bp->flow_stat->rx_fc_out_tbl.ctx_id != BNXT_CTX_VAL_INVAL)
5515                 bnxt_hwrm_ctx_unrgtr(bp, bp->flow_stat->rx_fc_out_tbl.ctx_id);
5516         bp->flow_stat->rx_fc_out_tbl.ctx_id = BNXT_CTX_VAL_INVAL;
5517
5518         if (bp->flow_stat->tx_fc_in_tbl.ctx_id != BNXT_CTX_VAL_INVAL)
5519                 bnxt_hwrm_ctx_unrgtr(bp, bp->flow_stat->tx_fc_in_tbl.ctx_id);
5520         bp->flow_stat->tx_fc_in_tbl.ctx_id = BNXT_CTX_VAL_INVAL;
5521
5522         if (bp->flow_stat->tx_fc_out_tbl.ctx_id != BNXT_CTX_VAL_INVAL)
5523                 bnxt_hwrm_ctx_unrgtr(bp, bp->flow_stat->tx_fc_out_tbl.ctx_id);
5524         bp->flow_stat->tx_fc_out_tbl.ctx_id = BNXT_CTX_VAL_INVAL;
5525 }
5526
5527 static void bnxt_uninit_fc_ctx_mem(struct bnxt *bp)
5528 {
5529         bnxt_unregister_fc_ctx_mem(bp);
5530
5531         bnxt_free_ctx_mem_buf(&bp->flow_stat->rx_fc_in_tbl);
5532         bnxt_free_ctx_mem_buf(&bp->flow_stat->rx_fc_out_tbl);
5533         bnxt_free_ctx_mem_buf(&bp->flow_stat->tx_fc_in_tbl);
5534         bnxt_free_ctx_mem_buf(&bp->flow_stat->tx_fc_out_tbl);
5535 }
5536
5537 static void bnxt_uninit_ctx_mem(struct bnxt *bp)
5538 {
5539         if (BNXT_FLOW_XSTATS_EN(bp))
5540                 bnxt_uninit_fc_ctx_mem(bp);
5541 }
5542
5543 static void
5544 bnxt_free_error_recovery_info(struct bnxt *bp)
5545 {
5546         rte_free(bp->recovery_info);
5547         bp->recovery_info = NULL;
5548         bp->fw_cap &= ~BNXT_FW_CAP_ERROR_RECOVERY;
5549 }
5550
5551 static void
5552 bnxt_uninit_locks(struct bnxt *bp)
5553 {
5554         pthread_mutex_destroy(&bp->flow_lock);
5555         pthread_mutex_destroy(&bp->def_cp_lock);
5556 }
5557
5558 static int
5559 bnxt_uninit_resources(struct bnxt *bp, bool reconfig_dev)
5560 {
5561         int rc;
5562
5563         bnxt_free_int(bp);
5564         bnxt_free_mem(bp, reconfig_dev);
5565         bnxt_hwrm_func_buf_unrgtr(bp);
5566         rc = bnxt_hwrm_func_driver_unregister(bp, 0);
5567         bp->flags &= ~BNXT_FLAG_REGISTERED;
5568         bnxt_free_ctx_mem(bp);
5569         if (!reconfig_dev) {
5570                 bnxt_free_hwrm_resources(bp);
5571                 bnxt_free_error_recovery_info(bp);
5572         }
5573
5574         bnxt_uninit_ctx_mem(bp);
5575
5576         bnxt_uninit_locks(bp);
5577         rte_free(bp->ptp_cfg);
5578         bp->ptp_cfg = NULL;
5579         return rc;
5580 }
5581
5582 static int
5583 bnxt_dev_uninit(struct rte_eth_dev *eth_dev)
5584 {
5585         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
5586                 return -EPERM;
5587
5588         PMD_DRV_LOG(DEBUG, "Calling Device uninit\n");
5589
5590         if (eth_dev->state != RTE_ETH_DEV_UNUSED)
5591                 bnxt_dev_close_op(eth_dev);
5592
5593         return 0;
5594 }
5595
5596 static int bnxt_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
5597         struct rte_pci_device *pci_dev)
5598 {
5599         return rte_eth_dev_pci_generic_probe(pci_dev, sizeof(struct bnxt),
5600                 bnxt_dev_init);
5601 }
5602
5603 static int bnxt_pci_remove(struct rte_pci_device *pci_dev)
5604 {
5605         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
5606                 return rte_eth_dev_pci_generic_remove(pci_dev,
5607                                 bnxt_dev_uninit);
5608         else
5609                 return rte_eth_dev_pci_generic_remove(pci_dev, NULL);
5610 }
5611
5612 static struct rte_pci_driver bnxt_rte_pmd = {
5613         .id_table = bnxt_pci_id_map,
5614         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
5615         .probe = bnxt_pci_probe,
5616         .remove = bnxt_pci_remove,
5617 };
5618
5619 static bool
5620 is_device_supported(struct rte_eth_dev *dev, struct rte_pci_driver *drv)
5621 {
5622         if (strcmp(dev->device->driver->name, drv->driver.name))
5623                 return false;
5624
5625         return true;
5626 }
5627
5628 bool is_bnxt_supported(struct rte_eth_dev *dev)
5629 {
5630         return is_device_supported(dev, &bnxt_rte_pmd);
5631 }
5632
5633 RTE_INIT(bnxt_init_log)
5634 {
5635         bnxt_logtype_driver = rte_log_register("pmd.net.bnxt.driver");
5636         if (bnxt_logtype_driver >= 0)
5637                 rte_log_set_level(bnxt_logtype_driver, RTE_LOG_NOTICE);
5638 }
5639
5640 RTE_PMD_REGISTER_PCI(net_bnxt, bnxt_rte_pmd);
5641 RTE_PMD_REGISTER_PCI_TABLE(net_bnxt, bnxt_pci_id_map);
5642 RTE_PMD_REGISTER_KMOD_DEP(net_bnxt, "* igb_uio | uio_pci_generic | vfio-pci");