drivers/net/bnxt/bnxt_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2014-2018 Broadcom
   3  * All rights reserved.
   4  */
   5
   6 #include <inttypes.h>
   7 #include <stdbool.h>
   8
   9 #include <rte_dev.h>
  10 #include <rte_ethdev_driver.h>
  11 #include <rte_ethdev_pci.h>
  12 #include <rte_malloc.h>
  13 #include <rte_cycles.h>
  14 #include <rte_alarm.h>
  15
  16 #include "bnxt.h"
  17 #include "bnxt_filter.h"
  18 #include "bnxt_hwrm.h"
  19 #include "bnxt_irq.h"
  20 #include "bnxt_ring.h"
  21 #include "bnxt_rxq.h"
  22 #include "bnxt_rxr.h"
  23 #include "bnxt_stats.h"
  24 #include "bnxt_txq.h"
  25 #include "bnxt_txr.h"
  26 #include "bnxt_vnic.h"
  27 #include "hsi_struct_def_dpdk.h"
  28 #include "bnxt_nvm_defs.h"
  29
  30 #define DRV_MODULE_NAME         "bnxt"
  31 static const char bnxt_version[] =
  32         "Broadcom NetXtreme driver " DRV_MODULE_NAME;
  33 int bnxt_logtype_driver;
  34
  35 /*
  36  * The set of PCI devices this driver supports
  37  */
  38 static const struct rte_pci_id bnxt_pci_id_map[] = {
  39         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM,
  40                          BROADCOM_DEV_ID_STRATUS_NIC_VF1) },
  41         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM,
  42                          BROADCOM_DEV_ID_STRATUS_NIC_VF2) },
  43         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_STRATUS_NIC) },
  44         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57414_VF) },
  45         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57301) },
  46         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57302) },
  47         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57304_PF) },
  48         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57304_VF) },
  49         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_NS2) },
  50         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57402) },
  51         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57404) },
  52         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57406_PF) },
  53         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57406_VF) },
  54         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57402_MF) },
  55         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57407_RJ45) },
  56         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57404_MF) },
  57         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57406_MF) },
  58         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57407_SFP) },
  59         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57407_MF) },
  60         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_5741X_VF) },
  61         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_5731X_VF) },
  62         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57314) },
  63         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57417_MF) },
  64         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57311) },
  65         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57312) },
  66         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57412) },
  67         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57414) },
  68         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57416_RJ45) },
  69         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57417_RJ45) },
  70         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57412_MF) },
  71         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57317_RJ45) },
  72         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57417_SFP) },
  73         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57416_SFP) },
  74         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57317_SFP) },
  75         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57414_MF) },
  76         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57416_MF) },
  77         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58802) },
  78         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58804) },
  79         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58808) },
  80         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_58802_VF) },
  81         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57508) },
  82         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57504) },
  83         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57502) },
  84         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57500_VF1) },
  85         { RTE_PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, BROADCOM_DEV_ID_57500_VF2) },
  86         { .vendor_id = 0, /* sentinel */ },
  87 };
  88
  89 #define BNXT_ETH_RSS_SUPPORT (  \
  90         ETH_RSS_IPV4 |          \
  91         ETH_RSS_NONFRAG_IPV4_TCP |      \
  92         ETH_RSS_NONFRAG_IPV4_UDP |      \
  93         ETH_RSS_IPV6 |          \
  94         ETH_RSS_NONFRAG_IPV6_TCP |      \
  95         ETH_RSS_NONFRAG_IPV6_UDP)
  96
  97 #define BNXT_DEV_TX_OFFLOAD_SUPPORT (DEV_TX_OFFLOAD_VLAN_INSERT | \
  98                                      DEV_TX_OFFLOAD_IPV4_CKSUM | \
  99                                      DEV_TX_OFFLOAD_TCP_CKSUM | \
 100                                      DEV_TX_OFFLOAD_UDP_CKSUM | \
 101                                      DEV_TX_OFFLOAD_TCP_TSO | \
 102                                      DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM | \
 103                                      DEV_TX_OFFLOAD_VXLAN_TNL_TSO | \
 104                                      DEV_TX_OFFLOAD_GRE_TNL_TSO | \
 105                                      DEV_TX_OFFLOAD_IPIP_TNL_TSO | \
 106                                      DEV_TX_OFFLOAD_GENEVE_TNL_TSO | \
 107                                      DEV_TX_OFFLOAD_QINQ_INSERT | \
 108                                      DEV_TX_OFFLOAD_MULTI_SEGS)
 109
 110 #define BNXT_DEV_RX_OFFLOAD_SUPPORT (DEV_RX_OFFLOAD_VLAN_FILTER | \
 111                                      DEV_RX_OFFLOAD_VLAN_STRIP | \
 112                                      DEV_RX_OFFLOAD_IPV4_CKSUM | \
 113                                      DEV_RX_OFFLOAD_UDP_CKSUM | \
 114                                      DEV_RX_OFFLOAD_TCP_CKSUM | \
 115                                      DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM | \
 116                                      DEV_RX_OFFLOAD_JUMBO_FRAME | \
 117                                      DEV_RX_OFFLOAD_KEEP_CRC | \
 118                                      DEV_RX_OFFLOAD_VLAN_EXTEND | \
 119                                      DEV_RX_OFFLOAD_TCP_LRO | \
 120                                      DEV_RX_OFFLOAD_SCATTER)
 121
 122 static int bnxt_vlan_offload_set_op(struct rte_eth_dev *dev, int mask);
 123 static void bnxt_print_link_info(struct rte_eth_dev *eth_dev);
 124 static int bnxt_dev_uninit(struct rte_eth_dev *eth_dev);
 125 static int bnxt_init_resources(struct bnxt *bp, bool reconfig_dev);
 126 static int bnxt_uninit_resources(struct bnxt *bp, bool reconfig_dev);
 127 static void bnxt_cancel_fw_health_check(struct bnxt *bp);
 128
 129 int is_bnxt_in_error(struct bnxt *bp)
 130 {
 131         if (bp->flags & BNXT_FLAG_FATAL_ERROR)
 132                 return -EIO;
 133         if (bp->flags & BNXT_FLAG_FW_RESET)
 134                 return -EBUSY;
 135
 136         return 0;
 137 }
 138
 139 /***********************/
 140
 141 /*
 142  * High level utility functions
 143  */
 144
 145 uint16_t bnxt_rss_ctxts(const struct bnxt *bp)
 146 {
 147         if (!BNXT_CHIP_THOR(bp))
 148                 return 1;
 149
 150         return RTE_ALIGN_MUL_CEIL(bp->rx_nr_rings,
 151                                   BNXT_RSS_ENTRIES_PER_CTX_THOR) /
 152                                     BNXT_RSS_ENTRIES_PER_CTX_THOR;
 153 }
 154
 155 static uint16_t  bnxt_rss_hash_tbl_size(const struct bnxt *bp)
 156 {
 157         if (!BNXT_CHIP_THOR(bp))
 158                 return HW_HASH_INDEX_SIZE;
 159
 160         return bnxt_rss_ctxts(bp) * BNXT_RSS_ENTRIES_PER_CTX_THOR;
 161 }
 162
 163 static void bnxt_free_mem(struct bnxt *bp, bool reconfig)
 164 {
 165         bnxt_free_filter_mem(bp);
 166         bnxt_free_vnic_attributes(bp);
 167         bnxt_free_vnic_mem(bp);
 168
 169         /* tx/rx rings are configured as part of *_queue_setup callbacks.
 170          * If the number of rings change across fw update,
 171          * we don't have much choice except to warn the user.
 172          */
 173         if (!reconfig) {
 174                 bnxt_free_stats(bp);
 175                 bnxt_free_tx_rings(bp);
 176                 bnxt_free_rx_rings(bp);
 177         }
 178         bnxt_free_async_cp_ring(bp);
 179         bnxt_free_rxtx_nq_ring(bp);
 180 }
 181
 182 static int bnxt_alloc_mem(struct bnxt *bp, bool reconfig)
 183 {
 184         int rc;
 185
 186         rc = bnxt_alloc_ring_grps(bp);
 187         if (rc)
 188                 goto alloc_mem_err;
 189
 190         rc = bnxt_alloc_async_ring_struct(bp);
 191         if (rc)
 192                 goto alloc_mem_err;
 193
 194         rc = bnxt_alloc_vnic_mem(bp);
 195         if (rc)
 196                 goto alloc_mem_err;
 197
 198         rc = bnxt_alloc_vnic_attributes(bp);
 199         if (rc)
 200                 goto alloc_mem_err;
 201
 202         rc = bnxt_alloc_filter_mem(bp);
 203         if (rc)
 204                 goto alloc_mem_err;
 205
 206         rc = bnxt_alloc_async_cp_ring(bp);
 207         if (rc)
 208                 goto alloc_mem_err;
 209
 210         rc = bnxt_alloc_rxtx_nq_ring(bp);
 211         if (rc)
 212                 goto alloc_mem_err;
 213
 214         return 0;
 215
 216 alloc_mem_err:
 217         bnxt_free_mem(bp, reconfig);
 218         return rc;
 219 }
 220
 221 static int bnxt_init_chip(struct bnxt *bp)
 222 {
 223         struct bnxt_rx_queue *rxq;
 224         struct rte_eth_link new;
 225         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(bp->eth_dev);
 226         struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
 227         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 228         uint64_t rx_offloads = dev_conf->rxmode.offloads;
 229         uint32_t intr_vector = 0;
 230         uint32_t queue_id, base = BNXT_MISC_VEC_ID;
 231         uint32_t vec = BNXT_MISC_VEC_ID;
 232         unsigned int i, j;
 233         int rc;
 234
 235         if (bp->eth_dev->data->mtu > RTE_ETHER_MTU) {
 236                 bp->eth_dev->data->dev_conf.rxmode.offloads |=
 237                         DEV_RX_OFFLOAD_JUMBO_FRAME;
 238                 bp->flags |= BNXT_FLAG_JUMBO;
 239         } else {
 240                 bp->eth_dev->data->dev_conf.rxmode.offloads &=
 241                         ~DEV_RX_OFFLOAD_JUMBO_FRAME;
 242                 bp->flags &= ~BNXT_FLAG_JUMBO;
 243         }
 244
 245         /* THOR does not support ring groups.
 246          * But we will use the array to save RSS context IDs.
 247          */
 248         if (BNXT_CHIP_THOR(bp))
 249                 bp->max_ring_grps = BNXT_MAX_RSS_CTXTS_THOR;
 250
 251         rc = bnxt_alloc_all_hwrm_stat_ctxs(bp);
 252         if (rc) {
 253                 PMD_DRV_LOG(ERR, "HWRM stat ctx alloc failure rc: %x\n", rc);
 254                 goto err_out;
 255         }
 256
 257         rc = bnxt_alloc_hwrm_rings(bp);
 258         if (rc) {
 259                 PMD_DRV_LOG(ERR, "HWRM ring alloc failure rc: %x\n", rc);
 260                 goto err_out;
 261         }
 262
 263         rc = bnxt_alloc_all_hwrm_ring_grps(bp);
 264         if (rc) {
 265                 PMD_DRV_LOG(ERR, "HWRM ring grp alloc failure: %x\n", rc);
 266                 goto err_out;
 267         }
 268
 269         if (!(bp->vnic_cap_flags & BNXT_VNIC_CAP_COS_CLASSIFY))
 270                 goto skip_cosq_cfg;
 271
 272         for (j = 0, i = 0; i < BNXT_COS_QUEUE_COUNT; i++) {
 273                 if (bp->rx_cos_queue[i].id != 0xff) {
 274                         struct bnxt_vnic_info *vnic = &bp->vnic_info[j++];
 275
 276                         if (!vnic) {
 277                                 PMD_DRV_LOG(ERR,
 278                                             "Num pools more than FW profile\n");
 279                                 rc = -EINVAL;
 280                                 goto err_out;
 281                         }
 282                         vnic->cos_queue_id = bp->rx_cos_queue[i].id;
 283                         bp->rx_cosq_cnt++;
 284                 }
 285         }
 286
 287 skip_cosq_cfg:
 288         rc = bnxt_mq_rx_configure(bp);
 289         if (rc) {
 290                 PMD_DRV_LOG(ERR, "MQ mode configure failure rc: %x\n", rc);
 291                 goto err_out;
 292         }
 293
 294         /* VNIC configuration */
 295         for (i = 0; i < bp->nr_vnics; i++) {
 296                 struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
 297                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
 298
 299                 rc = bnxt_vnic_grp_alloc(bp, vnic);
 300                 if (rc)
 301                         goto err_out;
 302
 303                 PMD_DRV_LOG(DEBUG, "vnic[%d] = %p vnic->fw_grp_ids = %p\n",
 304                             i, vnic, vnic->fw_grp_ids);
 305
 306                 rc = bnxt_hwrm_vnic_alloc(bp, vnic);
 307                 if (rc) {
 308                         PMD_DRV_LOG(ERR, "HWRM vnic %d alloc failure rc: %x\n",
 309                                 i, rc);
 310                         goto err_out;
 311                 }
 312
 313                 /* Alloc RSS context only if RSS mode is enabled */
 314                 if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS) {
 315                         int j, nr_ctxs = bnxt_rss_ctxts(bp);
 316
 317                         rc = 0;
 318                         for (j = 0; j < nr_ctxs; j++) {
 319                                 rc = bnxt_hwrm_vnic_ctx_alloc(bp, vnic, j);
 320                                 if (rc)
 321                                         break;
 322                         }
 323                         if (rc) {
 324                                 PMD_DRV_LOG(ERR,
 325                                   "HWRM vnic %d ctx %d alloc failure rc: %x\n",
 326                                   i, j, rc);
 327                                 goto err_out;
 328                         }
 329                         vnic->num_lb_ctxts = nr_ctxs;
 330                 }
 331
 332                 /*
 333                  * Firmware sets pf pair in default vnic cfg. If the VLAN strip
 334                  * setting is not available at this time, it will not be
 335                  * configured correctly in the CFA.
 336                  */
 337                 if (rx_offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
 338                         vnic->vlan_strip = true;
 339                 else
 340                         vnic->vlan_strip = false;
 341
 342                 rc = bnxt_hwrm_vnic_cfg(bp, vnic);
 343                 if (rc) {
 344                         PMD_DRV_LOG(ERR, "HWRM vnic %d cfg failure rc: %x\n",
 345                                 i, rc);
 346                         goto err_out;
 347                 }
 348
 349                 rc = bnxt_set_hwrm_vnic_filters(bp, vnic);
 350                 if (rc) {
 351                         PMD_DRV_LOG(ERR,
 352                                 "HWRM vnic %d filter failure rc: %x\n",
 353                                 i, rc);
 354                         goto err_out;
 355                 }
 356
 357                 for (j = 0; j < bp->rx_num_qs_per_vnic; j++) {
 358                         rxq = bp->eth_dev->data->rx_queues[j];
 359
 360                         PMD_DRV_LOG(DEBUG,
 361                                     "rxq[%d]->vnic=%p vnic->fw_grp_ids=%p\n",
 362                                     j, rxq->vnic, rxq->vnic->fw_grp_ids);
 363
 364                         if (BNXT_HAS_RING_GRPS(bp) && rxq->rx_deferred_start)
 365                                 rxq->vnic->fw_grp_ids[j] = INVALID_HW_RING_ID;
 366                 }
 367
 368                 rc = bnxt_vnic_rss_configure(bp, vnic);
 369                 if (rc) {
 370                         PMD_DRV_LOG(ERR,
 371                                     "HWRM vnic set RSS failure rc: %x\n", rc);
 372                         goto err_out;
 373                 }
 374
 375                 bnxt_hwrm_vnic_plcmode_cfg(bp, vnic);
 376
 377                 if (bp->eth_dev->data->dev_conf.rxmode.offloads &
 378                     DEV_RX_OFFLOAD_TCP_LRO)
 379                         bnxt_hwrm_vnic_tpa_cfg(bp, vnic, 1);
 380                 else
 381                         bnxt_hwrm_vnic_tpa_cfg(bp, vnic, 0);
 382         }
 383         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, &bp->vnic_info[0], 0, NULL);
 384         if (rc) {
 385                 PMD_DRV_LOG(ERR,
 386                         "HWRM cfa l2 rx mask failure rc: %x\n", rc);
 387                 goto err_out;
 388         }
 389
 390         /* check and configure queue intr-vector mapping */
 391         if ((rte_intr_cap_multiple(intr_handle) ||
 392              !RTE_ETH_DEV_SRIOV(bp->eth_dev).active) &&
 393             bp->eth_dev->data->dev_conf.intr_conf.rxq != 0) {
 394                 intr_vector = bp->eth_dev->data->nb_rx_queues;
 395                 PMD_DRV_LOG(DEBUG, "intr_vector = %d\n", intr_vector);
 396                 if (intr_vector > bp->rx_cp_nr_rings) {
 397                         PMD_DRV_LOG(ERR, "At most %d intr queues supported",
 398                                         bp->rx_cp_nr_rings);
 399                         return -ENOTSUP;
 400                 }
 401                 rc = rte_intr_efd_enable(intr_handle, intr_vector);
 402                 if (rc)
 403                         return rc;
 404         }
 405
 406         if (rte_intr_dp_is_en(intr_handle) && !intr_handle->intr_vec) {
 407                 intr_handle->intr_vec =
 408                         rte_zmalloc("intr_vec",
 409                                     bp->eth_dev->data->nb_rx_queues *
 410                                     sizeof(int), 0);
 411                 if (intr_handle->intr_vec == NULL) {
 412                         PMD_DRV_LOG(ERR, "Failed to allocate %d rx_queues"
 413                                 " intr_vec", bp->eth_dev->data->nb_rx_queues);
 414                         rc = -ENOMEM;
 415                         goto err_disable;
 416                 }
 417                 PMD_DRV_LOG(DEBUG, "intr_handle->intr_vec = %p "
 418                         "intr_handle->nb_efd = %d intr_handle->max_intr = %d\n",
 419                          intr_handle->intr_vec, intr_handle->nb_efd,
 420                         intr_handle->max_intr);
 421                 for (queue_id = 0; queue_id < bp->eth_dev->data->nb_rx_queues;
 422                      queue_id++) {
 423                         intr_handle->intr_vec[queue_id] =
 424                                                         vec + BNXT_RX_VEC_START;
 425                         if (vec < base + intr_handle->nb_efd - 1)
 426                                 vec++;
 427                 }
 428         }
 429
 430         /* enable uio/vfio intr/eventfd mapping */
 431         rc = rte_intr_enable(intr_handle);
 432         if (rc)
 433                 goto err_free;
 434
 435         rc = bnxt_get_hwrm_link_config(bp, &new);
 436         if (rc) {
 437                 PMD_DRV_LOG(ERR, "HWRM Get link config failure rc: %x\n", rc);
 438                 goto err_free;
 439         }
 440
 441         if (!bp->link_info.link_up) {
 442                 rc = bnxt_set_hwrm_link_config(bp, true);
 443                 if (rc) {
 444                         PMD_DRV_LOG(ERR,
 445                                 "HWRM link config failure rc: %x\n", rc);
 446                         goto err_free;
 447                 }
 448         }
 449         bnxt_print_link_info(bp->eth_dev);
 450
 451         return 0;
 452
 453 err_free:
 454         rte_free(intr_handle->intr_vec);
 455 err_disable:
 456         rte_intr_efd_disable(intr_handle);
 457 err_out:
 458         /* Some of the error status returned by FW may not be from errno.h */
 459         if (rc > 0)
 460                 rc = -EIO;
 461
 462         return rc;
 463 }
 464
 465 static int bnxt_shutdown_nic(struct bnxt *bp)
 466 {
 467         bnxt_free_all_hwrm_resources(bp);
 468         bnxt_free_all_filters(bp);
 469         bnxt_free_all_vnics(bp);
 470         return 0;
 471 }
 472
 473 static int bnxt_init_nic(struct bnxt *bp)
 474 {
 475         int rc;
 476
 477         if (BNXT_HAS_RING_GRPS(bp)) {
 478                 rc = bnxt_init_ring_grps(bp);
 479                 if (rc)
 480                         return rc;
 481         }
 482
 483         bnxt_init_vnics(bp);
 484         bnxt_init_filters(bp);
 485
 486         return 0;
 487 }
 488
 489 /*
 490  * Device configuration and status function
 491  */
 492
 493 static int bnxt_dev_info_get_op(struct rte_eth_dev *eth_dev,
 494                                 struct rte_eth_dev_info *dev_info)
 495 {
 496         struct rte_pci_device *pdev = RTE_DEV_TO_PCI(eth_dev->device);
 497         struct bnxt *bp = eth_dev->data->dev_private;
 498         uint16_t max_vnics, i, j, vpool, vrxq;
 499         unsigned int max_rx_rings;
 500         int rc;
 501
 502         rc = is_bnxt_in_error(bp);
 503         if (rc)
 504                 return rc;
 505
 506         /* MAC Specifics */
 507         dev_info->max_mac_addrs = bp->max_l2_ctx;
 508         dev_info->max_hash_mac_addrs = 0;
 509
 510         /* PF/VF specifics */
 511         if (BNXT_PF(bp))
 512                 dev_info->max_vfs = pdev->max_vfs;
 513
 514         max_rx_rings = RTE_MIN(bp->max_rx_rings, bp->max_stat_ctx);
 515         /* For the sake of symmetry, max_rx_queues = max_tx_queues */
 516         dev_info->max_rx_queues = max_rx_rings;
 517         dev_info->max_tx_queues = max_rx_rings;
 518         dev_info->reta_size = bnxt_rss_hash_tbl_size(bp);
 519         dev_info->hash_key_size = 40;
 520         max_vnics = bp->max_vnics;
 521
 522         /* MTU specifics */
 523         dev_info->min_mtu = RTE_ETHER_MIN_MTU;
 524         dev_info->max_mtu = BNXT_MAX_MTU;
 525
 526         /* Fast path specifics */
 527         dev_info->min_rx_bufsize = 1;
 528         dev_info->max_rx_pktlen = BNXT_MAX_PKT_LEN;
 529
 530         dev_info->rx_offload_capa = BNXT_DEV_RX_OFFLOAD_SUPPORT;
 531         if (bp->flags & BNXT_FLAG_PTP_SUPPORTED)
 532                 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TIMESTAMP;
 533         dev_info->tx_offload_capa = BNXT_DEV_TX_OFFLOAD_SUPPORT;
 534         dev_info->flow_type_rss_offloads = BNXT_ETH_RSS_SUPPORT;
 535
 536         /* *INDENT-OFF* */
 537         dev_info->default_rxconf = (struct rte_eth_rxconf) {
 538                 .rx_thresh = {
 539                         .pthresh = 8,
 540                         .hthresh = 8,
 541                         .wthresh = 0,
 542                 },
 543                 .rx_free_thresh = 32,
 544                 /* If no descriptors available, pkts are dropped by default */
 545                 .rx_drop_en = 1,
 546         };
 547
 548         dev_info->default_txconf = (struct rte_eth_txconf) {
 549                 .tx_thresh = {
 550                         .pthresh = 32,
 551                         .hthresh = 0,
 552                         .wthresh = 0,
 553                 },
 554                 .tx_free_thresh = 32,
 555                 .tx_rs_thresh = 32,
 556         };
 557         eth_dev->data->dev_conf.intr_conf.lsc = 1;
 558
 559         eth_dev->data->dev_conf.intr_conf.rxq = 1;
 560         dev_info->rx_desc_lim.nb_min = BNXT_MIN_RING_DESC;
 561         dev_info->rx_desc_lim.nb_max = BNXT_MAX_RX_RING_DESC;
 562         dev_info->tx_desc_lim.nb_min = BNXT_MIN_RING_DESC;
 563         dev_info->tx_desc_lim.nb_max = BNXT_MAX_TX_RING_DESC;
 564
 565         /* *INDENT-ON* */
 566
 567         /*
 568          * TODO: default_rxconf, default_txconf, rx_desc_lim, and tx_desc_lim
 569          *       need further investigation.
 570          */
 571
 572         /* VMDq resources */
 573         vpool = 64; /* ETH_64_POOLS */
 574         vrxq = 128; /* ETH_VMDQ_DCB_NUM_QUEUES */
 575         for (i = 0; i < 4; vpool >>= 1, i++) {
 576                 if (max_vnics > vpool) {
 577                         for (j = 0; j < 5; vrxq >>= 1, j++) {
 578                                 if (dev_info->max_rx_queues > vrxq) {
 579                                         if (vpool > vrxq)
 580                                                 vpool = vrxq;
 581                                         goto found;
 582                                 }
 583                         }
 584                         /* Not enough resources to support VMDq */
 585                         break;
 586                 }
 587         }
 588         /* Not enough resources to support VMDq */
 589         vpool = 0;
 590         vrxq = 0;
 591 found:
 592         dev_info->max_vmdq_pools = vpool;
 593         dev_info->vmdq_queue_num = vrxq;
 594
 595         dev_info->vmdq_pool_base = 0;
 596         dev_info->vmdq_queue_base = 0;
 597
 598         return 0;
 599 }
 600
 601 /* Configure the device based on the configuration provided */
 602 static int bnxt_dev_configure_op(struct rte_eth_dev *eth_dev)
 603 {
 604         struct bnxt *bp = eth_dev->data->dev_private;
 605         uint64_t rx_offloads = eth_dev->data->dev_conf.rxmode.offloads;
 606         int rc;
 607
 608         bp->rx_queues = (void *)eth_dev->data->rx_queues;
 609         bp->tx_queues = (void *)eth_dev->data->tx_queues;
 610         bp->tx_nr_rings = eth_dev->data->nb_tx_queues;
 611         bp->rx_nr_rings = eth_dev->data->nb_rx_queues;
 612
 613         rc = is_bnxt_in_error(bp);
 614         if (rc)
 615                 return rc;
 616
 617         if (BNXT_VF(bp) && (bp->flags & BNXT_FLAG_NEW_RM)) {
 618                 rc = bnxt_hwrm_check_vf_rings(bp);
 619                 if (rc) {
 620                         PMD_DRV_LOG(ERR, "HWRM insufficient resources\n");
 621                         return -ENOSPC;
 622                 }
 623
 624                 /* If a resource has already been allocated - in this case
 625                  * it is the async completion ring, free it. Reallocate it after
 626                  * resource reservation. This will ensure the resource counts
 627                  * are calculated correctly.
 628                  */
 629
 630                 pthread_mutex_lock(&bp->def_cp_lock);
 631
 632                 if (!BNXT_HAS_NQ(bp) && bp->async_cp_ring) {
 633                         bnxt_disable_int(bp);
 634                         bnxt_free_cp_ring(bp, bp->async_cp_ring);
 635                 }
 636
 637                 rc = bnxt_hwrm_func_reserve_vf_resc(bp, false);
 638                 if (rc) {
 639                         PMD_DRV_LOG(ERR, "HWRM resource alloc fail:%x\n", rc);
 640                         pthread_mutex_unlock(&bp->def_cp_lock);
 641                         return -ENOSPC;
 642                 }
 643
 644                 if (!BNXT_HAS_NQ(bp) && bp->async_cp_ring) {
 645                         rc = bnxt_alloc_async_cp_ring(bp);
 646                         if (rc) {
 647                                 pthread_mutex_unlock(&bp->def_cp_lock);
 648                                 return rc;
 649                         }
 650                         bnxt_enable_int(bp);
 651                 }
 652
 653                 pthread_mutex_unlock(&bp->def_cp_lock);
 654         } else {
 655                 /* legacy driver needs to get updated values */
 656                 rc = bnxt_hwrm_func_qcaps(bp);
 657                 if (rc) {
 658                         PMD_DRV_LOG(ERR, "hwrm func qcaps fail:%d\n", rc);
 659                         return rc;
 660                 }
 661         }
 662
 663         /* Inherit new configurations */
 664         if (eth_dev->data->nb_rx_queues > bp->max_rx_rings ||
 665             eth_dev->data->nb_tx_queues > bp->max_tx_rings ||
 666             eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues
 667                 + BNXT_NUM_ASYNC_CPR(bp) > bp->max_cp_rings ||
 668             eth_dev->data->nb_rx_queues + eth_dev->data->nb_tx_queues >
 669             bp->max_stat_ctx)
 670                 goto resource_error;
 671
 672         if (BNXT_HAS_RING_GRPS(bp) &&
 673             (uint32_t)(eth_dev->data->nb_rx_queues) > bp->max_ring_grps)
 674                 goto resource_error;
 675
 676         if (!(eth_dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS) &&
 677             bp->max_vnics < eth_dev->data->nb_rx_queues)
 678                 goto resource_error;
 679
 680         bp->rx_cp_nr_rings = bp->rx_nr_rings;
 681         bp->tx_cp_nr_rings = bp->tx_nr_rings;
 682
 683         if (rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
 684                 eth_dev->data->mtu =
 685                         eth_dev->data->dev_conf.rxmode.max_rx_pkt_len -
 686                         RTE_ETHER_HDR_LEN - RTE_ETHER_CRC_LEN - VLAN_TAG_SIZE *
 687                         BNXT_NUM_VLANS;
 688                 bnxt_mtu_set_op(eth_dev, eth_dev->data->mtu);
 689         }
 690         return 0;
 691
 692 resource_error:
 693         PMD_DRV_LOG(ERR,
 694                     "Insufficient resources to support requested config\n");
 695         PMD_DRV_LOG(ERR,
 696                     "Num Queues Requested: Tx %d, Rx %d\n",
 697                     eth_dev->data->nb_tx_queues,
 698                     eth_dev->data->nb_rx_queues);
 699         PMD_DRV_LOG(ERR,
 700                     "MAX: TxQ %d, RxQ %d, CQ %d Stat %d, Grp %d, Vnic %d\n",
 701                     bp->max_tx_rings, bp->max_rx_rings, bp->max_cp_rings,
 702                     bp->max_stat_ctx, bp->max_ring_grps, bp->max_vnics);
 703         return -ENOSPC;
 704 }
 705
 706 static void bnxt_print_link_info(struct rte_eth_dev *eth_dev)
 707 {
 708         struct rte_eth_link *link = &eth_dev->data->dev_link;
 709
 710         if (link->link_status)
 711                 PMD_DRV_LOG(INFO, "Port %d Link Up - speed %u Mbps - %s\n",
 712                         eth_dev->data->port_id,
 713                         (uint32_t)link->link_speed,
 714                         (link->link_duplex == ETH_LINK_FULL_DUPLEX) ?
 715                         ("full-duplex") : ("half-duplex\n"));
 716         else
 717                 PMD_DRV_LOG(INFO, "Port %d Link Down\n",
 718                         eth_dev->data->port_id);
 719 }
 720
 721 /*
 722  * Determine whether the current configuration requires support for scattered
 723  * receive; return 1 if scattered receive is required and 0 if not.
 724  */
 725 static int bnxt_scattered_rx(struct rte_eth_dev *eth_dev)
 726 {
 727         uint16_t buf_size;
 728         int i;
 729
 730         if (eth_dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_SCATTER)
 731                 return 1;
 732
 733         for (i = 0; i < eth_dev->data->nb_rx_queues; i++) {
 734                 struct bnxt_rx_queue *rxq = eth_dev->data->rx_queues[i];
 735
 736                 buf_size = (uint16_t)(rte_pktmbuf_data_room_size(rxq->mb_pool) -
 737                                       RTE_PKTMBUF_HEADROOM);
 738                 if (eth_dev->data->dev_conf.rxmode.max_rx_pkt_len > buf_size)
 739                         return 1;
 740         }
 741         return 0;
 742 }
 743
 744 static eth_rx_burst_t
 745 bnxt_receive_function(__rte_unused struct rte_eth_dev *eth_dev)
 746 {
 747 #ifdef RTE_ARCH_X86
 748 #ifndef RTE_LIBRTE_IEEE1588
 749         /*
 750          * Vector mode receive can be enabled only if scatter rx is not
 751          * in use and rx offloads are limited to VLAN stripping and
 752          * CRC stripping.
 753          */
 754         if (!eth_dev->data->scattered_rx &&
 755             !(eth_dev->data->dev_conf.rxmode.offloads &
 756               ~(DEV_RX_OFFLOAD_VLAN_STRIP |
 757                 DEV_RX_OFFLOAD_KEEP_CRC |
 758                 DEV_RX_OFFLOAD_JUMBO_FRAME |
 759                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 760                 DEV_RX_OFFLOAD_UDP_CKSUM |
 761                 DEV_RX_OFFLOAD_TCP_CKSUM |
 762                 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
 763                 DEV_RX_OFFLOAD_VLAN_FILTER))) {
 764                 PMD_DRV_LOG(INFO, "Using vector mode receive for port %d\n",
 765                             eth_dev->data->port_id);
 766                 return bnxt_recv_pkts_vec;
 767         }
 768         PMD_DRV_LOG(INFO, "Vector mode receive disabled for port %d\n",
 769                     eth_dev->data->port_id);
 770         PMD_DRV_LOG(INFO,
 771                     "Port %d scatter: %d rx offload: %" PRIX64 "\n",
 772                     eth_dev->data->port_id,
 773                     eth_dev->data->scattered_rx,
 774                     eth_dev->data->dev_conf.rxmode.offloads);
 775 #endif
 776 #endif
 777         return bnxt_recv_pkts;
 778 }
 779
 780 static eth_tx_burst_t
 781 bnxt_transmit_function(__rte_unused struct rte_eth_dev *eth_dev)
 782 {
 783 #ifdef RTE_ARCH_X86
 784 #ifndef RTE_LIBRTE_IEEE1588
 785         /*
 786          * Vector mode transmit can be enabled only if not using scatter rx
 787          * or tx offloads.
 788          */
 789         if (!eth_dev->data->scattered_rx &&
 790             !eth_dev->data->dev_conf.txmode.offloads) {
 791                 PMD_DRV_LOG(INFO, "Using vector mode transmit for port %d\n",
 792                             eth_dev->data->port_id);
 793                 return bnxt_xmit_pkts_vec;
 794         }
 795         PMD_DRV_LOG(INFO, "Vector mode transmit disabled for port %d\n",
 796                     eth_dev->data->port_id);
 797         PMD_DRV_LOG(INFO,
 798                     "Port %d scatter: %d tx offload: %" PRIX64 "\n",
 799                     eth_dev->data->port_id,
 800                     eth_dev->data->scattered_rx,
 801                     eth_dev->data->dev_conf.txmode.offloads);
 802 #endif
 803 #endif
 804         return bnxt_xmit_pkts;
 805 }
 806
 807 static int bnxt_handle_if_change_status(struct bnxt *bp)
 808 {
 809         int rc;
 810
 811         /* Since fw has undergone a reset and lost all contexts,
 812          * set fatal flag to not issue hwrm during cleanup
 813          */
 814         bp->flags |= BNXT_FLAG_FATAL_ERROR;
 815         bnxt_uninit_resources(bp, true);
 816
 817         /* clear fatal flag so that re-init happens */
 818         bp->flags &= ~BNXT_FLAG_FATAL_ERROR;
 819         rc = bnxt_init_resources(bp, true);
 820
 821         bp->flags &= ~BNXT_FLAG_IF_CHANGE_HOT_FW_RESET_DONE;
 822
 823         return rc;
 824 }
 825
 826 static int bnxt_dev_start_op(struct rte_eth_dev *eth_dev)
 827 {
 828         struct bnxt *bp = eth_dev->data->dev_private;
 829         uint64_t rx_offloads = eth_dev->data->dev_conf.rxmode.offloads;
 830         int vlan_mask = 0;
 831         int rc;
 832
 833         if (!eth_dev->data->nb_tx_queues || !eth_dev->data->nb_rx_queues) {
 834                 PMD_DRV_LOG(ERR, "Queues are not configured yet!\n");
 835                 return -EINVAL;
 836         }
 837
 838         if (bp->rx_cp_nr_rings > RTE_ETHDEV_QUEUE_STAT_CNTRS) {
 839                 PMD_DRV_LOG(ERR,
 840                         "RxQ cnt %d > CONFIG_RTE_ETHDEV_QUEUE_STAT_CNTRS %d\n",
 841                         bp->rx_cp_nr_rings, RTE_ETHDEV_QUEUE_STAT_CNTRS);
 842         }
 843
 844         rc = bnxt_hwrm_if_change(bp, 1);
 845         if (!rc) {
 846                 if (bp->flags & BNXT_FLAG_IF_CHANGE_HOT_FW_RESET_DONE) {
 847                         rc = bnxt_handle_if_change_status(bp);
 848                         if (rc)
 849                                 return rc;
 850                 }
 851         }
 852         bnxt_enable_int(bp);
 853
 854         rc = bnxt_init_chip(bp);
 855         if (rc)
 856                 goto error;
 857
 858         eth_dev->data->scattered_rx = bnxt_scattered_rx(eth_dev);
 859
 860         bnxt_link_update_op(eth_dev, 1);
 861
 862         if (rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
 863                 vlan_mask |= ETH_VLAN_FILTER_MASK;
 864         if (rx_offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
 865                 vlan_mask |= ETH_VLAN_STRIP_MASK;
 866         rc = bnxt_vlan_offload_set_op(eth_dev, vlan_mask);
 867         if (rc)
 868                 goto error;
 869
 870         eth_dev->rx_pkt_burst = bnxt_receive_function(eth_dev);
 871         eth_dev->tx_pkt_burst = bnxt_transmit_function(eth_dev);
 872
 873         bp->flags |= BNXT_FLAG_INIT_DONE;
 874         eth_dev->data->dev_started = 1;
 875         bp->dev_stopped = 0;
 876         pthread_mutex_lock(&bp->def_cp_lock);
 877         bnxt_schedule_fw_health_check(bp);
 878         pthread_mutex_unlock(&bp->def_cp_lock);
 879         return 0;
 880
 881 error:
 882         bnxt_hwrm_if_change(bp, 0);
 883         bnxt_shutdown_nic(bp);
 884         bnxt_free_tx_mbufs(bp);
 885         bnxt_free_rx_mbufs(bp);
 886         return rc;
 887 }
 888
 889 static int bnxt_dev_set_link_up_op(struct rte_eth_dev *eth_dev)
 890 {
 891         struct bnxt *bp = eth_dev->data->dev_private;
 892         int rc = 0;
 893
 894         if (!bp->link_info.link_up)
 895                 rc = bnxt_set_hwrm_link_config(bp, true);
 896         if (!rc)
 897                 eth_dev->data->dev_link.link_status = 1;
 898
 899         bnxt_print_link_info(eth_dev);
 900         return rc;
 901 }
 902
 903 static int bnxt_dev_set_link_down_op(struct rte_eth_dev *eth_dev)
 904 {
 905         struct bnxt *bp = eth_dev->data->dev_private;
 906
 907         eth_dev->data->dev_link.link_status = 0;
 908         bnxt_set_hwrm_link_config(bp, false);
 909         bp->link_info.link_up = 0;
 910
 911         return 0;
 912 }
 913
 914 /* Unload the driver, release resources */
 915 static void bnxt_dev_stop_op(struct rte_eth_dev *eth_dev)
 916 {
 917         struct bnxt *bp = eth_dev->data->dev_private;
 918         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
 919         struct rte_intr_handle *intr_handle = &pci_dev->intr_handle;
 920
 921         eth_dev->data->dev_started = 0;
 922         /* Prevent crashes when queues are still in use */
 923         eth_dev->rx_pkt_burst = &bnxt_dummy_recv_pkts;
 924         eth_dev->tx_pkt_burst = &bnxt_dummy_xmit_pkts;
 925
 926         bnxt_disable_int(bp);
 927
 928         /* disable uio/vfio intr/eventfd mapping */
 929         rte_intr_disable(intr_handle);
 930
 931         bnxt_cancel_fw_health_check(bp);
 932
 933         bp->flags &= ~BNXT_FLAG_INIT_DONE;
 934         if (bp->eth_dev->data->dev_started) {
 935                 /* TBD: STOP HW queues DMA */
 936                 eth_dev->data->dev_link.link_status = 0;
 937         }
 938         bnxt_dev_set_link_down_op(eth_dev);
 939
 940         /* Wait for link to be reset and the async notification to process.
 941          * During reset recovery, there is no need to wait
 942          */
 943         if (!is_bnxt_in_error(bp))
 944                 rte_delay_ms(BNXT_LINK_WAIT_INTERVAL * 2);
 945
 946         /* Clean queue intr-vector mapping */
 947         rte_intr_efd_disable(intr_handle);
 948         if (intr_handle->intr_vec != NULL) {
 949                 rte_free(intr_handle->intr_vec);
 950                 intr_handle->intr_vec = NULL;
 951         }
 952
 953         bnxt_hwrm_port_clr_stats(bp);
 954         bnxt_free_tx_mbufs(bp);
 955         bnxt_free_rx_mbufs(bp);
 956         /* Process any remaining notifications in default completion queue */
 957         bnxt_int_handler(eth_dev);
 958         bnxt_shutdown_nic(bp);
 959         bnxt_hwrm_if_change(bp, 0);
 960         bp->dev_stopped = 1;
 961 }
 962
 963 static void bnxt_dev_close_op(struct rte_eth_dev *eth_dev)
 964 {
 965         struct bnxt *bp = eth_dev->data->dev_private;
 966
 967         if (bp->dev_stopped == 0)
 968                 bnxt_dev_stop_op(eth_dev);
 969
 970         if (eth_dev->data->mac_addrs != NULL) {
 971                 rte_free(eth_dev->data->mac_addrs);
 972                 eth_dev->data->mac_addrs = NULL;
 973         }
 974         if (bp->grp_info != NULL) {
 975                 rte_free(bp->grp_info);
 976                 bp->grp_info = NULL;
 977         }
 978
 979         bnxt_dev_uninit(eth_dev);
 980 }
 981
 982 static void bnxt_mac_addr_remove_op(struct rte_eth_dev *eth_dev,
 983                                     uint32_t index)
 984 {
 985         struct bnxt *bp = eth_dev->data->dev_private;
 986         uint64_t pool_mask = eth_dev->data->mac_pool_sel[index];
 987         struct bnxt_vnic_info *vnic;
 988         struct bnxt_filter_info *filter, *temp_filter;
 989         uint32_t i;
 990
 991         if (is_bnxt_in_error(bp))
 992                 return;
 993
 994         /*
 995          * Loop through all VNICs from the specified filter flow pools to
 996          * remove the corresponding MAC addr filter
 997          */
 998         for (i = 0; i < bp->nr_vnics; i++) {
 999                 if (!(pool_mask & (1ULL << i)))
1000                         continue;
1001
1002                 vnic = &bp->vnic_info[i];
1003                 filter = STAILQ_FIRST(&vnic->filter);
1004                 while (filter) {
1005                         temp_filter = STAILQ_NEXT(filter, next);
1006                         if (filter->mac_index == index) {
1007                                 STAILQ_REMOVE(&vnic->filter, filter,
1008                                                 bnxt_filter_info, next);
1009                                 bnxt_hwrm_clear_l2_filter(bp, filter);
1010                                 filter->mac_index = INVALID_MAC_INDEX;
1011                                 memset(&filter->l2_addr, 0, RTE_ETHER_ADDR_LEN);
1012                                 STAILQ_INSERT_TAIL(&bp->free_filter_list,
1013                                                    filter, next);
1014                         }
1015                         filter = temp_filter;
1016                 }
1017         }
1018 }
1019
1020 static int bnxt_add_mac_filter(struct bnxt *bp, struct bnxt_vnic_info *vnic,
1021                                struct rte_ether_addr *mac_addr, uint32_t index)
1022 {
1023         struct bnxt_filter_info *filter;
1024         int rc = 0;
1025
1026         filter = STAILQ_FIRST(&vnic->filter);
1027         /* During bnxt_mac_addr_add_op, default MAC is
1028          * already programmed, so skip it. But, when
1029          * hw-vlan-filter is turned OFF from ON, default
1030          * MAC filter should be restored
1031          */
1032         if (index == 0 && filter->dflt)
1033                 return 0;
1034
1035         filter = bnxt_alloc_filter(bp);
1036         if (!filter) {
1037                 PMD_DRV_LOG(ERR, "L2 filter alloc failed\n");
1038                 return -ENODEV;
1039         }
1040
1041         filter->mac_index = index;
1042         /* bnxt_alloc_filter copies default MAC to filter->l2_addr. So,
1043          * if the MAC that's been programmed now is a different one, then,
1044          * copy that addr to filter->l2_addr
1045          */
1046         if (mac_addr)
1047                 memcpy(filter->l2_addr, mac_addr, RTE_ETHER_ADDR_LEN);
1048         filter->flags |= HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_OUTERMOST;
1049
1050         rc = bnxt_hwrm_set_l2_filter(bp, vnic->fw_vnic_id, filter);
1051         if (!rc) {
1052                 if (filter->mac_index == 0) {
1053                         filter->dflt = true;
1054                         STAILQ_INSERT_HEAD(&vnic->filter, filter, next);
1055                 } else {
1056                         STAILQ_INSERT_TAIL(&vnic->filter, filter, next);
1057                 }
1058         } else {
1059                 filter->mac_index = INVALID_MAC_INDEX;
1060                 memset(&filter->l2_addr, 0, RTE_ETHER_ADDR_LEN);
1061                 bnxt_free_filter(bp, filter);
1062         }
1063
1064         return rc;
1065 }
1066
1067 static int bnxt_mac_addr_add_op(struct rte_eth_dev *eth_dev,
1068                                 struct rte_ether_addr *mac_addr,
1069                                 uint32_t index, uint32_t pool)
1070 {
1071         struct bnxt *bp = eth_dev->data->dev_private;
1072         struct bnxt_vnic_info *vnic = &bp->vnic_info[pool];
1073         struct bnxt_filter_info *filter;
1074         int rc = 0;
1075
1076         rc = is_bnxt_in_error(bp);
1077         if (rc)
1078                 return rc;
1079
1080         if (BNXT_VF(bp) & !BNXT_VF_IS_TRUSTED(bp)) {
1081                 PMD_DRV_LOG(ERR, "Cannot add MAC address to a VF interface\n");
1082                 return -ENOTSUP;
1083         }
1084
1085         if (!vnic) {
1086                 PMD_DRV_LOG(ERR, "VNIC not found for pool %d!\n", pool);
1087                 return -EINVAL;
1088         }
1089         /* Attach requested MAC address to the new l2_filter */
1090         STAILQ_FOREACH(filter, &vnic->filter, next) {
1091                 if (filter->mac_index == index) {
1092                         PMD_DRV_LOG(ERR,
1093                                 "MAC addr already existed for pool %d\n", pool);
1094                         return 0;
1095                 }
1096         }
1097
1098         rc = bnxt_add_mac_filter(bp, vnic, mac_addr, index);
1099
1100         return rc;
1101 }
1102
1103 int bnxt_link_update_op(struct rte_eth_dev *eth_dev, int wait_to_complete)
1104 {
1105         int rc = 0;
1106         struct bnxt *bp = eth_dev->data->dev_private;
1107         struct rte_eth_link new;
1108         unsigned int cnt = BNXT_LINK_WAIT_CNT;
1109
1110         rc = is_bnxt_in_error(bp);
1111         if (rc)
1112                 return rc;
1113
1114         memset(&new, 0, sizeof(new));
1115         do {
1116                 /* Retrieve link info from hardware */
1117                 rc = bnxt_get_hwrm_link_config(bp, &new);
1118                 if (rc) {
1119                         new.link_speed = ETH_LINK_SPEED_100M;
1120                         new.link_duplex = ETH_LINK_FULL_DUPLEX;
1121                         PMD_DRV_LOG(ERR,
1122                                 "Failed to retrieve link rc = 0x%x!\n", rc);
1123                         goto out;
1124                 }
1125
1126                 if (!wait_to_complete || new.link_status)
1127                         break;
1128
1129                 rte_delay_ms(BNXT_LINK_WAIT_INTERVAL);
1130         } while (cnt--);
1131
1132 out:
1133         /* Timed out or success */
1134         if (new.link_status != eth_dev->data->dev_link.link_status ||
1135         new.link_speed != eth_dev->data->dev_link.link_speed) {
1136                 rte_eth_linkstatus_set(eth_dev, &new);
1137
1138                 _rte_eth_dev_callback_process(eth_dev,
1139                                               RTE_ETH_EVENT_INTR_LSC,
1140                                               NULL);
1141
1142                 bnxt_print_link_info(eth_dev);
1143         }
1144
1145         return rc;
1146 }
1147
1148 static int bnxt_promiscuous_enable_op(struct rte_eth_dev *eth_dev)
1149 {
1150         struct bnxt *bp = eth_dev->data->dev_private;
1151         struct bnxt_vnic_info *vnic;
1152         uint32_t old_flags;
1153         int rc;
1154
1155         rc = is_bnxt_in_error(bp);
1156         if (rc)
1157                 return rc;
1158
1159         if (bp->vnic_info == NULL)
1160                 return 0;
1161
1162         vnic = BNXT_GET_DEFAULT_VNIC(bp);
1163
1164         old_flags = vnic->flags;
1165         vnic->flags |= BNXT_VNIC_INFO_PROMISC;
1166         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
1167         if (rc != 0)
1168                 vnic->flags = old_flags;
1169
1170         return rc;
1171 }
1172
1173 static int bnxt_promiscuous_disable_op(struct rte_eth_dev *eth_dev)
1174 {
1175         struct bnxt *bp = eth_dev->data->dev_private;
1176         struct bnxt_vnic_info *vnic;
1177         uint32_t old_flags;
1178         int rc;
1179
1180         rc = is_bnxt_in_error(bp);
1181         if (rc)
1182                 return rc;
1183
1184         if (bp->vnic_info == NULL)
1185                 return 0;
1186
1187         vnic = BNXT_GET_DEFAULT_VNIC(bp);
1188
1189         old_flags = vnic->flags;
1190         vnic->flags &= ~BNXT_VNIC_INFO_PROMISC;
1191         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
1192         if (rc != 0)
1193                 vnic->flags = old_flags;
1194
1195         return rc;
1196 }
1197
1198 static int bnxt_allmulticast_enable_op(struct rte_eth_dev *eth_dev)
1199 {
1200         struct bnxt *bp = eth_dev->data->dev_private;
1201         struct bnxt_vnic_info *vnic;
1202         uint32_t old_flags;
1203         int rc;
1204
1205         rc = is_bnxt_in_error(bp);
1206         if (rc)
1207                 return rc;
1208
1209         if (bp->vnic_info == NULL)
1210                 return 0;
1211
1212         vnic = BNXT_GET_DEFAULT_VNIC(bp);
1213
1214         old_flags = vnic->flags;
1215         vnic->flags |= BNXT_VNIC_INFO_ALLMULTI;
1216         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
1217         if (rc != 0)
1218                 vnic->flags = old_flags;
1219
1220         return rc;
1221 }
1222
1223 static int bnxt_allmulticast_disable_op(struct rte_eth_dev *eth_dev)
1224 {
1225         struct bnxt *bp = eth_dev->data->dev_private;
1226         struct bnxt_vnic_info *vnic;
1227         uint32_t old_flags;
1228         int rc;
1229
1230         rc = is_bnxt_in_error(bp);
1231         if (rc)
1232                 return rc;
1233
1234         if (bp->vnic_info == NULL)
1235                 return 0;
1236
1237         vnic = BNXT_GET_DEFAULT_VNIC(bp);
1238
1239         old_flags = vnic->flags;
1240         vnic->flags &= ~BNXT_VNIC_INFO_ALLMULTI;
1241         rc = bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
1242         if (rc != 0)
1243                 vnic->flags = old_flags;
1244
1245         return rc;
1246 }
1247
1248 /* Return bnxt_rx_queue pointer corresponding to a given rxq. */
1249 static struct bnxt_rx_queue *bnxt_qid_to_rxq(struct bnxt *bp, uint16_t qid)
1250 {
1251         if (qid >= bp->rx_nr_rings)
1252                 return NULL;
1253
1254         return bp->eth_dev->data->rx_queues[qid];
1255 }
1256
1257 /* Return rxq corresponding to a given rss table ring/group ID. */
1258 static uint16_t bnxt_rss_to_qid(struct bnxt *bp, uint16_t fwr)
1259 {
1260         struct bnxt_rx_queue *rxq;
1261         unsigned int i;
1262
1263         if (!BNXT_HAS_RING_GRPS(bp)) {
1264                 for (i = 0; i < bp->rx_nr_rings; i++) {
1265                         rxq = bp->eth_dev->data->rx_queues[i];
1266                         if (rxq->rx_ring->rx_ring_struct->fw_ring_id == fwr)
1267                                 return rxq->index;
1268                 }
1269         } else {
1270                 for (i = 0; i < bp->rx_nr_rings; i++) {
1271                         if (bp->grp_info[i].fw_grp_id == fwr)
1272                                 return i;
1273                 }
1274         }
1275
1276         return INVALID_HW_RING_ID;
1277 }
1278
1279 static int bnxt_reta_update_op(struct rte_eth_dev *eth_dev,
1280                             struct rte_eth_rss_reta_entry64 *reta_conf,
1281                             uint16_t reta_size)
1282 {
1283         struct bnxt *bp = eth_dev->data->dev_private;
1284         struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
1285         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
1286         uint16_t tbl_size = bnxt_rss_hash_tbl_size(bp);
1287         uint16_t idx, sft;
1288         int i, rc;
1289
1290         rc = is_bnxt_in_error(bp);
1291         if (rc)
1292                 return rc;
1293
1294         if (!vnic->rss_table)
1295                 return -EINVAL;
1296
1297         if (!(dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG))
1298                 return -EINVAL;
1299
1300         if (reta_size != tbl_size) {
1301                 PMD_DRV_LOG(ERR, "The configured hash table lookup size "
1302                         "(%d) must equal the size supported by the hardware "
1303                         "(%d)\n", reta_size, tbl_size);
1304                 return -EINVAL;
1305         }
1306
1307         for (i = 0; i < reta_size; i++) {
1308                 struct bnxt_rx_queue *rxq;
1309
1310                 idx = i / RTE_RETA_GROUP_SIZE;
1311                 sft = i % RTE_RETA_GROUP_SIZE;
1312
1313                 if (!(reta_conf[idx].mask & (1ULL << sft)))
1314                         continue;
1315
1316                 rxq = bnxt_qid_to_rxq(bp, reta_conf[idx].reta[sft]);
1317                 if (!rxq) {
1318                         PMD_DRV_LOG(ERR, "Invalid ring in reta_conf.\n");
1319                         return -EINVAL;
1320                 }
1321
1322                 if (BNXT_CHIP_THOR(bp)) {
1323                         vnic->rss_table[i * 2] =
1324                                 rxq->rx_ring->rx_ring_struct->fw_ring_id;
1325                         vnic->rss_table[i * 2 + 1] =
1326                                 rxq->cp_ring->cp_ring_struct->fw_ring_id;
1327                 } else {
1328                         vnic->rss_table[i] =
1329                             vnic->fw_grp_ids[reta_conf[idx].reta[sft]];
1330                 }
1331         }
1332
1333         bnxt_hwrm_vnic_rss_cfg(bp, vnic);
1334         return 0;
1335 }
1336
1337 static int bnxt_reta_query_op(struct rte_eth_dev *eth_dev,
1338                               struct rte_eth_rss_reta_entry64 *reta_conf,
1339                               uint16_t reta_size)
1340 {
1341         struct bnxt *bp = eth_dev->data->dev_private;
1342         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
1343         uint16_t tbl_size = bnxt_rss_hash_tbl_size(bp);
1344         uint16_t idx, sft, i;
1345         int rc;
1346
1347         rc = is_bnxt_in_error(bp);
1348         if (rc)
1349                 return rc;
1350
1351         /* Retrieve from the default VNIC */
1352         if (!vnic)
1353                 return -EINVAL;
1354         if (!vnic->rss_table)
1355                 return -EINVAL;
1356
1357         if (reta_size != tbl_size) {
1358                 PMD_DRV_LOG(ERR, "The configured hash table lookup size "
1359                         "(%d) must equal the size supported by the hardware "
1360                         "(%d)\n", reta_size, tbl_size);
1361                 return -EINVAL;
1362         }
1363
1364         for (idx = 0, i = 0; i < reta_size; i++) {
1365                 idx = i / RTE_RETA_GROUP_SIZE;
1366                 sft = i % RTE_RETA_GROUP_SIZE;
1367
1368                 if (reta_conf[idx].mask & (1ULL << sft)) {
1369                         uint16_t qid;
1370
1371                         if (BNXT_CHIP_THOR(bp))
1372                                 qid = bnxt_rss_to_qid(bp,
1373                                                       vnic->rss_table[i * 2]);
1374                         else
1375                                 qid = bnxt_rss_to_qid(bp, vnic->rss_table[i]);
1376
1377                         if (qid == INVALID_HW_RING_ID) {
1378                                 PMD_DRV_LOG(ERR, "Inv. entry in rss table.\n");
1379                                 return -EINVAL;
1380                         }
1381                         reta_conf[idx].reta[sft] = qid;
1382                 }
1383         }
1384
1385         return 0;
1386 }
1387
1388 static int bnxt_rss_hash_update_op(struct rte_eth_dev *eth_dev,
1389                                    struct rte_eth_rss_conf *rss_conf)
1390 {
1391         struct bnxt *bp = eth_dev->data->dev_private;
1392         struct rte_eth_conf *dev_conf = &bp->eth_dev->data->dev_conf;
1393         struct bnxt_vnic_info *vnic;
1394         int rc;
1395
1396         rc = is_bnxt_in_error(bp);
1397         if (rc)
1398                 return rc;
1399
1400         /*
1401          * If RSS enablement were different than dev_configure,
1402          * then return -EINVAL
1403          */
1404         if (dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) {
1405                 if (!rss_conf->rss_hf)
1406                         PMD_DRV_LOG(ERR, "Hash type NONE\n");
1407         } else {
1408                 if (rss_conf->rss_hf & BNXT_ETH_RSS_SUPPORT)
1409                         return -EINVAL;
1410         }
1411
1412         bp->flags |= BNXT_FLAG_UPDATE_HASH;
1413         memcpy(&bp->rss_conf, rss_conf, sizeof(*rss_conf));
1414
1415         /* Update the default RSS VNIC(s) */
1416         vnic = &bp->vnic_info[0];
1417         vnic->hash_type = bnxt_rte_to_hwrm_hash_types(rss_conf->rss_hf);
1418
1419         /*
1420          * If hashkey is not specified, use the previously configured
1421          * hashkey
1422          */
1423         if (!rss_conf->rss_key)
1424                 goto rss_config;
1425
1426         if (rss_conf->rss_key_len != HW_HASH_KEY_SIZE) {
1427                 PMD_DRV_LOG(ERR,
1428                             "Invalid hashkey length, should be 16 bytes\n");
1429                 return -EINVAL;
1430         }
1431         memcpy(vnic->rss_hash_key, rss_conf->rss_key, rss_conf->rss_key_len);
1432
1433 rss_config:
1434         bnxt_hwrm_vnic_rss_cfg(bp, vnic);
1435         return 0;
1436 }
1437
1438 static int bnxt_rss_hash_conf_get_op(struct rte_eth_dev *eth_dev,
1439                                      struct rte_eth_rss_conf *rss_conf)
1440 {
1441         struct bnxt *bp = eth_dev->data->dev_private;
1442         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
1443         int len, rc;
1444         uint32_t hash_types;
1445
1446         rc = is_bnxt_in_error(bp);
1447         if (rc)
1448                 return rc;
1449
1450         /* RSS configuration is the same for all VNICs */
1451         if (vnic && vnic->rss_hash_key) {
1452                 if (rss_conf->rss_key) {
1453                         len = rss_conf->rss_key_len <= HW_HASH_KEY_SIZE ?
1454                               rss_conf->rss_key_len : HW_HASH_KEY_SIZE;
1455                         memcpy(rss_conf->rss_key, vnic->rss_hash_key, len);
1456                 }
1457
1458                 hash_types = vnic->hash_type;
1459                 rss_conf->rss_hf = 0;
1460                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV4) {
1461                         rss_conf->rss_hf |= ETH_RSS_IPV4;
1462                         hash_types &= ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV4;
1463                 }
1464                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV4) {
1465                         rss_conf->rss_hf |= ETH_RSS_NONFRAG_IPV4_TCP;
1466                         hash_types &=
1467                                 ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV4;
1468                 }
1469                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV4) {
1470                         rss_conf->rss_hf |= ETH_RSS_NONFRAG_IPV4_UDP;
1471                         hash_types &=
1472                                 ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV4;
1473                 }
1474                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV6) {
1475                         rss_conf->rss_hf |= ETH_RSS_IPV6;
1476                         hash_types &= ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_IPV6;
1477                 }
1478                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV6) {
1479                         rss_conf->rss_hf |= ETH_RSS_NONFRAG_IPV6_TCP;
1480                         hash_types &=
1481                                 ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_TCP_IPV6;
1482                 }
1483                 if (hash_types & HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV6) {
1484                         rss_conf->rss_hf |= ETH_RSS_NONFRAG_IPV6_UDP;
1485                         hash_types &=
1486                                 ~HWRM_VNIC_RSS_CFG_INPUT_HASH_TYPE_UDP_IPV6;
1487                 }
1488                 if (hash_types) {
1489                         PMD_DRV_LOG(ERR,
1490                                 "Unknwon RSS config from firmware (%08x), RSS disabled",
1491                                 vnic->hash_type);
1492                         return -ENOTSUP;
1493                 }
1494         } else {
1495                 rss_conf->rss_hf = 0;
1496         }
1497         return 0;
1498 }
1499
1500 static int bnxt_flow_ctrl_get_op(struct rte_eth_dev *dev,
1501                                struct rte_eth_fc_conf *fc_conf)
1502 {
1503         struct bnxt *bp = dev->data->dev_private;
1504         struct rte_eth_link link_info;
1505         int rc;
1506
1507         rc = is_bnxt_in_error(bp);
1508         if (rc)
1509                 return rc;
1510
1511         rc = bnxt_get_hwrm_link_config(bp, &link_info);
1512         if (rc)
1513                 return rc;
1514
1515         memset(fc_conf, 0, sizeof(*fc_conf));
1516         if (bp->link_info.auto_pause)
1517                 fc_conf->autoneg = 1;
1518         switch (bp->link_info.pause) {
1519         case 0:
1520                 fc_conf->mode = RTE_FC_NONE;
1521                 break;
1522         case HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX:
1523                 fc_conf->mode = RTE_FC_TX_PAUSE;
1524                 break;
1525         case HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX:
1526                 fc_conf->mode = RTE_FC_RX_PAUSE;
1527                 break;
1528         case (HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_TX |
1529                         HWRM_PORT_PHY_QCFG_OUTPUT_PAUSE_RX):
1530                 fc_conf->mode = RTE_FC_FULL;
1531                 break;
1532         }
1533         return 0;
1534 }
1535
1536 static int bnxt_flow_ctrl_set_op(struct rte_eth_dev *dev,
1537                                struct rte_eth_fc_conf *fc_conf)
1538 {
1539         struct bnxt *bp = dev->data->dev_private;
1540         int rc;
1541
1542         rc = is_bnxt_in_error(bp);
1543         if (rc)
1544                 return rc;
1545
1546         if (!BNXT_SINGLE_PF(bp) || BNXT_VF(bp)) {
1547                 PMD_DRV_LOG(ERR, "Flow Control Settings cannot be modified\n");
1548                 return -ENOTSUP;
1549         }
1550
1551         switch (fc_conf->mode) {
1552         case RTE_FC_NONE:
1553                 bp->link_info.auto_pause = 0;
1554                 bp->link_info.force_pause = 0;
1555                 break;
1556         case RTE_FC_RX_PAUSE:
1557                 if (fc_conf->autoneg) {
1558                         bp->link_info.auto_pause =
1559                                         HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_RX;
1560                         bp->link_info.force_pause = 0;
1561                 } else {
1562                         bp->link_info.auto_pause = 0;
1563                         bp->link_info.force_pause =
1564                                         HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_RX;
1565                 }
1566                 break;
1567         case RTE_FC_TX_PAUSE:
1568                 if (fc_conf->autoneg) {
1569                         bp->link_info.auto_pause =
1570                                         HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_TX;
1571                         bp->link_info.force_pause = 0;
1572                 } else {
1573                         bp->link_info.auto_pause = 0;
1574                         bp->link_info.force_pause =
1575                                         HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_TX;
1576                 }
1577                 break;
1578         case RTE_FC_FULL:
1579                 if (fc_conf->autoneg) {
1580                         bp->link_info.auto_pause =
1581                                         HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_TX |
1582                                         HWRM_PORT_PHY_CFG_INPUT_AUTO_PAUSE_RX;
1583                         bp->link_info.force_pause = 0;
1584                 } else {
1585                         bp->link_info.auto_pause = 0;
1586                         bp->link_info.force_pause =
1587                                         HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_TX |
1588                                         HWRM_PORT_PHY_CFG_INPUT_FORCE_PAUSE_RX;
1589                 }
1590                 break;
1591         }
1592         return bnxt_set_hwrm_link_config(bp, true);
1593 }
1594
1595 /* Add UDP tunneling port */
1596 static int
1597 bnxt_udp_tunnel_port_add_op(struct rte_eth_dev *eth_dev,
1598                          struct rte_eth_udp_tunnel *udp_tunnel)
1599 {
1600         struct bnxt *bp = eth_dev->data->dev_private;
1601         uint16_t tunnel_type = 0;
1602         int rc = 0;
1603
1604         rc = is_bnxt_in_error(bp);
1605         if (rc)
1606                 return rc;
1607
1608         switch (udp_tunnel->prot_type) {
1609         case RTE_TUNNEL_TYPE_VXLAN:
1610                 if (bp->vxlan_port_cnt) {
1611                         PMD_DRV_LOG(ERR, "Tunnel Port %d already programmed\n",
1612                                 udp_tunnel->udp_port);
1613                         if (bp->vxlan_port != udp_tunnel->udp_port) {
1614                                 PMD_DRV_LOG(ERR, "Only one port allowed\n");
1615                                 return -ENOSPC;
1616                         }
1617                         bp->vxlan_port_cnt++;
1618                         return 0;
1619                 }
1620                 tunnel_type =
1621                         HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_VXLAN;
1622                 bp->vxlan_port_cnt++;
1623                 break;
1624         case RTE_TUNNEL_TYPE_GENEVE:
1625                 if (bp->geneve_port_cnt) {
1626                         PMD_DRV_LOG(ERR, "Tunnel Port %d already programmed\n",
1627                                 udp_tunnel->udp_port);
1628                         if (bp->geneve_port != udp_tunnel->udp_port) {
1629                                 PMD_DRV_LOG(ERR, "Only one port allowed\n");
1630                                 return -ENOSPC;
1631                         }
1632                         bp->geneve_port_cnt++;
1633                         return 0;
1634                 }
1635                 tunnel_type =
1636                         HWRM_TUNNEL_DST_PORT_ALLOC_INPUT_TUNNEL_TYPE_GENEVE;
1637                 bp->geneve_port_cnt++;
1638                 break;
1639         default:
1640                 PMD_DRV_LOG(ERR, "Tunnel type is not supported\n");
1641                 return -ENOTSUP;
1642         }
1643         rc = bnxt_hwrm_tunnel_dst_port_alloc(bp, udp_tunnel->udp_port,
1644                                              tunnel_type);
1645         return rc;
1646 }
1647
1648 static int
1649 bnxt_udp_tunnel_port_del_op(struct rte_eth_dev *eth_dev,
1650                          struct rte_eth_udp_tunnel *udp_tunnel)
1651 {
1652         struct bnxt *bp = eth_dev->data->dev_private;
1653         uint16_t tunnel_type = 0;
1654         uint16_t port = 0;
1655         int rc = 0;
1656
1657         rc = is_bnxt_in_error(bp);
1658         if (rc)
1659                 return rc;
1660
1661         switch (udp_tunnel->prot_type) {
1662         case RTE_TUNNEL_TYPE_VXLAN:
1663                 if (!bp->vxlan_port_cnt) {
1664                         PMD_DRV_LOG(ERR, "No Tunnel port configured yet\n");
1665                         return -EINVAL;
1666                 }
1667                 if (bp->vxlan_port != udp_tunnel->udp_port) {
1668                         PMD_DRV_LOG(ERR, "Req Port: %d. Configured port: %d\n",
1669                                 udp_tunnel->udp_port, bp->vxlan_port);
1670                         return -EINVAL;
1671                 }
1672                 if (--bp->vxlan_port_cnt)
1673                         return 0;
1674
1675                 tunnel_type =
1676                         HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_VXLAN;
1677                 port = bp->vxlan_fw_dst_port_id;
1678                 break;
1679         case RTE_TUNNEL_TYPE_GENEVE:
1680                 if (!bp->geneve_port_cnt) {
1681                         PMD_DRV_LOG(ERR, "No Tunnel port configured yet\n");
1682                         return -EINVAL;
1683                 }
1684                 if (bp->geneve_port != udp_tunnel->udp_port) {
1685                         PMD_DRV_LOG(ERR, "Req Port: %d. Configured port: %d\n",
1686                                 udp_tunnel->udp_port, bp->geneve_port);
1687                         return -EINVAL;
1688                 }
1689                 if (--bp->geneve_port_cnt)
1690                         return 0;
1691
1692                 tunnel_type =
1693                         HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_GENEVE;
1694                 port = bp->geneve_fw_dst_port_id;
1695                 break;
1696         default:
1697                 PMD_DRV_LOG(ERR, "Tunnel type is not supported\n");
1698                 return -ENOTSUP;
1699         }
1700
1701         rc = bnxt_hwrm_tunnel_dst_port_free(bp, port, tunnel_type);
1702         if (!rc) {
1703                 if (tunnel_type ==
1704                     HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_VXLAN)
1705                         bp->vxlan_port = 0;
1706                 if (tunnel_type ==
1707                     HWRM_TUNNEL_DST_PORT_FREE_INPUT_TUNNEL_TYPE_GENEVE)
1708                         bp->geneve_port = 0;
1709         }
1710         return rc;
1711 }
1712
1713 static int bnxt_del_vlan_filter(struct bnxt *bp, uint16_t vlan_id)
1714 {
1715         struct bnxt_filter_info *filter;
1716         struct bnxt_vnic_info *vnic;
1717         int rc = 0;
1718         uint32_t chk = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN;
1719
1720         /* if VLAN exists && VLAN matches vlan_id
1721          *      remove the MAC+VLAN filter
1722          *      add a new MAC only filter
1723          * else
1724          *      VLAN filter doesn't exist, just skip and continue
1725          */
1726         vnic = BNXT_GET_DEFAULT_VNIC(bp);
1727         filter = STAILQ_FIRST(&vnic->filter);
1728         while (filter) {
1729                 /* Search for this matching MAC+VLAN filter */
1730                 if ((filter->enables & chk) &&
1731                     (filter->l2_ivlan == vlan_id &&
1732                      filter->l2_ivlan_mask != 0) &&
1733                     !memcmp(filter->l2_addr, bp->mac_addr,
1734                             RTE_ETHER_ADDR_LEN)) {
1735                         /* Delete the filter */
1736                         rc = bnxt_hwrm_clear_l2_filter(bp, filter);
1737                         if (rc)
1738                                 return rc;
1739                         STAILQ_REMOVE(&vnic->filter, filter,
1740                                       bnxt_filter_info, next);
1741                         STAILQ_INSERT_TAIL(&bp->free_filter_list, filter, next);
1742
1743                         PMD_DRV_LOG(INFO,
1744                                     "Del Vlan filter for %d\n",
1745                                     vlan_id);
1746                         return rc;
1747                 }
1748                 filter = STAILQ_NEXT(filter, next);
1749         }
1750         return -ENOENT;
1751 }
1752
1753 static int bnxt_add_vlan_filter(struct bnxt *bp, uint16_t vlan_id)
1754 {
1755         struct bnxt_filter_info *filter;
1756         struct bnxt_vnic_info *vnic;
1757         int rc = 0;
1758         uint32_t en = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN |
1759                 HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN_MASK;
1760         uint32_t chk = HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_IVLAN;
1761
1762         /* Implementation notes on the use of VNIC in this command:
1763          *
1764          * By default, these filters belong to default vnic for the function.
1765          * Once these filters are set up, only destination VNIC can be modified.
1766          * If the destination VNIC is not specified in this command,
1767          * then the HWRM shall only create an l2 context id.
1768          */
1769
1770         vnic = BNXT_GET_DEFAULT_VNIC(bp);
1771         filter = STAILQ_FIRST(&vnic->filter);
1772         /* Check if the VLAN has already been added */
1773         while (filter) {
1774                 if ((filter->enables & chk) &&
1775                     (filter->l2_ivlan == vlan_id &&
1776                      filter->l2_ivlan_mask == 0x0FFF) &&
1777                      !memcmp(filter->l2_addr, bp->mac_addr,
1778                              RTE_ETHER_ADDR_LEN))
1779                         return -EEXIST;
1780
1781                 filter = STAILQ_NEXT(filter, next);
1782         }
1783
1784         /* No match found. Alloc a fresh filter and issue the L2_FILTER_ALLOC
1785          * command to create MAC+VLAN filter with the right flags, enables set.
1786          */
1787         filter = bnxt_alloc_filter(bp);
1788         if (!filter) {
1789                 PMD_DRV_LOG(ERR,
1790                             "MAC/VLAN filter alloc failed\n");
1791                 return -ENOMEM;
1792         }
1793         /* MAC + VLAN ID filter */
1794         /* If l2_ivlan == 0 and l2_ivlan_mask != 0, only
1795          * untagged packets are received
1796          *
1797          * If l2_ivlan != 0 and l2_ivlan_mask != 0, untagged
1798          * packets and only the programmed vlan's packets are received
1799          */
1800         filter->l2_ivlan = vlan_id;
1801         filter->l2_ivlan_mask = 0x0FFF;
1802         filter->enables |= en;
1803         filter->flags |= HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_OUTERMOST;
1804
1805         rc = bnxt_hwrm_set_l2_filter(bp, vnic->fw_vnic_id, filter);
1806         if (rc) {
1807                 /* Free the newly allocated filter as we were
1808                  * not able to create the filter in hardware.
1809                  */
1810                 filter->fw_l2_filter_id = UINT64_MAX;
1811                 STAILQ_INSERT_TAIL(&bp->free_filter_list, filter, next);
1812                 return rc;
1813         } else {
1814                 /* Add this new filter to the list */
1815                 if (vlan_id == 0) {
1816                         filter->dflt = true;
1817                         STAILQ_INSERT_HEAD(&vnic->filter, filter, next);
1818                 } else {
1819                         STAILQ_INSERT_TAIL(&vnic->filter, filter, next);
1820                 }
1821         }
1822
1823         PMD_DRV_LOG(INFO,
1824                     "Added Vlan filter for %d\n", vlan_id);
1825         return rc;
1826 }
1827
1828 static int bnxt_vlan_filter_set_op(struct rte_eth_dev *eth_dev,
1829                 uint16_t vlan_id, int on)
1830 {
1831         struct bnxt *bp = eth_dev->data->dev_private;
1832         int rc;
1833
1834         rc = is_bnxt_in_error(bp);
1835         if (rc)
1836                 return rc;
1837
1838         /* These operations apply to ALL existing MAC/VLAN filters */
1839         if (on)
1840                 return bnxt_add_vlan_filter(bp, vlan_id);
1841         else
1842                 return bnxt_del_vlan_filter(bp, vlan_id);
1843 }
1844
1845 static int bnxt_del_dflt_mac_filter(struct bnxt *bp,
1846                                     struct bnxt_vnic_info *vnic)
1847 {
1848         struct bnxt_filter_info *filter;
1849         int rc;
1850
1851         filter = STAILQ_FIRST(&vnic->filter);
1852         while (filter) {
1853                 if (filter->dflt &&
1854                     !memcmp(filter->l2_addr, bp->mac_addr,
1855                             RTE_ETHER_ADDR_LEN)) {
1856                         rc = bnxt_hwrm_clear_l2_filter(bp, filter);
1857                         if (rc)
1858                                 return rc;
1859                         filter->dflt = false;
1860                         STAILQ_REMOVE(&vnic->filter, filter,
1861                                       bnxt_filter_info, next);
1862                         STAILQ_INSERT_TAIL(&bp->free_filter_list,
1863                                            filter, next);
1864                         filter->fw_l2_filter_id = -1;
1865                         break;
1866                 }
1867                 filter = STAILQ_NEXT(filter, next);
1868         }
1869         return 0;
1870 }
1871
1872 static int
1873 bnxt_vlan_offload_set_op(struct rte_eth_dev *dev, int mask)
1874 {
1875         struct bnxt *bp = dev->data->dev_private;
1876         uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads;
1877         struct bnxt_vnic_info *vnic;
1878         unsigned int i;
1879         int rc;
1880
1881         rc = is_bnxt_in_error(bp);
1882         if (rc)
1883                 return rc;
1884
1885         vnic = BNXT_GET_DEFAULT_VNIC(bp);
1886         if (!(rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER)) {
1887                 /* Remove any VLAN filters programmed */
1888                 for (i = 0; i < 4095; i++)
1889                         bnxt_del_vlan_filter(bp, i);
1890
1891                 rc = bnxt_add_mac_filter(bp, vnic, NULL, 0);
1892                 if (rc)
1893                         return rc;
1894         } else {
1895                 /* Default filter will allow packets that match the
1896                  * dest mac. So, it has to be deleted, otherwise, we
1897                  * will endup receiving vlan packets for which the
1898                  * filter is not programmed, when hw-vlan-filter
1899                  * configuration is ON
1900                  */
1901                 bnxt_del_dflt_mac_filter(bp, vnic);
1902                 /* This filter will allow only untagged packets */
1903                 bnxt_add_vlan_filter(bp, 0);
1904         }
1905         PMD_DRV_LOG(DEBUG, "VLAN Filtering: %d\n",
1906                     !!(rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER));
1907
1908         if (mask & ETH_VLAN_STRIP_MASK) {
1909                 /* Enable or disable VLAN stripping */
1910                 for (i = 0; i < bp->nr_vnics; i++) {
1911                         struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
1912                         if (rx_offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
1913                                 vnic->vlan_strip = true;
1914                         else
1915                                 vnic->vlan_strip = false;
1916                         bnxt_hwrm_vnic_cfg(bp, vnic);
1917                 }
1918                 PMD_DRV_LOG(DEBUG, "VLAN Strip Offload: %d\n",
1919                         !!(rx_offloads & DEV_RX_OFFLOAD_VLAN_STRIP));
1920         }
1921
1922         if (mask & ETH_VLAN_EXTEND_MASK) {
1923                 if (rx_offloads & DEV_RX_OFFLOAD_VLAN_EXTEND)
1924                         PMD_DRV_LOG(DEBUG, "Extend VLAN supported\n");
1925                 else
1926                         PMD_DRV_LOG(INFO, "Extend VLAN unsupported\n");
1927         }
1928
1929         return 0;
1930 }
1931
1932 static int
1933 bnxt_vlan_tpid_set_op(struct rte_eth_dev *dev, enum rte_vlan_type vlan_type,
1934                       uint16_t tpid)
1935 {
1936         struct bnxt *bp = dev->data->dev_private;
1937         int qinq = dev->data->dev_conf.rxmode.offloads &
1938                    DEV_RX_OFFLOAD_VLAN_EXTEND;
1939
1940         if (vlan_type != ETH_VLAN_TYPE_INNER &&
1941             vlan_type != ETH_VLAN_TYPE_OUTER) {
1942                 PMD_DRV_LOG(ERR,
1943                             "Unsupported vlan type.");
1944                 return -EINVAL;
1945         }
1946         if (!qinq) {
1947                 PMD_DRV_LOG(ERR,
1948                             "QinQ not enabled. Needs to be ON as we can "
1949                             "accelerate only outer vlan\n");
1950                 return -EINVAL;
1951         }
1952
1953         if (vlan_type == ETH_VLAN_TYPE_OUTER) {
1954                 switch (tpid) {
1955                 case RTE_ETHER_TYPE_QINQ:
1956                         bp->outer_tpid_bd =
1957                                 TX_BD_LONG_CFA_META_VLAN_TPID_TPID88A8;
1958                                 break;
1959                 case RTE_ETHER_TYPE_VLAN:
1960                         bp->outer_tpid_bd =
1961                                 TX_BD_LONG_CFA_META_VLAN_TPID_TPID8100;
1962                                 break;
1963                 case 0x9100:
1964                         bp->outer_tpid_bd =
1965                                 TX_BD_LONG_CFA_META_VLAN_TPID_TPID9100;
1966                                 break;
1967                 case 0x9200:
1968                         bp->outer_tpid_bd =
1969                                 TX_BD_LONG_CFA_META_VLAN_TPID_TPID9200;
1970                                 break;
1971                 case 0x9300:
1972                         bp->outer_tpid_bd =
1973                                  TX_BD_LONG_CFA_META_VLAN_TPID_TPID9300;
1974                                 break;
1975                 default:
1976                         PMD_DRV_LOG(ERR, "Invalid TPID: %x\n", tpid);
1977                         return -EINVAL;
1978                 }
1979                 bp->outer_tpid_bd |= tpid;
1980                 PMD_DRV_LOG(INFO, "outer_tpid_bd = %x\n", bp->outer_tpid_bd);
1981         } else if (vlan_type == ETH_VLAN_TYPE_INNER) {
1982                 PMD_DRV_LOG(ERR,
1983                             "Can accelerate only outer vlan in QinQ\n");
1984                 return -EINVAL;
1985         }
1986
1987         return 0;
1988 }
1989
1990 static int
1991 bnxt_set_default_mac_addr_op(struct rte_eth_dev *dev,
1992                              struct rte_ether_addr *addr)
1993 {
1994         struct bnxt *bp = dev->data->dev_private;
1995         /* Default Filter is tied to VNIC 0 */
1996         struct bnxt_vnic_info *vnic = &bp->vnic_info[0];
1997         struct bnxt_filter_info *filter;
1998         int rc;
1999
2000         rc = is_bnxt_in_error(bp);
2001         if (rc)
2002                 return rc;
2003
2004         if (BNXT_VF(bp) && !BNXT_VF_IS_TRUSTED(bp))
2005                 return -EPERM;
2006
2007         if (rte_is_zero_ether_addr(addr))
2008                 return -EINVAL;
2009
2010         STAILQ_FOREACH(filter, &vnic->filter, next) {
2011                 /* Default Filter is at Index 0 */
2012                 if (filter->mac_index != 0)
2013                         continue;
2014
2015                 memcpy(filter->l2_addr, addr, RTE_ETHER_ADDR_LEN);
2016                 memset(filter->l2_addr_mask, 0xff, RTE_ETHER_ADDR_LEN);
2017                 filter->flags |= HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_PATH_RX |
2018                         HWRM_CFA_L2_FILTER_ALLOC_INPUT_FLAGS_OUTERMOST;
2019                 filter->enables |=
2020                         HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR |
2021                         HWRM_CFA_L2_FILTER_ALLOC_INPUT_ENABLES_L2_ADDR_MASK;
2022
2023                 rc = bnxt_hwrm_set_l2_filter(bp, vnic->fw_vnic_id, filter);
2024                 if (rc) {
2025                         memcpy(filter->l2_addr, bp->mac_addr,
2026                                RTE_ETHER_ADDR_LEN);
2027                         return rc;
2028                 }
2029
2030                 memcpy(bp->mac_addr, addr, RTE_ETHER_ADDR_LEN);
2031                 PMD_DRV_LOG(DEBUG, "Set MAC addr\n");
2032                 return 0;
2033         }
2034
2035         return 0;
2036 }
2037
2038 static int
2039 bnxt_dev_set_mc_addr_list_op(struct rte_eth_dev *eth_dev,
2040                           struct rte_ether_addr *mc_addr_set,
2041                           uint32_t nb_mc_addr)
2042 {
2043         struct bnxt *bp = eth_dev->data->dev_private;
2044         char *mc_addr_list = (char *)mc_addr_set;
2045         struct bnxt_vnic_info *vnic;
2046         uint32_t off = 0, i = 0;
2047         int rc;
2048
2049         rc = is_bnxt_in_error(bp);
2050         if (rc)
2051                 return rc;
2052
2053         vnic = BNXT_GET_DEFAULT_VNIC(bp);
2054
2055         if (nb_mc_addr > BNXT_MAX_MC_ADDRS) {
2056                 vnic->flags |= BNXT_VNIC_INFO_ALLMULTI;
2057                 goto allmulti;
2058         }
2059
2060         /* TODO Check for Duplicate mcast addresses */
2061         vnic->flags &= ~BNXT_VNIC_INFO_ALLMULTI;
2062         for (i = 0; i < nb_mc_addr; i++) {
2063                 memcpy(vnic->mc_list + off, &mc_addr_list[i],
2064                         RTE_ETHER_ADDR_LEN);
2065                 off += RTE_ETHER_ADDR_LEN;
2066         }
2067
2068         vnic->mc_addr_cnt = i;
2069         if (vnic->mc_addr_cnt)
2070                 vnic->flags |= BNXT_VNIC_INFO_MCAST;
2071         else
2072                 vnic->flags &= ~BNXT_VNIC_INFO_MCAST;
2073
2074 allmulti:
2075         return bnxt_hwrm_cfa_l2_set_rx_mask(bp, vnic, 0, NULL);
2076 }
2077
2078 static int
2079 bnxt_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size)
2080 {
2081         struct bnxt *bp = dev->data->dev_private;
2082         uint8_t fw_major = (bp->fw_ver >> 24) & 0xff;
2083         uint8_t fw_minor = (bp->fw_ver >> 16) & 0xff;
2084         uint8_t fw_updt = (bp->fw_ver >> 8) & 0xff;
2085         int ret;
2086
2087         ret = snprintf(fw_version, fw_size, "%d.%d.%d",
2088                         fw_major, fw_minor, fw_updt);
2089
2090         ret += 1; /* add the size of '\0' */
2091         if (fw_size < (uint32_t)ret)
2092                 return ret;
2093         else
2094                 return 0;
2095 }
2096
2097 static void
2098 bnxt_rxq_info_get_op(struct rte_eth_dev *dev, uint16_t queue_id,
2099         struct rte_eth_rxq_info *qinfo)
2100 {
2101         struct bnxt_rx_queue *rxq;
2102
2103         rxq = dev->data->rx_queues[queue_id];
2104
2105         qinfo->mp = rxq->mb_pool;
2106         qinfo->scattered_rx = dev->data->scattered_rx;
2107         qinfo->nb_desc = rxq->nb_rx_desc;
2108
2109         qinfo->conf.rx_free_thresh = rxq->rx_free_thresh;
2110         qinfo->conf.rx_drop_en = 0;
2111         qinfo->conf.rx_deferred_start = rxq->rx_deferred_start;
2112 }
2113
2114 static void
2115 bnxt_txq_info_get_op(struct rte_eth_dev *dev, uint16_t queue_id,
2116         struct rte_eth_txq_info *qinfo)
2117 {
2118         struct bnxt_tx_queue *txq;
2119
2120         txq = dev->data->tx_queues[queue_id];
2121
2122         qinfo->nb_desc = txq->nb_tx_desc;
2123
2124         qinfo->conf.tx_thresh.pthresh = txq->pthresh;
2125         qinfo->conf.tx_thresh.hthresh = txq->hthresh;
2126         qinfo->conf.tx_thresh.wthresh = txq->wthresh;
2127
2128         qinfo->conf.tx_free_thresh = txq->tx_free_thresh;
2129         qinfo->conf.tx_rs_thresh = 0;
2130         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
2131 }
2132
2133 int bnxt_mtu_set_op(struct rte_eth_dev *eth_dev, uint16_t new_mtu)
2134 {
2135         struct bnxt *bp = eth_dev->data->dev_private;
2136         uint32_t new_pkt_size;
2137         uint32_t rc = 0;
2138         uint32_t i;
2139
2140         rc = is_bnxt_in_error(bp);
2141         if (rc)
2142                 return rc;
2143
2144         /* Exit if receive queues are not configured yet */
2145         if (!eth_dev->data->nb_rx_queues)
2146                 return rc;
2147
2148         new_pkt_size = new_mtu + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN +
2149                        VLAN_TAG_SIZE * BNXT_NUM_VLANS;
2150
2151 #ifdef RTE_ARCH_X86
2152         /*
2153          * If vector-mode tx/rx is active, disallow any MTU change that would
2154          * require scattered receive support.
2155          */
2156         if (eth_dev->data->dev_started &&
2157             (eth_dev->rx_pkt_burst == bnxt_recv_pkts_vec ||
2158              eth_dev->tx_pkt_burst == bnxt_xmit_pkts_vec) &&
2159             (new_pkt_size >
2160              eth_dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM)) {
2161                 PMD_DRV_LOG(ERR,
2162                             "MTU change would require scattered rx support. ");
2163                 PMD_DRV_LOG(ERR, "Stop port before changing MTU.\n");
2164                 return -EINVAL;
2165         }
2166 #endif
2167
2168         if (new_mtu > RTE_ETHER_MTU) {
2169                 bp->flags |= BNXT_FLAG_JUMBO;
2170                 bp->eth_dev->data->dev_conf.rxmode.offloads |=
2171                         DEV_RX_OFFLOAD_JUMBO_FRAME;
2172         } else {
2173                 bp->eth_dev->data->dev_conf.rxmode.offloads &=
2174                         ~DEV_RX_OFFLOAD_JUMBO_FRAME;
2175                 bp->flags &= ~BNXT_FLAG_JUMBO;
2176         }
2177
2178         /* Is there a change in mtu setting? */
2179         if (eth_dev->data->dev_conf.rxmode.max_rx_pkt_len == new_pkt_size)
2180                 return rc;
2181
2182         for (i = 0; i < bp->nr_vnics; i++) {
2183                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
2184                 uint16_t size = 0;
2185
2186                 vnic->mru = BNXT_VNIC_MRU(new_mtu);
2187                 rc = bnxt_hwrm_vnic_cfg(bp, vnic);
2188                 if (rc)
2189                         break;
2190
2191                 size = rte_pktmbuf_data_room_size(bp->rx_queues[0]->mb_pool);
2192                 size -= RTE_PKTMBUF_HEADROOM;
2193
2194                 if (size < new_mtu) {
2195                         rc = bnxt_hwrm_vnic_plcmode_cfg(bp, vnic);
2196                         if (rc)
2197                                 return rc;
2198                 }
2199         }
2200
2201         if (!rc)
2202                 eth_dev->data->dev_conf.rxmode.max_rx_pkt_len = new_pkt_size;
2203
2204         PMD_DRV_LOG(INFO, "New MTU is %d\n", new_mtu);
2205
2206         return rc;
2207 }
2208
2209 static int
2210 bnxt_vlan_pvid_set_op(struct rte_eth_dev *dev, uint16_t pvid, int on)
2211 {
2212         struct bnxt *bp = dev->data->dev_private;
2213         uint16_t vlan = bp->vlan;
2214         int rc;
2215
2216         rc = is_bnxt_in_error(bp);
2217         if (rc)
2218                 return rc;
2219
2220         if (!BNXT_SINGLE_PF(bp) || BNXT_VF(bp)) {
2221                 PMD_DRV_LOG(ERR,
2222                         "PVID cannot be modified for this function\n");
2223                 return -ENOTSUP;
2224         }
2225         bp->vlan = on ? pvid : 0;
2226
2227         rc = bnxt_hwrm_set_default_vlan(bp, 0, 0);
2228         if (rc)
2229                 bp->vlan = vlan;
2230         return rc;
2231 }
2232
2233 static int
2234 bnxt_dev_led_on_op(struct rte_eth_dev *dev)
2235 {
2236         struct bnxt *bp = dev->data->dev_private;
2237         int rc;
2238
2239         rc = is_bnxt_in_error(bp);
2240         if (rc)
2241                 return rc;
2242
2243         return bnxt_hwrm_port_led_cfg(bp, true);
2244 }
2245
2246 static int
2247 bnxt_dev_led_off_op(struct rte_eth_dev *dev)
2248 {
2249         struct bnxt *bp = dev->data->dev_private;
2250         int rc;
2251
2252         rc = is_bnxt_in_error(bp);
2253         if (rc)
2254                 return rc;
2255
2256         return bnxt_hwrm_port_led_cfg(bp, false);
2257 }
2258
2259 static uint32_t
2260 bnxt_rx_queue_count_op(struct rte_eth_dev *dev, uint16_t rx_queue_id)
2261 {
2262         struct bnxt *bp = (struct bnxt *)dev->data->dev_private;
2263         uint32_t desc = 0, raw_cons = 0, cons;
2264         struct bnxt_cp_ring_info *cpr;
2265         struct bnxt_rx_queue *rxq;
2266         struct rx_pkt_cmpl *rxcmp;
2267         int rc;
2268
2269         rc = is_bnxt_in_error(bp);
2270         if (rc)
2271                 return rc;
2272
2273         rxq = dev->data->rx_queues[rx_queue_id];
2274         cpr = rxq->cp_ring;
2275         raw_cons = cpr->cp_raw_cons;
2276
2277         while (1) {
2278                 cons = RING_CMP(cpr->cp_ring_struct, raw_cons);
2279                 rte_prefetch0(&cpr->cp_desc_ring[cons]);
2280                 rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons];
2281
2282                 if (!CMP_VALID(rxcmp, raw_cons, cpr->cp_ring_struct)) {
2283                         break;
2284                 } else {
2285                         raw_cons++;
2286                         desc++;
2287                 }
2288         }
2289
2290         return desc;
2291 }
2292
2293 static int
2294 bnxt_rx_descriptor_status_op(void *rx_queue, uint16_t offset)
2295 {
2296         struct bnxt_rx_queue *rxq = (struct bnxt_rx_queue *)rx_queue;
2297         struct bnxt_rx_ring_info *rxr;
2298         struct bnxt_cp_ring_info *cpr;
2299         struct bnxt_sw_rx_bd *rx_buf;
2300         struct rx_pkt_cmpl *rxcmp;
2301         uint32_t cons, cp_cons;
2302         int rc;
2303
2304         if (!rxq)
2305                 return -EINVAL;
2306
2307         rc = is_bnxt_in_error(rxq->bp);
2308         if (rc)
2309                 return rc;
2310
2311         cpr = rxq->cp_ring;
2312         rxr = rxq->rx_ring;
2313
2314         if (offset >= rxq->nb_rx_desc)
2315                 return -EINVAL;
2316
2317         cons = RING_CMP(cpr->cp_ring_struct, offset);
2318         cp_cons = cpr->cp_raw_cons;
2319         rxcmp = (struct rx_pkt_cmpl *)&cpr->cp_desc_ring[cons];
2320
2321         if (cons > cp_cons) {
2322                 if (CMPL_VALID(rxcmp, cpr->valid))
2323                         return RTE_ETH_RX_DESC_DONE;
2324         } else {
2325                 if (CMPL_VALID(rxcmp, !cpr->valid))
2326                         return RTE_ETH_RX_DESC_DONE;
2327         }
2328         rx_buf = &rxr->rx_buf_ring[cons];
2329         if (rx_buf->mbuf == NULL)
2330                 return RTE_ETH_RX_DESC_UNAVAIL;
2331
2332
2333         return RTE_ETH_RX_DESC_AVAIL;
2334 }
2335
2336 static int
2337 bnxt_tx_descriptor_status_op(void *tx_queue, uint16_t offset)
2338 {
2339         struct bnxt_tx_queue *txq = (struct bnxt_tx_queue *)tx_queue;
2340         struct bnxt_tx_ring_info *txr;
2341         struct bnxt_cp_ring_info *cpr;
2342         struct bnxt_sw_tx_bd *tx_buf;
2343         struct tx_pkt_cmpl *txcmp;
2344         uint32_t cons, cp_cons;
2345         int rc;
2346
2347         if (!txq)
2348                 return -EINVAL;
2349
2350         rc = is_bnxt_in_error(txq->bp);
2351         if (rc)
2352                 return rc;
2353
2354         cpr = txq->cp_ring;
2355         txr = txq->tx_ring;
2356
2357         if (offset >= txq->nb_tx_desc)
2358                 return -EINVAL;
2359
2360         cons = RING_CMP(cpr->cp_ring_struct, offset);
2361         txcmp = (struct tx_pkt_cmpl *)&cpr->cp_desc_ring[cons];
2362         cp_cons = cpr->cp_raw_cons;
2363
2364         if (cons > cp_cons) {
2365                 if (CMPL_VALID(txcmp, cpr->valid))
2366                         return RTE_ETH_TX_DESC_UNAVAIL;
2367         } else {
2368                 if (CMPL_VALID(txcmp, !cpr->valid))
2369                         return RTE_ETH_TX_DESC_UNAVAIL;
2370         }
2371         tx_buf = &txr->tx_buf_ring[cons];
2372         if (tx_buf->mbuf == NULL)
2373                 return RTE_ETH_TX_DESC_DONE;
2374
2375         return RTE_ETH_TX_DESC_FULL;
2376 }
2377
2378 static struct bnxt_filter_info *
2379 bnxt_match_and_validate_ether_filter(struct bnxt *bp,
2380                                 struct rte_eth_ethertype_filter *efilter,
2381                                 struct bnxt_vnic_info *vnic0,
2382                                 struct bnxt_vnic_info *vnic,
2383                                 int *ret)
2384 {
2385         struct bnxt_filter_info *mfilter = NULL;
2386         int match = 0;
2387         *ret = 0;
2388
2389         if (efilter->ether_type == RTE_ETHER_TYPE_IPV4 ||
2390                 efilter->ether_type == RTE_ETHER_TYPE_IPV6) {
2391                 PMD_DRV_LOG(ERR, "invalid ether_type(0x%04x) in"
2392                         " ethertype filter.", efilter->ether_type);
2393                 *ret = -EINVAL;
2394                 goto exit;
2395         }
2396         if (efilter->queue >= bp->rx_nr_rings) {
2397                 PMD_DRV_LOG(ERR, "Invalid queue %d\n", efilter->queue);
2398                 *ret = -EINVAL;
2399                 goto exit;
2400         }
2401
2402         vnic0 = &bp->vnic_info[0];
2403         vnic = &bp->vnic_info[efilter->queue];
2404         if (vnic == NULL) {
2405                 PMD_DRV_LOG(ERR, "Invalid queue %d\n", efilter->queue);
2406                 *ret = -EINVAL;
2407                 goto exit;
2408         }
2409
2410         if (efilter->flags & RTE_ETHTYPE_FLAGS_DROP) {
2411                 STAILQ_FOREACH(mfilter, &vnic0->filter, next) {
2412                         if ((!memcmp(efilter->mac_addr.addr_bytes,
2413                                      mfilter->l2_addr, RTE_ETHER_ADDR_LEN) &&
2414                              mfilter->flags ==
2415                              HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_DROP &&
2416                              mfilter->ethertype == efilter->ether_type)) {
2417                                 match = 1;
2418                                 break;
2419                         }
2420                 }
2421         } else {
2422                 STAILQ_FOREACH(mfilter, &vnic->filter, next)
2423                         if ((!memcmp(efilter->mac_addr.addr_bytes,
2424                                      mfilter->l2_addr, RTE_ETHER_ADDR_LEN) &&
2425                              mfilter->ethertype == efilter->ether_type &&
2426                              mfilter->flags ==
2427                              HWRM_CFA_L2_FILTER_CFG_INPUT_FLAGS_PATH_RX)) {
2428                                 match = 1;
2429                                 break;
2430                         }
2431         }
2432
2433         if (match)
2434                 *ret = -EEXIST;
2435
2436 exit:
2437         return mfilter;
2438 }
2439
2440 static int
2441 bnxt_ethertype_filter(struct rte_eth_dev *dev,
2442                         enum rte_filter_op filter_op,
2443                         void *arg)
2444 {
2445         struct bnxt *bp = dev->data->dev_private;
2446         struct rte_eth_ethertype_filter *efilter =
2447                         (struct rte_eth_ethertype_filter *)arg;
2448         struct bnxt_filter_info *bfilter, *filter1;
2449         struct bnxt_vnic_info *vnic, *vnic0;
2450         int ret;
2451
2452         if (filter_op == RTE_ETH_FILTER_NOP)
2453                 return 0;
2454
2455         if (arg == NULL) {
2456                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
2457                             filter_op);
2458                 return -EINVAL;
2459         }
2460
2461         vnic0 = &bp->vnic_info[0];
2462         vnic = &bp->vnic_info[efilter->queue];
2463
2464         switch (filter_op) {
2465         case RTE_ETH_FILTER_ADD:
2466                 bnxt_match_and_validate_ether_filter(bp, efilter,
2467                                                         vnic0, vnic, &ret);
2468                 if (ret < 0)
2469                         return ret;
2470
2471                 bfilter = bnxt_get_unused_filter(bp);
2472                 if (bfilter == NULL) {
2473                         PMD_DRV_LOG(ERR,
2474                                 "Not enough resources for a new filter.\n");
2475                         return -ENOMEM;
2476                 }
2477                 bfilter->filter_type = HWRM_CFA_NTUPLE_FILTER;
2478                 memcpy(bfilter->l2_addr, efilter->mac_addr.addr_bytes,
2479                        RTE_ETHER_ADDR_LEN);
2480                 memcpy(bfilter->dst_macaddr, efilter->mac_addr.addr_bytes,
2481                        RTE_ETHER_ADDR_LEN);
2482                 bfilter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_MACADDR;
2483                 bfilter->ethertype = efilter->ether_type;
2484                 bfilter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE;
2485
2486                 filter1 = bnxt_get_l2_filter(bp, bfilter, vnic0);
2487                 if (filter1 == NULL) {
2488                         ret = -EINVAL;
2489                         goto cleanup;
2490                 }
2491                 bfilter->enables |=
2492                         HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_L2_FILTER_ID;
2493                 bfilter->fw_l2_filter_id = filter1->fw_l2_filter_id;
2494
2495                 bfilter->dst_id = vnic->fw_vnic_id;
2496
2497                 if (efilter->flags & RTE_ETHTYPE_FLAGS_DROP) {
2498                         bfilter->flags =
2499                                 HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_DROP;
2500                 }
2501
2502                 ret = bnxt_hwrm_set_ntuple_filter(bp, bfilter->dst_id, bfilter);
2503                 if (ret)
2504                         goto cleanup;
2505                 STAILQ_INSERT_TAIL(&vnic->filter, bfilter, next);
2506                 break;
2507         case RTE_ETH_FILTER_DELETE:
2508                 filter1 = bnxt_match_and_validate_ether_filter(bp, efilter,
2509                                                         vnic0, vnic, &ret);
2510                 if (ret == -EEXIST) {
2511                         ret = bnxt_hwrm_clear_ntuple_filter(bp, filter1);
2512
2513                         STAILQ_REMOVE(&vnic->filter, filter1, bnxt_filter_info,
2514                                       next);
2515                         bnxt_free_filter(bp, filter1);
2516                 } else if (ret == 0) {
2517                         PMD_DRV_LOG(ERR, "No matching filter found\n");
2518                 }
2519                 break;
2520         default:
2521                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
2522                 ret = -EINVAL;
2523                 goto error;
2524         }
2525         return ret;
2526 cleanup:
2527         bnxt_free_filter(bp, bfilter);
2528 error:
2529         return ret;
2530 }
2531
2532 static inline int
2533 parse_ntuple_filter(struct bnxt *bp,
2534                     struct rte_eth_ntuple_filter *nfilter,
2535                     struct bnxt_filter_info *bfilter)
2536 {
2537         uint32_t en = 0;
2538
2539         if (nfilter->queue >= bp->rx_nr_rings) {
2540                 PMD_DRV_LOG(ERR, "Invalid queue %d\n", nfilter->queue);
2541                 return -EINVAL;
2542         }
2543
2544         switch (nfilter->dst_port_mask) {
2545         case UINT16_MAX:
2546                 bfilter->dst_port_mask = -1;
2547                 bfilter->dst_port = nfilter->dst_port;
2548                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT |
2549                         NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK;
2550                 break;
2551         default:
2552                 PMD_DRV_LOG(ERR, "invalid dst_port mask.");
2553                 return -EINVAL;
2554         }
2555
2556         bfilter->ip_addr_type = NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV4;
2557         en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO;
2558
2559         switch (nfilter->proto_mask) {
2560         case UINT8_MAX:
2561                 if (nfilter->proto == 17) /* IPPROTO_UDP */
2562                         bfilter->ip_protocol = 17;
2563                 else if (nfilter->proto == 6) /* IPPROTO_TCP */
2564                         bfilter->ip_protocol = 6;
2565                 else
2566                         return -EINVAL;
2567                 en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO;
2568                 break;
2569         default:
2570                 PMD_DRV_LOG(ERR, "invalid protocol mask.");
2571                 return -EINVAL;
2572         }
2573
2574         switch (nfilter->dst_ip_mask) {
2575         case UINT32_MAX:
2576                 bfilter->dst_ipaddr_mask[0] = -1;
2577                 bfilter->dst_ipaddr[0] = nfilter->dst_ip;
2578                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR |
2579                         NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK;
2580                 break;
2581         default:
2582                 PMD_DRV_LOG(ERR, "invalid dst_ip mask.");
2583                 return -EINVAL;
2584         }
2585
2586         switch (nfilter->src_ip_mask) {
2587         case UINT32_MAX:
2588                 bfilter->src_ipaddr_mask[0] = -1;
2589                 bfilter->src_ipaddr[0] = nfilter->src_ip;
2590                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR |
2591                         NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK;
2592                 break;
2593         default:
2594                 PMD_DRV_LOG(ERR, "invalid src_ip mask.");
2595                 return -EINVAL;
2596         }
2597
2598         switch (nfilter->src_port_mask) {
2599         case UINT16_MAX:
2600                 bfilter->src_port_mask = -1;
2601                 bfilter->src_port = nfilter->src_port;
2602                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT |
2603                         NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK;
2604                 break;
2605         default:
2606                 PMD_DRV_LOG(ERR, "invalid src_port mask.");
2607                 return -EINVAL;
2608         }
2609
2610         //TODO Priority
2611         //nfilter->priority = (uint8_t)filter->priority;
2612
2613         bfilter->enables = en;
2614         return 0;
2615 }
2616
2617 static struct bnxt_filter_info*
2618 bnxt_match_ntuple_filter(struct bnxt *bp,
2619                          struct bnxt_filter_info *bfilter,
2620                          struct bnxt_vnic_info **mvnic)
2621 {
2622         struct bnxt_filter_info *mfilter = NULL;
2623         int i;
2624
2625         for (i = bp->nr_vnics - 1; i >= 0; i--) {
2626                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
2627                 STAILQ_FOREACH(mfilter, &vnic->filter, next) {
2628                         if (bfilter->src_ipaddr[0] == mfilter->src_ipaddr[0] &&
2629                             bfilter->src_ipaddr_mask[0] ==
2630                             mfilter->src_ipaddr_mask[0] &&
2631                             bfilter->src_port == mfilter->src_port &&
2632                             bfilter->src_port_mask == mfilter->src_port_mask &&
2633                             bfilter->dst_ipaddr[0] == mfilter->dst_ipaddr[0] &&
2634                             bfilter->dst_ipaddr_mask[0] ==
2635                             mfilter->dst_ipaddr_mask[0] &&
2636                             bfilter->dst_port == mfilter->dst_port &&
2637                             bfilter->dst_port_mask == mfilter->dst_port_mask &&
2638                             bfilter->flags == mfilter->flags &&
2639                             bfilter->enables == mfilter->enables) {
2640                                 if (mvnic)
2641                                         *mvnic = vnic;
2642                                 return mfilter;
2643                         }
2644                 }
2645         }
2646         return NULL;
2647 }
2648
2649 static int
2650 bnxt_cfg_ntuple_filter(struct bnxt *bp,
2651                        struct rte_eth_ntuple_filter *nfilter,
2652                        enum rte_filter_op filter_op)
2653 {
2654         struct bnxt_filter_info *bfilter, *mfilter, *filter1;
2655         struct bnxt_vnic_info *vnic, *vnic0, *mvnic;
2656         int ret;
2657
2658         if (nfilter->flags != RTE_5TUPLE_FLAGS) {
2659                 PMD_DRV_LOG(ERR, "only 5tuple is supported.");
2660                 return -EINVAL;
2661         }
2662
2663         if (nfilter->flags & RTE_NTUPLE_FLAGS_TCP_FLAG) {
2664                 PMD_DRV_LOG(ERR, "Ntuple filter: TCP flags not supported\n");
2665                 return -EINVAL;
2666         }
2667
2668         bfilter = bnxt_get_unused_filter(bp);
2669         if (bfilter == NULL) {
2670                 PMD_DRV_LOG(ERR,
2671                         "Not enough resources for a new filter.\n");
2672                 return -ENOMEM;
2673         }
2674         ret = parse_ntuple_filter(bp, nfilter, bfilter);
2675         if (ret < 0)
2676                 goto free_filter;
2677
2678         vnic = &bp->vnic_info[nfilter->queue];
2679         vnic0 = &bp->vnic_info[0];
2680         filter1 = STAILQ_FIRST(&vnic0->filter);
2681         if (filter1 == NULL) {
2682                 ret = -EINVAL;
2683                 goto free_filter;
2684         }
2685
2686         bfilter->dst_id = vnic->fw_vnic_id;
2687         bfilter->fw_l2_filter_id = filter1->fw_l2_filter_id;
2688         bfilter->enables |=
2689                 HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_L2_FILTER_ID;
2690         bfilter->ethertype = 0x800;
2691         bfilter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE;
2692
2693         mfilter = bnxt_match_ntuple_filter(bp, bfilter, &mvnic);
2694
2695         if (mfilter != NULL && filter_op == RTE_ETH_FILTER_ADD &&
2696             bfilter->dst_id == mfilter->dst_id) {
2697                 PMD_DRV_LOG(ERR, "filter exists.\n");
2698                 ret = -EEXIST;
2699                 goto free_filter;
2700         } else if (mfilter != NULL && filter_op == RTE_ETH_FILTER_ADD &&
2701                    bfilter->dst_id != mfilter->dst_id) {
2702                 mfilter->dst_id = vnic->fw_vnic_id;
2703                 ret = bnxt_hwrm_set_ntuple_filter(bp, mfilter->dst_id, mfilter);
2704                 STAILQ_REMOVE(&mvnic->filter, mfilter, bnxt_filter_info, next);
2705                 STAILQ_INSERT_TAIL(&vnic->filter, mfilter, next);
2706                 PMD_DRV_LOG(ERR, "filter with matching pattern exists.\n");
2707                 PMD_DRV_LOG(ERR, " Updated it to the new destination queue\n");
2708                 goto free_filter;
2709         }
2710         if (mfilter == NULL && filter_op == RTE_ETH_FILTER_DELETE) {
2711                 PMD_DRV_LOG(ERR, "filter doesn't exist.");
2712                 ret = -ENOENT;
2713                 goto free_filter;
2714         }
2715
2716         if (filter_op == RTE_ETH_FILTER_ADD) {
2717                 bfilter->filter_type = HWRM_CFA_NTUPLE_FILTER;
2718                 ret = bnxt_hwrm_set_ntuple_filter(bp, bfilter->dst_id, bfilter);
2719                 if (ret)
2720                         goto free_filter;
2721                 STAILQ_INSERT_TAIL(&vnic->filter, bfilter, next);
2722         } else {
2723                 if (mfilter == NULL) {
2724                         /* This should not happen. But for Coverity! */
2725                         ret = -ENOENT;
2726                         goto free_filter;
2727                 }
2728                 ret = bnxt_hwrm_clear_ntuple_filter(bp, mfilter);
2729
2730                 STAILQ_REMOVE(&vnic->filter, mfilter, bnxt_filter_info, next);
2731                 bnxt_free_filter(bp, mfilter);
2732                 mfilter->fw_l2_filter_id = -1;
2733                 bnxt_free_filter(bp, bfilter);
2734                 bfilter->fw_l2_filter_id = -1;
2735         }
2736
2737         return 0;
2738 free_filter:
2739         bfilter->fw_l2_filter_id = -1;
2740         bnxt_free_filter(bp, bfilter);
2741         return ret;
2742 }
2743
2744 static int
2745 bnxt_ntuple_filter(struct rte_eth_dev *dev,
2746                         enum rte_filter_op filter_op,
2747                         void *arg)
2748 {
2749         struct bnxt *bp = dev->data->dev_private;
2750         int ret;
2751
2752         if (filter_op == RTE_ETH_FILTER_NOP)
2753                 return 0;
2754
2755         if (arg == NULL) {
2756                 PMD_DRV_LOG(ERR, "arg shouldn't be NULL for operation %u.",
2757                             filter_op);
2758                 return -EINVAL;
2759         }
2760
2761         switch (filter_op) {
2762         case RTE_ETH_FILTER_ADD:
2763                 ret = bnxt_cfg_ntuple_filter(bp,
2764                         (struct rte_eth_ntuple_filter *)arg,
2765                         filter_op);
2766                 break;
2767         case RTE_ETH_FILTER_DELETE:
2768                 ret = bnxt_cfg_ntuple_filter(bp,
2769                         (struct rte_eth_ntuple_filter *)arg,
2770                         filter_op);
2771                 break;
2772         default:
2773                 PMD_DRV_LOG(ERR, "unsupported operation %u.", filter_op);
2774                 ret = -EINVAL;
2775                 break;
2776         }
2777         return ret;
2778 }
2779
2780 static int
2781 bnxt_parse_fdir_filter(struct bnxt *bp,
2782                        struct rte_eth_fdir_filter *fdir,
2783                        struct bnxt_filter_info *filter)
2784 {
2785         enum rte_fdir_mode fdir_mode =
2786                 bp->eth_dev->data->dev_conf.fdir_conf.mode;
2787         struct bnxt_vnic_info *vnic0, *vnic;
2788         struct bnxt_filter_info *filter1;
2789         uint32_t en = 0;
2790         int i;
2791
2792         if (fdir_mode == RTE_FDIR_MODE_PERFECT_TUNNEL)
2793                 return -EINVAL;
2794
2795         filter->l2_ovlan = fdir->input.flow_ext.vlan_tci;
2796         en |= EM_FLOW_ALLOC_INPUT_EN_OVLAN_VID;
2797
2798         switch (fdir->input.flow_type) {
2799         case RTE_ETH_FLOW_IPV4:
2800         case RTE_ETH_FLOW_NONFRAG_IPV4_OTHER:
2801                 /* FALLTHROUGH */
2802                 filter->src_ipaddr[0] = fdir->input.flow.ip4_flow.src_ip;
2803                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR;
2804                 filter->dst_ipaddr[0] = fdir->input.flow.ip4_flow.dst_ip;
2805                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR;
2806                 filter->ip_protocol = fdir->input.flow.ip4_flow.proto;
2807                 en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO;
2808                 filter->ip_addr_type =
2809                         NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV4;
2810                 filter->src_ipaddr_mask[0] = 0xffffffff;
2811                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK;
2812                 filter->dst_ipaddr_mask[0] = 0xffffffff;
2813                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK;
2814                 filter->ethertype = 0x800;
2815                 filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE;
2816                 break;
2817         case RTE_ETH_FLOW_NONFRAG_IPV4_TCP:
2818                 filter->src_port = fdir->input.flow.tcp4_flow.src_port;
2819                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT;
2820                 filter->dst_port = fdir->input.flow.tcp4_flow.dst_port;
2821                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT;
2822                 filter->dst_port_mask = 0xffff;
2823                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK;
2824                 filter->src_port_mask = 0xffff;
2825                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK;
2826                 filter->src_ipaddr[0] = fdir->input.flow.tcp4_flow.ip.src_ip;
2827                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR;
2828                 filter->dst_ipaddr[0] = fdir->input.flow.tcp4_flow.ip.dst_ip;
2829                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR;
2830                 filter->ip_protocol = 6;
2831                 en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO;
2832                 filter->ip_addr_type =
2833                         NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV4;
2834                 filter->src_ipaddr_mask[0] = 0xffffffff;
2835                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK;
2836                 filter->dst_ipaddr_mask[0] = 0xffffffff;
2837                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK;
2838                 filter->ethertype = 0x800;
2839                 filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE;
2840                 break;
2841         case RTE_ETH_FLOW_NONFRAG_IPV4_UDP:
2842                 filter->src_port = fdir->input.flow.udp4_flow.src_port;
2843                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT;
2844                 filter->dst_port = fdir->input.flow.udp4_flow.dst_port;
2845                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT;
2846                 filter->dst_port_mask = 0xffff;
2847                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK;
2848                 filter->src_port_mask = 0xffff;
2849                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK;
2850                 filter->src_ipaddr[0] = fdir->input.flow.udp4_flow.ip.src_ip;
2851                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR;
2852                 filter->dst_ipaddr[0] = fdir->input.flow.udp4_flow.ip.dst_ip;
2853                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR;
2854                 filter->ip_protocol = 17;
2855                 en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO;
2856                 filter->ip_addr_type =
2857                         NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV4;
2858                 filter->src_ipaddr_mask[0] = 0xffffffff;
2859                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK;
2860                 filter->dst_ipaddr_mask[0] = 0xffffffff;
2861                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK;
2862                 filter->ethertype = 0x800;
2863                 filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE;
2864                 break;
2865         case RTE_ETH_FLOW_IPV6:
2866         case RTE_ETH_FLOW_NONFRAG_IPV6_OTHER:
2867                 /* FALLTHROUGH */
2868                 filter->ip_addr_type =
2869                         NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV6;
2870                 filter->ip_protocol = fdir->input.flow.ipv6_flow.proto;
2871                 en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO;
2872                 rte_memcpy(filter->src_ipaddr,
2873                            fdir->input.flow.ipv6_flow.src_ip, 16);
2874                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR;
2875                 rte_memcpy(filter->dst_ipaddr,
2876                            fdir->input.flow.ipv6_flow.dst_ip, 16);
2877                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR;
2878                 memset(filter->dst_ipaddr_mask, 0xff, 16);
2879                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK;
2880                 memset(filter->src_ipaddr_mask, 0xff, 16);
2881                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK;
2882                 filter->ethertype = 0x86dd;
2883                 filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE;
2884                 break;
2885         case RTE_ETH_FLOW_NONFRAG_IPV6_TCP:
2886                 filter->src_port = fdir->input.flow.tcp6_flow.src_port;
2887                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT;
2888                 filter->dst_port = fdir->input.flow.tcp6_flow.dst_port;
2889                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT;
2890                 filter->dst_port_mask = 0xffff;
2891                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK;
2892                 filter->src_port_mask = 0xffff;
2893                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK;
2894                 filter->ip_addr_type =
2895                         NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV6;
2896                 filter->ip_protocol = fdir->input.flow.tcp6_flow.ip.proto;
2897                 en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO;
2898                 rte_memcpy(filter->src_ipaddr,
2899                            fdir->input.flow.tcp6_flow.ip.src_ip, 16);
2900                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR;
2901                 rte_memcpy(filter->dst_ipaddr,
2902                            fdir->input.flow.tcp6_flow.ip.dst_ip, 16);
2903                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR;
2904                 memset(filter->dst_ipaddr_mask, 0xff, 16);
2905                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK;
2906                 memset(filter->src_ipaddr_mask, 0xff, 16);
2907                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK;
2908                 filter->ethertype = 0x86dd;
2909                 filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE;
2910                 break;
2911         case RTE_ETH_FLOW_NONFRAG_IPV6_UDP:
2912                 filter->src_port = fdir->input.flow.udp6_flow.src_port;
2913                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT;
2914                 filter->dst_port = fdir->input.flow.udp6_flow.dst_port;
2915                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT;
2916                 filter->dst_port_mask = 0xffff;
2917                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_PORT_MASK;
2918                 filter->src_port_mask = 0xffff;
2919                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_PORT_MASK;
2920                 filter->ip_addr_type =
2921                         NTUPLE_FLTR_ALLOC_INPUT_IP_ADDR_TYPE_IPV6;
2922                 filter->ip_protocol = fdir->input.flow.udp6_flow.ip.proto;
2923                 en |= NTUPLE_FLTR_ALLOC_IN_EN_IP_PROTO;
2924                 rte_memcpy(filter->src_ipaddr,
2925                            fdir->input.flow.udp6_flow.ip.src_ip, 16);
2926                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR;
2927                 rte_memcpy(filter->dst_ipaddr,
2928                            fdir->input.flow.udp6_flow.ip.dst_ip, 16);
2929                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR;
2930                 memset(filter->dst_ipaddr_mask, 0xff, 16);
2931                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_IPADDR_MASK;
2932                 memset(filter->src_ipaddr_mask, 0xff, 16);
2933                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_SRC_IPADDR_MASK;
2934                 filter->ethertype = 0x86dd;
2935                 filter->enables |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE;
2936                 break;
2937         case RTE_ETH_FLOW_L2_PAYLOAD:
2938                 filter->ethertype = fdir->input.flow.l2_flow.ether_type;
2939                 en |= NTUPLE_FLTR_ALLOC_INPUT_EN_ETHERTYPE;
2940                 break;
2941         case RTE_ETH_FLOW_VXLAN:
2942                 if (fdir->action.behavior == RTE_ETH_FDIR_REJECT)
2943                         return -EINVAL;
2944                 filter->vni = fdir->input.flow.tunnel_flow.tunnel_id;
2945                 filter->tunnel_type =
2946                         CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_VXLAN;
2947                 en |= HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_TUNNEL_TYPE;
2948                 break;
2949         case RTE_ETH_FLOW_NVGRE:
2950                 if (fdir->action.behavior == RTE_ETH_FDIR_REJECT)
2951                         return -EINVAL;
2952                 filter->vni = fdir->input.flow.tunnel_flow.tunnel_id;
2953                 filter->tunnel_type =
2954                         CFA_NTUPLE_FILTER_ALLOC_REQ_TUNNEL_TYPE_NVGRE;
2955                 en |= HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_TUNNEL_TYPE;
2956                 break;
2957         case RTE_ETH_FLOW_UNKNOWN:
2958         case RTE_ETH_FLOW_RAW:
2959         case RTE_ETH_FLOW_FRAG_IPV4:
2960         case RTE_ETH_FLOW_NONFRAG_IPV4_SCTP:
2961         case RTE_ETH_FLOW_FRAG_IPV6:
2962         case RTE_ETH_FLOW_NONFRAG_IPV6_SCTP:
2963         case RTE_ETH_FLOW_IPV6_EX:
2964         case RTE_ETH_FLOW_IPV6_TCP_EX:
2965         case RTE_ETH_FLOW_IPV6_UDP_EX:
2966         case RTE_ETH_FLOW_GENEVE:
2967                 /* FALLTHROUGH */
2968         default:
2969                 return -EINVAL;
2970         }
2971
2972         vnic0 = &bp->vnic_info[0];
2973         vnic = &bp->vnic_info[fdir->action.rx_queue];
2974         if (vnic == NULL) {
2975                 PMD_DRV_LOG(ERR, "Invalid queue %d\n", fdir->action.rx_queue);
2976                 return -EINVAL;
2977         }
2978
2979         if (fdir_mode == RTE_FDIR_MODE_PERFECT_MAC_VLAN) {
2980                 rte_memcpy(filter->dst_macaddr,
2981                         fdir->input.flow.mac_vlan_flow.mac_addr.addr_bytes, 6);
2982                         en |= NTUPLE_FLTR_ALLOC_INPUT_EN_DST_MACADDR;
2983         }
2984
2985         if (fdir->action.behavior == RTE_ETH_FDIR_REJECT) {
2986                 filter->flags = HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_FLAGS_DROP;
2987                 filter1 = STAILQ_FIRST(&vnic0->filter);
2988                 //filter1 = bnxt_get_l2_filter(bp, filter, vnic0);
2989         } else {
2990                 filter->dst_id = vnic->fw_vnic_id;
2991                 for (i = 0; i < RTE_ETHER_ADDR_LEN; i++)
2992                         if (filter->dst_macaddr[i] == 0x00)
2993                                 filter1 = STAILQ_FIRST(&vnic0->filter);
2994                         else
2995                                 filter1 = bnxt_get_l2_filter(bp, filter, vnic);
2996         }
2997
2998         if (filter1 == NULL)
2999                 return -EINVAL;
3000
3001         en |= HWRM_CFA_NTUPLE_FILTER_ALLOC_INPUT_ENABLES_L2_FILTER_ID;
3002         filter->fw_l2_filter_id = filter1->fw_l2_filter_id;
3003
3004         filter->enables = en;
3005
3006         return 0;
3007 }
3008
3009 static struct bnxt_filter_info *
3010 bnxt_match_fdir(struct bnxt *bp, struct bnxt_filter_info *nf,
3011                 struct bnxt_vnic_info **mvnic)
3012 {
3013         struct bnxt_filter_info *mf = NULL;
3014         int i;
3015
3016         for (i = bp->nr_vnics - 1; i >= 0; i--) {
3017                 struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3018
3019                 STAILQ_FOREACH(mf, &vnic->filter, next) {
3020                         if (mf->filter_type == nf->filter_type &&
3021                             mf->flags == nf->flags &&
3022                             mf->src_port == nf->src_port &&
3023                             mf->src_port_mask == nf->src_port_mask &&
3024                             mf->dst_port == nf->dst_port &&
3025                             mf->dst_port_mask == nf->dst_port_mask &&
3026                             mf->ip_protocol == nf->ip_protocol &&
3027                             mf->ip_addr_type == nf->ip_addr_type &&
3028                             mf->ethertype == nf->ethertype &&
3029                             mf->vni == nf->vni &&
3030                             mf->tunnel_type == nf->tunnel_type &&
3031                             mf->l2_ovlan == nf->l2_ovlan &&
3032                             mf->l2_ovlan_mask == nf->l2_ovlan_mask &&
3033                             mf->l2_ivlan == nf->l2_ivlan &&
3034                             mf->l2_ivlan_mask == nf->l2_ivlan_mask &&
3035                             !memcmp(mf->l2_addr, nf->l2_addr,
3036                                     RTE_ETHER_ADDR_LEN) &&
3037                             !memcmp(mf->l2_addr_mask, nf->l2_addr_mask,
3038                                     RTE_ETHER_ADDR_LEN) &&
3039                             !memcmp(mf->src_macaddr, nf->src_macaddr,
3040                                     RTE_ETHER_ADDR_LEN) &&
3041                             !memcmp(mf->dst_macaddr, nf->dst_macaddr,
3042                                     RTE_ETHER_ADDR_LEN) &&
3043                             !memcmp(mf->src_ipaddr, nf->src_ipaddr,
3044                                     sizeof(nf->src_ipaddr)) &&
3045                             !memcmp(mf->src_ipaddr_mask, nf->src_ipaddr_mask,
3046                                     sizeof(nf->src_ipaddr_mask)) &&
3047                             !memcmp(mf->dst_ipaddr, nf->dst_ipaddr,
3048                                     sizeof(nf->dst_ipaddr)) &&
3049                             !memcmp(mf->dst_ipaddr_mask, nf->dst_ipaddr_mask,
3050                                     sizeof(nf->dst_ipaddr_mask))) {
3051                                 if (mvnic)
3052                                         *mvnic = vnic;
3053                                 return mf;
3054                         }
3055                 }
3056         }
3057         return NULL;
3058 }
3059
3060 static int
3061 bnxt_fdir_filter(struct rte_eth_dev *dev,
3062                  enum rte_filter_op filter_op,
3063                  void *arg)
3064 {
3065         struct bnxt *bp = dev->data->dev_private;
3066         struct rte_eth_fdir_filter *fdir  = (struct rte_eth_fdir_filter *)arg;
3067         struct bnxt_filter_info *filter, *match;
3068         struct bnxt_vnic_info *vnic, *mvnic;
3069         int ret = 0, i;
3070
3071         if (filter_op == RTE_ETH_FILTER_NOP)
3072                 return 0;
3073
3074         if (arg == NULL && filter_op != RTE_ETH_FILTER_FLUSH)
3075                 return -EINVAL;
3076
3077         switch (filter_op) {
3078         case RTE_ETH_FILTER_ADD:
3079         case RTE_ETH_FILTER_DELETE:
3080                 /* FALLTHROUGH */
3081                 filter = bnxt_get_unused_filter(bp);
3082                 if (filter == NULL) {
3083                         PMD_DRV_LOG(ERR,
3084                                 "Not enough resources for a new flow.\n");
3085                         return -ENOMEM;
3086                 }
3087
3088                 ret = bnxt_parse_fdir_filter(bp, fdir, filter);
3089                 if (ret != 0)
3090                         goto free_filter;
3091                 filter->filter_type = HWRM_CFA_NTUPLE_FILTER;
3092
3093                 if (fdir->action.behavior == RTE_ETH_FDIR_REJECT)
3094                         vnic = &bp->vnic_info[0];
3095                 else
3096                         vnic = &bp->vnic_info[fdir->action.rx_queue];
3097
3098                 match = bnxt_match_fdir(bp, filter, &mvnic);
3099                 if (match != NULL && filter_op == RTE_ETH_FILTER_ADD) {
3100                         if (match->dst_id == vnic->fw_vnic_id) {
3101                                 PMD_DRV_LOG(ERR, "Flow already exists.\n");
3102                                 ret = -EEXIST;
3103                                 goto free_filter;
3104                         } else {
3105                                 match->dst_id = vnic->fw_vnic_id;
3106                                 ret = bnxt_hwrm_set_ntuple_filter(bp,
3107                                                                   match->dst_id,
3108                                                                   match);
3109                                 STAILQ_REMOVE(&mvnic->filter, match,
3110                                               bnxt_filter_info, next);
3111                                 STAILQ_INSERT_TAIL(&vnic->filter, match, next);
3112                                 PMD_DRV_LOG(ERR,
3113                                         "Filter with matching pattern exist\n");
3114                                 PMD_DRV_LOG(ERR,
3115                                         "Updated it to new destination q\n");
3116                                 goto free_filter;
3117                         }
3118                 }
3119                 if (match == NULL && filter_op == RTE_ETH_FILTER_DELETE) {
3120                         PMD_DRV_LOG(ERR, "Flow does not exist.\n");
3121                         ret = -ENOENT;
3122                         goto free_filter;
3123                 }
3124
3125                 if (filter_op == RTE_ETH_FILTER_ADD) {
3126                         ret = bnxt_hwrm_set_ntuple_filter(bp,
3127                                                           filter->dst_id,
3128                                                           filter);
3129                         if (ret)
3130                                 goto free_filter;
3131                         STAILQ_INSERT_TAIL(&vnic->filter, filter, next);
3132                 } else {
3133                         ret = bnxt_hwrm_clear_ntuple_filter(bp, match);
3134                         STAILQ_REMOVE(&vnic->filter, match,
3135                                       bnxt_filter_info, next);
3136                         bnxt_free_filter(bp, match);
3137                         filter->fw_l2_filter_id = -1;
3138                         bnxt_free_filter(bp, filter);
3139                 }
3140                 break;
3141         case RTE_ETH_FILTER_FLUSH:
3142                 for (i = bp->nr_vnics - 1; i >= 0; i--) {
3143                         struct bnxt_vnic_info *vnic = &bp->vnic_info[i];
3144
3145                         STAILQ_FOREACH(filter, &vnic->filter, next) {
3146                                 if (filter->filter_type ==
3147                                     HWRM_CFA_NTUPLE_FILTER) {
3148                                         ret =
3149                                         bnxt_hwrm_clear_ntuple_filter(bp,
3150                                                                       filter);
3151                                         STAILQ_REMOVE(&vnic->filter, filter,
3152                                                       bnxt_filter_info, next);
3153                                 }
3154                         }
3155                 }
3156                 return ret;
3157         case RTE_ETH_FILTER_UPDATE:
3158         case RTE_ETH_FILTER_STATS:
3159         case RTE_ETH_FILTER_INFO:
3160                 PMD_DRV_LOG(ERR, "operation %u not implemented", filter_op);
3161                 break;
3162         default:
3163                 PMD_DRV_LOG(ERR, "unknown operation %u", filter_op);
3164                 ret = -EINVAL;
3165                 break;
3166         }
3167         return ret;
3168
3169 free_filter:
3170         filter->fw_l2_filter_id = -1;
3171         bnxt_free_filter(bp, filter);
3172         return ret;
3173 }
3174
3175 static int
3176 bnxt_filter_ctrl_op(struct rte_eth_dev *dev __rte_unused,
3177                     enum rte_filter_type filter_type,
3178                     enum rte_filter_op filter_op, void *arg)
3179 {
3180         int ret = 0;
3181
3182         ret = is_bnxt_in_error(dev->data->dev_private);
3183         if (ret)
3184                 return ret;
3185
3186         switch (filter_type) {
3187         case RTE_ETH_FILTER_TUNNEL:
3188                 PMD_DRV_LOG(ERR,
3189                         "filter type: %d: To be implemented\n", filter_type);
3190                 break;
3191         case RTE_ETH_FILTER_FDIR:
3192                 ret = bnxt_fdir_filter(dev, filter_op, arg);
3193                 break;
3194         case RTE_ETH_FILTER_NTUPLE:
3195                 ret = bnxt_ntuple_filter(dev, filter_op, arg);
3196                 break;
3197         case RTE_ETH_FILTER_ETHERTYPE:
3198                 ret = bnxt_ethertype_filter(dev, filter_op, arg);
3199                 break;
3200         case RTE_ETH_FILTER_GENERIC:
3201                 if (filter_op != RTE_ETH_FILTER_GET)
3202                         return -EINVAL;
3203                 *(const void **)arg = &bnxt_flow_ops;
3204                 break;
3205         default:
3206                 PMD_DRV_LOG(ERR,
3207                         "Filter type (%d) not supported", filter_type);
3208                 ret = -EINVAL;
3209                 break;
3210         }
3211         return ret;
3212 }
3213
3214 static const uint32_t *
3215 bnxt_dev_supported_ptypes_get_op(struct rte_eth_dev *dev)
3216 {
3217         static const uint32_t ptypes[] = {
3218                 RTE_PTYPE_L2_ETHER_VLAN,
3219                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
3220                 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
3221                 RTE_PTYPE_L4_ICMP,
3222                 RTE_PTYPE_L4_TCP,
3223                 RTE_PTYPE_L4_UDP,
3224                 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
3225                 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
3226                 RTE_PTYPE_INNER_L4_ICMP,
3227                 RTE_PTYPE_INNER_L4_TCP,
3228                 RTE_PTYPE_INNER_L4_UDP,
3229                 RTE_PTYPE_UNKNOWN
3230         };
3231
3232         if (!dev->rx_pkt_burst)
3233                 return NULL;
3234
3235         return ptypes;
3236 }
3237
3238 static int bnxt_map_regs(struct bnxt *bp, uint32_t *reg_arr, int count,
3239                          int reg_win)
3240 {
3241         uint32_t reg_base = *reg_arr & 0xfffff000;
3242         uint32_t win_off;
3243         int i;
3244
3245         for (i = 0; i < count; i++) {
3246                 if ((reg_arr[i] & 0xfffff000) != reg_base)
3247                         return -ERANGE;
3248         }
3249         win_off = BNXT_GRCPF_REG_WINDOW_BASE_OUT + (reg_win - 1) * 4;
3250         rte_write32(reg_base, (uint8_t *)bp->bar0 + win_off);
3251         return 0;
3252 }
3253
3254 static int bnxt_map_ptp_regs(struct bnxt *bp)
3255 {
3256         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3257         uint32_t *reg_arr;
3258         int rc, i;
3259
3260         reg_arr = ptp->rx_regs;
3261         rc = bnxt_map_regs(bp, reg_arr, BNXT_PTP_RX_REGS, 5);
3262         if (rc)
3263                 return rc;
3264
3265         reg_arr = ptp->tx_regs;
3266         rc = bnxt_map_regs(bp, reg_arr, BNXT_PTP_TX_REGS, 6);
3267         if (rc)
3268                 return rc;
3269
3270         for (i = 0; i < BNXT_PTP_RX_REGS; i++)
3271                 ptp->rx_mapped_regs[i] = 0x5000 + (ptp->rx_regs[i] & 0xfff);
3272
3273         for (i = 0; i < BNXT_PTP_TX_REGS; i++)
3274                 ptp->tx_mapped_regs[i] = 0x6000 + (ptp->tx_regs[i] & 0xfff);
3275
3276         return 0;
3277 }
3278
3279 static void bnxt_unmap_ptp_regs(struct bnxt *bp)
3280 {
3281         rte_write32(0, (uint8_t *)bp->bar0 +
3282                          BNXT_GRCPF_REG_WINDOW_BASE_OUT + 16);
3283         rte_write32(0, (uint8_t *)bp->bar0 +
3284                          BNXT_GRCPF_REG_WINDOW_BASE_OUT + 20);
3285 }
3286
3287 static uint64_t bnxt_cc_read(struct bnxt *bp)
3288 {
3289         uint64_t ns;
3290
3291         ns = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3292                               BNXT_GRCPF_REG_SYNC_TIME));
3293         ns |= (uint64_t)(rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3294                                           BNXT_GRCPF_REG_SYNC_TIME + 4))) << 32;
3295         return ns;
3296 }
3297
3298 static int bnxt_get_tx_ts(struct bnxt *bp, uint64_t *ts)
3299 {
3300         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3301         uint32_t fifo;
3302
3303         fifo = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3304                                 ptp->tx_mapped_regs[BNXT_PTP_TX_FIFO]));
3305         if (fifo & BNXT_PTP_TX_FIFO_EMPTY)
3306                 return -EAGAIN;
3307
3308         fifo = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3309                                 ptp->tx_mapped_regs[BNXT_PTP_TX_FIFO]));
3310         *ts = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3311                                 ptp->tx_mapped_regs[BNXT_PTP_TX_TS_L]));
3312         *ts |= (uint64_t)rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3313                                 ptp->tx_mapped_regs[BNXT_PTP_TX_TS_H])) << 32;
3314
3315         return 0;
3316 }
3317
3318 static int bnxt_get_rx_ts(struct bnxt *bp, uint64_t *ts)
3319 {
3320         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3321         struct bnxt_pf_info *pf = &bp->pf;
3322         uint16_t port_id;
3323         uint32_t fifo;
3324
3325         if (!ptp)
3326                 return -ENODEV;
3327
3328         fifo = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3329                                 ptp->rx_mapped_regs[BNXT_PTP_RX_FIFO]));
3330         if (!(fifo & BNXT_PTP_RX_FIFO_PENDING))
3331                 return -EAGAIN;
3332
3333         port_id = pf->port_id;
3334         rte_write32(1 << port_id, (uint8_t *)bp->bar0 +
3335                ptp->rx_mapped_regs[BNXT_PTP_RX_FIFO_ADV]);
3336
3337         fifo = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3338                                    ptp->rx_mapped_regs[BNXT_PTP_RX_FIFO]));
3339         if (fifo & BNXT_PTP_RX_FIFO_PENDING) {
3340 /*              bnxt_clr_rx_ts(bp);       TBD  */
3341                 return -EBUSY;
3342         }
3343
3344         *ts = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3345                                 ptp->rx_mapped_regs[BNXT_PTP_RX_TS_L]));
3346         *ts |= (uint64_t)rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3347                                 ptp->rx_mapped_regs[BNXT_PTP_RX_TS_H])) << 32;
3348
3349         return 0;
3350 }
3351
3352 static int
3353 bnxt_timesync_write_time(struct rte_eth_dev *dev, const struct timespec *ts)
3354 {
3355         uint64_t ns;
3356         struct bnxt *bp = dev->data->dev_private;
3357         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3358
3359         if (!ptp)
3360                 return 0;
3361
3362         ns = rte_timespec_to_ns(ts);
3363         /* Set the timecounters to a new value. */
3364         ptp->tc.nsec = ns;
3365
3366         return 0;
3367 }
3368
3369 static int
3370 bnxt_timesync_read_time(struct rte_eth_dev *dev, struct timespec *ts)
3371 {
3372         struct bnxt *bp = dev->data->dev_private;
3373         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3374         uint64_t ns, systime_cycles = 0;
3375         int rc = 0;
3376
3377         if (!ptp)
3378                 return 0;
3379
3380         if (BNXT_CHIP_THOR(bp))
3381                 rc = bnxt_hwrm_port_ts_query(bp, BNXT_PTP_FLAGS_CURRENT_TIME,
3382                                              &systime_cycles);
3383         else
3384                 systime_cycles = bnxt_cc_read(bp);
3385
3386         ns = rte_timecounter_update(&ptp->tc, systime_cycles);
3387         *ts = rte_ns_to_timespec(ns);
3388
3389         return rc;
3390 }
3391 static int
3392 bnxt_timesync_enable(struct rte_eth_dev *dev)
3393 {
3394         struct bnxt *bp = dev->data->dev_private;
3395         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3396         uint32_t shift = 0;
3397         int rc;
3398
3399         if (!ptp)
3400                 return 0;
3401
3402         ptp->rx_filter = 1;
3403         ptp->tx_tstamp_en = 1;
3404         ptp->rxctl = BNXT_PTP_MSG_EVENTS;
3405
3406         rc = bnxt_hwrm_ptp_cfg(bp);
3407         if (rc)
3408                 return rc;
3409
3410         memset(&ptp->tc, 0, sizeof(struct rte_timecounter));
3411         memset(&ptp->rx_tstamp_tc, 0, sizeof(struct rte_timecounter));
3412         memset(&ptp->tx_tstamp_tc, 0, sizeof(struct rte_timecounter));
3413
3414         ptp->tc.cc_mask = BNXT_CYCLECOUNTER_MASK;
3415         ptp->tc.cc_shift = shift;
3416         ptp->tc.nsec_mask = (1ULL << shift) - 1;
3417
3418         ptp->rx_tstamp_tc.cc_mask = BNXT_CYCLECOUNTER_MASK;
3419         ptp->rx_tstamp_tc.cc_shift = shift;
3420         ptp->rx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
3421
3422         ptp->tx_tstamp_tc.cc_mask = BNXT_CYCLECOUNTER_MASK;
3423         ptp->tx_tstamp_tc.cc_shift = shift;
3424         ptp->tx_tstamp_tc.nsec_mask = (1ULL << shift) - 1;
3425
3426         if (!BNXT_CHIP_THOR(bp))
3427                 bnxt_map_ptp_regs(bp);
3428
3429         return 0;
3430 }
3431
3432 static int
3433 bnxt_timesync_disable(struct rte_eth_dev *dev)
3434 {
3435         struct bnxt *bp = dev->data->dev_private;
3436         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3437
3438         if (!ptp)
3439                 return 0;
3440
3441         ptp->rx_filter = 0;
3442         ptp->tx_tstamp_en = 0;
3443         ptp->rxctl = 0;
3444
3445         bnxt_hwrm_ptp_cfg(bp);
3446
3447         if (!BNXT_CHIP_THOR(bp))
3448                 bnxt_unmap_ptp_regs(bp);
3449
3450         return 0;
3451 }
3452
3453 static int
3454 bnxt_timesync_read_rx_timestamp(struct rte_eth_dev *dev,
3455                                  struct timespec *timestamp,
3456                                  uint32_t flags __rte_unused)
3457 {
3458         struct bnxt *bp = dev->data->dev_private;
3459         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3460         uint64_t rx_tstamp_cycles = 0;
3461         uint64_t ns;
3462
3463         if (!ptp)
3464                 return 0;
3465
3466         if (BNXT_CHIP_THOR(bp))
3467                 rx_tstamp_cycles = ptp->rx_timestamp;
3468         else
3469                 bnxt_get_rx_ts(bp, &rx_tstamp_cycles);
3470
3471         ns = rte_timecounter_update(&ptp->rx_tstamp_tc, rx_tstamp_cycles);
3472         *timestamp = rte_ns_to_timespec(ns);
3473         return  0;
3474 }
3475
3476 static int
3477 bnxt_timesync_read_tx_timestamp(struct rte_eth_dev *dev,
3478                                  struct timespec *timestamp)
3479 {
3480         struct bnxt *bp = dev->data->dev_private;
3481         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3482         uint64_t tx_tstamp_cycles = 0;
3483         uint64_t ns;
3484         int rc = 0;
3485
3486         if (!ptp)
3487                 return 0;
3488
3489         if (BNXT_CHIP_THOR(bp))
3490                 rc = bnxt_hwrm_port_ts_query(bp, BNXT_PTP_FLAGS_PATH_TX,
3491                                              &tx_tstamp_cycles);
3492         else
3493                 rc = bnxt_get_tx_ts(bp, &tx_tstamp_cycles);
3494
3495         ns = rte_timecounter_update(&ptp->tx_tstamp_tc, tx_tstamp_cycles);
3496         *timestamp = rte_ns_to_timespec(ns);
3497
3498         return rc;
3499 }
3500
3501 static int
3502 bnxt_timesync_adjust_time(struct rte_eth_dev *dev, int64_t delta)
3503 {
3504         struct bnxt *bp = dev->data->dev_private;
3505         struct bnxt_ptp_cfg *ptp = bp->ptp_cfg;
3506
3507         if (!ptp)
3508                 return 0;
3509
3510         ptp->tc.nsec += delta;
3511
3512         return 0;
3513 }
3514
3515 static int
3516 bnxt_get_eeprom_length_op(struct rte_eth_dev *dev)
3517 {
3518         struct bnxt *bp = dev->data->dev_private;
3519         int rc;
3520         uint32_t dir_entries;
3521         uint32_t entry_length;
3522
3523         rc = is_bnxt_in_error(bp);
3524         if (rc)
3525                 return rc;
3526
3527         PMD_DRV_LOG(INFO, "%04x:%02x:%02x:%02x\n",
3528                 bp->pdev->addr.domain, bp->pdev->addr.bus,
3529                 bp->pdev->addr.devid, bp->pdev->addr.function);
3530
3531         rc = bnxt_hwrm_nvm_get_dir_info(bp, &dir_entries, &entry_length);
3532         if (rc != 0)
3533                 return rc;
3534
3535         return dir_entries * entry_length;
3536 }
3537
3538 static int
3539 bnxt_get_eeprom_op(struct rte_eth_dev *dev,
3540                 struct rte_dev_eeprom_info *in_eeprom)
3541 {
3542         struct bnxt *bp = dev->data->dev_private;
3543         uint32_t index;
3544         uint32_t offset;
3545         int rc;
3546
3547         rc = is_bnxt_in_error(bp);
3548         if (rc)
3549                 return rc;
3550
3551         PMD_DRV_LOG(INFO, "%04x:%02x:%02x:%02x in_eeprom->offset = %d "
3552                 "len = %d\n", bp->pdev->addr.domain,
3553                 bp->pdev->addr.bus, bp->pdev->addr.devid,
3554                 bp->pdev->addr.function, in_eeprom->offset, in_eeprom->length);
3555
3556         if (in_eeprom->offset == 0) /* special offset value to get directory */
3557                 return bnxt_get_nvram_directory(bp, in_eeprom->length,
3558                                                 in_eeprom->data);
3559
3560         index = in_eeprom->offset >> 24;
3561         offset = in_eeprom->offset & 0xffffff;
3562
3563         if (index != 0)
3564                 return bnxt_hwrm_get_nvram_item(bp, index - 1, offset,
3565                                            in_eeprom->length, in_eeprom->data);
3566
3567         return 0;
3568 }
3569
3570 static bool bnxt_dir_type_is_ape_bin_format(uint16_t dir_type)
3571 {
3572         switch (dir_type) {
3573         case BNX_DIR_TYPE_CHIMP_PATCH:
3574         case BNX_DIR_TYPE_BOOTCODE:
3575         case BNX_DIR_TYPE_BOOTCODE_2:
3576         case BNX_DIR_TYPE_APE_FW:
3577         case BNX_DIR_TYPE_APE_PATCH:
3578         case BNX_DIR_TYPE_KONG_FW:
3579         case BNX_DIR_TYPE_KONG_PATCH:
3580         case BNX_DIR_TYPE_BONO_FW:
3581         case BNX_DIR_TYPE_BONO_PATCH:
3582                 /* FALLTHROUGH */
3583                 return true;
3584         }
3585
3586         return false;
3587 }
3588
3589 static bool bnxt_dir_type_is_other_exec_format(uint16_t dir_type)
3590 {
3591         switch (dir_type) {
3592         case BNX_DIR_TYPE_AVS:
3593         case BNX_DIR_TYPE_EXP_ROM_MBA:
3594         case BNX_DIR_TYPE_PCIE:
3595         case BNX_DIR_TYPE_TSCF_UCODE:
3596         case BNX_DIR_TYPE_EXT_PHY:
3597         case BNX_DIR_TYPE_CCM:
3598         case BNX_DIR_TYPE_ISCSI_BOOT:
3599         case BNX_DIR_TYPE_ISCSI_BOOT_IPV6:
3600         case BNX_DIR_TYPE_ISCSI_BOOT_IPV4N6:
3601                 /* FALLTHROUGH */
3602                 return true;
3603         }
3604
3605         return false;
3606 }
3607
3608 static bool bnxt_dir_type_is_executable(uint16_t dir_type)
3609 {
3610         return bnxt_dir_type_is_ape_bin_format(dir_type) ||
3611                 bnxt_dir_type_is_other_exec_format(dir_type);
3612 }
3613
3614 static int
3615 bnxt_set_eeprom_op(struct rte_eth_dev *dev,
3616                 struct rte_dev_eeprom_info *in_eeprom)
3617 {
3618         struct bnxt *bp = dev->data->dev_private;
3619         uint8_t index, dir_op;
3620         uint16_t type, ext, ordinal, attr;
3621         int rc;
3622
3623         rc = is_bnxt_in_error(bp);
3624         if (rc)
3625                 return rc;
3626
3627         PMD_DRV_LOG(INFO, "%04x:%02x:%02x:%02x in_eeprom->offset = %d "
3628                 "len = %d\n", bp->pdev->addr.domain,
3629                 bp->pdev->addr.bus, bp->pdev->addr.devid,
3630                 bp->pdev->addr.function, in_eeprom->offset, in_eeprom->length);
3631
3632         if (!BNXT_PF(bp)) {
3633                 PMD_DRV_LOG(ERR, "NVM write not supported from a VF\n");
3634                 return -EINVAL;
3635         }
3636
3637         type = in_eeprom->magic >> 16;
3638
3639         if (type == 0xffff) { /* special value for directory operations */
3640                 index = in_eeprom->magic & 0xff;
3641                 dir_op = in_eeprom->magic >> 8;
3642                 if (index == 0)
3643                         return -EINVAL;
3644                 switch (dir_op) {
3645                 case 0x0e: /* erase */
3646                         if (in_eeprom->offset != ~in_eeprom->magic)
3647                                 return -EINVAL;
3648                         return bnxt_hwrm_erase_nvram_directory(bp, index - 1);
3649                 default:
3650                         return -EINVAL;
3651                 }
3652         }
3653
3654         /* Create or re-write an NVM item: */
3655         if (bnxt_dir_type_is_executable(type) == true)
3656                 return -EOPNOTSUPP;
3657         ext = in_eeprom->magic & 0xffff;
3658         ordinal = in_eeprom->offset >> 16;
3659         attr = in_eeprom->offset & 0xffff;
3660
3661         return bnxt_hwrm_flash_nvram(bp, type, ordinal, ext, attr,
3662                                      in_eeprom->data, in_eeprom->length);
3663 }
3664
3665 /*
3666  * Initialization
3667  */
3668
3669 static const struct eth_dev_ops bnxt_dev_ops = {
3670         .dev_infos_get = bnxt_dev_info_get_op,
3671         .dev_close = bnxt_dev_close_op,
3672         .dev_configure = bnxt_dev_configure_op,
3673         .dev_start = bnxt_dev_start_op,
3674         .dev_stop = bnxt_dev_stop_op,
3675         .dev_set_link_up = bnxt_dev_set_link_up_op,
3676         .dev_set_link_down = bnxt_dev_set_link_down_op,
3677         .stats_get = bnxt_stats_get_op,
3678         .stats_reset = bnxt_stats_reset_op,
3679         .rx_queue_setup = bnxt_rx_queue_setup_op,
3680         .rx_queue_release = bnxt_rx_queue_release_op,
3681         .tx_queue_setup = bnxt_tx_queue_setup_op,
3682         .tx_queue_release = bnxt_tx_queue_release_op,
3683         .rx_queue_intr_enable = bnxt_rx_queue_intr_enable_op,
3684         .rx_queue_intr_disable = bnxt_rx_queue_intr_disable_op,
3685         .reta_update = bnxt_reta_update_op,
3686         .reta_query = bnxt_reta_query_op,
3687         .rss_hash_update = bnxt_rss_hash_update_op,
3688         .rss_hash_conf_get = bnxt_rss_hash_conf_get_op,
3689         .link_update = bnxt_link_update_op,
3690         .promiscuous_enable = bnxt_promiscuous_enable_op,
3691         .promiscuous_disable = bnxt_promiscuous_disable_op,
3692         .allmulticast_enable = bnxt_allmulticast_enable_op,
3693         .allmulticast_disable = bnxt_allmulticast_disable_op,
3694         .mac_addr_add = bnxt_mac_addr_add_op,
3695         .mac_addr_remove = bnxt_mac_addr_remove_op,
3696         .flow_ctrl_get = bnxt_flow_ctrl_get_op,
3697         .flow_ctrl_set = bnxt_flow_ctrl_set_op,
3698         .udp_tunnel_port_add  = bnxt_udp_tunnel_port_add_op,
3699         .udp_tunnel_port_del  = bnxt_udp_tunnel_port_del_op,
3700         .vlan_filter_set = bnxt_vlan_filter_set_op,
3701         .vlan_offload_set = bnxt_vlan_offload_set_op,
3702         .vlan_tpid_set = bnxt_vlan_tpid_set_op,
3703         .vlan_pvid_set = bnxt_vlan_pvid_set_op,
3704         .mtu_set = bnxt_mtu_set_op,
3705         .mac_addr_set = bnxt_set_default_mac_addr_op,
3706         .xstats_get = bnxt_dev_xstats_get_op,
3707         .xstats_get_names = bnxt_dev_xstats_get_names_op,
3708         .xstats_reset = bnxt_dev_xstats_reset_op,
3709         .fw_version_get = bnxt_fw_version_get,
3710         .set_mc_addr_list = bnxt_dev_set_mc_addr_list_op,
3711         .rxq_info_get = bnxt_rxq_info_get_op,
3712         .txq_info_get = bnxt_txq_info_get_op,
3713         .dev_led_on = bnxt_dev_led_on_op,
3714         .dev_led_off = bnxt_dev_led_off_op,
3715         .xstats_get_by_id = bnxt_dev_xstats_get_by_id_op,
3716         .xstats_get_names_by_id = bnxt_dev_xstats_get_names_by_id_op,
3717         .rx_queue_count = bnxt_rx_queue_count_op,
3718         .rx_descriptor_status = bnxt_rx_descriptor_status_op,
3719         .tx_descriptor_status = bnxt_tx_descriptor_status_op,
3720         .rx_queue_start = bnxt_rx_queue_start,
3721         .rx_queue_stop = bnxt_rx_queue_stop,
3722         .tx_queue_start = bnxt_tx_queue_start,
3723         .tx_queue_stop = bnxt_tx_queue_stop,
3724         .filter_ctrl = bnxt_filter_ctrl_op,
3725         .dev_supported_ptypes_get = bnxt_dev_supported_ptypes_get_op,
3726         .get_eeprom_length    = bnxt_get_eeprom_length_op,
3727         .get_eeprom           = bnxt_get_eeprom_op,
3728         .set_eeprom           = bnxt_set_eeprom_op,
3729         .timesync_enable      = bnxt_timesync_enable,
3730         .timesync_disable     = bnxt_timesync_disable,
3731         .timesync_read_time   = bnxt_timesync_read_time,
3732         .timesync_write_time   = bnxt_timesync_write_time,
3733         .timesync_adjust_time = bnxt_timesync_adjust_time,
3734         .timesync_read_rx_timestamp = bnxt_timesync_read_rx_timestamp,
3735         .timesync_read_tx_timestamp = bnxt_timesync_read_tx_timestamp,
3736 };
3737
3738 static uint32_t bnxt_map_reset_regs(struct bnxt *bp, uint32_t reg)
3739 {
3740         uint32_t offset;
3741
3742         /* Only pre-map the reset GRC registers using window 3 */
3743         rte_write32(reg & 0xfffff000, (uint8_t *)bp->bar0 +
3744                     BNXT_GRCPF_REG_WINDOW_BASE_OUT + 8);
3745
3746         offset = BNXT_GRCP_WINDOW_3_BASE + (reg & 0xffc);
3747
3748         return offset;
3749 }
3750
3751 int bnxt_map_fw_health_status_regs(struct bnxt *bp)
3752 {
3753         struct bnxt_error_recovery_info *info = bp->recovery_info;
3754         uint32_t reg_base = 0xffffffff;
3755         int i;
3756
3757         /* Only pre-map the monitoring GRC registers using window 2 */
3758         for (i = 0; i < BNXT_FW_STATUS_REG_CNT; i++) {
3759                 uint32_t reg = info->status_regs[i];
3760
3761                 if (BNXT_FW_STATUS_REG_TYPE(reg) != BNXT_FW_STATUS_REG_TYPE_GRC)
3762                         continue;
3763
3764                 if (reg_base == 0xffffffff)
3765                         reg_base = reg & 0xfffff000;
3766                 if ((reg & 0xfffff000) != reg_base)
3767                         return -ERANGE;
3768
3769                 /* Use mask 0xffc as the Lower 2 bits indicates
3770                  * address space location
3771                  */
3772                 info->mapped_status_regs[i] = BNXT_GRCP_WINDOW_2_BASE +
3773                                                 (reg & 0xffc);
3774         }
3775
3776         if (reg_base == 0xffffffff)
3777                 return 0;
3778
3779         rte_write32(reg_base, (uint8_t *)bp->bar0 +
3780                     BNXT_GRCPF_REG_WINDOW_BASE_OUT + 4);
3781
3782         return 0;
3783 }
3784
3785 static void bnxt_write_fw_reset_reg(struct bnxt *bp, uint32_t index)
3786 {
3787         struct bnxt_error_recovery_info *info = bp->recovery_info;
3788         uint32_t delay = info->delay_after_reset[index];
3789         uint32_t val = info->reset_reg_val[index];
3790         uint32_t reg = info->reset_reg[index];
3791         uint32_t type, offset;
3792
3793         type = BNXT_FW_STATUS_REG_TYPE(reg);
3794         offset = BNXT_FW_STATUS_REG_OFF(reg);
3795
3796         switch (type) {
3797         case BNXT_FW_STATUS_REG_TYPE_CFG:
3798                 rte_pci_write_config(bp->pdev, &val, sizeof(val), offset);
3799                 break;
3800         case BNXT_FW_STATUS_REG_TYPE_GRC:
3801                 offset = bnxt_map_reset_regs(bp, offset);
3802                 rte_write32(val, (uint8_t *)bp->bar0 + offset);
3803                 break;
3804         case BNXT_FW_STATUS_REG_TYPE_BAR0:
3805                 rte_write32(val, (uint8_t *)bp->bar0 + offset);
3806                 break;
3807         }
3808         /* wait on a specific interval of time until core reset is complete */
3809         if (delay)
3810                 rte_delay_ms(delay);
3811 }
3812
3813 static void bnxt_dev_cleanup(struct bnxt *bp)
3814 {
3815         bnxt_set_hwrm_link_config(bp, false);
3816         bp->link_info.link_up = 0;
3817         if (bp->dev_stopped == 0)
3818                 bnxt_dev_stop_op(bp->eth_dev);
3819
3820         bnxt_uninit_resources(bp, true);
3821 }
3822
3823 static int bnxt_restore_filters(struct bnxt *bp)
3824 {
3825         struct rte_eth_dev *dev = bp->eth_dev;
3826         int ret = 0;
3827
3828         if (dev->data->all_multicast)
3829                 ret = bnxt_allmulticast_enable_op(dev);
3830         if (dev->data->promiscuous)
3831                 ret = bnxt_promiscuous_enable_op(dev);
3832
3833         /* TODO restore other filters as well */
3834         return ret;
3835 }
3836
3837 static void bnxt_dev_recover(void *arg)
3838 {
3839         struct bnxt *bp = arg;
3840         int timeout = bp->fw_reset_max_msecs;
3841         int rc = 0;
3842
3843         /* Clear Error flag so that device re-init should happen */
3844         bp->flags &= ~BNXT_FLAG_FATAL_ERROR;
3845
3846         do {
3847                 rc = bnxt_hwrm_ver_get(bp);
3848                 if (rc == 0)
3849                         break;
3850                 rte_delay_ms(BNXT_FW_READY_WAIT_INTERVAL);
3851                 timeout -= BNXT_FW_READY_WAIT_INTERVAL;
3852         } while (rc && timeout);
3853
3854         if (rc) {
3855                 PMD_DRV_LOG(ERR, "FW is not Ready after reset\n");
3856                 goto err;
3857         }
3858
3859         rc = bnxt_init_resources(bp, true);
3860         if (rc) {
3861                 PMD_DRV_LOG(ERR,
3862                             "Failed to initialize resources after reset\n");
3863                 goto err;
3864         }
3865         /* clear reset flag as the device is initialized now */
3866         bp->flags &= ~BNXT_FLAG_FW_RESET;
3867
3868         rc = bnxt_dev_start_op(bp->eth_dev);
3869         if (rc) {
3870                 PMD_DRV_LOG(ERR, "Failed to start port after reset\n");
3871                 goto err;
3872         }
3873
3874         rc = bnxt_restore_filters(bp);
3875         if (rc)
3876                 goto err;
3877
3878         PMD_DRV_LOG(INFO, "Recovered from FW reset\n");
3879         return;
3880 err:
3881         bp->flags |= BNXT_FLAG_FATAL_ERROR;
3882         bnxt_uninit_resources(bp, false);
3883         PMD_DRV_LOG(ERR, "Failed to recover from FW reset\n");
3884 }
3885
3886 void bnxt_dev_reset_and_resume(void *arg)
3887 {
3888         struct bnxt *bp = arg;
3889         int rc;
3890
3891         bnxt_dev_cleanup(bp);
3892
3893         bnxt_wait_for_device_shutdown(bp);
3894
3895         rc = rte_eal_alarm_set(US_PER_MS * bp->fw_reset_min_msecs,
3896                                bnxt_dev_recover, (void *)bp);
3897         if (rc)
3898                 PMD_DRV_LOG(ERR, "Error setting recovery alarm");
3899 }
3900
3901 uint32_t bnxt_read_fw_status_reg(struct bnxt *bp, uint32_t index)
3902 {
3903         struct bnxt_error_recovery_info *info = bp->recovery_info;
3904         uint32_t reg = info->status_regs[index];
3905         uint32_t type, offset, val = 0;
3906
3907         type = BNXT_FW_STATUS_REG_TYPE(reg);
3908         offset = BNXT_FW_STATUS_REG_OFF(reg);
3909
3910         switch (type) {
3911         case BNXT_FW_STATUS_REG_TYPE_CFG:
3912                 rte_pci_read_config(bp->pdev, &val, sizeof(val), offset);
3913                 break;
3914         case BNXT_FW_STATUS_REG_TYPE_GRC:
3915                 offset = info->mapped_status_regs[index];
3916                 /* FALLTHROUGH */
3917         case BNXT_FW_STATUS_REG_TYPE_BAR0:
3918                 val = rte_le_to_cpu_32(rte_read32((uint8_t *)bp->bar0 +
3919                                        offset));
3920                 break;
3921         }
3922
3923         return val;
3924 }
3925
3926 static int bnxt_fw_reset_all(struct bnxt *bp)
3927 {
3928         struct bnxt_error_recovery_info *info = bp->recovery_info;
3929         uint32_t i;
3930         int rc = 0;
3931
3932         if (info->flags & BNXT_FLAG_ERROR_RECOVERY_HOST) {
3933                 /* Reset through master function driver */
3934                 for (i = 0; i < info->reg_array_cnt; i++)
3935                         bnxt_write_fw_reset_reg(bp, i);
3936                 /* Wait for time specified by FW after triggering reset */
3937                 rte_delay_ms(info->master_func_wait_period_after_reset);
3938         } else if (info->flags & BNXT_FLAG_ERROR_RECOVERY_CO_CPU) {
3939                 /* Reset with the help of Kong processor */
3940                 rc = bnxt_hwrm_fw_reset(bp);
3941                 if (rc)
3942                         PMD_DRV_LOG(ERR, "Failed to reset FW\n");
3943         }
3944
3945         return rc;
3946 }
3947
3948 static void bnxt_fw_reset_cb(void *arg)
3949 {
3950         struct bnxt *bp = arg;
3951         struct bnxt_error_recovery_info *info = bp->recovery_info;
3952         int rc = 0;
3953
3954         /* Only Master function can do FW reset */
3955         if (bnxt_is_master_func(bp) &&
3956             bnxt_is_recovery_enabled(bp)) {
3957                 rc = bnxt_fw_reset_all(bp);
3958                 if (rc) {
3959                         PMD_DRV_LOG(ERR, "Adapter recovery failed\n");
3960                         return;
3961                 }
3962         }
3963
3964         /* if recovery method is ERROR_RECOVERY_CO_CPU, KONG will send
3965          * EXCEPTION_FATAL_ASYNC event to all the functions
3966          * (including MASTER FUNC). After receiving this Async, all the active
3967          * drivers should treat this case as FW initiated recovery
3968          */
3969         if (info->flags & BNXT_FLAG_ERROR_RECOVERY_HOST) {
3970                 bp->fw_reset_min_msecs = BNXT_MIN_FW_READY_TIMEOUT;
3971                 bp->fw_reset_max_msecs = BNXT_MAX_FW_RESET_TIMEOUT;
3972
3973                 /* To recover from error */
3974                 rte_eal_alarm_set(US_PER_MS, bnxt_dev_reset_and_resume,
3975                                   (void *)bp);
3976         }
3977 }
3978
3979 /* Driver should poll FW heartbeat, reset_counter with the frequency
3980  * advertised by FW in HWRM_ERROR_RECOVERY_QCFG.
3981  * When the driver detects heartbeat stop or change in reset_counter,
3982  * it has to trigger a reset to recover from the error condition.
3983  * A “master PF” is the function who will have the privilege to
3984  * initiate the chimp reset. The master PF will be elected by the
3985  * firmware and will be notified through async message.
3986  */
3987 static void bnxt_check_fw_health(void *arg)
3988 {
3989         struct bnxt *bp = arg;
3990         struct bnxt_error_recovery_info *info = bp->recovery_info;
3991         uint32_t val = 0, wait_msec;
3992
3993         if (!info || !bnxt_is_recovery_enabled(bp) ||
3994             is_bnxt_in_error(bp))
3995                 return;
3996
3997         val = bnxt_read_fw_status_reg(bp, BNXT_FW_HEARTBEAT_CNT_REG);
3998         if (val == info->last_heart_beat)
3999                 goto reset;
4000
4001         info->last_heart_beat = val;
4002
4003         val = bnxt_read_fw_status_reg(bp, BNXT_FW_RECOVERY_CNT_REG);
4004         if (val != info->last_reset_counter)
4005                 goto reset;
4006
4007         info->last_reset_counter = val;
4008
4009         rte_eal_alarm_set(US_PER_MS * info->driver_polling_freq,
4010                           bnxt_check_fw_health, (void *)bp);
4011
4012         return;
4013 reset:
4014         /* Stop DMA to/from device */
4015         bp->flags |= BNXT_FLAG_FATAL_ERROR;
4016         bp->flags |= BNXT_FLAG_FW_RESET;
4017
4018         PMD_DRV_LOG(ERR, "Detected FW dead condition\n");
4019
4020         if (bnxt_is_master_func(bp))
4021                 wait_msec = info->master_func_wait_period;
4022         else
4023                 wait_msec = info->normal_func_wait_period;
4024
4025         rte_eal_alarm_set(US_PER_MS * wait_msec,
4026                           bnxt_fw_reset_cb, (void *)bp);
4027 }
4028
4029 void bnxt_schedule_fw_health_check(struct bnxt *bp)
4030 {
4031         uint32_t polling_freq;
4032
4033         if (!bnxt_is_recovery_enabled(bp))
4034                 return;
4035
4036         if (bp->flags & BNXT_FLAG_FW_HEALTH_CHECK_SCHEDULED)
4037                 return;
4038
4039         polling_freq = bp->recovery_info->driver_polling_freq;
4040
4041         rte_eal_alarm_set(US_PER_MS * polling_freq,
4042                           bnxt_check_fw_health, (void *)bp);
4043         bp->flags |= BNXT_FLAG_FW_HEALTH_CHECK_SCHEDULED;
4044 }
4045
4046 static void bnxt_cancel_fw_health_check(struct bnxt *bp)
4047 {
4048         if (!bnxt_is_recovery_enabled(bp))
4049                 return;
4050
4051         rte_eal_alarm_cancel(bnxt_check_fw_health, (void *)bp);
4052         bp->flags &= ~BNXT_FLAG_FW_HEALTH_CHECK_SCHEDULED;
4053 }
4054
4055 static bool bnxt_vf_pciid(uint16_t id)
4056 {
4057         if (id == BROADCOM_DEV_ID_57304_VF ||
4058             id == BROADCOM_DEV_ID_57406_VF ||
4059             id == BROADCOM_DEV_ID_5731X_VF ||
4060             id == BROADCOM_DEV_ID_5741X_VF ||
4061             id == BROADCOM_DEV_ID_57414_VF ||
4062             id == BROADCOM_DEV_ID_STRATUS_NIC_VF1 ||
4063             id == BROADCOM_DEV_ID_STRATUS_NIC_VF2 ||
4064             id == BROADCOM_DEV_ID_58802_VF ||
4065             id == BROADCOM_DEV_ID_57500_VF1 ||
4066             id == BROADCOM_DEV_ID_57500_VF2)
4067                 return true;
4068         return false;
4069 }
4070
4071 bool bnxt_stratus_device(struct bnxt *bp)
4072 {
4073         uint16_t id = bp->pdev->id.device_id;
4074
4075         if (id == BROADCOM_DEV_ID_STRATUS_NIC ||
4076             id == BROADCOM_DEV_ID_STRATUS_NIC_VF1 ||
4077             id == BROADCOM_DEV_ID_STRATUS_NIC_VF2)
4078                 return true;
4079         return false;
4080 }
4081
4082 static int bnxt_init_board(struct rte_eth_dev *eth_dev)
4083 {
4084         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
4085         struct bnxt *bp = eth_dev->data->dev_private;
4086
4087         /* enable device (incl. PCI PM wakeup), and bus-mastering */
4088         bp->bar0 = (void *)pci_dev->mem_resource[0].addr;
4089         bp->doorbell_base = (void *)pci_dev->mem_resource[2].addr;
4090         if (!bp->bar0 || !bp->doorbell_base) {
4091                 PMD_DRV_LOG(ERR, "Unable to access Hardware\n");
4092                 return -ENODEV;
4093         }
4094
4095         bp->eth_dev = eth_dev;
4096         bp->pdev = pci_dev;
4097
4098         return 0;
4099 }
4100
4101 static int bnxt_alloc_ctx_mem_blk(__rte_unused struct bnxt *bp,
4102                                   struct bnxt_ctx_pg_info *ctx_pg,
4103                                   uint32_t mem_size,
4104                                   const char *suffix,
4105                                   uint16_t idx)
4106 {
4107         struct bnxt_ring_mem_info *rmem = &ctx_pg->ring_mem;
4108         const struct rte_memzone *mz = NULL;
4109         char mz_name[RTE_MEMZONE_NAMESIZE];
4110         rte_iova_t mz_phys_addr;
4111         uint64_t valid_bits = 0;
4112         uint32_t sz;
4113         int i;
4114
4115         if (!mem_size)
4116                 return 0;
4117
4118         rmem->nr_pages = RTE_ALIGN_MUL_CEIL(mem_size, BNXT_PAGE_SIZE) /
4119                          BNXT_PAGE_SIZE;
4120         rmem->page_size = BNXT_PAGE_SIZE;
4121         rmem->pg_arr = ctx_pg->ctx_pg_arr;
4122         rmem->dma_arr = ctx_pg->ctx_dma_arr;
4123         rmem->flags = BNXT_RMEM_VALID_PTE_FLAG;
4124
4125         valid_bits = PTU_PTE_VALID;
4126
4127         if (rmem->nr_pages > 1) {
4128                 snprintf(mz_name, RTE_MEMZONE_NAMESIZE,
4129                          "bnxt_ctx_pg_tbl%s_%x_%d",
4130                          suffix, idx, bp->eth_dev->data->port_id);
4131                 mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0;
4132                 mz = rte_memzone_lookup(mz_name);
4133                 if (!mz) {
4134                         mz = rte_memzone_reserve_aligned(mz_name,
4135                                                 rmem->nr_pages * 8,
4136                                                 SOCKET_ID_ANY,
4137                                                 RTE_MEMZONE_2MB |
4138                                                 RTE_MEMZONE_SIZE_HINT_ONLY |
4139                                                 RTE_MEMZONE_IOVA_CONTIG,
4140                                                 BNXT_PAGE_SIZE);
4141                         if (mz == NULL)
4142                                 return -ENOMEM;
4143                 }
4144
4145                 memset(mz->addr, 0, mz->len);
4146                 mz_phys_addr = mz->iova;
4147                 if ((unsigned long)mz->addr == mz_phys_addr) {
4148                         PMD_DRV_LOG(DEBUG,
4149                                     "physical address same as virtual\n");
4150                         PMD_DRV_LOG(DEBUG, "Using rte_mem_virt2iova()\n");
4151                         mz_phys_addr = rte_mem_virt2iova(mz->addr);
4152                         if (mz_phys_addr == RTE_BAD_IOVA) {
4153                                 PMD_DRV_LOG(ERR,
4154                                         "unable to map addr to phys memory\n");
4155                                 return -ENOMEM;
4156                         }
4157                 }
4158                 rte_mem_lock_page(((char *)mz->addr));
4159
4160                 rmem->pg_tbl = mz->addr;
4161                 rmem->pg_tbl_map = mz_phys_addr;
4162                 rmem->pg_tbl_mz = mz;
4163         }
4164
4165         snprintf(mz_name, RTE_MEMZONE_NAMESIZE, "bnxt_ctx_%s_%x_%d",
4166                  suffix, idx, bp->eth_dev->data->port_id);
4167         mz = rte_memzone_lookup(mz_name);
4168         if (!mz) {
4169                 mz = rte_memzone_reserve_aligned(mz_name,
4170                                                  mem_size,
4171                                                  SOCKET_ID_ANY,
4172                                                  RTE_MEMZONE_1GB |
4173                                                  RTE_MEMZONE_SIZE_HINT_ONLY |
4174                                                  RTE_MEMZONE_IOVA_CONTIG,
4175                                                  BNXT_PAGE_SIZE);
4176                 if (mz == NULL)
4177                         return -ENOMEM;
4178         }
4179
4180         memset(mz->addr, 0, mz->len);
4181         mz_phys_addr = mz->iova;
4182         if ((unsigned long)mz->addr == mz_phys_addr) {
4183                 PMD_DRV_LOG(DEBUG,
4184                             "Memzone physical address same as virtual.\n");
4185                 PMD_DRV_LOG(DEBUG, "Using rte_mem_virt2iova()\n");
4186                 for (sz = 0; sz < mem_size; sz += BNXT_PAGE_SIZE)
4187                         rte_mem_lock_page(((char *)mz->addr) + sz);
4188                 mz_phys_addr = rte_mem_virt2iova(mz->addr);
4189                 if (mz_phys_addr == RTE_BAD_IOVA) {
4190                         PMD_DRV_LOG(ERR,
4191                                     "unable to map addr to phys memory\n");
4192                         return -ENOMEM;
4193                 }
4194         }
4195
4196         for (sz = 0, i = 0; sz < mem_size; sz += BNXT_PAGE_SIZE, i++) {
4197                 rte_mem_lock_page(((char *)mz->addr) + sz);
4198                 rmem->pg_arr[i] = ((char *)mz->addr) + sz;
4199                 rmem->dma_arr[i] = mz_phys_addr + sz;
4200
4201                 if (rmem->nr_pages > 1) {
4202                         if (i == rmem->nr_pages - 2 &&
4203                             (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
4204                                 valid_bits |= PTU_PTE_NEXT_TO_LAST;
4205                         else if (i == rmem->nr_pages - 1 &&
4206                                  (rmem->flags & BNXT_RMEM_RING_PTE_FLAG))
4207                                 valid_bits |= PTU_PTE_LAST;
4208
4209                         rmem->pg_tbl[i] = rte_cpu_to_le_64(rmem->dma_arr[i] |
4210                                                            valid_bits);
4211                 }
4212         }
4213
4214         rmem->mz = mz;
4215         if (rmem->vmem_size)
4216                 rmem->vmem = (void **)mz->addr;
4217         rmem->dma_arr[0] = mz_phys_addr;
4218         return 0;
4219 }
4220
4221 static void bnxt_free_ctx_mem(struct bnxt *bp)
4222 {
4223         int i;
4224
4225         if (!bp->ctx || !(bp->ctx->flags & BNXT_CTX_FLAG_INITED))
4226                 return;
4227
4228         bp->ctx->flags &= ~BNXT_CTX_FLAG_INITED;
4229         rte_memzone_free(bp->ctx->qp_mem.ring_mem.mz);
4230         rte_memzone_free(bp->ctx->srq_mem.ring_mem.mz);
4231         rte_memzone_free(bp->ctx->cq_mem.ring_mem.mz);
4232         rte_memzone_free(bp->ctx->vnic_mem.ring_mem.mz);
4233         rte_memzone_free(bp->ctx->stat_mem.ring_mem.mz);
4234         rte_memzone_free(bp->ctx->qp_mem.ring_mem.pg_tbl_mz);
4235         rte_memzone_free(bp->ctx->srq_mem.ring_mem.pg_tbl_mz);
4236         rte_memzone_free(bp->ctx->cq_mem.ring_mem.pg_tbl_mz);
4237         rte_memzone_free(bp->ctx->vnic_mem.ring_mem.pg_tbl_mz);
4238         rte_memzone_free(bp->ctx->stat_mem.ring_mem.pg_tbl_mz);
4239
4240         for (i = 0; i < BNXT_MAX_Q; i++) {
4241                 if (bp->ctx->tqm_mem[i])
4242                         rte_memzone_free(bp->ctx->tqm_mem[i]->ring_mem.mz);
4243         }
4244
4245         rte_free(bp->ctx);
4246         bp->ctx = NULL;
4247 }
4248
4249 #define bnxt_roundup(x, y)   ((((x) + ((y) - 1)) / (y)) * (y))
4250
4251 #define min_t(type, x, y) ({                    \
4252         type __min1 = (x);                      \
4253         type __min2 = (y);                      \
4254         __min1 < __min2 ? __min1 : __min2; })
4255
4256 #define max_t(type, x, y) ({                    \
4257         type __max1 = (x);                      \
4258         type __max2 = (y);                      \
4259         __max1 > __max2 ? __max1 : __max2; })
4260
4261 #define clamp_t(type, _x, min, max)     min_t(type, max_t(type, _x, min), max)
4262
4263 int bnxt_alloc_ctx_mem(struct bnxt *bp)
4264 {
4265         struct bnxt_ctx_pg_info *ctx_pg;
4266         struct bnxt_ctx_mem_info *ctx;
4267         uint32_t mem_size, ena, entries;
4268         int i, rc;
4269
4270         rc = bnxt_hwrm_func_backing_store_qcaps(bp);
4271         if (rc) {
4272                 PMD_DRV_LOG(ERR, "Query context mem capability failed\n");
4273                 return rc;
4274         }
4275         ctx = bp->ctx;
4276         if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED))
4277                 return 0;
4278
4279         ctx_pg = &ctx->qp_mem;
4280         ctx_pg->entries = ctx->qp_min_qp1_entries + ctx->qp_max_l2_entries;
4281         mem_size = ctx->qp_entry_size * ctx_pg->entries;
4282         rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size, "qp_mem", 0);
4283         if (rc)
4284                 return rc;
4285
4286         ctx_pg = &ctx->srq_mem;
4287         ctx_pg->entries = ctx->srq_max_l2_entries;
4288         mem_size = ctx->srq_entry_size * ctx_pg->entries;
4289         rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size, "srq_mem", 0);
4290         if (rc)
4291                 return rc;
4292
4293         ctx_pg = &ctx->cq_mem;
4294         ctx_pg->entries = ctx->cq_max_l2_entries;
4295         mem_size = ctx->cq_entry_size * ctx_pg->entries;
4296         rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size, "cq_mem", 0);
4297         if (rc)
4298                 return rc;
4299
4300         ctx_pg = &ctx->vnic_mem;
4301         ctx_pg->entries = ctx->vnic_max_vnic_entries +
4302                 ctx->vnic_max_ring_table_entries;
4303         mem_size = ctx->vnic_entry_size * ctx_pg->entries;
4304         rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size, "vnic_mem", 0);
4305         if (rc)
4306                 return rc;
4307
4308         ctx_pg = &ctx->stat_mem;
4309         ctx_pg->entries = ctx->stat_max_entries;
4310         mem_size = ctx->stat_entry_size * ctx_pg->entries;
4311         rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size, "stat_mem", 0);
4312         if (rc)
4313                 return rc;
4314
4315         entries = ctx->qp_max_l2_entries +
4316                   ctx->vnic_max_vnic_entries +
4317                   ctx->tqm_min_entries_per_ring;
4318         entries = bnxt_roundup(entries, ctx->tqm_entries_multiple);
4319         entries = clamp_t(uint32_t, entries, ctx->tqm_min_entries_per_ring,
4320                           ctx->tqm_max_entries_per_ring);
4321         for (i = 0, ena = 0; i < BNXT_MAX_Q; i++) {
4322                 ctx_pg = ctx->tqm_mem[i];
4323                 /* use min tqm entries for now. */
4324                 ctx_pg->entries = entries;
4325                 mem_size = ctx->tqm_entry_size * ctx_pg->entries;
4326                 rc = bnxt_alloc_ctx_mem_blk(bp, ctx_pg, mem_size, "tqm_mem", i);
4327                 if (rc)
4328                         return rc;
4329                 ena |= HWRM_FUNC_BACKING_STORE_CFG_INPUT_ENABLES_TQM_SP << i;
4330         }
4331
4332         ena |= FUNC_BACKING_STORE_CFG_INPUT_DFLT_ENABLES;
4333         rc = bnxt_hwrm_func_backing_store_cfg(bp, ena);
4334         if (rc)
4335                 PMD_DRV_LOG(ERR,
4336                             "Failed to configure context mem: rc = %d\n", rc);
4337         else
4338                 ctx->flags |= BNXT_CTX_FLAG_INITED;
4339
4340         return rc;
4341 }
4342
4343 static int bnxt_alloc_stats_mem(struct bnxt *bp)
4344 {
4345         struct rte_pci_device *pci_dev = bp->pdev;
4346         char mz_name[RTE_MEMZONE_NAMESIZE];
4347         const struct rte_memzone *mz = NULL;
4348         uint32_t total_alloc_len;
4349         rte_iova_t mz_phys_addr;
4350
4351         if (pci_dev->id.device_id == BROADCOM_DEV_ID_NS2)
4352                 return 0;
4353
4354         snprintf(mz_name, RTE_MEMZONE_NAMESIZE,
4355                  "bnxt_" PCI_PRI_FMT "-%s", pci_dev->addr.domain,
4356                  pci_dev->addr.bus, pci_dev->addr.devid,
4357                  pci_dev->addr.function, "rx_port_stats");
4358         mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0;
4359         mz = rte_memzone_lookup(mz_name);
4360         total_alloc_len =
4361                 RTE_CACHE_LINE_ROUNDUP(sizeof(struct rx_port_stats) +
4362                                        sizeof(struct rx_port_stats_ext) + 512);
4363         if (!mz) {
4364                 mz = rte_memzone_reserve(mz_name, total_alloc_len,
4365                                          SOCKET_ID_ANY,
4366                                          RTE_MEMZONE_2MB |
4367                                          RTE_MEMZONE_SIZE_HINT_ONLY |
4368                                          RTE_MEMZONE_IOVA_CONTIG);
4369                 if (mz == NULL)
4370                         return -ENOMEM;
4371         }
4372         memset(mz->addr, 0, mz->len);
4373         mz_phys_addr = mz->iova;
4374         if ((unsigned long)mz->addr == mz_phys_addr) {
4375                 PMD_DRV_LOG(DEBUG,
4376                             "Memzone physical address same as virtual.\n");
4377                 PMD_DRV_LOG(DEBUG,
4378                             "Using rte_mem_virt2iova()\n");
4379                 mz_phys_addr = rte_mem_virt2iova(mz->addr);
4380                 if (mz_phys_addr == RTE_BAD_IOVA) {
4381                         PMD_DRV_LOG(ERR,
4382                                     "Can't map address to physical memory\n");
4383                         return -ENOMEM;
4384                 }
4385         }
4386
4387         bp->rx_mem_zone = (const void *)mz;
4388         bp->hw_rx_port_stats = mz->addr;
4389         bp->hw_rx_port_stats_map = mz_phys_addr;
4390
4391         snprintf(mz_name, RTE_MEMZONE_NAMESIZE,
4392                  "bnxt_" PCI_PRI_FMT "-%s", pci_dev->addr.domain,
4393                  pci_dev->addr.bus, pci_dev->addr.devid,
4394                  pci_dev->addr.function, "tx_port_stats");
4395         mz_name[RTE_MEMZONE_NAMESIZE - 1] = 0;
4396         mz = rte_memzone_lookup(mz_name);
4397         total_alloc_len =
4398                 RTE_CACHE_LINE_ROUNDUP(sizeof(struct tx_port_stats) +
4399                                        sizeof(struct tx_port_stats_ext) + 512);
4400         if (!mz) {
4401                 mz = rte_memzone_reserve(mz_name,
4402                                          total_alloc_len,
4403                                          SOCKET_ID_ANY,
4404                                          RTE_MEMZONE_2MB |
4405                                          RTE_MEMZONE_SIZE_HINT_ONLY |
4406                                          RTE_MEMZONE_IOVA_CONTIG);
4407                 if (mz == NULL)
4408                         return -ENOMEM;
4409         }
4410         memset(mz->addr, 0, mz->len);
4411         mz_phys_addr = mz->iova;
4412         if ((unsigned long)mz->addr == mz_phys_addr) {
4413                 PMD_DRV_LOG(DEBUG,
4414                             "Memzone physical address same as virtual\n");
4415                 PMD_DRV_LOG(DEBUG, "Using rte_mem_virt2iova()\n");
4416                 mz_phys_addr = rte_mem_virt2iova(mz->addr);
4417                 if (mz_phys_addr == RTE_BAD_IOVA) {
4418                         PMD_DRV_LOG(ERR,
4419                                     "Can't map address to physical memory\n");
4420                         return -ENOMEM;
4421                 }
4422         }
4423
4424         bp->tx_mem_zone = (const void *)mz;
4425         bp->hw_tx_port_stats = mz->addr;
4426         bp->hw_tx_port_stats_map = mz_phys_addr;
4427         bp->flags |= BNXT_FLAG_PORT_STATS;
4428
4429         /* Display extended statistics if FW supports it */
4430         if (bp->hwrm_spec_code < HWRM_SPEC_CODE_1_8_4 ||
4431             bp->hwrm_spec_code == HWRM_SPEC_CODE_1_9_0 ||
4432             !(bp->flags & BNXT_FLAG_EXT_STATS_SUPPORTED))
4433                 return 0;
4434
4435         bp->hw_rx_port_stats_ext = (void *)
4436                 ((uint8_t *)bp->hw_rx_port_stats +
4437                  sizeof(struct rx_port_stats));
4438         bp->hw_rx_port_stats_ext_map = bp->hw_rx_port_stats_map +
4439                 sizeof(struct rx_port_stats);
4440         bp->flags |= BNXT_FLAG_EXT_RX_PORT_STATS;
4441
4442         if (bp->hwrm_spec_code < HWRM_SPEC_CODE_1_9_2 ||
4443             bp->flags & BNXT_FLAG_EXT_STATS_SUPPORTED) {
4444                 bp->hw_tx_port_stats_ext = (void *)
4445                         ((uint8_t *)bp->hw_tx_port_stats +
4446                          sizeof(struct tx_port_stats));
4447                 bp->hw_tx_port_stats_ext_map =
4448                         bp->hw_tx_port_stats_map +
4449                         sizeof(struct tx_port_stats);
4450                 bp->flags |= BNXT_FLAG_EXT_TX_PORT_STATS;
4451         }
4452
4453         return 0;
4454 }
4455
4456 static int bnxt_setup_mac_addr(struct rte_eth_dev *eth_dev)
4457 {
4458         struct bnxt *bp = eth_dev->data->dev_private;
4459         int rc = 0;
4460
4461         eth_dev->data->mac_addrs = rte_zmalloc("bnxt_mac_addr_tbl",
4462                                                RTE_ETHER_ADDR_LEN *
4463                                                bp->max_l2_ctx,
4464                                                0);
4465         if (eth_dev->data->mac_addrs == NULL) {
4466                 PMD_DRV_LOG(ERR, "Failed to alloc MAC addr tbl\n");
4467                 return -ENOMEM;
4468         }
4469
4470         if (bnxt_check_zero_bytes(bp->dflt_mac_addr, RTE_ETHER_ADDR_LEN)) {
4471                 if (BNXT_PF(bp))
4472                         return -EINVAL;
4473
4474                 /* Generate a random MAC address, if none was assigned by PF */
4475                 PMD_DRV_LOG(INFO, "VF MAC address not assigned by Host PF\n");
4476                 bnxt_eth_hw_addr_random(bp->mac_addr);
4477                 PMD_DRV_LOG(INFO,
4478                             "Assign random MAC:%02X:%02X:%02X:%02X:%02X:%02X\n",
4479                             bp->mac_addr[0], bp->mac_addr[1], bp->mac_addr[2],
4480                             bp->mac_addr[3], bp->mac_addr[4], bp->mac_addr[5]);
4481
4482                 rc = bnxt_hwrm_set_mac(bp);
4483                 if (!rc)
4484                         memcpy(&bp->eth_dev->data->mac_addrs[0], bp->mac_addr,
4485                                RTE_ETHER_ADDR_LEN);
4486                 return rc;
4487         }
4488
4489         /* Copy the permanent MAC from the FUNC_QCAPS response */
4490         memcpy(bp->mac_addr, bp->dflt_mac_addr, RTE_ETHER_ADDR_LEN);
4491         memcpy(&eth_dev->data->mac_addrs[0], bp->mac_addr, RTE_ETHER_ADDR_LEN);
4492
4493         return rc;
4494 }
4495
4496 static int bnxt_restore_dflt_mac(struct bnxt *bp)
4497 {
4498         int rc = 0;
4499
4500         /* MAC is already configured in FW */
4501         if (!bnxt_check_zero_bytes(bp->dflt_mac_addr, RTE_ETHER_ADDR_LEN))
4502                 return 0;
4503
4504         /* Restore the old MAC configured */
4505         rc = bnxt_hwrm_set_mac(bp);
4506         if (rc)
4507                 PMD_DRV_LOG(ERR, "Failed to restore MAC address\n");
4508
4509         return rc;
4510 }
4511
4512 static void bnxt_config_vf_req_fwd(struct bnxt *bp)
4513 {
4514         if (!BNXT_PF(bp))
4515                 return;
4516
4517 #define ALLOW_FUNC(x)   \
4518         { \
4519                 uint32_t arg = (x); \
4520                 bp->pf.vf_req_fwd[((arg) >> 5)] &= \
4521                 ~rte_cpu_to_le_32(1 << ((arg) & 0x1f)); \
4522         }
4523
4524         /* Forward all requests if firmware is new enough */
4525         if (((bp->fw_ver >= ((20 << 24) | (6 << 16) | (100 << 8))) &&
4526              (bp->fw_ver < ((20 << 24) | (7 << 16)))) ||
4527             ((bp->fw_ver >= ((20 << 24) | (8 << 16))))) {
4528                 memset(bp->pf.vf_req_fwd, 0xff, sizeof(bp->pf.vf_req_fwd));
4529         } else {
4530                 PMD_DRV_LOG(WARNING,
4531                             "Firmware too old for VF mailbox functionality\n");
4532                 memset(bp->pf.vf_req_fwd, 0, sizeof(bp->pf.vf_req_fwd));
4533         }
4534
4535         /*
4536          * The following are used for driver cleanup. If we disallow these,
4537          * VF drivers can't clean up cleanly.
4538          */
4539         ALLOW_FUNC(HWRM_FUNC_DRV_UNRGTR);
4540         ALLOW_FUNC(HWRM_VNIC_FREE);
4541         ALLOW_FUNC(HWRM_RING_FREE);
4542         ALLOW_FUNC(HWRM_RING_GRP_FREE);
4543         ALLOW_FUNC(HWRM_VNIC_RSS_COS_LB_CTX_FREE);
4544         ALLOW_FUNC(HWRM_CFA_L2_FILTER_FREE);
4545         ALLOW_FUNC(HWRM_STAT_CTX_FREE);
4546         ALLOW_FUNC(HWRM_PORT_PHY_QCFG);
4547         ALLOW_FUNC(HWRM_VNIC_TPA_CFG);
4548 }
4549
4550 static int bnxt_init_fw(struct bnxt *bp)
4551 {
4552         uint16_t mtu;
4553         int rc = 0;
4554
4555         rc = bnxt_hwrm_ver_get(bp);
4556         if (rc)
4557                 return rc;
4558
4559         rc = bnxt_hwrm_func_reset(bp);
4560         if (rc)
4561                 return -EIO;
4562
4563         rc = bnxt_hwrm_vnic_qcaps(bp);
4564         if (rc)
4565                 return rc;
4566
4567         rc = bnxt_hwrm_queue_qportcfg(bp);
4568         if (rc)
4569                 return rc;
4570
4571         /* Get the MAX capabilities for this function.
4572          * This function also allocates context memory for TQM rings and
4573          * informs the firmware about this allocated backing store memory.
4574          */
4575         rc = bnxt_hwrm_func_qcaps(bp);
4576         if (rc)
4577                 return rc;
4578
4579         rc = bnxt_hwrm_func_qcfg(bp, &mtu);
4580         if (rc)
4581                 return rc;
4582
4583         rc = bnxt_hwrm_cfa_adv_flow_mgmt_qcaps(bp);
4584         if (rc)
4585                 return rc;
4586
4587         /* Get the adapter error recovery support info */
4588         rc = bnxt_hwrm_error_recovery_qcfg(bp);
4589         if (rc)
4590                 bp->flags &= ~BNXT_FLAG_FW_CAP_ERROR_RECOVERY;
4591
4592         if (mtu >= RTE_ETHER_MIN_MTU && mtu <= BNXT_MAX_MTU &&
4593             mtu != bp->eth_dev->data->mtu)
4594                 bp->eth_dev->data->mtu = mtu;
4595
4596         bnxt_hwrm_port_led_qcaps(bp);
4597
4598         return 0;
4599 }
4600
4601 static int
4602 bnxt_init_locks(struct bnxt *bp)
4603 {
4604         int err;
4605
4606         err = pthread_mutex_init(&bp->flow_lock, NULL);
4607         if (err) {
4608                 PMD_DRV_LOG(ERR, "Unable to initialize flow_lock\n");
4609                 return err;
4610         }
4611
4612         err = pthread_mutex_init(&bp->def_cp_lock, NULL);
4613         if (err)
4614                 PMD_DRV_LOG(ERR, "Unable to initialize def_cp_lock\n");
4615         return err;
4616 }
4617
4618 static int bnxt_init_resources(struct bnxt *bp, bool reconfig_dev)
4619 {
4620         int rc;
4621
4622         rc = bnxt_init_fw(bp);
4623         if (rc)
4624                 return rc;
4625
4626         if (!reconfig_dev) {
4627                 rc = bnxt_setup_mac_addr(bp->eth_dev);
4628                 if (rc)
4629                         return rc;
4630         } else {
4631                 rc = bnxt_restore_dflt_mac(bp);
4632                 if (rc)
4633                         return rc;
4634         }
4635
4636         bnxt_config_vf_req_fwd(bp);
4637
4638         rc = bnxt_hwrm_func_driver_register(bp);
4639         if (rc) {
4640                 PMD_DRV_LOG(ERR, "Failed to register driver");
4641                 return -EBUSY;
4642         }
4643
4644         if (BNXT_PF(bp)) {
4645                 if (bp->pdev->max_vfs) {
4646                         rc = bnxt_hwrm_allocate_vfs(bp, bp->pdev->max_vfs);
4647                         if (rc) {
4648                                 PMD_DRV_LOG(ERR, "Failed to allocate VFs\n");
4649                                 return rc;
4650                         }
4651                 } else {
4652                         rc = bnxt_hwrm_allocate_pf_only(bp);
4653                         if (rc) {
4654                                 PMD_DRV_LOG(ERR,
4655                                             "Failed to allocate PF resources");
4656                                 return rc;
4657                         }
4658                 }
4659         }
4660
4661         rc = bnxt_alloc_mem(bp, reconfig_dev);
4662         if (rc)
4663                 return rc;
4664
4665         rc = bnxt_setup_int(bp);
4666         if (rc)
4667                 return rc;
4668
4669         bnxt_init_nic(bp);
4670
4671         rc = bnxt_request_int(bp);
4672         if (rc)
4673                 return rc;
4674
4675         rc = bnxt_init_locks(bp);
4676         if (rc)
4677                 return rc;
4678
4679         return 0;
4680 }
4681
4682 static int
4683 bnxt_dev_init(struct rte_eth_dev *eth_dev)
4684 {
4685         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
4686         static int version_printed;
4687         struct bnxt *bp;
4688         int rc;
4689
4690         if (version_printed++ == 0)
4691                 PMD_DRV_LOG(INFO, "%s\n", bnxt_version);
4692
4693         eth_dev->dev_ops = &bnxt_dev_ops;
4694         eth_dev->rx_pkt_burst = &bnxt_recv_pkts;
4695         eth_dev->tx_pkt_burst = &bnxt_xmit_pkts;
4696
4697         /*
4698          * For secondary processes, we don't initialise any further
4699          * as primary has already done this work.
4700          */
4701         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
4702                 return 0;
4703
4704         rte_eth_copy_pci_info(eth_dev, pci_dev);
4705
4706         bp = eth_dev->data->dev_private;
4707
4708         bp->dev_stopped = 1;
4709
4710         if (bnxt_vf_pciid(pci_dev->id.device_id))
4711                 bp->flags |= BNXT_FLAG_VF;
4712
4713         if (pci_dev->id.device_id == BROADCOM_DEV_ID_57508 ||
4714             pci_dev->id.device_id == BROADCOM_DEV_ID_57504 ||
4715             pci_dev->id.device_id == BROADCOM_DEV_ID_57502 ||
4716             pci_dev->id.device_id == BROADCOM_DEV_ID_57500_VF1 ||
4717             pci_dev->id.device_id == BROADCOM_DEV_ID_57500_VF2)
4718                 bp->flags |= BNXT_FLAG_THOR_CHIP;
4719
4720         if (pci_dev->id.device_id == BROADCOM_DEV_ID_58802 ||
4721             pci_dev->id.device_id == BROADCOM_DEV_ID_58804 ||
4722             pci_dev->id.device_id == BROADCOM_DEV_ID_58808 ||
4723             pci_dev->id.device_id == BROADCOM_DEV_ID_58802_VF)
4724                 bp->flags |= BNXT_FLAG_STINGRAY;
4725
4726         rc = bnxt_init_board(eth_dev);
4727         if (rc) {
4728                 PMD_DRV_LOG(ERR,
4729                             "Failed to initialize board rc: %x\n", rc);
4730                 return rc;
4731         }
4732
4733         rc = bnxt_alloc_hwrm_resources(bp);
4734         if (rc) {
4735                 PMD_DRV_LOG(ERR,
4736                             "Failed to allocate hwrm resource rc: %x\n", rc);
4737                 goto error_free;
4738         }
4739         rc = bnxt_init_resources(bp, false);
4740         if (rc)
4741                 goto error_free;
4742
4743         rc = bnxt_alloc_stats_mem(bp);
4744         if (rc)
4745                 goto error_free;
4746
4747         PMD_DRV_LOG(INFO,
4748                     DRV_MODULE_NAME "found at mem %" PRIX64 ", node addr %pM\n",
4749                     pci_dev->mem_resource[0].phys_addr,
4750                     pci_dev->mem_resource[0].addr);
4751
4752         return 0;
4753
4754 error_free:
4755         bnxt_dev_uninit(eth_dev);
4756         return rc;
4757 }
4758
4759 static void
4760 bnxt_uninit_locks(struct bnxt *bp)
4761 {
4762         pthread_mutex_destroy(&bp->flow_lock);
4763         pthread_mutex_destroy(&bp->def_cp_lock);
4764 }
4765
4766 static int
4767 bnxt_uninit_resources(struct bnxt *bp, bool reconfig_dev)
4768 {
4769         int rc;
4770
4771         bnxt_free_int(bp);
4772         bnxt_free_mem(bp, reconfig_dev);
4773         bnxt_hwrm_func_buf_unrgtr(bp);
4774         rc = bnxt_hwrm_func_driver_unregister(bp, 0);
4775         bp->flags &= ~BNXT_FLAG_REGISTERED;
4776         bnxt_free_ctx_mem(bp);
4777         if (!reconfig_dev) {
4778                 bnxt_free_hwrm_resources(bp);
4779
4780                 if (bp->recovery_info != NULL) {
4781                         rte_free(bp->recovery_info);
4782                         bp->recovery_info = NULL;
4783                 }
4784         }
4785
4786         rte_free(bp->ptp_cfg);
4787         bp->ptp_cfg = NULL;
4788         return rc;
4789 }
4790
4791 static int
4792 bnxt_dev_uninit(struct rte_eth_dev *eth_dev)
4793 {
4794         struct bnxt *bp = eth_dev->data->dev_private;
4795         int rc;
4796
4797         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
4798                 return -EPERM;
4799
4800         PMD_DRV_LOG(DEBUG, "Calling Device uninit\n");
4801
4802         rc = bnxt_uninit_resources(bp, false);
4803
4804         if (bp->grp_info != NULL) {
4805                 rte_free(bp->grp_info);
4806                 bp->grp_info = NULL;
4807         }
4808
4809         if (bp->tx_mem_zone) {
4810                 rte_memzone_free((const struct rte_memzone *)bp->tx_mem_zone);
4811                 bp->tx_mem_zone = NULL;
4812         }
4813
4814         if (bp->rx_mem_zone) {
4815                 rte_memzone_free((const struct rte_memzone *)bp->rx_mem_zone);
4816                 bp->rx_mem_zone = NULL;
4817         }
4818
4819         if (bp->dev_stopped == 0)
4820                 bnxt_dev_close_op(eth_dev);
4821         if (bp->pf.vf_info)
4822                 rte_free(bp->pf.vf_info);
4823         eth_dev->dev_ops = NULL;
4824         eth_dev->rx_pkt_burst = NULL;
4825         eth_dev->tx_pkt_burst = NULL;
4826
4827         bnxt_uninit_locks(bp);
4828
4829         return rc;
4830 }
4831
4832 static int bnxt_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
4833         struct rte_pci_device *pci_dev)
4834 {
4835         return rte_eth_dev_pci_generic_probe(pci_dev, sizeof(struct bnxt),
4836                 bnxt_dev_init);
4837 }
4838
4839 static int bnxt_pci_remove(struct rte_pci_device *pci_dev)
4840 {
4841         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
4842                 return rte_eth_dev_pci_generic_remove(pci_dev,
4843                                 bnxt_dev_uninit);
4844         else
4845                 return rte_eth_dev_pci_generic_remove(pci_dev, NULL);
4846 }
4847
4848 static struct rte_pci_driver bnxt_rte_pmd = {
4849         .id_table = bnxt_pci_id_map,
4850         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
4851         .probe = bnxt_pci_probe,
4852         .remove = bnxt_pci_remove,
4853 };
4854
4855 static bool
4856 is_device_supported(struct rte_eth_dev *dev, struct rte_pci_driver *drv)
4857 {
4858         if (strcmp(dev->device->driver->name, drv->driver.name))
4859                 return false;
4860
4861         return true;
4862 }
4863
4864 bool is_bnxt_supported(struct rte_eth_dev *dev)
4865 {
4866         return is_device_supported(dev, &bnxt_rte_pmd);
4867 }
4868
4869 RTE_INIT(bnxt_init_log)
4870 {
4871         bnxt_logtype_driver = rte_log_register("pmd.net.bnxt.driver");
4872         if (bnxt_logtype_driver >= 0)
4873                 rte_log_set_level(bnxt_logtype_driver, RTE_LOG_NOTICE);
4874 }
4875
4876 RTE_PMD_REGISTER_PCI(net_bnxt, bnxt_rte_pmd);
4877 RTE_PMD_REGISTER_PCI_TABLE(net_bnxt, bnxt_pci_id_map);
4878 RTE_PMD_REGISTER_KMOD_DEP(net_bnxt, "* igb_uio | uio_pci_generic | vfio-pci");