drivers/net/hinic/hinic_pmd_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Huawei Technologies Co., Ltd
   3  */
   4
   5 #include <rte_pci.h>
   6 #include <rte_bus_pci.h>
   7 #include <rte_ethdev_pci.h>
   8 #include <rte_mbuf.h>
   9 #include <rte_malloc.h>
  10 #include <rte_memcpy.h>
  11 #include <rte_mempool.h>
  12 #include <rte_errno.h>
  13 #include <rte_ether.h>
  14
  15 #include "base/hinic_compat.h"
  16 #include "base/hinic_pmd_hwdev.h"
  17 #include "base/hinic_pmd_hwif.h"
  18 #include "base/hinic_pmd_wq.h"
  19 #include "base/hinic_pmd_cfg.h"
  20 #include "base/hinic_pmd_mgmt.h"
  21 #include "base/hinic_pmd_cmdq.h"
  22 #include "base/hinic_pmd_niccfg.h"
  23 #include "base/hinic_pmd_nicio.h"
  24 #include "base/hinic_pmd_mbox.h"
  25 #include "hinic_pmd_ethdev.h"
  26 #include "hinic_pmd_tx.h"
  27 #include "hinic_pmd_rx.h"
  28
  29 /* Vendor ID used by Huawei devices */
  30 #define HINIC_HUAWEI_VENDOR_ID          0x19E5
  31
  32 /* Hinic devices */
  33 #define HINIC_DEV_ID_PRD                0x1822
  34 #define HINIC_DEV_ID_VF                 0x375E
  35 #define HINIC_DEV_ID_VF_HV              0x379E
  36
  37 /* Mezz card for Blade Server */
  38 #define HINIC_DEV_ID_MEZZ_25GE          0x0210
  39 #define HINIC_DEV_ID_MEZZ_40GE          0x020D
  40 #define HINIC_DEV_ID_MEZZ_100GE         0x0205
  41
  42 /* 2*25G and 2*100G card */
  43 #define HINIC_DEV_ID_1822_DUAL_25GE     0x0206
  44 #define HINIC_DEV_ID_1822_100GE         0x0200
  45
  46 #define HINIC_SERVICE_MODE_NIC          2
  47
  48 #define HINIC_INTR_CB_UNREG_MAX_RETRIES 10
  49
  50 #define DEFAULT_BASE_COS                4
  51 #define NR_MAX_COS                      8
  52
  53 #define HINIC_MIN_RX_BUF_SIZE           1024
  54 #define HINIC_MAX_UC_MAC_ADDRS          128
  55 #define HINIC_MAX_MC_MAC_ADDRS          2048
  56
  57 #define HINIC_DEFAULT_BURST_SIZE        32
  58 #define HINIC_DEFAULT_NB_QUEUES         1
  59 #define HINIC_DEFAULT_RING_SIZE         1024
  60 #define HINIC_MAX_LRO_SIZE              65536
  61
  62 /*
  63  * vlan_id is a 12 bit number.
  64  * The VFTA array is actually a 4096 bit array, 128 of 32bit elements.
  65  * 2^5 = 32. The val of lower 5 bits specifies the bit in the 32bit element.
  66  * The higher 7 bit val specifies VFTA array index.
  67  */
  68 #define HINIC_VFTA_BIT(vlan_id)    (1 << ((vlan_id) & 0x1F))
  69 #define HINIC_VFTA_IDX(vlan_id)    ((vlan_id) >> 5)
  70
  71 #define HINIC_VLAN_FILTER_EN            (1U << 0)
  72
  73 #define HINIC_MTU_TO_PKTLEN(mtu)        \
  74         ((mtu) + ETH_HLEN + ETH_CRC_LEN)
  75
  76 #define HINIC_PKTLEN_TO_MTU(pktlen)     \
  77         ((pktlen) - (ETH_HLEN + ETH_CRC_LEN))
  78
  79 /* lro numer limit for one packet */
  80 #define HINIC_LRO_WQE_NUM_DEFAULT       8
  81
  82 /* Driver-specific log messages type */
  83 int hinic_logtype;
  84
  85 struct hinic_xstats_name_off {
  86         char name[RTE_ETH_XSTATS_NAME_SIZE];
  87         u32  offset;
  88 };
  89
  90 #define HINIC_FUNC_STAT(_stat_item) {   \
  91         .name = #_stat_item, \
  92         .offset = offsetof(struct hinic_vport_stats, _stat_item) \
  93 }
  94
  95 #define HINIC_PORT_STAT(_stat_item) { \
  96         .name = #_stat_item, \
  97         .offset = offsetof(struct hinic_phy_port_stats, _stat_item) \
  98 }
  99
 100 static const struct hinic_xstats_name_off hinic_vport_stats_strings[] = {
 101         HINIC_FUNC_STAT(tx_unicast_pkts_vport),
 102         HINIC_FUNC_STAT(tx_unicast_bytes_vport),
 103         HINIC_FUNC_STAT(tx_multicast_pkts_vport),
 104         HINIC_FUNC_STAT(tx_multicast_bytes_vport),
 105         HINIC_FUNC_STAT(tx_broadcast_pkts_vport),
 106         HINIC_FUNC_STAT(tx_broadcast_bytes_vport),
 107
 108         HINIC_FUNC_STAT(rx_unicast_pkts_vport),
 109         HINIC_FUNC_STAT(rx_unicast_bytes_vport),
 110         HINIC_FUNC_STAT(rx_multicast_pkts_vport),
 111         HINIC_FUNC_STAT(rx_multicast_bytes_vport),
 112         HINIC_FUNC_STAT(rx_broadcast_pkts_vport),
 113         HINIC_FUNC_STAT(rx_broadcast_bytes_vport),
 114
 115         HINIC_FUNC_STAT(tx_discard_vport),
 116         HINIC_FUNC_STAT(rx_discard_vport),
 117         HINIC_FUNC_STAT(tx_err_vport),
 118         HINIC_FUNC_STAT(rx_err_vport),
 119 };
 120
 121 #define HINIC_VPORT_XSTATS_NUM (sizeof(hinic_vport_stats_strings) / \
 122                 sizeof(hinic_vport_stats_strings[0]))
 123
 124 static const struct hinic_xstats_name_off hinic_phyport_stats_strings[] = {
 125         HINIC_PORT_STAT(mac_rx_total_pkt_num),
 126         HINIC_PORT_STAT(mac_rx_total_oct_num),
 127         HINIC_PORT_STAT(mac_rx_bad_pkt_num),
 128         HINIC_PORT_STAT(mac_rx_bad_oct_num),
 129         HINIC_PORT_STAT(mac_rx_good_pkt_num),
 130         HINIC_PORT_STAT(mac_rx_good_oct_num),
 131         HINIC_PORT_STAT(mac_rx_uni_pkt_num),
 132         HINIC_PORT_STAT(mac_rx_multi_pkt_num),
 133         HINIC_PORT_STAT(mac_rx_broad_pkt_num),
 134         HINIC_PORT_STAT(mac_tx_total_pkt_num),
 135         HINIC_PORT_STAT(mac_tx_total_oct_num),
 136         HINIC_PORT_STAT(mac_tx_bad_pkt_num),
 137         HINIC_PORT_STAT(mac_tx_bad_oct_num),
 138         HINIC_PORT_STAT(mac_tx_good_pkt_num),
 139         HINIC_PORT_STAT(mac_tx_good_oct_num),
 140         HINIC_PORT_STAT(mac_tx_uni_pkt_num),
 141         HINIC_PORT_STAT(mac_tx_multi_pkt_num),
 142         HINIC_PORT_STAT(mac_tx_broad_pkt_num),
 143         HINIC_PORT_STAT(mac_rx_fragment_pkt_num),
 144         HINIC_PORT_STAT(mac_rx_undersize_pkt_num),
 145         HINIC_PORT_STAT(mac_rx_undermin_pkt_num),
 146         HINIC_PORT_STAT(mac_rx_64_oct_pkt_num),
 147         HINIC_PORT_STAT(mac_rx_65_127_oct_pkt_num),
 148         HINIC_PORT_STAT(mac_rx_128_255_oct_pkt_num),
 149         HINIC_PORT_STAT(mac_rx_256_511_oct_pkt_num),
 150         HINIC_PORT_STAT(mac_rx_512_1023_oct_pkt_num),
 151         HINIC_PORT_STAT(mac_rx_1024_1518_oct_pkt_num),
 152         HINIC_PORT_STAT(mac_rx_1519_2047_oct_pkt_num),
 153         HINIC_PORT_STAT(mac_rx_2048_4095_oct_pkt_num),
 154         HINIC_PORT_STAT(mac_rx_4096_8191_oct_pkt_num),
 155         HINIC_PORT_STAT(mac_rx_8192_9216_oct_pkt_num),
 156         HINIC_PORT_STAT(mac_rx_9217_12287_oct_pkt_num),
 157         HINIC_PORT_STAT(mac_rx_12288_16383_oct_pkt_num),
 158         HINIC_PORT_STAT(mac_rx_1519_max_bad_pkt_num),
 159         HINIC_PORT_STAT(mac_rx_1519_max_good_pkt_num),
 160         HINIC_PORT_STAT(mac_rx_oversize_pkt_num),
 161         HINIC_PORT_STAT(mac_rx_jabber_pkt_num),
 162         HINIC_PORT_STAT(mac_rx_mac_pause_num),
 163         HINIC_PORT_STAT(mac_rx_pfc_pkt_num),
 164         HINIC_PORT_STAT(mac_rx_pfc_pri0_pkt_num),
 165         HINIC_PORT_STAT(mac_rx_pfc_pri1_pkt_num),
 166         HINIC_PORT_STAT(mac_rx_pfc_pri2_pkt_num),
 167         HINIC_PORT_STAT(mac_rx_pfc_pri3_pkt_num),
 168         HINIC_PORT_STAT(mac_rx_pfc_pri4_pkt_num),
 169         HINIC_PORT_STAT(mac_rx_pfc_pri5_pkt_num),
 170         HINIC_PORT_STAT(mac_rx_pfc_pri6_pkt_num),
 171         HINIC_PORT_STAT(mac_rx_pfc_pri7_pkt_num),
 172         HINIC_PORT_STAT(mac_rx_mac_control_pkt_num),
 173         HINIC_PORT_STAT(mac_rx_sym_err_pkt_num),
 174         HINIC_PORT_STAT(mac_rx_fcs_err_pkt_num),
 175         HINIC_PORT_STAT(mac_rx_send_app_good_pkt_num),
 176         HINIC_PORT_STAT(mac_rx_send_app_bad_pkt_num),
 177         HINIC_PORT_STAT(mac_tx_fragment_pkt_num),
 178         HINIC_PORT_STAT(mac_tx_undersize_pkt_num),
 179         HINIC_PORT_STAT(mac_tx_undermin_pkt_num),
 180         HINIC_PORT_STAT(mac_tx_64_oct_pkt_num),
 181         HINIC_PORT_STAT(mac_tx_65_127_oct_pkt_num),
 182         HINIC_PORT_STAT(mac_tx_128_255_oct_pkt_num),
 183         HINIC_PORT_STAT(mac_tx_256_511_oct_pkt_num),
 184         HINIC_PORT_STAT(mac_tx_512_1023_oct_pkt_num),
 185         HINIC_PORT_STAT(mac_tx_1024_1518_oct_pkt_num),
 186         HINIC_PORT_STAT(mac_tx_1519_2047_oct_pkt_num),
 187         HINIC_PORT_STAT(mac_tx_2048_4095_oct_pkt_num),
 188         HINIC_PORT_STAT(mac_tx_4096_8191_oct_pkt_num),
 189         HINIC_PORT_STAT(mac_tx_8192_9216_oct_pkt_num),
 190         HINIC_PORT_STAT(mac_tx_9217_12287_oct_pkt_num),
 191         HINIC_PORT_STAT(mac_tx_12288_16383_oct_pkt_num),
 192         HINIC_PORT_STAT(mac_tx_1519_max_bad_pkt_num),
 193         HINIC_PORT_STAT(mac_tx_1519_max_good_pkt_num),
 194         HINIC_PORT_STAT(mac_tx_oversize_pkt_num),
 195         HINIC_PORT_STAT(mac_trans_jabber_pkt_num),
 196         HINIC_PORT_STAT(mac_tx_mac_pause_num),
 197         HINIC_PORT_STAT(mac_tx_pfc_pkt_num),
 198         HINIC_PORT_STAT(mac_tx_pfc_pri0_pkt_num),
 199         HINIC_PORT_STAT(mac_tx_pfc_pri1_pkt_num),
 200         HINIC_PORT_STAT(mac_tx_pfc_pri2_pkt_num),
 201         HINIC_PORT_STAT(mac_tx_pfc_pri3_pkt_num),
 202         HINIC_PORT_STAT(mac_tx_pfc_pri4_pkt_num),
 203         HINIC_PORT_STAT(mac_tx_pfc_pri5_pkt_num),
 204         HINIC_PORT_STAT(mac_tx_pfc_pri6_pkt_num),
 205         HINIC_PORT_STAT(mac_tx_pfc_pri7_pkt_num),
 206         HINIC_PORT_STAT(mac_tx_mac_control_pkt_num),
 207         HINIC_PORT_STAT(mac_tx_err_all_pkt_num),
 208         HINIC_PORT_STAT(mac_tx_from_app_good_pkt_num),
 209         HINIC_PORT_STAT(mac_tx_from_app_bad_pkt_num),
 210 };
 211
 212 #define HINIC_PHYPORT_XSTATS_NUM (sizeof(hinic_phyport_stats_strings) / \
 213                 sizeof(hinic_phyport_stats_strings[0]))
 214
 215 static const struct hinic_xstats_name_off hinic_rxq_stats_strings[] = {
 216         {"rx_nombuf", offsetof(struct hinic_rxq_stats, rx_nombuf)},
 217         {"burst_pkt", offsetof(struct hinic_rxq_stats, burst_pkts)},
 218 };
 219
 220 #define HINIC_RXQ_XSTATS_NUM (sizeof(hinic_rxq_stats_strings) / \
 221                 sizeof(hinic_rxq_stats_strings[0]))
 222
 223 static const struct hinic_xstats_name_off hinic_txq_stats_strings[] = {
 224         {"tx_busy", offsetof(struct hinic_txq_stats, tx_busy)},
 225         {"offload_errors", offsetof(struct hinic_txq_stats, off_errs)},
 226         {"copy_pkts", offsetof(struct hinic_txq_stats, cpy_pkts)},
 227         {"rl_drop", offsetof(struct hinic_txq_stats, rl_drop)},
 228         {"burst_pkts", offsetof(struct hinic_txq_stats, burst_pkts)},
 229 };
 230
 231 #define HINIC_TXQ_XSTATS_NUM (sizeof(hinic_txq_stats_strings) / \
 232                 sizeof(hinic_txq_stats_strings[0]))
 233
 234 static int hinic_xstats_calc_num(struct hinic_nic_dev *nic_dev)
 235 {
 236         if (HINIC_IS_VF(nic_dev->hwdev)) {
 237                 return (HINIC_VPORT_XSTATS_NUM +
 238                         HINIC_RXQ_XSTATS_NUM * nic_dev->num_rq +
 239                         HINIC_TXQ_XSTATS_NUM * nic_dev->num_sq);
 240         } else {
 241                 return (HINIC_VPORT_XSTATS_NUM +
 242                         HINIC_PHYPORT_XSTATS_NUM +
 243                         HINIC_RXQ_XSTATS_NUM * nic_dev->num_rq +
 244                         HINIC_TXQ_XSTATS_NUM * nic_dev->num_sq);
 245         }
 246 }
 247
 248 static const struct rte_eth_desc_lim hinic_rx_desc_lim = {
 249         .nb_max = HINIC_MAX_QUEUE_DEPTH,
 250         .nb_min = HINIC_MIN_QUEUE_DEPTH,
 251         .nb_align = HINIC_RXD_ALIGN,
 252 };
 253
 254 static const struct rte_eth_desc_lim hinic_tx_desc_lim = {
 255         .nb_max = HINIC_MAX_QUEUE_DEPTH,
 256         .nb_min = HINIC_MIN_QUEUE_DEPTH,
 257         .nb_align = HINIC_TXD_ALIGN,
 258 };
 259
 260 static int hinic_vlan_offload_set(struct rte_eth_dev *dev, int mask);
 261
 262 /**
 263  * Interrupt handler triggered by NIC  for handling
 264  * specific event.
 265  *
 266  * @param: The address of parameter (struct rte_eth_dev *) regsitered before.
 267  */
 268 static void hinic_dev_interrupt_handler(void *param)
 269 {
 270         struct rte_eth_dev *dev = param;
 271         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 272
 273         if (!hinic_test_bit(HINIC_DEV_INTR_EN, &nic_dev->dev_status)) {
 274                 PMD_DRV_LOG(WARNING, "Device's interrupt is disabled, ignore interrupt event, dev_name: %s, port_id: %d",
 275                             nic_dev->proc_dev_name, dev->data->port_id);
 276                 return;
 277         }
 278
 279         /* aeq0 msg handler */
 280         hinic_dev_handle_aeq_event(nic_dev->hwdev, param);
 281 }
 282
 283 /**
 284  * Ethernet device configuration.
 285  *
 286  * Prepare the driver for a given number of TX and RX queues, mtu size
 287  * and configure RSS.
 288  *
 289  * @param dev
 290  *   Pointer to Ethernet device structure.
 291  *
 292  * @return
 293  *   0 on success, negative error value otherwise.
 294  */
 295 static int hinic_dev_configure(struct rte_eth_dev *dev)
 296 {
 297         struct hinic_nic_dev *nic_dev;
 298         struct hinic_nic_io *nic_io;
 299         int err;
 300
 301         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 302         nic_io = nic_dev->hwdev->nic_io;
 303
 304         nic_dev->num_sq =  dev->data->nb_tx_queues;
 305         nic_dev->num_rq = dev->data->nb_rx_queues;
 306
 307         nic_io->num_sqs =  dev->data->nb_tx_queues;
 308         nic_io->num_rqs = dev->data->nb_rx_queues;
 309
 310         /* queue pair is max_num(sq, rq) */
 311         nic_dev->num_qps = (nic_dev->num_sq > nic_dev->num_rq) ?
 312                         nic_dev->num_sq : nic_dev->num_rq;
 313         nic_io->num_qps = nic_dev->num_qps;
 314
 315         if (nic_dev->num_qps > nic_io->max_qps) {
 316                 PMD_DRV_LOG(ERR,
 317                         "Queue number out of range, get queue_num:%d, max_queue_num:%d",
 318                         nic_dev->num_qps, nic_io->max_qps);
 319                 return -EINVAL;
 320         }
 321
 322         if (dev->data->dev_conf.rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG)
 323                 dev->data->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_RSS_HASH;
 324
 325         /* mtu size is 256~9600 */
 326         if (dev->data->dev_conf.rxmode.max_rx_pkt_len < HINIC_MIN_FRAME_SIZE ||
 327             dev->data->dev_conf.rxmode.max_rx_pkt_len >
 328             HINIC_MAX_JUMBO_FRAME_SIZE) {
 329                 PMD_DRV_LOG(ERR,
 330                         "Max rx pkt len out of range, get max_rx_pkt_len:%d, "
 331                         "expect between %d and %d",
 332                         dev->data->dev_conf.rxmode.max_rx_pkt_len,
 333                         HINIC_MIN_FRAME_SIZE, HINIC_MAX_JUMBO_FRAME_SIZE);
 334                 return -EINVAL;
 335         }
 336
 337         nic_dev->mtu_size =
 338                 HINIC_PKTLEN_TO_MTU(dev->data->dev_conf.rxmode.max_rx_pkt_len);
 339
 340         /* rss template */
 341         err = hinic_config_mq_mode(dev, TRUE);
 342         if (err) {
 343                 PMD_DRV_LOG(ERR, "Config multi-queue failed");
 344                 return err;
 345         }
 346
 347         /* init vlan offoad */
 348         err = hinic_vlan_offload_set(dev,
 349                                 ETH_VLAN_STRIP_MASK | ETH_VLAN_FILTER_MASK);
 350         if (err) {
 351                 PMD_DRV_LOG(ERR, "Initialize vlan filter and strip failed\n");
 352                 (void)hinic_config_mq_mode(dev, FALSE);
 353                 return err;
 354         }
 355
 356         /*clear fdir filter flag in function table*/
 357         hinic_free_fdir_filter(nic_dev);
 358
 359         return HINIC_OK;
 360 }
 361
 362 /**
 363  * DPDK callback to create the receive queue.
 364  *
 365  * @param dev
 366  *   Pointer to Ethernet device structure.
 367  * @param queue_idx
 368  *   RX queue index.
 369  * @param nb_desc
 370  *   Number of descriptors for receive queue.
 371  * @param socket_id
 372  *   NUMA socket on which memory must be allocated.
 373  * @param rx_conf
 374  *   Thresholds parameters (unused_).
 375  * @param mp
 376  *   Memory pool for buffer allocations.
 377  *
 378  * @return
 379  *   0 on success, negative error value otherwise.
 380  */
 381 static int hinic_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 382                          uint16_t nb_desc, unsigned int socket_id,
 383                          __rte_unused const struct rte_eth_rxconf *rx_conf,
 384                          struct rte_mempool *mp)
 385 {
 386         int rc;
 387         struct hinic_nic_dev *nic_dev;
 388         struct hinic_hwdev *hwdev;
 389         struct hinic_rxq *rxq;
 390         u16 rq_depth, rx_free_thresh;
 391         u32 buf_size;
 392
 393         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 394         hwdev = nic_dev->hwdev;
 395
 396         /* queue depth must be power of 2, otherwise will be aligned up */
 397         rq_depth = (nb_desc & (nb_desc - 1)) ?
 398                 ((u16)(1U << (ilog2(nb_desc) + 1))) : nb_desc;
 399
 400         /*
 401          * Validate number of receive descriptors.
 402          * It must not exceed hardware maximum and minimum.
 403          */
 404         if (rq_depth > HINIC_MAX_QUEUE_DEPTH ||
 405                 rq_depth < HINIC_MIN_QUEUE_DEPTH) {
 406                 PMD_DRV_LOG(ERR, "RX queue depth is out of range from %d to %d, (nb_desc=%d, q_depth=%d, port=%d queue=%d)",
 407                             HINIC_MIN_QUEUE_DEPTH, HINIC_MAX_QUEUE_DEPTH,
 408                             (int)nb_desc, (int)rq_depth,
 409                             (int)dev->data->port_id, (int)queue_idx);
 410                 return -EINVAL;
 411         }
 412
 413         /*
 414          * The RX descriptor ring will be cleaned after rxq->rx_free_thresh
 415          * descriptors are used or if the number of descriptors required
 416          * to transmit a packet is greater than the number of free RX
 417          * descriptors.
 418          * The following constraints must be satisfied:
 419          *  rx_free_thresh must be greater than 0.
 420          *  rx_free_thresh must be less than the size of the ring minus 1.
 421          * When set to zero use default values.
 422          */
 423         rx_free_thresh = (u16)((rx_conf->rx_free_thresh) ?
 424                         rx_conf->rx_free_thresh : HINIC_DEFAULT_RX_FREE_THRESH);
 425         if (rx_free_thresh >= (rq_depth - 1)) {
 426                 PMD_DRV_LOG(ERR, "rx_free_thresh must be less than the number of RX descriptors minus 1. (rx_free_thresh=%u port=%d queue=%d)",
 427                             (unsigned int)rx_free_thresh,
 428                             (int)dev->data->port_id,
 429                             (int)queue_idx);
 430                 return -EINVAL;
 431         }
 432
 433         rxq = rte_zmalloc_socket("hinic_rx_queue", sizeof(struct hinic_rxq),
 434                                  RTE_CACHE_LINE_SIZE, socket_id);
 435         if (!rxq) {
 436                 PMD_DRV_LOG(ERR, "Allocate rxq[%d] failed, dev_name: %s",
 437                             queue_idx, dev->data->name);
 438                 return -ENOMEM;
 439         }
 440         nic_dev->rxqs[queue_idx] = rxq;
 441
 442         /* alloc rx sq hw wqepage*/
 443         rc = hinic_create_rq(hwdev, queue_idx, rq_depth, socket_id);
 444         if (rc) {
 445                 PMD_DRV_LOG(ERR, "Create rxq[%d] failed, dev_name: %s, rq_depth: %d",
 446                             queue_idx, dev->data->name, rq_depth);
 447                 goto ceate_rq_fail;
 448         }
 449
 450         /* mbuf pool must be assigned before setup rx resources */
 451         rxq->mb_pool = mp;
 452
 453         rc =
 454         hinic_convert_rx_buf_size(rte_pktmbuf_data_room_size(rxq->mb_pool) -
 455                                   RTE_PKTMBUF_HEADROOM, &buf_size);
 456         if (rc) {
 457                 PMD_DRV_LOG(ERR, "Adjust buf size failed, dev_name: %s",
 458                             dev->data->name);
 459                 goto adjust_bufsize_fail;
 460         }
 461
 462         /* rx queue info, rearm control */
 463         rxq->wq = &hwdev->nic_io->rq_wq[queue_idx];
 464         rxq->pi_virt_addr = hwdev->nic_io->qps[queue_idx].rq.pi_virt_addr;
 465         rxq->nic_dev = nic_dev;
 466         rxq->q_id = queue_idx;
 467         rxq->q_depth = rq_depth;
 468         rxq->buf_len = (u16)buf_size;
 469         rxq->rx_free_thresh = rx_free_thresh;
 470         rxq->socket_id = socket_id;
 471
 472         /* the last point cant do mbuf rearm in bulk */
 473         rxq->rxinfo_align_end = rxq->q_depth - rxq->rx_free_thresh;
 474
 475         /* device port identifier */
 476         rxq->port_id = dev->data->port_id;
 477
 478         /* alloc rx_cqe and prepare rq_wqe */
 479         rc = hinic_setup_rx_resources(rxq);
 480         if (rc) {
 481                 PMD_DRV_LOG(ERR, "Setup rxq[%d] rx_resources failed, dev_name:%s",
 482                             queue_idx, dev->data->name);
 483                 goto setup_rx_res_err;
 484         }
 485
 486         /* record nic_dev rxq in rte_eth rx_queues */
 487         dev->data->rx_queues[queue_idx] = rxq;
 488
 489         return 0;
 490
 491 setup_rx_res_err:
 492 adjust_bufsize_fail:
 493         hinic_destroy_rq(hwdev, queue_idx);
 494
 495 ceate_rq_fail:
 496         rte_free(rxq);
 497
 498         return rc;
 499 }
 500
 501 static void hinic_reset_rx_queue(struct rte_eth_dev *dev)
 502 {
 503         struct hinic_rxq *rxq;
 504         struct hinic_nic_dev *nic_dev;
 505         int q_id = 0;
 506
 507         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 508
 509         for (q_id = 0; q_id < nic_dev->num_rq; q_id++) {
 510                 rxq = dev->data->rx_queues[q_id];
 511
 512                 rxq->wq->cons_idx = 0;
 513                 rxq->wq->prod_idx = 0;
 514                 rxq->wq->delta = rxq->q_depth;
 515                 rxq->wq->mask = rxq->q_depth - 1;
 516
 517                 /* alloc mbuf to rq */
 518                 hinic_rx_alloc_pkts(rxq);
 519         }
 520 }
 521
 522 /**
 523  * DPDK callback to configure the transmit queue.
 524  *
 525  * @param dev
 526  *   Pointer to Ethernet device structure.
 527  * @param queue_idx
 528  *   Transmit queue index.
 529  * @param nb_desc
 530  *   Number of descriptors for transmit queue.
 531  * @param socket_id
 532  *   NUMA socket on which memory must be allocated.
 533  * @param tx_conf
 534  *   Tx queue configuration parameters.
 535  *
 536  * @return
 537  *   0 on success, negative error value otherwise.
 538  */
 539 static int hinic_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 540                          uint16_t nb_desc, unsigned int socket_id,
 541                          __rte_unused const struct rte_eth_txconf *tx_conf)
 542 {
 543         int rc;
 544         struct hinic_nic_dev *nic_dev;
 545         struct hinic_hwdev *hwdev;
 546         struct hinic_txq *txq;
 547         u16 sq_depth, tx_free_thresh;
 548
 549         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 550         hwdev = nic_dev->hwdev;
 551
 552         /* queue depth must be power of 2, otherwise will be aligned up */
 553         sq_depth = (nb_desc & (nb_desc - 1)) ?
 554                         ((u16)(1U << (ilog2(nb_desc) + 1))) : nb_desc;
 555
 556         /*
 557          * Validate number of transmit descriptors.
 558          * It must not exceed hardware maximum and minimum.
 559          */
 560         if (sq_depth > HINIC_MAX_QUEUE_DEPTH ||
 561                 sq_depth < HINIC_MIN_QUEUE_DEPTH) {
 562                 PMD_DRV_LOG(ERR, "TX queue depth is out of range from %d to %d, (nb_desc=%d, q_depth=%d, port=%d queue=%d)",
 563                           HINIC_MIN_QUEUE_DEPTH, HINIC_MAX_QUEUE_DEPTH,
 564                           (int)nb_desc, (int)sq_depth,
 565                           (int)dev->data->port_id, (int)queue_idx);
 566                 return -EINVAL;
 567         }
 568
 569         /*
 570          * The TX descriptor ring will be cleaned after txq->tx_free_thresh
 571          * descriptors are used or if the number of descriptors required
 572          * to transmit a packet is greater than the number of free TX
 573          * descriptors.
 574          * The following constraints must be satisfied:
 575          *  tx_free_thresh must be greater than 0.
 576          *  tx_free_thresh must be less than the size of the ring minus 1.
 577          * When set to zero use default values.
 578          */
 579         tx_free_thresh = (u16)((tx_conf->tx_free_thresh) ?
 580                         tx_conf->tx_free_thresh : HINIC_DEFAULT_TX_FREE_THRESH);
 581         if (tx_free_thresh >= (sq_depth - 1)) {
 582                 PMD_DRV_LOG(ERR, "tx_free_thresh must be less than the number of TX descriptors minus 1. (tx_free_thresh=%u port=%d queue=%d)",
 583                         (unsigned int)tx_free_thresh, (int)dev->data->port_id,
 584                         (int)queue_idx);
 585                 return -EINVAL;
 586         }
 587
 588         txq = rte_zmalloc_socket("hinic_tx_queue", sizeof(struct hinic_txq),
 589                                  RTE_CACHE_LINE_SIZE, socket_id);
 590         if (!txq) {
 591                 PMD_DRV_LOG(ERR, "Allocate txq[%d] failed, dev_name: %s",
 592                             queue_idx, dev->data->name);
 593                 return -ENOMEM;
 594         }
 595         nic_dev->txqs[queue_idx] = txq;
 596
 597         /* alloc tx sq hw wqepage */
 598         rc = hinic_create_sq(hwdev, queue_idx, sq_depth, socket_id);
 599         if (rc) {
 600                 PMD_DRV_LOG(ERR, "Create txq[%d] failed, dev_name: %s, sq_depth: %d",
 601                             queue_idx, dev->data->name, sq_depth);
 602                 goto create_sq_fail;
 603         }
 604
 605         txq->q_id = queue_idx;
 606         txq->q_depth = sq_depth;
 607         txq->port_id = dev->data->port_id;
 608         txq->tx_free_thresh = tx_free_thresh;
 609         txq->nic_dev = nic_dev;
 610         txq->wq = &hwdev->nic_io->sq_wq[queue_idx];
 611         txq->sq = &hwdev->nic_io->qps[queue_idx].sq;
 612         txq->cons_idx_addr = hwdev->nic_io->qps[queue_idx].sq.cons_idx_addr;
 613         txq->sq_head_addr = HINIC_GET_WQ_HEAD(txq);
 614         txq->sq_bot_sge_addr = HINIC_GET_WQ_TAIL(txq) -
 615                                         sizeof(struct hinic_sq_bufdesc);
 616         txq->cos = nic_dev->default_cos;
 617         txq->socket_id = socket_id;
 618
 619         /* alloc software txinfo */
 620         rc = hinic_setup_tx_resources(txq);
 621         if (rc) {
 622                 PMD_DRV_LOG(ERR, "Setup txq[%d] tx_resources failed, dev_name: %s",
 623                             queue_idx, dev->data->name);
 624                 goto setup_tx_res_fail;
 625         }
 626
 627         /* record nic_dev txq in rte_eth tx_queues */
 628         dev->data->tx_queues[queue_idx] = txq;
 629
 630         return HINIC_OK;
 631
 632 setup_tx_res_fail:
 633         hinic_destroy_sq(hwdev, queue_idx);
 634
 635 create_sq_fail:
 636         rte_free(txq);
 637
 638         return rc;
 639 }
 640
 641 static void hinic_reset_tx_queue(struct rte_eth_dev *dev)
 642 {
 643         struct hinic_nic_dev *nic_dev;
 644         struct hinic_txq *txq;
 645         struct hinic_nic_io *nic_io;
 646         struct hinic_hwdev *hwdev;
 647         volatile u32 *ci_addr;
 648         int q_id = 0;
 649
 650         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 651         hwdev = nic_dev->hwdev;
 652         nic_io = hwdev->nic_io;
 653
 654         for (q_id = 0; q_id < nic_dev->num_sq; q_id++) {
 655                 txq = dev->data->tx_queues[q_id];
 656
 657                 txq->wq->cons_idx = 0;
 658                 txq->wq->prod_idx = 0;
 659                 txq->wq->delta = txq->q_depth;
 660                 txq->wq->mask  = txq->q_depth - 1;
 661
 662                 /* clear hardware ci */
 663                 ci_addr = (volatile u32 *)HINIC_CI_VADDR(nic_io->ci_vaddr_base,
 664                                                         q_id);
 665                 *ci_addr = 0;
 666         }
 667 }
 668
 669 /**
 670  * Get link speed from NIC.
 671  *
 672  * @param dev
 673  *   Pointer to Ethernet device structure.
 674  * @param speed_capa
 675  *   Pointer to link speed structure.
 676  */
 677 static void hinic_get_speed_capa(struct rte_eth_dev *dev, uint32_t *speed_capa)
 678 {
 679         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 680         u32 supported_link, advertised_link;
 681         int err;
 682
 683 #define HINIC_LINK_MODE_SUPPORT_1G      (1U << HINIC_GE_BASE_KX)
 684
 685 #define HINIC_LINK_MODE_SUPPORT_10G     (1U << HINIC_10GE_BASE_KR)
 686
 687 #define HINIC_LINK_MODE_SUPPORT_25G     ((1U << HINIC_25GE_BASE_KR_S) | \
 688                                         (1U << HINIC_25GE_BASE_CR_S) | \
 689                                         (1U << HINIC_25GE_BASE_KR) | \
 690                                         (1U << HINIC_25GE_BASE_CR))
 691
 692 #define HINIC_LINK_MODE_SUPPORT_40G     ((1U << HINIC_40GE_BASE_KR4) | \
 693                                         (1U << HINIC_40GE_BASE_CR4))
 694
 695 #define HINIC_LINK_MODE_SUPPORT_100G    ((1U << HINIC_100GE_BASE_KR4) | \
 696                                         (1U << HINIC_100GE_BASE_CR4))
 697
 698         err = hinic_get_link_mode(nic_dev->hwdev,
 699                                   &supported_link, &advertised_link);
 700         if (err || supported_link == HINIC_SUPPORTED_UNKNOWN ||
 701             advertised_link == HINIC_SUPPORTED_UNKNOWN) {
 702                 PMD_DRV_LOG(WARNING, "Get speed capability info failed, device: %s, port_id: %u",
 703                           nic_dev->proc_dev_name, dev->data->port_id);
 704         } else {
 705                 *speed_capa = 0;
 706                 if (!!(supported_link & HINIC_LINK_MODE_SUPPORT_1G))
 707                         *speed_capa |= ETH_LINK_SPEED_1G;
 708                 if (!!(supported_link & HINIC_LINK_MODE_SUPPORT_10G))
 709                         *speed_capa |= ETH_LINK_SPEED_10G;
 710                 if (!!(supported_link & HINIC_LINK_MODE_SUPPORT_25G))
 711                         *speed_capa |= ETH_LINK_SPEED_25G;
 712                 if (!!(supported_link & HINIC_LINK_MODE_SUPPORT_40G))
 713                         *speed_capa |= ETH_LINK_SPEED_40G;
 714                 if (!!(supported_link & HINIC_LINK_MODE_SUPPORT_100G))
 715                         *speed_capa |= ETH_LINK_SPEED_100G;
 716         }
 717 }
 718
 719 /**
 720  * DPDK callback to get information about the device.
 721  *
 722  * @param dev
 723  *   Pointer to Ethernet device structure.
 724  * @param info
 725  *   Pointer to Info structure output buffer.
 726  */
 727 static int
 728 hinic_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
 729 {
 730         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 731
 732         info->max_rx_queues  = nic_dev->nic_cap.max_rqs;
 733         info->max_tx_queues  = nic_dev->nic_cap.max_sqs;
 734         info->min_rx_bufsize = HINIC_MIN_RX_BUF_SIZE;
 735         info->max_rx_pktlen  = HINIC_MAX_JUMBO_FRAME_SIZE;
 736         info->max_mac_addrs  = HINIC_MAX_UC_MAC_ADDRS;
 737         info->min_mtu = HINIC_MIN_MTU_SIZE;
 738         info->max_mtu = HINIC_MAX_MTU_SIZE;
 739
 740         hinic_get_speed_capa(dev, &info->speed_capa);
 741         info->rx_queue_offload_capa = 0;
 742         info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
 743                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 744                                 DEV_RX_OFFLOAD_UDP_CKSUM |
 745                                 DEV_RX_OFFLOAD_TCP_CKSUM |
 746                                 DEV_RX_OFFLOAD_VLAN_FILTER |
 747                                 DEV_RX_OFFLOAD_SCATTER |
 748                                 DEV_RX_OFFLOAD_JUMBO_FRAME |
 749                                 DEV_RX_OFFLOAD_TCP_LRO |
 750                                 DEV_RX_OFFLOAD_RSS_HASH;
 751
 752         info->tx_queue_offload_capa = 0;
 753         info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
 754                                 DEV_TX_OFFLOAD_IPV4_CKSUM |
 755                                 DEV_TX_OFFLOAD_UDP_CKSUM |
 756                                 DEV_TX_OFFLOAD_TCP_CKSUM |
 757                                 DEV_TX_OFFLOAD_SCTP_CKSUM |
 758                                 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
 759                                 DEV_TX_OFFLOAD_TCP_TSO |
 760                                 DEV_TX_OFFLOAD_MULTI_SEGS;
 761
 762         info->hash_key_size = HINIC_RSS_KEY_SIZE;
 763         info->reta_size = HINIC_RSS_INDIR_SIZE;
 764         info->flow_type_rss_offloads = HINIC_RSS_OFFLOAD_ALL;
 765         info->rx_desc_lim = hinic_rx_desc_lim;
 766         info->tx_desc_lim = hinic_tx_desc_lim;
 767
 768         /* Driver-preferred Rx/Tx parameters */
 769         info->default_rxportconf.burst_size = HINIC_DEFAULT_BURST_SIZE;
 770         info->default_txportconf.burst_size = HINIC_DEFAULT_BURST_SIZE;
 771         info->default_rxportconf.nb_queues = HINIC_DEFAULT_NB_QUEUES;
 772         info->default_txportconf.nb_queues = HINIC_DEFAULT_NB_QUEUES;
 773         info->default_rxportconf.ring_size = HINIC_DEFAULT_RING_SIZE;
 774         info->default_txportconf.ring_size = HINIC_DEFAULT_RING_SIZE;
 775
 776         return 0;
 777 }
 778
 779 static int hinic_fw_version_get(struct rte_eth_dev *dev, char *fw_version,
 780                                 size_t fw_size)
 781 {
 782         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 783         char fw_ver[HINIC_MGMT_VERSION_MAX_LEN] = {0};
 784         int err;
 785
 786         err = hinic_get_mgmt_version(nic_dev->hwdev, fw_ver);
 787         if (err) {
 788                 PMD_DRV_LOG(ERR, "Failed to get fw version\n");
 789                 return -EINVAL;
 790         }
 791
 792         if (fw_size < strlen(fw_ver) + 1)
 793                 return (strlen(fw_ver) + 1);
 794
 795         snprintf(fw_version, fw_size, "%s", fw_ver);
 796
 797         return 0;
 798 }
 799
 800 static int hinic_config_rx_mode(struct hinic_nic_dev *nic_dev, u32 rx_mode_ctrl)
 801 {
 802         int err;
 803
 804         err = hinic_set_rx_mode(nic_dev->hwdev, rx_mode_ctrl);
 805         if (err) {
 806                 PMD_DRV_LOG(ERR, "Failed to set rx mode");
 807                 return -EINVAL;
 808         }
 809         nic_dev->rx_mode_status = rx_mode_ctrl;
 810
 811         return 0;
 812 }
 813
 814
 815 static int hinic_rxtx_configure(struct rte_eth_dev *dev)
 816 {
 817         int err;
 818         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 819         bool lro_en;
 820
 821         /* rx configure, if rss enable, need to init default configuration */
 822         err = hinic_rx_configure(dev);
 823         if (err) {
 824                 PMD_DRV_LOG(ERR, "Configure rss failed");
 825                 return err;
 826         }
 827
 828         /* rx mode init */
 829         err = hinic_config_rx_mode(nic_dev, HINIC_DEFAULT_RX_MODE);
 830         if (err) {
 831                 PMD_DRV_LOG(ERR, "Configure rx_mode:0x%x failed",
 832                         HINIC_DEFAULT_RX_MODE);
 833                 goto set_rx_mode_fail;
 834         }
 835
 836         /* config lro */
 837         lro_en = dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_TCP_LRO ?
 838                         true : false;
 839
 840         err = hinic_set_rx_lro(nic_dev->hwdev, lro_en, lro_en,
 841                                 HINIC_LRO_WQE_NUM_DEFAULT);
 842         if (err) {
 843                 PMD_DRV_LOG(ERR, "%s lro failed, err: %d",
 844                         lro_en ? "Enable" : "Disable", err);
 845                 goto set_rx_mode_fail;
 846         }
 847
 848         return HINIC_OK;
 849
 850 set_rx_mode_fail:
 851         hinic_rx_remove_configure(dev);
 852
 853         return err;
 854 }
 855
 856 static void hinic_remove_rxtx_configure(struct rte_eth_dev *dev)
 857 {
 858         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 859
 860         (void)hinic_config_rx_mode(nic_dev, 0);
 861         hinic_rx_remove_configure(dev);
 862 }
 863
 864 static int hinic_priv_get_dev_link_status(struct hinic_nic_dev *nic_dev,
 865                                           struct rte_eth_link *link)
 866 {
 867         int rc;
 868         u8 port_link_status = 0;
 869         struct nic_port_info port_link_info;
 870         struct hinic_hwdev *nic_hwdev = nic_dev->hwdev;
 871         uint32_t port_speed[LINK_SPEED_MAX] = {ETH_SPEED_NUM_10M,
 872                                         ETH_SPEED_NUM_100M, ETH_SPEED_NUM_1G,
 873                                         ETH_SPEED_NUM_10G, ETH_SPEED_NUM_25G,
 874                                         ETH_SPEED_NUM_40G, ETH_SPEED_NUM_100G};
 875
 876         rc = hinic_get_link_status(nic_hwdev, &port_link_status);
 877         if (rc)
 878                 return rc;
 879
 880         if (!port_link_status) {
 881                 link->link_status = ETH_LINK_DOWN;
 882                 link->link_speed = 0;
 883                 link->link_duplex = ETH_LINK_HALF_DUPLEX;
 884                 link->link_autoneg = ETH_LINK_FIXED;
 885                 return HINIC_OK;
 886         }
 887
 888         memset(&port_link_info, 0, sizeof(port_link_info));
 889         rc = hinic_get_port_info(nic_hwdev, &port_link_info);
 890         if (rc)
 891                 return rc;
 892
 893         link->link_speed = port_speed[port_link_info.speed % LINK_SPEED_MAX];
 894         link->link_duplex = port_link_info.duplex;
 895         link->link_autoneg = port_link_info.autoneg_state;
 896         link->link_status = port_link_status;
 897
 898         return HINIC_OK;
 899 }
 900
 901 /**
 902  * DPDK callback to retrieve physical link information.
 903  *
 904  * @param dev
 905  *   Pointer to Ethernet device structure.
 906  * @param wait_to_complete
 907  *   Wait for request completion.
 908  *
 909  * @return
 910  *   0 link status changed, -1 link status not changed
 911  */
 912 static int hinic_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 913 {
 914 #define CHECK_INTERVAL 10  /* 10ms */
 915 #define MAX_REPEAT_TIME 100  /* 1s (100 * 10ms) in total */
 916         int rc = HINIC_OK;
 917         struct rte_eth_link link;
 918         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 919         unsigned int rep_cnt = MAX_REPEAT_TIME;
 920
 921         memset(&link, 0, sizeof(link));
 922         do {
 923                 /* Get link status information from hardware */
 924                 rc = hinic_priv_get_dev_link_status(nic_dev, &link);
 925                 if (rc != HINIC_OK) {
 926                         link.link_speed = ETH_SPEED_NUM_NONE;
 927                         link.link_duplex = ETH_LINK_FULL_DUPLEX;
 928                         PMD_DRV_LOG(ERR, "Get link status failed");
 929                         goto out;
 930                 }
 931
 932                 if (!wait_to_complete || link.link_status)
 933                         break;
 934
 935                 rte_delay_ms(CHECK_INTERVAL);
 936         } while (rep_cnt--);
 937
 938 out:
 939         rc = rte_eth_linkstatus_set(dev, &link);
 940         return rc;
 941 }
 942
 943 /**
 944  * DPDK callback to bring the link UP.
 945  *
 946  * @param dev
 947  *   Pointer to Ethernet device structure.
 948  *
 949  * @return
 950  *   0 on success, negative errno value on failure.
 951  */
 952 static int hinic_dev_set_link_up(struct rte_eth_dev *dev)
 953 {
 954         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 955         int ret;
 956
 957         ret = hinic_set_xsfp_tx_status(nic_dev->hwdev, true);
 958         if (ret) {
 959                 PMD_DRV_LOG(ERR, "Enable port tx xsfp failed, dev_name: %s, port_id: %d",
 960                             nic_dev->proc_dev_name, dev->data->port_id);
 961                 return ret;
 962         }
 963
 964         /* link status follow phy port status, up will open pma */
 965         ret = hinic_set_port_enable(nic_dev->hwdev, true);
 966         if (ret)
 967                 PMD_DRV_LOG(ERR, "Set mac link up failed, dev_name: %s, port_id: %d",
 968                             nic_dev->proc_dev_name, dev->data->port_id);
 969
 970         return ret;
 971 }
 972
 973 /**
 974  * DPDK callback to bring the link DOWN.
 975  *
 976  * @param dev
 977  *   Pointer to Ethernet device structure.
 978  *
 979  * @return
 980  *   0 on success, negative errno value on failure.
 981  */
 982 static int hinic_dev_set_link_down(struct rte_eth_dev *dev)
 983 {
 984         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 985         int ret;
 986
 987         ret = hinic_set_xsfp_tx_status(nic_dev->hwdev, false);
 988         if (ret) {
 989                 PMD_DRV_LOG(ERR, "Disable port tx xsfp failed, dev_name: %s, port_id: %d",
 990                             nic_dev->proc_dev_name, dev->data->port_id);
 991                 return ret;
 992         }
 993
 994         /* link status follow phy port status, up will close pma */
 995         ret = hinic_set_port_enable(nic_dev->hwdev, false);
 996         if (ret)
 997                 PMD_DRV_LOG(ERR, "Set mac link down failed, dev_name: %s, port_id: %d",
 998                             nic_dev->proc_dev_name, dev->data->port_id);
 999
1000         return ret;
1001 }
1002
1003 /**
1004  * DPDK callback to start the device.
1005  *
1006  * @param dev
1007  *   Pointer to Ethernet device structure.
1008  *
1009  * @return
1010  *   0 on success, negative errno value on failure.
1011  */
1012 static int hinic_dev_start(struct rte_eth_dev *dev)
1013 {
1014         int rc;
1015         char *name;
1016         struct hinic_nic_dev *nic_dev;
1017
1018         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1019         name = dev->data->name;
1020
1021         /* reset rx and tx queue */
1022         hinic_reset_rx_queue(dev);
1023         hinic_reset_tx_queue(dev);
1024
1025         /* get func rx buf size */
1026         hinic_get_func_rx_buf_size(nic_dev);
1027
1028         /* init txq and rxq context */
1029         rc = hinic_init_qp_ctxts(nic_dev->hwdev);
1030         if (rc) {
1031                 PMD_DRV_LOG(ERR, "Initialize qp context failed, dev_name:%s",
1032                             name);
1033                 goto init_qp_fail;
1034         }
1035
1036         /* rss template */
1037         rc = hinic_config_mq_mode(dev, TRUE);
1038         if (rc) {
1039                 PMD_DRV_LOG(ERR, "Configure mq mode failed, dev_name: %s",
1040                             name);
1041                 goto cfg_mq_mode_fail;
1042         }
1043
1044         /* set default mtu */
1045         rc = hinic_set_port_mtu(nic_dev->hwdev, nic_dev->mtu_size);
1046         if (rc) {
1047                 PMD_DRV_LOG(ERR, "Set mtu_size[%d] failed, dev_name: %s",
1048                             nic_dev->mtu_size, name);
1049                 goto set_mtu_fail;
1050         }
1051
1052         /* configure rss rx_mode and other rx or tx default feature */
1053         rc = hinic_rxtx_configure(dev);
1054         if (rc) {
1055                 PMD_DRV_LOG(ERR, "Configure tx and rx failed, dev_name: %s",
1056                             name);
1057                 goto cfg_rxtx_fail;
1058         }
1059
1060         /* reactive pf status, so that uP report asyn event */
1061         hinic_set_pf_status(nic_dev->hwdev->hwif, HINIC_PF_STATUS_ACTIVE_FLAG);
1062
1063         /* open virtual port and ready to start packet receiving */
1064         rc = hinic_set_vport_enable(nic_dev->hwdev, true);
1065         if (rc) {
1066                 PMD_DRV_LOG(ERR, "Enable vport failed, dev_name:%s", name);
1067                 goto en_vport_fail;
1068         }
1069
1070         /* open physical port and start packet receiving */
1071         rc = hinic_set_port_enable(nic_dev->hwdev, true);
1072         if (rc) {
1073                 PMD_DRV_LOG(ERR, "Enable physical port failed, dev_name:%s",
1074                             name);
1075                 goto en_port_fail;
1076         }
1077
1078         /* update eth_dev link status */
1079         if (dev->data->dev_conf.intr_conf.lsc != 0)
1080                 (void)hinic_link_update(dev, 0);
1081
1082         hinic_set_bit(HINIC_DEV_START, &nic_dev->dev_status);
1083
1084         return 0;
1085
1086 en_port_fail:
1087         (void)hinic_set_vport_enable(nic_dev->hwdev, false);
1088
1089 en_vport_fail:
1090         hinic_set_pf_status(nic_dev->hwdev->hwif, HINIC_PF_STATUS_INIT);
1091
1092         /* Flush tx && rx chip resources in case of set vport fake fail */
1093         (void)hinic_flush_qp_res(nic_dev->hwdev);
1094         rte_delay_ms(100);
1095
1096         hinic_remove_rxtx_configure(dev);
1097
1098 cfg_rxtx_fail:
1099 set_mtu_fail:
1100 cfg_mq_mode_fail:
1101         hinic_free_qp_ctxts(nic_dev->hwdev);
1102
1103 init_qp_fail:
1104         hinic_free_all_rx_mbuf(dev);
1105         hinic_free_all_tx_mbuf(dev);
1106
1107         return rc;
1108 }
1109
1110 /**
1111  * DPDK callback to release the receive queue.
1112  *
1113  * @param queue
1114  *   Generic receive queue pointer.
1115  */
1116 static void hinic_rx_queue_release(void *queue)
1117 {
1118         struct hinic_rxq *rxq = queue;
1119         struct hinic_nic_dev *nic_dev;
1120
1121         if (!rxq) {
1122                 PMD_DRV_LOG(WARNING, "Rxq is null when release");
1123                 return;
1124         }
1125         nic_dev = rxq->nic_dev;
1126
1127         /* free rxq_pkt mbuf */
1128         hinic_free_all_rx_mbufs(rxq);
1129
1130         /* free rxq_cqe, rxq_info */
1131         hinic_free_rx_resources(rxq);
1132
1133         /* free root rq wq */
1134         hinic_destroy_rq(nic_dev->hwdev, rxq->q_id);
1135
1136         nic_dev->rxqs[rxq->q_id] = NULL;
1137
1138         /* free rxq */
1139         rte_free(rxq);
1140 }
1141
1142 /**
1143  * DPDK callback to release the transmit queue.
1144  *
1145  * @param queue
1146  *   Generic transmit queue pointer.
1147  */
1148 static void hinic_tx_queue_release(void *queue)
1149 {
1150         struct hinic_txq *txq = queue;
1151         struct hinic_nic_dev *nic_dev;
1152
1153         if (!txq) {
1154                 PMD_DRV_LOG(WARNING, "Txq is null when release");
1155                 return;
1156         }
1157         nic_dev = txq->nic_dev;
1158
1159         /* free txq_pkt mbuf */
1160         hinic_free_all_tx_mbufs(txq);
1161
1162         /* free txq_info */
1163         hinic_free_tx_resources(txq);
1164
1165         /* free root sq wq */
1166         hinic_destroy_sq(nic_dev->hwdev, txq->q_id);
1167         nic_dev->txqs[txq->q_id] = NULL;
1168
1169         /* free txq */
1170         rte_free(txq);
1171 }
1172
1173 static void hinic_free_all_rq(struct hinic_nic_dev *nic_dev)
1174 {
1175         u16 q_id;
1176
1177         for (q_id = 0; q_id < nic_dev->num_rq; q_id++)
1178                 hinic_destroy_rq(nic_dev->hwdev, q_id);
1179 }
1180
1181 static void hinic_free_all_sq(struct hinic_nic_dev *nic_dev)
1182 {
1183         u16 q_id;
1184
1185         for (q_id = 0; q_id < nic_dev->num_sq; q_id++)
1186                 hinic_destroy_sq(nic_dev->hwdev, q_id);
1187 }
1188
1189 /**
1190  * DPDK callback to stop the device.
1191  *
1192  * @param dev
1193  *   Pointer to Ethernet device structure.
1194  */
1195 static void hinic_dev_stop(struct rte_eth_dev *dev)
1196 {
1197         int rc;
1198         char *name;
1199         uint16_t port_id;
1200         struct hinic_nic_dev *nic_dev;
1201         struct rte_eth_link link;
1202
1203         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1204         name = dev->data->name;
1205         port_id = dev->data->port_id;
1206
1207         if (!hinic_test_and_clear_bit(HINIC_DEV_START, &nic_dev->dev_status)) {
1208                 PMD_DRV_LOG(INFO, "Device %s already stopped", name);
1209                 return;
1210         }
1211
1212         /* just stop phy port and vport */
1213         rc = hinic_set_port_enable(nic_dev->hwdev, false);
1214         if (rc)
1215                 PMD_DRV_LOG(WARNING, "Disable phy port failed, error: %d, dev_name:%s, port_id:%d",
1216                           rc, name, port_id);
1217
1218         rc = hinic_set_vport_enable(nic_dev->hwdev, false);
1219         if (rc)
1220                 PMD_DRV_LOG(WARNING, "Disable vport failed, error: %d, dev_name:%s, port_id:%d",
1221                           rc, name, port_id);
1222
1223         /* Clear recorded link status */
1224         memset(&link, 0, sizeof(link));
1225         (void)rte_eth_linkstatus_set(dev, &link);
1226
1227         /* flush pending io request */
1228         rc = hinic_rx_tx_flush(nic_dev->hwdev);
1229         if (rc)
1230                 PMD_DRV_LOG(WARNING, "Flush pending io failed, error: %d, dev_name: %s, port_id: %d",
1231                             rc, name, port_id);
1232
1233         /* clean rss table and rx_mode */
1234         hinic_remove_rxtx_configure(dev);
1235
1236         /* clean root context */
1237         hinic_free_qp_ctxts(nic_dev->hwdev);
1238
1239         hinic_free_fdir_filter(nic_dev);
1240
1241         /* free mbuf */
1242         hinic_free_all_rx_mbuf(dev);
1243         hinic_free_all_tx_mbuf(dev);
1244 }
1245
1246 static void hinic_disable_interrupt(struct rte_eth_dev *dev)
1247 {
1248         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1249         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1250         int ret, retries = 0;
1251
1252         hinic_clear_bit(HINIC_DEV_INTR_EN, &nic_dev->dev_status);
1253
1254         /* disable msix interrupt in hardware */
1255         hinic_set_msix_state(nic_dev->hwdev, 0, HINIC_MSIX_DISABLE);
1256
1257         /* disable rte interrupt */
1258         ret = rte_intr_disable(&pci_dev->intr_handle);
1259         if (ret)
1260                 PMD_DRV_LOG(ERR, "Disable intr failed: %d", ret);
1261
1262         do {
1263                 ret =
1264                 rte_intr_callback_unregister(&pci_dev->intr_handle,
1265                                              hinic_dev_interrupt_handler, dev);
1266                 if (ret >= 0) {
1267                         break;
1268                 } else if (ret == -EAGAIN) {
1269                         rte_delay_ms(100);
1270                         retries++;
1271                 } else {
1272                         PMD_DRV_LOG(ERR, "intr callback unregister failed: %d",
1273                                     ret);
1274                         break;
1275                 }
1276         } while (retries < HINIC_INTR_CB_UNREG_MAX_RETRIES);
1277
1278         if (retries == HINIC_INTR_CB_UNREG_MAX_RETRIES)
1279                 PMD_DRV_LOG(ERR, "Unregister intr callback failed after %d retries",
1280                             retries);
1281 }
1282
1283 static int hinic_set_dev_promiscuous(struct hinic_nic_dev *nic_dev, bool enable)
1284 {
1285         u32 rx_mode_ctrl = nic_dev->rx_mode_status;
1286
1287         if (enable)
1288                 rx_mode_ctrl |= HINIC_RX_MODE_PROMISC;
1289         else
1290                 rx_mode_ctrl &= (~HINIC_RX_MODE_PROMISC);
1291
1292         return hinic_config_rx_mode(nic_dev, rx_mode_ctrl);
1293 }
1294
1295 /**
1296  * DPDK callback to get device statistics.
1297  *
1298  * @param dev
1299  *   Pointer to Ethernet device structure.
1300  * @param stats
1301  *   Stats structure output buffer.
1302  *
1303  * @return
1304  *   0 on success and stats is filled,
1305  *   negative error value otherwise.
1306  */
1307 static int
1308 hinic_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1309 {
1310         int i, err, q_num;
1311         u64 rx_discards_pmd = 0;
1312         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1313         struct hinic_vport_stats vport_stats;
1314         struct hinic_rxq        *rxq = NULL;
1315         struct hinic_rxq_stats rxq_stats;
1316         struct hinic_txq        *txq = NULL;
1317         struct hinic_txq_stats txq_stats;
1318
1319         err = hinic_get_vport_stats(nic_dev->hwdev, &vport_stats);
1320         if (err) {
1321                 PMD_DRV_LOG(ERR, "Get vport stats from fw failed, nic_dev: %s",
1322                         nic_dev->proc_dev_name);
1323                 return err;
1324         }
1325
1326         /* rx queue stats */
1327         q_num = (nic_dev->num_rq < RTE_ETHDEV_QUEUE_STAT_CNTRS) ?
1328                         nic_dev->num_rq : RTE_ETHDEV_QUEUE_STAT_CNTRS;
1329         for (i = 0; i < q_num; i++) {
1330                 rxq = nic_dev->rxqs[i];
1331                 hinic_rxq_get_stats(rxq, &rxq_stats);
1332                 stats->q_ipackets[i] = rxq_stats.packets;
1333                 stats->q_ibytes[i] = rxq_stats.bytes;
1334                 stats->q_errors[i] = rxq_stats.rx_discards;
1335
1336                 stats->ierrors += rxq_stats.errors;
1337                 rx_discards_pmd += rxq_stats.rx_discards;
1338                 dev->data->rx_mbuf_alloc_failed += rxq_stats.rx_nombuf;
1339         }
1340
1341         /* tx queue stats */
1342         q_num = (nic_dev->num_sq < RTE_ETHDEV_QUEUE_STAT_CNTRS) ?
1343                 nic_dev->num_sq : RTE_ETHDEV_QUEUE_STAT_CNTRS;
1344         for (i = 0; i < q_num; i++) {
1345                 txq = nic_dev->txqs[i];
1346                 hinic_txq_get_stats(txq, &txq_stats);
1347                 stats->q_opackets[i] = txq_stats.packets;
1348                 stats->q_obytes[i] = txq_stats.bytes;
1349                 stats->oerrors += (txq_stats.tx_busy + txq_stats.off_errs);
1350         }
1351
1352         /* vport stats */
1353         stats->oerrors += vport_stats.tx_discard_vport;
1354
1355         stats->imissed = vport_stats.rx_discard_vport + rx_discards_pmd;
1356
1357         stats->ipackets = (vport_stats.rx_unicast_pkts_vport +
1358                         vport_stats.rx_multicast_pkts_vport +
1359                         vport_stats.rx_broadcast_pkts_vport -
1360                         rx_discards_pmd);
1361
1362         stats->opackets = (vport_stats.tx_unicast_pkts_vport +
1363                         vport_stats.tx_multicast_pkts_vport +
1364                         vport_stats.tx_broadcast_pkts_vport);
1365
1366         stats->ibytes = (vport_stats.rx_unicast_bytes_vport +
1367                         vport_stats.rx_multicast_bytes_vport +
1368                         vport_stats.rx_broadcast_bytes_vport);
1369
1370         stats->obytes = (vport_stats.tx_unicast_bytes_vport +
1371                         vport_stats.tx_multicast_bytes_vport +
1372                         vport_stats.tx_broadcast_bytes_vport);
1373         return 0;
1374 }
1375
1376 /**
1377  * DPDK callback to clear device statistics.
1378  *
1379  * @param dev
1380  *   Pointer to Ethernet device structure.
1381  */
1382 static int hinic_dev_stats_reset(struct rte_eth_dev *dev)
1383 {
1384         int qid;
1385         struct hinic_rxq        *rxq = NULL;
1386         struct hinic_txq        *txq = NULL;
1387         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1388         int ret;
1389
1390         ret = hinic_clear_vport_stats(nic_dev->hwdev);
1391         if (ret != 0)
1392                 return ret;
1393
1394         for (qid = 0; qid < nic_dev->num_rq; qid++) {
1395                 rxq = nic_dev->rxqs[qid];
1396                 hinic_rxq_stats_reset(rxq);
1397         }
1398
1399         for (qid = 0; qid < nic_dev->num_sq; qid++) {
1400                 txq = nic_dev->txqs[qid];
1401                 hinic_txq_stats_reset(txq);
1402         }
1403
1404         return 0;
1405 }
1406
1407 /**
1408  * DPDK callback to clear device extended statistics.
1409  *
1410  * @param dev
1411  *   Pointer to Ethernet device structure.
1412  */
1413 static int hinic_dev_xstats_reset(struct rte_eth_dev *dev)
1414 {
1415         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1416         int ret;
1417
1418         ret = hinic_dev_stats_reset(dev);
1419         if (ret != 0)
1420                 return ret;
1421
1422         if (hinic_func_type(nic_dev->hwdev) != TYPE_VF) {
1423                 ret = hinic_clear_phy_port_stats(nic_dev->hwdev);
1424                 if (ret != 0)
1425                         return ret;
1426         }
1427
1428         return 0;
1429 }
1430
1431 static void hinic_gen_random_mac_addr(struct rte_ether_addr *mac_addr)
1432 {
1433         uint64_t random_value;
1434
1435         /* Set Organizationally Unique Identifier (OUI) prefix */
1436         mac_addr->addr_bytes[0] = 0x00;
1437         mac_addr->addr_bytes[1] = 0x09;
1438         mac_addr->addr_bytes[2] = 0xC0;
1439         /* Force indication of locally assigned MAC address. */
1440         mac_addr->addr_bytes[0] |= RTE_ETHER_LOCAL_ADMIN_ADDR;
1441         /* Generate the last 3 bytes of the MAC address with a random number. */
1442         random_value = rte_rand();
1443         memcpy(&mac_addr->addr_bytes[3], &random_value, 3);
1444 }
1445
1446 /**
1447  * Init mac_vlan table in NIC.
1448  *
1449  * @param dev
1450  *   Pointer to Ethernet device structure.
1451  *
1452  * @return
1453  *   0 on success and stats is filled,
1454  *   negative error value otherwise.
1455  */
1456 static int hinic_init_mac_addr(struct rte_eth_dev *eth_dev)
1457 {
1458         struct hinic_nic_dev *nic_dev =
1459                                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
1460         uint8_t addr_bytes[RTE_ETHER_ADDR_LEN];
1461         u16 func_id = 0;
1462         int rc = 0;
1463
1464         rc = hinic_get_default_mac(nic_dev->hwdev, addr_bytes);
1465         if (rc)
1466                 return rc;
1467
1468         rte_ether_addr_copy((struct rte_ether_addr *)addr_bytes,
1469                 &eth_dev->data->mac_addrs[0]);
1470         if (rte_is_zero_ether_addr(&eth_dev->data->mac_addrs[0]))
1471                 hinic_gen_random_mac_addr(&eth_dev->data->mac_addrs[0]);
1472
1473         func_id = hinic_global_func_id(nic_dev->hwdev);
1474         rc = hinic_set_mac(nic_dev->hwdev,
1475                         eth_dev->data->mac_addrs[0].addr_bytes,
1476                         0, func_id);
1477         if (rc && rc != HINIC_PF_SET_VF_ALREADY)
1478                 return rc;
1479
1480         rte_ether_addr_copy(&eth_dev->data->mac_addrs[0],
1481                         &nic_dev->default_addr);
1482
1483         return 0;
1484 }
1485
1486 static void hinic_delete_mc_addr_list(struct hinic_nic_dev *nic_dev)
1487 {
1488         u16 func_id;
1489         u32 i;
1490
1491         func_id = hinic_global_func_id(nic_dev->hwdev);
1492
1493         for (i = 0; i < HINIC_MAX_MC_MAC_ADDRS; i++) {
1494                 if (rte_is_zero_ether_addr(&nic_dev->mc_list[i]))
1495                         break;
1496
1497                 hinic_del_mac(nic_dev->hwdev, nic_dev->mc_list[i].addr_bytes,
1498                               0, func_id);
1499                 memset(&nic_dev->mc_list[i], 0, sizeof(struct rte_ether_addr));
1500         }
1501 }
1502
1503 /**
1504  * Deinit mac_vlan table in NIC.
1505  *
1506  * @param dev
1507  *   Pointer to Ethernet device structure.
1508  *
1509  * @return
1510  *   0 on success and stats is filled,
1511  *   negative error value otherwise.
1512  */
1513 static void hinic_deinit_mac_addr(struct rte_eth_dev *eth_dev)
1514 {
1515         struct hinic_nic_dev *nic_dev =
1516                                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
1517         u16 func_id = 0;
1518         int rc;
1519         int i;
1520
1521         func_id = hinic_global_func_id(nic_dev->hwdev);
1522
1523         for (i = 0; i < HINIC_MAX_UC_MAC_ADDRS; i++) {
1524                 if (rte_is_zero_ether_addr(&eth_dev->data->mac_addrs[i]))
1525                         continue;
1526
1527                 rc = hinic_del_mac(nic_dev->hwdev,
1528                                    eth_dev->data->mac_addrs[i].addr_bytes,
1529                                    0, func_id);
1530                 if (rc && rc != HINIC_PF_SET_VF_ALREADY)
1531                         PMD_DRV_LOG(ERR, "Delete mac table failed, dev_name: %s",
1532                                     eth_dev->data->name);
1533
1534                 memset(&eth_dev->data->mac_addrs[i], 0,
1535                        sizeof(struct rte_ether_addr));
1536         }
1537
1538         /* delete multicast mac addrs */
1539         hinic_delete_mc_addr_list(nic_dev);
1540 }
1541
1542 static int hinic_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
1543 {
1544         int ret = 0;
1545         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1546
1547         PMD_DRV_LOG(INFO, "Set port mtu, port_id: %d, mtu: %d, max_pkt_len: %d",
1548                         dev->data->port_id, mtu, HINIC_MTU_TO_PKTLEN(mtu));
1549
1550         if (mtu < HINIC_MIN_MTU_SIZE || mtu > HINIC_MAX_MTU_SIZE) {
1551                 PMD_DRV_LOG(ERR, "Invalid mtu: %d, must between %d and %d",
1552                                 mtu, HINIC_MIN_MTU_SIZE, HINIC_MAX_MTU_SIZE);
1553                 return -EINVAL;
1554         }
1555
1556         ret = hinic_set_port_mtu(nic_dev->hwdev, mtu);
1557         if (ret) {
1558                 PMD_DRV_LOG(ERR, "Set port mtu failed, ret: %d", ret);
1559                 return ret;
1560         }
1561
1562         /* update max frame size */
1563         dev->data->dev_conf.rxmode.max_rx_pkt_len = HINIC_MTU_TO_PKTLEN(mtu);
1564         nic_dev->mtu_size = mtu;
1565
1566         return ret;
1567 }
1568
1569 static void hinic_store_vlan_filter(struct hinic_nic_dev *nic_dev,
1570                                         u16 vlan_id, bool on)
1571 {
1572         u32 vid_idx, vid_bit;
1573
1574         vid_idx = HINIC_VFTA_IDX(vlan_id);
1575         vid_bit = HINIC_VFTA_BIT(vlan_id);
1576
1577         if (on)
1578                 nic_dev->vfta[vid_idx] |= vid_bit;
1579         else
1580                 nic_dev->vfta[vid_idx] &= ~vid_bit;
1581 }
1582
1583 static bool hinic_find_vlan_filter(struct hinic_nic_dev *nic_dev,
1584                                 uint16_t vlan_id)
1585 {
1586         u32 vid_idx, vid_bit;
1587
1588         vid_idx = HINIC_VFTA_IDX(vlan_id);
1589         vid_bit = HINIC_VFTA_BIT(vlan_id);
1590
1591         return (nic_dev->vfta[vid_idx] & vid_bit) ? TRUE : FALSE;
1592 }
1593
1594 /**
1595  * DPDK callback to set vlan filter.
1596  *
1597  * @param dev
1598  *   Pointer to Ethernet device structure.
1599  * @param vlan_id
1600  *   vlan id is used to filter vlan packets
1601  * @param enable
1602  *   enable disable or enable vlan filter function
1603  */
1604 static int hinic_vlan_filter_set(struct rte_eth_dev *dev,
1605                                 uint16_t vlan_id, int enable)
1606 {
1607         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1608         int err = 0;
1609         u16 func_id;
1610
1611         if (vlan_id > RTE_ETHER_MAX_VLAN_ID)
1612                 return -EINVAL;
1613
1614         func_id = hinic_global_func_id(nic_dev->hwdev);
1615
1616         if (enable) {
1617                 /* If vlanid is already set, just return */
1618                 if (hinic_find_vlan_filter(nic_dev, vlan_id)) {
1619                         PMD_DRV_LOG(INFO, "Vlan %u has been added, device: %s",
1620                                   vlan_id, nic_dev->proc_dev_name);
1621                         return 0;
1622                 }
1623
1624                 err = hinic_add_remove_vlan(nic_dev->hwdev, vlan_id,
1625                                             func_id, TRUE);
1626         } else {
1627                 /* If vlanid can't be found, just return */
1628                 if (!hinic_find_vlan_filter(nic_dev, vlan_id)) {
1629                         PMD_DRV_LOG(INFO, "Vlan %u is not in the vlan filter list, device: %s",
1630                                   vlan_id, nic_dev->proc_dev_name);
1631                         return 0;
1632                 }
1633
1634                 err = hinic_add_remove_vlan(nic_dev->hwdev, vlan_id,
1635                                             func_id, FALSE);
1636         }
1637
1638         if (err) {
1639                 PMD_DRV_LOG(ERR, "%s vlan failed, func_id: %d, vlan_id: %d, err: %d",
1640                       enable ? "Add" : "Remove", func_id, vlan_id, err);
1641                 return err;
1642         }
1643
1644         hinic_store_vlan_filter(nic_dev, vlan_id, enable);
1645
1646         PMD_DRV_LOG(INFO, "%s vlan %u succeed, device: %s",
1647                   enable ? "Add" : "Remove", vlan_id, nic_dev->proc_dev_name);
1648         return 0;
1649 }
1650
1651 /**
1652  * DPDK callback to enable or disable vlan offload.
1653  *
1654  * @param dev
1655  *   Pointer to Ethernet device structure.
1656  * @param mask
1657  *   Definitions used for VLAN setting
1658  */
1659 static int hinic_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1660 {
1661         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1662         struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
1663         bool on;
1664         int err;
1665
1666         /* Enable or disable VLAN filter */
1667         if (mask & ETH_VLAN_FILTER_MASK) {
1668                 on = (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_FILTER) ?
1669                         TRUE : FALSE;
1670                 err = hinic_config_vlan_filter(nic_dev->hwdev, on);
1671                 if (err == HINIC_MGMT_CMD_UNSUPPORTED) {
1672                         PMD_DRV_LOG(WARNING,
1673                                 "Current matching version does not support vlan filter configuration, device: %s, port_id: %d",
1674                                   nic_dev->proc_dev_name, dev->data->port_id);
1675                 } else if (err) {
1676                         PMD_DRV_LOG(ERR, "Failed to %s vlan filter, device: %s, port_id: %d, err: %d",
1677                                   on ? "enable" : "disable",
1678                                   nic_dev->proc_dev_name,
1679                                   dev->data->port_id, err);
1680                         return err;
1681                 }
1682
1683                 PMD_DRV_LOG(INFO, "%s vlan filter succeed, device: %s, port_id: %d",
1684                           on ? "Enable" : "Disable",
1685                           nic_dev->proc_dev_name, dev->data->port_id);
1686         }
1687
1688         /* Enable or disable VLAN stripping */
1689         if (mask & ETH_VLAN_STRIP_MASK) {
1690                 on = (rxmode->offloads & DEV_RX_OFFLOAD_VLAN_STRIP) ?
1691                         TRUE : FALSE;
1692                 err = hinic_set_rx_vlan_offload(nic_dev->hwdev, on);
1693                 if (err) {
1694                         PMD_DRV_LOG(ERR, "Failed to %s vlan strip, device: %s, port_id: %d, err: %d",
1695                                   on ? "enable" : "disable",
1696                                   nic_dev->proc_dev_name,
1697                                   dev->data->port_id, err);
1698                         return err;
1699                 }
1700
1701                 PMD_DRV_LOG(INFO, "%s vlan strip succeed, device: %s, port_id: %d",
1702                           on ? "Enable" : "Disable",
1703                           nic_dev->proc_dev_name, dev->data->port_id);
1704         }
1705
1706         if (mask & ETH_VLAN_EXTEND_MASK) {
1707                 PMD_DRV_LOG(ERR, "Don't support vlan qinq, device: %s, port_id: %d",
1708                           nic_dev->proc_dev_name, dev->data->port_id);
1709                 return -ENOTSUP;
1710         }
1711
1712         return 0;
1713 }
1714
1715 static void hinic_remove_all_vlanid(struct rte_eth_dev *eth_dev)
1716 {
1717         struct hinic_nic_dev *nic_dev =
1718                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
1719         u16 func_id;
1720         int i;
1721
1722         func_id = hinic_global_func_id(nic_dev->hwdev);
1723         for (i = 0; i <= RTE_ETHER_MAX_VLAN_ID; i++) {
1724                 /* If can't find it, continue */
1725                 if (!hinic_find_vlan_filter(nic_dev, i))
1726                         continue;
1727
1728                 (void)hinic_add_remove_vlan(nic_dev->hwdev, i, func_id, FALSE);
1729                 hinic_store_vlan_filter(nic_dev, i, false);
1730         }
1731 }
1732
1733 static int hinic_set_dev_allmulticast(struct hinic_nic_dev *nic_dev,
1734                                 bool enable)
1735 {
1736         u32 rx_mode_ctrl = nic_dev->rx_mode_status;
1737
1738         if (enable)
1739                 rx_mode_ctrl |= HINIC_RX_MODE_MC_ALL;
1740         else
1741                 rx_mode_ctrl &= (~HINIC_RX_MODE_MC_ALL);
1742
1743         return hinic_config_rx_mode(nic_dev, rx_mode_ctrl);
1744 }
1745
1746 /**
1747  * DPDK callback to enable allmulticast mode.
1748  *
1749  * @param dev
1750  *   Pointer to Ethernet device structure.
1751  *
1752  * @return
1753  *   0 on success,
1754  *   negative error value otherwise.
1755  */
1756 static int hinic_dev_allmulticast_enable(struct rte_eth_dev *dev)
1757 {
1758         int ret = HINIC_OK;
1759         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1760
1761         ret = hinic_set_dev_allmulticast(nic_dev, true);
1762         if (ret) {
1763                 PMD_DRV_LOG(ERR, "Enable allmulticast failed, error: %d", ret);
1764                 return ret;
1765         }
1766
1767         PMD_DRV_LOG(INFO, "Enable allmulticast succeed, nic_dev: %s, port_id: %d",
1768                 nic_dev->proc_dev_name, dev->data->port_id);
1769         return 0;
1770 }
1771
1772 /**
1773  * DPDK callback to disable allmulticast mode.
1774  *
1775  * @param dev
1776  *   Pointer to Ethernet device structure.
1777  *
1778  * @return
1779  *   0 on success,
1780  *   negative error value otherwise.
1781  */
1782 static int hinic_dev_allmulticast_disable(struct rte_eth_dev *dev)
1783 {
1784         int ret = HINIC_OK;
1785         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1786
1787         ret = hinic_set_dev_allmulticast(nic_dev, false);
1788         if (ret) {
1789                 PMD_DRV_LOG(ERR, "Disable allmulticast failed, error: %d", ret);
1790                 return ret;
1791         }
1792
1793         PMD_DRV_LOG(INFO, "Disable allmulticast succeed, nic_dev: %s, port_id: %d",
1794                 nic_dev->proc_dev_name, dev->data->port_id);
1795         return 0;
1796 }
1797
1798 /**
1799  * DPDK callback to enable promiscuous mode.
1800  *
1801  * @param dev
1802  *   Pointer to Ethernet device structure.
1803  *
1804  * @return
1805  *   0 on success,
1806  *   negative error value otherwise.
1807  */
1808 static int hinic_dev_promiscuous_enable(struct rte_eth_dev *dev)
1809 {
1810         int rc = HINIC_OK;
1811         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1812
1813         PMD_DRV_LOG(INFO, "Enable promiscuous, nic_dev: %s, port_id: %d, promisc: %d",
1814                     nic_dev->proc_dev_name, dev->data->port_id,
1815                     dev->data->promiscuous);
1816
1817         rc = hinic_set_dev_promiscuous(nic_dev, true);
1818         if (rc)
1819                 PMD_DRV_LOG(ERR, "Enable promiscuous failed");
1820
1821         return rc;
1822 }
1823
1824 /**
1825  * DPDK callback to disable promiscuous mode.
1826  *
1827  * @param dev
1828  *   Pointer to Ethernet device structure.
1829  *
1830  * @return
1831  *   0 on success,
1832  *   negative error value otherwise.
1833  */
1834 static int hinic_dev_promiscuous_disable(struct rte_eth_dev *dev)
1835 {
1836         int rc = HINIC_OK;
1837         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1838
1839         PMD_DRV_LOG(INFO, "Disable promiscuous, nic_dev: %s, port_id: %d, promisc: %d",
1840                     nic_dev->proc_dev_name, dev->data->port_id,
1841                     dev->data->promiscuous);
1842
1843         rc = hinic_set_dev_promiscuous(nic_dev, false);
1844         if (rc)
1845                 PMD_DRV_LOG(ERR, "Disable promiscuous failed");
1846
1847         return rc;
1848 }
1849
1850 /**
1851  * DPDK callback to update the RSS hash key and RSS hash type.
1852  *
1853  * @param dev
1854  *   Pointer to Ethernet device structure.
1855  * @param rss_conf
1856  *   RSS configuration data.
1857  *
1858  * @return
1859  *   0 on success, negative error value otherwise.
1860  */
1861 static int hinic_rss_hash_update(struct rte_eth_dev *dev,
1862                           struct rte_eth_rss_conf *rss_conf)
1863 {
1864         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1865         u8 tmpl_idx = nic_dev->rss_tmpl_idx;
1866         u8 hashkey[HINIC_RSS_KEY_SIZE] = {0};
1867         u8 prio_tc[HINIC_DCB_UP_MAX] = {0};
1868         u64 rss_hf = rss_conf->rss_hf;
1869         struct nic_rss_type rss_type = {0};
1870         int err = 0;
1871
1872         if (!(nic_dev->flags & ETH_MQ_RX_RSS_FLAG)) {
1873                 PMD_DRV_LOG(WARNING, "RSS is not enabled");
1874                 return HINIC_OK;
1875         }
1876
1877         if (rss_conf->rss_key_len > HINIC_RSS_KEY_SIZE) {
1878                 PMD_DRV_LOG(ERR, "Invalid rss key, rss_key_len:%d",
1879                             rss_conf->rss_key_len);
1880                 return HINIC_ERROR;
1881         }
1882
1883         if (rss_conf->rss_key) {
1884                 memcpy(hashkey, rss_conf->rss_key, rss_conf->rss_key_len);
1885                 err = hinic_rss_set_template_tbl(nic_dev->hwdev, tmpl_idx,
1886                                                  hashkey);
1887                 if (err) {
1888                         PMD_DRV_LOG(ERR, "Set rss template table failed");
1889                         goto disable_rss;
1890                 }
1891         }
1892
1893         rss_type.ipv4 = (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4)) ? 1 : 0;
1894         rss_type.tcp_ipv4 = (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) ? 1 : 0;
1895         rss_type.ipv6 = (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6)) ? 1 : 0;
1896         rss_type.ipv6_ext = (rss_hf & ETH_RSS_IPV6_EX) ? 1 : 0;
1897         rss_type.tcp_ipv6 = (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) ? 1 : 0;
1898         rss_type.tcp_ipv6_ext = (rss_hf & ETH_RSS_IPV6_TCP_EX) ? 1 : 0;
1899         rss_type.udp_ipv4 = (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) ? 1 : 0;
1900         rss_type.udp_ipv6 = (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) ? 1 : 0;
1901
1902         err = hinic_set_rss_type(nic_dev->hwdev, tmpl_idx, rss_type);
1903         if (err) {
1904                 PMD_DRV_LOG(ERR, "Set rss type table failed");
1905                 goto disable_rss;
1906         }
1907
1908         return 0;
1909
1910 disable_rss:
1911         memset(prio_tc, 0, sizeof(prio_tc));
1912         (void)hinic_rss_cfg(nic_dev->hwdev, 0, tmpl_idx, 0, prio_tc);
1913         return err;
1914 }
1915
1916 /**
1917  * DPDK callback to get the RSS hash configuration.
1918  *
1919  * @param dev
1920  *   Pointer to Ethernet device structure.
1921  * @param rss_conf
1922  *   RSS configuration data.
1923  *
1924  * @return
1925  *   0 on success, negative error value otherwise.
1926  */
1927 static int hinic_rss_conf_get(struct rte_eth_dev *dev,
1928                        struct rte_eth_rss_conf *rss_conf)
1929 {
1930         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1931         u8 tmpl_idx = nic_dev->rss_tmpl_idx;
1932         u8 hashkey[HINIC_RSS_KEY_SIZE] = {0};
1933         struct nic_rss_type rss_type = {0};
1934         int err;
1935
1936         if (!(nic_dev->flags & ETH_MQ_RX_RSS_FLAG)) {
1937                 PMD_DRV_LOG(WARNING, "RSS is not enabled");
1938                 return HINIC_ERROR;
1939         }
1940
1941         err = hinic_rss_get_template_tbl(nic_dev->hwdev, tmpl_idx, hashkey);
1942         if (err)
1943                 return err;
1944
1945         if (rss_conf->rss_key &&
1946             rss_conf->rss_key_len >= HINIC_RSS_KEY_SIZE) {
1947                 memcpy(rss_conf->rss_key, hashkey, sizeof(hashkey));
1948                 rss_conf->rss_key_len = sizeof(hashkey);
1949         }
1950
1951         err = hinic_get_rss_type(nic_dev->hwdev, tmpl_idx, &rss_type);
1952         if (err)
1953                 return err;
1954
1955         rss_conf->rss_hf = 0;
1956         rss_conf->rss_hf |=  rss_type.ipv4 ?
1957                 (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4) : 0;
1958         rss_conf->rss_hf |=  rss_type.tcp_ipv4 ? ETH_RSS_NONFRAG_IPV4_TCP : 0;
1959         rss_conf->rss_hf |=  rss_type.ipv6 ?
1960                 (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6) : 0;
1961         rss_conf->rss_hf |=  rss_type.ipv6_ext ? ETH_RSS_IPV6_EX : 0;
1962         rss_conf->rss_hf |=  rss_type.tcp_ipv6 ? ETH_RSS_NONFRAG_IPV6_TCP : 0;
1963         rss_conf->rss_hf |=  rss_type.tcp_ipv6_ext ? ETH_RSS_IPV6_TCP_EX : 0;
1964         rss_conf->rss_hf |=  rss_type.udp_ipv4 ? ETH_RSS_NONFRAG_IPV4_UDP : 0;
1965         rss_conf->rss_hf |=  rss_type.udp_ipv6 ? ETH_RSS_NONFRAG_IPV6_UDP : 0;
1966
1967         return HINIC_OK;
1968 }
1969
1970 /**
1971  * DPDK callback to update the RETA indirection table.
1972  *
1973  * @param dev
1974  *   Pointer to Ethernet device structure.
1975  * @param reta_conf
1976  *   Pointer to RETA configuration structure array.
1977  * @param reta_size
1978  *   Size of the RETA table.
1979  *
1980  * @return
1981  *   0 on success, negative error value otherwise.
1982  */
1983 static int hinic_rss_indirtbl_update(struct rte_eth_dev *dev,
1984                               struct rte_eth_rss_reta_entry64 *reta_conf,
1985                               uint16_t reta_size)
1986 {
1987         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1988         u8 tmpl_idx = nic_dev->rss_tmpl_idx;
1989         u8 prio_tc[HINIC_DCB_UP_MAX] = {0};
1990         u32 indirtbl[NIC_RSS_INDIR_SIZE] = {0};
1991         int err = 0;
1992         u16 i = 0;
1993         u16 idx, shift;
1994
1995         if (!(nic_dev->flags & ETH_MQ_RX_RSS_FLAG))
1996                 return HINIC_OK;
1997
1998         if (reta_size != NIC_RSS_INDIR_SIZE) {
1999                 PMD_DRV_LOG(ERR, "Invalid reta size, reta_size:%d", reta_size);
2000                 return HINIC_ERROR;
2001         }
2002
2003         err = hinic_rss_get_indir_tbl(nic_dev->hwdev, tmpl_idx, indirtbl);
2004         if (err)
2005                 return err;
2006
2007         /* update rss indir_tbl */
2008         for (i = 0; i < reta_size; i++) {
2009                 idx = i / RTE_RETA_GROUP_SIZE;
2010                 shift = i % RTE_RETA_GROUP_SIZE;
2011                 if (reta_conf[idx].mask & (1ULL << shift))
2012                         indirtbl[i] = reta_conf[idx].reta[shift];
2013         }
2014
2015         for (i = 0 ; i < reta_size; i++) {
2016                 if (indirtbl[i] >= nic_dev->num_rq) {
2017                         PMD_DRV_LOG(ERR, "Invalid reta entry, index:%d, num_rq:%d",
2018                                     i, nic_dev->num_rq);
2019                         goto disable_rss;
2020                 }
2021         }
2022
2023         err = hinic_rss_set_indir_tbl(nic_dev->hwdev, tmpl_idx, indirtbl);
2024         if (err)
2025                 goto disable_rss;
2026
2027         nic_dev->rss_indir_flag = true;
2028
2029         return 0;
2030
2031 disable_rss:
2032         memset(prio_tc, 0, sizeof(prio_tc));
2033         (void)hinic_rss_cfg(nic_dev->hwdev, 0, tmpl_idx, 0, prio_tc);
2034
2035         return HINIC_ERROR;
2036 }
2037
2038
2039 /**
2040  * DPDK callback to get the RETA indirection table.
2041  *
2042  * @param dev
2043  *   Pointer to Ethernet device structure.
2044  * @param reta_conf
2045  *   Pointer to RETA configuration structure array.
2046  * @param reta_size
2047  *   Size of the RETA table.
2048  *
2049  * @return
2050  *   0 on success, negative error value otherwise.
2051  */
2052 static int hinic_rss_indirtbl_query(struct rte_eth_dev *dev,
2053                              struct rte_eth_rss_reta_entry64 *reta_conf,
2054                              uint16_t reta_size)
2055 {
2056         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
2057         u8 tmpl_idx = nic_dev->rss_tmpl_idx;
2058         int err = 0;
2059         u32 indirtbl[NIC_RSS_INDIR_SIZE] = {0};
2060         u16 idx, shift;
2061         u16 i = 0;
2062
2063         if (reta_size != NIC_RSS_INDIR_SIZE) {
2064                 PMD_DRV_LOG(ERR, "Invalid reta size, reta_size:%d", reta_size);
2065                 return HINIC_ERROR;
2066         }
2067
2068         err = hinic_rss_get_indir_tbl(nic_dev->hwdev, tmpl_idx, indirtbl);
2069         if (err) {
2070                 PMD_DRV_LOG(ERR, "Get rss indirect table failed, error:%d",
2071                             err);
2072                 return err;
2073         }
2074
2075         for (i = 0; i < reta_size; i++) {
2076                 idx = i / RTE_RETA_GROUP_SIZE;
2077                 shift = i % RTE_RETA_GROUP_SIZE;
2078                 if (reta_conf[idx].mask & (1ULL << shift))
2079                         reta_conf[idx].reta[shift] = (uint16_t)indirtbl[i];
2080         }
2081
2082         return HINIC_OK;
2083 }
2084
2085 /**
2086  * DPDK callback to get extended device statistics.
2087  *
2088  * @param dev
2089  *   Pointer to Ethernet device.
2090  * @param xstats
2091  *   Pointer to rte extended stats table.
2092  * @param n
2093  *   The size of the stats table.
2094  *
2095  * @return
2096  *   Number of extended stats on success and stats is filled,
2097  *   negative error value otherwise.
2098  */
2099 static int hinic_dev_xstats_get(struct rte_eth_dev *dev,
2100                          struct rte_eth_xstat *xstats,
2101                          unsigned int n)
2102 {
2103         u16 qid = 0;
2104         u32 i;
2105         int err, count;
2106         struct hinic_nic_dev *nic_dev;
2107         struct hinic_phy_port_stats port_stats;
2108         struct hinic_vport_stats vport_stats;
2109         struct hinic_rxq        *rxq = NULL;
2110         struct hinic_rxq_stats rxq_stats;
2111         struct hinic_txq        *txq = NULL;
2112         struct hinic_txq_stats txq_stats;
2113
2114         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
2115         count = hinic_xstats_calc_num(nic_dev);
2116         if ((int)n < count)
2117                 return count;
2118
2119         count = 0;
2120
2121         /* Get stats from hinic_rxq_stats */
2122         for (qid = 0; qid < nic_dev->num_rq; qid++) {
2123                 rxq = nic_dev->rxqs[qid];
2124                 hinic_rxq_get_stats(rxq, &rxq_stats);
2125
2126                 for (i = 0; i < HINIC_RXQ_XSTATS_NUM; i++) {
2127                         xstats[count].value =
2128                                 *(uint64_t *)(((char *)&rxq_stats) +
2129                                 hinic_rxq_stats_strings[i].offset);
2130                         xstats[count].id = count;
2131                         count++;
2132                 }
2133         }
2134
2135         /* Get stats from hinic_txq_stats */
2136         for (qid = 0; qid < nic_dev->num_sq; qid++) {
2137                 txq = nic_dev->txqs[qid];
2138                 hinic_txq_get_stats(txq, &txq_stats);
2139
2140                 for (i = 0; i < HINIC_TXQ_XSTATS_NUM; i++) {
2141                         xstats[count].value =
2142                                 *(uint64_t *)(((char *)&txq_stats) +
2143                                 hinic_txq_stats_strings[i].offset);
2144                         xstats[count].id = count;
2145                         count++;
2146                 }
2147         }
2148
2149         /* Get stats from hinic_vport_stats */
2150         err = hinic_get_vport_stats(nic_dev->hwdev, &vport_stats);
2151         if (err)
2152                 return err;
2153
2154         for (i = 0; i < HINIC_VPORT_XSTATS_NUM; i++) {
2155                 xstats[count].value =
2156                         *(uint64_t *)(((char *)&vport_stats) +
2157                         hinic_vport_stats_strings[i].offset);
2158                 xstats[count].id = count;
2159                 count++;
2160         }
2161
2162         if (HINIC_IS_VF(nic_dev->hwdev))
2163                 return count;
2164
2165         /* Get stats from hinic_phy_port_stats */
2166         err = hinic_get_phy_port_stats(nic_dev->hwdev, &port_stats);
2167         if (err)
2168                 return err;
2169
2170         for (i = 0; i < HINIC_PHYPORT_XSTATS_NUM; i++) {
2171                 xstats[count].value = *(uint64_t *)(((char *)&port_stats) +
2172                                 hinic_phyport_stats_strings[i].offset);
2173                 xstats[count].id = count;
2174                 count++;
2175         }
2176
2177         return count;
2178 }
2179
2180 static void hinic_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
2181                                 struct rte_eth_rxq_info *qinfo)
2182 {
2183         struct hinic_rxq  *rxq = dev->data->rx_queues[queue_id];
2184
2185         qinfo->mp = rxq->mb_pool;
2186         qinfo->nb_desc = rxq->q_depth;
2187 }
2188
2189 static void hinic_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
2190                                 struct rte_eth_txq_info *qinfo)
2191 {
2192         struct hinic_txq  *txq = dev->data->tx_queues[queue_id];
2193
2194         qinfo->nb_desc = txq->q_depth;
2195 }
2196
2197 /**
2198  * DPDK callback to retrieve names of extended device statistics
2199  *
2200  * @param dev
2201  *   Pointer to Ethernet device structure.
2202  * @param xstats_names
2203  *   Buffer to insert names into.
2204  *
2205  * @return
2206  *   Number of xstats names.
2207  */
2208 static int hinic_dev_xstats_get_names(struct rte_eth_dev *dev,
2209                                struct rte_eth_xstat_name *xstats_names,
2210                                __rte_unused unsigned int limit)
2211 {
2212         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
2213         int count = 0;
2214         u16 i = 0, q_num;
2215
2216         if (xstats_names == NULL)
2217                 return hinic_xstats_calc_num(nic_dev);
2218
2219         /* get pmd rxq stats */
2220         for (q_num = 0; q_num < nic_dev->num_rq; q_num++) {
2221                 for (i = 0; i < HINIC_RXQ_XSTATS_NUM; i++) {
2222                         snprintf(xstats_names[count].name,
2223                                  sizeof(xstats_names[count].name),
2224                                  "rxq%d_%s_pmd",
2225                                  q_num, hinic_rxq_stats_strings[i].name);
2226                         count++;
2227                 }
2228         }
2229
2230         /* get pmd txq stats */
2231         for (q_num = 0; q_num < nic_dev->num_sq; q_num++) {
2232                 for (i = 0; i < HINIC_TXQ_XSTATS_NUM; i++) {
2233                         snprintf(xstats_names[count].name,
2234                                  sizeof(xstats_names[count].name),
2235                                  "txq%d_%s_pmd",
2236                                  q_num, hinic_txq_stats_strings[i].name);
2237                         count++;
2238                 }
2239         }
2240
2241         /* get vport stats */
2242         for (i = 0; i < HINIC_VPORT_XSTATS_NUM; i++) {
2243                 snprintf(xstats_names[count].name,
2244                          sizeof(xstats_names[count].name),
2245                          "%s",
2246                          hinic_vport_stats_strings[i].name);
2247                 count++;
2248         }
2249
2250         if (HINIC_IS_VF(nic_dev->hwdev))
2251                 return count;
2252
2253         /* get phy port stats */
2254         for (i = 0; i < HINIC_PHYPORT_XSTATS_NUM; i++) {
2255                 snprintf(xstats_names[count].name,
2256                          sizeof(xstats_names[count].name),
2257                          "%s",
2258                          hinic_phyport_stats_strings[i].name);
2259                 count++;
2260         }
2261
2262         return count;
2263 }
2264 /**
2265  *  DPDK callback to set mac address
2266  *
2267  * @param dev
2268  *   Pointer to Ethernet device structure.
2269  * @param addr
2270  *   Pointer to mac address
2271  * @return
2272  *   0 on success, negative error value otherwise.
2273  */
2274 static int hinic_set_mac_addr(struct rte_eth_dev *dev,
2275                               struct rte_ether_addr *addr)
2276 {
2277         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
2278         u16 func_id;
2279         int err;
2280
2281         func_id = hinic_global_func_id(nic_dev->hwdev);
2282         err = hinic_update_mac(nic_dev->hwdev, nic_dev->default_addr.addr_bytes,
2283                                addr->addr_bytes, 0, func_id);
2284         if (err)
2285                 return err;
2286
2287         rte_ether_addr_copy(addr, &nic_dev->default_addr);
2288
2289         PMD_DRV_LOG(INFO, "Set new mac address %02x:%02x:%02x:%02x:%02x:%02x\n",
2290                     addr->addr_bytes[0], addr->addr_bytes[1],
2291                     addr->addr_bytes[2], addr->addr_bytes[3],
2292                     addr->addr_bytes[4], addr->addr_bytes[5]);
2293
2294         return 0;
2295 }
2296
2297 /**
2298  * DPDK callback to remove a MAC address.
2299  *
2300  * @param dev
2301  *   Pointer to Ethernet device structure.
2302  * @param index
2303  *   MAC address index.
2304  */
2305 static void hinic_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
2306 {
2307         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
2308         u16 func_id;
2309         int ret;
2310
2311         if (index >= HINIC_MAX_UC_MAC_ADDRS) {
2312                 PMD_DRV_LOG(INFO, "Remove mac index(%u) is out of range",
2313                             index);
2314                 return;
2315         }
2316
2317         func_id = hinic_global_func_id(nic_dev->hwdev);
2318         ret = hinic_del_mac(nic_dev->hwdev,
2319                             dev->data->mac_addrs[index].addr_bytes, 0, func_id);
2320         if (ret)
2321                 return;
2322
2323         memset(&dev->data->mac_addrs[index], 0, sizeof(struct rte_ether_addr));
2324 }
2325
2326 /**
2327  * DPDK callback to add a MAC address.
2328  *
2329  * @param dev
2330  *   Pointer to Ethernet device structure.
2331  * @param mac_addr
2332  *   MAC address to register.
2333  * @param index
2334  *   MAC address index.
2335  * @param vmdq
2336  *   VMDq pool index to associate address with (ignored).
2337  *
2338  * @return
2339  *   0 on success, a negative errno value otherwise and rte_errno is set.
2340  */
2341
2342 static int hinic_mac_addr_add(struct rte_eth_dev *dev,
2343                               struct rte_ether_addr *mac_addr, uint32_t index,
2344                               __rte_unused uint32_t vmdq)
2345 {
2346         struct hinic_nic_dev  *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
2347         unsigned int i;
2348         u16 func_id;
2349         int ret;
2350
2351         if (index >= HINIC_MAX_UC_MAC_ADDRS) {
2352                 PMD_DRV_LOG(INFO, "Add mac index(%u) is out of range,", index);
2353                 return -EINVAL;
2354         }
2355
2356         /* First, make sure this address isn't already configured. */
2357         for (i = 0; (i != HINIC_MAX_UC_MAC_ADDRS); ++i) {
2358                 /* Skip this index, it's going to be reconfigured. */
2359                 if (i == index)
2360                         continue;
2361
2362                 if (memcmp(&dev->data->mac_addrs[i],
2363                         mac_addr, sizeof(*mac_addr)))
2364                         continue;
2365
2366                 PMD_DRV_LOG(INFO, "MAC address already configured");
2367                 return -EADDRINUSE;
2368         }
2369
2370         func_id = hinic_global_func_id(nic_dev->hwdev);
2371         ret = hinic_set_mac(nic_dev->hwdev, mac_addr->addr_bytes, 0, func_id);
2372         if (ret)
2373                 return ret;
2374
2375         dev->data->mac_addrs[index] = *mac_addr;
2376         return 0;
2377 }
2378
2379 /**
2380  *  DPDK callback to set multicast mac address
2381  *
2382  * @param dev
2383  *   Pointer to Ethernet device structure.
2384  * @param mc_addr_set
2385  *   Pointer to multicast mac address
2386  * @param nb_mc_addr
2387  *   mc addr count
2388  * @return
2389  *   0 on success, negative error value otherwise.
2390  */
2391 static int hinic_set_mc_addr_list(struct rte_eth_dev *dev,
2392                                   struct rte_ether_addr *mc_addr_set,
2393                                   uint32_t nb_mc_addr)
2394 {
2395         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
2396         u16 func_id;
2397         int ret;
2398         u32 i;
2399
2400         func_id = hinic_global_func_id(nic_dev->hwdev);
2401
2402         /* delete old multi_cast addrs firstly */
2403         hinic_delete_mc_addr_list(nic_dev);
2404
2405         if (nb_mc_addr > HINIC_MAX_MC_MAC_ADDRS)
2406                 goto allmulti;
2407
2408         for (i = 0; i < nb_mc_addr; i++) {
2409                 ret = hinic_set_mac(nic_dev->hwdev, mc_addr_set[i].addr_bytes,
2410                                     0, func_id);
2411                 /* if add mc addr failed, set all multi_cast */
2412                 if (ret) {
2413                         hinic_delete_mc_addr_list(nic_dev);
2414                         goto allmulti;
2415                 }
2416
2417                 rte_ether_addr_copy(&mc_addr_set[i], &nic_dev->mc_list[i]);
2418         }
2419
2420         return 0;
2421
2422 allmulti:
2423         hinic_dev_allmulticast_enable(dev);
2424
2425         return 0;
2426 }
2427
2428 /**
2429  * DPDK callback to manage filter operations
2430  *
2431  * @param dev
2432  *   Pointer to Ethernet device structure.
2433  * @param filter_type
2434  *   Filter type.
2435  * @param filter_op
2436  *   Operation to perform.
2437  * @param arg
2438  *   Pointer to operation-specific structure.
2439  *
2440  * @return
2441  *   0 on success, negative errno value on failure.
2442  */
2443 static int hinic_dev_filter_ctrl(struct rte_eth_dev *dev,
2444                      enum rte_filter_type filter_type,
2445                      enum rte_filter_op filter_op,
2446                      void *arg)
2447 {
2448         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
2449         int func_id = hinic_global_func_id(nic_dev->hwdev);
2450
2451         switch (filter_type) {
2452         case RTE_ETH_FILTER_GENERIC:
2453                 if (filter_op != RTE_ETH_FILTER_GET)
2454                         return -EINVAL;
2455                 *(const void **)arg = &hinic_flow_ops;
2456                 break;
2457         default:
2458                 PMD_DRV_LOG(INFO, "Filter type (%d) not supported",
2459                         filter_type);
2460                 return -EINVAL;
2461         }
2462
2463         PMD_DRV_LOG(INFO, "Set filter_ctrl succeed, func_id: 0x%x, filter_type: 0x%x,"
2464                         "filter_op: 0x%x.", func_id, filter_type, filter_op);
2465         return 0;
2466 }
2467
2468 static int hinic_set_default_pause_feature(struct hinic_nic_dev *nic_dev)
2469 {
2470         struct nic_pause_config pause_config = {0};
2471
2472         pause_config.auto_neg = 0;
2473         pause_config.rx_pause = HINIC_DEFAUT_PAUSE_CONFIG;
2474         pause_config.tx_pause = HINIC_DEFAUT_PAUSE_CONFIG;
2475
2476         return hinic_set_pause_config(nic_dev->hwdev, pause_config);
2477 }
2478
2479 static int hinic_set_default_dcb_feature(struct hinic_nic_dev *nic_dev)
2480 {
2481         u8 up_tc[HINIC_DCB_UP_MAX] = {0};
2482         u8 up_pgid[HINIC_DCB_UP_MAX] = {0};
2483         u8 up_bw[HINIC_DCB_UP_MAX] = {0};
2484         u8 pg_bw[HINIC_DCB_UP_MAX] = {0};
2485         u8 up_strict[HINIC_DCB_UP_MAX] = {0};
2486         int i = 0;
2487
2488         pg_bw[0] = 100;
2489         for (i = 0; i < HINIC_DCB_UP_MAX; i++)
2490                 up_bw[i] = 100;
2491
2492         return hinic_dcb_set_ets(nic_dev->hwdev, up_tc, pg_bw,
2493                                         up_pgid, up_bw, up_strict);
2494 }
2495
2496 static int hinic_init_default_cos(struct hinic_nic_dev *nic_dev)
2497 {
2498         u8 cos_id = 0;
2499         int err;
2500
2501         if (!HINIC_IS_VF(nic_dev->hwdev)) {
2502                 nic_dev->default_cos =
2503                                 (hinic_global_func_id(nic_dev->hwdev) +
2504                                                 DEFAULT_BASE_COS) % NR_MAX_COS;
2505         } else {
2506                 err = hinic_vf_get_default_cos(nic_dev->hwdev, &cos_id);
2507                 if (err) {
2508                         PMD_DRV_LOG(ERR, "Get VF default cos failed, err: %d",
2509                                         err);
2510                         return HINIC_ERROR;
2511                 }
2512
2513                 nic_dev->default_cos = cos_id;
2514         }
2515
2516         return 0;
2517 }
2518
2519 static int hinic_set_default_hw_feature(struct hinic_nic_dev *nic_dev)
2520 {
2521         int err;
2522
2523         err = hinic_init_default_cos(nic_dev);
2524         if (err)
2525                 return err;
2526
2527         if (hinic_func_type(nic_dev->hwdev) == TYPE_VF)
2528                 return 0;
2529
2530         /* Restore DCB configure to default status */
2531         err = hinic_set_default_dcb_feature(nic_dev);
2532         if (err)
2533                 return err;
2534
2535         /* Set pause enable, and up will disable pfc. */
2536         err = hinic_set_default_pause_feature(nic_dev);
2537         if (err)
2538                 return err;
2539
2540         err = hinic_reset_port_link_cfg(nic_dev->hwdev);
2541         if (err)
2542                 return err;
2543
2544         err = hinic_set_link_status_follow(nic_dev->hwdev,
2545                                            HINIC_LINK_FOLLOW_PORT);
2546         if (err == HINIC_MGMT_CMD_UNSUPPORTED)
2547                 PMD_DRV_LOG(WARNING, "Don't support to set link status follow phy port status");
2548         else if (err)
2549                 return err;
2550
2551         return hinic_set_anti_attack(nic_dev->hwdev, true);
2552 }
2553
2554 static int32_t hinic_card_workmode_check(struct hinic_nic_dev *nic_dev)
2555 {
2556         struct hinic_board_info info = { 0 };
2557         int rc;
2558
2559         if (hinic_func_type(nic_dev->hwdev) == TYPE_VF)
2560                 return 0;
2561
2562         rc = hinic_get_board_info(nic_dev->hwdev, &info);
2563         if (rc)
2564                 return rc;
2565
2566         return (info.service_mode == HINIC_SERVICE_MODE_NIC ? HINIC_OK :
2567                                                 HINIC_ERROR);
2568 }
2569
2570 static int hinic_copy_mempool_init(struct hinic_nic_dev *nic_dev)
2571 {
2572         nic_dev->cpy_mpool = rte_mempool_lookup(nic_dev->proc_dev_name);
2573         if (nic_dev->cpy_mpool == NULL) {
2574                 nic_dev->cpy_mpool =
2575                 rte_pktmbuf_pool_create(nic_dev->proc_dev_name,
2576                                         HINIC_COPY_MEMPOOL_DEPTH,
2577                                         0, 0,
2578                                         HINIC_COPY_MBUF_SIZE,
2579                                         rte_socket_id());
2580                 if (!nic_dev->cpy_mpool) {
2581                         PMD_DRV_LOG(ERR, "Create copy mempool failed, errno: %d, dev_name: %s",
2582                                     rte_errno, nic_dev->proc_dev_name);
2583                         return -ENOMEM;
2584                 }
2585         }
2586
2587         return 0;
2588 }
2589
2590 static void hinic_copy_mempool_uninit(struct hinic_nic_dev *nic_dev)
2591 {
2592         if (nic_dev->cpy_mpool != NULL)
2593                 rte_mempool_free(nic_dev->cpy_mpool);
2594 }
2595
2596 static int hinic_init_sw_rxtxqs(struct hinic_nic_dev *nic_dev)
2597 {
2598         u32 txq_size;
2599         u32 rxq_size;
2600
2601         /* allocate software txq array */
2602         txq_size = nic_dev->nic_cap.max_sqs * sizeof(*nic_dev->txqs);
2603         nic_dev->txqs = kzalloc_aligned(txq_size, GFP_KERNEL);
2604         if (!nic_dev->txqs) {
2605                 PMD_DRV_LOG(ERR, "Allocate txqs failed");
2606                 return -ENOMEM;
2607         }
2608
2609         /* allocate software rxq array */
2610         rxq_size = nic_dev->nic_cap.max_rqs * sizeof(*nic_dev->rxqs);
2611         nic_dev->rxqs = kzalloc_aligned(rxq_size, GFP_KERNEL);
2612         if (!nic_dev->rxqs) {
2613                 /* free txqs */
2614                 kfree(nic_dev->txqs);
2615                 nic_dev->txqs = NULL;
2616
2617                 PMD_DRV_LOG(ERR, "Allocate rxqs failed");
2618                 return -ENOMEM;
2619         }
2620
2621         return HINIC_OK;
2622 }
2623
2624 static void hinic_deinit_sw_rxtxqs(struct hinic_nic_dev *nic_dev)
2625 {
2626         kfree(nic_dev->txqs);
2627         nic_dev->txqs = NULL;
2628
2629         kfree(nic_dev->rxqs);
2630         nic_dev->rxqs = NULL;
2631 }
2632
2633 static int hinic_nic_dev_create(struct rte_eth_dev *eth_dev)
2634 {
2635         struct hinic_nic_dev *nic_dev =
2636                                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
2637         int rc;
2638
2639         nic_dev->hwdev = rte_zmalloc("hinic_hwdev", sizeof(*nic_dev->hwdev),
2640                                      RTE_CACHE_LINE_SIZE);
2641         if (!nic_dev->hwdev) {
2642                 PMD_DRV_LOG(ERR, "Allocate hinic hwdev memory failed, dev_name: %s",
2643                             eth_dev->data->name);
2644                 return -ENOMEM;
2645         }
2646         nic_dev->hwdev->pcidev_hdl = RTE_ETH_DEV_TO_PCI(eth_dev);
2647
2648         /* init osdep*/
2649         rc = hinic_osdep_init(nic_dev->hwdev);
2650         if (rc) {
2651                 PMD_DRV_LOG(ERR, "Initialize os_dep failed, dev_name: %s",
2652                             eth_dev->data->name);
2653                 goto init_osdep_fail;
2654         }
2655
2656         /* init_hwif */
2657         rc = hinic_hwif_res_init(nic_dev->hwdev);
2658         if (rc) {
2659                 PMD_DRV_LOG(ERR, "Initialize hwif failed, dev_name: %s",
2660                             eth_dev->data->name);
2661                 goto init_hwif_fail;
2662         }
2663
2664         /* init_cfg_mgmt */
2665         rc = init_cfg_mgmt(nic_dev->hwdev);
2666         if (rc) {
2667                 PMD_DRV_LOG(ERR, "Initialize cfg_mgmt failed, dev_name: %s",
2668                             eth_dev->data->name);
2669                 goto init_cfgmgnt_fail;
2670         }
2671
2672         /* init_aeqs */
2673         rc = hinic_comm_aeqs_init(nic_dev->hwdev);
2674         if (rc) {
2675                 PMD_DRV_LOG(ERR, "Initialize aeqs failed, dev_name: %s",
2676                             eth_dev->data->name);
2677                 goto init_aeqs_fail;
2678         }
2679
2680         /* init_pf_to_mgnt */
2681         rc = hinic_comm_pf_to_mgmt_init(nic_dev->hwdev);
2682         if (rc) {
2683                 PMD_DRV_LOG(ERR, "Initialize pf_to_mgmt failed, dev_name: %s",
2684                             eth_dev->data->name);
2685                 goto init_pf_to_mgmt_fail;
2686         }
2687
2688         /* init mailbox */
2689         rc = hinic_comm_func_to_func_init(nic_dev->hwdev);
2690         if (rc) {
2691                 PMD_DRV_LOG(ERR, "Initialize func_to_func failed, dev_name: %s",
2692                             eth_dev->data->name);
2693                 goto init_func_to_func_fail;
2694         }
2695
2696         rc = hinic_card_workmode_check(nic_dev);
2697         if (rc) {
2698                 PMD_DRV_LOG(ERR, "Check card workmode failed, dev_name: %s",
2699                             eth_dev->data->name);
2700                 goto workmode_check_fail;
2701         }
2702
2703         /* do l2nic reset to make chip clear */
2704         rc = hinic_l2nic_reset(nic_dev->hwdev);
2705         if (rc) {
2706                 PMD_DRV_LOG(ERR, "Do l2nic reset failed, dev_name: %s",
2707                             eth_dev->data->name);
2708                 goto l2nic_reset_fail;
2709         }
2710
2711         /* init dma and aeq msix attribute table */
2712         (void)hinic_init_attr_table(nic_dev->hwdev);
2713
2714         /* init_cmdqs */
2715         rc = hinic_comm_cmdqs_init(nic_dev->hwdev);
2716         if (rc) {
2717                 PMD_DRV_LOG(ERR, "Initialize cmdq failed, dev_name: %s",
2718                             eth_dev->data->name);
2719                 goto init_cmdq_fail;
2720         }
2721
2722         /* set hardware state active */
2723         rc = hinic_activate_hwdev_state(nic_dev->hwdev);
2724         if (rc) {
2725                 PMD_DRV_LOG(ERR, "Initialize resources state failed, dev_name: %s",
2726                             eth_dev->data->name);
2727                 goto init_resources_state_fail;
2728         }
2729
2730         /* init_capability */
2731         rc = hinic_init_capability(nic_dev->hwdev);
2732         if (rc) {
2733                 PMD_DRV_LOG(ERR, "Initialize capability failed, dev_name: %s",
2734                             eth_dev->data->name);
2735                 goto init_cap_fail;
2736         }
2737
2738         /* get nic capability */
2739         if (!hinic_support_nic(nic_dev->hwdev, &nic_dev->nic_cap))
2740                 goto nic_check_fail;
2741
2742         /* init root cla and function table */
2743         rc = hinic_init_nicio(nic_dev->hwdev);
2744         if (rc) {
2745                 PMD_DRV_LOG(ERR, "Initialize nic_io failed, dev_name: %s",
2746                             eth_dev->data->name);
2747                 goto init_nicio_fail;
2748         }
2749
2750         /* init_software_txrxq */
2751         rc = hinic_init_sw_rxtxqs(nic_dev);
2752         if (rc) {
2753                 PMD_DRV_LOG(ERR, "Initialize sw_rxtxqs failed, dev_name: %s",
2754                             eth_dev->data->name);
2755                 goto init_sw_rxtxqs_fail;
2756         }
2757
2758         rc = hinic_copy_mempool_init(nic_dev);
2759         if (rc) {
2760                 PMD_DRV_LOG(ERR, "Create copy mempool failed, dev_name: %s",
2761                          eth_dev->data->name);
2762                 goto init_mpool_fail;
2763         }
2764
2765         /* set hardware feature to default status */
2766         rc = hinic_set_default_hw_feature(nic_dev);
2767         if (rc) {
2768                 PMD_DRV_LOG(ERR, "Initialize hardware default features failed, dev_name: %s",
2769                             eth_dev->data->name);
2770                 goto set_default_hw_feature_fail;
2771         }
2772
2773         return 0;
2774
2775 set_default_hw_feature_fail:
2776         hinic_copy_mempool_uninit(nic_dev);
2777
2778 init_mpool_fail:
2779         hinic_deinit_sw_rxtxqs(nic_dev);
2780
2781 init_sw_rxtxqs_fail:
2782         hinic_deinit_nicio(nic_dev->hwdev);
2783
2784 nic_check_fail:
2785 init_nicio_fail:
2786 init_cap_fail:
2787         hinic_deactivate_hwdev_state(nic_dev->hwdev);
2788
2789 init_resources_state_fail:
2790         hinic_comm_cmdqs_free(nic_dev->hwdev);
2791
2792 init_cmdq_fail:
2793 l2nic_reset_fail:
2794 workmode_check_fail:
2795         hinic_comm_func_to_func_free(nic_dev->hwdev);
2796
2797 init_func_to_func_fail:
2798         hinic_comm_pf_to_mgmt_free(nic_dev->hwdev);
2799
2800 init_pf_to_mgmt_fail:
2801         hinic_comm_aeqs_free(nic_dev->hwdev);
2802
2803 init_aeqs_fail:
2804         free_cfg_mgmt(nic_dev->hwdev);
2805
2806 init_cfgmgnt_fail:
2807         hinic_hwif_res_free(nic_dev->hwdev);
2808
2809 init_hwif_fail:
2810         hinic_osdep_deinit(nic_dev->hwdev);
2811
2812 init_osdep_fail:
2813         rte_free(nic_dev->hwdev);
2814         nic_dev->hwdev = NULL;
2815
2816         return rc;
2817 }
2818
2819 static void hinic_nic_dev_destroy(struct rte_eth_dev *eth_dev)
2820 {
2821         struct hinic_nic_dev *nic_dev =
2822                         HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
2823
2824         (void)hinic_set_link_status_follow(nic_dev->hwdev,
2825                                            HINIC_LINK_FOLLOW_DEFAULT);
2826         hinic_copy_mempool_uninit(nic_dev);
2827         hinic_deinit_sw_rxtxqs(nic_dev);
2828         hinic_deinit_nicio(nic_dev->hwdev);
2829         hinic_deactivate_hwdev_state(nic_dev->hwdev);
2830         hinic_comm_cmdqs_free(nic_dev->hwdev);
2831         hinic_comm_func_to_func_free(nic_dev->hwdev);
2832         hinic_comm_pf_to_mgmt_free(nic_dev->hwdev);
2833         hinic_comm_aeqs_free(nic_dev->hwdev);
2834         free_cfg_mgmt(nic_dev->hwdev);
2835         hinic_hwif_res_free(nic_dev->hwdev);
2836         hinic_osdep_deinit(nic_dev->hwdev);
2837         rte_free(nic_dev->hwdev);
2838         nic_dev->hwdev = NULL;
2839 }
2840
2841 /**
2842  * DPDK callback to close the device.
2843  *
2844  * @param dev
2845  *   Pointer to Ethernet device structure.
2846  */
2847 static void hinic_dev_close(struct rte_eth_dev *dev)
2848 {
2849         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
2850
2851         if (hinic_test_and_set_bit(HINIC_DEV_CLOSE, &nic_dev->dev_status)) {
2852                 PMD_DRV_LOG(WARNING, "Device %s already closed",
2853                             dev->data->name);
2854                 return;
2855         }
2856
2857         /* stop device first */
2858         hinic_dev_stop(dev);
2859
2860         /* rx_cqe, rx_info */
2861         hinic_free_all_rx_resources(dev);
2862
2863         /* tx_info */
2864         hinic_free_all_tx_resources(dev);
2865
2866         /* free wq, pi_dma_addr */
2867         hinic_free_all_rq(nic_dev);
2868
2869         /* free wq, db_addr */
2870         hinic_free_all_sq(nic_dev);
2871
2872         /* deinit mac vlan tbl */
2873         hinic_deinit_mac_addr(dev);
2874         hinic_remove_all_vlanid(dev);
2875
2876         /* disable hardware and uio interrupt */
2877         hinic_disable_interrupt(dev);
2878
2879         /* deinit nic hardware device */
2880         hinic_nic_dev_destroy(dev);
2881 }
2882
2883 static const struct eth_dev_ops hinic_pmd_ops = {
2884         .dev_configure                 = hinic_dev_configure,
2885         .dev_infos_get                 = hinic_dev_infos_get,
2886         .fw_version_get                = hinic_fw_version_get,
2887         .rx_queue_setup                = hinic_rx_queue_setup,
2888         .tx_queue_setup                = hinic_tx_queue_setup,
2889         .dev_start                     = hinic_dev_start,
2890         .dev_set_link_up               = hinic_dev_set_link_up,
2891         .dev_set_link_down             = hinic_dev_set_link_down,
2892         .link_update                   = hinic_link_update,
2893         .rx_queue_release              = hinic_rx_queue_release,
2894         .tx_queue_release              = hinic_tx_queue_release,
2895         .dev_stop                      = hinic_dev_stop,
2896         .dev_close                     = hinic_dev_close,
2897         .mtu_set                       = hinic_dev_set_mtu,
2898         .vlan_filter_set               = hinic_vlan_filter_set,
2899         .vlan_offload_set              = hinic_vlan_offload_set,
2900         .allmulticast_enable           = hinic_dev_allmulticast_enable,
2901         .allmulticast_disable          = hinic_dev_allmulticast_disable,
2902         .promiscuous_enable            = hinic_dev_promiscuous_enable,
2903         .promiscuous_disable           = hinic_dev_promiscuous_disable,
2904         .rss_hash_update               = hinic_rss_hash_update,
2905         .rss_hash_conf_get             = hinic_rss_conf_get,
2906         .reta_update                   = hinic_rss_indirtbl_update,
2907         .reta_query                    = hinic_rss_indirtbl_query,
2908         .stats_get                     = hinic_dev_stats_get,
2909         .stats_reset                   = hinic_dev_stats_reset,
2910         .xstats_get                    = hinic_dev_xstats_get,
2911         .xstats_reset                  = hinic_dev_xstats_reset,
2912         .xstats_get_names              = hinic_dev_xstats_get_names,
2913         .rxq_info_get                  = hinic_rxq_info_get,
2914         .txq_info_get                  = hinic_txq_info_get,
2915         .mac_addr_set                  = hinic_set_mac_addr,
2916         .mac_addr_remove               = hinic_mac_addr_remove,
2917         .mac_addr_add                  = hinic_mac_addr_add,
2918         .set_mc_addr_list              = hinic_set_mc_addr_list,
2919         .filter_ctrl                   = hinic_dev_filter_ctrl,
2920 };
2921
2922 static const struct eth_dev_ops hinic_pmd_vf_ops = {
2923         .dev_configure                 = hinic_dev_configure,
2924         .dev_infos_get                 = hinic_dev_infos_get,
2925         .fw_version_get                = hinic_fw_version_get,
2926         .rx_queue_setup                = hinic_rx_queue_setup,
2927         .tx_queue_setup                = hinic_tx_queue_setup,
2928         .dev_start                     = hinic_dev_start,
2929         .link_update                   = hinic_link_update,
2930         .rx_queue_release              = hinic_rx_queue_release,
2931         .tx_queue_release              = hinic_tx_queue_release,
2932         .dev_stop                      = hinic_dev_stop,
2933         .dev_close                     = hinic_dev_close,
2934         .mtu_set                       = hinic_dev_set_mtu,
2935         .vlan_filter_set               = hinic_vlan_filter_set,
2936         .vlan_offload_set              = hinic_vlan_offload_set,
2937         .allmulticast_enable           = hinic_dev_allmulticast_enable,
2938         .allmulticast_disable          = hinic_dev_allmulticast_disable,
2939         .rss_hash_update               = hinic_rss_hash_update,
2940         .rss_hash_conf_get             = hinic_rss_conf_get,
2941         .reta_update                   = hinic_rss_indirtbl_update,
2942         .reta_query                    = hinic_rss_indirtbl_query,
2943         .stats_get                     = hinic_dev_stats_get,
2944         .stats_reset                   = hinic_dev_stats_reset,
2945         .xstats_get                    = hinic_dev_xstats_get,
2946         .xstats_reset                  = hinic_dev_xstats_reset,
2947         .xstats_get_names              = hinic_dev_xstats_get_names,
2948         .rxq_info_get                  = hinic_rxq_info_get,
2949         .txq_info_get                  = hinic_txq_info_get,
2950         .mac_addr_set                  = hinic_set_mac_addr,
2951         .mac_addr_remove               = hinic_mac_addr_remove,
2952         .mac_addr_add                  = hinic_mac_addr_add,
2953         .set_mc_addr_list              = hinic_set_mc_addr_list,
2954         .filter_ctrl                   = hinic_dev_filter_ctrl,
2955 };
2956
2957 static int hinic_func_init(struct rte_eth_dev *eth_dev)
2958 {
2959         struct rte_pci_device *pci_dev;
2960         struct rte_ether_addr *eth_addr;
2961         struct hinic_nic_dev *nic_dev;
2962         struct hinic_filter_info *filter_info;
2963         u32 mac_size;
2964         int rc;
2965
2966         pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
2967
2968         /* EAL is SECONDARY and eth_dev is already created */
2969         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2970                 PMD_DRV_LOG(INFO, "Initialize %s in secondary process",
2971                             eth_dev->data->name);
2972
2973                 return 0;
2974         }
2975
2976         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
2977         memset(nic_dev, 0, sizeof(*nic_dev));
2978
2979         snprintf(nic_dev->proc_dev_name,
2980                  sizeof(nic_dev->proc_dev_name),
2981                  "hinic-%.4x:%.2x:%.2x.%x",
2982                  pci_dev->addr.domain, pci_dev->addr.bus,
2983                  pci_dev->addr.devid, pci_dev->addr.function);
2984
2985         /* alloc mac_addrs */
2986         mac_size = HINIC_MAX_UC_MAC_ADDRS * sizeof(struct rte_ether_addr);
2987         eth_addr = rte_zmalloc("hinic_mac", mac_size, 0);
2988         if (!eth_addr) {
2989                 PMD_DRV_LOG(ERR, "Allocate ethernet addresses' memory failed, dev_name: %s",
2990                             eth_dev->data->name);
2991                 rc = -ENOMEM;
2992                 goto eth_addr_fail;
2993         }
2994         eth_dev->data->mac_addrs = eth_addr;
2995
2996         mac_size = HINIC_MAX_MC_MAC_ADDRS * sizeof(struct rte_ether_addr);
2997         nic_dev->mc_list = rte_zmalloc("hinic_mc", mac_size, 0);
2998         if (!nic_dev->mc_list) {
2999                 PMD_DRV_LOG(ERR, "Allocate mcast address' memory failed, dev_name: %s",
3000                             eth_dev->data->name);
3001                 rc = -ENOMEM;
3002                 goto mc_addr_fail;
3003         }
3004
3005         /*
3006          * Pass the information to the rte_eth_dev_close() that it should also
3007          * release the private port resources.
3008          */
3009         eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
3010
3011         /* create hardware nic_device */
3012         rc = hinic_nic_dev_create(eth_dev);
3013         if (rc) {
3014                 PMD_DRV_LOG(ERR, "Create nic device failed, dev_name: %s",
3015                             eth_dev->data->name);
3016                 goto create_nic_dev_fail;
3017         }
3018
3019         if (HINIC_IS_VF(nic_dev->hwdev))
3020                 eth_dev->dev_ops = &hinic_pmd_vf_ops;
3021         else
3022                 eth_dev->dev_ops = &hinic_pmd_ops;
3023
3024         rc = hinic_init_mac_addr(eth_dev);
3025         if (rc) {
3026                 PMD_DRV_LOG(ERR, "Initialize mac table failed, dev_name: %s",
3027                             eth_dev->data->name);
3028                 goto init_mac_fail;
3029         }
3030
3031         /* register callback func to eal lib */
3032         rc = rte_intr_callback_register(&pci_dev->intr_handle,
3033                                         hinic_dev_interrupt_handler,
3034                                         (void *)eth_dev);
3035         if (rc) {
3036                 PMD_DRV_LOG(ERR, "Register rte interrupt callback failed, dev_name: %s",
3037                             eth_dev->data->name);
3038                 goto reg_intr_cb_fail;
3039         }
3040
3041         /* enable uio/vfio intr/eventfd mapping */
3042         rc = rte_intr_enable(&pci_dev->intr_handle);
3043         if (rc) {
3044                 PMD_DRV_LOG(ERR, "Enable rte interrupt failed, dev_name: %s",
3045                             eth_dev->data->name);
3046                 goto enable_intr_fail;
3047         }
3048         hinic_set_bit(HINIC_DEV_INTR_EN, &nic_dev->dev_status);
3049
3050         /* initialize filter info */
3051         filter_info = &nic_dev->filter;
3052         memset(filter_info, 0, sizeof(struct hinic_filter_info));
3053         /* initialize 5tuple filter list */
3054         TAILQ_INIT(&filter_info->fivetuple_list);
3055         TAILQ_INIT(&nic_dev->filter_ntuple_list);
3056         TAILQ_INIT(&nic_dev->filter_ethertype_list);
3057         TAILQ_INIT(&nic_dev->filter_fdir_rule_list);
3058         TAILQ_INIT(&nic_dev->hinic_flow_list);
3059
3060         hinic_set_bit(HINIC_DEV_INIT, &nic_dev->dev_status);
3061         PMD_DRV_LOG(INFO, "Initialize %s in primary successfully",
3062                     eth_dev->data->name);
3063
3064         return 0;
3065
3066 enable_intr_fail:
3067         (void)rte_intr_callback_unregister(&pci_dev->intr_handle,
3068                                            hinic_dev_interrupt_handler,
3069                                            (void *)eth_dev);
3070
3071 reg_intr_cb_fail:
3072         hinic_deinit_mac_addr(eth_dev);
3073
3074 init_mac_fail:
3075         eth_dev->dev_ops = NULL;
3076         hinic_nic_dev_destroy(eth_dev);
3077
3078 create_nic_dev_fail:
3079         rte_free(nic_dev->mc_list);
3080         nic_dev->mc_list = NULL;
3081
3082 mc_addr_fail:
3083         rte_free(eth_addr);
3084         eth_dev->data->mac_addrs = NULL;
3085
3086 eth_addr_fail:
3087         PMD_DRV_LOG(ERR, "Initialize %s in primary failed",
3088                     eth_dev->data->name);
3089         return rc;
3090 }
3091
3092 static int hinic_dev_init(struct rte_eth_dev *eth_dev)
3093 {
3094         struct rte_pci_device *pci_dev;
3095
3096         pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
3097
3098         PMD_DRV_LOG(INFO, "Initializing pf hinic-%.4x:%.2x:%.2x.%x in %s process",
3099                     pci_dev->addr.domain, pci_dev->addr.bus,
3100                     pci_dev->addr.devid, pci_dev->addr.function,
3101                     (rte_eal_process_type() == RTE_PROC_PRIMARY) ?
3102                     "primary" : "secondary");
3103
3104         /* rte_eth_dev rx_burst and tx_burst */
3105         eth_dev->rx_pkt_burst = hinic_recv_pkts;
3106         eth_dev->tx_pkt_burst = hinic_xmit_pkts;
3107
3108         return hinic_func_init(eth_dev);
3109 }
3110
3111 static int hinic_dev_uninit(struct rte_eth_dev *dev)
3112 {
3113         struct hinic_nic_dev *nic_dev;
3114
3115         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
3116         hinic_clear_bit(HINIC_DEV_INIT, &nic_dev->dev_status);
3117
3118         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
3119                 return 0;
3120
3121         hinic_dev_close(dev);
3122
3123         dev->dev_ops = NULL;
3124         dev->rx_pkt_burst = NULL;
3125         dev->tx_pkt_burst = NULL;
3126
3127         rte_free(nic_dev->mc_list);
3128
3129         rte_free(dev->data->mac_addrs);
3130         dev->data->mac_addrs = NULL;
3131
3132         return HINIC_OK;
3133 }
3134
3135 static struct rte_pci_id pci_id_hinic_map[] = {
3136         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_PRD) },
3137         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_MEZZ_25GE) },
3138         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_MEZZ_40GE) },
3139         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_MEZZ_100GE) },
3140         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_VF) },
3141         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_VF_HV) },
3142         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_1822_DUAL_25GE) },
3143         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_1822_100GE) },
3144         {.vendor_id = 0},
3145 };
3146
3147 static int hinic_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
3148                            struct rte_pci_device *pci_dev)
3149 {
3150         return rte_eth_dev_pci_generic_probe(pci_dev,
3151                 sizeof(struct hinic_nic_dev), hinic_dev_init);
3152 }
3153
3154 static int hinic_pci_remove(struct rte_pci_device *pci_dev)
3155 {
3156         return rte_eth_dev_pci_generic_remove(pci_dev, hinic_dev_uninit);
3157 }
3158
3159 static struct rte_pci_driver rte_hinic_pmd = {
3160         .id_table = pci_id_hinic_map,
3161         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
3162         .probe = hinic_pci_probe,
3163         .remove = hinic_pci_remove,
3164 };
3165
3166 RTE_PMD_REGISTER_PCI(net_hinic, rte_hinic_pmd);
3167 RTE_PMD_REGISTER_PCI_TABLE(net_hinic, pci_id_hinic_map);
3168
3169 RTE_INIT(hinic_init_log)
3170 {
3171         hinic_logtype = rte_log_register("pmd.net.hinic");
3172         if (hinic_logtype >= 0)
3173                 rte_log_set_level(hinic_logtype, RTE_LOG_INFO);
3174 }