drivers/net/hinic/hinic_pmd_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Huawei Technologies Co., Ltd
   3  */
   4
   5 #include <rte_pci.h>
   6 #include <rte_bus_pci.h>
   7 #include <rte_ethdev_pci.h>
   8 #include <rte_mbuf.h>
   9 #include <rte_malloc.h>
  10 #include <rte_memcpy.h>
  11 #include <rte_mempool.h>
  12 #include <rte_errno.h>
  13
  14 #include "base/hinic_compat.h"
  15 #include "base/hinic_pmd_hwdev.h"
  16 #include "base/hinic_pmd_hwif.h"
  17 #include "base/hinic_pmd_wq.h"
  18 #include "base/hinic_pmd_cfg.h"
  19 #include "base/hinic_pmd_mgmt.h"
  20 #include "base/hinic_pmd_cmdq.h"
  21 #include "base/hinic_pmd_niccfg.h"
  22 #include "base/hinic_pmd_nicio.h"
  23 #include "hinic_pmd_ethdev.h"
  24 #include "hinic_pmd_tx.h"
  25 #include "hinic_pmd_rx.h"
  26
  27 /* Vendor ID used by Huawei devices */
  28 #define HINIC_HUAWEI_VENDOR_ID          0x19E5
  29
  30 /* Hinic devices */
  31 #define HINIC_DEV_ID_PRD                0x1822
  32 #define HINIC_DEV_ID_MEZZ_25GE          0x0210
  33 #define HINIC_DEV_ID_MEZZ_40GE          0x020D
  34 #define HINIC_DEV_ID_MEZZ_100GE         0x0205
  35
  36 #define HINIC_SERVICE_MODE_NIC          2
  37
  38 #define HINIC_INTR_CB_UNREG_MAX_RETRIES         10
  39
  40 #define DEFAULT_BASE_COS                4
  41 #define NR_MAX_COS                      8
  42
  43 #define HINIC_MIN_RX_BUF_SIZE           1024
  44 #define HINIC_MAX_MAC_ADDRS             1
  45
  46 /** Driver-specific log messages type. */
  47 int hinic_logtype;
  48
  49 struct hinic_xstats_name_off {
  50         char name[RTE_ETH_XSTATS_NAME_SIZE];
  51         u32  offset;
  52 };
  53
  54 #define HINIC_FUNC_STAT(_stat_item) {   \
  55         .name = #_stat_item, \
  56         .offset = offsetof(struct hinic_vport_stats, _stat_item) \
  57 }
  58
  59 #define HINIC_PORT_STAT(_stat_item) { \
  60         .name = #_stat_item, \
  61         .offset = offsetof(struct hinic_phy_port_stats, _stat_item) \
  62 }
  63
  64 static const struct hinic_xstats_name_off hinic_vport_stats_strings[] = {
  65         HINIC_FUNC_STAT(tx_unicast_pkts_vport),
  66         HINIC_FUNC_STAT(tx_unicast_bytes_vport),
  67         HINIC_FUNC_STAT(tx_multicast_pkts_vport),
  68         HINIC_FUNC_STAT(tx_multicast_bytes_vport),
  69         HINIC_FUNC_STAT(tx_broadcast_pkts_vport),
  70         HINIC_FUNC_STAT(tx_broadcast_bytes_vport),
  71
  72         HINIC_FUNC_STAT(rx_unicast_pkts_vport),
  73         HINIC_FUNC_STAT(rx_unicast_bytes_vport),
  74         HINIC_FUNC_STAT(rx_multicast_pkts_vport),
  75         HINIC_FUNC_STAT(rx_multicast_bytes_vport),
  76         HINIC_FUNC_STAT(rx_broadcast_pkts_vport),
  77         HINIC_FUNC_STAT(rx_broadcast_bytes_vport),
  78
  79         HINIC_FUNC_STAT(tx_discard_vport),
  80         HINIC_FUNC_STAT(rx_discard_vport),
  81         HINIC_FUNC_STAT(tx_err_vport),
  82         HINIC_FUNC_STAT(rx_err_vport),
  83 };
  84
  85 #define HINIC_VPORT_XSTATS_NUM (sizeof(hinic_vport_stats_strings) / \
  86                 sizeof(hinic_vport_stats_strings[0]))
  87
  88 static const struct hinic_xstats_name_off hinic_phyport_stats_strings[] = {
  89         HINIC_PORT_STAT(mac_rx_total_pkt_num),
  90         HINIC_PORT_STAT(mac_rx_total_oct_num),
  91         HINIC_PORT_STAT(mac_rx_bad_pkt_num),
  92         HINIC_PORT_STAT(mac_rx_bad_oct_num),
  93         HINIC_PORT_STAT(mac_rx_good_pkt_num),
  94         HINIC_PORT_STAT(mac_rx_good_oct_num),
  95         HINIC_PORT_STAT(mac_rx_uni_pkt_num),
  96         HINIC_PORT_STAT(mac_rx_multi_pkt_num),
  97         HINIC_PORT_STAT(mac_rx_broad_pkt_num),
  98         HINIC_PORT_STAT(mac_tx_total_pkt_num),
  99         HINIC_PORT_STAT(mac_tx_total_oct_num),
 100         HINIC_PORT_STAT(mac_tx_bad_pkt_num),
 101         HINIC_PORT_STAT(mac_tx_bad_oct_num),
 102         HINIC_PORT_STAT(mac_tx_good_pkt_num),
 103         HINIC_PORT_STAT(mac_tx_good_oct_num),
 104         HINIC_PORT_STAT(mac_tx_uni_pkt_num),
 105         HINIC_PORT_STAT(mac_tx_multi_pkt_num),
 106         HINIC_PORT_STAT(mac_tx_broad_pkt_num),
 107         HINIC_PORT_STAT(mac_rx_fragment_pkt_num),
 108         HINIC_PORT_STAT(mac_rx_undersize_pkt_num),
 109         HINIC_PORT_STAT(mac_rx_undermin_pkt_num),
 110         HINIC_PORT_STAT(mac_rx_64_oct_pkt_num),
 111         HINIC_PORT_STAT(mac_rx_65_127_oct_pkt_num),
 112         HINIC_PORT_STAT(mac_rx_128_255_oct_pkt_num),
 113         HINIC_PORT_STAT(mac_rx_256_511_oct_pkt_num),
 114         HINIC_PORT_STAT(mac_rx_512_1023_oct_pkt_num),
 115         HINIC_PORT_STAT(mac_rx_1024_1518_oct_pkt_num),
 116         HINIC_PORT_STAT(mac_rx_1519_2047_oct_pkt_num),
 117         HINIC_PORT_STAT(mac_rx_2048_4095_oct_pkt_num),
 118         HINIC_PORT_STAT(mac_rx_4096_8191_oct_pkt_num),
 119         HINIC_PORT_STAT(mac_rx_8192_9216_oct_pkt_num),
 120         HINIC_PORT_STAT(mac_rx_9217_12287_oct_pkt_num),
 121         HINIC_PORT_STAT(mac_rx_12288_16383_oct_pkt_num),
 122         HINIC_PORT_STAT(mac_rx_1519_max_bad_pkt_num),
 123         HINIC_PORT_STAT(mac_rx_1519_max_good_pkt_num),
 124         HINIC_PORT_STAT(mac_rx_oversize_pkt_num),
 125         HINIC_PORT_STAT(mac_rx_jabber_pkt_num),
 126         HINIC_PORT_STAT(mac_rx_mac_pause_num),
 127         HINIC_PORT_STAT(mac_rx_pfc_pkt_num),
 128         HINIC_PORT_STAT(mac_rx_pfc_pri0_pkt_num),
 129         HINIC_PORT_STAT(mac_rx_pfc_pri1_pkt_num),
 130         HINIC_PORT_STAT(mac_rx_pfc_pri2_pkt_num),
 131         HINIC_PORT_STAT(mac_rx_pfc_pri3_pkt_num),
 132         HINIC_PORT_STAT(mac_rx_pfc_pri4_pkt_num),
 133         HINIC_PORT_STAT(mac_rx_pfc_pri5_pkt_num),
 134         HINIC_PORT_STAT(mac_rx_pfc_pri6_pkt_num),
 135         HINIC_PORT_STAT(mac_rx_pfc_pri7_pkt_num),
 136         HINIC_PORT_STAT(mac_rx_mac_control_pkt_num),
 137         HINIC_PORT_STAT(mac_rx_sym_err_pkt_num),
 138         HINIC_PORT_STAT(mac_rx_fcs_err_pkt_num),
 139         HINIC_PORT_STAT(mac_rx_send_app_good_pkt_num),
 140         HINIC_PORT_STAT(mac_rx_send_app_bad_pkt_num),
 141         HINIC_PORT_STAT(mac_tx_fragment_pkt_num),
 142         HINIC_PORT_STAT(mac_tx_undersize_pkt_num),
 143         HINIC_PORT_STAT(mac_tx_undermin_pkt_num),
 144         HINIC_PORT_STAT(mac_tx_64_oct_pkt_num),
 145         HINIC_PORT_STAT(mac_tx_65_127_oct_pkt_num),
 146         HINIC_PORT_STAT(mac_tx_128_255_oct_pkt_num),
 147         HINIC_PORT_STAT(mac_tx_256_511_oct_pkt_num),
 148         HINIC_PORT_STAT(mac_tx_512_1023_oct_pkt_num),
 149         HINIC_PORT_STAT(mac_tx_1024_1518_oct_pkt_num),
 150         HINIC_PORT_STAT(mac_tx_1519_2047_oct_pkt_num),
 151         HINIC_PORT_STAT(mac_tx_2048_4095_oct_pkt_num),
 152         HINIC_PORT_STAT(mac_tx_4096_8191_oct_pkt_num),
 153         HINIC_PORT_STAT(mac_tx_8192_9216_oct_pkt_num),
 154         HINIC_PORT_STAT(mac_tx_9217_12287_oct_pkt_num),
 155         HINIC_PORT_STAT(mac_tx_12288_16383_oct_pkt_num),
 156         HINIC_PORT_STAT(mac_tx_1519_max_bad_pkt_num),
 157         HINIC_PORT_STAT(mac_tx_1519_max_good_pkt_num),
 158         HINIC_PORT_STAT(mac_tx_oversize_pkt_num),
 159         HINIC_PORT_STAT(mac_trans_jabber_pkt_num),
 160         HINIC_PORT_STAT(mac_tx_mac_pause_num),
 161         HINIC_PORT_STAT(mac_tx_pfc_pkt_num),
 162         HINIC_PORT_STAT(mac_tx_pfc_pri0_pkt_num),
 163         HINIC_PORT_STAT(mac_tx_pfc_pri1_pkt_num),
 164         HINIC_PORT_STAT(mac_tx_pfc_pri2_pkt_num),
 165         HINIC_PORT_STAT(mac_tx_pfc_pri3_pkt_num),
 166         HINIC_PORT_STAT(mac_tx_pfc_pri4_pkt_num),
 167         HINIC_PORT_STAT(mac_tx_pfc_pri5_pkt_num),
 168         HINIC_PORT_STAT(mac_tx_pfc_pri6_pkt_num),
 169         HINIC_PORT_STAT(mac_tx_pfc_pri7_pkt_num),
 170         HINIC_PORT_STAT(mac_tx_mac_control_pkt_num),
 171         HINIC_PORT_STAT(mac_tx_err_all_pkt_num),
 172         HINIC_PORT_STAT(mac_tx_from_app_good_pkt_num),
 173         HINIC_PORT_STAT(mac_tx_from_app_bad_pkt_num),
 174 };
 175
 176 #define HINIC_PHYPORT_XSTATS_NUM (sizeof(hinic_phyport_stats_strings) / \
 177                 sizeof(hinic_phyport_stats_strings[0]))
 178
 179 static const struct hinic_xstats_name_off hinic_rxq_stats_strings[] = {
 180         {"rx_nombuf", offsetof(struct hinic_rxq_stats, rx_nombuf)},
 181         {"burst_pkt", offsetof(struct hinic_rxq_stats, burst_pkts)},
 182 };
 183
 184 #define HINIC_RXQ_XSTATS_NUM (sizeof(hinic_rxq_stats_strings) / \
 185                 sizeof(hinic_rxq_stats_strings[0]))
 186
 187 static const struct hinic_xstats_name_off hinic_txq_stats_strings[] = {
 188         {"tx_busy", offsetof(struct hinic_txq_stats, tx_busy)},
 189         {"offload_errors", offsetof(struct hinic_txq_stats, off_errs)},
 190         {"copy_pkts", offsetof(struct hinic_txq_stats, cpy_pkts)},
 191         {"rl_drop", offsetof(struct hinic_txq_stats, rl_drop)},
 192         {"burst_pkts", offsetof(struct hinic_txq_stats, burst_pkts)},
 193 };
 194
 195 #define HINIC_TXQ_XSTATS_NUM (sizeof(hinic_txq_stats_strings) / \
 196                 sizeof(hinic_txq_stats_strings[0]))
 197
 198 static int hinic_xstats_calc_num(struct hinic_nic_dev *nic_dev)
 199 {
 200         return (HINIC_VPORT_XSTATS_NUM +
 201                 HINIC_PHYPORT_XSTATS_NUM +
 202                 HINIC_RXQ_XSTATS_NUM * nic_dev->num_rq +
 203                 HINIC_TXQ_XSTATS_NUM * nic_dev->num_sq);
 204 }
 205
 206 static const struct rte_eth_desc_lim hinic_rx_desc_lim = {
 207         .nb_max = HINIC_MAX_QUEUE_DEPTH,
 208         .nb_min = HINIC_MIN_QUEUE_DEPTH,
 209         .nb_align = HINIC_RXD_ALIGN,
 210 };
 211
 212 static const struct rte_eth_desc_lim hinic_tx_desc_lim = {
 213         .nb_max = HINIC_MAX_QUEUE_DEPTH,
 214         .nb_min = HINIC_MIN_QUEUE_DEPTH,
 215         .nb_align = HINIC_TXD_ALIGN,
 216 };
 217
 218
 219 /**
 220  * Interrupt handler triggered by NIC  for handling
 221  * specific event.
 222  *
 223  * @param: The address of parameter (struct rte_eth_dev *) regsitered before.
 224  **/
 225 static void hinic_dev_interrupt_handler(void *param)
 226 {
 227         struct rte_eth_dev *dev = param;
 228         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 229
 230         if (!hinic_test_bit(HINIC_DEV_INTR_EN, &nic_dev->dev_status)) {
 231                 PMD_DRV_LOG(WARNING, "Device's interrupt is disabled, ignore interrupt event, dev_name: %s, port_id: %d",
 232                             nic_dev->proc_dev_name, dev->data->port_id);
 233                 return;
 234         }
 235
 236         /* aeq0 msg handler */
 237         hinic_dev_handle_aeq_event(nic_dev->hwdev, param);
 238 }
 239
 240 /**
 241  * Ethernet device configuration.
 242  *
 243  * Prepare the driver for a given number of TX and RX queues, mtu size
 244  * and configure RSS.
 245  *
 246  * @param dev
 247  *   Pointer to Ethernet device structure.
 248  *
 249  * @return
 250  *   0 on success, negative error value otherwise.
 251  */
 252 static int hinic_dev_configure(struct rte_eth_dev *dev)
 253 {
 254         struct hinic_nic_dev *nic_dev;
 255         struct hinic_nic_io *nic_io;
 256         int err;
 257
 258         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 259         nic_io = nic_dev->hwdev->nic_io;
 260
 261         nic_dev->num_sq =  dev->data->nb_tx_queues;
 262         nic_dev->num_rq = dev->data->nb_rx_queues;
 263
 264         nic_io->num_sqs =  dev->data->nb_tx_queues;
 265         nic_io->num_rqs = dev->data->nb_rx_queues;
 266
 267         /* queue pair is max_num(sq, rq) */
 268         nic_dev->num_qps = (nic_dev->num_sq > nic_dev->num_rq) ?
 269                         nic_dev->num_sq : nic_dev->num_rq;
 270         nic_io->num_qps = nic_dev->num_qps;
 271
 272         if (nic_dev->num_qps > nic_io->max_qps) {
 273                 PMD_DRV_LOG(ERR,
 274                         "Queue number out of range, get queue_num:%d, max_queue_num:%d",
 275                         nic_dev->num_qps, nic_io->max_qps);
 276                 return -EINVAL;
 277         }
 278
 279         /* mtu size is 256~9600 */
 280         if (dev->data->dev_conf.rxmode.max_rx_pkt_len < HINIC_MIN_FRAME_SIZE ||
 281             dev->data->dev_conf.rxmode.max_rx_pkt_len >
 282             HINIC_MAX_JUMBO_FRAME_SIZE) {
 283                 PMD_DRV_LOG(ERR,
 284                         "Max rx pkt len out of range, get max_rx_pkt_len:%d, "
 285                         "expect between %d and %d",
 286                         dev->data->dev_conf.rxmode.max_rx_pkt_len,
 287                         HINIC_MIN_FRAME_SIZE, HINIC_MAX_JUMBO_FRAME_SIZE);
 288                 return -EINVAL;
 289         }
 290
 291         nic_dev->mtu_size =
 292                 HINIC_PKTLEN_TO_MTU(dev->data->dev_conf.rxmode.max_rx_pkt_len);
 293
 294         /* rss template */
 295         err = hinic_config_mq_mode(dev, TRUE);
 296         if (err) {
 297                 PMD_DRV_LOG(ERR, "Config multi-queue failed");
 298                 return err;
 299         }
 300
 301         return HINIC_OK;
 302 }
 303
 304 /**
 305  * DPDK callback to create the receive queue.
 306  *
 307  * @param dev
 308  *   Pointer to Ethernet device structure.
 309  * @param queue_idx
 310  *   RX queue index.
 311  * @param nb_desc
 312  *   Number of descriptors for receive queue.
 313  * @param socket_id
 314  *   NUMA socket on which memory must be allocated.
 315  * @param rx_conf
 316  *   Thresholds parameters (unused_).
 317  * @param mp
 318  *   Memory pool for buffer allocations.
 319  *
 320  * @return
 321  *   0 on success, negative error value otherwise.
 322  */
 323 static int hinic_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 324                          uint16_t nb_desc, unsigned int socket_id,
 325                          __rte_unused const struct rte_eth_rxconf *rx_conf,
 326                          struct rte_mempool *mp)
 327 {
 328         int rc;
 329         struct hinic_nic_dev *nic_dev;
 330         struct hinic_hwdev *hwdev;
 331         struct hinic_rxq *rxq;
 332         u16 rq_depth, rx_free_thresh;
 333         u32 buf_size;
 334
 335         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 336         hwdev = nic_dev->hwdev;
 337
 338         /* queue depth must be power of 2, otherwise will be aligned up */
 339         rq_depth = (nb_desc & (nb_desc - 1)) ?
 340                 ((u16)(1U << (ilog2(nb_desc) + 1))) : nb_desc;
 341
 342         /*
 343          * Validate number of receive descriptors.
 344          * It must not exceed hardware maximum and minimum.
 345          */
 346         if (rq_depth > HINIC_MAX_QUEUE_DEPTH ||
 347                 rq_depth < HINIC_MIN_QUEUE_DEPTH) {
 348                 PMD_DRV_LOG(ERR, "RX queue depth is out of range from %d to %d, (nb_desc=%d, q_depth=%d, port=%d queue=%d)",
 349                             HINIC_MIN_QUEUE_DEPTH, HINIC_MAX_QUEUE_DEPTH,
 350                             (int)nb_desc, (int)rq_depth,
 351                             (int)dev->data->port_id, (int)queue_idx);
 352                 return -EINVAL;
 353         }
 354
 355         /*
 356          * The RX descriptor ring will be cleaned after rxq->rx_free_thresh
 357          * descriptors are used or if the number of descriptors required
 358          * to transmit a packet is greater than the number of free RX
 359          * descriptors.
 360          * The following constraints must be satisfied:
 361          *  rx_free_thresh must be greater than 0.
 362          *  rx_free_thresh must be less than the size of the ring minus 1.
 363          * When set to zero use default values.
 364          */
 365         rx_free_thresh = (u16)((rx_conf->rx_free_thresh) ?
 366                         rx_conf->rx_free_thresh : HINIC_DEFAULT_RX_FREE_THRESH);
 367         if (rx_free_thresh >= (rq_depth - 1)) {
 368                 PMD_DRV_LOG(ERR, "rx_free_thresh must be less than the number of RX descriptors minus 1. (rx_free_thresh=%u port=%d queue=%d)",
 369                             (unsigned int)rx_free_thresh,
 370                             (int)dev->data->port_id,
 371                             (int)queue_idx);
 372                 return -EINVAL;
 373         }
 374
 375         rxq = rte_zmalloc_socket("hinic_rx_queue", sizeof(struct hinic_rxq),
 376                                  RTE_CACHE_LINE_SIZE, socket_id);
 377         if (!rxq) {
 378                 PMD_DRV_LOG(ERR, "Allocate rxq[%d] failed, dev_name: %s",
 379                             queue_idx, dev->data->name);
 380                 return -ENOMEM;
 381         }
 382         nic_dev->rxqs[queue_idx] = rxq;
 383
 384         /* alloc rx sq hw wqepage*/
 385         rc = hinic_create_rq(hwdev, queue_idx, rq_depth);
 386         if (rc) {
 387                 PMD_DRV_LOG(ERR, "Create rxq[%d] failed, dev_name: %s, rq_depth: %d",
 388                             queue_idx, dev->data->name, rq_depth);
 389                 goto ceate_rq_fail;
 390         }
 391
 392         /* mbuf pool must be assigned before setup rx resources */
 393         rxq->mb_pool = mp;
 394
 395         rc =
 396         hinic_convert_rx_buf_size(rte_pktmbuf_data_room_size(rxq->mb_pool) -
 397                                   RTE_PKTMBUF_HEADROOM, &buf_size);
 398         if (rc) {
 399                 PMD_DRV_LOG(ERR, "Adjust buf size failed, dev_name: %s",
 400                             dev->data->name);
 401                 goto adjust_bufsize_fail;
 402         }
 403
 404         /* rx queue info, rearm control */
 405         rxq->wq = &hwdev->nic_io->rq_wq[queue_idx];
 406         rxq->pi_virt_addr = hwdev->nic_io->qps[queue_idx].rq.pi_virt_addr;
 407         rxq->nic_dev = nic_dev;
 408         rxq->q_id = queue_idx;
 409         rxq->q_depth = rq_depth;
 410         rxq->buf_len = (u16)buf_size;
 411         rxq->rx_free_thresh = rx_free_thresh;
 412
 413         /* the last point cant do mbuf rearm in bulk */
 414         rxq->rxinfo_align_end = rxq->q_depth - rxq->rx_free_thresh;
 415
 416         /* device port identifier */
 417         rxq->port_id = dev->data->port_id;
 418
 419         /* alloc rx_cqe and prepare rq_wqe */
 420         rc = hinic_setup_rx_resources(rxq);
 421         if (rc) {
 422                 PMD_DRV_LOG(ERR, "Setup rxq[%d] rx_resources failed, dev_name:%s",
 423                             queue_idx, dev->data->name);
 424                 goto setup_rx_res_err;
 425         }
 426
 427         /* record nic_dev rxq in rte_eth rx_queues */
 428         dev->data->rx_queues[queue_idx] = rxq;
 429
 430         return 0;
 431
 432 setup_rx_res_err:
 433 adjust_bufsize_fail:
 434         hinic_destroy_rq(hwdev, queue_idx);
 435
 436 ceate_rq_fail:
 437         rte_free(rxq);
 438
 439         return rc;
 440 }
 441
 442 static void hinic_reset_rx_queue(struct rte_eth_dev *dev)
 443 {
 444         struct hinic_rxq *rxq;
 445         struct hinic_nic_dev *nic_dev;
 446         int q_id = 0;
 447
 448         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 449
 450         for (q_id = 0; q_id < nic_dev->num_rq; q_id++) {
 451                 rxq = dev->data->rx_queues[q_id];
 452
 453                 rxq->wq->cons_idx = 0;
 454                 rxq->wq->prod_idx = 0;
 455                 rxq->wq->delta = rxq->q_depth;
 456                 rxq->wq->mask = rxq->q_depth - 1;
 457
 458                 /* alloc mbuf to rq */
 459                 hinic_rx_alloc_pkts(rxq);
 460         }
 461 }
 462
 463 /**
 464  * DPDK callback to configure the transmit queue.
 465  *
 466  * @param dev
 467  *   Pointer to Ethernet device structure.
 468  * @param queue_idx
 469  *   Transmit queue index.
 470  * @param nb_desc
 471  *   Number of descriptors for transmit queue.
 472  * @param socket_id
 473  *   NUMA socket on which memory must be allocated.
 474  * @param tx_conf
 475  *   Tx queue configuration parameters.
 476  *
 477  * @return
 478  *   0 on success, negative error value otherwise.
 479  */
 480 static int hinic_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 481                          uint16_t nb_desc, unsigned int socket_id,
 482                          __rte_unused const struct rte_eth_txconf *tx_conf)
 483 {
 484         int rc;
 485         struct hinic_nic_dev *nic_dev;
 486         struct hinic_hwdev *hwdev;
 487         struct hinic_txq *txq;
 488         u16 sq_depth, tx_free_thresh;
 489
 490         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 491         hwdev = nic_dev->hwdev;
 492
 493         /* queue depth must be power of 2, otherwise will be aligned up */
 494         sq_depth = (nb_desc & (nb_desc - 1)) ?
 495                         ((u16)(1U << (ilog2(nb_desc) + 1))) : nb_desc;
 496
 497         /*
 498          * Validate number of transmit descriptors.
 499          * It must not exceed hardware maximum and minimum.
 500          */
 501         if (sq_depth > HINIC_MAX_QUEUE_DEPTH ||
 502                 sq_depth < HINIC_MIN_QUEUE_DEPTH) {
 503                 PMD_DRV_LOG(ERR, "TX queue depth is out of range from %d to %d, (nb_desc=%d, q_depth=%d, port=%d queue=%d)",
 504                           HINIC_MIN_QUEUE_DEPTH, HINIC_MAX_QUEUE_DEPTH,
 505                           (int)nb_desc, (int)sq_depth,
 506                           (int)dev->data->port_id, (int)queue_idx);
 507                 return -EINVAL;
 508         }
 509
 510         /*
 511          * The TX descriptor ring will be cleaned after txq->tx_free_thresh
 512          * descriptors are used or if the number of descriptors required
 513          * to transmit a packet is greater than the number of free TX
 514          * descriptors.
 515          * The following constraints must be satisfied:
 516          *  tx_free_thresh must be greater than 0.
 517          *  tx_free_thresh must be less than the size of the ring minus 1.
 518          * When set to zero use default values.
 519          */
 520         tx_free_thresh = (u16)((tx_conf->tx_free_thresh) ?
 521                         tx_conf->tx_free_thresh : HINIC_DEFAULT_TX_FREE_THRESH);
 522         if (tx_free_thresh >= (sq_depth - 1)) {
 523                 PMD_DRV_LOG(ERR, "tx_free_thresh must be less than the number of TX descriptors minus 1. (tx_free_thresh=%u port=%d queue=%d)",
 524                         (unsigned int)tx_free_thresh, (int)dev->data->port_id,
 525                         (int)queue_idx);
 526                 return -EINVAL;
 527         }
 528
 529         txq = rte_zmalloc_socket("hinic_tx_queue", sizeof(struct hinic_txq),
 530                                  RTE_CACHE_LINE_SIZE, socket_id);
 531         if (!txq) {
 532                 PMD_DRV_LOG(ERR, "Allocate txq[%d] failed, dev_name: %s",
 533                             queue_idx, dev->data->name);
 534                 return -ENOMEM;
 535         }
 536         nic_dev->txqs[queue_idx] = txq;
 537
 538         /* alloc tx sq hw wqepage */
 539         rc = hinic_create_sq(hwdev, queue_idx, sq_depth);
 540         if (rc) {
 541                 PMD_DRV_LOG(ERR, "Create txq[%d] failed, dev_name: %s, sq_depth: %d",
 542                             queue_idx, dev->data->name, sq_depth);
 543                 goto create_sq_fail;
 544         }
 545
 546         txq->q_id = queue_idx;
 547         txq->q_depth = sq_depth;
 548         txq->port_id = dev->data->port_id;
 549         txq->tx_free_thresh = tx_free_thresh;
 550         txq->nic_dev = nic_dev;
 551         txq->wq = &hwdev->nic_io->sq_wq[queue_idx];
 552         txq->sq = &hwdev->nic_io->qps[queue_idx].sq;
 553         txq->cons_idx_addr = hwdev->nic_io->qps[queue_idx].sq.cons_idx_addr;
 554         txq->sq_head_addr = HINIC_GET_WQ_HEAD(txq);
 555         txq->sq_bot_sge_addr = HINIC_GET_WQ_TAIL(txq) -
 556                                         sizeof(struct hinic_sq_bufdesc);
 557         txq->cos = nic_dev->default_cos;
 558
 559         /* alloc software txinfo */
 560         rc = hinic_setup_tx_resources(txq);
 561         if (rc) {
 562                 PMD_DRV_LOG(ERR, "Setup txq[%d] tx_resources failed, dev_name: %s",
 563                             queue_idx, dev->data->name);
 564                 goto setup_tx_res_fail;
 565         }
 566
 567         /* record nic_dev txq in rte_eth tx_queues */
 568         dev->data->tx_queues[queue_idx] = txq;
 569
 570         return HINIC_OK;
 571
 572 setup_tx_res_fail:
 573         hinic_destroy_sq(hwdev, queue_idx);
 574
 575 create_sq_fail:
 576         rte_free(txq);
 577
 578         return rc;
 579 }
 580
 581 static void hinic_reset_tx_queue(struct rte_eth_dev *dev)
 582 {
 583         struct hinic_nic_dev *nic_dev;
 584         struct hinic_txq *txq;
 585         struct hinic_nic_io *nic_io;
 586         struct hinic_hwdev *hwdev;
 587         volatile u32 *ci_addr;
 588         int q_id = 0;
 589
 590         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 591         hwdev = nic_dev->hwdev;
 592         nic_io = hwdev->nic_io;
 593
 594         for (q_id = 0; q_id < nic_dev->num_sq; q_id++) {
 595                 txq = dev->data->tx_queues[q_id];
 596
 597                 txq->wq->cons_idx = 0;
 598                 txq->wq->prod_idx = 0;
 599                 txq->wq->delta = txq->q_depth;
 600                 txq->wq->mask  = txq->q_depth - 1;
 601
 602                 /*clear hardware ci*/
 603                 ci_addr = (volatile u32 *)HINIC_CI_VADDR(nic_io->ci_vaddr_base,
 604                                                         q_id);
 605                 *ci_addr = 0;
 606         }
 607 }
 608
 609 /**
 610  * Get link speed from NIC.
 611  *
 612  * @param dev
 613  *   Pointer to Ethernet device structure.
 614  * @param speed_capa
 615  *   Pointer to link speed structure.
 616  */
 617 static void hinic_get_speed_capa(struct rte_eth_dev *dev, uint32_t *speed_capa)
 618 {
 619         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 620         u32 supported_link, advertised_link;
 621         int err;
 622
 623 #define HINIC_LINK_MODE_SUPPORT_1G      (1U << HINIC_GE_BASE_KX)
 624
 625 #define HINIC_LINK_MODE_SUPPORT_10G     (1U << HINIC_10GE_BASE_KR)
 626
 627 #define HINIC_LINK_MODE_SUPPORT_25G     ((1U << HINIC_25GE_BASE_KR_S) | \
 628                                         (1U << HINIC_25GE_BASE_CR_S) | \
 629                                         (1U << HINIC_25GE_BASE_KR) | \
 630                                         (1U << HINIC_25GE_BASE_CR))
 631
 632 #define HINIC_LINK_MODE_SUPPORT_40G     ((1U << HINIC_40GE_BASE_KR4) | \
 633                                         (1U << HINIC_40GE_BASE_CR4))
 634
 635 #define HINIC_LINK_MODE_SUPPORT_100G    ((1U << HINIC_100GE_BASE_KR4) | \
 636                                         (1U << HINIC_100GE_BASE_CR4))
 637
 638         err = hinic_get_link_mode(nic_dev->hwdev,
 639                                   &supported_link, &advertised_link);
 640         if (err || supported_link == HINIC_SUPPORTED_UNKNOWN ||
 641             advertised_link == HINIC_SUPPORTED_UNKNOWN) {
 642                 PMD_DRV_LOG(WARNING, "Get speed capability info failed, device: %s, port_id: %u",
 643                           nic_dev->proc_dev_name, dev->data->port_id);
 644         } else {
 645                 *speed_capa = 0;
 646                 if (!!(supported_link & HINIC_LINK_MODE_SUPPORT_1G))
 647                         *speed_capa |= ETH_LINK_SPEED_1G;
 648                 if (!!(supported_link & HINIC_LINK_MODE_SUPPORT_10G))
 649                         *speed_capa |= ETH_LINK_SPEED_10G;
 650                 if (!!(supported_link & HINIC_LINK_MODE_SUPPORT_25G))
 651                         *speed_capa |= ETH_LINK_SPEED_25G;
 652                 if (!!(supported_link & HINIC_LINK_MODE_SUPPORT_40G))
 653                         *speed_capa |= ETH_LINK_SPEED_40G;
 654                 if (!!(supported_link & HINIC_LINK_MODE_SUPPORT_100G))
 655                         *speed_capa |= ETH_LINK_SPEED_100G;
 656         }
 657 }
 658
 659 /**
 660  * DPDK callback to get information about the device.
 661  *
 662  * @param dev
 663  *   Pointer to Ethernet device structure.
 664  * @param info
 665  *   Pointer to Info structure output buffer.
 666  */
 667 static int
 668 hinic_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
 669 {
 670         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 671
 672         info->max_rx_queues  = nic_dev->nic_cap.max_rqs;
 673         info->max_tx_queues  = nic_dev->nic_cap.max_sqs;
 674         info->min_rx_bufsize = HINIC_MIN_RX_BUF_SIZE;
 675         info->max_rx_pktlen  = HINIC_MAX_JUMBO_FRAME_SIZE;
 676         info->max_mac_addrs  = HINIC_MAX_MAC_ADDRS;
 677
 678         hinic_get_speed_capa(dev, &info->speed_capa);
 679         info->rx_queue_offload_capa = 0;
 680         info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
 681                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 682                                 DEV_RX_OFFLOAD_UDP_CKSUM |
 683                                 DEV_RX_OFFLOAD_TCP_CKSUM;
 684
 685         info->tx_queue_offload_capa = 0;
 686         info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
 687                                 DEV_TX_OFFLOAD_IPV4_CKSUM |
 688                                 DEV_TX_OFFLOAD_UDP_CKSUM |
 689                                 DEV_TX_OFFLOAD_TCP_CKSUM |
 690                                 DEV_TX_OFFLOAD_SCTP_CKSUM |
 691                                 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
 692                                 DEV_TX_OFFLOAD_TCP_TSO |
 693                                 DEV_TX_OFFLOAD_MULTI_SEGS;
 694
 695         info->hash_key_size = HINIC_RSS_KEY_SIZE;
 696         info->reta_size = HINIC_RSS_INDIR_SIZE;
 697         info->flow_type_rss_offloads = HINIC_RSS_OFFLOAD_ALL;
 698         info->rx_desc_lim = hinic_rx_desc_lim;
 699         info->tx_desc_lim = hinic_tx_desc_lim;
 700
 701         return 0;
 702 }
 703
 704 static int hinic_config_rx_mode(struct hinic_nic_dev *nic_dev, u32 rx_mode_ctrl)
 705 {
 706         int err;
 707
 708         err = hinic_set_rx_mode(nic_dev->hwdev, rx_mode_ctrl);
 709         if (err) {
 710                 PMD_DRV_LOG(ERR, "Failed to set rx mode");
 711                 return -EINVAL;
 712         }
 713         nic_dev->rx_mode_status = rx_mode_ctrl;
 714
 715         return 0;
 716 }
 717
 718
 719 static int hinic_rxtx_configure(struct rte_eth_dev *dev)
 720 {
 721         int err;
 722         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 723
 724         /* rx configure, if rss enable, need to init default configuration */
 725         err = hinic_rx_configure(dev);
 726         if (err) {
 727                 PMD_DRV_LOG(ERR, "Configure rss failed");
 728                 return err;
 729         }
 730
 731         /* rx mode init */
 732         err = hinic_config_rx_mode(nic_dev, HINIC_DEFAULT_RX_MODE);
 733         if (err) {
 734                 PMD_DRV_LOG(ERR, "Configure rx_mode:0x%x failed",
 735                         HINIC_DEFAULT_RX_MODE);
 736                 goto set_rx_mode_fail;
 737         }
 738
 739         return HINIC_OK;
 740
 741 set_rx_mode_fail:
 742         hinic_rx_remove_configure(dev);
 743
 744         return err;
 745 }
 746
 747 static void hinic_remove_rxtx_configure(struct rte_eth_dev *dev)
 748 {
 749         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 750
 751         (void)hinic_config_rx_mode(nic_dev, 0);
 752         hinic_rx_remove_configure(dev);
 753 }
 754
 755 static int hinic_priv_get_dev_link_status(struct hinic_nic_dev *nic_dev,
 756                                           struct rte_eth_link *link)
 757 {
 758         int rc;
 759         u8 port_link_status = 0;
 760         struct nic_port_info port_link_info;
 761         struct hinic_hwdev *nic_hwdev = nic_dev->hwdev;
 762         uint32_t port_speed[LINK_SPEED_MAX] = {ETH_SPEED_NUM_10M,
 763                                         ETH_SPEED_NUM_100M, ETH_SPEED_NUM_1G,
 764                                         ETH_SPEED_NUM_10G, ETH_SPEED_NUM_25G,
 765                                         ETH_SPEED_NUM_40G, ETH_SPEED_NUM_100G};
 766
 767         rc = hinic_get_link_status(nic_hwdev, &port_link_status);
 768         if (rc)
 769                 return rc;
 770
 771         if (!port_link_status) {
 772                 link->link_status = ETH_LINK_DOWN;
 773                 link->link_speed = 0;
 774                 link->link_duplex = ETH_LINK_HALF_DUPLEX;
 775                 link->link_autoneg = ETH_LINK_FIXED;
 776                 return HINIC_OK;
 777         }
 778
 779         memset(&port_link_info, 0, sizeof(port_link_info));
 780         rc = hinic_get_port_info(nic_hwdev, &port_link_info);
 781         if (rc)
 782                 return rc;
 783
 784         link->link_speed = port_speed[port_link_info.speed % LINK_SPEED_MAX];
 785         link->link_duplex = port_link_info.duplex;
 786         link->link_autoneg = port_link_info.autoneg_state;
 787         link->link_status = port_link_status;
 788
 789         return HINIC_OK;
 790 }
 791
 792 /**
 793  * DPDK callback to retrieve physical link information.
 794  *
 795  * @param dev
 796  *   Pointer to Ethernet device structure.
 797  * @param wait_to_complete
 798  *   Wait for request completion.
 799  *
 800  * @return
 801  *   0 link status changed, -1 link status not changed
 802  */
 803 static int hinic_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 804 {
 805 #define CHECK_INTERVAL 10  /* 10ms */
 806 #define MAX_REPEAT_TIME 100  /* 1s (100 * 10ms) in total */
 807         int rc = HINIC_OK;
 808         struct rte_eth_link link;
 809         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 810         unsigned int rep_cnt = MAX_REPEAT_TIME;
 811
 812         memset(&link, 0, sizeof(link));
 813         do {
 814                 /* Get link status information from hardware */
 815                 rc = hinic_priv_get_dev_link_status(nic_dev, &link);
 816                 if (rc != HINIC_OK) {
 817                         link.link_speed = ETH_SPEED_NUM_NONE;
 818                         link.link_duplex = ETH_LINK_FULL_DUPLEX;
 819                         PMD_DRV_LOG(ERR, "Get link status failed");
 820                         goto out;
 821                 }
 822
 823                 if (!wait_to_complete || link.link_status)
 824                         break;
 825
 826                 rte_delay_ms(CHECK_INTERVAL);
 827         } while (rep_cnt--);
 828
 829 out:
 830         rc = rte_eth_linkstatus_set(dev, &link);
 831         return rc;
 832 }
 833
 834 /**
 835  * DPDK callback to start the device.
 836  *
 837  * @param dev
 838  *   Pointer to Ethernet device structure.
 839  *
 840  * @return
 841  *   0 on success, negative errno value on failure.
 842  */
 843 static int hinic_dev_start(struct rte_eth_dev *dev)
 844 {
 845         int rc;
 846         char *name;
 847         struct hinic_nic_dev *nic_dev;
 848
 849         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 850         name = dev->data->name;
 851
 852         /* reset rx and tx queue */
 853         hinic_reset_rx_queue(dev);
 854         hinic_reset_tx_queue(dev);
 855
 856         /* get func rx buf size */
 857         hinic_get_func_rx_buf_size(nic_dev);
 858
 859         /* init txq and rxq context */
 860         rc = hinic_init_qp_ctxts(nic_dev->hwdev);
 861         if (rc) {
 862                 PMD_DRV_LOG(ERR, "Initialize qp context failed, dev_name:%s",
 863                             name);
 864                 goto init_qp_fail;
 865         }
 866
 867         /* rss template */
 868         rc = hinic_config_mq_mode(dev, TRUE);
 869         if (rc) {
 870                 PMD_DRV_LOG(ERR, "Configure mq mode failed, dev_name: %s",
 871                             name);
 872                 goto cfg_mq_mode_fail;
 873         }
 874
 875         /* set default mtu */
 876         rc = hinic_set_port_mtu(nic_dev->hwdev, nic_dev->mtu_size);
 877         if (rc) {
 878                 PMD_DRV_LOG(ERR, "Set mtu_size[%d] failed, dev_name: %s",
 879                             nic_dev->mtu_size, name);
 880                 goto set_mtu_fail;
 881         }
 882
 883         /* configure rss rx_mode and other rx or tx default feature */
 884         rc = hinic_rxtx_configure(dev);
 885         if (rc) {
 886                 PMD_DRV_LOG(ERR, "Configure tx and rx failed, dev_name: %s",
 887                             name);
 888                 goto cfg_rxtx_fail;
 889         }
 890
 891         /* open virtual port and ready to start packet receiving */
 892         rc = hinic_set_vport_enable(nic_dev->hwdev, true);
 893         if (rc) {
 894                 PMD_DRV_LOG(ERR, "Enable vport failed, dev_name:%s", name);
 895                 goto en_vport_fail;
 896         }
 897
 898         /* open physical port and start packet receiving */
 899         rc = hinic_set_port_enable(nic_dev->hwdev, true);
 900         if (rc) {
 901                 PMD_DRV_LOG(ERR, "Enable physical port failed, dev_name:%s",
 902                             name);
 903                 goto en_port_fail;
 904         }
 905
 906         /* update eth_dev link status */
 907         if (dev->data->dev_conf.intr_conf.lsc != 0)
 908                 (void)hinic_link_update(dev, 0);
 909
 910         hinic_set_bit(HINIC_DEV_START, &nic_dev->dev_status);
 911
 912         return 0;
 913
 914 en_port_fail:
 915         (void)hinic_set_vport_enable(nic_dev->hwdev, false);
 916
 917 en_vport_fail:
 918         /* Flush tx && rx chip resources in case of set vport fake fail */
 919         (void)hinic_flush_qp_res(nic_dev->hwdev);
 920         rte_delay_ms(100);
 921
 922         hinic_remove_rxtx_configure(dev);
 923
 924 cfg_rxtx_fail:
 925 set_mtu_fail:
 926 cfg_mq_mode_fail:
 927         hinic_free_qp_ctxts(nic_dev->hwdev);
 928
 929 init_qp_fail:
 930         hinic_free_all_rx_mbuf(dev);
 931         hinic_free_all_tx_mbuf(dev);
 932
 933         return rc;
 934 }
 935
 936 /**
 937  * DPDK callback to release the receive queue.
 938  *
 939  * @param queue
 940  *   Generic receive queue pointer.
 941  */
 942 static void hinic_rx_queue_release(void *queue)
 943 {
 944         struct hinic_rxq *rxq = queue;
 945         struct hinic_nic_dev *nic_dev;
 946
 947         if (!rxq) {
 948                 PMD_DRV_LOG(WARNING, "Rxq is null when release");
 949                 return;
 950         }
 951         nic_dev = rxq->nic_dev;
 952
 953         /* free rxq_pkt mbuf */
 954         hinic_free_all_rx_skbs(rxq);
 955
 956         /* free rxq_cqe, rxq_info */
 957         hinic_free_rx_resources(rxq);
 958
 959         /* free root rq wq */
 960         hinic_destroy_rq(nic_dev->hwdev, rxq->q_id);
 961
 962         nic_dev->rxqs[rxq->q_id] = NULL;
 963
 964         /* free rxq */
 965         rte_free(rxq);
 966 }
 967
 968 /**
 969  * DPDK callback to release the transmit queue.
 970  *
 971  * @param queue
 972  *   Generic transmit queue pointer.
 973  */
 974 static void hinic_tx_queue_release(void *queue)
 975 {
 976         struct hinic_txq *txq = queue;
 977         struct hinic_nic_dev *nic_dev;
 978
 979         if (!txq) {
 980                 PMD_DRV_LOG(WARNING, "Txq is null when release");
 981                 return;
 982         }
 983         nic_dev = txq->nic_dev;
 984
 985         /* free txq_pkt mbuf */
 986         hinic_free_all_tx_skbs(txq);
 987
 988         /* free txq_info */
 989         hinic_free_tx_resources(txq);
 990
 991         /* free root sq wq */
 992         hinic_destroy_sq(nic_dev->hwdev, txq->q_id);
 993         nic_dev->txqs[txq->q_id] = NULL;
 994
 995         /* free txq */
 996         rte_free(txq);
 997 }
 998
 999 static void hinic_free_all_rq(struct hinic_nic_dev *nic_dev)
1000 {
1001         u16 q_id;
1002
1003         for (q_id = 0; q_id < nic_dev->num_rq; q_id++)
1004                 hinic_destroy_rq(nic_dev->hwdev, q_id);
1005 }
1006
1007 static void hinic_free_all_sq(struct hinic_nic_dev *nic_dev)
1008 {
1009         u16 q_id;
1010
1011         for (q_id = 0; q_id < nic_dev->num_sq; q_id++)
1012                 hinic_destroy_sq(nic_dev->hwdev, q_id);
1013 }
1014
1015 /**
1016  * DPDK callback to stop the device.
1017  *
1018  * @param dev
1019  *   Pointer to Ethernet device structure.
1020  */
1021 static void hinic_dev_stop(struct rte_eth_dev *dev)
1022 {
1023         int rc;
1024         char *name;
1025         uint16_t port_id;
1026         struct hinic_nic_dev *nic_dev;
1027         struct rte_eth_link link;
1028
1029         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1030         name = dev->data->name;
1031         port_id = dev->data->port_id;
1032
1033         if (!hinic_test_and_clear_bit(HINIC_DEV_START, &nic_dev->dev_status)) {
1034                 PMD_DRV_LOG(INFO, "Device %s already stopped", name);
1035                 return;
1036         }
1037
1038         /* just stop phy port and vport */
1039         rc = hinic_set_port_enable(nic_dev->hwdev, false);
1040         if (rc)
1041                 PMD_DRV_LOG(WARNING, "Disable phy port failed, error: %d, dev_name:%s, port_id:%d",
1042                           rc, name, port_id);
1043
1044         rc = hinic_set_vport_enable(nic_dev->hwdev, false);
1045         if (rc)
1046                 PMD_DRV_LOG(WARNING, "Disable vport failed, error: %d, dev_name:%s, port_id:%d",
1047                           rc, name, port_id);
1048
1049         /* Clear recorded link status */
1050         memset(&link, 0, sizeof(link));
1051         (void)rte_eth_linkstatus_set(dev, &link);
1052
1053         /* flush pending io request */
1054         rc = hinic_rx_tx_flush(nic_dev->hwdev);
1055         if (rc)
1056                 PMD_DRV_LOG(WARNING, "Flush pending io failed, error: %d, dev_name: %s, port_id: %d",
1057                             rc, name, port_id);
1058
1059         /* clean rss table and rx_mode */
1060         hinic_remove_rxtx_configure(dev);
1061
1062         /* clean root context */
1063         hinic_free_qp_ctxts(nic_dev->hwdev);
1064
1065         /* free mbuf */
1066         hinic_free_all_rx_mbuf(dev);
1067         hinic_free_all_tx_mbuf(dev);
1068 }
1069
1070 static void hinic_disable_interrupt(struct rte_eth_dev *dev)
1071 {
1072         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1073         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1074         int ret, retries = 0;
1075
1076         hinic_clear_bit(HINIC_DEV_INTR_EN, &nic_dev->dev_status);
1077
1078         /* disable msix interrupt in hardware */
1079         hinic_set_msix_state(nic_dev->hwdev, 0, HINIC_MSIX_DISABLE);
1080
1081         /* disable rte interrupt */
1082         ret = rte_intr_disable(&pci_dev->intr_handle);
1083         if (ret)
1084                 PMD_DRV_LOG(ERR, "Disable intr failed: %d", ret);
1085
1086         do {
1087                 ret =
1088                 rte_intr_callback_unregister(&pci_dev->intr_handle,
1089                                              hinic_dev_interrupt_handler, dev);
1090                 if (ret >= 0) {
1091                         break;
1092                 } else if (ret == -EAGAIN) {
1093                         rte_delay_ms(100);
1094                         retries++;
1095                 } else {
1096                         PMD_DRV_LOG(ERR, "intr callback unregister failed: %d",
1097                                     ret);
1098                         break;
1099                 }
1100         } while (retries < HINIC_INTR_CB_UNREG_MAX_RETRIES);
1101
1102         if (retries == HINIC_INTR_CB_UNREG_MAX_RETRIES)
1103                 PMD_DRV_LOG(ERR, "Unregister intr callback failed after %d retries",
1104                             retries);
1105 }
1106
1107 static int hinic_set_dev_promiscuous(struct hinic_nic_dev *nic_dev, bool enable)
1108 {
1109         u32 rx_mode_ctrl = nic_dev->rx_mode_status;
1110
1111         if (enable)
1112                 rx_mode_ctrl |= HINIC_RX_MODE_PROMISC;
1113         else
1114                 rx_mode_ctrl &= (~HINIC_RX_MODE_PROMISC);
1115
1116         return hinic_config_rx_mode(nic_dev, rx_mode_ctrl);
1117 }
1118
1119 /**
1120  * DPDK callback to get device statistics.
1121  *
1122  * @param dev
1123  *   Pointer to Ethernet device structure.
1124  * @param stats
1125  *   Stats structure output buffer.
1126  *
1127  * @return
1128  *   0 on success and stats is filled,
1129  *   negative error value otherwise.
1130  */
1131 static int
1132 hinic_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1133 {
1134         int i, err, q_num;
1135         u64 rx_discards_pmd = 0;
1136         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1137         struct hinic_vport_stats vport_stats;
1138         struct hinic_rxq        *rxq = NULL;
1139         struct hinic_rxq_stats rxq_stats;
1140         struct hinic_txq        *txq = NULL;
1141         struct hinic_txq_stats txq_stats;
1142
1143         err = hinic_get_vport_stats(nic_dev->hwdev, &vport_stats);
1144         if (err) {
1145                 PMD_DRV_LOG(ERR, "Get vport stats from fw failed, nic_dev: %s",
1146                         nic_dev->proc_dev_name);
1147                 return err;
1148         }
1149
1150         /* rx queue stats */
1151         q_num = (nic_dev->num_rq < RTE_ETHDEV_QUEUE_STAT_CNTRS) ?
1152                         nic_dev->num_rq : RTE_ETHDEV_QUEUE_STAT_CNTRS;
1153         for (i = 0; i < q_num; i++) {
1154                 rxq = nic_dev->rxqs[i];
1155                 hinic_rxq_get_stats(rxq, &rxq_stats);
1156                 stats->q_ipackets[i] = rxq_stats.packets;
1157                 stats->q_ibytes[i] = rxq_stats.bytes;
1158                 stats->q_errors[i] = rxq_stats.rx_discards;
1159
1160                 stats->ierrors += rxq_stats.errors;
1161                 rx_discards_pmd += rxq_stats.rx_discards;
1162                 dev->data->rx_mbuf_alloc_failed += rxq_stats.rx_nombuf;
1163         }
1164
1165         /* tx queue stats */
1166         q_num = (nic_dev->num_sq < RTE_ETHDEV_QUEUE_STAT_CNTRS) ?
1167                 nic_dev->num_sq : RTE_ETHDEV_QUEUE_STAT_CNTRS;
1168         for (i = 0; i < q_num; i++) {
1169                 txq = nic_dev->txqs[i];
1170                 hinic_txq_get_stats(txq, &txq_stats);
1171                 stats->q_opackets[i] = txq_stats.packets;
1172                 stats->q_obytes[i] = txq_stats.bytes;
1173                 stats->oerrors += (txq_stats.tx_busy + txq_stats.off_errs);
1174         }
1175
1176         /* vport stats */
1177         stats->oerrors += vport_stats.tx_discard_vport;
1178
1179         stats->imissed = vport_stats.rx_discard_vport + rx_discards_pmd;
1180
1181         stats->ipackets = (vport_stats.rx_unicast_pkts_vport +
1182                         vport_stats.rx_multicast_pkts_vport +
1183                         vport_stats.rx_broadcast_pkts_vport -
1184                         rx_discards_pmd);
1185
1186         stats->opackets = (vport_stats.tx_unicast_pkts_vport +
1187                         vport_stats.tx_multicast_pkts_vport +
1188                         vport_stats.tx_broadcast_pkts_vport);
1189
1190         stats->ibytes = (vport_stats.rx_unicast_bytes_vport +
1191                         vport_stats.rx_multicast_bytes_vport +
1192                         vport_stats.rx_broadcast_bytes_vport);
1193
1194         stats->obytes = (vport_stats.tx_unicast_bytes_vport +
1195                         vport_stats.tx_multicast_bytes_vport +
1196                         vport_stats.tx_broadcast_bytes_vport);
1197         return 0;
1198 }
1199
1200 /**
1201  * DPDK callback to clear device statistics.
1202  *
1203  * @param dev
1204  *   Pointer to Ethernet device structure.
1205  */
1206 static void hinic_dev_stats_reset(struct rte_eth_dev *dev)
1207 {
1208         int qid;
1209         struct hinic_rxq        *rxq = NULL;
1210         struct hinic_txq        *txq = NULL;
1211         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1212
1213         hinic_clear_vport_stats(nic_dev->hwdev);
1214
1215         for (qid = 0; qid < nic_dev->num_rq; qid++) {
1216                 rxq = nic_dev->rxqs[qid];
1217                 hinic_rxq_stats_reset(rxq);
1218         }
1219
1220         for (qid = 0; qid < nic_dev->num_sq; qid++) {
1221                 txq = nic_dev->txqs[qid];
1222                 hinic_txq_stats_reset(txq);
1223         }
1224 }
1225
1226 /**
1227  * DPDK callback to clear device extended statistics.
1228  *
1229  * @param dev
1230  *   Pointer to Ethernet device structure.
1231  **/
1232 static void hinic_dev_xstats_reset(struct rte_eth_dev *dev)
1233 {
1234         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1235
1236         hinic_dev_stats_reset(dev);
1237
1238         if (hinic_func_type(nic_dev->hwdev) != TYPE_VF)
1239                 hinic_clear_phy_port_stats(nic_dev->hwdev);
1240 }
1241
1242 static void hinic_gen_random_mac_addr(struct rte_ether_addr *mac_addr)
1243 {
1244         uint64_t random_value;
1245
1246         /* Set Organizationally Unique Identifier (OUI) prefix */
1247         mac_addr->addr_bytes[0] = 0x00;
1248         mac_addr->addr_bytes[1] = 0x09;
1249         mac_addr->addr_bytes[2] = 0xC0;
1250         /* Force indication of locally assigned MAC address. */
1251         mac_addr->addr_bytes[0] |= RTE_ETHER_LOCAL_ADMIN_ADDR;
1252         /* Generate the last 3 bytes of the MAC address with a random number. */
1253         random_value = rte_rand();
1254         memcpy(&mac_addr->addr_bytes[3], &random_value, 3);
1255 }
1256
1257 /**
1258  * Init mac_vlan table in NIC.
1259  *
1260  * @param dev
1261  *   Pointer to Ethernet device structure.
1262  *
1263  * @return
1264  *   0 on success and stats is filled,
1265  *   negative error value otherwise.
1266  */
1267 static int hinic_init_mac_addr(struct rte_eth_dev *eth_dev)
1268 {
1269         struct hinic_nic_dev *nic_dev =
1270                                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
1271         uint8_t addr_bytes[RTE_ETHER_ADDR_LEN];
1272         u16 func_id = 0;
1273         int rc = 0;
1274
1275         rc = hinic_get_default_mac(nic_dev->hwdev, addr_bytes);
1276         if (rc)
1277                 return rc;
1278
1279         memmove(eth_dev->data->mac_addrs->addr_bytes,
1280                 addr_bytes, RTE_ETHER_ADDR_LEN);
1281
1282         if (rte_is_zero_ether_addr(eth_dev->data->mac_addrs))
1283                 hinic_gen_random_mac_addr(eth_dev->data->mac_addrs);
1284
1285         func_id = hinic_global_func_id(nic_dev->hwdev);
1286         rc = hinic_set_mac(nic_dev->hwdev, eth_dev->data->mac_addrs->addr_bytes,
1287                            0, func_id);
1288         if (rc && rc != HINIC_PF_SET_VF_ALREADY)
1289                 return rc;
1290
1291         return 0;
1292 }
1293
1294 /**
1295  * Deinit mac_vlan table in NIC.
1296  *
1297  * @param dev
1298  *   Pointer to Ethernet device structure.
1299  *
1300  * @return
1301  *   0 on success and stats is filled,
1302  *   negative error value otherwise.
1303  */
1304 static void hinic_deinit_mac_addr(struct rte_eth_dev *eth_dev)
1305 {
1306         struct hinic_nic_dev *nic_dev =
1307                                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
1308         int rc;
1309         u16 func_id = 0;
1310
1311         if (rte_is_zero_ether_addr(eth_dev->data->mac_addrs))
1312                 return;
1313
1314         func_id = hinic_global_func_id(nic_dev->hwdev);
1315         rc = hinic_del_mac(nic_dev->hwdev,
1316                            eth_dev->data->mac_addrs->addr_bytes,
1317                            0, func_id);
1318         if (rc && rc != HINIC_PF_SET_VF_ALREADY)
1319                 PMD_DRV_LOG(ERR, "Delete mac table failed, dev_name: %s",
1320                             eth_dev->data->name);
1321 }
1322
1323 /**
1324  * DPDK callback to enable promiscuous mode.
1325  *
1326  * @param dev
1327  *   Pointer to Ethernet device structure.
1328  *
1329  * @return
1330  *   0 on success,
1331  *   negative error value otherwise.
1332  */
1333 static int hinic_dev_promiscuous_enable(struct rte_eth_dev *dev)
1334 {
1335         int rc = HINIC_OK;
1336         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1337
1338         PMD_DRV_LOG(INFO, "Enable promiscuous, nic_dev: %s, port_id: %d, promisc: %d",
1339                     nic_dev->proc_dev_name, dev->data->port_id,
1340                     dev->data->promiscuous);
1341
1342         rc = hinic_set_dev_promiscuous(nic_dev, true);
1343         if (rc)
1344                 PMD_DRV_LOG(ERR, "Enable promiscuous failed");
1345
1346         return rc;
1347 }
1348
1349 /**
1350  * DPDK callback to disable promiscuous mode.
1351  *
1352  * @param dev
1353  *   Pointer to Ethernet device structure.
1354  *
1355  * @return
1356  *   0 on success,
1357  *   negative error value otherwise.
1358  */
1359 static int hinic_dev_promiscuous_disable(struct rte_eth_dev *dev)
1360 {
1361         int rc = HINIC_OK;
1362         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1363
1364         PMD_DRV_LOG(INFO, "Disable promiscuous, nic_dev: %s, port_id: %d, promisc: %d",
1365                     nic_dev->proc_dev_name, dev->data->port_id,
1366                     dev->data->promiscuous);
1367
1368         rc = hinic_set_dev_promiscuous(nic_dev, false);
1369         if (rc)
1370                 PMD_DRV_LOG(ERR, "Disable promiscuous failed");
1371
1372         return rc;
1373 }
1374
1375 /**
1376  * DPDK callback to update the RSS hash key and RSS hash type.
1377  *
1378  * @param dev
1379  *   Pointer to Ethernet device structure.
1380  * @param rss_conf
1381  *   RSS configuration data.
1382  *
1383  * @return
1384  *   0 on success, negative error value otherwise.
1385  */
1386 static int hinic_rss_hash_update(struct rte_eth_dev *dev,
1387                           struct rte_eth_rss_conf *rss_conf)
1388 {
1389         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1390         u8 tmpl_idx = nic_dev->rss_tmpl_idx;
1391         u8 hashkey[HINIC_RSS_KEY_SIZE] = {0};
1392         u8 prio_tc[HINIC_DCB_UP_MAX] = {0};
1393         u64 rss_hf = rss_conf->rss_hf;
1394         struct nic_rss_type rss_type = {0};
1395         int err = 0;
1396
1397         if (!(nic_dev->flags & ETH_MQ_RX_RSS_FLAG)) {
1398                 PMD_DRV_LOG(WARNING, "RSS is not enabled");
1399                 return HINIC_OK;
1400         }
1401
1402         if (rss_conf->rss_key_len > HINIC_RSS_KEY_SIZE) {
1403                 PMD_DRV_LOG(ERR, "Invalid rss key, rss_key_len:%d",
1404                             rss_conf->rss_key_len);
1405                 return HINIC_ERROR;
1406         }
1407
1408         if (rss_conf->rss_key) {
1409                 memcpy(hashkey, rss_conf->rss_key, rss_conf->rss_key_len);
1410                 err = hinic_rss_set_template_tbl(nic_dev->hwdev, tmpl_idx,
1411                                                  hashkey);
1412                 if (err) {
1413                         PMD_DRV_LOG(ERR, "Set rss template table failed");
1414                         goto disable_rss;
1415                 }
1416         }
1417
1418         rss_type.ipv4 = (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4)) ? 1 : 0;
1419         rss_type.tcp_ipv4 = (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) ? 1 : 0;
1420         rss_type.ipv6 = (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6)) ? 1 : 0;
1421         rss_type.ipv6_ext = (rss_hf & ETH_RSS_IPV6_EX) ? 1 : 0;
1422         rss_type.tcp_ipv6 = (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) ? 1 : 0;
1423         rss_type.tcp_ipv6_ext = (rss_hf & ETH_RSS_IPV6_TCP_EX) ? 1 : 0;
1424         rss_type.udp_ipv4 = (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) ? 1 : 0;
1425         rss_type.udp_ipv6 = (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) ? 1 : 0;
1426
1427         err = hinic_set_rss_type(nic_dev->hwdev, tmpl_idx, rss_type);
1428         if (err) {
1429                 PMD_DRV_LOG(ERR, "Set rss type table failed");
1430                 goto disable_rss;
1431         }
1432
1433         return 0;
1434
1435 disable_rss:
1436         memset(prio_tc, 0, sizeof(prio_tc));
1437         (void)hinic_rss_cfg(nic_dev->hwdev, 0, tmpl_idx, 0, prio_tc);
1438         return err;
1439 }
1440
1441 /**
1442  * DPDK callback to get the RSS hash configuration.
1443  *
1444  * @param dev
1445  *   Pointer to Ethernet device structure.
1446  * @param rss_conf
1447  *   RSS configuration data.
1448  *
1449  * @return
1450  *   0 on success, negative error value otherwise.
1451  */
1452 static int hinic_rss_conf_get(struct rte_eth_dev *dev,
1453                        struct rte_eth_rss_conf *rss_conf)
1454 {
1455         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1456         u8 tmpl_idx = nic_dev->rss_tmpl_idx;
1457         u8 hashkey[HINIC_RSS_KEY_SIZE] = {0};
1458         struct nic_rss_type rss_type = {0};
1459         int err;
1460
1461         if (!(nic_dev->flags & ETH_MQ_RX_RSS_FLAG)) {
1462                 PMD_DRV_LOG(WARNING, "RSS is not enabled");
1463                 return HINIC_ERROR;
1464         }
1465
1466         err = hinic_rss_get_template_tbl(nic_dev->hwdev, tmpl_idx, hashkey);
1467         if (err)
1468                 return err;
1469
1470         if (rss_conf->rss_key &&
1471             rss_conf->rss_key_len >= HINIC_RSS_KEY_SIZE) {
1472                 memcpy(rss_conf->rss_key, hashkey, sizeof(hashkey));
1473                 rss_conf->rss_key_len = sizeof(hashkey);
1474         }
1475
1476         err = hinic_get_rss_type(nic_dev->hwdev, tmpl_idx, &rss_type);
1477         if (err)
1478                 return err;
1479
1480         rss_conf->rss_hf = 0;
1481         rss_conf->rss_hf |=  rss_type.ipv4 ?
1482                 (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4) : 0;
1483         rss_conf->rss_hf |=  rss_type.tcp_ipv4 ? ETH_RSS_NONFRAG_IPV4_TCP : 0;
1484         rss_conf->rss_hf |=  rss_type.ipv6 ?
1485                 (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6) : 0;
1486         rss_conf->rss_hf |=  rss_type.ipv6_ext ? ETH_RSS_IPV6_EX : 0;
1487         rss_conf->rss_hf |=  rss_type.tcp_ipv6 ? ETH_RSS_NONFRAG_IPV6_TCP : 0;
1488         rss_conf->rss_hf |=  rss_type.tcp_ipv6_ext ? ETH_RSS_IPV6_TCP_EX : 0;
1489         rss_conf->rss_hf |=  rss_type.udp_ipv4 ? ETH_RSS_NONFRAG_IPV4_UDP : 0;
1490         rss_conf->rss_hf |=  rss_type.udp_ipv6 ? ETH_RSS_NONFRAG_IPV6_UDP : 0;
1491
1492         return HINIC_OK;
1493 }
1494
1495 /**
1496  * DPDK callback to update the RETA indirection table.
1497  *
1498  * @param dev
1499  *   Pointer to Ethernet device structure.
1500  * @param reta_conf
1501  *   Pointer to RETA configuration structure array.
1502  * @param reta_size
1503  *   Size of the RETA table.
1504  *
1505  * @return
1506  *   0 on success, negative error value otherwise.
1507  */
1508 static int hinic_rss_indirtbl_update(struct rte_eth_dev *dev,
1509                               struct rte_eth_rss_reta_entry64 *reta_conf,
1510                               uint16_t reta_size)
1511 {
1512         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1513         u8 tmpl_idx = nic_dev->rss_tmpl_idx;
1514         u8 prio_tc[HINIC_DCB_UP_MAX] = {0};
1515         u32 indirtbl[NIC_RSS_INDIR_SIZE] = {0};
1516         int err = 0;
1517         u16 i = 0;
1518         u16 idx, shift;
1519
1520         if (!(nic_dev->flags & ETH_MQ_RX_RSS_FLAG))
1521                 return HINIC_OK;
1522
1523         if (reta_size != NIC_RSS_INDIR_SIZE) {
1524                 PMD_DRV_LOG(ERR, "Invalid reta size, reta_size:%d", reta_size);
1525                 return HINIC_ERROR;
1526         }
1527
1528         err = hinic_rss_get_indir_tbl(nic_dev->hwdev, tmpl_idx, indirtbl);
1529         if (err)
1530                 return err;
1531
1532         /* update rss indir_tbl */
1533         for (i = 0; i < reta_size; i++) {
1534                 idx = i / RTE_RETA_GROUP_SIZE;
1535                 shift = i % RTE_RETA_GROUP_SIZE;
1536                 if (reta_conf[idx].mask & (1ULL << shift))
1537                         indirtbl[i] = reta_conf[idx].reta[shift];
1538         }
1539
1540         for (i = 0 ; i < reta_size; i++) {
1541                 if (indirtbl[i] >= nic_dev->num_rq) {
1542                         PMD_DRV_LOG(ERR, "Invalid reta entry, index:%d, num_rq:%d",
1543                                     i, nic_dev->num_rq);
1544                         goto disable_rss;
1545                 }
1546         }
1547
1548         err = hinic_rss_set_indir_tbl(nic_dev->hwdev, tmpl_idx, indirtbl);
1549         if (err)
1550                 goto disable_rss;
1551
1552         nic_dev->rss_indir_flag = true;
1553
1554         return 0;
1555
1556 disable_rss:
1557         memset(prio_tc, 0, sizeof(prio_tc));
1558         (void)hinic_rss_cfg(nic_dev->hwdev, 0, tmpl_idx, 0, prio_tc);
1559
1560         return HINIC_ERROR;
1561 }
1562
1563
1564 /**
1565  * DPDK callback to get the RETA indirection table.
1566  *
1567  * @param dev
1568  *   Pointer to Ethernet device structure.
1569  * @param reta_conf
1570  *   Pointer to RETA configuration structure array.
1571  * @param reta_size
1572  *   Size of the RETA table.
1573  *
1574  * @return
1575  *   0 on success, negative error value otherwise.
1576  */
1577 static int hinic_rss_indirtbl_query(struct rte_eth_dev *dev,
1578                              struct rte_eth_rss_reta_entry64 *reta_conf,
1579                              uint16_t reta_size)
1580 {
1581         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1582         u8 tmpl_idx = nic_dev->rss_tmpl_idx;
1583         int err = 0;
1584         u32 indirtbl[NIC_RSS_INDIR_SIZE] = {0};
1585         u16 idx, shift;
1586         u16 i = 0;
1587
1588         if (reta_size != NIC_RSS_INDIR_SIZE) {
1589                 PMD_DRV_LOG(ERR, "Invalid reta size, reta_size:%d", reta_size);
1590                 return HINIC_ERROR;
1591         }
1592
1593         err = hinic_rss_get_indir_tbl(nic_dev->hwdev, tmpl_idx, indirtbl);
1594         if (err) {
1595                 PMD_DRV_LOG(ERR, "Get rss indirect table failed, error:%d",
1596                             err);
1597                 return err;
1598         }
1599
1600         for (i = 0; i < reta_size; i++) {
1601                 idx = i / RTE_RETA_GROUP_SIZE;
1602                 shift = i % RTE_RETA_GROUP_SIZE;
1603                 if (reta_conf[idx].mask & (1ULL << shift))
1604                         reta_conf[idx].reta[shift] = (uint16_t)indirtbl[i];
1605         }
1606
1607         return HINIC_OK;
1608 }
1609
1610 /**
1611  * DPDK callback to get extended device statistics.
1612  *
1613  * @param dev
1614  *   Pointer to Ethernet device.
1615  * @param xstats
1616  *   Pointer to rte extended stats table.
1617  * @param n
1618  *   The size of the stats table.
1619  *
1620  * @return
1621  *   Number of extended stats on success and stats is filled,
1622  *   negative error value otherwise.
1623  */
1624 static int hinic_dev_xstats_get(struct rte_eth_dev *dev,
1625                          struct rte_eth_xstat *xstats,
1626                          unsigned int n)
1627 {
1628         u16 qid = 0;
1629         u32 i;
1630         int err, count;
1631         struct hinic_nic_dev *nic_dev;
1632         struct hinic_phy_port_stats port_stats;
1633         struct hinic_vport_stats vport_stats;
1634         struct hinic_rxq        *rxq = NULL;
1635         struct hinic_rxq_stats rxq_stats;
1636         struct hinic_txq        *txq = NULL;
1637         struct hinic_txq_stats txq_stats;
1638
1639         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1640         count = hinic_xstats_calc_num(nic_dev);
1641         if ((int)n < count)
1642                 return count;
1643
1644         count = 0;
1645
1646         /* Get stats from hinic_rxq_stats */
1647         for (qid = 0; qid < nic_dev->num_rq; qid++) {
1648                 rxq = nic_dev->rxqs[qid];
1649                 hinic_rxq_get_stats(rxq, &rxq_stats);
1650
1651                 for (i = 0; i < HINIC_RXQ_XSTATS_NUM; i++) {
1652                         xstats[count].value =
1653                                 *(uint64_t *)(((char *)&rxq_stats) +
1654                                 hinic_rxq_stats_strings[i].offset);
1655                         xstats[count].id = count;
1656                         count++;
1657                 }
1658         }
1659
1660         /* Get stats from hinic_txq_stats */
1661         for (qid = 0; qid < nic_dev->num_sq; qid++) {
1662                 txq = nic_dev->txqs[qid];
1663                 hinic_txq_get_stats(txq, &txq_stats);
1664
1665                 for (i = 0; i < HINIC_TXQ_XSTATS_NUM; i++) {
1666                         xstats[count].value =
1667                                 *(uint64_t *)(((char *)&txq_stats) +
1668                                 hinic_txq_stats_strings[i].offset);
1669                         xstats[count].id = count;
1670                         count++;
1671                 }
1672         }
1673
1674         /* Get stats from hinic_vport_stats */
1675         err = hinic_get_vport_stats(nic_dev->hwdev, &vport_stats);
1676         if (err)
1677                 return err;
1678
1679         for (i = 0; i < HINIC_VPORT_XSTATS_NUM; i++) {
1680                 xstats[count].value =
1681                         *(uint64_t *)(((char *)&vport_stats) +
1682                         hinic_vport_stats_strings[i].offset);
1683                 xstats[count].id = count;
1684                 count++;
1685         }
1686
1687         /* Get stats from hinic_phy_port_stats */
1688         err = hinic_get_phy_port_stats(nic_dev->hwdev, &port_stats);
1689         if (err)
1690                 return err;
1691
1692         for (i = 0; i < HINIC_PHYPORT_XSTATS_NUM; i++) {
1693                 xstats[count].value = *(uint64_t *)(((char *)&port_stats) +
1694                                 hinic_phyport_stats_strings[i].offset);
1695                 xstats[count].id = count;
1696                 count++;
1697         }
1698
1699         return count;
1700 }
1701
1702 /**
1703  * DPDK callback to retrieve names of extended device statistics
1704  *
1705  * @param dev
1706  *   Pointer to Ethernet device structure.
1707  * @param xstats_names
1708  *   Buffer to insert names into.
1709  *
1710  * @return
1711  *   Number of xstats names.
1712  */
1713 static int hinic_dev_xstats_get_names(struct rte_eth_dev *dev,
1714                                struct rte_eth_xstat_name *xstats_names,
1715                                __rte_unused unsigned int limit)
1716 {
1717         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1718         int count = 0;
1719         u16 i = 0, q_num;
1720
1721         if (xstats_names == NULL)
1722                 return hinic_xstats_calc_num(nic_dev);
1723
1724         /* get pmd rxq stats */
1725         for (q_num = 0; q_num < nic_dev->num_rq; q_num++) {
1726                 for (i = 0; i < HINIC_RXQ_XSTATS_NUM; i++) {
1727                         snprintf(xstats_names[count].name,
1728                                  sizeof(xstats_names[count].name),
1729                                  "rxq%d_%s_pmd",
1730                                  q_num, hinic_rxq_stats_strings[i].name);
1731                         count++;
1732                 }
1733         }
1734
1735         /* get pmd txq stats */
1736         for (q_num = 0; q_num < nic_dev->num_sq; q_num++) {
1737                 for (i = 0; i < HINIC_TXQ_XSTATS_NUM; i++) {
1738                         snprintf(xstats_names[count].name,
1739                                  sizeof(xstats_names[count].name),
1740                                  "txq%d_%s_pmd",
1741                                  q_num, hinic_txq_stats_strings[i].name);
1742                         count++;
1743                 }
1744         }
1745
1746         /* get vport stats */
1747         for (i = 0; i < HINIC_VPORT_XSTATS_NUM; i++) {
1748                 snprintf(xstats_names[count].name,
1749                          sizeof(xstats_names[count].name),
1750                          "%s",
1751                          hinic_vport_stats_strings[i].name);
1752                 count++;
1753         }
1754
1755         /* get phy port stats */
1756         for (i = 0; i < HINIC_PHYPORT_XSTATS_NUM; i++) {
1757                 snprintf(xstats_names[count].name,
1758                          sizeof(xstats_names[count].name),
1759                          "%s",
1760                          hinic_phyport_stats_strings[i].name);
1761                 count++;
1762         }
1763
1764         return count;
1765 }
1766
1767 static int hinic_set_default_pause_feature(struct hinic_nic_dev *nic_dev)
1768 {
1769         struct nic_pause_config pause_config = {0};
1770
1771         pause_config.auto_neg = 0;
1772         pause_config.rx_pause = HINIC_DEFAUT_PAUSE_CONFIG;
1773         pause_config.tx_pause = HINIC_DEFAUT_PAUSE_CONFIG;
1774
1775         return hinic_set_pause_config(nic_dev->hwdev, pause_config);
1776 }
1777
1778 static int hinic_set_default_dcb_feature(struct hinic_nic_dev *nic_dev)
1779 {
1780         u8 up_tc[HINIC_DCB_UP_MAX] = {0};
1781         u8 up_pgid[HINIC_DCB_UP_MAX] = {0};
1782         u8 up_bw[HINIC_DCB_UP_MAX] = {0};
1783         u8 pg_bw[HINIC_DCB_UP_MAX] = {0};
1784         u8 up_strict[HINIC_DCB_UP_MAX] = {0};
1785         int i = 0;
1786
1787         pg_bw[0] = 100;
1788         for (i = 0; i < HINIC_DCB_UP_MAX; i++)
1789                 up_bw[i] = 100;
1790
1791         return hinic_dcb_set_ets(nic_dev->hwdev, up_tc, pg_bw,
1792                                         up_pgid, up_bw, up_strict);
1793 }
1794
1795 static void hinic_init_default_cos(struct hinic_nic_dev *nic_dev)
1796 {
1797         nic_dev->default_cos =
1798                         (hinic_global_func_id(nic_dev->hwdev) +
1799                          DEFAULT_BASE_COS) % NR_MAX_COS;
1800 }
1801
1802 static int hinic_set_default_hw_feature(struct hinic_nic_dev *nic_dev)
1803 {
1804         int err;
1805
1806         hinic_init_default_cos(nic_dev);
1807
1808         /* Restore DCB configure to default status */
1809         err = hinic_set_default_dcb_feature(nic_dev);
1810         if (err)
1811                 return err;
1812
1813         /* disable LRO */
1814         err = hinic_set_rx_lro(nic_dev->hwdev, 0, 0, (u8)0);
1815         if (err)
1816                 return err;
1817
1818         /* Set pause enable, and up will disable pfc. */
1819         err = hinic_set_default_pause_feature(nic_dev);
1820         if (err)
1821                 return err;
1822
1823         err = hinic_reset_port_link_cfg(nic_dev->hwdev);
1824         if (err)
1825                 return err;
1826
1827         err = hinic_set_link_status_follow(nic_dev->hwdev,
1828                                            HINIC_LINK_FOLLOW_PORT);
1829         if (err == HINIC_MGMT_CMD_UNSUPPORTED)
1830                 PMD_DRV_LOG(WARNING, "Don't support to set link status follow phy port status");
1831         else if (err)
1832                 return err;
1833
1834         return hinic_set_anti_attack(nic_dev->hwdev, true);
1835 }
1836
1837 static int32_t hinic_card_workmode_check(struct hinic_nic_dev *nic_dev)
1838 {
1839         struct hinic_board_info info = { 0 };
1840         int rc;
1841
1842         rc = hinic_get_board_info(nic_dev->hwdev, &info);
1843         if (rc)
1844                 return rc;
1845
1846         return (info.service_mode == HINIC_SERVICE_MODE_NIC ? HINIC_OK :
1847                                                 HINIC_ERROR);
1848 }
1849
1850 static int hinic_copy_mempool_init(struct hinic_nic_dev *nic_dev)
1851 {
1852         nic_dev->cpy_mpool = rte_mempool_lookup(nic_dev->proc_dev_name);
1853         if (nic_dev->cpy_mpool == NULL) {
1854                 nic_dev->cpy_mpool =
1855                 rte_pktmbuf_pool_create(nic_dev->proc_dev_name,
1856                                         HINIC_COPY_MEMPOOL_DEPTH,
1857                                         RTE_CACHE_LINE_SIZE, 0,
1858                                         HINIC_COPY_MBUF_SIZE,
1859                                         rte_socket_id());
1860                 if (!nic_dev->cpy_mpool) {
1861                         PMD_DRV_LOG(ERR, "Create copy mempool failed, errno: %d, dev_name: %s",
1862                                     rte_errno, nic_dev->proc_dev_name);
1863                         return -ENOMEM;
1864                 }
1865         }
1866
1867         return 0;
1868 }
1869
1870 static void hinic_copy_mempool_uninit(struct hinic_nic_dev *nic_dev)
1871 {
1872         if (nic_dev->cpy_mpool != NULL)
1873                 rte_mempool_free(nic_dev->cpy_mpool);
1874 }
1875
1876 static int hinic_init_sw_rxtxqs(struct hinic_nic_dev *nic_dev)
1877 {
1878         u32 txq_size;
1879         u32 rxq_size;
1880
1881         /* allocate software txq array */
1882         txq_size = nic_dev->nic_cap.max_sqs * sizeof(*nic_dev->txqs);
1883         nic_dev->txqs = kzalloc_aligned(txq_size, GFP_KERNEL);
1884         if (!nic_dev->txqs) {
1885                 PMD_DRV_LOG(ERR, "Allocate txqs failed");
1886                 return -ENOMEM;
1887         }
1888
1889         /* allocate software rxq array */
1890         rxq_size = nic_dev->nic_cap.max_rqs * sizeof(*nic_dev->rxqs);
1891         nic_dev->rxqs = kzalloc_aligned(rxq_size, GFP_KERNEL);
1892         if (!nic_dev->rxqs) {
1893                 /* free txqs */
1894                 kfree(nic_dev->txqs);
1895                 nic_dev->txqs = NULL;
1896
1897                 PMD_DRV_LOG(ERR, "Allocate rxqs failed");
1898                 return -ENOMEM;
1899         }
1900
1901         return HINIC_OK;
1902 }
1903
1904 static void hinic_deinit_sw_rxtxqs(struct hinic_nic_dev *nic_dev)
1905 {
1906         kfree(nic_dev->txqs);
1907         nic_dev->txqs = NULL;
1908
1909         kfree(nic_dev->rxqs);
1910         nic_dev->rxqs = NULL;
1911 }
1912
1913 static int hinic_nic_dev_create(struct rte_eth_dev *eth_dev)
1914 {
1915         struct hinic_nic_dev *nic_dev =
1916                                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
1917         int rc;
1918
1919         nic_dev->hwdev = rte_zmalloc("hinic_hwdev", sizeof(*nic_dev->hwdev),
1920                                      RTE_CACHE_LINE_SIZE);
1921         if (!nic_dev->hwdev) {
1922                 PMD_DRV_LOG(ERR, "Allocate hinic hwdev memory failed, dev_name: %s",
1923                             eth_dev->data->name);
1924                 return -ENOMEM;
1925         }
1926         nic_dev->hwdev->pcidev_hdl = RTE_ETH_DEV_TO_PCI(eth_dev);
1927
1928         /* init osdep*/
1929         rc = hinic_osdep_init(nic_dev->hwdev);
1930         if (rc) {
1931                 PMD_DRV_LOG(ERR, "Initialize os_dep failed, dev_name: %s",
1932                             eth_dev->data->name);
1933                 goto init_osdep_fail;
1934         }
1935
1936         /* init_hwif */
1937         rc = hinic_hwif_res_init(nic_dev->hwdev);
1938         if (rc) {
1939                 PMD_DRV_LOG(ERR, "Initialize hwif failed, dev_name: %s",
1940                             eth_dev->data->name);
1941                 goto init_hwif_fail;
1942         }
1943
1944         /* init_cfg_mgmt */
1945         rc = init_cfg_mgmt(nic_dev->hwdev);
1946         if (rc) {
1947                 PMD_DRV_LOG(ERR, "Initialize cfg_mgmt failed, dev_name: %s",
1948                             eth_dev->data->name);
1949                 goto init_cfgmgnt_fail;
1950         }
1951
1952         /* init_aeqs */
1953         rc = hinic_comm_aeqs_init(nic_dev->hwdev);
1954         if (rc) {
1955                 PMD_DRV_LOG(ERR, "Initialize aeqs failed, dev_name: %s",
1956                             eth_dev->data->name);
1957                 goto init_aeqs_fail;
1958         }
1959
1960         /* init_pf_to_mgnt */
1961         rc = hinic_comm_pf_to_mgmt_init(nic_dev->hwdev);
1962         if (rc) {
1963                 PMD_DRV_LOG(ERR, "Initialize pf_to_mgmt failed, dev_name: %s",
1964                             eth_dev->data->name);
1965                 goto init_pf_to_mgmt_fail;
1966         }
1967
1968         rc = hinic_card_workmode_check(nic_dev);
1969         if (rc) {
1970                 PMD_DRV_LOG(ERR, "Check card workmode failed, dev_name: %s",
1971                             eth_dev->data->name);
1972                 goto workmode_check_fail;
1973         }
1974
1975         /* do l2nic reset to make chip clear */
1976         rc = hinic_l2nic_reset(nic_dev->hwdev);
1977         if (rc) {
1978                 PMD_DRV_LOG(ERR, "Do l2nic reset failed, dev_name: %s",
1979                             eth_dev->data->name);
1980                 goto l2nic_reset_fail;
1981         }
1982
1983         /* init dma and aeq msix attribute table */
1984         (void)hinic_init_attr_table(nic_dev->hwdev);
1985
1986         /* init_cmdqs */
1987         rc = hinic_comm_cmdqs_init(nic_dev->hwdev);
1988         if (rc) {
1989                 PMD_DRV_LOG(ERR, "Initialize cmdq failed, dev_name: %s",
1990                             eth_dev->data->name);
1991                 goto init_cmdq_fail;
1992         }
1993
1994         /* set hardware state active */
1995         rc = hinic_activate_hwdev_state(nic_dev->hwdev);
1996         if (rc) {
1997                 PMD_DRV_LOG(ERR, "Initialize resources state failed, dev_name: %s",
1998                             eth_dev->data->name);
1999                 goto init_resources_state_fail;
2000         }
2001
2002         /* init_capability */
2003         rc = hinic_init_capability(nic_dev->hwdev);
2004         if (rc) {
2005                 PMD_DRV_LOG(ERR, "Initialize capability failed, dev_name: %s",
2006                             eth_dev->data->name);
2007                 goto init_cap_fail;
2008         }
2009
2010         /* get nic capability */
2011         if (!hinic_support_nic(nic_dev->hwdev, &nic_dev->nic_cap))
2012                 goto nic_check_fail;
2013
2014         /* init root cla and function table */
2015         rc = hinic_init_nicio(nic_dev->hwdev);
2016         if (rc) {
2017                 PMD_DRV_LOG(ERR, "Initialize nic_io failed, dev_name: %s",
2018                             eth_dev->data->name);
2019                 goto init_nicio_fail;
2020         }
2021
2022         /* init_software_txrxq */
2023         rc = hinic_init_sw_rxtxqs(nic_dev);
2024         if (rc) {
2025                 PMD_DRV_LOG(ERR, "Initialize sw_rxtxqs failed, dev_name: %s",
2026                             eth_dev->data->name);
2027                 goto init_sw_rxtxqs_fail;
2028         }
2029
2030         rc = hinic_copy_mempool_init(nic_dev);
2031         if (rc) {
2032                 PMD_DRV_LOG(ERR, "Create copy mempool failed, dev_name: %s",
2033                          eth_dev->data->name);
2034                 goto init_mpool_fail;
2035         }
2036
2037         /* set hardware feature to default status */
2038         rc = hinic_set_default_hw_feature(nic_dev);
2039         if (rc) {
2040                 PMD_DRV_LOG(ERR, "Initialize hardware default features failed, dev_name: %s",
2041                             eth_dev->data->name);
2042                 goto set_default_hw_feature_fail;
2043         }
2044
2045         return 0;
2046
2047 set_default_hw_feature_fail:
2048         hinic_copy_mempool_uninit(nic_dev);
2049
2050 init_mpool_fail:
2051         hinic_deinit_sw_rxtxqs(nic_dev);
2052
2053 init_sw_rxtxqs_fail:
2054         hinic_deinit_nicio(nic_dev->hwdev);
2055
2056 nic_check_fail:
2057 init_nicio_fail:
2058 init_cap_fail:
2059         hinic_deactivate_hwdev_state(nic_dev->hwdev);
2060
2061 init_resources_state_fail:
2062         hinic_comm_cmdqs_free(nic_dev->hwdev);
2063
2064 init_cmdq_fail:
2065 l2nic_reset_fail:
2066 workmode_check_fail:
2067         hinic_comm_pf_to_mgmt_free(nic_dev->hwdev);
2068
2069 init_pf_to_mgmt_fail:
2070         hinic_comm_aeqs_free(nic_dev->hwdev);
2071
2072 init_aeqs_fail:
2073         free_cfg_mgmt(nic_dev->hwdev);
2074
2075 init_cfgmgnt_fail:
2076         hinic_hwif_res_free(nic_dev->hwdev);
2077
2078 init_hwif_fail:
2079         hinic_osdep_deinit(nic_dev->hwdev);
2080
2081 init_osdep_fail:
2082         rte_free(nic_dev->hwdev);
2083         nic_dev->hwdev = NULL;
2084
2085         return rc;
2086 }
2087
2088 static void hinic_nic_dev_destroy(struct rte_eth_dev *eth_dev)
2089 {
2090         struct hinic_nic_dev *nic_dev =
2091                         HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
2092
2093         (void)hinic_set_link_status_follow(nic_dev->hwdev,
2094                                            HINIC_LINK_FOLLOW_DEFAULT);
2095         hinic_copy_mempool_uninit(nic_dev);
2096         hinic_deinit_sw_rxtxqs(nic_dev);
2097         hinic_deinit_nicio(nic_dev->hwdev);
2098         hinic_deactivate_hwdev_state(nic_dev->hwdev);
2099         hinic_comm_cmdqs_free(nic_dev->hwdev);
2100         hinic_comm_pf_to_mgmt_free(nic_dev->hwdev);
2101         hinic_comm_aeqs_free(nic_dev->hwdev);
2102         free_cfg_mgmt(nic_dev->hwdev);
2103         hinic_hwif_res_free(nic_dev->hwdev);
2104         hinic_osdep_deinit(nic_dev->hwdev);
2105         rte_free(nic_dev->hwdev);
2106         nic_dev->hwdev = NULL;
2107 }
2108
2109 static int hinic_func_init(struct rte_eth_dev *eth_dev)
2110 {
2111         struct rte_pci_device *pci_dev;
2112         struct rte_ether_addr *eth_addr;
2113         struct hinic_nic_dev *nic_dev;
2114         int rc;
2115
2116         pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
2117
2118         /* EAL is SECONDARY and eth_dev is already created */
2119         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2120                 rc = rte_intr_callback_register(&pci_dev->intr_handle,
2121                                                 hinic_dev_interrupt_handler,
2122                                                 (void *)eth_dev);
2123                 if (rc)
2124                         PMD_DRV_LOG(ERR, "Initialize %s failed in secondary process",
2125                                     eth_dev->data->name);
2126
2127                 return rc;
2128         }
2129
2130         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
2131         memset(nic_dev, 0, sizeof(*nic_dev));
2132
2133         snprintf(nic_dev->proc_dev_name,
2134                  sizeof(nic_dev->proc_dev_name),
2135                  "hinic-%.4x:%.2x:%.2x.%x",
2136                  pci_dev->addr.domain, pci_dev->addr.bus,
2137                  pci_dev->addr.devid, pci_dev->addr.function);
2138
2139         /* alloc mac_addrs */
2140         eth_addr = rte_zmalloc("hinic_mac", sizeof(*eth_addr), 0);
2141         if (!eth_addr) {
2142                 PMD_DRV_LOG(ERR, "Allocate ethernet addresses' memory failed, dev_name: %s",
2143                             eth_dev->data->name);
2144                 rc = -ENOMEM;
2145                 goto eth_addr_fail;
2146         }
2147         eth_dev->data->mac_addrs = eth_addr;
2148
2149         /*
2150          * Pass the information to the rte_eth_dev_close() that it should also
2151          * release the private port resources.
2152          */
2153         eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
2154
2155         /* create hardware nic_device */
2156         rc = hinic_nic_dev_create(eth_dev);
2157         if (rc) {
2158                 PMD_DRV_LOG(ERR, "Create nic device failed, dev_name: %s",
2159                             eth_dev->data->name);
2160                 goto create_nic_dev_fail;
2161         }
2162
2163         rc = hinic_init_mac_addr(eth_dev);
2164         if (rc) {
2165                 PMD_DRV_LOG(ERR, "Initialize mac table failed, dev_name: %s",
2166                             eth_dev->data->name);
2167                 goto init_mac_fail;
2168         }
2169
2170         /* register callback func to eal lib */
2171         rc = rte_intr_callback_register(&pci_dev->intr_handle,
2172                                         hinic_dev_interrupt_handler,
2173                                         (void *)eth_dev);
2174         if (rc) {
2175                 PMD_DRV_LOG(ERR, "Register rte interrupt callback failed, dev_name: %s",
2176                             eth_dev->data->name);
2177                 goto reg_intr_cb_fail;
2178         }
2179
2180         /* enable uio/vfio intr/eventfd mapping */
2181         rc = rte_intr_enable(&pci_dev->intr_handle);
2182         if (rc) {
2183                 PMD_DRV_LOG(ERR, "Enable rte interrupt failed, dev_name: %s",
2184                             eth_dev->data->name);
2185                 goto enable_intr_fail;
2186         }
2187         hinic_set_bit(HINIC_DEV_INTR_EN, &nic_dev->dev_status);
2188
2189         hinic_set_bit(HINIC_DEV_INIT, &nic_dev->dev_status);
2190         PMD_DRV_LOG(INFO, "Initialize %s in primary successfully",
2191                     eth_dev->data->name);
2192
2193         return 0;
2194
2195 enable_intr_fail:
2196         (void)rte_intr_callback_unregister(&pci_dev->intr_handle,
2197                                            hinic_dev_interrupt_handler,
2198                                            (void *)eth_dev);
2199
2200 reg_intr_cb_fail:
2201         hinic_deinit_mac_addr(eth_dev);
2202
2203 init_mac_fail:
2204         hinic_nic_dev_destroy(eth_dev);
2205
2206 create_nic_dev_fail:
2207         rte_free(eth_addr);
2208         eth_dev->data->mac_addrs = NULL;
2209
2210 eth_addr_fail:
2211         PMD_DRV_LOG(ERR, "Initialize %s in primary failed",
2212                     eth_dev->data->name);
2213         return rc;
2214 }
2215
2216 /**
2217  * DPDK callback to close the device.
2218  *
2219  * @param dev
2220  *   Pointer to Ethernet device structure.
2221  */
2222 static void hinic_dev_close(struct rte_eth_dev *dev)
2223 {
2224         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
2225
2226         if (hinic_test_and_set_bit(HINIC_DEV_CLOSE, &nic_dev->dev_status)) {
2227                 PMD_DRV_LOG(WARNING, "Device %s already closed",
2228                             dev->data->name);
2229                 return;
2230         }
2231
2232         /* stop device first */
2233         hinic_dev_stop(dev);
2234
2235         /* rx_cqe, rx_info */
2236         hinic_free_all_rx_resources(dev);
2237
2238         /* tx_info */
2239         hinic_free_all_tx_resources(dev);
2240
2241         /* free wq, pi_dma_addr */
2242         hinic_free_all_rq(nic_dev);
2243
2244         /* free wq, db_addr */
2245         hinic_free_all_sq(nic_dev);
2246
2247         /* deinit mac vlan tbl */
2248         hinic_deinit_mac_addr(dev);
2249
2250         /* disable hardware and uio interrupt */
2251         hinic_disable_interrupt(dev);
2252
2253         /* deinit nic hardware device */
2254         hinic_nic_dev_destroy(dev);
2255 }
2256
2257 static const struct eth_dev_ops hinic_pmd_ops = {
2258         .dev_configure                 = hinic_dev_configure,
2259         .dev_infos_get                 = hinic_dev_infos_get,
2260         .rx_queue_setup                = hinic_rx_queue_setup,
2261         .tx_queue_setup                = hinic_tx_queue_setup,
2262         .dev_start                     = hinic_dev_start,
2263         .link_update                   = hinic_link_update,
2264         .rx_queue_release              = hinic_rx_queue_release,
2265         .tx_queue_release              = hinic_tx_queue_release,
2266         .dev_stop                      = hinic_dev_stop,
2267         .dev_close                     = hinic_dev_close,
2268         .promiscuous_enable            = hinic_dev_promiscuous_enable,
2269         .promiscuous_disable           = hinic_dev_promiscuous_disable,
2270         .rss_hash_update               = hinic_rss_hash_update,
2271         .rss_hash_conf_get             = hinic_rss_conf_get,
2272         .reta_update                   = hinic_rss_indirtbl_update,
2273         .reta_query                    = hinic_rss_indirtbl_query,
2274         .stats_get                     = hinic_dev_stats_get,
2275         .stats_reset                   = hinic_dev_stats_reset,
2276         .xstats_get                    = hinic_dev_xstats_get,
2277         .xstats_reset                  = hinic_dev_xstats_reset,
2278         .xstats_get_names              = hinic_dev_xstats_get_names,
2279 };
2280
2281 static int hinic_dev_init(struct rte_eth_dev *eth_dev)
2282 {
2283         struct rte_pci_device *pci_dev;
2284
2285         pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
2286
2287         PMD_DRV_LOG(INFO, "Initializing pf hinic-%.4x:%.2x:%.2x.%x in %s process",
2288                     pci_dev->addr.domain, pci_dev->addr.bus,
2289                     pci_dev->addr.devid, pci_dev->addr.function,
2290                     (rte_eal_process_type() == RTE_PROC_PRIMARY) ?
2291                     "primary" : "secondary");
2292
2293         /* rte_eth_dev ops, rx_burst and tx_burst */
2294         eth_dev->dev_ops = &hinic_pmd_ops;
2295         eth_dev->rx_pkt_burst = hinic_recv_pkts;
2296         eth_dev->tx_pkt_burst = hinic_xmit_pkts;
2297
2298         return hinic_func_init(eth_dev);
2299 }
2300
2301 static int hinic_dev_uninit(struct rte_eth_dev *dev)
2302 {
2303         struct hinic_nic_dev *nic_dev;
2304
2305         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
2306         hinic_clear_bit(HINIC_DEV_INIT, &nic_dev->dev_status);
2307
2308         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2309                 return 0;
2310
2311         hinic_dev_close(dev);
2312
2313         dev->dev_ops = NULL;
2314         dev->rx_pkt_burst = NULL;
2315         dev->tx_pkt_burst = NULL;
2316
2317         rte_free(dev->data->mac_addrs);
2318         dev->data->mac_addrs = NULL;
2319
2320         return HINIC_OK;
2321 }
2322
2323 static struct rte_pci_id pci_id_hinic_map[] = {
2324         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_PRD) },
2325         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_MEZZ_25GE) },
2326         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_MEZZ_40GE) },
2327         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_MEZZ_100GE) },
2328         {.vendor_id = 0},
2329 };
2330
2331 static int hinic_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
2332                            struct rte_pci_device *pci_dev)
2333 {
2334         return rte_eth_dev_pci_generic_probe(pci_dev,
2335                 sizeof(struct hinic_nic_dev), hinic_dev_init);
2336 }
2337
2338 static int hinic_pci_remove(struct rte_pci_device *pci_dev)
2339 {
2340         return rte_eth_dev_pci_generic_remove(pci_dev, hinic_dev_uninit);
2341 }
2342
2343 static struct rte_pci_driver rte_hinic_pmd = {
2344         .id_table = pci_id_hinic_map,
2345         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
2346         .probe = hinic_pci_probe,
2347         .remove = hinic_pci_remove,
2348 };
2349
2350 RTE_PMD_REGISTER_PCI(net_hinic, rte_hinic_pmd);
2351 RTE_PMD_REGISTER_PCI_TABLE(net_hinic, pci_id_hinic_map);
2352
2353 RTE_INIT(hinic_init_log)
2354 {
2355         hinic_logtype = rte_log_register("pmd.net.hinic");
2356         if (hinic_logtype >= 0)
2357                 rte_log_set_level(hinic_logtype, RTE_LOG_INFO);
2358 }