drivers/net/hinic/hinic_pmd_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Huawei Technologies Co., Ltd
   3  */
   4
   5 #include <rte_pci.h>
   6 #include <rte_bus_pci.h>
   7 #include <rte_ethdev_pci.h>
   8 #include <rte_mbuf.h>
   9 #include <rte_malloc.h>
  10 #include <rte_memcpy.h>
  11 #include <rte_mempool.h>
  12 #include <rte_errno.h>
  13
  14 #include "base/hinic_compat.h"
  15 #include "base/hinic_pmd_hwdev.h"
  16 #include "base/hinic_pmd_hwif.h"
  17 #include "base/hinic_pmd_wq.h"
  18 #include "base/hinic_pmd_cfg.h"
  19 #include "base/hinic_pmd_mgmt.h"
  20 #include "base/hinic_pmd_cmdq.h"
  21 #include "base/hinic_pmd_niccfg.h"
  22 #include "base/hinic_pmd_nicio.h"
  23 #include "base/hinic_pmd_mbox.h"
  24 #include "hinic_pmd_ethdev.h"
  25 #include "hinic_pmd_tx.h"
  26 #include "hinic_pmd_rx.h"
  27
  28 /* Vendor ID used by Huawei devices */
  29 #define HINIC_HUAWEI_VENDOR_ID          0x19E5
  30
  31 /* Hinic devices */
  32 #define HINIC_DEV_ID_PRD                0x1822
  33 #define HINIC_DEV_ID_VF                 0x375E
  34 #define HINIC_DEV_ID_VF_HV              0x379E
  35
  36 /* Mezz card for Blade Server */
  37 #define HINIC_DEV_ID_MEZZ_25GE          0x0210
  38 #define HINIC_DEV_ID_MEZZ_40GE          0x020D
  39 #define HINIC_DEV_ID_MEZZ_100GE         0x0205
  40
  41 /* 2*25G and 2*100G card */
  42 #define HINIC_DEV_ID_1822_DUAL_25GE     0x0206
  43 #define HINIC_DEV_ID_1822_100GE         0x0200
  44
  45 #define HINIC_SERVICE_MODE_NIC          2
  46
  47 #define HINIC_INTR_CB_UNREG_MAX_RETRIES 10
  48
  49 #define DEFAULT_BASE_COS                4
  50 #define NR_MAX_COS                      8
  51
  52 #define HINIC_MIN_RX_BUF_SIZE           1024
  53 #define HINIC_MAX_MAC_ADDRS             1
  54
  55 /* Driver-specific log messages type */
  56 int hinic_logtype;
  57
  58 struct hinic_xstats_name_off {
  59         char name[RTE_ETH_XSTATS_NAME_SIZE];
  60         u32  offset;
  61 };
  62
  63 #define HINIC_FUNC_STAT(_stat_item) {   \
  64         .name = #_stat_item, \
  65         .offset = offsetof(struct hinic_vport_stats, _stat_item) \
  66 }
  67
  68 #define HINIC_PORT_STAT(_stat_item) { \
  69         .name = #_stat_item, \
  70         .offset = offsetof(struct hinic_phy_port_stats, _stat_item) \
  71 }
  72
  73 static const struct hinic_xstats_name_off hinic_vport_stats_strings[] = {
  74         HINIC_FUNC_STAT(tx_unicast_pkts_vport),
  75         HINIC_FUNC_STAT(tx_unicast_bytes_vport),
  76         HINIC_FUNC_STAT(tx_multicast_pkts_vport),
  77         HINIC_FUNC_STAT(tx_multicast_bytes_vport),
  78         HINIC_FUNC_STAT(tx_broadcast_pkts_vport),
  79         HINIC_FUNC_STAT(tx_broadcast_bytes_vport),
  80
  81         HINIC_FUNC_STAT(rx_unicast_pkts_vport),
  82         HINIC_FUNC_STAT(rx_unicast_bytes_vport),
  83         HINIC_FUNC_STAT(rx_multicast_pkts_vport),
  84         HINIC_FUNC_STAT(rx_multicast_bytes_vport),
  85         HINIC_FUNC_STAT(rx_broadcast_pkts_vport),
  86         HINIC_FUNC_STAT(rx_broadcast_bytes_vport),
  87
  88         HINIC_FUNC_STAT(tx_discard_vport),
  89         HINIC_FUNC_STAT(rx_discard_vport),
  90         HINIC_FUNC_STAT(tx_err_vport),
  91         HINIC_FUNC_STAT(rx_err_vport),
  92 };
  93
  94 #define HINIC_VPORT_XSTATS_NUM (sizeof(hinic_vport_stats_strings) / \
  95                 sizeof(hinic_vport_stats_strings[0]))
  96
  97 static const struct hinic_xstats_name_off hinic_phyport_stats_strings[] = {
  98         HINIC_PORT_STAT(mac_rx_total_pkt_num),
  99         HINIC_PORT_STAT(mac_rx_total_oct_num),
 100         HINIC_PORT_STAT(mac_rx_bad_pkt_num),
 101         HINIC_PORT_STAT(mac_rx_bad_oct_num),
 102         HINIC_PORT_STAT(mac_rx_good_pkt_num),
 103         HINIC_PORT_STAT(mac_rx_good_oct_num),
 104         HINIC_PORT_STAT(mac_rx_uni_pkt_num),
 105         HINIC_PORT_STAT(mac_rx_multi_pkt_num),
 106         HINIC_PORT_STAT(mac_rx_broad_pkt_num),
 107         HINIC_PORT_STAT(mac_tx_total_pkt_num),
 108         HINIC_PORT_STAT(mac_tx_total_oct_num),
 109         HINIC_PORT_STAT(mac_tx_bad_pkt_num),
 110         HINIC_PORT_STAT(mac_tx_bad_oct_num),
 111         HINIC_PORT_STAT(mac_tx_good_pkt_num),
 112         HINIC_PORT_STAT(mac_tx_good_oct_num),
 113         HINIC_PORT_STAT(mac_tx_uni_pkt_num),
 114         HINIC_PORT_STAT(mac_tx_multi_pkt_num),
 115         HINIC_PORT_STAT(mac_tx_broad_pkt_num),
 116         HINIC_PORT_STAT(mac_rx_fragment_pkt_num),
 117         HINIC_PORT_STAT(mac_rx_undersize_pkt_num),
 118         HINIC_PORT_STAT(mac_rx_undermin_pkt_num),
 119         HINIC_PORT_STAT(mac_rx_64_oct_pkt_num),
 120         HINIC_PORT_STAT(mac_rx_65_127_oct_pkt_num),
 121         HINIC_PORT_STAT(mac_rx_128_255_oct_pkt_num),
 122         HINIC_PORT_STAT(mac_rx_256_511_oct_pkt_num),
 123         HINIC_PORT_STAT(mac_rx_512_1023_oct_pkt_num),
 124         HINIC_PORT_STAT(mac_rx_1024_1518_oct_pkt_num),
 125         HINIC_PORT_STAT(mac_rx_1519_2047_oct_pkt_num),
 126         HINIC_PORT_STAT(mac_rx_2048_4095_oct_pkt_num),
 127         HINIC_PORT_STAT(mac_rx_4096_8191_oct_pkt_num),
 128         HINIC_PORT_STAT(mac_rx_8192_9216_oct_pkt_num),
 129         HINIC_PORT_STAT(mac_rx_9217_12287_oct_pkt_num),
 130         HINIC_PORT_STAT(mac_rx_12288_16383_oct_pkt_num),
 131         HINIC_PORT_STAT(mac_rx_1519_max_bad_pkt_num),
 132         HINIC_PORT_STAT(mac_rx_1519_max_good_pkt_num),
 133         HINIC_PORT_STAT(mac_rx_oversize_pkt_num),
 134         HINIC_PORT_STAT(mac_rx_jabber_pkt_num),
 135         HINIC_PORT_STAT(mac_rx_mac_pause_num),
 136         HINIC_PORT_STAT(mac_rx_pfc_pkt_num),
 137         HINIC_PORT_STAT(mac_rx_pfc_pri0_pkt_num),
 138         HINIC_PORT_STAT(mac_rx_pfc_pri1_pkt_num),
 139         HINIC_PORT_STAT(mac_rx_pfc_pri2_pkt_num),
 140         HINIC_PORT_STAT(mac_rx_pfc_pri3_pkt_num),
 141         HINIC_PORT_STAT(mac_rx_pfc_pri4_pkt_num),
 142         HINIC_PORT_STAT(mac_rx_pfc_pri5_pkt_num),
 143         HINIC_PORT_STAT(mac_rx_pfc_pri6_pkt_num),
 144         HINIC_PORT_STAT(mac_rx_pfc_pri7_pkt_num),
 145         HINIC_PORT_STAT(mac_rx_mac_control_pkt_num),
 146         HINIC_PORT_STAT(mac_rx_sym_err_pkt_num),
 147         HINIC_PORT_STAT(mac_rx_fcs_err_pkt_num),
 148         HINIC_PORT_STAT(mac_rx_send_app_good_pkt_num),
 149         HINIC_PORT_STAT(mac_rx_send_app_bad_pkt_num),
 150         HINIC_PORT_STAT(mac_tx_fragment_pkt_num),
 151         HINIC_PORT_STAT(mac_tx_undersize_pkt_num),
 152         HINIC_PORT_STAT(mac_tx_undermin_pkt_num),
 153         HINIC_PORT_STAT(mac_tx_64_oct_pkt_num),
 154         HINIC_PORT_STAT(mac_tx_65_127_oct_pkt_num),
 155         HINIC_PORT_STAT(mac_tx_128_255_oct_pkt_num),
 156         HINIC_PORT_STAT(mac_tx_256_511_oct_pkt_num),
 157         HINIC_PORT_STAT(mac_tx_512_1023_oct_pkt_num),
 158         HINIC_PORT_STAT(mac_tx_1024_1518_oct_pkt_num),
 159         HINIC_PORT_STAT(mac_tx_1519_2047_oct_pkt_num),
 160         HINIC_PORT_STAT(mac_tx_2048_4095_oct_pkt_num),
 161         HINIC_PORT_STAT(mac_tx_4096_8191_oct_pkt_num),
 162         HINIC_PORT_STAT(mac_tx_8192_9216_oct_pkt_num),
 163         HINIC_PORT_STAT(mac_tx_9217_12287_oct_pkt_num),
 164         HINIC_PORT_STAT(mac_tx_12288_16383_oct_pkt_num),
 165         HINIC_PORT_STAT(mac_tx_1519_max_bad_pkt_num),
 166         HINIC_PORT_STAT(mac_tx_1519_max_good_pkt_num),
 167         HINIC_PORT_STAT(mac_tx_oversize_pkt_num),
 168         HINIC_PORT_STAT(mac_trans_jabber_pkt_num),
 169         HINIC_PORT_STAT(mac_tx_mac_pause_num),
 170         HINIC_PORT_STAT(mac_tx_pfc_pkt_num),
 171         HINIC_PORT_STAT(mac_tx_pfc_pri0_pkt_num),
 172         HINIC_PORT_STAT(mac_tx_pfc_pri1_pkt_num),
 173         HINIC_PORT_STAT(mac_tx_pfc_pri2_pkt_num),
 174         HINIC_PORT_STAT(mac_tx_pfc_pri3_pkt_num),
 175         HINIC_PORT_STAT(mac_tx_pfc_pri4_pkt_num),
 176         HINIC_PORT_STAT(mac_tx_pfc_pri5_pkt_num),
 177         HINIC_PORT_STAT(mac_tx_pfc_pri6_pkt_num),
 178         HINIC_PORT_STAT(mac_tx_pfc_pri7_pkt_num),
 179         HINIC_PORT_STAT(mac_tx_mac_control_pkt_num),
 180         HINIC_PORT_STAT(mac_tx_err_all_pkt_num),
 181         HINIC_PORT_STAT(mac_tx_from_app_good_pkt_num),
 182         HINIC_PORT_STAT(mac_tx_from_app_bad_pkt_num),
 183 };
 184
 185 #define HINIC_PHYPORT_XSTATS_NUM (sizeof(hinic_phyport_stats_strings) / \
 186                 sizeof(hinic_phyport_stats_strings[0]))
 187
 188 static const struct hinic_xstats_name_off hinic_rxq_stats_strings[] = {
 189         {"rx_nombuf", offsetof(struct hinic_rxq_stats, rx_nombuf)},
 190         {"burst_pkt", offsetof(struct hinic_rxq_stats, burst_pkts)},
 191 };
 192
 193 #define HINIC_RXQ_XSTATS_NUM (sizeof(hinic_rxq_stats_strings) / \
 194                 sizeof(hinic_rxq_stats_strings[0]))
 195
 196 static const struct hinic_xstats_name_off hinic_txq_stats_strings[] = {
 197         {"tx_busy", offsetof(struct hinic_txq_stats, tx_busy)},
 198         {"offload_errors", offsetof(struct hinic_txq_stats, off_errs)},
 199         {"copy_pkts", offsetof(struct hinic_txq_stats, cpy_pkts)},
 200         {"rl_drop", offsetof(struct hinic_txq_stats, rl_drop)},
 201         {"burst_pkts", offsetof(struct hinic_txq_stats, burst_pkts)},
 202 };
 203
 204 #define HINIC_TXQ_XSTATS_NUM (sizeof(hinic_txq_stats_strings) / \
 205                 sizeof(hinic_txq_stats_strings[0]))
 206
 207 static int hinic_xstats_calc_num(struct hinic_nic_dev *nic_dev)
 208 {
 209         if (HINIC_IS_VF(nic_dev->hwdev)) {
 210                 return (HINIC_VPORT_XSTATS_NUM +
 211                         HINIC_RXQ_XSTATS_NUM * nic_dev->num_rq +
 212                         HINIC_TXQ_XSTATS_NUM * nic_dev->num_sq);
 213         } else {
 214                 return (HINIC_VPORT_XSTATS_NUM +
 215                         HINIC_PHYPORT_XSTATS_NUM +
 216                         HINIC_RXQ_XSTATS_NUM * nic_dev->num_rq +
 217                         HINIC_TXQ_XSTATS_NUM * nic_dev->num_sq);
 218         }
 219 }
 220
 221 static const struct rte_eth_desc_lim hinic_rx_desc_lim = {
 222         .nb_max = HINIC_MAX_QUEUE_DEPTH,
 223         .nb_min = HINIC_MIN_QUEUE_DEPTH,
 224         .nb_align = HINIC_RXD_ALIGN,
 225 };
 226
 227 static const struct rte_eth_desc_lim hinic_tx_desc_lim = {
 228         .nb_max = HINIC_MAX_QUEUE_DEPTH,
 229         .nb_min = HINIC_MIN_QUEUE_DEPTH,
 230         .nb_align = HINIC_TXD_ALIGN,
 231 };
 232
 233
 234 /**
 235  * Interrupt handler triggered by NIC  for handling
 236  * specific event.
 237  *
 238  * @param: The address of parameter (struct rte_eth_dev *) regsitered before.
 239  **/
 240 static void hinic_dev_interrupt_handler(void *param)
 241 {
 242         struct rte_eth_dev *dev = param;
 243         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 244
 245         if (!hinic_test_bit(HINIC_DEV_INTR_EN, &nic_dev->dev_status)) {
 246                 PMD_DRV_LOG(WARNING, "Device's interrupt is disabled, ignore interrupt event, dev_name: %s, port_id: %d",
 247                             nic_dev->proc_dev_name, dev->data->port_id);
 248                 return;
 249         }
 250
 251         /* aeq0 msg handler */
 252         hinic_dev_handle_aeq_event(nic_dev->hwdev, param);
 253 }
 254
 255 /**
 256  * Ethernet device configuration.
 257  *
 258  * Prepare the driver for a given number of TX and RX queues, mtu size
 259  * and configure RSS.
 260  *
 261  * @param dev
 262  *   Pointer to Ethernet device structure.
 263  *
 264  * @return
 265  *   0 on success, negative error value otherwise.
 266  */
 267 static int hinic_dev_configure(struct rte_eth_dev *dev)
 268 {
 269         struct hinic_nic_dev *nic_dev;
 270         struct hinic_nic_io *nic_io;
 271         int err;
 272
 273         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 274         nic_io = nic_dev->hwdev->nic_io;
 275
 276         nic_dev->num_sq =  dev->data->nb_tx_queues;
 277         nic_dev->num_rq = dev->data->nb_rx_queues;
 278
 279         nic_io->num_sqs =  dev->data->nb_tx_queues;
 280         nic_io->num_rqs = dev->data->nb_rx_queues;
 281
 282         /* queue pair is max_num(sq, rq) */
 283         nic_dev->num_qps = (nic_dev->num_sq > nic_dev->num_rq) ?
 284                         nic_dev->num_sq : nic_dev->num_rq;
 285         nic_io->num_qps = nic_dev->num_qps;
 286
 287         if (nic_dev->num_qps > nic_io->max_qps) {
 288                 PMD_DRV_LOG(ERR,
 289                         "Queue number out of range, get queue_num:%d, max_queue_num:%d",
 290                         nic_dev->num_qps, nic_io->max_qps);
 291                 return -EINVAL;
 292         }
 293
 294         /* mtu size is 256~9600 */
 295         if (dev->data->dev_conf.rxmode.max_rx_pkt_len < HINIC_MIN_FRAME_SIZE ||
 296             dev->data->dev_conf.rxmode.max_rx_pkt_len >
 297             HINIC_MAX_JUMBO_FRAME_SIZE) {
 298                 PMD_DRV_LOG(ERR,
 299                         "Max rx pkt len out of range, get max_rx_pkt_len:%d, "
 300                         "expect between %d and %d",
 301                         dev->data->dev_conf.rxmode.max_rx_pkt_len,
 302                         HINIC_MIN_FRAME_SIZE, HINIC_MAX_JUMBO_FRAME_SIZE);
 303                 return -EINVAL;
 304         }
 305
 306         nic_dev->mtu_size =
 307                 HINIC_PKTLEN_TO_MTU(dev->data->dev_conf.rxmode.max_rx_pkt_len);
 308
 309         /* rss template */
 310         err = hinic_config_mq_mode(dev, TRUE);
 311         if (err) {
 312                 PMD_DRV_LOG(ERR, "Config multi-queue failed");
 313                 return err;
 314         }
 315
 316         return HINIC_OK;
 317 }
 318
 319 /**
 320  * DPDK callback to create the receive queue.
 321  *
 322  * @param dev
 323  *   Pointer to Ethernet device structure.
 324  * @param queue_idx
 325  *   RX queue index.
 326  * @param nb_desc
 327  *   Number of descriptors for receive queue.
 328  * @param socket_id
 329  *   NUMA socket on which memory must be allocated.
 330  * @param rx_conf
 331  *   Thresholds parameters (unused_).
 332  * @param mp
 333  *   Memory pool for buffer allocations.
 334  *
 335  * @return
 336  *   0 on success, negative error value otherwise.
 337  */
 338 static int hinic_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 339                          uint16_t nb_desc, unsigned int socket_id,
 340                          __rte_unused const struct rte_eth_rxconf *rx_conf,
 341                          struct rte_mempool *mp)
 342 {
 343         int rc;
 344         struct hinic_nic_dev *nic_dev;
 345         struct hinic_hwdev *hwdev;
 346         struct hinic_rxq *rxq;
 347         u16 rq_depth, rx_free_thresh;
 348         u32 buf_size;
 349
 350         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 351         hwdev = nic_dev->hwdev;
 352
 353         /* queue depth must be power of 2, otherwise will be aligned up */
 354         rq_depth = (nb_desc & (nb_desc - 1)) ?
 355                 ((u16)(1U << (ilog2(nb_desc) + 1))) : nb_desc;
 356
 357         /*
 358          * Validate number of receive descriptors.
 359          * It must not exceed hardware maximum and minimum.
 360          */
 361         if (rq_depth > HINIC_MAX_QUEUE_DEPTH ||
 362                 rq_depth < HINIC_MIN_QUEUE_DEPTH) {
 363                 PMD_DRV_LOG(ERR, "RX queue depth is out of range from %d to %d, (nb_desc=%d, q_depth=%d, port=%d queue=%d)",
 364                             HINIC_MIN_QUEUE_DEPTH, HINIC_MAX_QUEUE_DEPTH,
 365                             (int)nb_desc, (int)rq_depth,
 366                             (int)dev->data->port_id, (int)queue_idx);
 367                 return -EINVAL;
 368         }
 369
 370         /*
 371          * The RX descriptor ring will be cleaned after rxq->rx_free_thresh
 372          * descriptors are used or if the number of descriptors required
 373          * to transmit a packet is greater than the number of free RX
 374          * descriptors.
 375          * The following constraints must be satisfied:
 376          *  rx_free_thresh must be greater than 0.
 377          *  rx_free_thresh must be less than the size of the ring minus 1.
 378          * When set to zero use default values.
 379          */
 380         rx_free_thresh = (u16)((rx_conf->rx_free_thresh) ?
 381                         rx_conf->rx_free_thresh : HINIC_DEFAULT_RX_FREE_THRESH);
 382         if (rx_free_thresh >= (rq_depth - 1)) {
 383                 PMD_DRV_LOG(ERR, "rx_free_thresh must be less than the number of RX descriptors minus 1. (rx_free_thresh=%u port=%d queue=%d)",
 384                             (unsigned int)rx_free_thresh,
 385                             (int)dev->data->port_id,
 386                             (int)queue_idx);
 387                 return -EINVAL;
 388         }
 389
 390         rxq = rte_zmalloc_socket("hinic_rx_queue", sizeof(struct hinic_rxq),
 391                                  RTE_CACHE_LINE_SIZE, socket_id);
 392         if (!rxq) {
 393                 PMD_DRV_LOG(ERR, "Allocate rxq[%d] failed, dev_name: %s",
 394                             queue_idx, dev->data->name);
 395                 return -ENOMEM;
 396         }
 397         nic_dev->rxqs[queue_idx] = rxq;
 398
 399         /* alloc rx sq hw wqepage*/
 400         rc = hinic_create_rq(hwdev, queue_idx, rq_depth);
 401         if (rc) {
 402                 PMD_DRV_LOG(ERR, "Create rxq[%d] failed, dev_name: %s, rq_depth: %d",
 403                             queue_idx, dev->data->name, rq_depth);
 404                 goto ceate_rq_fail;
 405         }
 406
 407         /* mbuf pool must be assigned before setup rx resources */
 408         rxq->mb_pool = mp;
 409
 410         rc =
 411         hinic_convert_rx_buf_size(rte_pktmbuf_data_room_size(rxq->mb_pool) -
 412                                   RTE_PKTMBUF_HEADROOM, &buf_size);
 413         if (rc) {
 414                 PMD_DRV_LOG(ERR, "Adjust buf size failed, dev_name: %s",
 415                             dev->data->name);
 416                 goto adjust_bufsize_fail;
 417         }
 418
 419         /* rx queue info, rearm control */
 420         rxq->wq = &hwdev->nic_io->rq_wq[queue_idx];
 421         rxq->pi_virt_addr = hwdev->nic_io->qps[queue_idx].rq.pi_virt_addr;
 422         rxq->nic_dev = nic_dev;
 423         rxq->q_id = queue_idx;
 424         rxq->q_depth = rq_depth;
 425         rxq->buf_len = (u16)buf_size;
 426         rxq->rx_free_thresh = rx_free_thresh;
 427
 428         /* the last point cant do mbuf rearm in bulk */
 429         rxq->rxinfo_align_end = rxq->q_depth - rxq->rx_free_thresh;
 430
 431         /* device port identifier */
 432         rxq->port_id = dev->data->port_id;
 433
 434         /* alloc rx_cqe and prepare rq_wqe */
 435         rc = hinic_setup_rx_resources(rxq);
 436         if (rc) {
 437                 PMD_DRV_LOG(ERR, "Setup rxq[%d] rx_resources failed, dev_name:%s",
 438                             queue_idx, dev->data->name);
 439                 goto setup_rx_res_err;
 440         }
 441
 442         /* record nic_dev rxq in rte_eth rx_queues */
 443         dev->data->rx_queues[queue_idx] = rxq;
 444
 445         return 0;
 446
 447 setup_rx_res_err:
 448 adjust_bufsize_fail:
 449         hinic_destroy_rq(hwdev, queue_idx);
 450
 451 ceate_rq_fail:
 452         rte_free(rxq);
 453
 454         return rc;
 455 }
 456
 457 static void hinic_reset_rx_queue(struct rte_eth_dev *dev)
 458 {
 459         struct hinic_rxq *rxq;
 460         struct hinic_nic_dev *nic_dev;
 461         int q_id = 0;
 462
 463         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 464
 465         for (q_id = 0; q_id < nic_dev->num_rq; q_id++) {
 466                 rxq = dev->data->rx_queues[q_id];
 467
 468                 rxq->wq->cons_idx = 0;
 469                 rxq->wq->prod_idx = 0;
 470                 rxq->wq->delta = rxq->q_depth;
 471                 rxq->wq->mask = rxq->q_depth - 1;
 472
 473                 /* alloc mbuf to rq */
 474                 hinic_rx_alloc_pkts(rxq);
 475         }
 476 }
 477
 478 /**
 479  * DPDK callback to configure the transmit queue.
 480  *
 481  * @param dev
 482  *   Pointer to Ethernet device structure.
 483  * @param queue_idx
 484  *   Transmit queue index.
 485  * @param nb_desc
 486  *   Number of descriptors for transmit queue.
 487  * @param socket_id
 488  *   NUMA socket on which memory must be allocated.
 489  * @param tx_conf
 490  *   Tx queue configuration parameters.
 491  *
 492  * @return
 493  *   0 on success, negative error value otherwise.
 494  */
 495 static int hinic_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 496                          uint16_t nb_desc, unsigned int socket_id,
 497                          __rte_unused const struct rte_eth_txconf *tx_conf)
 498 {
 499         int rc;
 500         struct hinic_nic_dev *nic_dev;
 501         struct hinic_hwdev *hwdev;
 502         struct hinic_txq *txq;
 503         u16 sq_depth, tx_free_thresh;
 504
 505         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 506         hwdev = nic_dev->hwdev;
 507
 508         /* queue depth must be power of 2, otherwise will be aligned up */
 509         sq_depth = (nb_desc & (nb_desc - 1)) ?
 510                         ((u16)(1U << (ilog2(nb_desc) + 1))) : nb_desc;
 511
 512         /*
 513          * Validate number of transmit descriptors.
 514          * It must not exceed hardware maximum and minimum.
 515          */
 516         if (sq_depth > HINIC_MAX_QUEUE_DEPTH ||
 517                 sq_depth < HINIC_MIN_QUEUE_DEPTH) {
 518                 PMD_DRV_LOG(ERR, "TX queue depth is out of range from %d to %d, (nb_desc=%d, q_depth=%d, port=%d queue=%d)",
 519                           HINIC_MIN_QUEUE_DEPTH, HINIC_MAX_QUEUE_DEPTH,
 520                           (int)nb_desc, (int)sq_depth,
 521                           (int)dev->data->port_id, (int)queue_idx);
 522                 return -EINVAL;
 523         }
 524
 525         /*
 526          * The TX descriptor ring will be cleaned after txq->tx_free_thresh
 527          * descriptors are used or if the number of descriptors required
 528          * to transmit a packet is greater than the number of free TX
 529          * descriptors.
 530          * The following constraints must be satisfied:
 531          *  tx_free_thresh must be greater than 0.
 532          *  tx_free_thresh must be less than the size of the ring minus 1.
 533          * When set to zero use default values.
 534          */
 535         tx_free_thresh = (u16)((tx_conf->tx_free_thresh) ?
 536                         tx_conf->tx_free_thresh : HINIC_DEFAULT_TX_FREE_THRESH);
 537         if (tx_free_thresh >= (sq_depth - 1)) {
 538                 PMD_DRV_LOG(ERR, "tx_free_thresh must be less than the number of TX descriptors minus 1. (tx_free_thresh=%u port=%d queue=%d)",
 539                         (unsigned int)tx_free_thresh, (int)dev->data->port_id,
 540                         (int)queue_idx);
 541                 return -EINVAL;
 542         }
 543
 544         txq = rte_zmalloc_socket("hinic_tx_queue", sizeof(struct hinic_txq),
 545                                  RTE_CACHE_LINE_SIZE, socket_id);
 546         if (!txq) {
 547                 PMD_DRV_LOG(ERR, "Allocate txq[%d] failed, dev_name: %s",
 548                             queue_idx, dev->data->name);
 549                 return -ENOMEM;
 550         }
 551         nic_dev->txqs[queue_idx] = txq;
 552
 553         /* alloc tx sq hw wqepage */
 554         rc = hinic_create_sq(hwdev, queue_idx, sq_depth);
 555         if (rc) {
 556                 PMD_DRV_LOG(ERR, "Create txq[%d] failed, dev_name: %s, sq_depth: %d",
 557                             queue_idx, dev->data->name, sq_depth);
 558                 goto create_sq_fail;
 559         }
 560
 561         txq->q_id = queue_idx;
 562         txq->q_depth = sq_depth;
 563         txq->port_id = dev->data->port_id;
 564         txq->tx_free_thresh = tx_free_thresh;
 565         txq->nic_dev = nic_dev;
 566         txq->wq = &hwdev->nic_io->sq_wq[queue_idx];
 567         txq->sq = &hwdev->nic_io->qps[queue_idx].sq;
 568         txq->cons_idx_addr = hwdev->nic_io->qps[queue_idx].sq.cons_idx_addr;
 569         txq->sq_head_addr = HINIC_GET_WQ_HEAD(txq);
 570         txq->sq_bot_sge_addr = HINIC_GET_WQ_TAIL(txq) -
 571                                         sizeof(struct hinic_sq_bufdesc);
 572         txq->cos = nic_dev->default_cos;
 573
 574         /* alloc software txinfo */
 575         rc = hinic_setup_tx_resources(txq);
 576         if (rc) {
 577                 PMD_DRV_LOG(ERR, "Setup txq[%d] tx_resources failed, dev_name: %s",
 578                             queue_idx, dev->data->name);
 579                 goto setup_tx_res_fail;
 580         }
 581
 582         /* record nic_dev txq in rte_eth tx_queues */
 583         dev->data->tx_queues[queue_idx] = txq;
 584
 585         return HINIC_OK;
 586
 587 setup_tx_res_fail:
 588         hinic_destroy_sq(hwdev, queue_idx);
 589
 590 create_sq_fail:
 591         rte_free(txq);
 592
 593         return rc;
 594 }
 595
 596 static void hinic_reset_tx_queue(struct rte_eth_dev *dev)
 597 {
 598         struct hinic_nic_dev *nic_dev;
 599         struct hinic_txq *txq;
 600         struct hinic_nic_io *nic_io;
 601         struct hinic_hwdev *hwdev;
 602         volatile u32 *ci_addr;
 603         int q_id = 0;
 604
 605         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 606         hwdev = nic_dev->hwdev;
 607         nic_io = hwdev->nic_io;
 608
 609         for (q_id = 0; q_id < nic_dev->num_sq; q_id++) {
 610                 txq = dev->data->tx_queues[q_id];
 611
 612                 txq->wq->cons_idx = 0;
 613                 txq->wq->prod_idx = 0;
 614                 txq->wq->delta = txq->q_depth;
 615                 txq->wq->mask  = txq->q_depth - 1;
 616
 617                 /* clear hardware ci */
 618                 ci_addr = (volatile u32 *)HINIC_CI_VADDR(nic_io->ci_vaddr_base,
 619                                                         q_id);
 620                 *ci_addr = 0;
 621         }
 622 }
 623
 624 /**
 625  * Get link speed from NIC.
 626  *
 627  * @param dev
 628  *   Pointer to Ethernet device structure.
 629  * @param speed_capa
 630  *   Pointer to link speed structure.
 631  */
 632 static void hinic_get_speed_capa(struct rte_eth_dev *dev, uint32_t *speed_capa)
 633 {
 634         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 635         u32 supported_link, advertised_link;
 636         int err;
 637
 638 #define HINIC_LINK_MODE_SUPPORT_1G      (1U << HINIC_GE_BASE_KX)
 639
 640 #define HINIC_LINK_MODE_SUPPORT_10G     (1U << HINIC_10GE_BASE_KR)
 641
 642 #define HINIC_LINK_MODE_SUPPORT_25G     ((1U << HINIC_25GE_BASE_KR_S) | \
 643                                         (1U << HINIC_25GE_BASE_CR_S) | \
 644                                         (1U << HINIC_25GE_BASE_KR) | \
 645                                         (1U << HINIC_25GE_BASE_CR))
 646
 647 #define HINIC_LINK_MODE_SUPPORT_40G     ((1U << HINIC_40GE_BASE_KR4) | \
 648                                         (1U << HINIC_40GE_BASE_CR4))
 649
 650 #define HINIC_LINK_MODE_SUPPORT_100G    ((1U << HINIC_100GE_BASE_KR4) | \
 651                                         (1U << HINIC_100GE_BASE_CR4))
 652
 653         err = hinic_get_link_mode(nic_dev->hwdev,
 654                                   &supported_link, &advertised_link);
 655         if (err || supported_link == HINIC_SUPPORTED_UNKNOWN ||
 656             advertised_link == HINIC_SUPPORTED_UNKNOWN) {
 657                 PMD_DRV_LOG(WARNING, "Get speed capability info failed, device: %s, port_id: %u",
 658                           nic_dev->proc_dev_name, dev->data->port_id);
 659         } else {
 660                 *speed_capa = 0;
 661                 if (!!(supported_link & HINIC_LINK_MODE_SUPPORT_1G))
 662                         *speed_capa |= ETH_LINK_SPEED_1G;
 663                 if (!!(supported_link & HINIC_LINK_MODE_SUPPORT_10G))
 664                         *speed_capa |= ETH_LINK_SPEED_10G;
 665                 if (!!(supported_link & HINIC_LINK_MODE_SUPPORT_25G))
 666                         *speed_capa |= ETH_LINK_SPEED_25G;
 667                 if (!!(supported_link & HINIC_LINK_MODE_SUPPORT_40G))
 668                         *speed_capa |= ETH_LINK_SPEED_40G;
 669                 if (!!(supported_link & HINIC_LINK_MODE_SUPPORT_100G))
 670                         *speed_capa |= ETH_LINK_SPEED_100G;
 671         }
 672 }
 673
 674 /**
 675  * DPDK callback to get information about the device.
 676  *
 677  * @param dev
 678  *   Pointer to Ethernet device structure.
 679  * @param info
 680  *   Pointer to Info structure output buffer.
 681  */
 682 static int
 683 hinic_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
 684 {
 685         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 686
 687         info->max_rx_queues  = nic_dev->nic_cap.max_rqs;
 688         info->max_tx_queues  = nic_dev->nic_cap.max_sqs;
 689         info->min_rx_bufsize = HINIC_MIN_RX_BUF_SIZE;
 690         info->max_rx_pktlen  = HINIC_MAX_JUMBO_FRAME_SIZE;
 691         info->max_mac_addrs  = HINIC_MAX_MAC_ADDRS;
 692
 693         hinic_get_speed_capa(dev, &info->speed_capa);
 694         info->rx_queue_offload_capa = 0;
 695         info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
 696                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 697                                 DEV_RX_OFFLOAD_UDP_CKSUM |
 698                                 DEV_RX_OFFLOAD_TCP_CKSUM;
 699
 700         info->tx_queue_offload_capa = 0;
 701         info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
 702                                 DEV_TX_OFFLOAD_IPV4_CKSUM |
 703                                 DEV_TX_OFFLOAD_UDP_CKSUM |
 704                                 DEV_TX_OFFLOAD_TCP_CKSUM |
 705                                 DEV_TX_OFFLOAD_SCTP_CKSUM |
 706                                 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
 707                                 DEV_TX_OFFLOAD_TCP_TSO |
 708                                 DEV_TX_OFFLOAD_MULTI_SEGS;
 709
 710         info->hash_key_size = HINIC_RSS_KEY_SIZE;
 711         info->reta_size = HINIC_RSS_INDIR_SIZE;
 712         info->flow_type_rss_offloads = HINIC_RSS_OFFLOAD_ALL;
 713         info->rx_desc_lim = hinic_rx_desc_lim;
 714         info->tx_desc_lim = hinic_tx_desc_lim;
 715
 716         return 0;
 717 }
 718
 719 static int hinic_config_rx_mode(struct hinic_nic_dev *nic_dev, u32 rx_mode_ctrl)
 720 {
 721         int err;
 722
 723         err = hinic_set_rx_mode(nic_dev->hwdev, rx_mode_ctrl);
 724         if (err) {
 725                 PMD_DRV_LOG(ERR, "Failed to set rx mode");
 726                 return -EINVAL;
 727         }
 728         nic_dev->rx_mode_status = rx_mode_ctrl;
 729
 730         return 0;
 731 }
 732
 733
 734 static int hinic_rxtx_configure(struct rte_eth_dev *dev)
 735 {
 736         int err;
 737         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 738
 739         /* rx configure, if rss enable, need to init default configuration */
 740         err = hinic_rx_configure(dev);
 741         if (err) {
 742                 PMD_DRV_LOG(ERR, "Configure rss failed");
 743                 return err;
 744         }
 745
 746         /* rx mode init */
 747         err = hinic_config_rx_mode(nic_dev, HINIC_DEFAULT_RX_MODE);
 748         if (err) {
 749                 PMD_DRV_LOG(ERR, "Configure rx_mode:0x%x failed",
 750                         HINIC_DEFAULT_RX_MODE);
 751                 goto set_rx_mode_fail;
 752         }
 753
 754         return HINIC_OK;
 755
 756 set_rx_mode_fail:
 757         hinic_rx_remove_configure(dev);
 758
 759         return err;
 760 }
 761
 762 static void hinic_remove_rxtx_configure(struct rte_eth_dev *dev)
 763 {
 764         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 765
 766         (void)hinic_config_rx_mode(nic_dev, 0);
 767         hinic_rx_remove_configure(dev);
 768 }
 769
 770 static int hinic_priv_get_dev_link_status(struct hinic_nic_dev *nic_dev,
 771                                           struct rte_eth_link *link)
 772 {
 773         int rc;
 774         u8 port_link_status = 0;
 775         struct nic_port_info port_link_info;
 776         struct hinic_hwdev *nic_hwdev = nic_dev->hwdev;
 777         uint32_t port_speed[LINK_SPEED_MAX] = {ETH_SPEED_NUM_10M,
 778                                         ETH_SPEED_NUM_100M, ETH_SPEED_NUM_1G,
 779                                         ETH_SPEED_NUM_10G, ETH_SPEED_NUM_25G,
 780                                         ETH_SPEED_NUM_40G, ETH_SPEED_NUM_100G};
 781
 782         rc = hinic_get_link_status(nic_hwdev, &port_link_status);
 783         if (rc)
 784                 return rc;
 785
 786         if (!port_link_status) {
 787                 link->link_status = ETH_LINK_DOWN;
 788                 link->link_speed = 0;
 789                 link->link_duplex = ETH_LINK_HALF_DUPLEX;
 790                 link->link_autoneg = ETH_LINK_FIXED;
 791                 return HINIC_OK;
 792         }
 793
 794         memset(&port_link_info, 0, sizeof(port_link_info));
 795         rc = hinic_get_port_info(nic_hwdev, &port_link_info);
 796         if (rc)
 797                 return rc;
 798
 799         link->link_speed = port_speed[port_link_info.speed % LINK_SPEED_MAX];
 800         link->link_duplex = port_link_info.duplex;
 801         link->link_autoneg = port_link_info.autoneg_state;
 802         link->link_status = port_link_status;
 803
 804         return HINIC_OK;
 805 }
 806
 807 /**
 808  * DPDK callback to retrieve physical link information.
 809  *
 810  * @param dev
 811  *   Pointer to Ethernet device structure.
 812  * @param wait_to_complete
 813  *   Wait for request completion.
 814  *
 815  * @return
 816  *   0 link status changed, -1 link status not changed
 817  */
 818 static int hinic_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 819 {
 820 #define CHECK_INTERVAL 10  /* 10ms */
 821 #define MAX_REPEAT_TIME 100  /* 1s (100 * 10ms) in total */
 822         int rc = HINIC_OK;
 823         struct rte_eth_link link;
 824         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 825         unsigned int rep_cnt = MAX_REPEAT_TIME;
 826
 827         memset(&link, 0, sizeof(link));
 828         do {
 829                 /* Get link status information from hardware */
 830                 rc = hinic_priv_get_dev_link_status(nic_dev, &link);
 831                 if (rc != HINIC_OK) {
 832                         link.link_speed = ETH_SPEED_NUM_NONE;
 833                         link.link_duplex = ETH_LINK_FULL_DUPLEX;
 834                         PMD_DRV_LOG(ERR, "Get link status failed");
 835                         goto out;
 836                 }
 837
 838                 if (!wait_to_complete || link.link_status)
 839                         break;
 840
 841                 rte_delay_ms(CHECK_INTERVAL);
 842         } while (rep_cnt--);
 843
 844 out:
 845         rc = rte_eth_linkstatus_set(dev, &link);
 846         return rc;
 847 }
 848
 849 /**
 850  * DPDK callback to start the device.
 851  *
 852  * @param dev
 853  *   Pointer to Ethernet device structure.
 854  *
 855  * @return
 856  *   0 on success, negative errno value on failure.
 857  */
 858 static int hinic_dev_start(struct rte_eth_dev *dev)
 859 {
 860         int rc;
 861         char *name;
 862         struct hinic_nic_dev *nic_dev;
 863
 864         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 865         name = dev->data->name;
 866
 867         /* reset rx and tx queue */
 868         hinic_reset_rx_queue(dev);
 869         hinic_reset_tx_queue(dev);
 870
 871         /* get func rx buf size */
 872         hinic_get_func_rx_buf_size(nic_dev);
 873
 874         /* init txq and rxq context */
 875         rc = hinic_init_qp_ctxts(nic_dev->hwdev);
 876         if (rc) {
 877                 PMD_DRV_LOG(ERR, "Initialize qp context failed, dev_name:%s",
 878                             name);
 879                 goto init_qp_fail;
 880         }
 881
 882         /* rss template */
 883         rc = hinic_config_mq_mode(dev, TRUE);
 884         if (rc) {
 885                 PMD_DRV_LOG(ERR, "Configure mq mode failed, dev_name: %s",
 886                             name);
 887                 goto cfg_mq_mode_fail;
 888         }
 889
 890         /* set default mtu */
 891         rc = hinic_set_port_mtu(nic_dev->hwdev, nic_dev->mtu_size);
 892         if (rc) {
 893                 PMD_DRV_LOG(ERR, "Set mtu_size[%d] failed, dev_name: %s",
 894                             nic_dev->mtu_size, name);
 895                 goto set_mtu_fail;
 896         }
 897
 898         /* configure rss rx_mode and other rx or tx default feature */
 899         rc = hinic_rxtx_configure(dev);
 900         if (rc) {
 901                 PMD_DRV_LOG(ERR, "Configure tx and rx failed, dev_name: %s",
 902                             name);
 903                 goto cfg_rxtx_fail;
 904         }
 905
 906         /* reactive pf status, so that uP report asyn event */
 907         hinic_set_pf_status(nic_dev->hwdev->hwif, HINIC_PF_STATUS_ACTIVE_FLAG);
 908
 909         /* open virtual port and ready to start packet receiving */
 910         rc = hinic_set_vport_enable(nic_dev->hwdev, true);
 911         if (rc) {
 912                 PMD_DRV_LOG(ERR, "Enable vport failed, dev_name:%s", name);
 913                 goto en_vport_fail;
 914         }
 915
 916         /* open physical port and start packet receiving */
 917         rc = hinic_set_port_enable(nic_dev->hwdev, true);
 918         if (rc) {
 919                 PMD_DRV_LOG(ERR, "Enable physical port failed, dev_name:%s",
 920                             name);
 921                 goto en_port_fail;
 922         }
 923
 924         /* update eth_dev link status */
 925         if (dev->data->dev_conf.intr_conf.lsc != 0)
 926                 (void)hinic_link_update(dev, 0);
 927
 928         hinic_set_bit(HINIC_DEV_START, &nic_dev->dev_status);
 929
 930         return 0;
 931
 932 en_port_fail:
 933         (void)hinic_set_vport_enable(nic_dev->hwdev, false);
 934
 935 en_vport_fail:
 936         hinic_set_pf_status(nic_dev->hwdev->hwif, HINIC_PF_STATUS_INIT);
 937
 938         /* Flush tx && rx chip resources in case of set vport fake fail */
 939         (void)hinic_flush_qp_res(nic_dev->hwdev);
 940         rte_delay_ms(100);
 941
 942         hinic_remove_rxtx_configure(dev);
 943
 944 cfg_rxtx_fail:
 945 set_mtu_fail:
 946 cfg_mq_mode_fail:
 947         hinic_free_qp_ctxts(nic_dev->hwdev);
 948
 949 init_qp_fail:
 950         hinic_free_all_rx_mbuf(dev);
 951         hinic_free_all_tx_mbuf(dev);
 952
 953         return rc;
 954 }
 955
 956 /**
 957  * DPDK callback to release the receive queue.
 958  *
 959  * @param queue
 960  *   Generic receive queue pointer.
 961  */
 962 static void hinic_rx_queue_release(void *queue)
 963 {
 964         struct hinic_rxq *rxq = queue;
 965         struct hinic_nic_dev *nic_dev;
 966
 967         if (!rxq) {
 968                 PMD_DRV_LOG(WARNING, "Rxq is null when release");
 969                 return;
 970         }
 971         nic_dev = rxq->nic_dev;
 972
 973         /* free rxq_pkt mbuf */
 974         hinic_free_all_rx_skbs(rxq);
 975
 976         /* free rxq_cqe, rxq_info */
 977         hinic_free_rx_resources(rxq);
 978
 979         /* free root rq wq */
 980         hinic_destroy_rq(nic_dev->hwdev, rxq->q_id);
 981
 982         nic_dev->rxqs[rxq->q_id] = NULL;
 983
 984         /* free rxq */
 985         rte_free(rxq);
 986 }
 987
 988 /**
 989  * DPDK callback to release the transmit queue.
 990  *
 991  * @param queue
 992  *   Generic transmit queue pointer.
 993  */
 994 static void hinic_tx_queue_release(void *queue)
 995 {
 996         struct hinic_txq *txq = queue;
 997         struct hinic_nic_dev *nic_dev;
 998
 999         if (!txq) {
1000                 PMD_DRV_LOG(WARNING, "Txq is null when release");
1001                 return;
1002         }
1003         nic_dev = txq->nic_dev;
1004
1005         /* free txq_pkt mbuf */
1006         hinic_free_all_tx_skbs(txq);
1007
1008         /* free txq_info */
1009         hinic_free_tx_resources(txq);
1010
1011         /* free root sq wq */
1012         hinic_destroy_sq(nic_dev->hwdev, txq->q_id);
1013         nic_dev->txqs[txq->q_id] = NULL;
1014
1015         /* free txq */
1016         rte_free(txq);
1017 }
1018
1019 static void hinic_free_all_rq(struct hinic_nic_dev *nic_dev)
1020 {
1021         u16 q_id;
1022
1023         for (q_id = 0; q_id < nic_dev->num_rq; q_id++)
1024                 hinic_destroy_rq(nic_dev->hwdev, q_id);
1025 }
1026
1027 static void hinic_free_all_sq(struct hinic_nic_dev *nic_dev)
1028 {
1029         u16 q_id;
1030
1031         for (q_id = 0; q_id < nic_dev->num_sq; q_id++)
1032                 hinic_destroy_sq(nic_dev->hwdev, q_id);
1033 }
1034
1035 /**
1036  * DPDK callback to stop the device.
1037  *
1038  * @param dev
1039  *   Pointer to Ethernet device structure.
1040  */
1041 static void hinic_dev_stop(struct rte_eth_dev *dev)
1042 {
1043         int rc;
1044         char *name;
1045         uint16_t port_id;
1046         struct hinic_nic_dev *nic_dev;
1047         struct rte_eth_link link;
1048
1049         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1050         name = dev->data->name;
1051         port_id = dev->data->port_id;
1052
1053         if (!hinic_test_and_clear_bit(HINIC_DEV_START, &nic_dev->dev_status)) {
1054                 PMD_DRV_LOG(INFO, "Device %s already stopped", name);
1055                 return;
1056         }
1057
1058         /* just stop phy port and vport */
1059         rc = hinic_set_port_enable(nic_dev->hwdev, false);
1060         if (rc)
1061                 PMD_DRV_LOG(WARNING, "Disable phy port failed, error: %d, dev_name:%s, port_id:%d",
1062                           rc, name, port_id);
1063
1064         rc = hinic_set_vport_enable(nic_dev->hwdev, false);
1065         if (rc)
1066                 PMD_DRV_LOG(WARNING, "Disable vport failed, error: %d, dev_name:%s, port_id:%d",
1067                           rc, name, port_id);
1068
1069         /* Clear recorded link status */
1070         memset(&link, 0, sizeof(link));
1071         (void)rte_eth_linkstatus_set(dev, &link);
1072
1073         /* flush pending io request */
1074         rc = hinic_rx_tx_flush(nic_dev->hwdev);
1075         if (rc)
1076                 PMD_DRV_LOG(WARNING, "Flush pending io failed, error: %d, dev_name: %s, port_id: %d",
1077                             rc, name, port_id);
1078
1079         /* clean rss table and rx_mode */
1080         hinic_remove_rxtx_configure(dev);
1081
1082         /* clean root context */
1083         hinic_free_qp_ctxts(nic_dev->hwdev);
1084
1085         /* free mbuf */
1086         hinic_free_all_rx_mbuf(dev);
1087         hinic_free_all_tx_mbuf(dev);
1088 }
1089
1090 static void hinic_disable_interrupt(struct rte_eth_dev *dev)
1091 {
1092         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1093         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1094         int ret, retries = 0;
1095
1096         hinic_clear_bit(HINIC_DEV_INTR_EN, &nic_dev->dev_status);
1097
1098         /* disable msix interrupt in hardware */
1099         hinic_set_msix_state(nic_dev->hwdev, 0, HINIC_MSIX_DISABLE);
1100
1101         /* disable rte interrupt */
1102         ret = rte_intr_disable(&pci_dev->intr_handle);
1103         if (ret)
1104                 PMD_DRV_LOG(ERR, "Disable intr failed: %d", ret);
1105
1106         do {
1107                 ret =
1108                 rte_intr_callback_unregister(&pci_dev->intr_handle,
1109                                              hinic_dev_interrupt_handler, dev);
1110                 if (ret >= 0) {
1111                         break;
1112                 } else if (ret == -EAGAIN) {
1113                         rte_delay_ms(100);
1114                         retries++;
1115                 } else {
1116                         PMD_DRV_LOG(ERR, "intr callback unregister failed: %d",
1117                                     ret);
1118                         break;
1119                 }
1120         } while (retries < HINIC_INTR_CB_UNREG_MAX_RETRIES);
1121
1122         if (retries == HINIC_INTR_CB_UNREG_MAX_RETRIES)
1123                 PMD_DRV_LOG(ERR, "Unregister intr callback failed after %d retries",
1124                             retries);
1125 }
1126
1127 static int hinic_set_dev_promiscuous(struct hinic_nic_dev *nic_dev, bool enable)
1128 {
1129         u32 rx_mode_ctrl = nic_dev->rx_mode_status;
1130
1131         if (enable)
1132                 rx_mode_ctrl |= HINIC_RX_MODE_PROMISC;
1133         else
1134                 rx_mode_ctrl &= (~HINIC_RX_MODE_PROMISC);
1135
1136         return hinic_config_rx_mode(nic_dev, rx_mode_ctrl);
1137 }
1138
1139 /**
1140  * DPDK callback to get device statistics.
1141  *
1142  * @param dev
1143  *   Pointer to Ethernet device structure.
1144  * @param stats
1145  *   Stats structure output buffer.
1146  *
1147  * @return
1148  *   0 on success and stats is filled,
1149  *   negative error value otherwise.
1150  */
1151 static int
1152 hinic_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1153 {
1154         int i, err, q_num;
1155         u64 rx_discards_pmd = 0;
1156         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1157         struct hinic_vport_stats vport_stats;
1158         struct hinic_rxq        *rxq = NULL;
1159         struct hinic_rxq_stats rxq_stats;
1160         struct hinic_txq        *txq = NULL;
1161         struct hinic_txq_stats txq_stats;
1162
1163         err = hinic_get_vport_stats(nic_dev->hwdev, &vport_stats);
1164         if (err) {
1165                 PMD_DRV_LOG(ERR, "Get vport stats from fw failed, nic_dev: %s",
1166                         nic_dev->proc_dev_name);
1167                 return err;
1168         }
1169
1170         /* rx queue stats */
1171         q_num = (nic_dev->num_rq < RTE_ETHDEV_QUEUE_STAT_CNTRS) ?
1172                         nic_dev->num_rq : RTE_ETHDEV_QUEUE_STAT_CNTRS;
1173         for (i = 0; i < q_num; i++) {
1174                 rxq = nic_dev->rxqs[i];
1175                 hinic_rxq_get_stats(rxq, &rxq_stats);
1176                 stats->q_ipackets[i] = rxq_stats.packets;
1177                 stats->q_ibytes[i] = rxq_stats.bytes;
1178                 stats->q_errors[i] = rxq_stats.rx_discards;
1179
1180                 stats->ierrors += rxq_stats.errors;
1181                 rx_discards_pmd += rxq_stats.rx_discards;
1182                 dev->data->rx_mbuf_alloc_failed += rxq_stats.rx_nombuf;
1183         }
1184
1185         /* tx queue stats */
1186         q_num = (nic_dev->num_sq < RTE_ETHDEV_QUEUE_STAT_CNTRS) ?
1187                 nic_dev->num_sq : RTE_ETHDEV_QUEUE_STAT_CNTRS;
1188         for (i = 0; i < q_num; i++) {
1189                 txq = nic_dev->txqs[i];
1190                 hinic_txq_get_stats(txq, &txq_stats);
1191                 stats->q_opackets[i] = txq_stats.packets;
1192                 stats->q_obytes[i] = txq_stats.bytes;
1193                 stats->oerrors += (txq_stats.tx_busy + txq_stats.off_errs);
1194         }
1195
1196         /* vport stats */
1197         stats->oerrors += vport_stats.tx_discard_vport;
1198
1199         stats->imissed = vport_stats.rx_discard_vport + rx_discards_pmd;
1200
1201         stats->ipackets = (vport_stats.rx_unicast_pkts_vport +
1202                         vport_stats.rx_multicast_pkts_vport +
1203                         vport_stats.rx_broadcast_pkts_vport -
1204                         rx_discards_pmd);
1205
1206         stats->opackets = (vport_stats.tx_unicast_pkts_vport +
1207                         vport_stats.tx_multicast_pkts_vport +
1208                         vport_stats.tx_broadcast_pkts_vport);
1209
1210         stats->ibytes = (vport_stats.rx_unicast_bytes_vport +
1211                         vport_stats.rx_multicast_bytes_vport +
1212                         vport_stats.rx_broadcast_bytes_vport);
1213
1214         stats->obytes = (vport_stats.tx_unicast_bytes_vport +
1215                         vport_stats.tx_multicast_bytes_vport +
1216                         vport_stats.tx_broadcast_bytes_vport);
1217         return 0;
1218 }
1219
1220 /**
1221  * DPDK callback to clear device statistics.
1222  *
1223  * @param dev
1224  *   Pointer to Ethernet device structure.
1225  */
1226 static int hinic_dev_stats_reset(struct rte_eth_dev *dev)
1227 {
1228         int qid;
1229         struct hinic_rxq        *rxq = NULL;
1230         struct hinic_txq        *txq = NULL;
1231         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1232         int ret;
1233
1234         ret = hinic_clear_vport_stats(nic_dev->hwdev);
1235         if (ret != 0)
1236                 return ret;
1237
1238         for (qid = 0; qid < nic_dev->num_rq; qid++) {
1239                 rxq = nic_dev->rxqs[qid];
1240                 hinic_rxq_stats_reset(rxq);
1241         }
1242
1243         for (qid = 0; qid < nic_dev->num_sq; qid++) {
1244                 txq = nic_dev->txqs[qid];
1245                 hinic_txq_stats_reset(txq);
1246         }
1247
1248         return 0;
1249 }
1250
1251 /**
1252  * DPDK callback to clear device extended statistics.
1253  *
1254  * @param dev
1255  *   Pointer to Ethernet device structure.
1256  */
1257 static int hinic_dev_xstats_reset(struct rte_eth_dev *dev)
1258 {
1259         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1260         int ret;
1261
1262         ret = hinic_dev_stats_reset(dev);
1263         if (ret != 0)
1264                 return ret;
1265
1266         if (hinic_func_type(nic_dev->hwdev) != TYPE_VF) {
1267                 ret = hinic_clear_phy_port_stats(nic_dev->hwdev);
1268                 if (ret != 0)
1269                         return ret;
1270         }
1271
1272         return 0;
1273 }
1274
1275 static void hinic_gen_random_mac_addr(struct rte_ether_addr *mac_addr)
1276 {
1277         uint64_t random_value;
1278
1279         /* Set Organizationally Unique Identifier (OUI) prefix */
1280         mac_addr->addr_bytes[0] = 0x00;
1281         mac_addr->addr_bytes[1] = 0x09;
1282         mac_addr->addr_bytes[2] = 0xC0;
1283         /* Force indication of locally assigned MAC address. */
1284         mac_addr->addr_bytes[0] |= RTE_ETHER_LOCAL_ADMIN_ADDR;
1285         /* Generate the last 3 bytes of the MAC address with a random number. */
1286         random_value = rte_rand();
1287         memcpy(&mac_addr->addr_bytes[3], &random_value, 3);
1288 }
1289
1290 /**
1291  * Init mac_vlan table in NIC.
1292  *
1293  * @param dev
1294  *   Pointer to Ethernet device structure.
1295  *
1296  * @return
1297  *   0 on success and stats is filled,
1298  *   negative error value otherwise.
1299  */
1300 static int hinic_init_mac_addr(struct rte_eth_dev *eth_dev)
1301 {
1302         struct hinic_nic_dev *nic_dev =
1303                                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
1304         uint8_t addr_bytes[RTE_ETHER_ADDR_LEN];
1305         u16 func_id = 0;
1306         int rc = 0;
1307
1308         rc = hinic_get_default_mac(nic_dev->hwdev, addr_bytes);
1309         if (rc)
1310                 return rc;
1311
1312         memmove(eth_dev->data->mac_addrs->addr_bytes,
1313                 addr_bytes, RTE_ETHER_ADDR_LEN);
1314
1315         if (rte_is_zero_ether_addr(eth_dev->data->mac_addrs))
1316                 hinic_gen_random_mac_addr(eth_dev->data->mac_addrs);
1317
1318         func_id = hinic_global_func_id(nic_dev->hwdev);
1319         rc = hinic_set_mac(nic_dev->hwdev, eth_dev->data->mac_addrs->addr_bytes,
1320                            0, func_id);
1321         if (rc && rc != HINIC_PF_SET_VF_ALREADY)
1322                 return rc;
1323
1324         return 0;
1325 }
1326
1327 /**
1328  * Deinit mac_vlan table in NIC.
1329  *
1330  * @param dev
1331  *   Pointer to Ethernet device structure.
1332  *
1333  * @return
1334  *   0 on success and stats is filled,
1335  *   negative error value otherwise.
1336  */
1337 static void hinic_deinit_mac_addr(struct rte_eth_dev *eth_dev)
1338 {
1339         struct hinic_nic_dev *nic_dev =
1340                                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
1341         int rc;
1342         u16 func_id = 0;
1343
1344         if (rte_is_zero_ether_addr(eth_dev->data->mac_addrs))
1345                 return;
1346
1347         func_id = hinic_global_func_id(nic_dev->hwdev);
1348         rc = hinic_del_mac(nic_dev->hwdev,
1349                            eth_dev->data->mac_addrs->addr_bytes,
1350                            0, func_id);
1351         if (rc && rc != HINIC_PF_SET_VF_ALREADY)
1352                 PMD_DRV_LOG(ERR, "Delete mac table failed, dev_name: %s",
1353                             eth_dev->data->name);
1354 }
1355
1356 /**
1357  * DPDK callback to enable promiscuous mode.
1358  *
1359  * @param dev
1360  *   Pointer to Ethernet device structure.
1361  *
1362  * @return
1363  *   0 on success,
1364  *   negative error value otherwise.
1365  */
1366 static int hinic_dev_promiscuous_enable(struct rte_eth_dev *dev)
1367 {
1368         int rc = HINIC_OK;
1369         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1370
1371         PMD_DRV_LOG(INFO, "Enable promiscuous, nic_dev: %s, port_id: %d, promisc: %d",
1372                     nic_dev->proc_dev_name, dev->data->port_id,
1373                     dev->data->promiscuous);
1374
1375         rc = hinic_set_dev_promiscuous(nic_dev, true);
1376         if (rc)
1377                 PMD_DRV_LOG(ERR, "Enable promiscuous failed");
1378
1379         return rc;
1380 }
1381
1382 /**
1383  * DPDK callback to disable promiscuous mode.
1384  *
1385  * @param dev
1386  *   Pointer to Ethernet device structure.
1387  *
1388  * @return
1389  *   0 on success,
1390  *   negative error value otherwise.
1391  */
1392 static int hinic_dev_promiscuous_disable(struct rte_eth_dev *dev)
1393 {
1394         int rc = HINIC_OK;
1395         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1396
1397         PMD_DRV_LOG(INFO, "Disable promiscuous, nic_dev: %s, port_id: %d, promisc: %d",
1398                     nic_dev->proc_dev_name, dev->data->port_id,
1399                     dev->data->promiscuous);
1400
1401         rc = hinic_set_dev_promiscuous(nic_dev, false);
1402         if (rc)
1403                 PMD_DRV_LOG(ERR, "Disable promiscuous failed");
1404
1405         return rc;
1406 }
1407
1408 /**
1409  * DPDK callback to update the RSS hash key and RSS hash type.
1410  *
1411  * @param dev
1412  *   Pointer to Ethernet device structure.
1413  * @param rss_conf
1414  *   RSS configuration data.
1415  *
1416  * @return
1417  *   0 on success, negative error value otherwise.
1418  */
1419 static int hinic_rss_hash_update(struct rte_eth_dev *dev,
1420                           struct rte_eth_rss_conf *rss_conf)
1421 {
1422         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1423         u8 tmpl_idx = nic_dev->rss_tmpl_idx;
1424         u8 hashkey[HINIC_RSS_KEY_SIZE] = {0};
1425         u8 prio_tc[HINIC_DCB_UP_MAX] = {0};
1426         u64 rss_hf = rss_conf->rss_hf;
1427         struct nic_rss_type rss_type = {0};
1428         int err = 0;
1429
1430         if (!(nic_dev->flags & ETH_MQ_RX_RSS_FLAG)) {
1431                 PMD_DRV_LOG(WARNING, "RSS is not enabled");
1432                 return HINIC_OK;
1433         }
1434
1435         if (rss_conf->rss_key_len > HINIC_RSS_KEY_SIZE) {
1436                 PMD_DRV_LOG(ERR, "Invalid rss key, rss_key_len:%d",
1437                             rss_conf->rss_key_len);
1438                 return HINIC_ERROR;
1439         }
1440
1441         if (rss_conf->rss_key) {
1442                 memcpy(hashkey, rss_conf->rss_key, rss_conf->rss_key_len);
1443                 err = hinic_rss_set_template_tbl(nic_dev->hwdev, tmpl_idx,
1444                                                  hashkey);
1445                 if (err) {
1446                         PMD_DRV_LOG(ERR, "Set rss template table failed");
1447                         goto disable_rss;
1448                 }
1449         }
1450
1451         rss_type.ipv4 = (rss_hf & (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4)) ? 1 : 0;
1452         rss_type.tcp_ipv4 = (rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) ? 1 : 0;
1453         rss_type.ipv6 = (rss_hf & (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6)) ? 1 : 0;
1454         rss_type.ipv6_ext = (rss_hf & ETH_RSS_IPV6_EX) ? 1 : 0;
1455         rss_type.tcp_ipv6 = (rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) ? 1 : 0;
1456         rss_type.tcp_ipv6_ext = (rss_hf & ETH_RSS_IPV6_TCP_EX) ? 1 : 0;
1457         rss_type.udp_ipv4 = (rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) ? 1 : 0;
1458         rss_type.udp_ipv6 = (rss_hf & ETH_RSS_NONFRAG_IPV6_UDP) ? 1 : 0;
1459
1460         err = hinic_set_rss_type(nic_dev->hwdev, tmpl_idx, rss_type);
1461         if (err) {
1462                 PMD_DRV_LOG(ERR, "Set rss type table failed");
1463                 goto disable_rss;
1464         }
1465
1466         return 0;
1467
1468 disable_rss:
1469         memset(prio_tc, 0, sizeof(prio_tc));
1470         (void)hinic_rss_cfg(nic_dev->hwdev, 0, tmpl_idx, 0, prio_tc);
1471         return err;
1472 }
1473
1474 /**
1475  * DPDK callback to get the RSS hash configuration.
1476  *
1477  * @param dev
1478  *   Pointer to Ethernet device structure.
1479  * @param rss_conf
1480  *   RSS configuration data.
1481  *
1482  * @return
1483  *   0 on success, negative error value otherwise.
1484  */
1485 static int hinic_rss_conf_get(struct rte_eth_dev *dev,
1486                        struct rte_eth_rss_conf *rss_conf)
1487 {
1488         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1489         u8 tmpl_idx = nic_dev->rss_tmpl_idx;
1490         u8 hashkey[HINIC_RSS_KEY_SIZE] = {0};
1491         struct nic_rss_type rss_type = {0};
1492         int err;
1493
1494         if (!(nic_dev->flags & ETH_MQ_RX_RSS_FLAG)) {
1495                 PMD_DRV_LOG(WARNING, "RSS is not enabled");
1496                 return HINIC_ERROR;
1497         }
1498
1499         err = hinic_rss_get_template_tbl(nic_dev->hwdev, tmpl_idx, hashkey);
1500         if (err)
1501                 return err;
1502
1503         if (rss_conf->rss_key &&
1504             rss_conf->rss_key_len >= HINIC_RSS_KEY_SIZE) {
1505                 memcpy(rss_conf->rss_key, hashkey, sizeof(hashkey));
1506                 rss_conf->rss_key_len = sizeof(hashkey);
1507         }
1508
1509         err = hinic_get_rss_type(nic_dev->hwdev, tmpl_idx, &rss_type);
1510         if (err)
1511                 return err;
1512
1513         rss_conf->rss_hf = 0;
1514         rss_conf->rss_hf |=  rss_type.ipv4 ?
1515                 (ETH_RSS_IPV4 | ETH_RSS_FRAG_IPV4) : 0;
1516         rss_conf->rss_hf |=  rss_type.tcp_ipv4 ? ETH_RSS_NONFRAG_IPV4_TCP : 0;
1517         rss_conf->rss_hf |=  rss_type.ipv6 ?
1518                 (ETH_RSS_IPV6 | ETH_RSS_FRAG_IPV6) : 0;
1519         rss_conf->rss_hf |=  rss_type.ipv6_ext ? ETH_RSS_IPV6_EX : 0;
1520         rss_conf->rss_hf |=  rss_type.tcp_ipv6 ? ETH_RSS_NONFRAG_IPV6_TCP : 0;
1521         rss_conf->rss_hf |=  rss_type.tcp_ipv6_ext ? ETH_RSS_IPV6_TCP_EX : 0;
1522         rss_conf->rss_hf |=  rss_type.udp_ipv4 ? ETH_RSS_NONFRAG_IPV4_UDP : 0;
1523         rss_conf->rss_hf |=  rss_type.udp_ipv6 ? ETH_RSS_NONFRAG_IPV6_UDP : 0;
1524
1525         return HINIC_OK;
1526 }
1527
1528 /**
1529  * DPDK callback to update the RETA indirection table.
1530  *
1531  * @param dev
1532  *   Pointer to Ethernet device structure.
1533  * @param reta_conf
1534  *   Pointer to RETA configuration structure array.
1535  * @param reta_size
1536  *   Size of the RETA table.
1537  *
1538  * @return
1539  *   0 on success, negative error value otherwise.
1540  */
1541 static int hinic_rss_indirtbl_update(struct rte_eth_dev *dev,
1542                               struct rte_eth_rss_reta_entry64 *reta_conf,
1543                               uint16_t reta_size)
1544 {
1545         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1546         u8 tmpl_idx = nic_dev->rss_tmpl_idx;
1547         u8 prio_tc[HINIC_DCB_UP_MAX] = {0};
1548         u32 indirtbl[NIC_RSS_INDIR_SIZE] = {0};
1549         int err = 0;
1550         u16 i = 0;
1551         u16 idx, shift;
1552
1553         if (!(nic_dev->flags & ETH_MQ_RX_RSS_FLAG))
1554                 return HINIC_OK;
1555
1556         if (reta_size != NIC_RSS_INDIR_SIZE) {
1557                 PMD_DRV_LOG(ERR, "Invalid reta size, reta_size:%d", reta_size);
1558                 return HINIC_ERROR;
1559         }
1560
1561         err = hinic_rss_get_indir_tbl(nic_dev->hwdev, tmpl_idx, indirtbl);
1562         if (err)
1563                 return err;
1564
1565         /* update rss indir_tbl */
1566         for (i = 0; i < reta_size; i++) {
1567                 idx = i / RTE_RETA_GROUP_SIZE;
1568                 shift = i % RTE_RETA_GROUP_SIZE;
1569                 if (reta_conf[idx].mask & (1ULL << shift))
1570                         indirtbl[i] = reta_conf[idx].reta[shift];
1571         }
1572
1573         for (i = 0 ; i < reta_size; i++) {
1574                 if (indirtbl[i] >= nic_dev->num_rq) {
1575                         PMD_DRV_LOG(ERR, "Invalid reta entry, index:%d, num_rq:%d",
1576                                     i, nic_dev->num_rq);
1577                         goto disable_rss;
1578                 }
1579         }
1580
1581         err = hinic_rss_set_indir_tbl(nic_dev->hwdev, tmpl_idx, indirtbl);
1582         if (err)
1583                 goto disable_rss;
1584
1585         nic_dev->rss_indir_flag = true;
1586
1587         return 0;
1588
1589 disable_rss:
1590         memset(prio_tc, 0, sizeof(prio_tc));
1591         (void)hinic_rss_cfg(nic_dev->hwdev, 0, tmpl_idx, 0, prio_tc);
1592
1593         return HINIC_ERROR;
1594 }
1595
1596
1597 /**
1598  * DPDK callback to get the RETA indirection table.
1599  *
1600  * @param dev
1601  *   Pointer to Ethernet device structure.
1602  * @param reta_conf
1603  *   Pointer to RETA configuration structure array.
1604  * @param reta_size
1605  *   Size of the RETA table.
1606  *
1607  * @return
1608  *   0 on success, negative error value otherwise.
1609  */
1610 static int hinic_rss_indirtbl_query(struct rte_eth_dev *dev,
1611                              struct rte_eth_rss_reta_entry64 *reta_conf,
1612                              uint16_t reta_size)
1613 {
1614         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1615         u8 tmpl_idx = nic_dev->rss_tmpl_idx;
1616         int err = 0;
1617         u32 indirtbl[NIC_RSS_INDIR_SIZE] = {0};
1618         u16 idx, shift;
1619         u16 i = 0;
1620
1621         if (reta_size != NIC_RSS_INDIR_SIZE) {
1622                 PMD_DRV_LOG(ERR, "Invalid reta size, reta_size:%d", reta_size);
1623                 return HINIC_ERROR;
1624         }
1625
1626         err = hinic_rss_get_indir_tbl(nic_dev->hwdev, tmpl_idx, indirtbl);
1627         if (err) {
1628                 PMD_DRV_LOG(ERR, "Get rss indirect table failed, error:%d",
1629                             err);
1630                 return err;
1631         }
1632
1633         for (i = 0; i < reta_size; i++) {
1634                 idx = i / RTE_RETA_GROUP_SIZE;
1635                 shift = i % RTE_RETA_GROUP_SIZE;
1636                 if (reta_conf[idx].mask & (1ULL << shift))
1637                         reta_conf[idx].reta[shift] = (uint16_t)indirtbl[i];
1638         }
1639
1640         return HINIC_OK;
1641 }
1642
1643 /**
1644  * DPDK callback to get extended device statistics.
1645  *
1646  * @param dev
1647  *   Pointer to Ethernet device.
1648  * @param xstats
1649  *   Pointer to rte extended stats table.
1650  * @param n
1651  *   The size of the stats table.
1652  *
1653  * @return
1654  *   Number of extended stats on success and stats is filled,
1655  *   negative error value otherwise.
1656  */
1657 static int hinic_dev_xstats_get(struct rte_eth_dev *dev,
1658                          struct rte_eth_xstat *xstats,
1659                          unsigned int n)
1660 {
1661         u16 qid = 0;
1662         u32 i;
1663         int err, count;
1664         struct hinic_nic_dev *nic_dev;
1665         struct hinic_phy_port_stats port_stats;
1666         struct hinic_vport_stats vport_stats;
1667         struct hinic_rxq        *rxq = NULL;
1668         struct hinic_rxq_stats rxq_stats;
1669         struct hinic_txq        *txq = NULL;
1670         struct hinic_txq_stats txq_stats;
1671
1672         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1673         count = hinic_xstats_calc_num(nic_dev);
1674         if ((int)n < count)
1675                 return count;
1676
1677         count = 0;
1678
1679         /* Get stats from hinic_rxq_stats */
1680         for (qid = 0; qid < nic_dev->num_rq; qid++) {
1681                 rxq = nic_dev->rxqs[qid];
1682                 hinic_rxq_get_stats(rxq, &rxq_stats);
1683
1684                 for (i = 0; i < HINIC_RXQ_XSTATS_NUM; i++) {
1685                         xstats[count].value =
1686                                 *(uint64_t *)(((char *)&rxq_stats) +
1687                                 hinic_rxq_stats_strings[i].offset);
1688                         xstats[count].id = count;
1689                         count++;
1690                 }
1691         }
1692
1693         /* Get stats from hinic_txq_stats */
1694         for (qid = 0; qid < nic_dev->num_sq; qid++) {
1695                 txq = nic_dev->txqs[qid];
1696                 hinic_txq_get_stats(txq, &txq_stats);
1697
1698                 for (i = 0; i < HINIC_TXQ_XSTATS_NUM; i++) {
1699                         xstats[count].value =
1700                                 *(uint64_t *)(((char *)&txq_stats) +
1701                                 hinic_txq_stats_strings[i].offset);
1702                         xstats[count].id = count;
1703                         count++;
1704                 }
1705         }
1706
1707         /* Get stats from hinic_vport_stats */
1708         err = hinic_get_vport_stats(nic_dev->hwdev, &vport_stats);
1709         if (err)
1710                 return err;
1711
1712         for (i = 0; i < HINIC_VPORT_XSTATS_NUM; i++) {
1713                 xstats[count].value =
1714                         *(uint64_t *)(((char *)&vport_stats) +
1715                         hinic_vport_stats_strings[i].offset);
1716                 xstats[count].id = count;
1717                 count++;
1718         }
1719
1720         if (HINIC_IS_VF(nic_dev->hwdev))
1721                 return count;
1722
1723         /* Get stats from hinic_phy_port_stats */
1724         err = hinic_get_phy_port_stats(nic_dev->hwdev, &port_stats);
1725         if (err)
1726                 return err;
1727
1728         for (i = 0; i < HINIC_PHYPORT_XSTATS_NUM; i++) {
1729                 xstats[count].value = *(uint64_t *)(((char *)&port_stats) +
1730                                 hinic_phyport_stats_strings[i].offset);
1731                 xstats[count].id = count;
1732                 count++;
1733         }
1734
1735         return count;
1736 }
1737
1738 /**
1739  * DPDK callback to retrieve names of extended device statistics
1740  *
1741  * @param dev
1742  *   Pointer to Ethernet device structure.
1743  * @param xstats_names
1744  *   Buffer to insert names into.
1745  *
1746  * @return
1747  *   Number of xstats names.
1748  */
1749 static int hinic_dev_xstats_get_names(struct rte_eth_dev *dev,
1750                                struct rte_eth_xstat_name *xstats_names,
1751                                __rte_unused unsigned int limit)
1752 {
1753         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1754         int count = 0;
1755         u16 i = 0, q_num;
1756
1757         if (xstats_names == NULL)
1758                 return hinic_xstats_calc_num(nic_dev);
1759
1760         /* get pmd rxq stats */
1761         for (q_num = 0; q_num < nic_dev->num_rq; q_num++) {
1762                 for (i = 0; i < HINIC_RXQ_XSTATS_NUM; i++) {
1763                         snprintf(xstats_names[count].name,
1764                                  sizeof(xstats_names[count].name),
1765                                  "rxq%d_%s_pmd",
1766                                  q_num, hinic_rxq_stats_strings[i].name);
1767                         count++;
1768                 }
1769         }
1770
1771         /* get pmd txq stats */
1772         for (q_num = 0; q_num < nic_dev->num_sq; q_num++) {
1773                 for (i = 0; i < HINIC_TXQ_XSTATS_NUM; i++) {
1774                         snprintf(xstats_names[count].name,
1775                                  sizeof(xstats_names[count].name),
1776                                  "txq%d_%s_pmd",
1777                                  q_num, hinic_txq_stats_strings[i].name);
1778                         count++;
1779                 }
1780         }
1781
1782         /* get vport stats */
1783         for (i = 0; i < HINIC_VPORT_XSTATS_NUM; i++) {
1784                 snprintf(xstats_names[count].name,
1785                          sizeof(xstats_names[count].name),
1786                          "%s",
1787                          hinic_vport_stats_strings[i].name);
1788                 count++;
1789         }
1790
1791         if (HINIC_IS_VF(nic_dev->hwdev))
1792                 return count;
1793
1794         /* get phy port stats */
1795         for (i = 0; i < HINIC_PHYPORT_XSTATS_NUM; i++) {
1796                 snprintf(xstats_names[count].name,
1797                          sizeof(xstats_names[count].name),
1798                          "%s",
1799                          hinic_phyport_stats_strings[i].name);
1800                 count++;
1801         }
1802
1803         return count;
1804 }
1805
1806 static int hinic_set_default_pause_feature(struct hinic_nic_dev *nic_dev)
1807 {
1808         struct nic_pause_config pause_config = {0};
1809
1810         pause_config.auto_neg = 0;
1811         pause_config.rx_pause = HINIC_DEFAUT_PAUSE_CONFIG;
1812         pause_config.tx_pause = HINIC_DEFAUT_PAUSE_CONFIG;
1813
1814         return hinic_set_pause_config(nic_dev->hwdev, pause_config);
1815 }
1816
1817 static int hinic_set_default_dcb_feature(struct hinic_nic_dev *nic_dev)
1818 {
1819         u8 up_tc[HINIC_DCB_UP_MAX] = {0};
1820         u8 up_pgid[HINIC_DCB_UP_MAX] = {0};
1821         u8 up_bw[HINIC_DCB_UP_MAX] = {0};
1822         u8 pg_bw[HINIC_DCB_UP_MAX] = {0};
1823         u8 up_strict[HINIC_DCB_UP_MAX] = {0};
1824         int i = 0;
1825
1826         pg_bw[0] = 100;
1827         for (i = 0; i < HINIC_DCB_UP_MAX; i++)
1828                 up_bw[i] = 100;
1829
1830         return hinic_dcb_set_ets(nic_dev->hwdev, up_tc, pg_bw,
1831                                         up_pgid, up_bw, up_strict);
1832 }
1833
1834 static int hinic_init_default_cos(struct hinic_nic_dev *nic_dev)
1835 {
1836         u8 cos_id = 0;
1837         int err;
1838
1839         if (!HINIC_IS_VF(nic_dev->hwdev)) {
1840                 nic_dev->default_cos =
1841                                 (hinic_global_func_id(nic_dev->hwdev) +
1842                                                 DEFAULT_BASE_COS) % NR_MAX_COS;
1843         } else {
1844                 err = hinic_vf_get_default_cos(nic_dev->hwdev, &cos_id);
1845                 if (err) {
1846                         PMD_DRV_LOG(ERR, "Get VF default cos failed, err: %d",
1847                                         err);
1848                         return HINIC_ERROR;
1849                 }
1850
1851                 nic_dev->default_cos = cos_id;
1852         }
1853
1854         return 0;
1855 }
1856
1857 static int hinic_set_default_hw_feature(struct hinic_nic_dev *nic_dev)
1858 {
1859         int err;
1860
1861         err = hinic_init_default_cos(nic_dev);
1862         if (err)
1863                 return err;
1864
1865         if (hinic_func_type(nic_dev->hwdev) == TYPE_VF)
1866                 return 0;
1867
1868         /* Restore DCB configure to default status */
1869         err = hinic_set_default_dcb_feature(nic_dev);
1870         if (err)
1871                 return err;
1872
1873         /* disable LRO */
1874         err = hinic_set_rx_lro(nic_dev->hwdev, 0, 0, (u8)0);
1875         if (err)
1876                 return err;
1877
1878         /* Set pause enable, and up will disable pfc. */
1879         err = hinic_set_default_pause_feature(nic_dev);
1880         if (err)
1881                 return err;
1882
1883         err = hinic_reset_port_link_cfg(nic_dev->hwdev);
1884         if (err)
1885                 return err;
1886
1887         err = hinic_set_link_status_follow(nic_dev->hwdev,
1888                                            HINIC_LINK_FOLLOW_PORT);
1889         if (err == HINIC_MGMT_CMD_UNSUPPORTED)
1890                 PMD_DRV_LOG(WARNING, "Don't support to set link status follow phy port status");
1891         else if (err)
1892                 return err;
1893
1894         return hinic_set_anti_attack(nic_dev->hwdev, true);
1895 }
1896
1897 static int32_t hinic_card_workmode_check(struct hinic_nic_dev *nic_dev)
1898 {
1899         struct hinic_board_info info = { 0 };
1900         int rc;
1901
1902         if (hinic_func_type(nic_dev->hwdev) == TYPE_VF)
1903                 return 0;
1904
1905         rc = hinic_get_board_info(nic_dev->hwdev, &info);
1906         if (rc)
1907                 return rc;
1908
1909         return (info.service_mode == HINIC_SERVICE_MODE_NIC ? HINIC_OK :
1910                                                 HINIC_ERROR);
1911 }
1912
1913 static int hinic_copy_mempool_init(struct hinic_nic_dev *nic_dev)
1914 {
1915         nic_dev->cpy_mpool = rte_mempool_lookup(nic_dev->proc_dev_name);
1916         if (nic_dev->cpy_mpool == NULL) {
1917                 nic_dev->cpy_mpool =
1918                 rte_pktmbuf_pool_create(nic_dev->proc_dev_name,
1919                                         HINIC_COPY_MEMPOOL_DEPTH,
1920                                         0, 0,
1921                                         HINIC_COPY_MBUF_SIZE,
1922                                         rte_socket_id());
1923                 if (!nic_dev->cpy_mpool) {
1924                         PMD_DRV_LOG(ERR, "Create copy mempool failed, errno: %d, dev_name: %s",
1925                                     rte_errno, nic_dev->proc_dev_name);
1926                         return -ENOMEM;
1927                 }
1928         }
1929
1930         return 0;
1931 }
1932
1933 static void hinic_copy_mempool_uninit(struct hinic_nic_dev *nic_dev)
1934 {
1935         if (nic_dev->cpy_mpool != NULL)
1936                 rte_mempool_free(nic_dev->cpy_mpool);
1937 }
1938
1939 static int hinic_init_sw_rxtxqs(struct hinic_nic_dev *nic_dev)
1940 {
1941         u32 txq_size;
1942         u32 rxq_size;
1943
1944         /* allocate software txq array */
1945         txq_size = nic_dev->nic_cap.max_sqs * sizeof(*nic_dev->txqs);
1946         nic_dev->txqs = kzalloc_aligned(txq_size, GFP_KERNEL);
1947         if (!nic_dev->txqs) {
1948                 PMD_DRV_LOG(ERR, "Allocate txqs failed");
1949                 return -ENOMEM;
1950         }
1951
1952         /* allocate software rxq array */
1953         rxq_size = nic_dev->nic_cap.max_rqs * sizeof(*nic_dev->rxqs);
1954         nic_dev->rxqs = kzalloc_aligned(rxq_size, GFP_KERNEL);
1955         if (!nic_dev->rxqs) {
1956                 /* free txqs */
1957                 kfree(nic_dev->txqs);
1958                 nic_dev->txqs = NULL;
1959
1960                 PMD_DRV_LOG(ERR, "Allocate rxqs failed");
1961                 return -ENOMEM;
1962         }
1963
1964         return HINIC_OK;
1965 }
1966
1967 static void hinic_deinit_sw_rxtxqs(struct hinic_nic_dev *nic_dev)
1968 {
1969         kfree(nic_dev->txqs);
1970         nic_dev->txqs = NULL;
1971
1972         kfree(nic_dev->rxqs);
1973         nic_dev->rxqs = NULL;
1974 }
1975
1976 static int hinic_nic_dev_create(struct rte_eth_dev *eth_dev)
1977 {
1978         struct hinic_nic_dev *nic_dev =
1979                                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
1980         int rc;
1981
1982         nic_dev->hwdev = rte_zmalloc("hinic_hwdev", sizeof(*nic_dev->hwdev),
1983                                      RTE_CACHE_LINE_SIZE);
1984         if (!nic_dev->hwdev) {
1985                 PMD_DRV_LOG(ERR, "Allocate hinic hwdev memory failed, dev_name: %s",
1986                             eth_dev->data->name);
1987                 return -ENOMEM;
1988         }
1989         nic_dev->hwdev->pcidev_hdl = RTE_ETH_DEV_TO_PCI(eth_dev);
1990
1991         /* init osdep*/
1992         rc = hinic_osdep_init(nic_dev->hwdev);
1993         if (rc) {
1994                 PMD_DRV_LOG(ERR, "Initialize os_dep failed, dev_name: %s",
1995                             eth_dev->data->name);
1996                 goto init_osdep_fail;
1997         }
1998
1999         /* init_hwif */
2000         rc = hinic_hwif_res_init(nic_dev->hwdev);
2001         if (rc) {
2002                 PMD_DRV_LOG(ERR, "Initialize hwif failed, dev_name: %s",
2003                             eth_dev->data->name);
2004                 goto init_hwif_fail;
2005         }
2006
2007         /* init_cfg_mgmt */
2008         rc = init_cfg_mgmt(nic_dev->hwdev);
2009         if (rc) {
2010                 PMD_DRV_LOG(ERR, "Initialize cfg_mgmt failed, dev_name: %s",
2011                             eth_dev->data->name);
2012                 goto init_cfgmgnt_fail;
2013         }
2014
2015         /* init_aeqs */
2016         rc = hinic_comm_aeqs_init(nic_dev->hwdev);
2017         if (rc) {
2018                 PMD_DRV_LOG(ERR, "Initialize aeqs failed, dev_name: %s",
2019                             eth_dev->data->name);
2020                 goto init_aeqs_fail;
2021         }
2022
2023         /* init_pf_to_mgnt */
2024         rc = hinic_comm_pf_to_mgmt_init(nic_dev->hwdev);
2025         if (rc) {
2026                 PMD_DRV_LOG(ERR, "Initialize pf_to_mgmt failed, dev_name: %s",
2027                             eth_dev->data->name);
2028                 goto init_pf_to_mgmt_fail;
2029         }
2030
2031         /* init mailbox */
2032         rc = hinic_comm_func_to_func_init(nic_dev->hwdev);
2033         if (rc) {
2034                 PMD_DRV_LOG(ERR, "Initialize func_to_func failed, dev_name: %s",
2035                             eth_dev->data->name);
2036                 goto init_func_to_func_fail;
2037         }
2038
2039         rc = hinic_card_workmode_check(nic_dev);
2040         if (rc) {
2041                 PMD_DRV_LOG(ERR, "Check card workmode failed, dev_name: %s",
2042                             eth_dev->data->name);
2043                 goto workmode_check_fail;
2044         }
2045
2046         /* do l2nic reset to make chip clear */
2047         rc = hinic_l2nic_reset(nic_dev->hwdev);
2048         if (rc) {
2049                 PMD_DRV_LOG(ERR, "Do l2nic reset failed, dev_name: %s",
2050                             eth_dev->data->name);
2051                 goto l2nic_reset_fail;
2052         }
2053
2054         /* init dma and aeq msix attribute table */
2055         (void)hinic_init_attr_table(nic_dev->hwdev);
2056
2057         /* init_cmdqs */
2058         rc = hinic_comm_cmdqs_init(nic_dev->hwdev);
2059         if (rc) {
2060                 PMD_DRV_LOG(ERR, "Initialize cmdq failed, dev_name: %s",
2061                             eth_dev->data->name);
2062                 goto init_cmdq_fail;
2063         }
2064
2065         /* set hardware state active */
2066         rc = hinic_activate_hwdev_state(nic_dev->hwdev);
2067         if (rc) {
2068                 PMD_DRV_LOG(ERR, "Initialize resources state failed, dev_name: %s",
2069                             eth_dev->data->name);
2070                 goto init_resources_state_fail;
2071         }
2072
2073         /* init_capability */
2074         rc = hinic_init_capability(nic_dev->hwdev);
2075         if (rc) {
2076                 PMD_DRV_LOG(ERR, "Initialize capability failed, dev_name: %s",
2077                             eth_dev->data->name);
2078                 goto init_cap_fail;
2079         }
2080
2081         /* get nic capability */
2082         if (!hinic_support_nic(nic_dev->hwdev, &nic_dev->nic_cap))
2083                 goto nic_check_fail;
2084
2085         /* init root cla and function table */
2086         rc = hinic_init_nicio(nic_dev->hwdev);
2087         if (rc) {
2088                 PMD_DRV_LOG(ERR, "Initialize nic_io failed, dev_name: %s",
2089                             eth_dev->data->name);
2090                 goto init_nicio_fail;
2091         }
2092
2093         /* init_software_txrxq */
2094         rc = hinic_init_sw_rxtxqs(nic_dev);
2095         if (rc) {
2096                 PMD_DRV_LOG(ERR, "Initialize sw_rxtxqs failed, dev_name: %s",
2097                             eth_dev->data->name);
2098                 goto init_sw_rxtxqs_fail;
2099         }
2100
2101         rc = hinic_copy_mempool_init(nic_dev);
2102         if (rc) {
2103                 PMD_DRV_LOG(ERR, "Create copy mempool failed, dev_name: %s",
2104                          eth_dev->data->name);
2105                 goto init_mpool_fail;
2106         }
2107
2108         /* set hardware feature to default status */
2109         rc = hinic_set_default_hw_feature(nic_dev);
2110         if (rc) {
2111                 PMD_DRV_LOG(ERR, "Initialize hardware default features failed, dev_name: %s",
2112                             eth_dev->data->name);
2113                 goto set_default_hw_feature_fail;
2114         }
2115
2116         return 0;
2117
2118 set_default_hw_feature_fail:
2119         hinic_copy_mempool_uninit(nic_dev);
2120
2121 init_mpool_fail:
2122         hinic_deinit_sw_rxtxqs(nic_dev);
2123
2124 init_sw_rxtxqs_fail:
2125         hinic_deinit_nicio(nic_dev->hwdev);
2126
2127 nic_check_fail:
2128 init_nicio_fail:
2129 init_cap_fail:
2130         hinic_deactivate_hwdev_state(nic_dev->hwdev);
2131
2132 init_resources_state_fail:
2133         hinic_comm_cmdqs_free(nic_dev->hwdev);
2134
2135 init_cmdq_fail:
2136 l2nic_reset_fail:
2137 workmode_check_fail:
2138         hinic_comm_func_to_func_free(nic_dev->hwdev);
2139
2140 init_func_to_func_fail:
2141         hinic_comm_pf_to_mgmt_free(nic_dev->hwdev);
2142
2143 init_pf_to_mgmt_fail:
2144         hinic_comm_aeqs_free(nic_dev->hwdev);
2145
2146 init_aeqs_fail:
2147         free_cfg_mgmt(nic_dev->hwdev);
2148
2149 init_cfgmgnt_fail:
2150         hinic_hwif_res_free(nic_dev->hwdev);
2151
2152 init_hwif_fail:
2153         hinic_osdep_deinit(nic_dev->hwdev);
2154
2155 init_osdep_fail:
2156         rte_free(nic_dev->hwdev);
2157         nic_dev->hwdev = NULL;
2158
2159         return rc;
2160 }
2161
2162 static void hinic_nic_dev_destroy(struct rte_eth_dev *eth_dev)
2163 {
2164         struct hinic_nic_dev *nic_dev =
2165                         HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
2166
2167         (void)hinic_set_link_status_follow(nic_dev->hwdev,
2168                                            HINIC_LINK_FOLLOW_DEFAULT);
2169         hinic_copy_mempool_uninit(nic_dev);
2170         hinic_deinit_sw_rxtxqs(nic_dev);
2171         hinic_deinit_nicio(nic_dev->hwdev);
2172         hinic_deactivate_hwdev_state(nic_dev->hwdev);
2173         hinic_comm_cmdqs_free(nic_dev->hwdev);
2174         hinic_comm_func_to_func_free(nic_dev->hwdev);
2175         hinic_comm_pf_to_mgmt_free(nic_dev->hwdev);
2176         hinic_comm_aeqs_free(nic_dev->hwdev);
2177         free_cfg_mgmt(nic_dev->hwdev);
2178         hinic_hwif_res_free(nic_dev->hwdev);
2179         hinic_osdep_deinit(nic_dev->hwdev);
2180         rte_free(nic_dev->hwdev);
2181         nic_dev->hwdev = NULL;
2182 }
2183
2184 /**
2185  * DPDK callback to close the device.
2186  *
2187  * @param dev
2188  *   Pointer to Ethernet device structure.
2189  */
2190 static void hinic_dev_close(struct rte_eth_dev *dev)
2191 {
2192         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
2193
2194         if (hinic_test_and_set_bit(HINIC_DEV_CLOSE, &nic_dev->dev_status)) {
2195                 PMD_DRV_LOG(WARNING, "Device %s already closed",
2196                             dev->data->name);
2197                 return;
2198         }
2199
2200         /* stop device first */
2201         hinic_dev_stop(dev);
2202
2203         /* rx_cqe, rx_info */
2204         hinic_free_all_rx_resources(dev);
2205
2206         /* tx_info */
2207         hinic_free_all_tx_resources(dev);
2208
2209         /* free wq, pi_dma_addr */
2210         hinic_free_all_rq(nic_dev);
2211
2212         /* free wq, db_addr */
2213         hinic_free_all_sq(nic_dev);
2214
2215         /* deinit mac vlan tbl */
2216         hinic_deinit_mac_addr(dev);
2217
2218         /* disable hardware and uio interrupt */
2219         hinic_disable_interrupt(dev);
2220
2221         /* deinit nic hardware device */
2222         hinic_nic_dev_destroy(dev);
2223 }
2224
2225 static const struct eth_dev_ops hinic_pmd_ops = {
2226         .dev_configure                 = hinic_dev_configure,
2227         .dev_infos_get                 = hinic_dev_infos_get,
2228         .rx_queue_setup                = hinic_rx_queue_setup,
2229         .tx_queue_setup                = hinic_tx_queue_setup,
2230         .dev_start                     = hinic_dev_start,
2231         .link_update                   = hinic_link_update,
2232         .rx_queue_release              = hinic_rx_queue_release,
2233         .tx_queue_release              = hinic_tx_queue_release,
2234         .dev_stop                      = hinic_dev_stop,
2235         .dev_close                     = hinic_dev_close,
2236         .promiscuous_enable            = hinic_dev_promiscuous_enable,
2237         .promiscuous_disable           = hinic_dev_promiscuous_disable,
2238         .rss_hash_update               = hinic_rss_hash_update,
2239         .rss_hash_conf_get             = hinic_rss_conf_get,
2240         .reta_update                   = hinic_rss_indirtbl_update,
2241         .reta_query                    = hinic_rss_indirtbl_query,
2242         .stats_get                     = hinic_dev_stats_get,
2243         .stats_reset                   = hinic_dev_stats_reset,
2244         .xstats_get                    = hinic_dev_xstats_get,
2245         .xstats_reset                  = hinic_dev_xstats_reset,
2246         .xstats_get_names              = hinic_dev_xstats_get_names,
2247 };
2248
2249 static const struct eth_dev_ops hinic_pmd_vf_ops = {
2250         .dev_configure                 = hinic_dev_configure,
2251         .dev_infos_get                 = hinic_dev_infos_get,
2252         .rx_queue_setup                = hinic_rx_queue_setup,
2253         .tx_queue_setup                = hinic_tx_queue_setup,
2254         .dev_start                     = hinic_dev_start,
2255         .link_update                   = hinic_link_update,
2256         .rx_queue_release              = hinic_rx_queue_release,
2257         .tx_queue_release              = hinic_tx_queue_release,
2258         .dev_stop                      = hinic_dev_stop,
2259         .dev_close                     = hinic_dev_close,
2260         .rss_hash_update               = hinic_rss_hash_update,
2261         .rss_hash_conf_get             = hinic_rss_conf_get,
2262         .reta_update                   = hinic_rss_indirtbl_update,
2263         .reta_query                    = hinic_rss_indirtbl_query,
2264         .stats_get                     = hinic_dev_stats_get,
2265         .stats_reset                   = hinic_dev_stats_reset,
2266         .xstats_get                    = hinic_dev_xstats_get,
2267         .xstats_reset                  = hinic_dev_xstats_reset,
2268         .xstats_get_names              = hinic_dev_xstats_get_names,
2269 };
2270
2271 static int hinic_func_init(struct rte_eth_dev *eth_dev)
2272 {
2273         struct rte_pci_device *pci_dev;
2274         struct rte_ether_addr *eth_addr;
2275         struct hinic_nic_dev *nic_dev;
2276         int rc;
2277
2278         pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
2279
2280         /* EAL is SECONDARY and eth_dev is already created */
2281         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2282                 rc = rte_intr_callback_register(&pci_dev->intr_handle,
2283                                                 hinic_dev_interrupt_handler,
2284                                                 (void *)eth_dev);
2285                 if (rc)
2286                         PMD_DRV_LOG(ERR, "Initialize %s failed in secondary process",
2287                                     eth_dev->data->name);
2288
2289                 return rc;
2290         }
2291
2292         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
2293         memset(nic_dev, 0, sizeof(*nic_dev));
2294
2295         snprintf(nic_dev->proc_dev_name,
2296                  sizeof(nic_dev->proc_dev_name),
2297                  "hinic-%.4x:%.2x:%.2x.%x",
2298                  pci_dev->addr.domain, pci_dev->addr.bus,
2299                  pci_dev->addr.devid, pci_dev->addr.function);
2300
2301         /* alloc mac_addrs */
2302         eth_addr = rte_zmalloc("hinic_mac", sizeof(*eth_addr), 0);
2303         if (!eth_addr) {
2304                 PMD_DRV_LOG(ERR, "Allocate ethernet addresses' memory failed, dev_name: %s",
2305                             eth_dev->data->name);
2306                 rc = -ENOMEM;
2307                 goto eth_addr_fail;
2308         }
2309         eth_dev->data->mac_addrs = eth_addr;
2310
2311         /*
2312          * Pass the information to the rte_eth_dev_close() that it should also
2313          * release the private port resources.
2314          */
2315         eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
2316
2317         /* create hardware nic_device */
2318         rc = hinic_nic_dev_create(eth_dev);
2319         if (rc) {
2320                 PMD_DRV_LOG(ERR, "Create nic device failed, dev_name: %s",
2321                             eth_dev->data->name);
2322                 goto create_nic_dev_fail;
2323         }
2324
2325         if (HINIC_IS_VF(nic_dev->hwdev))
2326                 eth_dev->dev_ops = &hinic_pmd_vf_ops;
2327         else
2328                 eth_dev->dev_ops = &hinic_pmd_ops;
2329
2330         rc = hinic_init_mac_addr(eth_dev);
2331         if (rc) {
2332                 PMD_DRV_LOG(ERR, "Initialize mac table failed, dev_name: %s",
2333                             eth_dev->data->name);
2334                 goto init_mac_fail;
2335         }
2336
2337         /* register callback func to eal lib */
2338         rc = rte_intr_callback_register(&pci_dev->intr_handle,
2339                                         hinic_dev_interrupt_handler,
2340                                         (void *)eth_dev);
2341         if (rc) {
2342                 PMD_DRV_LOG(ERR, "Register rte interrupt callback failed, dev_name: %s",
2343                             eth_dev->data->name);
2344                 goto reg_intr_cb_fail;
2345         }
2346
2347         /* enable uio/vfio intr/eventfd mapping */
2348         rc = rte_intr_enable(&pci_dev->intr_handle);
2349         if (rc) {
2350                 PMD_DRV_LOG(ERR, "Enable rte interrupt failed, dev_name: %s",
2351                             eth_dev->data->name);
2352                 goto enable_intr_fail;
2353         }
2354         hinic_set_bit(HINIC_DEV_INTR_EN, &nic_dev->dev_status);
2355
2356         hinic_set_bit(HINIC_DEV_INIT, &nic_dev->dev_status);
2357         PMD_DRV_LOG(INFO, "Initialize %s in primary successfully",
2358                     eth_dev->data->name);
2359
2360         return 0;
2361
2362 enable_intr_fail:
2363         (void)rte_intr_callback_unregister(&pci_dev->intr_handle,
2364                                            hinic_dev_interrupt_handler,
2365                                            (void *)eth_dev);
2366
2367 reg_intr_cb_fail:
2368         hinic_deinit_mac_addr(eth_dev);
2369
2370 init_mac_fail:
2371         eth_dev->dev_ops = NULL;
2372         hinic_nic_dev_destroy(eth_dev);
2373
2374 create_nic_dev_fail:
2375         rte_free(eth_addr);
2376         eth_dev->data->mac_addrs = NULL;
2377
2378 eth_addr_fail:
2379         PMD_DRV_LOG(ERR, "Initialize %s in primary failed",
2380                     eth_dev->data->name);
2381         return rc;
2382 }
2383
2384 static int hinic_dev_init(struct rte_eth_dev *eth_dev)
2385 {
2386         struct rte_pci_device *pci_dev;
2387
2388         pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
2389
2390         PMD_DRV_LOG(INFO, "Initializing pf hinic-%.4x:%.2x:%.2x.%x in %s process",
2391                     pci_dev->addr.domain, pci_dev->addr.bus,
2392                     pci_dev->addr.devid, pci_dev->addr.function,
2393                     (rte_eal_process_type() == RTE_PROC_PRIMARY) ?
2394                     "primary" : "secondary");
2395
2396         /* rte_eth_dev rx_burst and tx_burst */
2397         eth_dev->rx_pkt_burst = hinic_recv_pkts;
2398         eth_dev->tx_pkt_burst = hinic_xmit_pkts;
2399
2400         return hinic_func_init(eth_dev);
2401 }
2402
2403 static int hinic_dev_uninit(struct rte_eth_dev *dev)
2404 {
2405         struct hinic_nic_dev *nic_dev;
2406
2407         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
2408         hinic_clear_bit(HINIC_DEV_INIT, &nic_dev->dev_status);
2409
2410         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2411                 return 0;
2412
2413         hinic_dev_close(dev);
2414
2415         dev->dev_ops = NULL;
2416         dev->rx_pkt_burst = NULL;
2417         dev->tx_pkt_burst = NULL;
2418
2419         rte_free(dev->data->mac_addrs);
2420         dev->data->mac_addrs = NULL;
2421
2422         return HINIC_OK;
2423 }
2424
2425 static struct rte_pci_id pci_id_hinic_map[] = {
2426         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_PRD) },
2427         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_MEZZ_25GE) },
2428         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_MEZZ_40GE) },
2429         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_MEZZ_100GE) },
2430         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_VF) },
2431         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_VF_HV) },
2432         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_1822_DUAL_25GE) },
2433         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_1822_100GE) },
2434         {.vendor_id = 0},
2435 };
2436
2437 static int hinic_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
2438                            struct rte_pci_device *pci_dev)
2439 {
2440         return rte_eth_dev_pci_generic_probe(pci_dev,
2441                 sizeof(struct hinic_nic_dev), hinic_dev_init);
2442 }
2443
2444 static int hinic_pci_remove(struct rte_pci_device *pci_dev)
2445 {
2446         return rte_eth_dev_pci_generic_remove(pci_dev, hinic_dev_uninit);
2447 }
2448
2449 static struct rte_pci_driver rte_hinic_pmd = {
2450         .id_table = pci_id_hinic_map,
2451         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
2452         .probe = hinic_pci_probe,
2453         .remove = hinic_pci_remove,
2454 };
2455
2456 RTE_PMD_REGISTER_PCI(net_hinic, rte_hinic_pmd);
2457 RTE_PMD_REGISTER_PCI_TABLE(net_hinic, pci_id_hinic_map);
2458
2459 RTE_INIT(hinic_init_log)
2460 {
2461         hinic_logtype = rte_log_register("pmd.net.hinic");
2462         if (hinic_logtype >= 0)
2463                 rte_log_set_level(hinic_logtype, RTE_LOG_INFO);
2464 }