drivers/net/hinic/hinic_pmd_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Huawei Technologies Co., Ltd
   3  */
   4
   5 #include <rte_pci.h>
   6 #include <rte_bus_pci.h>
   7 #include <rte_ethdev_pci.h>
   8 #include <rte_mbuf.h>
   9 #include <rte_malloc.h>
  10 #include <rte_memcpy.h>
  11 #include <rte_mempool.h>
  12 #include <rte_errno.h>
  13
  14 #include "base/hinic_compat.h"
  15 #include "base/hinic_pmd_hwdev.h"
  16 #include "base/hinic_pmd_hwif.h"
  17 #include "base/hinic_pmd_wq.h"
  18 #include "base/hinic_pmd_cfg.h"
  19 #include "base/hinic_pmd_mgmt.h"
  20 #include "base/hinic_pmd_cmdq.h"
  21 #include "base/hinic_pmd_niccfg.h"
  22 #include "base/hinic_pmd_nicio.h"
  23 #include "hinic_pmd_ethdev.h"
  24 #include "hinic_pmd_tx.h"
  25 #include "hinic_pmd_rx.h"
  26
  27 /* Vendor ID used by Huawei devices */
  28 #define HINIC_HUAWEI_VENDOR_ID          0x19E5
  29
  30 /* Hinic devices */
  31 #define HINIC_DEV_ID_PRD                0x1822
  32 #define HINIC_DEV_ID_MEZZ_25GE          0x0210
  33 #define HINIC_DEV_ID_MEZZ_40GE          0x020D
  34 #define HINIC_DEV_ID_MEZZ_100GE         0x0205
  35
  36 #define HINIC_SERVICE_MODE_NIC          2
  37
  38 #define HINIC_INTR_CB_UNREG_MAX_RETRIES         10
  39
  40 #define DEFAULT_BASE_COS                4
  41 #define NR_MAX_COS                      8
  42
  43 #define HINIC_MIN_RX_BUF_SIZE           1024
  44 #define HINIC_MAX_MAC_ADDRS             1
  45
  46 /** Driver-specific log messages type. */
  47 int hinic_logtype;
  48
  49 static const struct rte_eth_desc_lim hinic_rx_desc_lim = {
  50         .nb_max = HINIC_MAX_QUEUE_DEPTH,
  51         .nb_min = HINIC_MIN_QUEUE_DEPTH,
  52         .nb_align = HINIC_RXD_ALIGN,
  53 };
  54
  55 static const struct rte_eth_desc_lim hinic_tx_desc_lim = {
  56         .nb_max = HINIC_MAX_QUEUE_DEPTH,
  57         .nb_min = HINIC_MIN_QUEUE_DEPTH,
  58         .nb_align = HINIC_TXD_ALIGN,
  59 };
  60
  61
  62 /**
  63  * Interrupt handler triggered by NIC  for handling
  64  * specific event.
  65  *
  66  * @param: The address of parameter (struct rte_eth_dev *) regsitered before.
  67  **/
  68 static void hinic_dev_interrupt_handler(void *param)
  69 {
  70         struct rte_eth_dev *dev = param;
  71         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
  72
  73         if (!hinic_test_bit(HINIC_DEV_INTR_EN, &nic_dev->dev_status)) {
  74                 PMD_DRV_LOG(WARNING, "Device's interrupt is disabled, ignore interrupt event, dev_name: %s, port_id: %d",
  75                             nic_dev->proc_dev_name, dev->data->port_id);
  76                 return;
  77         }
  78
  79         /* aeq0 msg handler */
  80         hinic_dev_handle_aeq_event(nic_dev->hwdev, param);
  81 }
  82
  83 /**
  84  * Ethernet device configuration.
  85  *
  86  * Prepare the driver for a given number of TX and RX queues, mtu size
  87  * and configure RSS.
  88  *
  89  * @param dev
  90  *   Pointer to Ethernet device structure.
  91  *
  92  * @return
  93  *   0 on success, negative error value otherwise.
  94  */
  95 static int hinic_dev_configure(struct rte_eth_dev *dev)
  96 {
  97         struct hinic_nic_dev *nic_dev;
  98         struct hinic_nic_io *nic_io;
  99         int err;
 100
 101         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 102         nic_io = nic_dev->hwdev->nic_io;
 103
 104         nic_dev->num_sq =  dev->data->nb_tx_queues;
 105         nic_dev->num_rq = dev->data->nb_rx_queues;
 106
 107         nic_io->num_sqs =  dev->data->nb_tx_queues;
 108         nic_io->num_rqs = dev->data->nb_rx_queues;
 109
 110         /* queue pair is max_num(sq, rq) */
 111         nic_dev->num_qps = (nic_dev->num_sq > nic_dev->num_rq) ?
 112                         nic_dev->num_sq : nic_dev->num_rq;
 113         nic_io->num_qps = nic_dev->num_qps;
 114
 115         if (nic_dev->num_qps > nic_io->max_qps) {
 116                 PMD_DRV_LOG(ERR,
 117                         "Queue number out of range, get queue_num:%d, max_queue_num:%d",
 118                         nic_dev->num_qps, nic_io->max_qps);
 119                 return -EINVAL;
 120         }
 121
 122         /* mtu size is 256~9600 */
 123         if (dev->data->dev_conf.rxmode.max_rx_pkt_len < HINIC_MIN_FRAME_SIZE ||
 124             dev->data->dev_conf.rxmode.max_rx_pkt_len >
 125             HINIC_MAX_JUMBO_FRAME_SIZE) {
 126                 PMD_DRV_LOG(ERR,
 127                         "Max rx pkt len out of range, get max_rx_pkt_len:%d, "
 128                         "expect between %d and %d",
 129                         dev->data->dev_conf.rxmode.max_rx_pkt_len,
 130                         HINIC_MIN_FRAME_SIZE, HINIC_MAX_JUMBO_FRAME_SIZE);
 131                 return -EINVAL;
 132         }
 133
 134         nic_dev->mtu_size =
 135                 HINIC_PKTLEN_TO_MTU(dev->data->dev_conf.rxmode.max_rx_pkt_len);
 136
 137         /* rss template */
 138         err = hinic_config_mq_mode(dev, TRUE);
 139         if (err) {
 140                 PMD_DRV_LOG(ERR, "Config multi-queue failed");
 141                 return err;
 142         }
 143
 144         return HINIC_OK;
 145 }
 146
 147 /**
 148  * DPDK callback to create the receive queue.
 149  *
 150  * @param dev
 151  *   Pointer to Ethernet device structure.
 152  * @param queue_idx
 153  *   RX queue index.
 154  * @param nb_desc
 155  *   Number of descriptors for receive queue.
 156  * @param socket_id
 157  *   NUMA socket on which memory must be allocated.
 158  * @param rx_conf
 159  *   Thresholds parameters (unused_).
 160  * @param mp
 161  *   Memory pool for buffer allocations.
 162  *
 163  * @return
 164  *   0 on success, negative error value otherwise.
 165  */
 166 static int hinic_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 167                          uint16_t nb_desc, unsigned int socket_id,
 168                          __rte_unused const struct rte_eth_rxconf *rx_conf,
 169                          struct rte_mempool *mp)
 170 {
 171         int rc;
 172         struct hinic_nic_dev *nic_dev;
 173         struct hinic_hwdev *hwdev;
 174         struct hinic_rxq *rxq;
 175         u16 rq_depth, rx_free_thresh;
 176         u32 buf_size;
 177
 178         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 179         hwdev = nic_dev->hwdev;
 180
 181         /* queue depth must be power of 2, otherwise will be aligned up */
 182         rq_depth = (nb_desc & (nb_desc - 1)) ?
 183                 ((u16)(1U << (ilog2(nb_desc) + 1))) : nb_desc;
 184
 185         /*
 186          * Validate number of receive descriptors.
 187          * It must not exceed hardware maximum and minimum.
 188          */
 189         if (rq_depth > HINIC_MAX_QUEUE_DEPTH ||
 190                 rq_depth < HINIC_MIN_QUEUE_DEPTH) {
 191                 PMD_DRV_LOG(ERR, "RX queue depth is out of range from %d to %d, (nb_desc=%d, q_depth=%d, port=%d queue=%d)",
 192                             HINIC_MIN_QUEUE_DEPTH, HINIC_MAX_QUEUE_DEPTH,
 193                             (int)nb_desc, (int)rq_depth,
 194                             (int)dev->data->port_id, (int)queue_idx);
 195                 return -EINVAL;
 196         }
 197
 198         /*
 199          * The RX descriptor ring will be cleaned after rxq->rx_free_thresh
 200          * descriptors are used or if the number of descriptors required
 201          * to transmit a packet is greater than the number of free RX
 202          * descriptors.
 203          * The following constraints must be satisfied:
 204          *  rx_free_thresh must be greater than 0.
 205          *  rx_free_thresh must be less than the size of the ring minus 1.
 206          * When set to zero use default values.
 207          */
 208         rx_free_thresh = (u16)((rx_conf->rx_free_thresh) ?
 209                         rx_conf->rx_free_thresh : HINIC_DEFAULT_RX_FREE_THRESH);
 210         if (rx_free_thresh >= (rq_depth - 1)) {
 211                 PMD_DRV_LOG(ERR, "rx_free_thresh must be less than the number of RX descriptors minus 1. (rx_free_thresh=%u port=%d queue=%d)",
 212                             (unsigned int)rx_free_thresh,
 213                             (int)dev->data->port_id,
 214                             (int)queue_idx);
 215                 return -EINVAL;
 216         }
 217
 218         rxq = rte_zmalloc_socket("hinic_rx_queue", sizeof(struct hinic_rxq),
 219                                  RTE_CACHE_LINE_SIZE, socket_id);
 220         if (!rxq) {
 221                 PMD_DRV_LOG(ERR, "Allocate rxq[%d] failed, dev_name: %s",
 222                             queue_idx, dev->data->name);
 223                 return -ENOMEM;
 224         }
 225         nic_dev->rxqs[queue_idx] = rxq;
 226
 227         /* alloc rx sq hw wqepage*/
 228         rc = hinic_create_rq(hwdev, queue_idx, rq_depth);
 229         if (rc) {
 230                 PMD_DRV_LOG(ERR, "Create rxq[%d] failed, dev_name: %s, rq_depth: %d",
 231                             queue_idx, dev->data->name, rq_depth);
 232                 goto ceate_rq_fail;
 233         }
 234
 235         /* mbuf pool must be assigned before setup rx resources */
 236         rxq->mb_pool = mp;
 237
 238         rc =
 239         hinic_convert_rx_buf_size(rte_pktmbuf_data_room_size(rxq->mb_pool) -
 240                                   RTE_PKTMBUF_HEADROOM, &buf_size);
 241         if (rc) {
 242                 PMD_DRV_LOG(ERR, "Adjust buf size failed, dev_name: %s",
 243                             dev->data->name);
 244                 goto adjust_bufsize_fail;
 245         }
 246
 247         /* rx queue info, rearm control */
 248         rxq->wq = &hwdev->nic_io->rq_wq[queue_idx];
 249         rxq->pi_virt_addr = hwdev->nic_io->qps[queue_idx].rq.pi_virt_addr;
 250         rxq->nic_dev = nic_dev;
 251         rxq->q_id = queue_idx;
 252         rxq->q_depth = rq_depth;
 253         rxq->buf_len = (u16)buf_size;
 254         rxq->rx_free_thresh = rx_free_thresh;
 255
 256         /* the last point cant do mbuf rearm in bulk */
 257         rxq->rxinfo_align_end = rxq->q_depth - rxq->rx_free_thresh;
 258
 259         /* device port identifier */
 260         rxq->port_id = dev->data->port_id;
 261
 262         /* alloc rx_cqe and prepare rq_wqe */
 263         rc = hinic_setup_rx_resources(rxq);
 264         if (rc) {
 265                 PMD_DRV_LOG(ERR, "Setup rxq[%d] rx_resources failed, dev_name:%s",
 266                             queue_idx, dev->data->name);
 267                 goto setup_rx_res_err;
 268         }
 269
 270         /* record nic_dev rxq in rte_eth rx_queues */
 271         dev->data->rx_queues[queue_idx] = rxq;
 272
 273         return 0;
 274
 275 setup_rx_res_err:
 276 adjust_bufsize_fail:
 277         hinic_destroy_rq(hwdev, queue_idx);
 278
 279 ceate_rq_fail:
 280         rte_free(rxq);
 281
 282         return rc;
 283 }
 284
 285 static void hinic_reset_rx_queue(struct rte_eth_dev *dev)
 286 {
 287         struct hinic_rxq *rxq;
 288         struct hinic_nic_dev *nic_dev;
 289         int q_id = 0;
 290
 291         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 292
 293         for (q_id = 0; q_id < nic_dev->num_rq; q_id++) {
 294                 rxq = dev->data->rx_queues[q_id];
 295
 296                 rxq->wq->cons_idx = 0;
 297                 rxq->wq->prod_idx = 0;
 298                 rxq->wq->delta = rxq->q_depth;
 299                 rxq->wq->mask = rxq->q_depth - 1;
 300
 301                 /* alloc mbuf to rq */
 302                 hinic_rx_alloc_pkts(rxq);
 303         }
 304 }
 305
 306 /**
 307  * DPDK callback to configure the transmit queue.
 308  *
 309  * @param dev
 310  *   Pointer to Ethernet device structure.
 311  * @param queue_idx
 312  *   Transmit queue index.
 313  * @param nb_desc
 314  *   Number of descriptors for transmit queue.
 315  * @param socket_id
 316  *   NUMA socket on which memory must be allocated.
 317  * @param tx_conf
 318  *   Tx queue configuration parameters.
 319  *
 320  * @return
 321  *   0 on success, negative error value otherwise.
 322  */
 323 static int hinic_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 324                          uint16_t nb_desc, unsigned int socket_id,
 325                          __rte_unused const struct rte_eth_txconf *tx_conf)
 326 {
 327         int rc;
 328         struct hinic_nic_dev *nic_dev;
 329         struct hinic_hwdev *hwdev;
 330         struct hinic_txq *txq;
 331         u16 sq_depth, tx_free_thresh;
 332
 333         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 334         hwdev = nic_dev->hwdev;
 335
 336         /* queue depth must be power of 2, otherwise will be aligned up */
 337         sq_depth = (nb_desc & (nb_desc - 1)) ?
 338                         ((u16)(1U << (ilog2(nb_desc) + 1))) : nb_desc;
 339
 340         /*
 341          * Validate number of transmit descriptors.
 342          * It must not exceed hardware maximum and minimum.
 343          */
 344         if (sq_depth > HINIC_MAX_QUEUE_DEPTH ||
 345                 sq_depth < HINIC_MIN_QUEUE_DEPTH) {
 346                 PMD_DRV_LOG(ERR, "TX queue depth is out of range from %d to %d, (nb_desc=%d, q_depth=%d, port=%d queue=%d)",
 347                           HINIC_MIN_QUEUE_DEPTH, HINIC_MAX_QUEUE_DEPTH,
 348                           (int)nb_desc, (int)sq_depth,
 349                           (int)dev->data->port_id, (int)queue_idx);
 350                 return -EINVAL;
 351         }
 352
 353         /*
 354          * The TX descriptor ring will be cleaned after txq->tx_free_thresh
 355          * descriptors are used or if the number of descriptors required
 356          * to transmit a packet is greater than the number of free TX
 357          * descriptors.
 358          * The following constraints must be satisfied:
 359          *  tx_free_thresh must be greater than 0.
 360          *  tx_free_thresh must be less than the size of the ring minus 1.
 361          * When set to zero use default values.
 362          */
 363         tx_free_thresh = (u16)((tx_conf->tx_free_thresh) ?
 364                         tx_conf->tx_free_thresh : HINIC_DEFAULT_TX_FREE_THRESH);
 365         if (tx_free_thresh >= (sq_depth - 1)) {
 366                 PMD_DRV_LOG(ERR, "tx_free_thresh must be less than the number of TX descriptors minus 1. (tx_free_thresh=%u port=%d queue=%d)",
 367                         (unsigned int)tx_free_thresh, (int)dev->data->port_id,
 368                         (int)queue_idx);
 369                 return -EINVAL;
 370         }
 371
 372         txq = rte_zmalloc_socket("hinic_tx_queue", sizeof(struct hinic_txq),
 373                                  RTE_CACHE_LINE_SIZE, socket_id);
 374         if (!txq) {
 375                 PMD_DRV_LOG(ERR, "Allocate txq[%d] failed, dev_name: %s",
 376                             queue_idx, dev->data->name);
 377                 return -ENOMEM;
 378         }
 379         nic_dev->txqs[queue_idx] = txq;
 380
 381         /* alloc tx sq hw wqepage */
 382         rc = hinic_create_sq(hwdev, queue_idx, sq_depth);
 383         if (rc) {
 384                 PMD_DRV_LOG(ERR, "Create txq[%d] failed, dev_name: %s, sq_depth: %d",
 385                             queue_idx, dev->data->name, sq_depth);
 386                 goto create_sq_fail;
 387         }
 388
 389         txq->q_id = queue_idx;
 390         txq->q_depth = sq_depth;
 391         txq->port_id = dev->data->port_id;
 392         txq->tx_free_thresh = tx_free_thresh;
 393         txq->nic_dev = nic_dev;
 394         txq->wq = &hwdev->nic_io->sq_wq[queue_idx];
 395         txq->sq = &hwdev->nic_io->qps[queue_idx].sq;
 396         txq->cons_idx_addr = hwdev->nic_io->qps[queue_idx].sq.cons_idx_addr;
 397         txq->sq_head_addr = HINIC_GET_WQ_HEAD(txq);
 398         txq->sq_bot_sge_addr = HINIC_GET_WQ_TAIL(txq) -
 399                                         sizeof(struct hinic_sq_bufdesc);
 400         txq->cos = nic_dev->default_cos;
 401
 402         /* alloc software txinfo */
 403         rc = hinic_setup_tx_resources(txq);
 404         if (rc) {
 405                 PMD_DRV_LOG(ERR, "Setup txq[%d] tx_resources failed, dev_name: %s",
 406                             queue_idx, dev->data->name);
 407                 goto setup_tx_res_fail;
 408         }
 409
 410         /* record nic_dev txq in rte_eth tx_queues */
 411         dev->data->tx_queues[queue_idx] = txq;
 412
 413         return HINIC_OK;
 414
 415 setup_tx_res_fail:
 416         hinic_destroy_sq(hwdev, queue_idx);
 417
 418 create_sq_fail:
 419         rte_free(txq);
 420
 421         return rc;
 422 }
 423
 424 static void hinic_reset_tx_queue(struct rte_eth_dev *dev)
 425 {
 426         struct hinic_nic_dev *nic_dev;
 427         struct hinic_txq *txq;
 428         struct hinic_nic_io *nic_io;
 429         struct hinic_hwdev *hwdev;
 430         volatile u32 *ci_addr;
 431         int q_id = 0;
 432
 433         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 434         hwdev = nic_dev->hwdev;
 435         nic_io = hwdev->nic_io;
 436
 437         for (q_id = 0; q_id < nic_dev->num_sq; q_id++) {
 438                 txq = dev->data->tx_queues[q_id];
 439
 440                 txq->wq->cons_idx = 0;
 441                 txq->wq->prod_idx = 0;
 442                 txq->wq->delta = txq->q_depth;
 443                 txq->wq->mask  = txq->q_depth - 1;
 444
 445                 /*clear hardware ci*/
 446                 ci_addr = (volatile u32 *)HINIC_CI_VADDR(nic_io->ci_vaddr_base,
 447                                                         q_id);
 448                 *ci_addr = 0;
 449         }
 450 }
 451
 452 /**
 453  * Get link speed from NIC.
 454  *
 455  * @param dev
 456  *   Pointer to Ethernet device structure.
 457  * @param speed_capa
 458  *   Pointer to link speed structure.
 459  */
 460 static void hinic_get_speed_capa(struct rte_eth_dev *dev, uint32_t *speed_capa)
 461 {
 462         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 463         u32 supported_link, advertised_link;
 464         int err;
 465
 466 #define HINIC_LINK_MODE_SUPPORT_1G      (1U << HINIC_GE_BASE_KX)
 467
 468 #define HINIC_LINK_MODE_SUPPORT_10G     (1U << HINIC_10GE_BASE_KR)
 469
 470 #define HINIC_LINK_MODE_SUPPORT_25G     ((1U << HINIC_25GE_BASE_KR_S) | \
 471                                         (1U << HINIC_25GE_BASE_CR_S) | \
 472                                         (1U << HINIC_25GE_BASE_KR) | \
 473                                         (1U << HINIC_25GE_BASE_CR))
 474
 475 #define HINIC_LINK_MODE_SUPPORT_40G     ((1U << HINIC_40GE_BASE_KR4) | \
 476                                         (1U << HINIC_40GE_BASE_CR4))
 477
 478 #define HINIC_LINK_MODE_SUPPORT_100G    ((1U << HINIC_100GE_BASE_KR4) | \
 479                                         (1U << HINIC_100GE_BASE_CR4))
 480
 481         err = hinic_get_link_mode(nic_dev->hwdev,
 482                                   &supported_link, &advertised_link);
 483         if (err || supported_link == HINIC_SUPPORTED_UNKNOWN ||
 484             advertised_link == HINIC_SUPPORTED_UNKNOWN) {
 485                 PMD_DRV_LOG(WARNING, "Get speed capability info failed, device: %s, port_id: %u",
 486                           nic_dev->proc_dev_name, dev->data->port_id);
 487         } else {
 488                 *speed_capa = 0;
 489                 if (!!(supported_link & HINIC_LINK_MODE_SUPPORT_1G))
 490                         *speed_capa |= ETH_LINK_SPEED_1G;
 491                 if (!!(supported_link & HINIC_LINK_MODE_SUPPORT_10G))
 492                         *speed_capa |= ETH_LINK_SPEED_10G;
 493                 if (!!(supported_link & HINIC_LINK_MODE_SUPPORT_25G))
 494                         *speed_capa |= ETH_LINK_SPEED_25G;
 495                 if (!!(supported_link & HINIC_LINK_MODE_SUPPORT_40G))
 496                         *speed_capa |= ETH_LINK_SPEED_40G;
 497                 if (!!(supported_link & HINIC_LINK_MODE_SUPPORT_100G))
 498                         *speed_capa |= ETH_LINK_SPEED_100G;
 499         }
 500 }
 501
 502 /**
 503  * DPDK callback to get information about the device.
 504  *
 505  * @param dev
 506  *   Pointer to Ethernet device structure.
 507  * @param info
 508  *   Pointer to Info structure output buffer.
 509  */
 510 static void
 511 hinic_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *info)
 512 {
 513         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 514
 515         info->max_rx_queues  = nic_dev->nic_cap.max_rqs;
 516         info->max_tx_queues  = nic_dev->nic_cap.max_sqs;
 517         info->min_rx_bufsize = HINIC_MIN_RX_BUF_SIZE;
 518         info->max_rx_pktlen  = HINIC_MAX_JUMBO_FRAME_SIZE;
 519         info->max_mac_addrs  = HINIC_MAX_MAC_ADDRS;
 520
 521         hinic_get_speed_capa(dev, &info->speed_capa);
 522         info->rx_queue_offload_capa = 0;
 523         info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP |
 524                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 525                                 DEV_RX_OFFLOAD_UDP_CKSUM |
 526                                 DEV_RX_OFFLOAD_TCP_CKSUM;
 527
 528         info->tx_queue_offload_capa = 0;
 529         info->tx_offload_capa = DEV_TX_OFFLOAD_VLAN_INSERT |
 530                                 DEV_TX_OFFLOAD_IPV4_CKSUM |
 531                                 DEV_TX_OFFLOAD_UDP_CKSUM |
 532                                 DEV_TX_OFFLOAD_TCP_CKSUM |
 533                                 DEV_TX_OFFLOAD_SCTP_CKSUM |
 534                                 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
 535                                 DEV_TX_OFFLOAD_TCP_TSO |
 536                                 DEV_TX_OFFLOAD_MULTI_SEGS;
 537
 538         info->hash_key_size = HINIC_RSS_KEY_SIZE;
 539         info->reta_size = HINIC_RSS_INDIR_SIZE;
 540         info->flow_type_rss_offloads = HINIC_RSS_OFFLOAD_ALL;
 541         info->rx_desc_lim = hinic_rx_desc_lim;
 542         info->tx_desc_lim = hinic_tx_desc_lim;
 543 }
 544
 545 static int hinic_config_rx_mode(struct hinic_nic_dev *nic_dev, u32 rx_mode_ctrl)
 546 {
 547         int err;
 548
 549         err = hinic_set_rx_mode(nic_dev->hwdev, rx_mode_ctrl);
 550         if (err) {
 551                 PMD_DRV_LOG(ERR, "Failed to set rx mode");
 552                 return -EINVAL;
 553         }
 554         nic_dev->rx_mode_status = rx_mode_ctrl;
 555
 556         return 0;
 557 }
 558
 559
 560 static int hinic_rxtx_configure(struct rte_eth_dev *dev)
 561 {
 562         int err;
 563         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 564
 565         /* rx configure, if rss enable, need to init default configuration */
 566         err = hinic_rx_configure(dev);
 567         if (err) {
 568                 PMD_DRV_LOG(ERR, "Configure rss failed");
 569                 return err;
 570         }
 571
 572         /* rx mode init */
 573         err = hinic_config_rx_mode(nic_dev, HINIC_DEFAULT_RX_MODE);
 574         if (err) {
 575                 PMD_DRV_LOG(ERR, "Configure rx_mode:0x%x failed",
 576                         HINIC_DEFAULT_RX_MODE);
 577                 goto set_rx_mode_fail;
 578         }
 579
 580         return HINIC_OK;
 581
 582 set_rx_mode_fail:
 583         hinic_rx_remove_configure(dev);
 584
 585         return err;
 586 }
 587
 588 static void hinic_remove_rxtx_configure(struct rte_eth_dev *dev)
 589 {
 590         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 591
 592         (void)hinic_config_rx_mode(nic_dev, 0);
 593         hinic_rx_remove_configure(dev);
 594 }
 595
 596 static int hinic_priv_get_dev_link_status(struct hinic_nic_dev *nic_dev,
 597                                           struct rte_eth_link *link)
 598 {
 599         int rc;
 600         u8 port_link_status = 0;
 601         struct nic_port_info port_link_info;
 602         struct hinic_hwdev *nic_hwdev = nic_dev->hwdev;
 603         uint32_t port_speed[LINK_SPEED_MAX] = {ETH_SPEED_NUM_10M,
 604                                         ETH_SPEED_NUM_100M, ETH_SPEED_NUM_1G,
 605                                         ETH_SPEED_NUM_10G, ETH_SPEED_NUM_25G,
 606                                         ETH_SPEED_NUM_40G, ETH_SPEED_NUM_100G};
 607
 608         rc = hinic_get_link_status(nic_hwdev, &port_link_status);
 609         if (rc)
 610                 return rc;
 611
 612         if (!port_link_status) {
 613                 link->link_status = ETH_LINK_DOWN;
 614                 link->link_speed = 0;
 615                 link->link_duplex = ETH_LINK_HALF_DUPLEX;
 616                 link->link_autoneg = ETH_LINK_FIXED;
 617                 return HINIC_OK;
 618         }
 619
 620         memset(&port_link_info, 0, sizeof(port_link_info));
 621         rc = hinic_get_port_info(nic_hwdev, &port_link_info);
 622         if (rc)
 623                 return rc;
 624
 625         link->link_speed = port_speed[port_link_info.speed % LINK_SPEED_MAX];
 626         link->link_duplex = port_link_info.duplex;
 627         link->link_autoneg = port_link_info.autoneg_state;
 628         link->link_status = port_link_status;
 629
 630         return HINIC_OK;
 631 }
 632
 633 /**
 634  * DPDK callback to retrieve physical link information.
 635  *
 636  * @param dev
 637  *   Pointer to Ethernet device structure.
 638  * @param wait_to_complete
 639  *   Wait for request completion.
 640  *
 641  * @return
 642  *   0 link status changed, -1 link status not changed
 643  */
 644 static int hinic_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 645 {
 646 #define CHECK_INTERVAL 10  /* 10ms */
 647 #define MAX_REPEAT_TIME 100  /* 1s (100 * 10ms) in total */
 648         int rc = HINIC_OK;
 649         struct rte_eth_link link;
 650         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 651         unsigned int rep_cnt = MAX_REPEAT_TIME;
 652
 653         memset(&link, 0, sizeof(link));
 654         do {
 655                 /* Get link status information from hardware */
 656                 rc = hinic_priv_get_dev_link_status(nic_dev, &link);
 657                 if (rc != HINIC_OK) {
 658                         link.link_speed = ETH_SPEED_NUM_NONE;
 659                         link.link_duplex = ETH_LINK_FULL_DUPLEX;
 660                         PMD_DRV_LOG(ERR, "Get link status failed");
 661                         goto out;
 662                 }
 663
 664                 if (!wait_to_complete || link.link_status)
 665                         break;
 666
 667                 rte_delay_ms(CHECK_INTERVAL);
 668         } while (rep_cnt--);
 669
 670 out:
 671         rc = rte_eth_linkstatus_set(dev, &link);
 672         return rc;
 673 }
 674
 675 /**
 676  * DPDK callback to start the device.
 677  *
 678  * @param dev
 679  *   Pointer to Ethernet device structure.
 680  *
 681  * @return
 682  *   0 on success, negative errno value on failure.
 683  */
 684 static int hinic_dev_start(struct rte_eth_dev *dev)
 685 {
 686         int rc;
 687         char *name;
 688         struct hinic_nic_dev *nic_dev;
 689
 690         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 691         name = dev->data->name;
 692
 693         /* reset rx and tx queue */
 694         hinic_reset_rx_queue(dev);
 695         hinic_reset_tx_queue(dev);
 696
 697         /* get func rx buf size */
 698         hinic_get_func_rx_buf_size(nic_dev);
 699
 700         /* init txq and rxq context */
 701         rc = hinic_init_qp_ctxts(nic_dev->hwdev);
 702         if (rc) {
 703                 PMD_DRV_LOG(ERR, "Initialize qp context failed, dev_name:%s",
 704                             name);
 705                 goto init_qp_fail;
 706         }
 707
 708         /* rss template */
 709         rc = hinic_config_mq_mode(dev, TRUE);
 710         if (rc) {
 711                 PMD_DRV_LOG(ERR, "Configure mq mode failed, dev_name: %s",
 712                             name);
 713                 goto cfg_mq_mode_fail;
 714         }
 715
 716         /* set default mtu */
 717         rc = hinic_set_port_mtu(nic_dev->hwdev, nic_dev->mtu_size);
 718         if (rc) {
 719                 PMD_DRV_LOG(ERR, "Set mtu_size[%d] failed, dev_name: %s",
 720                             nic_dev->mtu_size, name);
 721                 goto set_mtu_fail;
 722         }
 723
 724         /* configure rss rx_mode and other rx or tx default feature */
 725         rc = hinic_rxtx_configure(dev);
 726         if (rc) {
 727                 PMD_DRV_LOG(ERR, "Configure tx and rx failed, dev_name: %s",
 728                             name);
 729                 goto cfg_rxtx_fail;
 730         }
 731
 732         /* open virtual port and ready to start packet receiving */
 733         rc = hinic_set_vport_enable(nic_dev->hwdev, true);
 734         if (rc) {
 735                 PMD_DRV_LOG(ERR, "Enable vport failed, dev_name:%s", name);
 736                 goto en_vport_fail;
 737         }
 738
 739         /* open physical port and start packet receiving */
 740         rc = hinic_set_port_enable(nic_dev->hwdev, true);
 741         if (rc) {
 742                 PMD_DRV_LOG(ERR, "Enable physical port failed, dev_name:%s",
 743                             name);
 744                 goto en_port_fail;
 745         }
 746
 747         /* update eth_dev link status */
 748         if (dev->data->dev_conf.intr_conf.lsc != 0)
 749                 (void)hinic_link_update(dev, 0);
 750
 751         hinic_set_bit(HINIC_DEV_START, &nic_dev->dev_status);
 752
 753         return 0;
 754
 755 en_port_fail:
 756         (void)hinic_set_vport_enable(nic_dev->hwdev, false);
 757
 758 en_vport_fail:
 759         /* Flush tx && rx chip resources in case of set vport fake fail */
 760         (void)hinic_flush_qp_res(nic_dev->hwdev);
 761         rte_delay_ms(100);
 762
 763         hinic_remove_rxtx_configure(dev);
 764
 765 cfg_rxtx_fail:
 766 set_mtu_fail:
 767 cfg_mq_mode_fail:
 768         hinic_free_qp_ctxts(nic_dev->hwdev);
 769
 770 init_qp_fail:
 771         hinic_free_all_rx_mbuf(dev);
 772         hinic_free_all_tx_mbuf(dev);
 773
 774         return rc;
 775 }
 776
 777 /**
 778  * DPDK callback to release the receive queue.
 779  *
 780  * @param queue
 781  *   Generic receive queue pointer.
 782  */
 783 static void hinic_rx_queue_release(void *queue)
 784 {
 785         struct hinic_rxq *rxq = queue;
 786         struct hinic_nic_dev *nic_dev;
 787
 788         if (!rxq) {
 789                 PMD_DRV_LOG(WARNING, "Rxq is null when release");
 790                 return;
 791         }
 792         nic_dev = rxq->nic_dev;
 793
 794         /* free rxq_pkt mbuf */
 795         hinic_free_all_rx_skbs(rxq);
 796
 797         /* free rxq_cqe, rxq_info */
 798         hinic_free_rx_resources(rxq);
 799
 800         /* free root rq wq */
 801         hinic_destroy_rq(nic_dev->hwdev, rxq->q_id);
 802
 803         nic_dev->rxqs[rxq->q_id] = NULL;
 804
 805         /* free rxq */
 806         rte_free(rxq);
 807 }
 808
 809 /**
 810  * DPDK callback to release the transmit queue.
 811  *
 812  * @param queue
 813  *   Generic transmit queue pointer.
 814  */
 815 static void hinic_tx_queue_release(void *queue)
 816 {
 817         struct hinic_txq *txq = queue;
 818         struct hinic_nic_dev *nic_dev;
 819
 820         if (!txq) {
 821                 PMD_DRV_LOG(WARNING, "Txq is null when release");
 822                 return;
 823         }
 824         nic_dev = txq->nic_dev;
 825
 826         /* free txq_pkt mbuf */
 827         hinic_free_all_tx_skbs(txq);
 828
 829         /* free txq_info */
 830         hinic_free_tx_resources(txq);
 831
 832         /* free root sq wq */
 833         hinic_destroy_sq(nic_dev->hwdev, txq->q_id);
 834         nic_dev->txqs[txq->q_id] = NULL;
 835
 836         /* free txq */
 837         rte_free(txq);
 838 }
 839
 840 static void hinic_free_all_rq(struct hinic_nic_dev *nic_dev)
 841 {
 842         u16 q_id;
 843
 844         for (q_id = 0; q_id < nic_dev->num_rq; q_id++)
 845                 hinic_destroy_rq(nic_dev->hwdev, q_id);
 846 }
 847
 848 static void hinic_free_all_sq(struct hinic_nic_dev *nic_dev)
 849 {
 850         u16 q_id;
 851
 852         for (q_id = 0; q_id < nic_dev->num_sq; q_id++)
 853                 hinic_destroy_sq(nic_dev->hwdev, q_id);
 854 }
 855
 856 /**
 857  * DPDK callback to stop the device.
 858  *
 859  * @param dev
 860  *   Pointer to Ethernet device structure.
 861  */
 862 static void hinic_dev_stop(struct rte_eth_dev *dev)
 863 {
 864         int rc;
 865         char *name;
 866         uint16_t port_id;
 867         struct hinic_nic_dev *nic_dev;
 868         struct rte_eth_link link;
 869
 870         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 871         name = dev->data->name;
 872         port_id = dev->data->port_id;
 873
 874         if (!hinic_test_and_clear_bit(HINIC_DEV_START, &nic_dev->dev_status)) {
 875                 PMD_DRV_LOG(INFO, "Device %s already stopped", name);
 876                 return;
 877         }
 878
 879         /* just stop phy port and vport */
 880         rc = hinic_set_port_enable(nic_dev->hwdev, false);
 881         if (rc)
 882                 PMD_DRV_LOG(WARNING, "Disable phy port failed, error: %d, dev_name:%s, port_id:%d",
 883                           rc, name, port_id);
 884
 885         rc = hinic_set_vport_enable(nic_dev->hwdev, false);
 886         if (rc)
 887                 PMD_DRV_LOG(WARNING, "Disable vport failed, error: %d, dev_name:%s, port_id:%d",
 888                           rc, name, port_id);
 889
 890         /* Clear recorded link status */
 891         memset(&link, 0, sizeof(link));
 892         (void)rte_eth_linkstatus_set(dev, &link);
 893
 894         /* flush pending io request */
 895         rc = hinic_rx_tx_flush(nic_dev->hwdev);
 896         if (rc)
 897                 PMD_DRV_LOG(WARNING, "Flush pending io failed, error: %d, dev_name: %s, port_id: %d",
 898                             rc, name, port_id);
 899
 900         /* clean rss table and rx_mode */
 901         hinic_remove_rxtx_configure(dev);
 902
 903         /* clean root context */
 904         hinic_free_qp_ctxts(nic_dev->hwdev);
 905
 906         /* free mbuf */
 907         hinic_free_all_rx_mbuf(dev);
 908         hinic_free_all_tx_mbuf(dev);
 909 }
 910
 911 static void hinic_disable_interrupt(struct rte_eth_dev *dev)
 912 {
 913         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
 914         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 915         int ret, retries = 0;
 916
 917         hinic_clear_bit(HINIC_DEV_INTR_EN, &nic_dev->dev_status);
 918
 919         /* disable msix interrupt in hardware */
 920         hinic_set_msix_state(nic_dev->hwdev, 0, HINIC_MSIX_DISABLE);
 921
 922         /* disable rte interrupt */
 923         ret = rte_intr_disable(&pci_dev->intr_handle);
 924         if (ret)
 925                 PMD_DRV_LOG(ERR, "Disable intr failed: %d", ret);
 926
 927         do {
 928                 ret =
 929                 rte_intr_callback_unregister(&pci_dev->intr_handle,
 930                                              hinic_dev_interrupt_handler, dev);
 931                 if (ret >= 0) {
 932                         break;
 933                 } else if (ret == -EAGAIN) {
 934                         rte_delay_ms(100);
 935                         retries++;
 936                 } else {
 937                         PMD_DRV_LOG(ERR, "intr callback unregister failed: %d",
 938                                     ret);
 939                         break;
 940                 }
 941         } while (retries < HINIC_INTR_CB_UNREG_MAX_RETRIES);
 942
 943         if (retries == HINIC_INTR_CB_UNREG_MAX_RETRIES)
 944                 PMD_DRV_LOG(ERR, "Unregister intr callback failed after %d retries",
 945                             retries);
 946 }
 947
 948 static void hinic_gen_random_mac_addr(struct rte_ether_addr *mac_addr)
 949 {
 950         uint64_t random_value;
 951
 952         /* Set Organizationally Unique Identifier (OUI) prefix */
 953         mac_addr->addr_bytes[0] = 0x00;
 954         mac_addr->addr_bytes[1] = 0x09;
 955         mac_addr->addr_bytes[2] = 0xC0;
 956         /* Force indication of locally assigned MAC address. */
 957         mac_addr->addr_bytes[0] |= RTE_ETHER_LOCAL_ADMIN_ADDR;
 958         /* Generate the last 3 bytes of the MAC address with a random number. */
 959         random_value = rte_rand();
 960         memcpy(&mac_addr->addr_bytes[3], &random_value, 3);
 961 }
 962
 963 /**
 964  * Init mac_vlan table in NIC.
 965  *
 966  * @param dev
 967  *   Pointer to Ethernet device structure.
 968  *
 969  * @return
 970  *   0 on success and stats is filled,
 971  *   negative error value otherwise.
 972  */
 973 static int hinic_init_mac_addr(struct rte_eth_dev *eth_dev)
 974 {
 975         struct hinic_nic_dev *nic_dev =
 976                                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
 977         uint8_t addr_bytes[RTE_ETHER_ADDR_LEN];
 978         u16 func_id = 0;
 979         int rc = 0;
 980
 981         rc = hinic_get_default_mac(nic_dev->hwdev, addr_bytes);
 982         if (rc)
 983                 return rc;
 984
 985         memmove(eth_dev->data->mac_addrs->addr_bytes,
 986                 addr_bytes, RTE_ETHER_ADDR_LEN);
 987
 988         if (rte_is_zero_ether_addr(eth_dev->data->mac_addrs))
 989                 hinic_gen_random_mac_addr(eth_dev->data->mac_addrs);
 990
 991         func_id = hinic_global_func_id(nic_dev->hwdev);
 992         rc = hinic_set_mac(nic_dev->hwdev, eth_dev->data->mac_addrs->addr_bytes,
 993                            0, func_id);
 994         if (rc && rc != HINIC_PF_SET_VF_ALREADY)
 995                 return rc;
 996
 997         return 0;
 998 }
 999
1000 /**
1001  * Deinit mac_vlan table in NIC.
1002  *
1003  * @param dev
1004  *   Pointer to Ethernet device structure.
1005  *
1006  * @return
1007  *   0 on success and stats is filled,
1008  *   negative error value otherwise.
1009  */
1010 static void hinic_deinit_mac_addr(struct rte_eth_dev *eth_dev)
1011 {
1012         struct hinic_nic_dev *nic_dev =
1013                                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
1014         int rc;
1015         u16 func_id = 0;
1016
1017         if (rte_is_zero_ether_addr(eth_dev->data->mac_addrs))
1018                 return;
1019
1020         func_id = hinic_global_func_id(nic_dev->hwdev);
1021         rc = hinic_del_mac(nic_dev->hwdev,
1022                            eth_dev->data->mac_addrs->addr_bytes,
1023                            0, func_id);
1024         if (rc && rc != HINIC_PF_SET_VF_ALREADY)
1025                 PMD_DRV_LOG(ERR, "Delete mac table failed, dev_name: %s",
1026                             eth_dev->data->name);
1027 }
1028
1029 static int hinic_set_default_pause_feature(struct hinic_nic_dev *nic_dev)
1030 {
1031         struct nic_pause_config pause_config = {0};
1032
1033         pause_config.auto_neg = 0;
1034         pause_config.rx_pause = HINIC_DEFAUT_PAUSE_CONFIG;
1035         pause_config.tx_pause = HINIC_DEFAUT_PAUSE_CONFIG;
1036
1037         return hinic_set_pause_config(nic_dev->hwdev, pause_config);
1038 }
1039
1040 static int hinic_set_default_dcb_feature(struct hinic_nic_dev *nic_dev)
1041 {
1042         u8 up_tc[HINIC_DCB_UP_MAX] = {0};
1043         u8 up_pgid[HINIC_DCB_UP_MAX] = {0};
1044         u8 up_bw[HINIC_DCB_UP_MAX] = {0};
1045         u8 pg_bw[HINIC_DCB_UP_MAX] = {0};
1046         u8 up_strict[HINIC_DCB_UP_MAX] = {0};
1047         int i = 0;
1048
1049         pg_bw[0] = 100;
1050         for (i = 0; i < HINIC_DCB_UP_MAX; i++)
1051                 up_bw[i] = 100;
1052
1053         return hinic_dcb_set_ets(nic_dev->hwdev, up_tc, pg_bw,
1054                                         up_pgid, up_bw, up_strict);
1055 }
1056
1057 static void hinic_init_default_cos(struct hinic_nic_dev *nic_dev)
1058 {
1059         nic_dev->default_cos =
1060                         (hinic_global_func_id(nic_dev->hwdev) +
1061                          DEFAULT_BASE_COS) % NR_MAX_COS;
1062 }
1063
1064 static int hinic_set_default_hw_feature(struct hinic_nic_dev *nic_dev)
1065 {
1066         int err;
1067
1068         hinic_init_default_cos(nic_dev);
1069
1070         /* Restore DCB configure to default status */
1071         err = hinic_set_default_dcb_feature(nic_dev);
1072         if (err)
1073                 return err;
1074
1075         /* disable LRO */
1076         err = hinic_set_rx_lro(nic_dev->hwdev, 0, 0, (u8)0);
1077         if (err)
1078                 return err;
1079
1080         /* Set pause enable, and up will disable pfc. */
1081         err = hinic_set_default_pause_feature(nic_dev);
1082         if (err)
1083                 return err;
1084
1085         err = hinic_reset_port_link_cfg(nic_dev->hwdev);
1086         if (err)
1087                 return err;
1088
1089         err = hinic_set_link_status_follow(nic_dev->hwdev,
1090                                            HINIC_LINK_FOLLOW_PORT);
1091         if (err == HINIC_MGMT_CMD_UNSUPPORTED)
1092                 PMD_DRV_LOG(WARNING, "Don't support to set link status follow phy port status");
1093         else if (err)
1094                 return err;
1095
1096         return hinic_set_anti_attack(nic_dev->hwdev, true);
1097 }
1098
1099 static int32_t hinic_card_workmode_check(struct hinic_nic_dev *nic_dev)
1100 {
1101         struct hinic_board_info info = { 0 };
1102         int rc;
1103
1104         rc = hinic_get_board_info(nic_dev->hwdev, &info);
1105         if (rc)
1106                 return rc;
1107
1108         return (info.service_mode == HINIC_SERVICE_MODE_NIC ? HINIC_OK :
1109                                                 HINIC_ERROR);
1110 }
1111
1112 static int hinic_copy_mempool_init(struct hinic_nic_dev *nic_dev)
1113 {
1114         nic_dev->cpy_mpool = rte_mempool_lookup(nic_dev->proc_dev_name);
1115         if (nic_dev->cpy_mpool == NULL) {
1116                 nic_dev->cpy_mpool =
1117                 rte_pktmbuf_pool_create(nic_dev->proc_dev_name,
1118                                         HINIC_COPY_MEMPOOL_DEPTH,
1119                                         RTE_CACHE_LINE_SIZE, 0,
1120                                         HINIC_COPY_MBUF_SIZE,
1121                                         rte_socket_id());
1122                 if (!nic_dev->cpy_mpool) {
1123                         PMD_DRV_LOG(ERR, "Create copy mempool failed, errno: %d, dev_name: %s",
1124                                     rte_errno, nic_dev->proc_dev_name);
1125                         return -ENOMEM;
1126                 }
1127         }
1128
1129         return 0;
1130 }
1131
1132 static void hinic_copy_mempool_uninit(struct hinic_nic_dev *nic_dev)
1133 {
1134         if (nic_dev->cpy_mpool != NULL)
1135                 rte_mempool_free(nic_dev->cpy_mpool);
1136 }
1137
1138 static int hinic_init_sw_rxtxqs(struct hinic_nic_dev *nic_dev)
1139 {
1140         u32 txq_size;
1141         u32 rxq_size;
1142
1143         /* allocate software txq array */
1144         txq_size = nic_dev->nic_cap.max_sqs * sizeof(*nic_dev->txqs);
1145         nic_dev->txqs = kzalloc_aligned(txq_size, GFP_KERNEL);
1146         if (!nic_dev->txqs) {
1147                 PMD_DRV_LOG(ERR, "Allocate txqs failed");
1148                 return -ENOMEM;
1149         }
1150
1151         /* allocate software rxq array */
1152         rxq_size = nic_dev->nic_cap.max_rqs * sizeof(*nic_dev->rxqs);
1153         nic_dev->rxqs = kzalloc_aligned(rxq_size, GFP_KERNEL);
1154         if (!nic_dev->rxqs) {
1155                 /* free txqs */
1156                 kfree(nic_dev->txqs);
1157                 nic_dev->txqs = NULL;
1158
1159                 PMD_DRV_LOG(ERR, "Allocate rxqs failed");
1160                 return -ENOMEM;
1161         }
1162
1163         return HINIC_OK;
1164 }
1165
1166 static void hinic_deinit_sw_rxtxqs(struct hinic_nic_dev *nic_dev)
1167 {
1168         kfree(nic_dev->txqs);
1169         nic_dev->txqs = NULL;
1170
1171         kfree(nic_dev->rxqs);
1172         nic_dev->rxqs = NULL;
1173 }
1174
1175 static int hinic_nic_dev_create(struct rte_eth_dev *eth_dev)
1176 {
1177         struct hinic_nic_dev *nic_dev =
1178                                 HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
1179         int rc;
1180
1181         nic_dev->hwdev = rte_zmalloc("hinic_hwdev", sizeof(*nic_dev->hwdev),
1182                                      RTE_CACHE_LINE_SIZE);
1183         if (!nic_dev->hwdev) {
1184                 PMD_DRV_LOG(ERR, "Allocate hinic hwdev memory failed, dev_name: %s",
1185                             eth_dev->data->name);
1186                 return -ENOMEM;
1187         }
1188         nic_dev->hwdev->pcidev_hdl = RTE_ETH_DEV_TO_PCI(eth_dev);
1189
1190         /* init osdep*/
1191         rc = hinic_osdep_init(nic_dev->hwdev);
1192         if (rc) {
1193                 PMD_DRV_LOG(ERR, "Initialize os_dep failed, dev_name: %s",
1194                             eth_dev->data->name);
1195                 goto init_osdep_fail;
1196         }
1197
1198         /* init_hwif */
1199         rc = hinic_hwif_res_init(nic_dev->hwdev);
1200         if (rc) {
1201                 PMD_DRV_LOG(ERR, "Initialize hwif failed, dev_name: %s",
1202                             eth_dev->data->name);
1203                 goto init_hwif_fail;
1204         }
1205
1206         /* init_cfg_mgmt */
1207         rc = init_cfg_mgmt(nic_dev->hwdev);
1208         if (rc) {
1209                 PMD_DRV_LOG(ERR, "Initialize cfg_mgmt failed, dev_name: %s",
1210                             eth_dev->data->name);
1211                 goto init_cfgmgnt_fail;
1212         }
1213
1214         /* init_aeqs */
1215         rc = hinic_comm_aeqs_init(nic_dev->hwdev);
1216         if (rc) {
1217                 PMD_DRV_LOG(ERR, "Initialize aeqs failed, dev_name: %s",
1218                             eth_dev->data->name);
1219                 goto init_aeqs_fail;
1220         }
1221
1222         /* init_pf_to_mgnt */
1223         rc = hinic_comm_pf_to_mgmt_init(nic_dev->hwdev);
1224         if (rc) {
1225                 PMD_DRV_LOG(ERR, "Initialize pf_to_mgmt failed, dev_name: %s",
1226                             eth_dev->data->name);
1227                 goto init_pf_to_mgmt_fail;
1228         }
1229
1230         rc = hinic_card_workmode_check(nic_dev);
1231         if (rc) {
1232                 PMD_DRV_LOG(ERR, "Check card workmode failed, dev_name: %s",
1233                             eth_dev->data->name);
1234                 goto workmode_check_fail;
1235         }
1236
1237         /* do l2nic reset to make chip clear */
1238         rc = hinic_l2nic_reset(nic_dev->hwdev);
1239         if (rc) {
1240                 PMD_DRV_LOG(ERR, "Do l2nic reset failed, dev_name: %s",
1241                             eth_dev->data->name);
1242                 goto l2nic_reset_fail;
1243         }
1244
1245         /* init dma and aeq msix attribute table */
1246         (void)hinic_init_attr_table(nic_dev->hwdev);
1247
1248         /* init_cmdqs */
1249         rc = hinic_comm_cmdqs_init(nic_dev->hwdev);
1250         if (rc) {
1251                 PMD_DRV_LOG(ERR, "Initialize cmdq failed, dev_name: %s",
1252                             eth_dev->data->name);
1253                 goto init_cmdq_fail;
1254         }
1255
1256         /* set hardware state active */
1257         rc = hinic_activate_hwdev_state(nic_dev->hwdev);
1258         if (rc) {
1259                 PMD_DRV_LOG(ERR, "Initialize resources state failed, dev_name: %s",
1260                             eth_dev->data->name);
1261                 goto init_resources_state_fail;
1262         }
1263
1264         /* init_capability */
1265         rc = hinic_init_capability(nic_dev->hwdev);
1266         if (rc) {
1267                 PMD_DRV_LOG(ERR, "Initialize capability failed, dev_name: %s",
1268                             eth_dev->data->name);
1269                 goto init_cap_fail;
1270         }
1271
1272         /* get nic capability */
1273         if (!hinic_support_nic(nic_dev->hwdev, &nic_dev->nic_cap))
1274                 goto nic_check_fail;
1275
1276         /* init root cla and function table */
1277         rc = hinic_init_nicio(nic_dev->hwdev);
1278         if (rc) {
1279                 PMD_DRV_LOG(ERR, "Initialize nic_io failed, dev_name: %s",
1280                             eth_dev->data->name);
1281                 goto init_nicio_fail;
1282         }
1283
1284         /* init_software_txrxq */
1285         rc = hinic_init_sw_rxtxqs(nic_dev);
1286         if (rc) {
1287                 PMD_DRV_LOG(ERR, "Initialize sw_rxtxqs failed, dev_name: %s",
1288                             eth_dev->data->name);
1289                 goto init_sw_rxtxqs_fail;
1290         }
1291
1292         rc = hinic_copy_mempool_init(nic_dev);
1293         if (rc) {
1294                 PMD_DRV_LOG(ERR, "Create copy mempool failed, dev_name: %s",
1295                          eth_dev->data->name);
1296                 goto init_mpool_fail;
1297         }
1298
1299         /* set hardware feature to default status */
1300         rc = hinic_set_default_hw_feature(nic_dev);
1301         if (rc) {
1302                 PMD_DRV_LOG(ERR, "Initialize hardware default features failed, dev_name: %s",
1303                             eth_dev->data->name);
1304                 goto set_default_hw_feature_fail;
1305         }
1306
1307         return 0;
1308
1309 set_default_hw_feature_fail:
1310         hinic_copy_mempool_uninit(nic_dev);
1311
1312 init_mpool_fail:
1313         hinic_deinit_sw_rxtxqs(nic_dev);
1314
1315 init_sw_rxtxqs_fail:
1316         hinic_deinit_nicio(nic_dev->hwdev);
1317
1318 nic_check_fail:
1319 init_nicio_fail:
1320 init_cap_fail:
1321         hinic_deactivate_hwdev_state(nic_dev->hwdev);
1322
1323 init_resources_state_fail:
1324         hinic_comm_cmdqs_free(nic_dev->hwdev);
1325
1326 init_cmdq_fail:
1327 l2nic_reset_fail:
1328 workmode_check_fail:
1329         hinic_comm_pf_to_mgmt_free(nic_dev->hwdev);
1330
1331 init_pf_to_mgmt_fail:
1332         hinic_comm_aeqs_free(nic_dev->hwdev);
1333
1334 init_aeqs_fail:
1335         free_cfg_mgmt(nic_dev->hwdev);
1336
1337 init_cfgmgnt_fail:
1338         hinic_hwif_res_free(nic_dev->hwdev);
1339
1340 init_hwif_fail:
1341         hinic_osdep_deinit(nic_dev->hwdev);
1342
1343 init_osdep_fail:
1344         rte_free(nic_dev->hwdev);
1345         nic_dev->hwdev = NULL;
1346
1347         return rc;
1348 }
1349
1350 static void hinic_nic_dev_destroy(struct rte_eth_dev *eth_dev)
1351 {
1352         struct hinic_nic_dev *nic_dev =
1353                         HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
1354
1355         (void)hinic_set_link_status_follow(nic_dev->hwdev,
1356                                            HINIC_LINK_FOLLOW_DEFAULT);
1357         hinic_copy_mempool_uninit(nic_dev);
1358         hinic_deinit_sw_rxtxqs(nic_dev);
1359         hinic_deinit_nicio(nic_dev->hwdev);
1360         hinic_deactivate_hwdev_state(nic_dev->hwdev);
1361         hinic_comm_cmdqs_free(nic_dev->hwdev);
1362         hinic_comm_pf_to_mgmt_free(nic_dev->hwdev);
1363         hinic_comm_aeqs_free(nic_dev->hwdev);
1364         free_cfg_mgmt(nic_dev->hwdev);
1365         hinic_hwif_res_free(nic_dev->hwdev);
1366         hinic_osdep_deinit(nic_dev->hwdev);
1367         rte_free(nic_dev->hwdev);
1368         nic_dev->hwdev = NULL;
1369 }
1370
1371 static int hinic_func_init(struct rte_eth_dev *eth_dev)
1372 {
1373         struct rte_pci_device *pci_dev;
1374         struct rte_ether_addr *eth_addr;
1375         struct hinic_nic_dev *nic_dev;
1376         int rc;
1377
1378         pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
1379
1380         /* EAL is SECONDARY and eth_dev is already created */
1381         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
1382                 rc = rte_intr_callback_register(&pci_dev->intr_handle,
1383                                                 hinic_dev_interrupt_handler,
1384                                                 (void *)eth_dev);
1385                 if (rc)
1386                         PMD_DRV_LOG(ERR, "Initialize %s failed in secondary process",
1387                                     eth_dev->data->name);
1388
1389                 return rc;
1390         }
1391
1392         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
1393         memset(nic_dev, 0, sizeof(*nic_dev));
1394
1395         snprintf(nic_dev->proc_dev_name,
1396                  sizeof(nic_dev->proc_dev_name),
1397                  "hinic-%.4x:%.2x:%.2x.%x",
1398                  pci_dev->addr.domain, pci_dev->addr.bus,
1399                  pci_dev->addr.devid, pci_dev->addr.function);
1400
1401         /* alloc mac_addrs */
1402         eth_addr = rte_zmalloc("hinic_mac", sizeof(*eth_addr), 0);
1403         if (!eth_addr) {
1404                 PMD_DRV_LOG(ERR, "Allocate ethernet addresses' memory failed, dev_name: %s",
1405                             eth_dev->data->name);
1406                 rc = -ENOMEM;
1407                 goto eth_addr_fail;
1408         }
1409         eth_dev->data->mac_addrs = eth_addr;
1410
1411         /*
1412          * Pass the information to the rte_eth_dev_close() that it should also
1413          * release the private port resources.
1414          */
1415         eth_dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
1416
1417         /* create hardware nic_device */
1418         rc = hinic_nic_dev_create(eth_dev);
1419         if (rc) {
1420                 PMD_DRV_LOG(ERR, "Create nic device failed, dev_name: %s",
1421                             eth_dev->data->name);
1422                 goto create_nic_dev_fail;
1423         }
1424
1425         rc = hinic_init_mac_addr(eth_dev);
1426         if (rc) {
1427                 PMD_DRV_LOG(ERR, "Initialize mac table failed, dev_name: %s",
1428                             eth_dev->data->name);
1429                 goto init_mac_fail;
1430         }
1431
1432         /* register callback func to eal lib */
1433         rc = rte_intr_callback_register(&pci_dev->intr_handle,
1434                                         hinic_dev_interrupt_handler,
1435                                         (void *)eth_dev);
1436         if (rc) {
1437                 PMD_DRV_LOG(ERR, "Register rte interrupt callback failed, dev_name: %s",
1438                             eth_dev->data->name);
1439                 goto reg_intr_cb_fail;
1440         }
1441
1442         /* enable uio/vfio intr/eventfd mapping */
1443         rc = rte_intr_enable(&pci_dev->intr_handle);
1444         if (rc) {
1445                 PMD_DRV_LOG(ERR, "Enable rte interrupt failed, dev_name: %s",
1446                             eth_dev->data->name);
1447                 goto enable_intr_fail;
1448         }
1449         hinic_set_bit(HINIC_DEV_INTR_EN, &nic_dev->dev_status);
1450
1451         hinic_set_bit(HINIC_DEV_INIT, &nic_dev->dev_status);
1452         PMD_DRV_LOG(INFO, "Initialize %s in primary successfully",
1453                     eth_dev->data->name);
1454
1455         return 0;
1456
1457 enable_intr_fail:
1458         (void)rte_intr_callback_unregister(&pci_dev->intr_handle,
1459                                            hinic_dev_interrupt_handler,
1460                                            (void *)eth_dev);
1461
1462 reg_intr_cb_fail:
1463         hinic_deinit_mac_addr(eth_dev);
1464
1465 init_mac_fail:
1466         hinic_nic_dev_destroy(eth_dev);
1467
1468 create_nic_dev_fail:
1469         rte_free(eth_addr);
1470         eth_dev->data->mac_addrs = NULL;
1471
1472 eth_addr_fail:
1473         PMD_DRV_LOG(ERR, "Initialize %s in primary failed",
1474                     eth_dev->data->name);
1475         return rc;
1476 }
1477
1478 /**
1479  * DPDK callback to close the device.
1480  *
1481  * @param dev
1482  *   Pointer to Ethernet device structure.
1483  */
1484 static void hinic_dev_close(struct rte_eth_dev *dev)
1485 {
1486         struct hinic_nic_dev *nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1487
1488         if (hinic_test_and_set_bit(HINIC_DEV_CLOSE, &nic_dev->dev_status)) {
1489                 PMD_DRV_LOG(WARNING, "Device %s already closed",
1490                             dev->data->name);
1491                 return;
1492         }
1493
1494         /* stop device first */
1495         hinic_dev_stop(dev);
1496
1497         /* rx_cqe, rx_info */
1498         hinic_free_all_rx_resources(dev);
1499
1500         /* tx_info */
1501         hinic_free_all_tx_resources(dev);
1502
1503         /* free wq, pi_dma_addr */
1504         hinic_free_all_rq(nic_dev);
1505
1506         /* free wq, db_addr */
1507         hinic_free_all_sq(nic_dev);
1508
1509         /* deinit mac vlan tbl */
1510         hinic_deinit_mac_addr(dev);
1511
1512         /* disable hardware and uio interrupt */
1513         hinic_disable_interrupt(dev);
1514
1515         /* deinit nic hardware device */
1516         hinic_nic_dev_destroy(dev);
1517 }
1518
1519 static const struct eth_dev_ops hinic_pmd_ops = {
1520         .dev_configure                 = hinic_dev_configure,
1521         .dev_infos_get                 = hinic_dev_infos_get,
1522         .rx_queue_setup                = hinic_rx_queue_setup,
1523         .tx_queue_setup                = hinic_tx_queue_setup,
1524         .dev_start                     = hinic_dev_start,
1525         .link_update                   = hinic_link_update,
1526         .rx_queue_release              = hinic_rx_queue_release,
1527         .tx_queue_release              = hinic_tx_queue_release,
1528         .dev_stop                      = hinic_dev_stop,
1529         .dev_close                     = hinic_dev_close,
1530 };
1531
1532 static int hinic_dev_init(struct rte_eth_dev *eth_dev)
1533 {
1534         struct rte_pci_device *pci_dev;
1535
1536         pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev);
1537
1538         PMD_DRV_LOG(INFO, "Initializing pf hinic-%.4x:%.2x:%.2x.%x in %s process",
1539                     pci_dev->addr.domain, pci_dev->addr.bus,
1540                     pci_dev->addr.devid, pci_dev->addr.function,
1541                     (rte_eal_process_type() == RTE_PROC_PRIMARY) ?
1542                     "primary" : "secondary");
1543
1544         /* rte_eth_dev ops, rx_burst and tx_burst */
1545         eth_dev->dev_ops = &hinic_pmd_ops;
1546
1547         return hinic_func_init(eth_dev);
1548 }
1549
1550 static int hinic_dev_uninit(struct rte_eth_dev *dev)
1551 {
1552         struct hinic_nic_dev *nic_dev;
1553
1554         nic_dev = HINIC_ETH_DEV_TO_PRIVATE_NIC_DEV(dev);
1555         hinic_clear_bit(HINIC_DEV_INIT, &nic_dev->dev_status);
1556
1557         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1558                 return 0;
1559
1560         hinic_dev_close(dev);
1561
1562         dev->dev_ops = NULL;
1563
1564         rte_free(dev->data->mac_addrs);
1565         dev->data->mac_addrs = NULL;
1566
1567         return HINIC_OK;
1568 }
1569
1570 static struct rte_pci_id pci_id_hinic_map[] = {
1571         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_PRD) },
1572         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_MEZZ_25GE) },
1573         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_MEZZ_40GE) },
1574         { RTE_PCI_DEVICE(HINIC_HUAWEI_VENDOR_ID, HINIC_DEV_ID_MEZZ_100GE) },
1575         {.vendor_id = 0},
1576 };
1577
1578 static int hinic_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
1579                            struct rte_pci_device *pci_dev)
1580 {
1581         return rte_eth_dev_pci_generic_probe(pci_dev,
1582                 sizeof(struct hinic_nic_dev), hinic_dev_init);
1583 }
1584
1585 static int hinic_pci_remove(struct rte_pci_device *pci_dev)
1586 {
1587         return rte_eth_dev_pci_generic_remove(pci_dev, hinic_dev_uninit);
1588 }
1589
1590 static struct rte_pci_driver rte_hinic_pmd = {
1591         .id_table = pci_id_hinic_map,
1592         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
1593         .probe = hinic_pci_probe,
1594         .remove = hinic_pci_remove,
1595 };
1596
1597 RTE_PMD_REGISTER_PCI(net_hinic, rte_hinic_pmd);
1598 RTE_PMD_REGISTER_PCI_TABLE(net_hinic, pci_id_hinic_map);
1599
1600 RTE_INIT(hinic_init_log)
1601 {
1602         hinic_logtype = rte_log_register("pmd.net.hinic");
1603         if (hinic_logtype >= 0)
1604                 rte_log_set_level(hinic_logtype, RTE_LOG_INFO);
1605 }