drivers/net/enic/enic_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2008-2017 Cisco Systems, Inc.  All rights reserved.
   3  * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
   4  */
   5
   6 #include <stdio.h>
   7 #include <stdint.h>
   8
   9 #include <rte_dev.h>
  10 #include <rte_pci.h>
  11 #include <rte_bus_pci.h>
  12 #include <rte_ethdev_driver.h>
  13 #include <rte_ethdev_pci.h>
  14 #include <rte_kvargs.h>
  15 #include <rte_string_fns.h>
  16
  17 #include "vnic_intr.h"
  18 #include "vnic_cq.h"
  19 #include "vnic_wq.h"
  20 #include "vnic_rq.h"
  21 #include "vnic_enet.h"
  22 #include "enic.h"
  23
  24 int enicpmd_logtype_init;
  25 int enicpmd_logtype_flow;
  26
  27 #define PMD_INIT_LOG(level, fmt, args...) \
  28         rte_log(RTE_LOG_ ## level, enicpmd_logtype_init, \
  29                 "%s" fmt "\n", __func__, ##args)
  30
  31 #define ENICPMD_FUNC_TRACE() PMD_INIT_LOG(DEBUG, " >>")
  32
  33 /*
  34  * The set of PCI devices this driver supports
  35  */
  36 #define CISCO_PCI_VENDOR_ID 0x1137
  37 static const struct rte_pci_id pci_id_enic_map[] = {
  38         { RTE_PCI_DEVICE(CISCO_PCI_VENDOR_ID, PCI_DEVICE_ID_CISCO_VIC_ENET) },
  39         { RTE_PCI_DEVICE(CISCO_PCI_VENDOR_ID, PCI_DEVICE_ID_CISCO_VIC_ENET_VF) },
  40         {.vendor_id = 0, /* sentinel */},
  41 };
  42
  43 #define ENIC_DEVARG_DISABLE_OVERLAY "disable-overlay"
  44 #define ENIC_DEVARG_IG_VLAN_REWRITE "ig-vlan-rewrite"
  45
  46 RTE_INIT(enicpmd_init_log);
  47 static void
  48 enicpmd_init_log(void)
  49 {
  50         enicpmd_logtype_init = rte_log_register("pmd.net.enic.init");
  51         if (enicpmd_logtype_init >= 0)
  52                 rte_log_set_level(enicpmd_logtype_init, RTE_LOG_NOTICE);
  53         enicpmd_logtype_flow = rte_log_register("pmd.net.enic.flow");
  54         if (enicpmd_logtype_flow >= 0)
  55                 rte_log_set_level(enicpmd_logtype_flow, RTE_LOG_NOTICE);
  56 }
  57
  58 static int
  59 enicpmd_fdir_ctrl_func(struct rte_eth_dev *eth_dev,
  60                         enum rte_filter_op filter_op, void *arg)
  61 {
  62         struct enic *enic = pmd_priv(eth_dev);
  63         int ret = 0;
  64
  65         ENICPMD_FUNC_TRACE();
  66         if (filter_op == RTE_ETH_FILTER_NOP)
  67                 return 0;
  68
  69         if (arg == NULL && filter_op != RTE_ETH_FILTER_FLUSH)
  70                 return -EINVAL;
  71
  72         switch (filter_op) {
  73         case RTE_ETH_FILTER_ADD:
  74         case RTE_ETH_FILTER_UPDATE:
  75                 ret = enic_fdir_add_fltr(enic,
  76                         (struct rte_eth_fdir_filter *)arg);
  77                 break;
  78
  79         case RTE_ETH_FILTER_DELETE:
  80                 ret = enic_fdir_del_fltr(enic,
  81                         (struct rte_eth_fdir_filter *)arg);
  82                 break;
  83
  84         case RTE_ETH_FILTER_STATS:
  85                 enic_fdir_stats_get(enic, (struct rte_eth_fdir_stats *)arg);
  86                 break;
  87
  88         case RTE_ETH_FILTER_FLUSH:
  89                 dev_warning(enic, "unsupported operation %u", filter_op);
  90                 ret = -ENOTSUP;
  91                 break;
  92         case RTE_ETH_FILTER_INFO:
  93                 enic_fdir_info_get(enic, (struct rte_eth_fdir_info *)arg);
  94                 break;
  95         default:
  96                 dev_err(enic, "unknown operation %u", filter_op);
  97                 ret = -EINVAL;
  98                 break;
  99         }
 100         return ret;
 101 }
 102
 103 static int
 104 enicpmd_dev_filter_ctrl(struct rte_eth_dev *dev,
 105                      enum rte_filter_type filter_type,
 106                      enum rte_filter_op filter_op,
 107                      void *arg)
 108 {
 109         int ret = 0;
 110
 111         ENICPMD_FUNC_TRACE();
 112
 113         switch (filter_type) {
 114         case RTE_ETH_FILTER_GENERIC:
 115                 if (filter_op != RTE_ETH_FILTER_GET)
 116                         return -EINVAL;
 117                 *(const void **)arg = &enic_flow_ops;
 118                 break;
 119         case RTE_ETH_FILTER_FDIR:
 120                 ret = enicpmd_fdir_ctrl_func(dev, filter_op, arg);
 121                 break;
 122         default:
 123                 dev_warning(enic, "Filter type (%d) not supported",
 124                         filter_type);
 125                 ret = -EINVAL;
 126                 break;
 127         }
 128
 129         return ret;
 130 }
 131
 132 static void enicpmd_dev_tx_queue_release(void *txq)
 133 {
 134         ENICPMD_FUNC_TRACE();
 135
 136         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 137                 return;
 138
 139         enic_free_wq(txq);
 140 }
 141
 142 static int enicpmd_dev_setup_intr(struct enic *enic)
 143 {
 144         int ret;
 145         unsigned int index;
 146
 147         ENICPMD_FUNC_TRACE();
 148
 149         /* Are we done with the init of all the queues? */
 150         for (index = 0; index < enic->cq_count; index++) {
 151                 if (!enic->cq[index].ctrl)
 152                         break;
 153         }
 154         if (enic->cq_count != index)
 155                 return 0;
 156         for (index = 0; index < enic->wq_count; index++) {
 157                 if (!enic->wq[index].ctrl)
 158                         break;
 159         }
 160         if (enic->wq_count != index)
 161                 return 0;
 162         /* check start of packet (SOP) RQs only in case scatter is disabled. */
 163         for (index = 0; index < enic->rq_count; index++) {
 164                 if (!enic->rq[enic_rte_rq_idx_to_sop_idx(index)].ctrl)
 165                         break;
 166         }
 167         if (enic->rq_count != index)
 168                 return 0;
 169
 170         ret = enic_alloc_intr_resources(enic);
 171         if (ret) {
 172                 dev_err(enic, "alloc intr failed\n");
 173                 return ret;
 174         }
 175         enic_init_vnic_resources(enic);
 176
 177         ret = enic_setup_finish(enic);
 178         if (ret)
 179                 dev_err(enic, "setup could not be finished\n");
 180
 181         return ret;
 182 }
 183
 184 static int enicpmd_dev_tx_queue_setup(struct rte_eth_dev *eth_dev,
 185         uint16_t queue_idx,
 186         uint16_t nb_desc,
 187         unsigned int socket_id,
 188         const struct rte_eth_txconf *tx_conf)
 189 {
 190         int ret;
 191         struct enic *enic = pmd_priv(eth_dev);
 192         struct vnic_wq *wq;
 193
 194         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 195                 return -E_RTE_SECONDARY;
 196
 197         ENICPMD_FUNC_TRACE();
 198         RTE_ASSERT(queue_idx < enic->conf_wq_count);
 199         wq = &enic->wq[queue_idx];
 200         wq->offloads = tx_conf->offloads |
 201                 eth_dev->data->dev_conf.txmode.offloads;
 202         eth_dev->data->tx_queues[queue_idx] = (void *)wq;
 203
 204         ret = enic_alloc_wq(enic, queue_idx, socket_id, nb_desc);
 205         if (ret) {
 206                 dev_err(enic, "error in allocating wq\n");
 207                 return ret;
 208         }
 209
 210         return enicpmd_dev_setup_intr(enic);
 211 }
 212
 213 static int enicpmd_dev_tx_queue_start(struct rte_eth_dev *eth_dev,
 214         uint16_t queue_idx)
 215 {
 216         struct enic *enic = pmd_priv(eth_dev);
 217
 218         ENICPMD_FUNC_TRACE();
 219
 220         enic_start_wq(enic, queue_idx);
 221
 222         return 0;
 223 }
 224
 225 static int enicpmd_dev_tx_queue_stop(struct rte_eth_dev *eth_dev,
 226         uint16_t queue_idx)
 227 {
 228         int ret;
 229         struct enic *enic = pmd_priv(eth_dev);
 230
 231         ENICPMD_FUNC_TRACE();
 232
 233         ret = enic_stop_wq(enic, queue_idx);
 234         if (ret)
 235                 dev_err(enic, "error in stopping wq %d\n", queue_idx);
 236
 237         return ret;
 238 }
 239
 240 static int enicpmd_dev_rx_queue_start(struct rte_eth_dev *eth_dev,
 241         uint16_t queue_idx)
 242 {
 243         struct enic *enic = pmd_priv(eth_dev);
 244
 245         ENICPMD_FUNC_TRACE();
 246
 247         enic_start_rq(enic, queue_idx);
 248
 249         return 0;
 250 }
 251
 252 static int enicpmd_dev_rx_queue_stop(struct rte_eth_dev *eth_dev,
 253         uint16_t queue_idx)
 254 {
 255         int ret;
 256         struct enic *enic = pmd_priv(eth_dev);
 257
 258         ENICPMD_FUNC_TRACE();
 259
 260         ret = enic_stop_rq(enic, queue_idx);
 261         if (ret)
 262                 dev_err(enic, "error in stopping rq %d\n", queue_idx);
 263
 264         return ret;
 265 }
 266
 267 static void enicpmd_dev_rx_queue_release(void *rxq)
 268 {
 269         ENICPMD_FUNC_TRACE();
 270
 271         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 272                 return;
 273
 274         enic_free_rq(rxq);
 275 }
 276
 277 static uint32_t enicpmd_dev_rx_queue_count(struct rte_eth_dev *dev,
 278                                            uint16_t rx_queue_id)
 279 {
 280         struct enic *enic = pmd_priv(dev);
 281         uint32_t queue_count = 0;
 282         struct vnic_cq *cq;
 283         uint32_t cq_tail;
 284         uint16_t cq_idx;
 285         int rq_num;
 286
 287         rq_num = enic_rte_rq_idx_to_sop_idx(rx_queue_id);
 288         cq = &enic->cq[enic_cq_rq(enic, rq_num)];
 289         cq_idx = cq->to_clean;
 290
 291         cq_tail = ioread32(&cq->ctrl->cq_tail);
 292
 293         if (cq_tail < cq_idx)
 294                 cq_tail += cq->ring.desc_count;
 295
 296         queue_count = cq_tail - cq_idx;
 297
 298         return queue_count;
 299 }
 300
 301 static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
 302         uint16_t queue_idx,
 303         uint16_t nb_desc,
 304         unsigned int socket_id,
 305         const struct rte_eth_rxconf *rx_conf,
 306         struct rte_mempool *mp)
 307 {
 308         int ret;
 309         struct enic *enic = pmd_priv(eth_dev);
 310
 311         ENICPMD_FUNC_TRACE();
 312
 313         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 314                 return -E_RTE_SECONDARY;
 315         RTE_ASSERT(enic_rte_rq_idx_to_sop_idx(queue_idx) < enic->conf_rq_count);
 316         eth_dev->data->rx_queues[queue_idx] =
 317                 (void *)&enic->rq[enic_rte_rq_idx_to_sop_idx(queue_idx)];
 318
 319         ret = enic_alloc_rq(enic, queue_idx, socket_id, mp, nb_desc,
 320                             rx_conf->rx_free_thresh);
 321         if (ret) {
 322                 dev_err(enic, "error in allocating rq\n");
 323                 return ret;
 324         }
 325
 326         return enicpmd_dev_setup_intr(enic);
 327 }
 328
 329 static int enicpmd_vlan_offload_set(struct rte_eth_dev *eth_dev, int mask)
 330 {
 331         struct enic *enic = pmd_priv(eth_dev);
 332         uint64_t offloads;
 333
 334         ENICPMD_FUNC_TRACE();
 335
 336         offloads = eth_dev->data->dev_conf.rxmode.offloads;
 337         if (mask & ETH_VLAN_STRIP_MASK) {
 338                 if (offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
 339                         enic->ig_vlan_strip_en = 1;
 340                 else
 341                         enic->ig_vlan_strip_en = 0;
 342         }
 343
 344         if ((mask & ETH_VLAN_FILTER_MASK) &&
 345             (offloads & DEV_RX_OFFLOAD_VLAN_FILTER)) {
 346                 dev_warning(enic,
 347                         "Configuration of VLAN filter is not supported\n");
 348         }
 349
 350         if ((mask & ETH_VLAN_EXTEND_MASK) &&
 351             (offloads & DEV_RX_OFFLOAD_VLAN_EXTEND)) {
 352                 dev_warning(enic,
 353                         "Configuration of extended VLAN is not supported\n");
 354         }
 355
 356         return enic_set_vlan_strip(enic);
 357 }
 358
 359 static int enicpmd_dev_configure(struct rte_eth_dev *eth_dev)
 360 {
 361         int ret;
 362         int mask;
 363         struct enic *enic = pmd_priv(eth_dev);
 364
 365         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 366                 return -E_RTE_SECONDARY;
 367
 368         ENICPMD_FUNC_TRACE();
 369         ret = enic_set_vnic_res(enic);
 370         if (ret) {
 371                 dev_err(enic, "Set vNIC resource num  failed, aborting\n");
 372                 return ret;
 373         }
 374
 375         enic->hw_ip_checksum = !!(eth_dev->data->dev_conf.rxmode.offloads &
 376                                   DEV_RX_OFFLOAD_CHECKSUM);
 377         /* All vlan offload masks to apply the current settings */
 378         mask = ETH_VLAN_STRIP_MASK |
 379                 ETH_VLAN_FILTER_MASK |
 380                 ETH_VLAN_EXTEND_MASK;
 381         ret = enicpmd_vlan_offload_set(eth_dev, mask);
 382         if (ret) {
 383                 dev_err(enic, "Failed to configure VLAN offloads\n");
 384                 return ret;
 385         }
 386         /*
 387          * Initialize RSS with the default reta and key. If the user key is
 388          * given (rx_adv_conf.rss_conf.rss_key), will use that instead of the
 389          * default key.
 390          */
 391         return enic_init_rss_nic_cfg(enic);
 392 }
 393
 394 /* Start the device.
 395  * It returns 0 on success.
 396  */
 397 static int enicpmd_dev_start(struct rte_eth_dev *eth_dev)
 398 {
 399         struct enic *enic = pmd_priv(eth_dev);
 400
 401         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 402                 return -E_RTE_SECONDARY;
 403
 404         ENICPMD_FUNC_TRACE();
 405         return enic_enable(enic);
 406 }
 407
 408 /*
 409  * Stop device: disable rx and tx functions to allow for reconfiguring.
 410  */
 411 static void enicpmd_dev_stop(struct rte_eth_dev *eth_dev)
 412 {
 413         struct rte_eth_link link;
 414         struct enic *enic = pmd_priv(eth_dev);
 415
 416         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 417                 return;
 418
 419         ENICPMD_FUNC_TRACE();
 420         enic_disable(enic);
 421
 422         memset(&link, 0, sizeof(link));
 423         rte_eth_linkstatus_set(eth_dev, &link);
 424 }
 425
 426 /*
 427  * Stop device.
 428  */
 429 static void enicpmd_dev_close(struct rte_eth_dev *eth_dev)
 430 {
 431         struct enic *enic = pmd_priv(eth_dev);
 432
 433         ENICPMD_FUNC_TRACE();
 434         enic_remove(enic);
 435 }
 436
 437 static int enicpmd_dev_link_update(struct rte_eth_dev *eth_dev,
 438         __rte_unused int wait_to_complete)
 439 {
 440         struct enic *enic = pmd_priv(eth_dev);
 441
 442         ENICPMD_FUNC_TRACE();
 443         return enic_link_update(enic);
 444 }
 445
 446 static int enicpmd_dev_stats_get(struct rte_eth_dev *eth_dev,
 447         struct rte_eth_stats *stats)
 448 {
 449         struct enic *enic = pmd_priv(eth_dev);
 450
 451         ENICPMD_FUNC_TRACE();
 452         return enic_dev_stats_get(enic, stats);
 453 }
 454
 455 static void enicpmd_dev_stats_reset(struct rte_eth_dev *eth_dev)
 456 {
 457         struct enic *enic = pmd_priv(eth_dev);
 458
 459         ENICPMD_FUNC_TRACE();
 460         enic_dev_stats_clear(enic);
 461 }
 462
 463 static void enicpmd_dev_info_get(struct rte_eth_dev *eth_dev,
 464         struct rte_eth_dev_info *device_info)
 465 {
 466         struct enic *enic = pmd_priv(eth_dev);
 467
 468         ENICPMD_FUNC_TRACE();
 469         /* Scattered Rx uses two receive queues per rx queue exposed to dpdk */
 470         device_info->max_rx_queues = enic->conf_rq_count / 2;
 471         device_info->max_tx_queues = enic->conf_wq_count;
 472         device_info->min_rx_bufsize = ENIC_MIN_MTU;
 473         /* "Max" mtu is not a typo. HW receives packet sizes up to the
 474          * max mtu regardless of the current mtu (vNIC's mtu). vNIC mtu is
 475          * a hint to the driver to size receive buffers accordingly so that
 476          * larger-than-vnic-mtu packets get truncated.. For DPDK, we let
 477          * the user decide the buffer size via rxmode.max_rx_pkt_len, basically
 478          * ignoring vNIC mtu.
 479          */
 480         device_info->max_rx_pktlen = enic_mtu_to_max_rx_pktlen(enic->max_mtu);
 481         device_info->max_mac_addrs = ENIC_MAX_MAC_ADDR;
 482         device_info->rx_offload_capa = enic->rx_offload_capa;
 483         device_info->tx_offload_capa = enic->tx_offload_capa;
 484         device_info->tx_queue_offload_capa = enic->tx_queue_offload_capa;
 485         device_info->default_rxconf = (struct rte_eth_rxconf) {
 486                 .rx_free_thresh = ENIC_DEFAULT_RX_FREE_THRESH
 487         };
 488         device_info->reta_size = enic->reta_size;
 489         device_info->hash_key_size = enic->hash_key_size;
 490         device_info->flow_type_rss_offloads = enic->flow_type_rss_offloads;
 491         device_info->rx_desc_lim = (struct rte_eth_desc_lim) {
 492                 .nb_max = enic->config.rq_desc_count,
 493                 .nb_min = ENIC_MIN_RQ_DESCS,
 494                 .nb_align = ENIC_ALIGN_DESCS,
 495         };
 496         device_info->tx_desc_lim = (struct rte_eth_desc_lim) {
 497                 .nb_max = enic->config.wq_desc_count,
 498                 .nb_min = ENIC_MIN_WQ_DESCS,
 499                 .nb_align = ENIC_ALIGN_DESCS,
 500                 .nb_seg_max = ENIC_TX_XMIT_MAX,
 501                 .nb_mtu_seg_max = ENIC_NON_TSO_MAX_DESC,
 502         };
 503         device_info->default_rxportconf = (struct rte_eth_dev_portconf) {
 504                 .burst_size = ENIC_DEFAULT_RX_BURST,
 505                 .ring_size = RTE_MIN(device_info->rx_desc_lim.nb_max,
 506                         ENIC_DEFAULT_RX_RING_SIZE),
 507                 .nb_queues = ENIC_DEFAULT_RX_RINGS,
 508         };
 509         device_info->default_txportconf = (struct rte_eth_dev_portconf) {
 510                 .burst_size = ENIC_DEFAULT_TX_BURST,
 511                 .ring_size = RTE_MIN(device_info->tx_desc_lim.nb_max,
 512                         ENIC_DEFAULT_TX_RING_SIZE),
 513                 .nb_queues = ENIC_DEFAULT_TX_RINGS,
 514         };
 515 }
 516
 517 static const uint32_t *enicpmd_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 518 {
 519         static const uint32_t ptypes[] = {
 520                 RTE_PTYPE_L2_ETHER,
 521                 RTE_PTYPE_L2_ETHER_VLAN,
 522                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
 523                 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
 524                 RTE_PTYPE_L4_TCP,
 525                 RTE_PTYPE_L4_UDP,
 526                 RTE_PTYPE_L4_FRAG,
 527                 RTE_PTYPE_L4_NONFRAG,
 528                 RTE_PTYPE_UNKNOWN
 529         };
 530
 531         if (dev->rx_pkt_burst == enic_recv_pkts)
 532                 return ptypes;
 533         return NULL;
 534 }
 535
 536 static void enicpmd_dev_promiscuous_enable(struct rte_eth_dev *eth_dev)
 537 {
 538         struct enic *enic = pmd_priv(eth_dev);
 539
 540         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 541                 return;
 542
 543         ENICPMD_FUNC_TRACE();
 544
 545         enic->promisc = 1;
 546         enic_add_packet_filter(enic);
 547 }
 548
 549 static void enicpmd_dev_promiscuous_disable(struct rte_eth_dev *eth_dev)
 550 {
 551         struct enic *enic = pmd_priv(eth_dev);
 552
 553         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 554                 return;
 555
 556         ENICPMD_FUNC_TRACE();
 557         enic->promisc = 0;
 558         enic_add_packet_filter(enic);
 559 }
 560
 561 static void enicpmd_dev_allmulticast_enable(struct rte_eth_dev *eth_dev)
 562 {
 563         struct enic *enic = pmd_priv(eth_dev);
 564
 565         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 566                 return;
 567
 568         ENICPMD_FUNC_TRACE();
 569         enic->allmulti = 1;
 570         enic_add_packet_filter(enic);
 571 }
 572
 573 static void enicpmd_dev_allmulticast_disable(struct rte_eth_dev *eth_dev)
 574 {
 575         struct enic *enic = pmd_priv(eth_dev);
 576
 577         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 578                 return;
 579
 580         ENICPMD_FUNC_TRACE();
 581         enic->allmulti = 0;
 582         enic_add_packet_filter(enic);
 583 }
 584
 585 static int enicpmd_add_mac_addr(struct rte_eth_dev *eth_dev,
 586         struct ether_addr *mac_addr,
 587         __rte_unused uint32_t index, __rte_unused uint32_t pool)
 588 {
 589         struct enic *enic = pmd_priv(eth_dev);
 590
 591         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 592                 return -E_RTE_SECONDARY;
 593
 594         ENICPMD_FUNC_TRACE();
 595         return enic_set_mac_address(enic, mac_addr->addr_bytes);
 596 }
 597
 598 static void enicpmd_remove_mac_addr(struct rte_eth_dev *eth_dev, uint32_t index)
 599 {
 600         struct enic *enic = pmd_priv(eth_dev);
 601
 602         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 603                 return;
 604
 605         ENICPMD_FUNC_TRACE();
 606         if (enic_del_mac_address(enic, index))
 607                 dev_err(enic, "del mac addr failed\n");
 608 }
 609
 610 static int enicpmd_set_mac_addr(struct rte_eth_dev *eth_dev,
 611                                 struct ether_addr *addr)
 612 {
 613         struct enic *enic = pmd_priv(eth_dev);
 614         int ret;
 615
 616         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 617                 return -E_RTE_SECONDARY;
 618
 619         ENICPMD_FUNC_TRACE();
 620         ret = enic_del_mac_address(enic, 0);
 621         if (ret)
 622                 return ret;
 623         return enic_set_mac_address(enic, addr->addr_bytes);
 624 }
 625
 626 static int enicpmd_mtu_set(struct rte_eth_dev *eth_dev, uint16_t mtu)
 627 {
 628         struct enic *enic = pmd_priv(eth_dev);
 629
 630         ENICPMD_FUNC_TRACE();
 631         return enic_set_mtu(enic, mtu);
 632 }
 633
 634 static int enicpmd_dev_rss_reta_query(struct rte_eth_dev *dev,
 635                                       struct rte_eth_rss_reta_entry64
 636                                       *reta_conf,
 637                                       uint16_t reta_size)
 638 {
 639         struct enic *enic = pmd_priv(dev);
 640         uint16_t i, idx, shift;
 641
 642         ENICPMD_FUNC_TRACE();
 643         if (reta_size != ENIC_RSS_RETA_SIZE) {
 644                 dev_err(enic, "reta_query: wrong reta_size. given=%u expected=%u\n",
 645                         reta_size, ENIC_RSS_RETA_SIZE);
 646                 return -EINVAL;
 647         }
 648
 649         for (i = 0; i < reta_size; i++) {
 650                 idx = i / RTE_RETA_GROUP_SIZE;
 651                 shift = i % RTE_RETA_GROUP_SIZE;
 652                 if (reta_conf[idx].mask & (1ULL << shift))
 653                         reta_conf[idx].reta[shift] = enic_sop_rq_idx_to_rte_idx(
 654                                 enic->rss_cpu.cpu[i / 4].b[i % 4]);
 655         }
 656
 657         return 0;
 658 }
 659
 660 static int enicpmd_dev_rss_reta_update(struct rte_eth_dev *dev,
 661                                        struct rte_eth_rss_reta_entry64
 662                                        *reta_conf,
 663                                        uint16_t reta_size)
 664 {
 665         struct enic *enic = pmd_priv(dev);
 666         union vnic_rss_cpu rss_cpu;
 667         uint16_t i, idx, shift;
 668
 669         ENICPMD_FUNC_TRACE();
 670         if (reta_size != ENIC_RSS_RETA_SIZE) {
 671                 dev_err(enic, "reta_update: wrong reta_size. given=%u"
 672                         " expected=%u\n",
 673                         reta_size, ENIC_RSS_RETA_SIZE);
 674                 return -EINVAL;
 675         }
 676         /*
 677          * Start with the current reta and modify it per reta_conf, as we
 678          * need to push the entire reta even if we only modify one entry.
 679          */
 680         rss_cpu = enic->rss_cpu;
 681         for (i = 0; i < reta_size; i++) {
 682                 idx = i / RTE_RETA_GROUP_SIZE;
 683                 shift = i % RTE_RETA_GROUP_SIZE;
 684                 if (reta_conf[idx].mask & (1ULL << shift))
 685                         rss_cpu.cpu[i / 4].b[i % 4] =
 686                                 enic_rte_rq_idx_to_sop_idx(
 687                                         reta_conf[idx].reta[shift]);
 688         }
 689         return enic_set_rss_reta(enic, &rss_cpu);
 690 }
 691
 692 static int enicpmd_dev_rss_hash_update(struct rte_eth_dev *dev,
 693                                        struct rte_eth_rss_conf *rss_conf)
 694 {
 695         struct enic *enic = pmd_priv(dev);
 696
 697         ENICPMD_FUNC_TRACE();
 698         return enic_set_rss_conf(enic, rss_conf);
 699 }
 700
 701 static int enicpmd_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
 702                                          struct rte_eth_rss_conf *rss_conf)
 703 {
 704         struct enic *enic = pmd_priv(dev);
 705
 706         ENICPMD_FUNC_TRACE();
 707         if (rss_conf == NULL)
 708                 return -EINVAL;
 709         if (rss_conf->rss_key != NULL &&
 710             rss_conf->rss_key_len < ENIC_RSS_HASH_KEY_SIZE) {
 711                 dev_err(enic, "rss_hash_conf_get: wrong rss_key_len. given=%u"
 712                         " expected=%u+\n",
 713                         rss_conf->rss_key_len, ENIC_RSS_HASH_KEY_SIZE);
 714                 return -EINVAL;
 715         }
 716         rss_conf->rss_hf = enic->rss_hf;
 717         if (rss_conf->rss_key != NULL) {
 718                 int i;
 719                 for (i = 0; i < ENIC_RSS_HASH_KEY_SIZE; i++) {
 720                         rss_conf->rss_key[i] =
 721                                 enic->rss_key.key[i / 10].b[i % 10];
 722                 }
 723                 rss_conf->rss_key_len = ENIC_RSS_HASH_KEY_SIZE;
 724         }
 725         return 0;
 726 }
 727
 728 static void enicpmd_dev_rxq_info_get(struct rte_eth_dev *dev,
 729                                      uint16_t rx_queue_id,
 730                                      struct rte_eth_rxq_info *qinfo)
 731 {
 732         struct enic *enic = pmd_priv(dev);
 733         struct vnic_rq *rq_sop;
 734         struct vnic_rq *rq_data;
 735         struct rte_eth_rxconf *conf;
 736         uint16_t sop_queue_idx;
 737         uint16_t data_queue_idx;
 738
 739         ENICPMD_FUNC_TRACE();
 740         sop_queue_idx = enic_rte_rq_idx_to_sop_idx(rx_queue_id);
 741         data_queue_idx = enic_rte_rq_idx_to_data_idx(rx_queue_id);
 742         rq_sop = &enic->rq[sop_queue_idx];
 743         rq_data = &enic->rq[data_queue_idx]; /* valid if data_queue_enable */
 744         qinfo->mp = rq_sop->mp;
 745         qinfo->scattered_rx = rq_sop->data_queue_enable;
 746         qinfo->nb_desc = rq_sop->ring.desc_count;
 747         if (qinfo->scattered_rx)
 748                 qinfo->nb_desc += rq_data->ring.desc_count;
 749         conf = &qinfo->conf;
 750         memset(conf, 0, sizeof(*conf));
 751         conf->rx_free_thresh = rq_sop->rx_free_thresh;
 752         conf->rx_drop_en = 1;
 753         /*
 754          * Except VLAN stripping (port setting), all the checksum offloads
 755          * are always enabled.
 756          */
 757         conf->offloads = enic->rx_offload_capa;
 758         if (!enic->ig_vlan_strip_en)
 759                 conf->offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP;
 760         /* rx_thresh and other fields are not applicable for enic */
 761 }
 762
 763 static void enicpmd_dev_txq_info_get(struct rte_eth_dev *dev,
 764                                      uint16_t tx_queue_id,
 765                                      struct rte_eth_txq_info *qinfo)
 766 {
 767         struct enic *enic = pmd_priv(dev);
 768         struct vnic_wq *wq = &enic->wq[tx_queue_id];
 769
 770         ENICPMD_FUNC_TRACE();
 771         qinfo->nb_desc = wq->ring.desc_count;
 772         memset(&qinfo->conf, 0, sizeof(qinfo->conf));
 773         qinfo->conf.offloads = wq->offloads;
 774         /* tx_thresh, and all the other fields are not applicable for enic */
 775 }
 776
 777 static int enicpmd_dev_rx_queue_intr_enable(struct rte_eth_dev *eth_dev,
 778                                             uint16_t rx_queue_id)
 779 {
 780         struct enic *enic = pmd_priv(eth_dev);
 781
 782         ENICPMD_FUNC_TRACE();
 783         vnic_intr_unmask(&enic->intr[rx_queue_id + ENICPMD_RXQ_INTR_OFFSET]);
 784         return 0;
 785 }
 786
 787 static int enicpmd_dev_rx_queue_intr_disable(struct rte_eth_dev *eth_dev,
 788                                              uint16_t rx_queue_id)
 789 {
 790         struct enic *enic = pmd_priv(eth_dev);
 791
 792         ENICPMD_FUNC_TRACE();
 793         vnic_intr_mask(&enic->intr[rx_queue_id + ENICPMD_RXQ_INTR_OFFSET]);
 794         return 0;
 795 }
 796
 797 static int udp_tunnel_common_check(struct enic *enic,
 798                                    struct rte_eth_udp_tunnel *tnl)
 799 {
 800         if (tnl->prot_type != RTE_TUNNEL_TYPE_VXLAN)
 801                 return -ENOTSUP;
 802         if (!enic->overlay_offload) {
 803                 PMD_INIT_LOG(DEBUG, " vxlan (overlay offload) is not "
 804                              "supported\n");
 805                 return -ENOTSUP;
 806         }
 807         return 0;
 808 }
 809
 810 static int update_vxlan_port(struct enic *enic, uint16_t port)
 811 {
 812         if (vnic_dev_overlay_offload_cfg(enic->vdev,
 813                                          OVERLAY_CFG_VXLAN_PORT_UPDATE,
 814                                          port)) {
 815                 PMD_INIT_LOG(DEBUG, " failed to update vxlan port\n");
 816                 return -EINVAL;
 817         }
 818         PMD_INIT_LOG(DEBUG, " updated vxlan port to %u\n", port);
 819         enic->vxlan_port = port;
 820         return 0;
 821 }
 822
 823 static int enicpmd_dev_udp_tunnel_port_add(struct rte_eth_dev *eth_dev,
 824                                            struct rte_eth_udp_tunnel *tnl)
 825 {
 826         struct enic *enic = pmd_priv(eth_dev);
 827         int ret;
 828
 829         ENICPMD_FUNC_TRACE();
 830         ret = udp_tunnel_common_check(enic, tnl);
 831         if (ret)
 832                 return ret;
 833         /*
 834          * The NIC has 1 configurable VXLAN port number. "Adding" a new port
 835          * number replaces it.
 836          */
 837         if (tnl->udp_port == enic->vxlan_port || tnl->udp_port == 0) {
 838                 PMD_INIT_LOG(DEBUG, " %u is already configured or invalid\n",
 839                              tnl->udp_port);
 840                 return -EINVAL;
 841         }
 842         return update_vxlan_port(enic, tnl->udp_port);
 843 }
 844
 845 static int enicpmd_dev_udp_tunnel_port_del(struct rte_eth_dev *eth_dev,
 846                                            struct rte_eth_udp_tunnel *tnl)
 847 {
 848         struct enic *enic = pmd_priv(eth_dev);
 849         int ret;
 850
 851         ENICPMD_FUNC_TRACE();
 852         ret = udp_tunnel_common_check(enic, tnl);
 853         if (ret)
 854                 return ret;
 855         /*
 856          * Clear the previously set port number and restore the
 857          * hardware default port number. Some drivers disable VXLAN
 858          * offloads when there are no configured port numbers. But
 859          * enic does not do that as VXLAN is part of overlay offload,
 860          * which is tied to inner RSS and TSO.
 861          */
 862         if (tnl->udp_port != enic->vxlan_port) {
 863                 PMD_INIT_LOG(DEBUG, " %u is not a configured vxlan port\n",
 864                              tnl->udp_port);
 865                 return -EINVAL;
 866         }
 867         return update_vxlan_port(enic, ENIC_DEFAULT_VXLAN_PORT);
 868 }
 869
 870 static const struct eth_dev_ops enicpmd_eth_dev_ops = {
 871         .dev_configure        = enicpmd_dev_configure,
 872         .dev_start            = enicpmd_dev_start,
 873         .dev_stop             = enicpmd_dev_stop,
 874         .dev_set_link_up      = NULL,
 875         .dev_set_link_down    = NULL,
 876         .dev_close            = enicpmd_dev_close,
 877         .promiscuous_enable   = enicpmd_dev_promiscuous_enable,
 878         .promiscuous_disable  = enicpmd_dev_promiscuous_disable,
 879         .allmulticast_enable  = enicpmd_dev_allmulticast_enable,
 880         .allmulticast_disable = enicpmd_dev_allmulticast_disable,
 881         .link_update          = enicpmd_dev_link_update,
 882         .stats_get            = enicpmd_dev_stats_get,
 883         .stats_reset          = enicpmd_dev_stats_reset,
 884         .queue_stats_mapping_set = NULL,
 885         .dev_infos_get        = enicpmd_dev_info_get,
 886         .dev_supported_ptypes_get = enicpmd_dev_supported_ptypes_get,
 887         .mtu_set              = enicpmd_mtu_set,
 888         .vlan_filter_set      = NULL,
 889         .vlan_tpid_set        = NULL,
 890         .vlan_offload_set     = enicpmd_vlan_offload_set,
 891         .vlan_strip_queue_set = NULL,
 892         .rx_queue_start       = enicpmd_dev_rx_queue_start,
 893         .rx_queue_stop        = enicpmd_dev_rx_queue_stop,
 894         .tx_queue_start       = enicpmd_dev_tx_queue_start,
 895         .tx_queue_stop        = enicpmd_dev_tx_queue_stop,
 896         .rx_queue_setup       = enicpmd_dev_rx_queue_setup,
 897         .rx_queue_release     = enicpmd_dev_rx_queue_release,
 898         .rx_queue_count       = enicpmd_dev_rx_queue_count,
 899         .rx_descriptor_done   = NULL,
 900         .tx_queue_setup       = enicpmd_dev_tx_queue_setup,
 901         .tx_queue_release     = enicpmd_dev_tx_queue_release,
 902         .rx_queue_intr_enable = enicpmd_dev_rx_queue_intr_enable,
 903         .rx_queue_intr_disable = enicpmd_dev_rx_queue_intr_disable,
 904         .rxq_info_get         = enicpmd_dev_rxq_info_get,
 905         .txq_info_get         = enicpmd_dev_txq_info_get,
 906         .dev_led_on           = NULL,
 907         .dev_led_off          = NULL,
 908         .flow_ctrl_get        = NULL,
 909         .flow_ctrl_set        = NULL,
 910         .priority_flow_ctrl_set = NULL,
 911         .mac_addr_add         = enicpmd_add_mac_addr,
 912         .mac_addr_remove      = enicpmd_remove_mac_addr,
 913         .mac_addr_set         = enicpmd_set_mac_addr,
 914         .filter_ctrl          = enicpmd_dev_filter_ctrl,
 915         .reta_query           = enicpmd_dev_rss_reta_query,
 916         .reta_update          = enicpmd_dev_rss_reta_update,
 917         .rss_hash_conf_get    = enicpmd_dev_rss_hash_conf_get,
 918         .rss_hash_update      = enicpmd_dev_rss_hash_update,
 919         .udp_tunnel_port_add  = enicpmd_dev_udp_tunnel_port_add,
 920         .udp_tunnel_port_del  = enicpmd_dev_udp_tunnel_port_del,
 921 };
 922
 923 static int enic_parse_disable_overlay(__rte_unused const char *key,
 924                                       const char *value,
 925                                       void *opaque)
 926 {
 927         struct enic *enic;
 928
 929         enic = (struct enic *)opaque;
 930         if (strcmp(value, "0") == 0) {
 931                 enic->disable_overlay = false;
 932         } else if (strcmp(value, "1") == 0) {
 933                 enic->disable_overlay = true;
 934         } else {
 935                 dev_err(enic, "Invalid value for " ENIC_DEVARG_DISABLE_OVERLAY
 936                         ": expected=0|1 given=%s\n", value);
 937                 return -EINVAL;
 938         }
 939         return 0;
 940 }
 941
 942 static int enic_parse_ig_vlan_rewrite(__rte_unused const char *key,
 943                                       const char *value,
 944                                       void *opaque)
 945 {
 946         struct enic *enic;
 947
 948         enic = (struct enic *)opaque;
 949         if (strcmp(value, "trunk") == 0) {
 950                 /* Trunk mode: always tag */
 951                 enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_DEFAULT_TRUNK;
 952         } else if (strcmp(value, "untag") == 0) {
 953                 /* Untag default VLAN mode: untag if VLAN = default VLAN */
 954                 enic->ig_vlan_rewrite_mode =
 955                         IG_VLAN_REWRITE_MODE_UNTAG_DEFAULT_VLAN;
 956         } else if (strcmp(value, "priority") == 0) {
 957                 /*
 958                  * Priority-tag default VLAN mode: priority tag (VLAN header
 959                  * with ID=0) if VLAN = default
 960                  */
 961                 enic->ig_vlan_rewrite_mode =
 962                         IG_VLAN_REWRITE_MODE_PRIORITY_TAG_DEFAULT_VLAN;
 963         } else if (strcmp(value, "pass") == 0) {
 964                 /* Pass through mode: do not touch tags */
 965                 enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_PASS_THRU;
 966         } else {
 967                 dev_err(enic, "Invalid value for " ENIC_DEVARG_IG_VLAN_REWRITE
 968                         ": expected=trunk|untag|priority|pass given=%s\n",
 969                         value);
 970                 return -EINVAL;
 971         }
 972         return 0;
 973 }
 974
 975 static int enic_check_devargs(struct rte_eth_dev *dev)
 976 {
 977         static const char *const valid_keys[] = {
 978                 ENIC_DEVARG_DISABLE_OVERLAY,
 979                 ENIC_DEVARG_IG_VLAN_REWRITE,
 980                 NULL};
 981         struct enic *enic = pmd_priv(dev);
 982         struct rte_kvargs *kvlist;
 983
 984         ENICPMD_FUNC_TRACE();
 985
 986         enic->disable_overlay = false;
 987         enic->ig_vlan_rewrite_mode = IG_VLAN_REWRITE_MODE_PASS_THRU;
 988         if (!dev->device->devargs)
 989                 return 0;
 990         kvlist = rte_kvargs_parse(dev->device->devargs->args, valid_keys);
 991         if (!kvlist)
 992                 return -EINVAL;
 993         if (rte_kvargs_process(kvlist, ENIC_DEVARG_DISABLE_OVERLAY,
 994                                enic_parse_disable_overlay, enic) < 0 ||
 995             rte_kvargs_process(kvlist, ENIC_DEVARG_IG_VLAN_REWRITE,
 996                                enic_parse_ig_vlan_rewrite, enic) < 0) {
 997                 rte_kvargs_free(kvlist);
 998                 return -EINVAL;
 999         }
1000         rte_kvargs_free(kvlist);
1001         return 0;
1002 }
1003
1004 struct enic *enicpmd_list_head = NULL;
1005 /* Initialize the driver
1006  * It returns 0 on success.
1007  */
1008 static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev)
1009 {
1010         struct rte_pci_device *pdev;
1011         struct rte_pci_addr *addr;
1012         struct enic *enic = pmd_priv(eth_dev);
1013         int err;
1014
1015         ENICPMD_FUNC_TRACE();
1016
1017         enic->port_id = eth_dev->data->port_id;
1018         enic->rte_dev = eth_dev;
1019         eth_dev->dev_ops = &enicpmd_eth_dev_ops;
1020         eth_dev->rx_pkt_burst = &enic_recv_pkts;
1021         eth_dev->tx_pkt_burst = &enic_xmit_pkts;
1022         eth_dev->tx_pkt_prepare = &enic_prep_pkts;
1023
1024         pdev = RTE_ETH_DEV_TO_PCI(eth_dev);
1025         rte_eth_copy_pci_info(eth_dev, pdev);
1026         enic->pdev = pdev;
1027         addr = &pdev->addr;
1028
1029         snprintf(enic->bdf_name, ENICPMD_BDF_LENGTH, "%04x:%02x:%02x.%x",
1030                 addr->domain, addr->bus, addr->devid, addr->function);
1031
1032         err = enic_check_devargs(eth_dev);
1033         if (err)
1034                 return err;
1035         return enic_probe(enic);
1036 }
1037
1038 static int eth_enic_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
1039         struct rte_pci_device *pci_dev)
1040 {
1041         return rte_eth_dev_pci_generic_probe(pci_dev, sizeof(struct enic),
1042                 eth_enicpmd_dev_init);
1043 }
1044
1045 static int eth_enic_pci_remove(struct rte_pci_device *pci_dev)
1046 {
1047         return rte_eth_dev_pci_generic_remove(pci_dev, NULL);
1048 }
1049
1050 static struct rte_pci_driver rte_enic_pmd = {
1051         .id_table = pci_id_enic_map,
1052         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
1053         .probe = eth_enic_pci_probe,
1054         .remove = eth_enic_pci_remove,
1055 };
1056
1057 RTE_PMD_REGISTER_PCI(net_enic, rte_enic_pmd);
1058 RTE_PMD_REGISTER_PCI_TABLE(net_enic, pci_id_enic_map);
1059 RTE_PMD_REGISTER_KMOD_DEP(net_enic, "* igb_uio | uio_pci_generic | vfio-pci");
1060 RTE_PMD_REGISTER_PARAM_STRING(net_enic,
1061         ENIC_DEVARG_DISABLE_OVERLAY "=0|1 "
1062         ENIC_DEVARG_IG_VLAN_REWRITE "=trunk|untag|priority|pass");