drivers/net/sfc/sfc_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (c) 2016-2018 Solarflare Communications Inc.
   4  * All rights reserved.
   5  *
   6  * This software was jointly developed between OKTET Labs (under contract
   7  * for Solarflare) and Solarflare Communications, Inc.
   8  */
   9
  10 #include <rte_dev.h>
  11 #include <rte_ethdev_driver.h>
  12 #include <rte_ethdev_pci.h>
  13 #include <rte_pci.h>
  14 #include <rte_bus_pci.h>
  15 #include <rte_errno.h>
  16
  17 #include "efx.h"
  18
  19 #include "sfc.h"
  20 #include "sfc_debug.h"
  21 #include "sfc_log.h"
  22 #include "sfc_kvargs.h"
  23 #include "sfc_ev.h"
  24 #include "sfc_rx.h"
  25 #include "sfc_tx.h"
  26 #include "sfc_flow.h"
  27 #include "sfc_dp.h"
  28 #include "sfc_dp_rx.h"
  29
  30 uint32_t sfc_logtype_driver;
  31
  32 static struct sfc_dp_list sfc_dp_head =
  33         TAILQ_HEAD_INITIALIZER(sfc_dp_head);
  34
  35 static int
  36 sfc_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size)
  37 {
  38         struct sfc_adapter *sa = dev->data->dev_private;
  39         efx_nic_fw_info_t enfi;
  40         int ret;
  41         int rc;
  42
  43         /*
  44          * Return value of the callback is likely supposed to be
  45          * equal to or greater than 0, nevertheless, if an error
  46          * occurs, it will be desirable to pass it to the caller
  47          */
  48         if ((fw_version == NULL) || (fw_size == 0))
  49                 return -EINVAL;
  50
  51         rc = efx_nic_get_fw_version(sa->nic, &enfi);
  52         if (rc != 0)
  53                 return -rc;
  54
  55         ret = snprintf(fw_version, fw_size,
  56                        "%" PRIu16 ".%" PRIu16 ".%" PRIu16 ".%" PRIu16,
  57                        enfi.enfi_mc_fw_version[0], enfi.enfi_mc_fw_version[1],
  58                        enfi.enfi_mc_fw_version[2], enfi.enfi_mc_fw_version[3]);
  59         if (ret < 0)
  60                 return ret;
  61
  62         if (enfi.enfi_dpcpu_fw_ids_valid) {
  63                 size_t dpcpu_fw_ids_offset = MIN(fw_size - 1, (size_t)ret);
  64                 int ret_extra;
  65
  66                 ret_extra = snprintf(fw_version + dpcpu_fw_ids_offset,
  67                                      fw_size - dpcpu_fw_ids_offset,
  68                                      " rx%" PRIx16 " tx%" PRIx16,
  69                                      enfi.enfi_rx_dpcpu_fw_id,
  70                                      enfi.enfi_tx_dpcpu_fw_id);
  71                 if (ret_extra < 0)
  72                         return ret_extra;
  73
  74                 ret += ret_extra;
  75         }
  76
  77         if (fw_size < (size_t)(++ret))
  78                 return ret;
  79         else
  80                 return 0;
  81 }
  82
  83 static void
  84 sfc_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
  85 {
  86         struct sfc_adapter *sa = dev->data->dev_private;
  87         const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
  88         uint64_t txq_offloads_def = 0;
  89
  90         sfc_log_init(sa, "entry");
  91
  92         dev_info->pci_dev = RTE_ETH_DEV_TO_PCI(dev);
  93         dev_info->max_rx_pktlen = EFX_MAC_PDU_MAX;
  94
  95         /* Autonegotiation may be disabled */
  96         dev_info->speed_capa = ETH_LINK_SPEED_FIXED;
  97         if (sa->port.phy_adv_cap_mask & EFX_PHY_CAP_1000FDX)
  98                 dev_info->speed_capa |= ETH_LINK_SPEED_1G;
  99         if (sa->port.phy_adv_cap_mask & EFX_PHY_CAP_10000FDX)
 100                 dev_info->speed_capa |= ETH_LINK_SPEED_10G;
 101         if (sa->port.phy_adv_cap_mask & EFX_PHY_CAP_25000FDX)
 102                 dev_info->speed_capa |= ETH_LINK_SPEED_25G;
 103         if (sa->port.phy_adv_cap_mask & EFX_PHY_CAP_40000FDX)
 104                 dev_info->speed_capa |= ETH_LINK_SPEED_40G;
 105         if (sa->port.phy_adv_cap_mask & EFX_PHY_CAP_50000FDX)
 106                 dev_info->speed_capa |= ETH_LINK_SPEED_50G;
 107         if (sa->port.phy_adv_cap_mask & EFX_PHY_CAP_100000FDX)
 108                 dev_info->speed_capa |= ETH_LINK_SPEED_100G;
 109
 110         dev_info->max_rx_queues = sa->rxq_max;
 111         dev_info->max_tx_queues = sa->txq_max;
 112
 113         /* By default packets are dropped if no descriptors are available */
 114         dev_info->default_rxconf.rx_drop_en = 1;
 115
 116         dev_info->rx_queue_offload_capa = sfc_rx_get_queue_offload_caps(sa);
 117
 118         /*
 119          * rx_offload_capa includes both device and queue offloads since
 120          * the latter may be requested on a per device basis which makes
 121          * sense when some offloads are needed to be set on all queues.
 122          */
 123         dev_info->rx_offload_capa = sfc_rx_get_dev_offload_caps(sa) |
 124                                     dev_info->rx_queue_offload_capa;
 125
 126         dev_info->tx_queue_offload_capa = sfc_tx_get_queue_offload_caps(sa);
 127
 128         /*
 129          * tx_offload_capa includes both device and queue offloads since
 130          * the latter may be requested on a per device basis which makes
 131          * sense when some offloads are needed to be set on all queues.
 132          */
 133         dev_info->tx_offload_capa = sfc_tx_get_dev_offload_caps(sa) |
 134                                     dev_info->tx_queue_offload_capa;
 135
 136         if (dev_info->tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 137                 txq_offloads_def |= DEV_TX_OFFLOAD_MBUF_FAST_FREE;
 138
 139         dev_info->default_txconf.offloads |= txq_offloads_def;
 140
 141         dev_info->default_txconf.txq_flags = ETH_TXQ_FLAGS_NOXSUMSCTP;
 142         if ((~sa->dp_tx->features & SFC_DP_TX_FEAT_VLAN_INSERT) ||
 143             !encp->enc_hw_tx_insert_vlan_enabled)
 144                 dev_info->default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOVLANOFFL;
 145
 146         if (~sa->dp_tx->features & SFC_DP_TX_FEAT_MULTI_SEG)
 147                 dev_info->default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOMULTSEGS;
 148
 149         if (~sa->dp_tx->features & SFC_DP_TX_FEAT_MULTI_POOL)
 150                 dev_info->default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOMULTMEMP;
 151
 152         if (~sa->dp_tx->features & SFC_DP_TX_FEAT_REFCNT)
 153                 dev_info->default_txconf.txq_flags |= ETH_TXQ_FLAGS_NOREFCOUNT;
 154
 155 #if EFSYS_OPT_RX_SCALE
 156         if (sa->rss_support != EFX_RX_SCALE_UNAVAILABLE) {
 157                 dev_info->reta_size = EFX_RSS_TBL_SIZE;
 158                 dev_info->hash_key_size = EFX_RSS_KEY_SIZE;
 159                 dev_info->flow_type_rss_offloads = SFC_RSS_OFFLOADS;
 160         }
 161 #endif
 162
 163         /* Initialize to hardware limits */
 164         dev_info->rx_desc_lim.nb_max = EFX_RXQ_MAXNDESCS;
 165         dev_info->rx_desc_lim.nb_min = EFX_RXQ_MINNDESCS;
 166         /* The RXQ hardware requires that the descriptor count is a power
 167          * of 2, but rx_desc_lim cannot properly describe that constraint.
 168          */
 169         dev_info->rx_desc_lim.nb_align = EFX_RXQ_MINNDESCS;
 170
 171         /* Initialize to hardware limits */
 172         dev_info->tx_desc_lim.nb_max = sa->txq_max_entries;
 173         dev_info->tx_desc_lim.nb_min = EFX_TXQ_MINNDESCS;
 174         /*
 175          * The TXQ hardware requires that the descriptor count is a power
 176          * of 2, but tx_desc_lim cannot properly describe that constraint
 177          */
 178         dev_info->tx_desc_lim.nb_align = EFX_TXQ_MINNDESCS;
 179
 180         if (sa->dp_rx->get_dev_info != NULL)
 181                 sa->dp_rx->get_dev_info(dev_info);
 182         if (sa->dp_tx->get_dev_info != NULL)
 183                 sa->dp_tx->get_dev_info(dev_info);
 184 }
 185
 186 static const uint32_t *
 187 sfc_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 188 {
 189         struct sfc_adapter *sa = dev->data->dev_private;
 190         const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
 191         uint32_t tunnel_encaps = encp->enc_tunnel_encapsulations_supported;
 192
 193         return sa->dp_rx->supported_ptypes_get(tunnel_encaps);
 194 }
 195
 196 static int
 197 sfc_dev_configure(struct rte_eth_dev *dev)
 198 {
 199         struct rte_eth_dev_data *dev_data = dev->data;
 200         struct sfc_adapter *sa = dev_data->dev_private;
 201         int rc;
 202
 203         sfc_log_init(sa, "entry n_rxq=%u n_txq=%u",
 204                      dev_data->nb_rx_queues, dev_data->nb_tx_queues);
 205
 206         sfc_adapter_lock(sa);
 207         switch (sa->state) {
 208         case SFC_ADAPTER_CONFIGURED:
 209                 /* FALLTHROUGH */
 210         case SFC_ADAPTER_INITIALIZED:
 211                 rc = sfc_configure(sa);
 212                 break;
 213         default:
 214                 sfc_err(sa, "unexpected adapter state %u to configure",
 215                         sa->state);
 216                 rc = EINVAL;
 217                 break;
 218         }
 219         sfc_adapter_unlock(sa);
 220
 221         sfc_log_init(sa, "done %d", rc);
 222         SFC_ASSERT(rc >= 0);
 223         return -rc;
 224 }
 225
 226 static int
 227 sfc_dev_start(struct rte_eth_dev *dev)
 228 {
 229         struct sfc_adapter *sa = dev->data->dev_private;
 230         int rc;
 231
 232         sfc_log_init(sa, "entry");
 233
 234         sfc_adapter_lock(sa);
 235         rc = sfc_start(sa);
 236         sfc_adapter_unlock(sa);
 237
 238         sfc_log_init(sa, "done %d", rc);
 239         SFC_ASSERT(rc >= 0);
 240         return -rc;
 241 }
 242
 243 static int
 244 sfc_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 245 {
 246         struct sfc_adapter *sa = dev->data->dev_private;
 247         struct rte_eth_link current_link;
 248         int ret;
 249
 250         sfc_log_init(sa, "entry");
 251
 252         if (sa->state != SFC_ADAPTER_STARTED) {
 253                 sfc_port_link_mode_to_info(EFX_LINK_UNKNOWN, &current_link);
 254         } else if (wait_to_complete) {
 255                 efx_link_mode_t link_mode;
 256
 257                 if (efx_port_poll(sa->nic, &link_mode) != 0)
 258                         link_mode = EFX_LINK_UNKNOWN;
 259                 sfc_port_link_mode_to_info(link_mode, &current_link);
 260
 261         } else {
 262                 sfc_ev_mgmt_qpoll(sa);
 263                 rte_eth_linkstatus_get(dev, &current_link);
 264         }
 265
 266         ret = rte_eth_linkstatus_set(dev, &current_link);
 267         if (ret == 0)
 268                 sfc_notice(sa, "Link status is %s",
 269                            current_link.link_status ? "UP" : "DOWN");
 270
 271         return ret;
 272 }
 273
 274 static void
 275 sfc_dev_stop(struct rte_eth_dev *dev)
 276 {
 277         struct sfc_adapter *sa = dev->data->dev_private;
 278
 279         sfc_log_init(sa, "entry");
 280
 281         sfc_adapter_lock(sa);
 282         sfc_stop(sa);
 283         sfc_adapter_unlock(sa);
 284
 285         sfc_log_init(sa, "done");
 286 }
 287
 288 static int
 289 sfc_dev_set_link_up(struct rte_eth_dev *dev)
 290 {
 291         struct sfc_adapter *sa = dev->data->dev_private;
 292         int rc;
 293
 294         sfc_log_init(sa, "entry");
 295
 296         sfc_adapter_lock(sa);
 297         rc = sfc_start(sa);
 298         sfc_adapter_unlock(sa);
 299
 300         SFC_ASSERT(rc >= 0);
 301         return -rc;
 302 }
 303
 304 static int
 305 sfc_dev_set_link_down(struct rte_eth_dev *dev)
 306 {
 307         struct sfc_adapter *sa = dev->data->dev_private;
 308
 309         sfc_log_init(sa, "entry");
 310
 311         sfc_adapter_lock(sa);
 312         sfc_stop(sa);
 313         sfc_adapter_unlock(sa);
 314
 315         return 0;
 316 }
 317
 318 static void
 319 sfc_dev_close(struct rte_eth_dev *dev)
 320 {
 321         struct sfc_adapter *sa = dev->data->dev_private;
 322
 323         sfc_log_init(sa, "entry");
 324
 325         sfc_adapter_lock(sa);
 326         switch (sa->state) {
 327         case SFC_ADAPTER_STARTED:
 328                 sfc_stop(sa);
 329                 SFC_ASSERT(sa->state == SFC_ADAPTER_CONFIGURED);
 330                 /* FALLTHROUGH */
 331         case SFC_ADAPTER_CONFIGURED:
 332                 sfc_close(sa);
 333                 SFC_ASSERT(sa->state == SFC_ADAPTER_INITIALIZED);
 334                 /* FALLTHROUGH */
 335         case SFC_ADAPTER_INITIALIZED:
 336                 break;
 337         default:
 338                 sfc_err(sa, "unexpected adapter state %u on close", sa->state);
 339                 break;
 340         }
 341         sfc_adapter_unlock(sa);
 342
 343         sfc_log_init(sa, "done");
 344 }
 345
 346 static void
 347 sfc_dev_filter_set(struct rte_eth_dev *dev, enum sfc_dev_filter_mode mode,
 348                    boolean_t enabled)
 349 {
 350         struct sfc_port *port;
 351         boolean_t *toggle;
 352         struct sfc_adapter *sa = dev->data->dev_private;
 353         boolean_t allmulti = (mode == SFC_DEV_FILTER_MODE_ALLMULTI);
 354         const char *desc = (allmulti) ? "all-multi" : "promiscuous";
 355
 356         sfc_adapter_lock(sa);
 357
 358         port = &sa->port;
 359         toggle = (allmulti) ? (&port->allmulti) : (&port->promisc);
 360
 361         if (*toggle != enabled) {
 362                 *toggle = enabled;
 363
 364                 if (port->isolated) {
 365                         sfc_warn(sa, "isolated mode is active on the port");
 366                         sfc_warn(sa, "the change is to be applied on the next "
 367                                      "start provided that isolated mode is "
 368                                      "disabled prior the next start");
 369                 } else if ((sa->state == SFC_ADAPTER_STARTED) &&
 370                            (sfc_set_rx_mode(sa) != 0)) {
 371                         *toggle = !(enabled);
 372                         sfc_warn(sa, "Failed to %s %s mode",
 373                                  ((enabled) ? "enable" : "disable"), desc);
 374                 }
 375         }
 376
 377         sfc_adapter_unlock(sa);
 378 }
 379
 380 static void
 381 sfc_dev_promisc_enable(struct rte_eth_dev *dev)
 382 {
 383         sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_PROMISC, B_TRUE);
 384 }
 385
 386 static void
 387 sfc_dev_promisc_disable(struct rte_eth_dev *dev)
 388 {
 389         sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_PROMISC, B_FALSE);
 390 }
 391
 392 static void
 393 sfc_dev_allmulti_enable(struct rte_eth_dev *dev)
 394 {
 395         sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_ALLMULTI, B_TRUE);
 396 }
 397
 398 static void
 399 sfc_dev_allmulti_disable(struct rte_eth_dev *dev)
 400 {
 401         sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_ALLMULTI, B_FALSE);
 402 }
 403
 404 static int
 405 sfc_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
 406                    uint16_t nb_rx_desc, unsigned int socket_id,
 407                    const struct rte_eth_rxconf *rx_conf,
 408                    struct rte_mempool *mb_pool)
 409 {
 410         struct sfc_adapter *sa = dev->data->dev_private;
 411         int rc;
 412
 413         sfc_log_init(sa, "RxQ=%u nb_rx_desc=%u socket_id=%u",
 414                      rx_queue_id, nb_rx_desc, socket_id);
 415
 416         sfc_adapter_lock(sa);
 417
 418         rc = sfc_rx_qinit(sa, rx_queue_id, nb_rx_desc, socket_id,
 419                           rx_conf, mb_pool);
 420         if (rc != 0)
 421                 goto fail_rx_qinit;
 422
 423         dev->data->rx_queues[rx_queue_id] = sa->rxq_info[rx_queue_id].rxq->dp;
 424
 425         sfc_adapter_unlock(sa);
 426
 427         return 0;
 428
 429 fail_rx_qinit:
 430         sfc_adapter_unlock(sa);
 431         SFC_ASSERT(rc > 0);
 432         return -rc;
 433 }
 434
 435 static void
 436 sfc_rx_queue_release(void *queue)
 437 {
 438         struct sfc_dp_rxq *dp_rxq = queue;
 439         struct sfc_rxq *rxq;
 440         struct sfc_adapter *sa;
 441         unsigned int sw_index;
 442
 443         if (dp_rxq == NULL)
 444                 return;
 445
 446         rxq = sfc_rxq_by_dp_rxq(dp_rxq);
 447         sa = rxq->evq->sa;
 448         sfc_adapter_lock(sa);
 449
 450         sw_index = sfc_rxq_sw_index(rxq);
 451
 452         sfc_log_init(sa, "RxQ=%u", sw_index);
 453
 454         sa->eth_dev->data->rx_queues[sw_index] = NULL;
 455
 456         sfc_rx_qfini(sa, sw_index);
 457
 458         sfc_adapter_unlock(sa);
 459 }
 460
 461 static int
 462 sfc_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id,
 463                    uint16_t nb_tx_desc, unsigned int socket_id,
 464                    const struct rte_eth_txconf *tx_conf)
 465 {
 466         struct sfc_adapter *sa = dev->data->dev_private;
 467         int rc;
 468
 469         sfc_log_init(sa, "TxQ = %u, nb_tx_desc = %u, socket_id = %u",
 470                      tx_queue_id, nb_tx_desc, socket_id);
 471
 472         sfc_adapter_lock(sa);
 473
 474         rc = sfc_tx_qinit(sa, tx_queue_id, nb_tx_desc, socket_id, tx_conf);
 475         if (rc != 0)
 476                 goto fail_tx_qinit;
 477
 478         dev->data->tx_queues[tx_queue_id] = sa->txq_info[tx_queue_id].txq->dp;
 479
 480         sfc_adapter_unlock(sa);
 481         return 0;
 482
 483 fail_tx_qinit:
 484         sfc_adapter_unlock(sa);
 485         SFC_ASSERT(rc > 0);
 486         return -rc;
 487 }
 488
 489 static void
 490 sfc_tx_queue_release(void *queue)
 491 {
 492         struct sfc_dp_txq *dp_txq = queue;
 493         struct sfc_txq *txq;
 494         unsigned int sw_index;
 495         struct sfc_adapter *sa;
 496
 497         if (dp_txq == NULL)
 498                 return;
 499
 500         txq = sfc_txq_by_dp_txq(dp_txq);
 501         sw_index = sfc_txq_sw_index(txq);
 502
 503         SFC_ASSERT(txq->evq != NULL);
 504         sa = txq->evq->sa;
 505
 506         sfc_log_init(sa, "TxQ = %u", sw_index);
 507
 508         sfc_adapter_lock(sa);
 509
 510         SFC_ASSERT(sw_index < sa->eth_dev->data->nb_tx_queues);
 511         sa->eth_dev->data->tx_queues[sw_index] = NULL;
 512
 513         sfc_tx_qfini(sa, sw_index);
 514
 515         sfc_adapter_unlock(sa);
 516 }
 517
 518 static int
 519 sfc_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 520 {
 521         struct sfc_adapter *sa = dev->data->dev_private;
 522         struct sfc_port *port = &sa->port;
 523         uint64_t *mac_stats;
 524         int ret;
 525
 526         rte_spinlock_lock(&port->mac_stats_lock);
 527
 528         ret = sfc_port_update_mac_stats(sa);
 529         if (ret != 0)
 530                 goto unlock;
 531
 532         mac_stats = port->mac_stats_buf;
 533
 534         if (EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask,
 535                                    EFX_MAC_VADAPTER_RX_UNICAST_PACKETS)) {
 536                 stats->ipackets =
 537                         mac_stats[EFX_MAC_VADAPTER_RX_UNICAST_PACKETS] +
 538                         mac_stats[EFX_MAC_VADAPTER_RX_MULTICAST_PACKETS] +
 539                         mac_stats[EFX_MAC_VADAPTER_RX_BROADCAST_PACKETS];
 540                 stats->opackets =
 541                         mac_stats[EFX_MAC_VADAPTER_TX_UNICAST_PACKETS] +
 542                         mac_stats[EFX_MAC_VADAPTER_TX_MULTICAST_PACKETS] +
 543                         mac_stats[EFX_MAC_VADAPTER_TX_BROADCAST_PACKETS];
 544                 stats->ibytes =
 545                         mac_stats[EFX_MAC_VADAPTER_RX_UNICAST_BYTES] +
 546                         mac_stats[EFX_MAC_VADAPTER_RX_MULTICAST_BYTES] +
 547                         mac_stats[EFX_MAC_VADAPTER_RX_BROADCAST_BYTES];
 548                 stats->obytes =
 549                         mac_stats[EFX_MAC_VADAPTER_TX_UNICAST_BYTES] +
 550                         mac_stats[EFX_MAC_VADAPTER_TX_MULTICAST_BYTES] +
 551                         mac_stats[EFX_MAC_VADAPTER_TX_BROADCAST_BYTES];
 552                 stats->imissed = mac_stats[EFX_MAC_VADAPTER_RX_OVERFLOW];
 553                 stats->ierrors = mac_stats[EFX_MAC_VADAPTER_RX_BAD_PACKETS];
 554                 stats->oerrors = mac_stats[EFX_MAC_VADAPTER_TX_BAD_PACKETS];
 555         } else {
 556                 stats->ipackets = mac_stats[EFX_MAC_RX_PKTS];
 557                 stats->opackets = mac_stats[EFX_MAC_TX_PKTS];
 558                 stats->ibytes = mac_stats[EFX_MAC_RX_OCTETS];
 559                 stats->obytes = mac_stats[EFX_MAC_TX_OCTETS];
 560                 /*
 561                  * Take into account stats which are whenever supported
 562                  * on EF10. If some stat is not supported by current
 563                  * firmware variant or HW revision, it is guaranteed
 564                  * to be zero in mac_stats.
 565                  */
 566                 stats->imissed =
 567                         mac_stats[EFX_MAC_RX_NODESC_DROP_CNT] +
 568                         mac_stats[EFX_MAC_PM_TRUNC_BB_OVERFLOW] +
 569                         mac_stats[EFX_MAC_PM_DISCARD_BB_OVERFLOW] +
 570                         mac_stats[EFX_MAC_PM_TRUNC_VFIFO_FULL] +
 571                         mac_stats[EFX_MAC_PM_DISCARD_VFIFO_FULL] +
 572                         mac_stats[EFX_MAC_PM_TRUNC_QBB] +
 573                         mac_stats[EFX_MAC_PM_DISCARD_QBB] +
 574                         mac_stats[EFX_MAC_PM_DISCARD_MAPPING] +
 575                         mac_stats[EFX_MAC_RXDP_Q_DISABLED_PKTS] +
 576                         mac_stats[EFX_MAC_RXDP_DI_DROPPED_PKTS];
 577                 stats->ierrors =
 578                         mac_stats[EFX_MAC_RX_FCS_ERRORS] +
 579                         mac_stats[EFX_MAC_RX_ALIGN_ERRORS] +
 580                         mac_stats[EFX_MAC_RX_JABBER_PKTS];
 581                 /* no oerrors counters supported on EF10 */
 582         }
 583
 584 unlock:
 585         rte_spinlock_unlock(&port->mac_stats_lock);
 586         SFC_ASSERT(ret >= 0);
 587         return -ret;
 588 }
 589
 590 static void
 591 sfc_stats_reset(struct rte_eth_dev *dev)
 592 {
 593         struct sfc_adapter *sa = dev->data->dev_private;
 594         struct sfc_port *port = &sa->port;
 595         int rc;
 596
 597         if (sa->state != SFC_ADAPTER_STARTED) {
 598                 /*
 599                  * The operation cannot be done if port is not started; it
 600                  * will be scheduled to be done during the next port start
 601                  */
 602                 port->mac_stats_reset_pending = B_TRUE;
 603                 return;
 604         }
 605
 606         rc = sfc_port_reset_mac_stats(sa);
 607         if (rc != 0)
 608                 sfc_err(sa, "failed to reset statistics (rc = %d)", rc);
 609 }
 610
 611 static int
 612 sfc_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
 613                unsigned int xstats_count)
 614 {
 615         struct sfc_adapter *sa = dev->data->dev_private;
 616         struct sfc_port *port = &sa->port;
 617         uint64_t *mac_stats;
 618         int rc;
 619         unsigned int i;
 620         int nstats = 0;
 621
 622         rte_spinlock_lock(&port->mac_stats_lock);
 623
 624         rc = sfc_port_update_mac_stats(sa);
 625         if (rc != 0) {
 626                 SFC_ASSERT(rc > 0);
 627                 nstats = -rc;
 628                 goto unlock;
 629         }
 630
 631         mac_stats = port->mac_stats_buf;
 632
 633         for (i = 0; i < EFX_MAC_NSTATS; ++i) {
 634                 if (EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i)) {
 635                         if (xstats != NULL && nstats < (int)xstats_count) {
 636                                 xstats[nstats].id = nstats;
 637                                 xstats[nstats].value = mac_stats[i];
 638                         }
 639                         nstats++;
 640                 }
 641         }
 642
 643 unlock:
 644         rte_spinlock_unlock(&port->mac_stats_lock);
 645
 646         return nstats;
 647 }
 648
 649 static int
 650 sfc_xstats_get_names(struct rte_eth_dev *dev,
 651                      struct rte_eth_xstat_name *xstats_names,
 652                      unsigned int xstats_count)
 653 {
 654         struct sfc_adapter *sa = dev->data->dev_private;
 655         struct sfc_port *port = &sa->port;
 656         unsigned int i;
 657         unsigned int nstats = 0;
 658
 659         for (i = 0; i < EFX_MAC_NSTATS; ++i) {
 660                 if (EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i)) {
 661                         if (xstats_names != NULL && nstats < xstats_count)
 662                                 strncpy(xstats_names[nstats].name,
 663                                         efx_mac_stat_name(sa->nic, i),
 664                                         sizeof(xstats_names[0].name));
 665                         nstats++;
 666                 }
 667         }
 668
 669         return nstats;
 670 }
 671
 672 static int
 673 sfc_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids,
 674                      uint64_t *values, unsigned int n)
 675 {
 676         struct sfc_adapter *sa = dev->data->dev_private;
 677         struct sfc_port *port = &sa->port;
 678         uint64_t *mac_stats;
 679         unsigned int nb_supported = 0;
 680         unsigned int nb_written = 0;
 681         unsigned int i;
 682         int ret;
 683         int rc;
 684
 685         if (unlikely(values == NULL) ||
 686             unlikely((ids == NULL) && (n < port->mac_stats_nb_supported)))
 687                 return port->mac_stats_nb_supported;
 688
 689         rte_spinlock_lock(&port->mac_stats_lock);
 690
 691         rc = sfc_port_update_mac_stats(sa);
 692         if (rc != 0) {
 693                 SFC_ASSERT(rc > 0);
 694                 ret = -rc;
 695                 goto unlock;
 696         }
 697
 698         mac_stats = port->mac_stats_buf;
 699
 700         for (i = 0; (i < EFX_MAC_NSTATS) && (nb_written < n); ++i) {
 701                 if (!EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i))
 702                         continue;
 703
 704                 if ((ids == NULL) || (ids[nb_written] == nb_supported))
 705                         values[nb_written++] = mac_stats[i];
 706
 707                 ++nb_supported;
 708         }
 709
 710         ret = nb_written;
 711
 712 unlock:
 713         rte_spinlock_unlock(&port->mac_stats_lock);
 714
 715         return ret;
 716 }
 717
 718 static int
 719 sfc_xstats_get_names_by_id(struct rte_eth_dev *dev,
 720                            struct rte_eth_xstat_name *xstats_names,
 721                            const uint64_t *ids, unsigned int size)
 722 {
 723         struct sfc_adapter *sa = dev->data->dev_private;
 724         struct sfc_port *port = &sa->port;
 725         unsigned int nb_supported = 0;
 726         unsigned int nb_written = 0;
 727         unsigned int i;
 728
 729         if (unlikely(xstats_names == NULL) ||
 730             unlikely((ids == NULL) && (size < port->mac_stats_nb_supported)))
 731                 return port->mac_stats_nb_supported;
 732
 733         for (i = 0; (i < EFX_MAC_NSTATS) && (nb_written < size); ++i) {
 734                 if (!EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i))
 735                         continue;
 736
 737                 if ((ids == NULL) || (ids[nb_written] == nb_supported)) {
 738                         char *name = xstats_names[nb_written++].name;
 739
 740                         strncpy(name, efx_mac_stat_name(sa->nic, i),
 741                                 sizeof(xstats_names[0].name));
 742                         name[sizeof(xstats_names[0].name) - 1] = '\0';
 743                 }
 744
 745                 ++nb_supported;
 746         }
 747
 748         return nb_written;
 749 }
 750
 751 static int
 752 sfc_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 753 {
 754         struct sfc_adapter *sa = dev->data->dev_private;
 755         unsigned int wanted_fc, link_fc;
 756
 757         memset(fc_conf, 0, sizeof(*fc_conf));
 758
 759         sfc_adapter_lock(sa);
 760
 761         if (sa->state == SFC_ADAPTER_STARTED)
 762                 efx_mac_fcntl_get(sa->nic, &wanted_fc, &link_fc);
 763         else
 764                 link_fc = sa->port.flow_ctrl;
 765
 766         switch (link_fc) {
 767         case 0:
 768                 fc_conf->mode = RTE_FC_NONE;
 769                 break;
 770         case EFX_FCNTL_RESPOND:
 771                 fc_conf->mode = RTE_FC_RX_PAUSE;
 772                 break;
 773         case EFX_FCNTL_GENERATE:
 774                 fc_conf->mode = RTE_FC_TX_PAUSE;
 775                 break;
 776         case (EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE):
 777                 fc_conf->mode = RTE_FC_FULL;
 778                 break;
 779         default:
 780                 sfc_err(sa, "%s: unexpected flow control value %#x",
 781                         __func__, link_fc);
 782         }
 783
 784         fc_conf->autoneg = sa->port.flow_ctrl_autoneg;
 785
 786         sfc_adapter_unlock(sa);
 787
 788         return 0;
 789 }
 790
 791 static int
 792 sfc_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 793 {
 794         struct sfc_adapter *sa = dev->data->dev_private;
 795         struct sfc_port *port = &sa->port;
 796         unsigned int fcntl;
 797         int rc;
 798
 799         if (fc_conf->high_water != 0 || fc_conf->low_water != 0 ||
 800             fc_conf->pause_time != 0 || fc_conf->send_xon != 0 ||
 801             fc_conf->mac_ctrl_frame_fwd != 0) {
 802                 sfc_err(sa, "unsupported flow control settings specified");
 803                 rc = EINVAL;
 804                 goto fail_inval;
 805         }
 806
 807         switch (fc_conf->mode) {
 808         case RTE_FC_NONE:
 809                 fcntl = 0;
 810                 break;
 811         case RTE_FC_RX_PAUSE:
 812                 fcntl = EFX_FCNTL_RESPOND;
 813                 break;
 814         case RTE_FC_TX_PAUSE:
 815                 fcntl = EFX_FCNTL_GENERATE;
 816                 break;
 817         case RTE_FC_FULL:
 818                 fcntl = EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE;
 819                 break;
 820         default:
 821                 rc = EINVAL;
 822                 goto fail_inval;
 823         }
 824
 825         sfc_adapter_lock(sa);
 826
 827         if (sa->state == SFC_ADAPTER_STARTED) {
 828                 rc = efx_mac_fcntl_set(sa->nic, fcntl, fc_conf->autoneg);
 829                 if (rc != 0)
 830                         goto fail_mac_fcntl_set;
 831         }
 832
 833         port->flow_ctrl = fcntl;
 834         port->flow_ctrl_autoneg = fc_conf->autoneg;
 835
 836         sfc_adapter_unlock(sa);
 837
 838         return 0;
 839
 840 fail_mac_fcntl_set:
 841         sfc_adapter_unlock(sa);
 842 fail_inval:
 843         SFC_ASSERT(rc > 0);
 844         return -rc;
 845 }
 846
 847 static int
 848 sfc_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 849 {
 850         struct sfc_adapter *sa = dev->data->dev_private;
 851         size_t pdu = EFX_MAC_PDU(mtu);
 852         size_t old_pdu;
 853         int rc;
 854
 855         sfc_log_init(sa, "mtu=%u", mtu);
 856
 857         rc = EINVAL;
 858         if (pdu < EFX_MAC_PDU_MIN) {
 859                 sfc_err(sa, "too small MTU %u (PDU size %u less than min %u)",
 860                         (unsigned int)mtu, (unsigned int)pdu,
 861                         EFX_MAC_PDU_MIN);
 862                 goto fail_inval;
 863         }
 864         if (pdu > EFX_MAC_PDU_MAX) {
 865                 sfc_err(sa, "too big MTU %u (PDU size %u greater than max %u)",
 866                         (unsigned int)mtu, (unsigned int)pdu,
 867                         EFX_MAC_PDU_MAX);
 868                 goto fail_inval;
 869         }
 870
 871         sfc_adapter_lock(sa);
 872
 873         if (pdu != sa->port.pdu) {
 874                 if (sa->state == SFC_ADAPTER_STARTED) {
 875                         sfc_stop(sa);
 876
 877                         old_pdu = sa->port.pdu;
 878                         sa->port.pdu = pdu;
 879                         rc = sfc_start(sa);
 880                         if (rc != 0)
 881                                 goto fail_start;
 882                 } else {
 883                         sa->port.pdu = pdu;
 884                 }
 885         }
 886
 887         /*
 888          * The driver does not use it, but other PMDs update jumbo_frame
 889          * flag and max_rx_pkt_len when MTU is set.
 890          */
 891         if (mtu > ETHER_MAX_LEN) {
 892                 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
 893
 894                 rxmode->offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
 895                 rxmode->jumbo_frame = 1;
 896         }
 897
 898         dev->data->dev_conf.rxmode.max_rx_pkt_len = sa->port.pdu;
 899
 900         sfc_adapter_unlock(sa);
 901
 902         sfc_log_init(sa, "done");
 903         return 0;
 904
 905 fail_start:
 906         sa->port.pdu = old_pdu;
 907         if (sfc_start(sa) != 0)
 908                 sfc_err(sa, "cannot start with neither new (%u) nor old (%u) "
 909                         "PDU max size - port is stopped",
 910                         (unsigned int)pdu, (unsigned int)old_pdu);
 911         sfc_adapter_unlock(sa);
 912
 913 fail_inval:
 914         sfc_log_init(sa, "failed %d", rc);
 915         SFC_ASSERT(rc > 0);
 916         return -rc;
 917 }
 918 static void
 919 sfc_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
 920 {
 921         struct sfc_adapter *sa = dev->data->dev_private;
 922         const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
 923         struct sfc_port *port = &sa->port;
 924         int rc;
 925
 926         sfc_adapter_lock(sa);
 927
 928         /*
 929          * Copy the address to the device private data so that
 930          * it could be recalled in the case of adapter restart.
 931          */
 932         ether_addr_copy(mac_addr, &port->default_mac_addr);
 933
 934         if (port->isolated) {
 935                 sfc_err(sa, "isolated mode is active on the port");
 936                 sfc_err(sa, "will not set MAC address");
 937                 goto unlock;
 938         }
 939
 940         if (sa->state != SFC_ADAPTER_STARTED) {
 941                 sfc_notice(sa, "the port is not started");
 942                 sfc_notice(sa, "the new MAC address will be set on port start");
 943
 944                 goto unlock;
 945         }
 946
 947         if (encp->enc_allow_set_mac_with_installed_filters) {
 948                 rc = efx_mac_addr_set(sa->nic, mac_addr->addr_bytes);
 949                 if (rc != 0) {
 950                         sfc_err(sa, "cannot set MAC address (rc = %u)", rc);
 951                         goto unlock;
 952                 }
 953
 954                 /*
 955                  * Changing the MAC address by means of MCDI request
 956                  * has no effect on received traffic, therefore
 957                  * we also need to update unicast filters
 958                  */
 959                 rc = sfc_set_rx_mode(sa);
 960                 if (rc != 0)
 961                         sfc_err(sa, "cannot set filter (rc = %u)", rc);
 962         } else {
 963                 sfc_warn(sa, "cannot set MAC address with filters installed");
 964                 sfc_warn(sa, "adapter will be restarted to pick the new MAC");
 965                 sfc_warn(sa, "(some traffic may be dropped)");
 966
 967                 /*
 968                  * Since setting MAC address with filters installed is not
 969                  * allowed on the adapter, the new MAC address will be set
 970                  * by means of adapter restart. sfc_start() shall retrieve
 971                  * the new address from the device private data and set it.
 972                  */
 973                 sfc_stop(sa);
 974                 rc = sfc_start(sa);
 975                 if (rc != 0)
 976                         sfc_err(sa, "cannot restart adapter (rc = %u)", rc);
 977         }
 978
 979 unlock:
 980         /*
 981          * In the case of failure sa->port->default_mac_addr does not
 982          * need rollback since no error code is returned, and the upper
 983          * API will anyway update the external MAC address storage.
 984          * To be consistent with that new value it is better to keep
 985          * the device private value the same.
 986          */
 987         sfc_adapter_unlock(sa);
 988 }
 989
 990
 991 static int
 992 sfc_set_mc_addr_list(struct rte_eth_dev *dev, struct ether_addr *mc_addr_set,
 993                      uint32_t nb_mc_addr)
 994 {
 995         struct sfc_adapter *sa = dev->data->dev_private;
 996         struct sfc_port *port = &sa->port;
 997         uint8_t *mc_addrs = port->mcast_addrs;
 998         int rc;
 999         unsigned int i;
1000
1001         if (port->isolated) {
1002                 sfc_err(sa, "isolated mode is active on the port");
1003                 sfc_err(sa, "will not set multicast address list");
1004                 return -ENOTSUP;
1005         }
1006
1007         if (mc_addrs == NULL)
1008                 return -ENOBUFS;
1009
1010         if (nb_mc_addr > port->max_mcast_addrs) {
1011                 sfc_err(sa, "too many multicast addresses: %u > %u",
1012                          nb_mc_addr, port->max_mcast_addrs);
1013                 return -EINVAL;
1014         }
1015
1016         for (i = 0; i < nb_mc_addr; ++i) {
1017                 rte_memcpy(mc_addrs, mc_addr_set[i].addr_bytes,
1018                                  EFX_MAC_ADDR_LEN);
1019                 mc_addrs += EFX_MAC_ADDR_LEN;
1020         }
1021
1022         port->nb_mcast_addrs = nb_mc_addr;
1023
1024         if (sa->state != SFC_ADAPTER_STARTED)
1025                 return 0;
1026
1027         rc = efx_mac_multicast_list_set(sa->nic, port->mcast_addrs,
1028                                         port->nb_mcast_addrs);
1029         if (rc != 0)
1030                 sfc_err(sa, "cannot set multicast address list (rc = %u)", rc);
1031
1032         SFC_ASSERT(rc > 0);
1033         return -rc;
1034 }
1035
1036 /*
1037  * The function is used by the secondary process as well. It must not
1038  * use any process-local pointers from the adapter data.
1039  */
1040 static void
1041 sfc_rx_queue_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1042                       struct rte_eth_rxq_info *qinfo)
1043 {
1044         struct sfc_adapter *sa = dev->data->dev_private;
1045         struct sfc_rxq_info *rxq_info;
1046         struct sfc_rxq *rxq;
1047
1048         sfc_adapter_lock(sa);
1049
1050         SFC_ASSERT(rx_queue_id < sa->rxq_count);
1051
1052         rxq_info = &sa->rxq_info[rx_queue_id];
1053         rxq = rxq_info->rxq;
1054         SFC_ASSERT(rxq != NULL);
1055
1056         qinfo->mp = rxq->refill_mb_pool;
1057         qinfo->conf.rx_free_thresh = rxq->refill_threshold;
1058         qinfo->conf.rx_drop_en = 1;
1059         qinfo->conf.rx_deferred_start = rxq_info->deferred_start;
1060         qinfo->conf.offloads = DEV_RX_OFFLOAD_IPV4_CKSUM |
1061                                DEV_RX_OFFLOAD_UDP_CKSUM |
1062                                DEV_RX_OFFLOAD_TCP_CKSUM;
1063         if (rxq_info->type_flags & EFX_RXQ_FLAG_SCATTER) {
1064                 qinfo->conf.offloads |= DEV_RX_OFFLOAD_SCATTER;
1065                 qinfo->scattered_rx = 1;
1066         }
1067         qinfo->nb_desc = rxq_info->entries;
1068
1069         sfc_adapter_unlock(sa);
1070 }
1071
1072 /*
1073  * The function is used by the secondary process as well. It must not
1074  * use any process-local pointers from the adapter data.
1075  */
1076 static void
1077 sfc_tx_queue_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1078                       struct rte_eth_txq_info *qinfo)
1079 {
1080         struct sfc_adapter *sa = dev->data->dev_private;
1081         struct sfc_txq_info *txq_info;
1082
1083         sfc_adapter_lock(sa);
1084
1085         SFC_ASSERT(tx_queue_id < sa->txq_count);
1086
1087         txq_info = &sa->txq_info[tx_queue_id];
1088         SFC_ASSERT(txq_info->txq != NULL);
1089
1090         memset(qinfo, 0, sizeof(*qinfo));
1091
1092         qinfo->conf.txq_flags = txq_info->txq->flags;
1093         qinfo->conf.offloads = txq_info->txq->offloads;
1094         qinfo->conf.tx_free_thresh = txq_info->txq->free_thresh;
1095         qinfo->conf.tx_deferred_start = txq_info->deferred_start;
1096         qinfo->nb_desc = txq_info->entries;
1097
1098         sfc_adapter_unlock(sa);
1099 }
1100
1101 static uint32_t
1102 sfc_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1103 {
1104         struct sfc_adapter *sa = dev->data->dev_private;
1105
1106         sfc_log_init(sa, "RxQ=%u", rx_queue_id);
1107
1108         return sfc_rx_qdesc_npending(sa, rx_queue_id);
1109 }
1110
1111 static int
1112 sfc_rx_descriptor_done(void *queue, uint16_t offset)
1113 {
1114         struct sfc_dp_rxq *dp_rxq = queue;
1115
1116         return sfc_rx_qdesc_done(dp_rxq, offset);
1117 }
1118
1119 static int
1120 sfc_rx_descriptor_status(void *queue, uint16_t offset)
1121 {
1122         struct sfc_dp_rxq *dp_rxq = queue;
1123         struct sfc_rxq *rxq = sfc_rxq_by_dp_rxq(dp_rxq);
1124
1125         return rxq->evq->sa->dp_rx->qdesc_status(dp_rxq, offset);
1126 }
1127
1128 static int
1129 sfc_tx_descriptor_status(void *queue, uint16_t offset)
1130 {
1131         struct sfc_dp_txq *dp_txq = queue;
1132         struct sfc_txq *txq = sfc_txq_by_dp_txq(dp_txq);
1133
1134         return txq->evq->sa->dp_tx->qdesc_status(dp_txq, offset);
1135 }
1136
1137 static int
1138 sfc_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1139 {
1140         struct sfc_adapter *sa = dev->data->dev_private;
1141         int rc;
1142
1143         sfc_log_init(sa, "RxQ=%u", rx_queue_id);
1144
1145         sfc_adapter_lock(sa);
1146
1147         rc = EINVAL;
1148         if (sa->state != SFC_ADAPTER_STARTED)
1149                 goto fail_not_started;
1150
1151         rc = sfc_rx_qstart(sa, rx_queue_id);
1152         if (rc != 0)
1153                 goto fail_rx_qstart;
1154
1155         sa->rxq_info[rx_queue_id].deferred_started = B_TRUE;
1156
1157         sfc_adapter_unlock(sa);
1158
1159         return 0;
1160
1161 fail_rx_qstart:
1162 fail_not_started:
1163         sfc_adapter_unlock(sa);
1164         SFC_ASSERT(rc > 0);
1165         return -rc;
1166 }
1167
1168 static int
1169 sfc_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1170 {
1171         struct sfc_adapter *sa = dev->data->dev_private;
1172
1173         sfc_log_init(sa, "RxQ=%u", rx_queue_id);
1174
1175         sfc_adapter_lock(sa);
1176         sfc_rx_qstop(sa, rx_queue_id);
1177
1178         sa->rxq_info[rx_queue_id].deferred_started = B_FALSE;
1179
1180         sfc_adapter_unlock(sa);
1181
1182         return 0;
1183 }
1184
1185 static int
1186 sfc_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1187 {
1188         struct sfc_adapter *sa = dev->data->dev_private;
1189         int rc;
1190
1191         sfc_log_init(sa, "TxQ = %u", tx_queue_id);
1192
1193         sfc_adapter_lock(sa);
1194
1195         rc = EINVAL;
1196         if (sa->state != SFC_ADAPTER_STARTED)
1197                 goto fail_not_started;
1198
1199         rc = sfc_tx_qstart(sa, tx_queue_id);
1200         if (rc != 0)
1201                 goto fail_tx_qstart;
1202
1203         sa->txq_info[tx_queue_id].deferred_started = B_TRUE;
1204
1205         sfc_adapter_unlock(sa);
1206         return 0;
1207
1208 fail_tx_qstart:
1209
1210 fail_not_started:
1211         sfc_adapter_unlock(sa);
1212         SFC_ASSERT(rc > 0);
1213         return -rc;
1214 }
1215
1216 static int
1217 sfc_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1218 {
1219         struct sfc_adapter *sa = dev->data->dev_private;
1220
1221         sfc_log_init(sa, "TxQ = %u", tx_queue_id);
1222
1223         sfc_adapter_lock(sa);
1224
1225         sfc_tx_qstop(sa, tx_queue_id);
1226
1227         sa->txq_info[tx_queue_id].deferred_started = B_FALSE;
1228
1229         sfc_adapter_unlock(sa);
1230         return 0;
1231 }
1232
1233 static efx_tunnel_protocol_t
1234 sfc_tunnel_rte_type_to_efx_udp_proto(enum rte_eth_tunnel_type rte_type)
1235 {
1236         switch (rte_type) {
1237         case RTE_TUNNEL_TYPE_VXLAN:
1238                 return EFX_TUNNEL_PROTOCOL_VXLAN;
1239         case RTE_TUNNEL_TYPE_GENEVE:
1240                 return EFX_TUNNEL_PROTOCOL_GENEVE;
1241         default:
1242                 return EFX_TUNNEL_NPROTOS;
1243         }
1244 }
1245
1246 enum sfc_udp_tunnel_op_e {
1247         SFC_UDP_TUNNEL_ADD_PORT,
1248         SFC_UDP_TUNNEL_DEL_PORT,
1249 };
1250
1251 static int
1252 sfc_dev_udp_tunnel_op(struct rte_eth_dev *dev,
1253                       struct rte_eth_udp_tunnel *tunnel_udp,
1254                       enum sfc_udp_tunnel_op_e op)
1255 {
1256         struct sfc_adapter *sa = dev->data->dev_private;
1257         efx_tunnel_protocol_t tunnel_proto;
1258         int rc;
1259
1260         sfc_log_init(sa, "%s udp_port=%u prot_type=%u",
1261                      (op == SFC_UDP_TUNNEL_ADD_PORT) ? "add" :
1262                      (op == SFC_UDP_TUNNEL_DEL_PORT) ? "delete" : "unknown",
1263                      tunnel_udp->udp_port, tunnel_udp->prot_type);
1264
1265         tunnel_proto =
1266                 sfc_tunnel_rte_type_to_efx_udp_proto(tunnel_udp->prot_type);
1267         if (tunnel_proto >= EFX_TUNNEL_NPROTOS) {
1268                 rc = ENOTSUP;
1269                 goto fail_bad_proto;
1270         }
1271
1272         sfc_adapter_lock(sa);
1273
1274         switch (op) {
1275         case SFC_UDP_TUNNEL_ADD_PORT:
1276                 rc = efx_tunnel_config_udp_add(sa->nic,
1277                                                tunnel_udp->udp_port,
1278                                                tunnel_proto);
1279                 break;
1280         case SFC_UDP_TUNNEL_DEL_PORT:
1281                 rc = efx_tunnel_config_udp_remove(sa->nic,
1282                                                   tunnel_udp->udp_port,
1283                                                   tunnel_proto);
1284                 break;
1285         default:
1286                 rc = EINVAL;
1287                 goto fail_bad_op;
1288         }
1289
1290         if (rc != 0)
1291                 goto fail_op;
1292
1293         if (sa->state == SFC_ADAPTER_STARTED) {
1294                 rc = efx_tunnel_reconfigure(sa->nic);
1295                 if (rc == EAGAIN) {
1296                         /*
1297                          * Configuration is accepted by FW and MC reboot
1298                          * is initiated to apply the changes. MC reboot
1299                          * will be handled in a usual way (MC reboot
1300                          * event on management event queue and adapter
1301                          * restart).
1302                          */
1303                         rc = 0;
1304                 } else if (rc != 0) {
1305                         goto fail_reconfigure;
1306                 }
1307         }
1308
1309         sfc_adapter_unlock(sa);
1310         return 0;
1311
1312 fail_reconfigure:
1313         /* Remove/restore entry since the change makes the trouble */
1314         switch (op) {
1315         case SFC_UDP_TUNNEL_ADD_PORT:
1316                 (void)efx_tunnel_config_udp_remove(sa->nic,
1317                                                    tunnel_udp->udp_port,
1318                                                    tunnel_proto);
1319                 break;
1320         case SFC_UDP_TUNNEL_DEL_PORT:
1321                 (void)efx_tunnel_config_udp_add(sa->nic,
1322                                                 tunnel_udp->udp_port,
1323                                                 tunnel_proto);
1324                 break;
1325         }
1326
1327 fail_op:
1328 fail_bad_op:
1329         sfc_adapter_unlock(sa);
1330
1331 fail_bad_proto:
1332         SFC_ASSERT(rc > 0);
1333         return -rc;
1334 }
1335
1336 static int
1337 sfc_dev_udp_tunnel_port_add(struct rte_eth_dev *dev,
1338                             struct rte_eth_udp_tunnel *tunnel_udp)
1339 {
1340         return sfc_dev_udp_tunnel_op(dev, tunnel_udp, SFC_UDP_TUNNEL_ADD_PORT);
1341 }
1342
1343 static int
1344 sfc_dev_udp_tunnel_port_del(struct rte_eth_dev *dev,
1345                             struct rte_eth_udp_tunnel *tunnel_udp)
1346 {
1347         return sfc_dev_udp_tunnel_op(dev, tunnel_udp, SFC_UDP_TUNNEL_DEL_PORT);
1348 }
1349
1350 #if EFSYS_OPT_RX_SCALE
1351 static int
1352 sfc_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
1353                           struct rte_eth_rss_conf *rss_conf)
1354 {
1355         struct sfc_adapter *sa = dev->data->dev_private;
1356         struct sfc_port *port = &sa->port;
1357
1358         if ((sa->rss_support != EFX_RX_SCALE_EXCLUSIVE) || port->isolated)
1359                 return -ENOTSUP;
1360
1361         if (sa->rss_channels == 0)
1362                 return -EINVAL;
1363
1364         sfc_adapter_lock(sa);
1365
1366         /*
1367          * Mapping of hash configuration between RTE and EFX is not one-to-one,
1368          * hence, conversion is done here to derive a correct set of ETH_RSS
1369          * flags which corresponds to the active EFX configuration stored
1370          * locally in 'sfc_adapter' and kept up-to-date
1371          */
1372         rss_conf->rss_hf = sfc_efx_to_rte_hash_type(sa->rss_hash_types);
1373         rss_conf->rss_key_len = EFX_RSS_KEY_SIZE;
1374         if (rss_conf->rss_key != NULL)
1375                 rte_memcpy(rss_conf->rss_key, sa->rss_key, EFX_RSS_KEY_SIZE);
1376
1377         sfc_adapter_unlock(sa);
1378
1379         return 0;
1380 }
1381
1382 static int
1383 sfc_dev_rss_hash_update(struct rte_eth_dev *dev,
1384                         struct rte_eth_rss_conf *rss_conf)
1385 {
1386         struct sfc_adapter *sa = dev->data->dev_private;
1387         struct sfc_port *port = &sa->port;
1388         unsigned int efx_hash_types;
1389         int rc = 0;
1390
1391         if (port->isolated)
1392                 return -ENOTSUP;
1393
1394         if (sa->rss_support != EFX_RX_SCALE_EXCLUSIVE) {
1395                 sfc_err(sa, "RSS is not available");
1396                 return -ENOTSUP;
1397         }
1398
1399         if (sa->rss_channels == 0) {
1400                 sfc_err(sa, "RSS is not configured");
1401                 return -EINVAL;
1402         }
1403
1404         if ((rss_conf->rss_key != NULL) &&
1405             (rss_conf->rss_key_len != sizeof(sa->rss_key))) {
1406                 sfc_err(sa, "RSS key size is wrong (should be %lu)",
1407                         sizeof(sa->rss_key));
1408                 return -EINVAL;
1409         }
1410
1411         if ((rss_conf->rss_hf & ~SFC_RSS_OFFLOADS) != 0) {
1412                 sfc_err(sa, "unsupported hash functions requested");
1413                 return -EINVAL;
1414         }
1415
1416         sfc_adapter_lock(sa);
1417
1418         efx_hash_types = sfc_rte_to_efx_hash_type(rss_conf->rss_hf);
1419
1420         rc = efx_rx_scale_mode_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT,
1421                                    EFX_RX_HASHALG_TOEPLITZ,
1422                                    efx_hash_types, B_TRUE);
1423         if (rc != 0)
1424                 goto fail_scale_mode_set;
1425
1426         if (rss_conf->rss_key != NULL) {
1427                 if (sa->state == SFC_ADAPTER_STARTED) {
1428                         rc = efx_rx_scale_key_set(sa->nic,
1429                                                   EFX_RSS_CONTEXT_DEFAULT,
1430                                                   rss_conf->rss_key,
1431                                                   sizeof(sa->rss_key));
1432                         if (rc != 0)
1433                                 goto fail_scale_key_set;
1434                 }
1435
1436                 rte_memcpy(sa->rss_key, rss_conf->rss_key, sizeof(sa->rss_key));
1437         }
1438
1439         sa->rss_hash_types = efx_hash_types;
1440
1441         sfc_adapter_unlock(sa);
1442
1443         return 0;
1444
1445 fail_scale_key_set:
1446         if (efx_rx_scale_mode_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT,
1447                                   EFX_RX_HASHALG_TOEPLITZ,
1448                                   sa->rss_hash_types, B_TRUE) != 0)
1449                 sfc_err(sa, "failed to restore RSS mode");
1450
1451 fail_scale_mode_set:
1452         sfc_adapter_unlock(sa);
1453         return -rc;
1454 }
1455
1456 static int
1457 sfc_dev_rss_reta_query(struct rte_eth_dev *dev,
1458                        struct rte_eth_rss_reta_entry64 *reta_conf,
1459                        uint16_t reta_size)
1460 {
1461         struct sfc_adapter *sa = dev->data->dev_private;
1462         struct sfc_port *port = &sa->port;
1463         int entry;
1464
1465         if ((sa->rss_support != EFX_RX_SCALE_EXCLUSIVE) || port->isolated)
1466                 return -ENOTSUP;
1467
1468         if (sa->rss_channels == 0)
1469                 return -EINVAL;
1470
1471         if (reta_size != EFX_RSS_TBL_SIZE)
1472                 return -EINVAL;
1473
1474         sfc_adapter_lock(sa);
1475
1476         for (entry = 0; entry < reta_size; entry++) {
1477                 int grp = entry / RTE_RETA_GROUP_SIZE;
1478                 int grp_idx = entry % RTE_RETA_GROUP_SIZE;
1479
1480                 if ((reta_conf[grp].mask >> grp_idx) & 1)
1481                         reta_conf[grp].reta[grp_idx] = sa->rss_tbl[entry];
1482         }
1483
1484         sfc_adapter_unlock(sa);
1485
1486         return 0;
1487 }
1488
1489 static int
1490 sfc_dev_rss_reta_update(struct rte_eth_dev *dev,
1491                         struct rte_eth_rss_reta_entry64 *reta_conf,
1492                         uint16_t reta_size)
1493 {
1494         struct sfc_adapter *sa = dev->data->dev_private;
1495         struct sfc_port *port = &sa->port;
1496         unsigned int *rss_tbl_new;
1497         uint16_t entry;
1498         int rc = 0;
1499
1500
1501         if (port->isolated)
1502                 return -ENOTSUP;
1503
1504         if (sa->rss_support != EFX_RX_SCALE_EXCLUSIVE) {
1505                 sfc_err(sa, "RSS is not available");
1506                 return -ENOTSUP;
1507         }
1508
1509         if (sa->rss_channels == 0) {
1510                 sfc_err(sa, "RSS is not configured");
1511                 return -EINVAL;
1512         }
1513
1514         if (reta_size != EFX_RSS_TBL_SIZE) {
1515                 sfc_err(sa, "RETA size is wrong (should be %u)",
1516                         EFX_RSS_TBL_SIZE);
1517                 return -EINVAL;
1518         }
1519
1520         rss_tbl_new = rte_zmalloc("rss_tbl_new", sizeof(sa->rss_tbl), 0);
1521         if (rss_tbl_new == NULL)
1522                 return -ENOMEM;
1523
1524         sfc_adapter_lock(sa);
1525
1526         rte_memcpy(rss_tbl_new, sa->rss_tbl, sizeof(sa->rss_tbl));
1527
1528         for (entry = 0; entry < reta_size; entry++) {
1529                 int grp_idx = entry % RTE_RETA_GROUP_SIZE;
1530                 struct rte_eth_rss_reta_entry64 *grp;
1531
1532                 grp = &reta_conf[entry / RTE_RETA_GROUP_SIZE];
1533
1534                 if (grp->mask & (1ull << grp_idx)) {
1535                         if (grp->reta[grp_idx] >= sa->rss_channels) {
1536                                 rc = EINVAL;
1537                                 goto bad_reta_entry;
1538                         }
1539                         rss_tbl_new[entry] = grp->reta[grp_idx];
1540                 }
1541         }
1542
1543         if (sa->state == SFC_ADAPTER_STARTED) {
1544                 rc = efx_rx_scale_tbl_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT,
1545                                           rss_tbl_new, EFX_RSS_TBL_SIZE);
1546                 if (rc != 0)
1547                         goto fail_scale_tbl_set;
1548         }
1549
1550         rte_memcpy(sa->rss_tbl, rss_tbl_new, sizeof(sa->rss_tbl));
1551
1552 fail_scale_tbl_set:
1553 bad_reta_entry:
1554         sfc_adapter_unlock(sa);
1555
1556         rte_free(rss_tbl_new);
1557
1558         SFC_ASSERT(rc >= 0);
1559         return -rc;
1560 }
1561 #endif
1562
1563 static int
1564 sfc_dev_filter_ctrl(struct rte_eth_dev *dev, enum rte_filter_type filter_type,
1565                     enum rte_filter_op filter_op,
1566                     void *arg)
1567 {
1568         struct sfc_adapter *sa = dev->data->dev_private;
1569         int rc = ENOTSUP;
1570
1571         sfc_log_init(sa, "entry");
1572
1573         switch (filter_type) {
1574         case RTE_ETH_FILTER_NONE:
1575                 sfc_err(sa, "Global filters configuration not supported");
1576                 break;
1577         case RTE_ETH_FILTER_MACVLAN:
1578                 sfc_err(sa, "MACVLAN filters not supported");
1579                 break;
1580         case RTE_ETH_FILTER_ETHERTYPE:
1581                 sfc_err(sa, "EtherType filters not supported");
1582                 break;
1583         case RTE_ETH_FILTER_FLEXIBLE:
1584                 sfc_err(sa, "Flexible filters not supported");
1585                 break;
1586         case RTE_ETH_FILTER_SYN:
1587                 sfc_err(sa, "SYN filters not supported");
1588                 break;
1589         case RTE_ETH_FILTER_NTUPLE:
1590                 sfc_err(sa, "NTUPLE filters not supported");
1591                 break;
1592         case RTE_ETH_FILTER_TUNNEL:
1593                 sfc_err(sa, "Tunnel filters not supported");
1594                 break;
1595         case RTE_ETH_FILTER_FDIR:
1596                 sfc_err(sa, "Flow Director filters not supported");
1597                 break;
1598         case RTE_ETH_FILTER_HASH:
1599                 sfc_err(sa, "Hash filters not supported");
1600                 break;
1601         case RTE_ETH_FILTER_GENERIC:
1602                 if (filter_op != RTE_ETH_FILTER_GET) {
1603                         rc = EINVAL;
1604                 } else {
1605                         *(const void **)arg = &sfc_flow_ops;
1606                         rc = 0;
1607                 }
1608                 break;
1609         default:
1610                 sfc_err(sa, "Unknown filter type %u", filter_type);
1611                 break;
1612         }
1613
1614         sfc_log_init(sa, "exit: %d", -rc);
1615         SFC_ASSERT(rc >= 0);
1616         return -rc;
1617 }
1618
1619 static const struct eth_dev_ops sfc_eth_dev_ops = {
1620         .dev_configure                  = sfc_dev_configure,
1621         .dev_start                      = sfc_dev_start,
1622         .dev_stop                       = sfc_dev_stop,
1623         .dev_set_link_up                = sfc_dev_set_link_up,
1624         .dev_set_link_down              = sfc_dev_set_link_down,
1625         .dev_close                      = sfc_dev_close,
1626         .promiscuous_enable             = sfc_dev_promisc_enable,
1627         .promiscuous_disable            = sfc_dev_promisc_disable,
1628         .allmulticast_enable            = sfc_dev_allmulti_enable,
1629         .allmulticast_disable           = sfc_dev_allmulti_disable,
1630         .link_update                    = sfc_dev_link_update,
1631         .stats_get                      = sfc_stats_get,
1632         .stats_reset                    = sfc_stats_reset,
1633         .xstats_get                     = sfc_xstats_get,
1634         .xstats_reset                   = sfc_stats_reset,
1635         .xstats_get_names               = sfc_xstats_get_names,
1636         .dev_infos_get                  = sfc_dev_infos_get,
1637         .dev_supported_ptypes_get       = sfc_dev_supported_ptypes_get,
1638         .mtu_set                        = sfc_dev_set_mtu,
1639         .rx_queue_start                 = sfc_rx_queue_start,
1640         .rx_queue_stop                  = sfc_rx_queue_stop,
1641         .tx_queue_start                 = sfc_tx_queue_start,
1642         .tx_queue_stop                  = sfc_tx_queue_stop,
1643         .rx_queue_setup                 = sfc_rx_queue_setup,
1644         .rx_queue_release               = sfc_rx_queue_release,
1645         .rx_queue_count                 = sfc_rx_queue_count,
1646         .rx_descriptor_done             = sfc_rx_descriptor_done,
1647         .rx_descriptor_status           = sfc_rx_descriptor_status,
1648         .tx_descriptor_status           = sfc_tx_descriptor_status,
1649         .tx_queue_setup                 = sfc_tx_queue_setup,
1650         .tx_queue_release               = sfc_tx_queue_release,
1651         .flow_ctrl_get                  = sfc_flow_ctrl_get,
1652         .flow_ctrl_set                  = sfc_flow_ctrl_set,
1653         .mac_addr_set                   = sfc_mac_addr_set,
1654         .udp_tunnel_port_add            = sfc_dev_udp_tunnel_port_add,
1655         .udp_tunnel_port_del            = sfc_dev_udp_tunnel_port_del,
1656 #if EFSYS_OPT_RX_SCALE
1657         .reta_update                    = sfc_dev_rss_reta_update,
1658         .reta_query                     = sfc_dev_rss_reta_query,
1659         .rss_hash_update                = sfc_dev_rss_hash_update,
1660         .rss_hash_conf_get              = sfc_dev_rss_hash_conf_get,
1661 #endif
1662         .filter_ctrl                    = sfc_dev_filter_ctrl,
1663         .set_mc_addr_list               = sfc_set_mc_addr_list,
1664         .rxq_info_get                   = sfc_rx_queue_info_get,
1665         .txq_info_get                   = sfc_tx_queue_info_get,
1666         .fw_version_get                 = sfc_fw_version_get,
1667         .xstats_get_by_id               = sfc_xstats_get_by_id,
1668         .xstats_get_names_by_id         = sfc_xstats_get_names_by_id,
1669 };
1670
1671 /**
1672  * Duplicate a string in potentially shared memory required for
1673  * multi-process support.
1674  *
1675  * strdup() allocates from process-local heap/memory.
1676  */
1677 static char *
1678 sfc_strdup(const char *str)
1679 {
1680         size_t size;
1681         char *copy;
1682
1683         if (str == NULL)
1684                 return NULL;
1685
1686         size = strlen(str) + 1;
1687         copy = rte_malloc(__func__, size, 0);
1688         if (copy != NULL)
1689                 rte_memcpy(copy, str, size);
1690
1691         return copy;
1692 }
1693
1694 static int
1695 sfc_eth_dev_set_ops(struct rte_eth_dev *dev)
1696 {
1697         struct sfc_adapter *sa = dev->data->dev_private;
1698         unsigned int avail_caps = 0;
1699         const char *rx_name = NULL;
1700         const char *tx_name = NULL;
1701         int rc;
1702
1703         switch (sa->family) {
1704         case EFX_FAMILY_HUNTINGTON:
1705         case EFX_FAMILY_MEDFORD:
1706         case EFX_FAMILY_MEDFORD2:
1707                 avail_caps |= SFC_DP_HW_FW_CAP_EF10;
1708                 break;
1709         default:
1710                 break;
1711         }
1712
1713         rc = sfc_kvargs_process(sa, SFC_KVARG_RX_DATAPATH,
1714                                 sfc_kvarg_string_handler, &rx_name);
1715         if (rc != 0)
1716                 goto fail_kvarg_rx_datapath;
1717
1718         if (rx_name != NULL) {
1719                 sa->dp_rx = sfc_dp_find_rx_by_name(&sfc_dp_head, rx_name);
1720                 if (sa->dp_rx == NULL) {
1721                         sfc_err(sa, "Rx datapath %s not found", rx_name);
1722                         rc = ENOENT;
1723                         goto fail_dp_rx;
1724                 }
1725                 if (!sfc_dp_match_hw_fw_caps(&sa->dp_rx->dp, avail_caps)) {
1726                         sfc_err(sa,
1727                                 "Insufficient Hw/FW capabilities to use Rx datapath %s",
1728                                 rx_name);
1729                         rc = EINVAL;
1730                         goto fail_dp_rx_caps;
1731                 }
1732         } else {
1733                 sa->dp_rx = sfc_dp_find_rx_by_caps(&sfc_dp_head, avail_caps);
1734                 if (sa->dp_rx == NULL) {
1735                         sfc_err(sa, "Rx datapath by caps %#x not found",
1736                                 avail_caps);
1737                         rc = ENOENT;
1738                         goto fail_dp_rx;
1739                 }
1740         }
1741
1742         sa->dp_rx_name = sfc_strdup(sa->dp_rx->dp.name);
1743         if (sa->dp_rx_name == NULL) {
1744                 rc = ENOMEM;
1745                 goto fail_dp_rx_name;
1746         }
1747
1748         sfc_notice(sa, "use %s Rx datapath", sa->dp_rx_name);
1749
1750         dev->rx_pkt_burst = sa->dp_rx->pkt_burst;
1751
1752         rc = sfc_kvargs_process(sa, SFC_KVARG_TX_DATAPATH,
1753                                 sfc_kvarg_string_handler, &tx_name);
1754         if (rc != 0)
1755                 goto fail_kvarg_tx_datapath;
1756
1757         if (tx_name != NULL) {
1758                 sa->dp_tx = sfc_dp_find_tx_by_name(&sfc_dp_head, tx_name);
1759                 if (sa->dp_tx == NULL) {
1760                         sfc_err(sa, "Tx datapath %s not found", tx_name);
1761                         rc = ENOENT;
1762                         goto fail_dp_tx;
1763                 }
1764                 if (!sfc_dp_match_hw_fw_caps(&sa->dp_tx->dp, avail_caps)) {
1765                         sfc_err(sa,
1766                                 "Insufficient Hw/FW capabilities to use Tx datapath %s",
1767                                 tx_name);
1768                         rc = EINVAL;
1769                         goto fail_dp_tx_caps;
1770                 }
1771         } else {
1772                 sa->dp_tx = sfc_dp_find_tx_by_caps(&sfc_dp_head, avail_caps);
1773                 if (sa->dp_tx == NULL) {
1774                         sfc_err(sa, "Tx datapath by caps %#x not found",
1775                                 avail_caps);
1776                         rc = ENOENT;
1777                         goto fail_dp_tx;
1778                 }
1779         }
1780
1781         sa->dp_tx_name = sfc_strdup(sa->dp_tx->dp.name);
1782         if (sa->dp_tx_name == NULL) {
1783                 rc = ENOMEM;
1784                 goto fail_dp_tx_name;
1785         }
1786
1787         sfc_notice(sa, "use %s Tx datapath", sa->dp_tx_name);
1788
1789         dev->tx_pkt_burst = sa->dp_tx->pkt_burst;
1790
1791         dev->dev_ops = &sfc_eth_dev_ops;
1792
1793         return 0;
1794
1795 fail_dp_tx_name:
1796 fail_dp_tx_caps:
1797         sa->dp_tx = NULL;
1798
1799 fail_dp_tx:
1800 fail_kvarg_tx_datapath:
1801         rte_free(sa->dp_rx_name);
1802         sa->dp_rx_name = NULL;
1803
1804 fail_dp_rx_name:
1805 fail_dp_rx_caps:
1806         sa->dp_rx = NULL;
1807
1808 fail_dp_rx:
1809 fail_kvarg_rx_datapath:
1810         return rc;
1811 }
1812
1813 static void
1814 sfc_eth_dev_clear_ops(struct rte_eth_dev *dev)
1815 {
1816         struct sfc_adapter *sa = dev->data->dev_private;
1817
1818         dev->dev_ops = NULL;
1819         dev->rx_pkt_burst = NULL;
1820         dev->tx_pkt_burst = NULL;
1821
1822         rte_free(sa->dp_tx_name);
1823         sa->dp_tx_name = NULL;
1824         sa->dp_tx = NULL;
1825
1826         rte_free(sa->dp_rx_name);
1827         sa->dp_rx_name = NULL;
1828         sa->dp_rx = NULL;
1829 }
1830
1831 static const struct eth_dev_ops sfc_eth_dev_secondary_ops = {
1832         .rxq_info_get                   = sfc_rx_queue_info_get,
1833         .txq_info_get                   = sfc_tx_queue_info_get,
1834 };
1835
1836 static int
1837 sfc_eth_dev_secondary_set_ops(struct rte_eth_dev *dev)
1838 {
1839         /*
1840          * Device private data has really many process-local pointers.
1841          * Below code should be extremely careful to use data located
1842          * in shared memory only.
1843          */
1844         struct sfc_adapter *sa = dev->data->dev_private;
1845         const struct sfc_dp_rx *dp_rx;
1846         const struct sfc_dp_tx *dp_tx;
1847         int rc;
1848
1849         dp_rx = sfc_dp_find_rx_by_name(&sfc_dp_head, sa->dp_rx_name);
1850         if (dp_rx == NULL) {
1851                 sfc_err(sa, "cannot find %s Rx datapath", sa->dp_tx_name);
1852                 rc = ENOENT;
1853                 goto fail_dp_rx;
1854         }
1855         if (~dp_rx->features & SFC_DP_RX_FEAT_MULTI_PROCESS) {
1856                 sfc_err(sa, "%s Rx datapath does not support multi-process",
1857                         sa->dp_tx_name);
1858                 rc = EINVAL;
1859                 goto fail_dp_rx_multi_process;
1860         }
1861
1862         dp_tx = sfc_dp_find_tx_by_name(&sfc_dp_head, sa->dp_tx_name);
1863         if (dp_tx == NULL) {
1864                 sfc_err(sa, "cannot find %s Tx datapath", sa->dp_tx_name);
1865                 rc = ENOENT;
1866                 goto fail_dp_tx;
1867         }
1868         if (~dp_tx->features & SFC_DP_TX_FEAT_MULTI_PROCESS) {
1869                 sfc_err(sa, "%s Tx datapath does not support multi-process",
1870                         sa->dp_tx_name);
1871                 rc = EINVAL;
1872                 goto fail_dp_tx_multi_process;
1873         }
1874
1875         dev->rx_pkt_burst = dp_rx->pkt_burst;
1876         dev->tx_pkt_burst = dp_tx->pkt_burst;
1877         dev->dev_ops = &sfc_eth_dev_secondary_ops;
1878
1879         return 0;
1880
1881 fail_dp_tx_multi_process:
1882 fail_dp_tx:
1883 fail_dp_rx_multi_process:
1884 fail_dp_rx:
1885         return rc;
1886 }
1887
1888 static void
1889 sfc_eth_dev_secondary_clear_ops(struct rte_eth_dev *dev)
1890 {
1891         dev->dev_ops = NULL;
1892         dev->tx_pkt_burst = NULL;
1893         dev->rx_pkt_burst = NULL;
1894 }
1895
1896 static void
1897 sfc_register_dp(void)
1898 {
1899         /* Register once */
1900         if (TAILQ_EMPTY(&sfc_dp_head)) {
1901                 /* Prefer EF10 datapath */
1902                 sfc_dp_register(&sfc_dp_head, &sfc_ef10_rx.dp);
1903                 sfc_dp_register(&sfc_dp_head, &sfc_efx_rx.dp);
1904
1905                 sfc_dp_register(&sfc_dp_head, &sfc_ef10_tx.dp);
1906                 sfc_dp_register(&sfc_dp_head, &sfc_efx_tx.dp);
1907                 sfc_dp_register(&sfc_dp_head, &sfc_ef10_simple_tx.dp);
1908         }
1909 }
1910
1911 static int
1912 sfc_eth_dev_init(struct rte_eth_dev *dev)
1913 {
1914         struct sfc_adapter *sa = dev->data->dev_private;
1915         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
1916         int rc;
1917         const efx_nic_cfg_t *encp;
1918         const struct ether_addr *from;
1919
1920         sfc_register_dp();
1921
1922         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1923                 return -sfc_eth_dev_secondary_set_ops(dev);
1924
1925         /* Required for logging */
1926         sa->pci_addr = pci_dev->addr;
1927         sa->port_id = dev->data->port_id;
1928
1929         sa->eth_dev = dev;
1930
1931         /* Copy PCI device info to the dev->data */
1932         rte_eth_copy_pci_info(dev, pci_dev);
1933
1934         sa->logtype_main = sfc_register_logtype(sa, SFC_LOGTYPE_MAIN_STR,
1935                                                 RTE_LOG_NOTICE);
1936
1937         rc = sfc_kvargs_parse(sa);
1938         if (rc != 0)
1939                 goto fail_kvargs_parse;
1940
1941         sfc_log_init(sa, "entry");
1942
1943         dev->data->mac_addrs = rte_zmalloc("sfc", ETHER_ADDR_LEN, 0);
1944         if (dev->data->mac_addrs == NULL) {
1945                 rc = ENOMEM;
1946                 goto fail_mac_addrs;
1947         }
1948
1949         sfc_adapter_lock_init(sa);
1950         sfc_adapter_lock(sa);
1951
1952         sfc_log_init(sa, "probing");
1953         rc = sfc_probe(sa);
1954         if (rc != 0)
1955                 goto fail_probe;
1956
1957         sfc_log_init(sa, "set device ops");
1958         rc = sfc_eth_dev_set_ops(dev);
1959         if (rc != 0)
1960                 goto fail_set_ops;
1961
1962         sfc_log_init(sa, "attaching");
1963         rc = sfc_attach(sa);
1964         if (rc != 0)
1965                 goto fail_attach;
1966
1967         encp = efx_nic_cfg_get(sa->nic);
1968
1969         /*
1970          * The arguments are really reverse order in comparison to
1971          * Linux kernel. Copy from NIC config to Ethernet device data.
1972          */
1973         from = (const struct ether_addr *)(encp->enc_mac_addr);
1974         ether_addr_copy(from, &dev->data->mac_addrs[0]);
1975
1976         sfc_adapter_unlock(sa);
1977
1978         sfc_log_init(sa, "done");
1979         return 0;
1980
1981 fail_attach:
1982         sfc_eth_dev_clear_ops(dev);
1983
1984 fail_set_ops:
1985         sfc_unprobe(sa);
1986
1987 fail_probe:
1988         sfc_adapter_unlock(sa);
1989         sfc_adapter_lock_fini(sa);
1990         rte_free(dev->data->mac_addrs);
1991         dev->data->mac_addrs = NULL;
1992
1993 fail_mac_addrs:
1994         sfc_kvargs_cleanup(sa);
1995
1996 fail_kvargs_parse:
1997         sfc_log_init(sa, "failed %d", rc);
1998         SFC_ASSERT(rc > 0);
1999         return -rc;
2000 }
2001
2002 static int
2003 sfc_eth_dev_uninit(struct rte_eth_dev *dev)
2004 {
2005         struct sfc_adapter *sa;
2006
2007         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2008                 sfc_eth_dev_secondary_clear_ops(dev);
2009                 return 0;
2010         }
2011
2012         sa = dev->data->dev_private;
2013         sfc_log_init(sa, "entry");
2014
2015         sfc_adapter_lock(sa);
2016
2017         sfc_eth_dev_clear_ops(dev);
2018
2019         sfc_detach(sa);
2020         sfc_unprobe(sa);
2021
2022         rte_free(dev->data->mac_addrs);
2023         dev->data->mac_addrs = NULL;
2024
2025         sfc_kvargs_cleanup(sa);
2026
2027         sfc_adapter_unlock(sa);
2028         sfc_adapter_lock_fini(sa);
2029
2030         sfc_log_init(sa, "done");
2031
2032         /* Required for logging, so cleanup last */
2033         sa->eth_dev = NULL;
2034         return 0;
2035 }
2036
2037 static const struct rte_pci_id pci_id_sfc_efx_map[] = {
2038         { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_FARMINGDALE) },
2039         { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_FARMINGDALE_VF) },
2040         { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_GREENPORT) },
2041         { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_GREENPORT_VF) },
2042         { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_MEDFORD) },
2043         { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_MEDFORD_VF) },
2044         { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_MEDFORD2) },
2045         { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_MEDFORD2_VF) },
2046         { .vendor_id = 0 /* sentinel */ }
2047 };
2048
2049 static int sfc_eth_dev_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
2050         struct rte_pci_device *pci_dev)
2051 {
2052         return rte_eth_dev_pci_generic_probe(pci_dev,
2053                 sizeof(struct sfc_adapter), sfc_eth_dev_init);
2054 }
2055
2056 static int sfc_eth_dev_pci_remove(struct rte_pci_device *pci_dev)
2057 {
2058         return rte_eth_dev_pci_generic_remove(pci_dev, sfc_eth_dev_uninit);
2059 }
2060
2061 static struct rte_pci_driver sfc_efx_pmd = {
2062         .id_table = pci_id_sfc_efx_map,
2063         .drv_flags =
2064                 RTE_PCI_DRV_INTR_LSC |
2065                 RTE_PCI_DRV_NEED_MAPPING,
2066         .probe = sfc_eth_dev_pci_probe,
2067         .remove = sfc_eth_dev_pci_remove,
2068 };
2069
2070 RTE_PMD_REGISTER_PCI(net_sfc_efx, sfc_efx_pmd);
2071 RTE_PMD_REGISTER_PCI_TABLE(net_sfc_efx, pci_id_sfc_efx_map);
2072 RTE_PMD_REGISTER_KMOD_DEP(net_sfc_efx, "* igb_uio | uio_pci_generic | vfio-pci");
2073 RTE_PMD_REGISTER_PARAM_STRING(net_sfc_efx,
2074         SFC_KVARG_RX_DATAPATH "=" SFC_KVARG_VALUES_RX_DATAPATH " "
2075         SFC_KVARG_TX_DATAPATH "=" SFC_KVARG_VALUES_TX_DATAPATH " "
2076         SFC_KVARG_PERF_PROFILE "=" SFC_KVARG_VALUES_PERF_PROFILE " "
2077         SFC_KVARG_FW_VARIANT "=" SFC_KVARG_VALUES_FW_VARIANT " "
2078         SFC_KVARG_STATS_UPDATE_PERIOD_MS "=<long>");
2079
2080 RTE_INIT(sfc_driver_register_logtype);
2081 static void
2082 sfc_driver_register_logtype(void)
2083 {
2084         int ret;
2085
2086         ret = rte_log_register_type_and_pick_level(SFC_LOGTYPE_PREFIX "driver",
2087                                                    RTE_LOG_NOTICE);
2088         sfc_logtype_driver = (ret < 0) ? RTE_LOGTYPE_PMD : ret;
2089 }