drivers/net/sfc/sfc_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (c) 2016-2018 Solarflare Communications Inc.
   4  * All rights reserved.
   5  *
   6  * This software was jointly developed between OKTET Labs (under contract
   7  * for Solarflare) and Solarflare Communications, Inc.
   8  */
   9
  10 #include <rte_dev.h>
  11 #include <rte_ethdev_driver.h>
  12 #include <rte_ethdev_pci.h>
  13 #include <rte_pci.h>
  14 #include <rte_bus_pci.h>
  15 #include <rte_errno.h>
  16 #include <rte_string_fns.h>
  17 #include <rte_ether.h>
  18
  19 #include "efx.h"
  20
  21 #include "sfc.h"
  22 #include "sfc_debug.h"
  23 #include "sfc_log.h"
  24 #include "sfc_kvargs.h"
  25 #include "sfc_ev.h"
  26 #include "sfc_rx.h"
  27 #include "sfc_tx.h"
  28 #include "sfc_flow.h"
  29 #include "sfc_dp.h"
  30 #include "sfc_dp_rx.h"
  31
  32 uint32_t sfc_logtype_driver;
  33
  34 static struct sfc_dp_list sfc_dp_head =
  35         TAILQ_HEAD_INITIALIZER(sfc_dp_head);
  36
  37
  38 static void sfc_eth_dev_clear_ops(struct rte_eth_dev *dev);
  39
  40
  41 static int
  42 sfc_fw_version_get(struct rte_eth_dev *dev, char *fw_version, size_t fw_size)
  43 {
  44         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
  45         efx_nic_fw_info_t enfi;
  46         int ret;
  47         int rc;
  48
  49         /*
  50          * Return value of the callback is likely supposed to be
  51          * equal to or greater than 0, nevertheless, if an error
  52          * occurs, it will be desirable to pass it to the caller
  53          */
  54         if ((fw_version == NULL) || (fw_size == 0))
  55                 return -EINVAL;
  56
  57         rc = efx_nic_get_fw_version(sa->nic, &enfi);
  58         if (rc != 0)
  59                 return -rc;
  60
  61         ret = snprintf(fw_version, fw_size,
  62                        "%" PRIu16 ".%" PRIu16 ".%" PRIu16 ".%" PRIu16,
  63                        enfi.enfi_mc_fw_version[0], enfi.enfi_mc_fw_version[1],
  64                        enfi.enfi_mc_fw_version[2], enfi.enfi_mc_fw_version[3]);
  65         if (ret < 0)
  66                 return ret;
  67
  68         if (enfi.enfi_dpcpu_fw_ids_valid) {
  69                 size_t dpcpu_fw_ids_offset = MIN(fw_size - 1, (size_t)ret);
  70                 int ret_extra;
  71
  72                 ret_extra = snprintf(fw_version + dpcpu_fw_ids_offset,
  73                                      fw_size - dpcpu_fw_ids_offset,
  74                                      " rx%" PRIx16 " tx%" PRIx16,
  75                                      enfi.enfi_rx_dpcpu_fw_id,
  76                                      enfi.enfi_tx_dpcpu_fw_id);
  77                 if (ret_extra < 0)
  78                         return ret_extra;
  79
  80                 ret += ret_extra;
  81         }
  82
  83         if (fw_size < (size_t)(++ret))
  84                 return ret;
  85         else
  86                 return 0;
  87 }
  88
  89 static void
  90 sfc_dev_infos_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
  91 {
  92         const struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(dev);
  93         struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
  94         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
  95         struct sfc_rss *rss = &sas->rss;
  96         uint64_t txq_offloads_def = 0;
  97
  98         sfc_log_init(sa, "entry");
  99
 100         dev_info->min_mtu = RTE_ETHER_MIN_MTU;
 101         dev_info->max_mtu = EFX_MAC_SDU_MAX;
 102
 103         dev_info->max_rx_pktlen = EFX_MAC_PDU_MAX;
 104
 105         /* Autonegotiation may be disabled */
 106         dev_info->speed_capa = ETH_LINK_SPEED_FIXED;
 107         if (sa->port.phy_adv_cap_mask & (1u << EFX_PHY_CAP_1000FDX))
 108                 dev_info->speed_capa |= ETH_LINK_SPEED_1G;
 109         if (sa->port.phy_adv_cap_mask & (1u << EFX_PHY_CAP_10000FDX))
 110                 dev_info->speed_capa |= ETH_LINK_SPEED_10G;
 111         if (sa->port.phy_adv_cap_mask & (1u << EFX_PHY_CAP_25000FDX))
 112                 dev_info->speed_capa |= ETH_LINK_SPEED_25G;
 113         if (sa->port.phy_adv_cap_mask & (1u << EFX_PHY_CAP_40000FDX))
 114                 dev_info->speed_capa |= ETH_LINK_SPEED_40G;
 115         if (sa->port.phy_adv_cap_mask & (1u << EFX_PHY_CAP_50000FDX))
 116                 dev_info->speed_capa |= ETH_LINK_SPEED_50G;
 117         if (sa->port.phy_adv_cap_mask & (1u << EFX_PHY_CAP_100000FDX))
 118                 dev_info->speed_capa |= ETH_LINK_SPEED_100G;
 119
 120         dev_info->max_rx_queues = sa->rxq_max;
 121         dev_info->max_tx_queues = sa->txq_max;
 122
 123         /* By default packets are dropped if no descriptors are available */
 124         dev_info->default_rxconf.rx_drop_en = 1;
 125
 126         dev_info->rx_queue_offload_capa = sfc_rx_get_queue_offload_caps(sa);
 127
 128         /*
 129          * rx_offload_capa includes both device and queue offloads since
 130          * the latter may be requested on a per device basis which makes
 131          * sense when some offloads are needed to be set on all queues.
 132          */
 133         dev_info->rx_offload_capa = sfc_rx_get_dev_offload_caps(sa) |
 134                                     dev_info->rx_queue_offload_capa;
 135
 136         dev_info->tx_queue_offload_capa = sfc_tx_get_queue_offload_caps(sa);
 137
 138         /*
 139          * tx_offload_capa includes both device and queue offloads since
 140          * the latter may be requested on a per device basis which makes
 141          * sense when some offloads are needed to be set on all queues.
 142          */
 143         dev_info->tx_offload_capa = sfc_tx_get_dev_offload_caps(sa) |
 144                                     dev_info->tx_queue_offload_capa;
 145
 146         if (dev_info->tx_offload_capa & DEV_TX_OFFLOAD_MBUF_FAST_FREE)
 147                 txq_offloads_def |= DEV_TX_OFFLOAD_MBUF_FAST_FREE;
 148
 149         dev_info->default_txconf.offloads |= txq_offloads_def;
 150
 151         if (rss->context_type != EFX_RX_SCALE_UNAVAILABLE) {
 152                 uint64_t rte_hf = 0;
 153                 unsigned int i;
 154
 155                 for (i = 0; i < rss->hf_map_nb_entries; ++i)
 156                         rte_hf |= rss->hf_map[i].rte;
 157
 158                 dev_info->reta_size = EFX_RSS_TBL_SIZE;
 159                 dev_info->hash_key_size = EFX_RSS_KEY_SIZE;
 160                 dev_info->flow_type_rss_offloads = rte_hf;
 161         }
 162
 163         /* Initialize to hardware limits */
 164         dev_info->rx_desc_lim.nb_max = sa->rxq_max_entries;
 165         dev_info->rx_desc_lim.nb_min = sa->rxq_min_entries;
 166         /* The RXQ hardware requires that the descriptor count is a power
 167          * of 2, but rx_desc_lim cannot properly describe that constraint.
 168          */
 169         dev_info->rx_desc_lim.nb_align = sa->rxq_min_entries;
 170
 171         /* Initialize to hardware limits */
 172         dev_info->tx_desc_lim.nb_max = sa->txq_max_entries;
 173         dev_info->tx_desc_lim.nb_min = sa->txq_min_entries;
 174         /*
 175          * The TXQ hardware requires that the descriptor count is a power
 176          * of 2, but tx_desc_lim cannot properly describe that constraint
 177          */
 178         dev_info->tx_desc_lim.nb_align = sa->txq_min_entries;
 179
 180         if (sap->dp_rx->get_dev_info != NULL)
 181                 sap->dp_rx->get_dev_info(dev_info);
 182         if (sap->dp_tx->get_dev_info != NULL)
 183                 sap->dp_tx->get_dev_info(dev_info);
 184
 185         dev_info->dev_capa = RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP |
 186                              RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP;
 187 }
 188
 189 static const uint32_t *
 190 sfc_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 191 {
 192         const struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(dev);
 193
 194         return sap->dp_rx->supported_ptypes_get(sap->shared->tunnel_encaps);
 195 }
 196
 197 static int
 198 sfc_dev_configure(struct rte_eth_dev *dev)
 199 {
 200         struct rte_eth_dev_data *dev_data = dev->data;
 201         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
 202         int rc;
 203
 204         sfc_log_init(sa, "entry n_rxq=%u n_txq=%u",
 205                      dev_data->nb_rx_queues, dev_data->nb_tx_queues);
 206
 207         sfc_adapter_lock(sa);
 208         switch (sa->state) {
 209         case SFC_ADAPTER_CONFIGURED:
 210                 /* FALLTHROUGH */
 211         case SFC_ADAPTER_INITIALIZED:
 212                 rc = sfc_configure(sa);
 213                 break;
 214         default:
 215                 sfc_err(sa, "unexpected adapter state %u to configure",
 216                         sa->state);
 217                 rc = EINVAL;
 218                 break;
 219         }
 220         sfc_adapter_unlock(sa);
 221
 222         sfc_log_init(sa, "done %d", rc);
 223         SFC_ASSERT(rc >= 0);
 224         return -rc;
 225 }
 226
 227 static int
 228 sfc_dev_start(struct rte_eth_dev *dev)
 229 {
 230         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
 231         int rc;
 232
 233         sfc_log_init(sa, "entry");
 234
 235         sfc_adapter_lock(sa);
 236         rc = sfc_start(sa);
 237         sfc_adapter_unlock(sa);
 238
 239         sfc_log_init(sa, "done %d", rc);
 240         SFC_ASSERT(rc >= 0);
 241         return -rc;
 242 }
 243
 244 static int
 245 sfc_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 246 {
 247         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
 248         struct rte_eth_link current_link;
 249         int ret;
 250
 251         sfc_log_init(sa, "entry");
 252
 253         if (sa->state != SFC_ADAPTER_STARTED) {
 254                 sfc_port_link_mode_to_info(EFX_LINK_UNKNOWN, &current_link);
 255         } else if (wait_to_complete) {
 256                 efx_link_mode_t link_mode;
 257
 258                 if (efx_port_poll(sa->nic, &link_mode) != 0)
 259                         link_mode = EFX_LINK_UNKNOWN;
 260                 sfc_port_link_mode_to_info(link_mode, &current_link);
 261
 262         } else {
 263                 sfc_ev_mgmt_qpoll(sa);
 264                 rte_eth_linkstatus_get(dev, &current_link);
 265         }
 266
 267         ret = rte_eth_linkstatus_set(dev, &current_link);
 268         if (ret == 0)
 269                 sfc_notice(sa, "Link status is %s",
 270                            current_link.link_status ? "UP" : "DOWN");
 271
 272         return ret;
 273 }
 274
 275 static void
 276 sfc_dev_stop(struct rte_eth_dev *dev)
 277 {
 278         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
 279
 280         sfc_log_init(sa, "entry");
 281
 282         sfc_adapter_lock(sa);
 283         sfc_stop(sa);
 284         sfc_adapter_unlock(sa);
 285
 286         sfc_log_init(sa, "done");
 287 }
 288
 289 static int
 290 sfc_dev_set_link_up(struct rte_eth_dev *dev)
 291 {
 292         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
 293         int rc;
 294
 295         sfc_log_init(sa, "entry");
 296
 297         sfc_adapter_lock(sa);
 298         rc = sfc_start(sa);
 299         sfc_adapter_unlock(sa);
 300
 301         SFC_ASSERT(rc >= 0);
 302         return -rc;
 303 }
 304
 305 static int
 306 sfc_dev_set_link_down(struct rte_eth_dev *dev)
 307 {
 308         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
 309
 310         sfc_log_init(sa, "entry");
 311
 312         sfc_adapter_lock(sa);
 313         sfc_stop(sa);
 314         sfc_adapter_unlock(sa);
 315
 316         return 0;
 317 }
 318
 319 static void
 320 sfc_dev_close(struct rte_eth_dev *dev)
 321 {
 322         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
 323
 324         sfc_log_init(sa, "entry");
 325
 326         sfc_adapter_lock(sa);
 327         switch (sa->state) {
 328         case SFC_ADAPTER_STARTED:
 329                 sfc_stop(sa);
 330                 SFC_ASSERT(sa->state == SFC_ADAPTER_CONFIGURED);
 331                 /* FALLTHROUGH */
 332         case SFC_ADAPTER_CONFIGURED:
 333                 sfc_close(sa);
 334                 SFC_ASSERT(sa->state == SFC_ADAPTER_INITIALIZED);
 335                 /* FALLTHROUGH */
 336         case SFC_ADAPTER_INITIALIZED:
 337                 break;
 338         default:
 339                 sfc_err(sa, "unexpected adapter state %u on close", sa->state);
 340                 break;
 341         }
 342
 343         /*
 344          * Cleanup all resources in accordance with RTE_ETH_DEV_CLOSE_REMOVE.
 345          * Rollback primary process sfc_eth_dev_init() below.
 346          */
 347
 348         sfc_eth_dev_clear_ops(dev);
 349
 350         sfc_detach(sa);
 351         sfc_unprobe(sa);
 352
 353         sfc_kvargs_cleanup(sa);
 354
 355         sfc_adapter_unlock(sa);
 356         sfc_adapter_lock_fini(sa);
 357
 358         sfc_log_init(sa, "done");
 359
 360         /* Required for logging, so cleanup last */
 361         sa->eth_dev = NULL;
 362
 363         dev->process_private = NULL;
 364         free(sa);
 365 }
 366
 367 static void
 368 sfc_dev_filter_set(struct rte_eth_dev *dev, enum sfc_dev_filter_mode mode,
 369                    boolean_t enabled)
 370 {
 371         struct sfc_port *port;
 372         boolean_t *toggle;
 373         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
 374         boolean_t allmulti = (mode == SFC_DEV_FILTER_MODE_ALLMULTI);
 375         const char *desc = (allmulti) ? "all-multi" : "promiscuous";
 376
 377         sfc_adapter_lock(sa);
 378
 379         port = &sa->port;
 380         toggle = (allmulti) ? (&port->allmulti) : (&port->promisc);
 381
 382         if (*toggle != enabled) {
 383                 *toggle = enabled;
 384
 385                 if (sfc_sa2shared(sa)->isolated) {
 386                         sfc_warn(sa, "isolated mode is active on the port");
 387                         sfc_warn(sa, "the change is to be applied on the next "
 388                                      "start provided that isolated mode is "
 389                                      "disabled prior the next start");
 390                 } else if ((sa->state == SFC_ADAPTER_STARTED) &&
 391                            (sfc_set_rx_mode(sa) != 0)) {
 392                         *toggle = !(enabled);
 393                         sfc_warn(sa, "Failed to %s %s mode",
 394                                  ((enabled) ? "enable" : "disable"), desc);
 395                 }
 396         }
 397
 398         sfc_adapter_unlock(sa);
 399 }
 400
 401 static void
 402 sfc_dev_promisc_enable(struct rte_eth_dev *dev)
 403 {
 404         sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_PROMISC, B_TRUE);
 405 }
 406
 407 static void
 408 sfc_dev_promisc_disable(struct rte_eth_dev *dev)
 409 {
 410         sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_PROMISC, B_FALSE);
 411 }
 412
 413 static void
 414 sfc_dev_allmulti_enable(struct rte_eth_dev *dev)
 415 {
 416         sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_ALLMULTI, B_TRUE);
 417 }
 418
 419 static void
 420 sfc_dev_allmulti_disable(struct rte_eth_dev *dev)
 421 {
 422         sfc_dev_filter_set(dev, SFC_DEV_FILTER_MODE_ALLMULTI, B_FALSE);
 423 }
 424
 425 static int
 426 sfc_rx_queue_setup(struct rte_eth_dev *dev, uint16_t rx_queue_id,
 427                    uint16_t nb_rx_desc, unsigned int socket_id,
 428                    const struct rte_eth_rxconf *rx_conf,
 429                    struct rte_mempool *mb_pool)
 430 {
 431         struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
 432         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
 433         int rc;
 434
 435         sfc_log_init(sa, "RxQ=%u nb_rx_desc=%u socket_id=%u",
 436                      rx_queue_id, nb_rx_desc, socket_id);
 437
 438         sfc_adapter_lock(sa);
 439
 440         rc = sfc_rx_qinit(sa, rx_queue_id, nb_rx_desc, socket_id,
 441                           rx_conf, mb_pool);
 442         if (rc != 0)
 443                 goto fail_rx_qinit;
 444
 445         dev->data->rx_queues[rx_queue_id] = sas->rxq_info[rx_queue_id].dp;
 446
 447         sfc_adapter_unlock(sa);
 448
 449         return 0;
 450
 451 fail_rx_qinit:
 452         sfc_adapter_unlock(sa);
 453         SFC_ASSERT(rc > 0);
 454         return -rc;
 455 }
 456
 457 static void
 458 sfc_rx_queue_release(void *queue)
 459 {
 460         struct sfc_dp_rxq *dp_rxq = queue;
 461         struct sfc_rxq *rxq;
 462         struct sfc_adapter *sa;
 463         unsigned int sw_index;
 464
 465         if (dp_rxq == NULL)
 466                 return;
 467
 468         rxq = sfc_rxq_by_dp_rxq(dp_rxq);
 469         sa = rxq->evq->sa;
 470         sfc_adapter_lock(sa);
 471
 472         sw_index = dp_rxq->dpq.queue_id;
 473
 474         sfc_log_init(sa, "RxQ=%u", sw_index);
 475
 476         sfc_rx_qfini(sa, sw_index);
 477
 478         sfc_adapter_unlock(sa);
 479 }
 480
 481 static int
 482 sfc_tx_queue_setup(struct rte_eth_dev *dev, uint16_t tx_queue_id,
 483                    uint16_t nb_tx_desc, unsigned int socket_id,
 484                    const struct rte_eth_txconf *tx_conf)
 485 {
 486         struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
 487         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
 488         int rc;
 489
 490         sfc_log_init(sa, "TxQ = %u, nb_tx_desc = %u, socket_id = %u",
 491                      tx_queue_id, nb_tx_desc, socket_id);
 492
 493         sfc_adapter_lock(sa);
 494
 495         rc = sfc_tx_qinit(sa, tx_queue_id, nb_tx_desc, socket_id, tx_conf);
 496         if (rc != 0)
 497                 goto fail_tx_qinit;
 498
 499         dev->data->tx_queues[tx_queue_id] = sas->txq_info[tx_queue_id].dp;
 500
 501         sfc_adapter_unlock(sa);
 502         return 0;
 503
 504 fail_tx_qinit:
 505         sfc_adapter_unlock(sa);
 506         SFC_ASSERT(rc > 0);
 507         return -rc;
 508 }
 509
 510 static void
 511 sfc_tx_queue_release(void *queue)
 512 {
 513         struct sfc_dp_txq *dp_txq = queue;
 514         struct sfc_txq *txq;
 515         unsigned int sw_index;
 516         struct sfc_adapter *sa;
 517
 518         if (dp_txq == NULL)
 519                 return;
 520
 521         txq = sfc_txq_by_dp_txq(dp_txq);
 522         sw_index = dp_txq->dpq.queue_id;
 523
 524         SFC_ASSERT(txq->evq != NULL);
 525         sa = txq->evq->sa;
 526
 527         sfc_log_init(sa, "TxQ = %u", sw_index);
 528
 529         sfc_adapter_lock(sa);
 530
 531         sfc_tx_qfini(sa, sw_index);
 532
 533         sfc_adapter_unlock(sa);
 534 }
 535
 536 /*
 537  * Some statistics are computed as A - B where A and B each increase
 538  * monotonically with some hardware counter(s) and the counters are read
 539  * asynchronously.
 540  *
 541  * If packet X is counted in A, but not counted in B yet, computed value is
 542  * greater than real.
 543  *
 544  * If packet X is not counted in A at the moment of reading the counter,
 545  * but counted in B at the moment of reading the counter, computed value
 546  * is less than real.
 547  *
 548  * However, counter which grows backward is worse evil than slightly wrong
 549  * value. So, let's try to guarantee that it never happens except may be
 550  * the case when the MAC stats are zeroed as a result of a NIC reset.
 551  */
 552 static void
 553 sfc_update_diff_stat(uint64_t *stat, uint64_t newval)
 554 {
 555         if ((int64_t)(newval - *stat) > 0 || newval == 0)
 556                 *stat = newval;
 557 }
 558
 559 static int
 560 sfc_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 561 {
 562         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
 563         struct sfc_port *port = &sa->port;
 564         uint64_t *mac_stats;
 565         int ret;
 566
 567         rte_spinlock_lock(&port->mac_stats_lock);
 568
 569         ret = sfc_port_update_mac_stats(sa);
 570         if (ret != 0)
 571                 goto unlock;
 572
 573         mac_stats = port->mac_stats_buf;
 574
 575         if (EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask,
 576                                    EFX_MAC_VADAPTER_RX_UNICAST_PACKETS)) {
 577                 stats->ipackets =
 578                         mac_stats[EFX_MAC_VADAPTER_RX_UNICAST_PACKETS] +
 579                         mac_stats[EFX_MAC_VADAPTER_RX_MULTICAST_PACKETS] +
 580                         mac_stats[EFX_MAC_VADAPTER_RX_BROADCAST_PACKETS];
 581                 stats->opackets =
 582                         mac_stats[EFX_MAC_VADAPTER_TX_UNICAST_PACKETS] +
 583                         mac_stats[EFX_MAC_VADAPTER_TX_MULTICAST_PACKETS] +
 584                         mac_stats[EFX_MAC_VADAPTER_TX_BROADCAST_PACKETS];
 585                 stats->ibytes =
 586                         mac_stats[EFX_MAC_VADAPTER_RX_UNICAST_BYTES] +
 587                         mac_stats[EFX_MAC_VADAPTER_RX_MULTICAST_BYTES] +
 588                         mac_stats[EFX_MAC_VADAPTER_RX_BROADCAST_BYTES];
 589                 stats->obytes =
 590                         mac_stats[EFX_MAC_VADAPTER_TX_UNICAST_BYTES] +
 591                         mac_stats[EFX_MAC_VADAPTER_TX_MULTICAST_BYTES] +
 592                         mac_stats[EFX_MAC_VADAPTER_TX_BROADCAST_BYTES];
 593                 stats->imissed = mac_stats[EFX_MAC_VADAPTER_RX_BAD_PACKETS];
 594                 stats->oerrors = mac_stats[EFX_MAC_VADAPTER_TX_BAD_PACKETS];
 595         } else {
 596                 stats->opackets = mac_stats[EFX_MAC_TX_PKTS];
 597                 stats->ibytes = mac_stats[EFX_MAC_RX_OCTETS];
 598                 stats->obytes = mac_stats[EFX_MAC_TX_OCTETS];
 599                 /*
 600                  * Take into account stats which are whenever supported
 601                  * on EF10. If some stat is not supported by current
 602                  * firmware variant or HW revision, it is guaranteed
 603                  * to be zero in mac_stats.
 604                  */
 605                 stats->imissed =
 606                         mac_stats[EFX_MAC_RX_NODESC_DROP_CNT] +
 607                         mac_stats[EFX_MAC_PM_TRUNC_BB_OVERFLOW] +
 608                         mac_stats[EFX_MAC_PM_DISCARD_BB_OVERFLOW] +
 609                         mac_stats[EFX_MAC_PM_TRUNC_VFIFO_FULL] +
 610                         mac_stats[EFX_MAC_PM_DISCARD_VFIFO_FULL] +
 611                         mac_stats[EFX_MAC_PM_TRUNC_QBB] +
 612                         mac_stats[EFX_MAC_PM_DISCARD_QBB] +
 613                         mac_stats[EFX_MAC_PM_DISCARD_MAPPING] +
 614                         mac_stats[EFX_MAC_RXDP_Q_DISABLED_PKTS] +
 615                         mac_stats[EFX_MAC_RXDP_DI_DROPPED_PKTS];
 616                 stats->ierrors =
 617                         mac_stats[EFX_MAC_RX_FCS_ERRORS] +
 618                         mac_stats[EFX_MAC_RX_ALIGN_ERRORS] +
 619                         mac_stats[EFX_MAC_RX_JABBER_PKTS];
 620                 /* no oerrors counters supported on EF10 */
 621
 622                 /* Exclude missed, errors and pauses from Rx packets */
 623                 sfc_update_diff_stat(&port->ipackets,
 624                         mac_stats[EFX_MAC_RX_PKTS] -
 625                         mac_stats[EFX_MAC_RX_PAUSE_PKTS] -
 626                         stats->imissed - stats->ierrors);
 627                 stats->ipackets = port->ipackets;
 628         }
 629
 630 unlock:
 631         rte_spinlock_unlock(&port->mac_stats_lock);
 632         SFC_ASSERT(ret >= 0);
 633         return -ret;
 634 }
 635
 636 static void
 637 sfc_stats_reset(struct rte_eth_dev *dev)
 638 {
 639         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
 640         struct sfc_port *port = &sa->port;
 641         int rc;
 642
 643         if (sa->state != SFC_ADAPTER_STARTED) {
 644                 /*
 645                  * The operation cannot be done if port is not started; it
 646                  * will be scheduled to be done during the next port start
 647                  */
 648                 port->mac_stats_reset_pending = B_TRUE;
 649                 return;
 650         }
 651
 652         rc = sfc_port_reset_mac_stats(sa);
 653         if (rc != 0)
 654                 sfc_err(sa, "failed to reset statistics (rc = %d)", rc);
 655 }
 656
 657 static int
 658 sfc_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
 659                unsigned int xstats_count)
 660 {
 661         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
 662         struct sfc_port *port = &sa->port;
 663         uint64_t *mac_stats;
 664         int rc;
 665         unsigned int i;
 666         int nstats = 0;
 667
 668         rte_spinlock_lock(&port->mac_stats_lock);
 669
 670         rc = sfc_port_update_mac_stats(sa);
 671         if (rc != 0) {
 672                 SFC_ASSERT(rc > 0);
 673                 nstats = -rc;
 674                 goto unlock;
 675         }
 676
 677         mac_stats = port->mac_stats_buf;
 678
 679         for (i = 0; i < EFX_MAC_NSTATS; ++i) {
 680                 if (EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i)) {
 681                         if (xstats != NULL && nstats < (int)xstats_count) {
 682                                 xstats[nstats].id = nstats;
 683                                 xstats[nstats].value = mac_stats[i];
 684                         }
 685                         nstats++;
 686                 }
 687         }
 688
 689 unlock:
 690         rte_spinlock_unlock(&port->mac_stats_lock);
 691
 692         return nstats;
 693 }
 694
 695 static int
 696 sfc_xstats_get_names(struct rte_eth_dev *dev,
 697                      struct rte_eth_xstat_name *xstats_names,
 698                      unsigned int xstats_count)
 699 {
 700         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
 701         struct sfc_port *port = &sa->port;
 702         unsigned int i;
 703         unsigned int nstats = 0;
 704
 705         for (i = 0; i < EFX_MAC_NSTATS; ++i) {
 706                 if (EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i)) {
 707                         if (xstats_names != NULL && nstats < xstats_count)
 708                                 strlcpy(xstats_names[nstats].name,
 709                                         efx_mac_stat_name(sa->nic, i),
 710                                         sizeof(xstats_names[0].name));
 711                         nstats++;
 712                 }
 713         }
 714
 715         return nstats;
 716 }
 717
 718 static int
 719 sfc_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids,
 720                      uint64_t *values, unsigned int n)
 721 {
 722         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
 723         struct sfc_port *port = &sa->port;
 724         uint64_t *mac_stats;
 725         unsigned int nb_supported = 0;
 726         unsigned int nb_written = 0;
 727         unsigned int i;
 728         int ret;
 729         int rc;
 730
 731         if (unlikely(values == NULL) ||
 732             unlikely((ids == NULL) && (n < port->mac_stats_nb_supported)))
 733                 return port->mac_stats_nb_supported;
 734
 735         rte_spinlock_lock(&port->mac_stats_lock);
 736
 737         rc = sfc_port_update_mac_stats(sa);
 738         if (rc != 0) {
 739                 SFC_ASSERT(rc > 0);
 740                 ret = -rc;
 741                 goto unlock;
 742         }
 743
 744         mac_stats = port->mac_stats_buf;
 745
 746         for (i = 0; (i < EFX_MAC_NSTATS) && (nb_written < n); ++i) {
 747                 if (!EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i))
 748                         continue;
 749
 750                 if ((ids == NULL) || (ids[nb_written] == nb_supported))
 751                         values[nb_written++] = mac_stats[i];
 752
 753                 ++nb_supported;
 754         }
 755
 756         ret = nb_written;
 757
 758 unlock:
 759         rte_spinlock_unlock(&port->mac_stats_lock);
 760
 761         return ret;
 762 }
 763
 764 static int
 765 sfc_xstats_get_names_by_id(struct rte_eth_dev *dev,
 766                            struct rte_eth_xstat_name *xstats_names,
 767                            const uint64_t *ids, unsigned int size)
 768 {
 769         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
 770         struct sfc_port *port = &sa->port;
 771         unsigned int nb_supported = 0;
 772         unsigned int nb_written = 0;
 773         unsigned int i;
 774
 775         if (unlikely(xstats_names == NULL) ||
 776             unlikely((ids == NULL) && (size < port->mac_stats_nb_supported)))
 777                 return port->mac_stats_nb_supported;
 778
 779         for (i = 0; (i < EFX_MAC_NSTATS) && (nb_written < size); ++i) {
 780                 if (!EFX_MAC_STAT_SUPPORTED(port->mac_stats_mask, i))
 781                         continue;
 782
 783                 if ((ids == NULL) || (ids[nb_written] == nb_supported)) {
 784                         char *name = xstats_names[nb_written++].name;
 785
 786                         strlcpy(name, efx_mac_stat_name(sa->nic, i),
 787                                 sizeof(xstats_names[0].name));
 788                 }
 789
 790                 ++nb_supported;
 791         }
 792
 793         return nb_written;
 794 }
 795
 796 static int
 797 sfc_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 798 {
 799         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
 800         unsigned int wanted_fc, link_fc;
 801
 802         memset(fc_conf, 0, sizeof(*fc_conf));
 803
 804         sfc_adapter_lock(sa);
 805
 806         if (sa->state == SFC_ADAPTER_STARTED)
 807                 efx_mac_fcntl_get(sa->nic, &wanted_fc, &link_fc);
 808         else
 809                 link_fc = sa->port.flow_ctrl;
 810
 811         switch (link_fc) {
 812         case 0:
 813                 fc_conf->mode = RTE_FC_NONE;
 814                 break;
 815         case EFX_FCNTL_RESPOND:
 816                 fc_conf->mode = RTE_FC_RX_PAUSE;
 817                 break;
 818         case EFX_FCNTL_GENERATE:
 819                 fc_conf->mode = RTE_FC_TX_PAUSE;
 820                 break;
 821         case (EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE):
 822                 fc_conf->mode = RTE_FC_FULL;
 823                 break;
 824         default:
 825                 sfc_err(sa, "%s: unexpected flow control value %#x",
 826                         __func__, link_fc);
 827         }
 828
 829         fc_conf->autoneg = sa->port.flow_ctrl_autoneg;
 830
 831         sfc_adapter_unlock(sa);
 832
 833         return 0;
 834 }
 835
 836 static int
 837 sfc_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
 838 {
 839         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
 840         struct sfc_port *port = &sa->port;
 841         unsigned int fcntl;
 842         int rc;
 843
 844         if (fc_conf->high_water != 0 || fc_conf->low_water != 0 ||
 845             fc_conf->pause_time != 0 || fc_conf->send_xon != 0 ||
 846             fc_conf->mac_ctrl_frame_fwd != 0) {
 847                 sfc_err(sa, "unsupported flow control settings specified");
 848                 rc = EINVAL;
 849                 goto fail_inval;
 850         }
 851
 852         switch (fc_conf->mode) {
 853         case RTE_FC_NONE:
 854                 fcntl = 0;
 855                 break;
 856         case RTE_FC_RX_PAUSE:
 857                 fcntl = EFX_FCNTL_RESPOND;
 858                 break;
 859         case RTE_FC_TX_PAUSE:
 860                 fcntl = EFX_FCNTL_GENERATE;
 861                 break;
 862         case RTE_FC_FULL:
 863                 fcntl = EFX_FCNTL_RESPOND | EFX_FCNTL_GENERATE;
 864                 break;
 865         default:
 866                 rc = EINVAL;
 867                 goto fail_inval;
 868         }
 869
 870         sfc_adapter_lock(sa);
 871
 872         if (sa->state == SFC_ADAPTER_STARTED) {
 873                 rc = efx_mac_fcntl_set(sa->nic, fcntl, fc_conf->autoneg);
 874                 if (rc != 0)
 875                         goto fail_mac_fcntl_set;
 876         }
 877
 878         port->flow_ctrl = fcntl;
 879         port->flow_ctrl_autoneg = fc_conf->autoneg;
 880
 881         sfc_adapter_unlock(sa);
 882
 883         return 0;
 884
 885 fail_mac_fcntl_set:
 886         sfc_adapter_unlock(sa);
 887 fail_inval:
 888         SFC_ASSERT(rc > 0);
 889         return -rc;
 890 }
 891
 892 static int
 893 sfc_check_scatter_on_all_rx_queues(struct sfc_adapter *sa, size_t pdu)
 894 {
 895         struct sfc_adapter_shared * const sas = sfc_sa2shared(sa);
 896         const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
 897         boolean_t scatter_enabled;
 898         const char *error;
 899         unsigned int i;
 900
 901         for (i = 0; i < sas->rxq_count; i++) {
 902                 if ((sas->rxq_info[i].state & SFC_RXQ_INITIALIZED) == 0)
 903                         continue;
 904
 905                 scatter_enabled = (sas->rxq_info[i].type_flags &
 906                                    EFX_RXQ_FLAG_SCATTER);
 907
 908                 if (!sfc_rx_check_scatter(pdu, sa->rxq_ctrl[i].buf_size,
 909                                           encp->enc_rx_prefix_size,
 910                                           scatter_enabled, &error)) {
 911                         sfc_err(sa, "MTU check for RxQ %u failed: %s", i,
 912                                 error);
 913                         return EINVAL;
 914                 }
 915         }
 916
 917         return 0;
 918 }
 919
 920 static int
 921 sfc_dev_set_mtu(struct rte_eth_dev *dev, uint16_t mtu)
 922 {
 923         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
 924         size_t pdu = EFX_MAC_PDU(mtu);
 925         size_t old_pdu;
 926         int rc;
 927
 928         sfc_log_init(sa, "mtu=%u", mtu);
 929
 930         rc = EINVAL;
 931         if (pdu < EFX_MAC_PDU_MIN) {
 932                 sfc_err(sa, "too small MTU %u (PDU size %u less than min %u)",
 933                         (unsigned int)mtu, (unsigned int)pdu,
 934                         EFX_MAC_PDU_MIN);
 935                 goto fail_inval;
 936         }
 937         if (pdu > EFX_MAC_PDU_MAX) {
 938                 sfc_err(sa, "too big MTU %u (PDU size %u greater than max %u)",
 939                         (unsigned int)mtu, (unsigned int)pdu,
 940                         (unsigned int)EFX_MAC_PDU_MAX);
 941                 goto fail_inval;
 942         }
 943
 944         sfc_adapter_lock(sa);
 945
 946         rc = sfc_check_scatter_on_all_rx_queues(sa, pdu);
 947         if (rc != 0)
 948                 goto fail_check_scatter;
 949
 950         if (pdu != sa->port.pdu) {
 951                 if (sa->state == SFC_ADAPTER_STARTED) {
 952                         sfc_stop(sa);
 953
 954                         old_pdu = sa->port.pdu;
 955                         sa->port.pdu = pdu;
 956                         rc = sfc_start(sa);
 957                         if (rc != 0)
 958                                 goto fail_start;
 959                 } else {
 960                         sa->port.pdu = pdu;
 961                 }
 962         }
 963
 964         /*
 965          * The driver does not use it, but other PMDs update jumbo frame
 966          * flag and max_rx_pkt_len when MTU is set.
 967          */
 968         if (mtu > RTE_ETHER_MAX_LEN) {
 969                 struct rte_eth_rxmode *rxmode = &dev->data->dev_conf.rxmode;
 970                 rxmode->offloads |= DEV_RX_OFFLOAD_JUMBO_FRAME;
 971         }
 972
 973         dev->data->dev_conf.rxmode.max_rx_pkt_len = sa->port.pdu;
 974
 975         sfc_adapter_unlock(sa);
 976
 977         sfc_log_init(sa, "done");
 978         return 0;
 979
 980 fail_start:
 981         sa->port.pdu = old_pdu;
 982         if (sfc_start(sa) != 0)
 983                 sfc_err(sa, "cannot start with neither new (%u) nor old (%u) "
 984                         "PDU max size - port is stopped",
 985                         (unsigned int)pdu, (unsigned int)old_pdu);
 986
 987 fail_check_scatter:
 988         sfc_adapter_unlock(sa);
 989
 990 fail_inval:
 991         sfc_log_init(sa, "failed %d", rc);
 992         SFC_ASSERT(rc > 0);
 993         return -rc;
 994 }
 995 static int
 996 sfc_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr)
 997 {
 998         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
 999         const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
1000         struct sfc_port *port = &sa->port;
1001         struct rte_ether_addr *old_addr = &dev->data->mac_addrs[0];
1002         int rc = 0;
1003
1004         sfc_adapter_lock(sa);
1005
1006         /*
1007          * Copy the address to the device private data so that
1008          * it could be recalled in the case of adapter restart.
1009          */
1010         rte_ether_addr_copy(mac_addr, &port->default_mac_addr);
1011
1012         /*
1013          * Neither of the two following checks can return
1014          * an error. The new MAC address is preserved in
1015          * the device private data and can be activated
1016          * on the next port start if the user prevents
1017          * isolated mode from being enabled.
1018          */
1019         if (sfc_sa2shared(sa)->isolated) {
1020                 sfc_warn(sa, "isolated mode is active on the port");
1021                 sfc_warn(sa, "will not set MAC address");
1022                 goto unlock;
1023         }
1024
1025         if (sa->state != SFC_ADAPTER_STARTED) {
1026                 sfc_notice(sa, "the port is not started");
1027                 sfc_notice(sa, "the new MAC address will be set on port start");
1028
1029                 goto unlock;
1030         }
1031
1032         if (encp->enc_allow_set_mac_with_installed_filters) {
1033                 rc = efx_mac_addr_set(sa->nic, mac_addr->addr_bytes);
1034                 if (rc != 0) {
1035                         sfc_err(sa, "cannot set MAC address (rc = %u)", rc);
1036                         goto unlock;
1037                 }
1038
1039                 /*
1040                  * Changing the MAC address by means of MCDI request
1041                  * has no effect on received traffic, therefore
1042                  * we also need to update unicast filters
1043                  */
1044                 rc = sfc_set_rx_mode(sa);
1045                 if (rc != 0) {
1046                         sfc_err(sa, "cannot set filter (rc = %u)", rc);
1047                         /* Rollback the old address */
1048                         (void)efx_mac_addr_set(sa->nic, old_addr->addr_bytes);
1049                         (void)sfc_set_rx_mode(sa);
1050                 }
1051         } else {
1052                 sfc_warn(sa, "cannot set MAC address with filters installed");
1053                 sfc_warn(sa, "adapter will be restarted to pick the new MAC");
1054                 sfc_warn(sa, "(some traffic may be dropped)");
1055
1056                 /*
1057                  * Since setting MAC address with filters installed is not
1058                  * allowed on the adapter, the new MAC address will be set
1059                  * by means of adapter restart. sfc_start() shall retrieve
1060                  * the new address from the device private data and set it.
1061                  */
1062                 sfc_stop(sa);
1063                 rc = sfc_start(sa);
1064                 if (rc != 0)
1065                         sfc_err(sa, "cannot restart adapter (rc = %u)", rc);
1066         }
1067
1068 unlock:
1069         if (rc != 0)
1070                 rte_ether_addr_copy(old_addr, &port->default_mac_addr);
1071
1072         sfc_adapter_unlock(sa);
1073
1074         SFC_ASSERT(rc >= 0);
1075         return -rc;
1076 }
1077
1078
1079 static int
1080 sfc_set_mc_addr_list(struct rte_eth_dev *dev,
1081                 struct rte_ether_addr *mc_addr_set, uint32_t nb_mc_addr)
1082 {
1083         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
1084         struct sfc_port *port = &sa->port;
1085         uint8_t *mc_addrs = port->mcast_addrs;
1086         int rc;
1087         unsigned int i;
1088
1089         if (sfc_sa2shared(sa)->isolated) {
1090                 sfc_err(sa, "isolated mode is active on the port");
1091                 sfc_err(sa, "will not set multicast address list");
1092                 return -ENOTSUP;
1093         }
1094
1095         if (mc_addrs == NULL)
1096                 return -ENOBUFS;
1097
1098         if (nb_mc_addr > port->max_mcast_addrs) {
1099                 sfc_err(sa, "too many multicast addresses: %u > %u",
1100                          nb_mc_addr, port->max_mcast_addrs);
1101                 return -EINVAL;
1102         }
1103
1104         for (i = 0; i < nb_mc_addr; ++i) {
1105                 rte_memcpy(mc_addrs, mc_addr_set[i].addr_bytes,
1106                                  EFX_MAC_ADDR_LEN);
1107                 mc_addrs += EFX_MAC_ADDR_LEN;
1108         }
1109
1110         port->nb_mcast_addrs = nb_mc_addr;
1111
1112         if (sa->state != SFC_ADAPTER_STARTED)
1113                 return 0;
1114
1115         rc = efx_mac_multicast_list_set(sa->nic, port->mcast_addrs,
1116                                         port->nb_mcast_addrs);
1117         if (rc != 0)
1118                 sfc_err(sa, "cannot set multicast address list (rc = %u)", rc);
1119
1120         SFC_ASSERT(rc >= 0);
1121         return -rc;
1122 }
1123
1124 /*
1125  * The function is used by the secondary process as well. It must not
1126  * use any process-local pointers from the adapter data.
1127  */
1128 static void
1129 sfc_rx_queue_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1130                       struct rte_eth_rxq_info *qinfo)
1131 {
1132         struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
1133         struct sfc_rxq_info *rxq_info;
1134
1135         SFC_ASSERT(rx_queue_id < sas->rxq_count);
1136
1137         rxq_info = &sas->rxq_info[rx_queue_id];
1138
1139         qinfo->mp = rxq_info->refill_mb_pool;
1140         qinfo->conf.rx_free_thresh = rxq_info->refill_threshold;
1141         qinfo->conf.rx_drop_en = 1;
1142         qinfo->conf.rx_deferred_start = rxq_info->deferred_start;
1143         qinfo->conf.offloads = dev->data->dev_conf.rxmode.offloads;
1144         if (rxq_info->type_flags & EFX_RXQ_FLAG_SCATTER) {
1145                 qinfo->conf.offloads |= DEV_RX_OFFLOAD_SCATTER;
1146                 qinfo->scattered_rx = 1;
1147         }
1148         qinfo->nb_desc = rxq_info->entries;
1149 }
1150
1151 /*
1152  * The function is used by the secondary process as well. It must not
1153  * use any process-local pointers from the adapter data.
1154  */
1155 static void
1156 sfc_tx_queue_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1157                       struct rte_eth_txq_info *qinfo)
1158 {
1159         struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
1160         struct sfc_txq_info *txq_info;
1161
1162         SFC_ASSERT(tx_queue_id < sas->txq_count);
1163
1164         txq_info = &sas->txq_info[tx_queue_id];
1165
1166         memset(qinfo, 0, sizeof(*qinfo));
1167
1168         qinfo->conf.offloads = txq_info->offloads;
1169         qinfo->conf.tx_free_thresh = txq_info->free_thresh;
1170         qinfo->conf.tx_deferred_start = txq_info->deferred_start;
1171         qinfo->nb_desc = txq_info->entries;
1172 }
1173
1174 /*
1175  * The function is used by the secondary process as well. It must not
1176  * use any process-local pointers from the adapter data.
1177  */
1178 static uint32_t
1179 sfc_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1180 {
1181         const struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(dev);
1182         struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
1183         struct sfc_rxq_info *rxq_info;
1184
1185         SFC_ASSERT(rx_queue_id < sas->rxq_count);
1186         rxq_info = &sas->rxq_info[rx_queue_id];
1187
1188         if ((rxq_info->state & SFC_RXQ_STARTED) == 0)
1189                 return 0;
1190
1191         return sap->dp_rx->qdesc_npending(rxq_info->dp);
1192 }
1193
1194 /*
1195  * The function is used by the secondary process as well. It must not
1196  * use any process-local pointers from the adapter data.
1197  */
1198 static int
1199 sfc_rx_descriptor_done(void *queue, uint16_t offset)
1200 {
1201         struct sfc_dp_rxq *dp_rxq = queue;
1202         const struct sfc_dp_rx *dp_rx;
1203
1204         dp_rx = sfc_dp_rx_by_dp_rxq(dp_rxq);
1205
1206         return offset < dp_rx->qdesc_npending(dp_rxq);
1207 }
1208
1209 /*
1210  * The function is used by the secondary process as well. It must not
1211  * use any process-local pointers from the adapter data.
1212  */
1213 static int
1214 sfc_rx_descriptor_status(void *queue, uint16_t offset)
1215 {
1216         struct sfc_dp_rxq *dp_rxq = queue;
1217         const struct sfc_dp_rx *dp_rx;
1218
1219         dp_rx = sfc_dp_rx_by_dp_rxq(dp_rxq);
1220
1221         return dp_rx->qdesc_status(dp_rxq, offset);
1222 }
1223
1224 /*
1225  * The function is used by the secondary process as well. It must not
1226  * use any process-local pointers from the adapter data.
1227  */
1228 static int
1229 sfc_tx_descriptor_status(void *queue, uint16_t offset)
1230 {
1231         struct sfc_dp_txq *dp_txq = queue;
1232         const struct sfc_dp_tx *dp_tx;
1233
1234         dp_tx = sfc_dp_tx_by_dp_txq(dp_txq);
1235
1236         return dp_tx->qdesc_status(dp_txq, offset);
1237 }
1238
1239 static int
1240 sfc_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1241 {
1242         struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
1243         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
1244         int rc;
1245
1246         sfc_log_init(sa, "RxQ=%u", rx_queue_id);
1247
1248         sfc_adapter_lock(sa);
1249
1250         rc = EINVAL;
1251         if (sa->state != SFC_ADAPTER_STARTED)
1252                 goto fail_not_started;
1253
1254         if (sas->rxq_info[rx_queue_id].state != SFC_RXQ_INITIALIZED)
1255                 goto fail_not_setup;
1256
1257         rc = sfc_rx_qstart(sa, rx_queue_id);
1258         if (rc != 0)
1259                 goto fail_rx_qstart;
1260
1261         sas->rxq_info[rx_queue_id].deferred_started = B_TRUE;
1262
1263         sfc_adapter_unlock(sa);
1264
1265         return 0;
1266
1267 fail_rx_qstart:
1268 fail_not_setup:
1269 fail_not_started:
1270         sfc_adapter_unlock(sa);
1271         SFC_ASSERT(rc > 0);
1272         return -rc;
1273 }
1274
1275 static int
1276 sfc_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1277 {
1278         struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
1279         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
1280
1281         sfc_log_init(sa, "RxQ=%u", rx_queue_id);
1282
1283         sfc_adapter_lock(sa);
1284         sfc_rx_qstop(sa, rx_queue_id);
1285
1286         sas->rxq_info[rx_queue_id].deferred_started = B_FALSE;
1287
1288         sfc_adapter_unlock(sa);
1289
1290         return 0;
1291 }
1292
1293 static int
1294 sfc_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1295 {
1296         struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
1297         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
1298         int rc;
1299
1300         sfc_log_init(sa, "TxQ = %u", tx_queue_id);
1301
1302         sfc_adapter_lock(sa);
1303
1304         rc = EINVAL;
1305         if (sa->state != SFC_ADAPTER_STARTED)
1306                 goto fail_not_started;
1307
1308         if (sas->txq_info[tx_queue_id].state != SFC_TXQ_INITIALIZED)
1309                 goto fail_not_setup;
1310
1311         rc = sfc_tx_qstart(sa, tx_queue_id);
1312         if (rc != 0)
1313                 goto fail_tx_qstart;
1314
1315         sas->txq_info[tx_queue_id].deferred_started = B_TRUE;
1316
1317         sfc_adapter_unlock(sa);
1318         return 0;
1319
1320 fail_tx_qstart:
1321
1322 fail_not_setup:
1323 fail_not_started:
1324         sfc_adapter_unlock(sa);
1325         SFC_ASSERT(rc > 0);
1326         return -rc;
1327 }
1328
1329 static int
1330 sfc_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
1331 {
1332         struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
1333         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
1334
1335         sfc_log_init(sa, "TxQ = %u", tx_queue_id);
1336
1337         sfc_adapter_lock(sa);
1338
1339         sfc_tx_qstop(sa, tx_queue_id);
1340
1341         sas->txq_info[tx_queue_id].deferred_started = B_FALSE;
1342
1343         sfc_adapter_unlock(sa);
1344         return 0;
1345 }
1346
1347 static efx_tunnel_protocol_t
1348 sfc_tunnel_rte_type_to_efx_udp_proto(enum rte_eth_tunnel_type rte_type)
1349 {
1350         switch (rte_type) {
1351         case RTE_TUNNEL_TYPE_VXLAN:
1352                 return EFX_TUNNEL_PROTOCOL_VXLAN;
1353         case RTE_TUNNEL_TYPE_GENEVE:
1354                 return EFX_TUNNEL_PROTOCOL_GENEVE;
1355         default:
1356                 return EFX_TUNNEL_NPROTOS;
1357         }
1358 }
1359
1360 enum sfc_udp_tunnel_op_e {
1361         SFC_UDP_TUNNEL_ADD_PORT,
1362         SFC_UDP_TUNNEL_DEL_PORT,
1363 };
1364
1365 static int
1366 sfc_dev_udp_tunnel_op(struct rte_eth_dev *dev,
1367                       struct rte_eth_udp_tunnel *tunnel_udp,
1368                       enum sfc_udp_tunnel_op_e op)
1369 {
1370         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
1371         efx_tunnel_protocol_t tunnel_proto;
1372         int rc;
1373
1374         sfc_log_init(sa, "%s udp_port=%u prot_type=%u",
1375                      (op == SFC_UDP_TUNNEL_ADD_PORT) ? "add" :
1376                      (op == SFC_UDP_TUNNEL_DEL_PORT) ? "delete" : "unknown",
1377                      tunnel_udp->udp_port, tunnel_udp->prot_type);
1378
1379         tunnel_proto =
1380                 sfc_tunnel_rte_type_to_efx_udp_proto(tunnel_udp->prot_type);
1381         if (tunnel_proto >= EFX_TUNNEL_NPROTOS) {
1382                 rc = ENOTSUP;
1383                 goto fail_bad_proto;
1384         }
1385
1386         sfc_adapter_lock(sa);
1387
1388         switch (op) {
1389         case SFC_UDP_TUNNEL_ADD_PORT:
1390                 rc = efx_tunnel_config_udp_add(sa->nic,
1391                                                tunnel_udp->udp_port,
1392                                                tunnel_proto);
1393                 break;
1394         case SFC_UDP_TUNNEL_DEL_PORT:
1395                 rc = efx_tunnel_config_udp_remove(sa->nic,
1396                                                   tunnel_udp->udp_port,
1397                                                   tunnel_proto);
1398                 break;
1399         default:
1400                 rc = EINVAL;
1401                 goto fail_bad_op;
1402         }
1403
1404         if (rc != 0)
1405                 goto fail_op;
1406
1407         if (sa->state == SFC_ADAPTER_STARTED) {
1408                 rc = efx_tunnel_reconfigure(sa->nic);
1409                 if (rc == EAGAIN) {
1410                         /*
1411                          * Configuration is accepted by FW and MC reboot
1412                          * is initiated to apply the changes. MC reboot
1413                          * will be handled in a usual way (MC reboot
1414                          * event on management event queue and adapter
1415                          * restart).
1416                          */
1417                         rc = 0;
1418                 } else if (rc != 0) {
1419                         goto fail_reconfigure;
1420                 }
1421         }
1422
1423         sfc_adapter_unlock(sa);
1424         return 0;
1425
1426 fail_reconfigure:
1427         /* Remove/restore entry since the change makes the trouble */
1428         switch (op) {
1429         case SFC_UDP_TUNNEL_ADD_PORT:
1430                 (void)efx_tunnel_config_udp_remove(sa->nic,
1431                                                    tunnel_udp->udp_port,
1432                                                    tunnel_proto);
1433                 break;
1434         case SFC_UDP_TUNNEL_DEL_PORT:
1435                 (void)efx_tunnel_config_udp_add(sa->nic,
1436                                                 tunnel_udp->udp_port,
1437                                                 tunnel_proto);
1438                 break;
1439         }
1440
1441 fail_op:
1442 fail_bad_op:
1443         sfc_adapter_unlock(sa);
1444
1445 fail_bad_proto:
1446         SFC_ASSERT(rc > 0);
1447         return -rc;
1448 }
1449
1450 static int
1451 sfc_dev_udp_tunnel_port_add(struct rte_eth_dev *dev,
1452                             struct rte_eth_udp_tunnel *tunnel_udp)
1453 {
1454         return sfc_dev_udp_tunnel_op(dev, tunnel_udp, SFC_UDP_TUNNEL_ADD_PORT);
1455 }
1456
1457 static int
1458 sfc_dev_udp_tunnel_port_del(struct rte_eth_dev *dev,
1459                             struct rte_eth_udp_tunnel *tunnel_udp)
1460 {
1461         return sfc_dev_udp_tunnel_op(dev, tunnel_udp, SFC_UDP_TUNNEL_DEL_PORT);
1462 }
1463
1464 /*
1465  * The function is used by the secondary process as well. It must not
1466  * use any process-local pointers from the adapter data.
1467  */
1468 static int
1469 sfc_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
1470                           struct rte_eth_rss_conf *rss_conf)
1471 {
1472         struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
1473         struct sfc_rss *rss = &sas->rss;
1474
1475         if (rss->context_type != EFX_RX_SCALE_EXCLUSIVE)
1476                 return -ENOTSUP;
1477
1478         /*
1479          * Mapping of hash configuration between RTE and EFX is not one-to-one,
1480          * hence, conversion is done here to derive a correct set of ETH_RSS
1481          * flags which corresponds to the active EFX configuration stored
1482          * locally in 'sfc_adapter' and kept up-to-date
1483          */
1484         rss_conf->rss_hf = sfc_rx_hf_efx_to_rte(rss, rss->hash_types);
1485         rss_conf->rss_key_len = EFX_RSS_KEY_SIZE;
1486         if (rss_conf->rss_key != NULL)
1487                 rte_memcpy(rss_conf->rss_key, rss->key, EFX_RSS_KEY_SIZE);
1488
1489         return 0;
1490 }
1491
1492 static int
1493 sfc_dev_rss_hash_update(struct rte_eth_dev *dev,
1494                         struct rte_eth_rss_conf *rss_conf)
1495 {
1496         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
1497         struct sfc_rss *rss = &sfc_sa2shared(sa)->rss;
1498         unsigned int efx_hash_types;
1499         int rc = 0;
1500
1501         if (sfc_sa2shared(sa)->isolated)
1502                 return -ENOTSUP;
1503
1504         if (rss->context_type != EFX_RX_SCALE_EXCLUSIVE) {
1505                 sfc_err(sa, "RSS is not available");
1506                 return -ENOTSUP;
1507         }
1508
1509         if (rss->channels == 0) {
1510                 sfc_err(sa, "RSS is not configured");
1511                 return -EINVAL;
1512         }
1513
1514         if ((rss_conf->rss_key != NULL) &&
1515             (rss_conf->rss_key_len != sizeof(rss->key))) {
1516                 sfc_err(sa, "RSS key size is wrong (should be %lu)",
1517                         sizeof(rss->key));
1518                 return -EINVAL;
1519         }
1520
1521         sfc_adapter_lock(sa);
1522
1523         rc = sfc_rx_hf_rte_to_efx(sa, rss_conf->rss_hf, &efx_hash_types);
1524         if (rc != 0)
1525                 goto fail_rx_hf_rte_to_efx;
1526
1527         rc = efx_rx_scale_mode_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT,
1528                                    rss->hash_alg, efx_hash_types, B_TRUE);
1529         if (rc != 0)
1530                 goto fail_scale_mode_set;
1531
1532         if (rss_conf->rss_key != NULL) {
1533                 if (sa->state == SFC_ADAPTER_STARTED) {
1534                         rc = efx_rx_scale_key_set(sa->nic,
1535                                                   EFX_RSS_CONTEXT_DEFAULT,
1536                                                   rss_conf->rss_key,
1537                                                   sizeof(rss->key));
1538                         if (rc != 0)
1539                                 goto fail_scale_key_set;
1540                 }
1541
1542                 rte_memcpy(rss->key, rss_conf->rss_key, sizeof(rss->key));
1543         }
1544
1545         rss->hash_types = efx_hash_types;
1546
1547         sfc_adapter_unlock(sa);
1548
1549         return 0;
1550
1551 fail_scale_key_set:
1552         if (efx_rx_scale_mode_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT,
1553                                   EFX_RX_HASHALG_TOEPLITZ,
1554                                   rss->hash_types, B_TRUE) != 0)
1555                 sfc_err(sa, "failed to restore RSS mode");
1556
1557 fail_scale_mode_set:
1558 fail_rx_hf_rte_to_efx:
1559         sfc_adapter_unlock(sa);
1560         return -rc;
1561 }
1562
1563 /*
1564  * The function is used by the secondary process as well. It must not
1565  * use any process-local pointers from the adapter data.
1566  */
1567 static int
1568 sfc_dev_rss_reta_query(struct rte_eth_dev *dev,
1569                        struct rte_eth_rss_reta_entry64 *reta_conf,
1570                        uint16_t reta_size)
1571 {
1572         struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
1573         struct sfc_rss *rss = &sas->rss;
1574         int entry;
1575
1576         if (rss->context_type != EFX_RX_SCALE_EXCLUSIVE || sas->isolated)
1577                 return -ENOTSUP;
1578
1579         if (rss->channels == 0)
1580                 return -EINVAL;
1581
1582         if (reta_size != EFX_RSS_TBL_SIZE)
1583                 return -EINVAL;
1584
1585         for (entry = 0; entry < reta_size; entry++) {
1586                 int grp = entry / RTE_RETA_GROUP_SIZE;
1587                 int grp_idx = entry % RTE_RETA_GROUP_SIZE;
1588
1589                 if ((reta_conf[grp].mask >> grp_idx) & 1)
1590                         reta_conf[grp].reta[grp_idx] = rss->tbl[entry];
1591         }
1592
1593         return 0;
1594 }
1595
1596 static int
1597 sfc_dev_rss_reta_update(struct rte_eth_dev *dev,
1598                         struct rte_eth_rss_reta_entry64 *reta_conf,
1599                         uint16_t reta_size)
1600 {
1601         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
1602         struct sfc_rss *rss = &sfc_sa2shared(sa)->rss;
1603         unsigned int *rss_tbl_new;
1604         uint16_t entry;
1605         int rc = 0;
1606
1607
1608         if (sfc_sa2shared(sa)->isolated)
1609                 return -ENOTSUP;
1610
1611         if (rss->context_type != EFX_RX_SCALE_EXCLUSIVE) {
1612                 sfc_err(sa, "RSS is not available");
1613                 return -ENOTSUP;
1614         }
1615
1616         if (rss->channels == 0) {
1617                 sfc_err(sa, "RSS is not configured");
1618                 return -EINVAL;
1619         }
1620
1621         if (reta_size != EFX_RSS_TBL_SIZE) {
1622                 sfc_err(sa, "RETA size is wrong (should be %u)",
1623                         EFX_RSS_TBL_SIZE);
1624                 return -EINVAL;
1625         }
1626
1627         rss_tbl_new = rte_zmalloc("rss_tbl_new", sizeof(rss->tbl), 0);
1628         if (rss_tbl_new == NULL)
1629                 return -ENOMEM;
1630
1631         sfc_adapter_lock(sa);
1632
1633         rte_memcpy(rss_tbl_new, rss->tbl, sizeof(rss->tbl));
1634
1635         for (entry = 0; entry < reta_size; entry++) {
1636                 int grp_idx = entry % RTE_RETA_GROUP_SIZE;
1637                 struct rte_eth_rss_reta_entry64 *grp;
1638
1639                 grp = &reta_conf[entry / RTE_RETA_GROUP_SIZE];
1640
1641                 if (grp->mask & (1ull << grp_idx)) {
1642                         if (grp->reta[grp_idx] >= rss->channels) {
1643                                 rc = EINVAL;
1644                                 goto bad_reta_entry;
1645                         }
1646                         rss_tbl_new[entry] = grp->reta[grp_idx];
1647                 }
1648         }
1649
1650         if (sa->state == SFC_ADAPTER_STARTED) {
1651                 rc = efx_rx_scale_tbl_set(sa->nic, EFX_RSS_CONTEXT_DEFAULT,
1652                                           rss_tbl_new, EFX_RSS_TBL_SIZE);
1653                 if (rc != 0)
1654                         goto fail_scale_tbl_set;
1655         }
1656
1657         rte_memcpy(rss->tbl, rss_tbl_new, sizeof(rss->tbl));
1658
1659 fail_scale_tbl_set:
1660 bad_reta_entry:
1661         sfc_adapter_unlock(sa);
1662
1663         rte_free(rss_tbl_new);
1664
1665         SFC_ASSERT(rc >= 0);
1666         return -rc;
1667 }
1668
1669 static int
1670 sfc_dev_filter_ctrl(struct rte_eth_dev *dev, enum rte_filter_type filter_type,
1671                     enum rte_filter_op filter_op,
1672                     void *arg)
1673 {
1674         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
1675         int rc = ENOTSUP;
1676
1677         sfc_log_init(sa, "entry");
1678
1679         switch (filter_type) {
1680         case RTE_ETH_FILTER_NONE:
1681                 sfc_err(sa, "Global filters configuration not supported");
1682                 break;
1683         case RTE_ETH_FILTER_MACVLAN:
1684                 sfc_err(sa, "MACVLAN filters not supported");
1685                 break;
1686         case RTE_ETH_FILTER_ETHERTYPE:
1687                 sfc_err(sa, "EtherType filters not supported");
1688                 break;
1689         case RTE_ETH_FILTER_FLEXIBLE:
1690                 sfc_err(sa, "Flexible filters not supported");
1691                 break;
1692         case RTE_ETH_FILTER_SYN:
1693                 sfc_err(sa, "SYN filters not supported");
1694                 break;
1695         case RTE_ETH_FILTER_NTUPLE:
1696                 sfc_err(sa, "NTUPLE filters not supported");
1697                 break;
1698         case RTE_ETH_FILTER_TUNNEL:
1699                 sfc_err(sa, "Tunnel filters not supported");
1700                 break;
1701         case RTE_ETH_FILTER_FDIR:
1702                 sfc_err(sa, "Flow Director filters not supported");
1703                 break;
1704         case RTE_ETH_FILTER_HASH:
1705                 sfc_err(sa, "Hash filters not supported");
1706                 break;
1707         case RTE_ETH_FILTER_GENERIC:
1708                 if (filter_op != RTE_ETH_FILTER_GET) {
1709                         rc = EINVAL;
1710                 } else {
1711                         *(const void **)arg = &sfc_flow_ops;
1712                         rc = 0;
1713                 }
1714                 break;
1715         default:
1716                 sfc_err(sa, "Unknown filter type %u", filter_type);
1717                 break;
1718         }
1719
1720         sfc_log_init(sa, "exit: %d", -rc);
1721         SFC_ASSERT(rc >= 0);
1722         return -rc;
1723 }
1724
1725 static int
1726 sfc_pool_ops_supported(struct rte_eth_dev *dev, const char *pool)
1727 {
1728         const struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(dev);
1729
1730         /*
1731          * If Rx datapath does not provide callback to check mempool,
1732          * all pools are supported.
1733          */
1734         if (sap->dp_rx->pool_ops_supported == NULL)
1735                 return 1;
1736
1737         return sap->dp_rx->pool_ops_supported(pool);
1738 }
1739
1740 static int
1741 sfc_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)
1742 {
1743         const struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(dev);
1744         struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
1745         struct sfc_rxq_info *rxq_info;
1746
1747         SFC_ASSERT(queue_id < sas->rxq_count);
1748         rxq_info = &sas->rxq_info[queue_id];
1749
1750         return sap->dp_rx->intr_enable(rxq_info->dp);
1751 }
1752
1753 static int
1754 sfc_rx_queue_intr_disable(struct rte_eth_dev *dev, uint16_t queue_id)
1755 {
1756         const struct sfc_adapter_priv *sap = sfc_adapter_priv_by_eth_dev(dev);
1757         struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
1758         struct sfc_rxq_info *rxq_info;
1759
1760         SFC_ASSERT(queue_id < sas->rxq_count);
1761         rxq_info = &sas->rxq_info[queue_id];
1762
1763         return sap->dp_rx->intr_disable(rxq_info->dp);
1764 }
1765
1766 static const struct eth_dev_ops sfc_eth_dev_ops = {
1767         .dev_configure                  = sfc_dev_configure,
1768         .dev_start                      = sfc_dev_start,
1769         .dev_stop                       = sfc_dev_stop,
1770         .dev_set_link_up                = sfc_dev_set_link_up,
1771         .dev_set_link_down              = sfc_dev_set_link_down,
1772         .dev_close                      = sfc_dev_close,
1773         .promiscuous_enable             = sfc_dev_promisc_enable,
1774         .promiscuous_disable            = sfc_dev_promisc_disable,
1775         .allmulticast_enable            = sfc_dev_allmulti_enable,
1776         .allmulticast_disable           = sfc_dev_allmulti_disable,
1777         .link_update                    = sfc_dev_link_update,
1778         .stats_get                      = sfc_stats_get,
1779         .stats_reset                    = sfc_stats_reset,
1780         .xstats_get                     = sfc_xstats_get,
1781         .xstats_reset                   = sfc_stats_reset,
1782         .xstats_get_names               = sfc_xstats_get_names,
1783         .dev_infos_get                  = sfc_dev_infos_get,
1784         .dev_supported_ptypes_get       = sfc_dev_supported_ptypes_get,
1785         .mtu_set                        = sfc_dev_set_mtu,
1786         .rx_queue_start                 = sfc_rx_queue_start,
1787         .rx_queue_stop                  = sfc_rx_queue_stop,
1788         .tx_queue_start                 = sfc_tx_queue_start,
1789         .tx_queue_stop                  = sfc_tx_queue_stop,
1790         .rx_queue_setup                 = sfc_rx_queue_setup,
1791         .rx_queue_release               = sfc_rx_queue_release,
1792         .rx_queue_count                 = sfc_rx_queue_count,
1793         .rx_descriptor_done             = sfc_rx_descriptor_done,
1794         .rx_descriptor_status           = sfc_rx_descriptor_status,
1795         .tx_descriptor_status           = sfc_tx_descriptor_status,
1796         .rx_queue_intr_enable           = sfc_rx_queue_intr_enable,
1797         .rx_queue_intr_disable          = sfc_rx_queue_intr_disable,
1798         .tx_queue_setup                 = sfc_tx_queue_setup,
1799         .tx_queue_release               = sfc_tx_queue_release,
1800         .flow_ctrl_get                  = sfc_flow_ctrl_get,
1801         .flow_ctrl_set                  = sfc_flow_ctrl_set,
1802         .mac_addr_set                   = sfc_mac_addr_set,
1803         .udp_tunnel_port_add            = sfc_dev_udp_tunnel_port_add,
1804         .udp_tunnel_port_del            = sfc_dev_udp_tunnel_port_del,
1805         .reta_update                    = sfc_dev_rss_reta_update,
1806         .reta_query                     = sfc_dev_rss_reta_query,
1807         .rss_hash_update                = sfc_dev_rss_hash_update,
1808         .rss_hash_conf_get              = sfc_dev_rss_hash_conf_get,
1809         .filter_ctrl                    = sfc_dev_filter_ctrl,
1810         .set_mc_addr_list               = sfc_set_mc_addr_list,
1811         .rxq_info_get                   = sfc_rx_queue_info_get,
1812         .txq_info_get                   = sfc_tx_queue_info_get,
1813         .fw_version_get                 = sfc_fw_version_get,
1814         .xstats_get_by_id               = sfc_xstats_get_by_id,
1815         .xstats_get_names_by_id         = sfc_xstats_get_names_by_id,
1816         .pool_ops_supported             = sfc_pool_ops_supported,
1817 };
1818
1819 /**
1820  * Duplicate a string in potentially shared memory required for
1821  * multi-process support.
1822  *
1823  * strdup() allocates from process-local heap/memory.
1824  */
1825 static char *
1826 sfc_strdup(const char *str)
1827 {
1828         size_t size;
1829         char *copy;
1830
1831         if (str == NULL)
1832                 return NULL;
1833
1834         size = strlen(str) + 1;
1835         copy = rte_malloc(__func__, size, 0);
1836         if (copy != NULL)
1837                 rte_memcpy(copy, str, size);
1838
1839         return copy;
1840 }
1841
1842 static int
1843 sfc_eth_dev_set_ops(struct rte_eth_dev *dev)
1844 {
1845         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
1846         struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
1847         const struct sfc_dp_rx *dp_rx;
1848         const struct sfc_dp_tx *dp_tx;
1849         const efx_nic_cfg_t *encp;
1850         unsigned int avail_caps = 0;
1851         const char *rx_name = NULL;
1852         const char *tx_name = NULL;
1853         int rc;
1854
1855         switch (sa->family) {
1856         case EFX_FAMILY_HUNTINGTON:
1857         case EFX_FAMILY_MEDFORD:
1858         case EFX_FAMILY_MEDFORD2:
1859                 avail_caps |= SFC_DP_HW_FW_CAP_EF10;
1860                 break;
1861         default:
1862                 break;
1863         }
1864
1865         encp = efx_nic_cfg_get(sa->nic);
1866         if (encp->enc_rx_es_super_buffer_supported)
1867                 avail_caps |= SFC_DP_HW_FW_CAP_RX_ES_SUPER_BUFFER;
1868
1869         rc = sfc_kvargs_process(sa, SFC_KVARG_RX_DATAPATH,
1870                                 sfc_kvarg_string_handler, &rx_name);
1871         if (rc != 0)
1872                 goto fail_kvarg_rx_datapath;
1873
1874         if (rx_name != NULL) {
1875                 dp_rx = sfc_dp_find_rx_by_name(&sfc_dp_head, rx_name);
1876                 if (dp_rx == NULL) {
1877                         sfc_err(sa, "Rx datapath %s not found", rx_name);
1878                         rc = ENOENT;
1879                         goto fail_dp_rx;
1880                 }
1881                 if (!sfc_dp_match_hw_fw_caps(&dp_rx->dp, avail_caps)) {
1882                         sfc_err(sa,
1883                                 "Insufficient Hw/FW capabilities to use Rx datapath %s",
1884                                 rx_name);
1885                         rc = EINVAL;
1886                         goto fail_dp_rx_caps;
1887                 }
1888         } else {
1889                 dp_rx = sfc_dp_find_rx_by_caps(&sfc_dp_head, avail_caps);
1890                 if (dp_rx == NULL) {
1891                         sfc_err(sa, "Rx datapath by caps %#x not found",
1892                                 avail_caps);
1893                         rc = ENOENT;
1894                         goto fail_dp_rx;
1895                 }
1896         }
1897
1898         sas->dp_rx_name = sfc_strdup(dp_rx->dp.name);
1899         if (sas->dp_rx_name == NULL) {
1900                 rc = ENOMEM;
1901                 goto fail_dp_rx_name;
1902         }
1903
1904         sfc_notice(sa, "use %s Rx datapath", sas->dp_rx_name);
1905
1906         rc = sfc_kvargs_process(sa, SFC_KVARG_TX_DATAPATH,
1907                                 sfc_kvarg_string_handler, &tx_name);
1908         if (rc != 0)
1909                 goto fail_kvarg_tx_datapath;
1910
1911         if (tx_name != NULL) {
1912                 dp_tx = sfc_dp_find_tx_by_name(&sfc_dp_head, tx_name);
1913                 if (dp_tx == NULL) {
1914                         sfc_err(sa, "Tx datapath %s not found", tx_name);
1915                         rc = ENOENT;
1916                         goto fail_dp_tx;
1917                 }
1918                 if (!sfc_dp_match_hw_fw_caps(&dp_tx->dp, avail_caps)) {
1919                         sfc_err(sa,
1920                                 "Insufficient Hw/FW capabilities to use Tx datapath %s",
1921                                 tx_name);
1922                         rc = EINVAL;
1923                         goto fail_dp_tx_caps;
1924                 }
1925         } else {
1926                 dp_tx = sfc_dp_find_tx_by_caps(&sfc_dp_head, avail_caps);
1927                 if (dp_tx == NULL) {
1928                         sfc_err(sa, "Tx datapath by caps %#x not found",
1929                                 avail_caps);
1930                         rc = ENOENT;
1931                         goto fail_dp_tx;
1932                 }
1933         }
1934
1935         sas->dp_tx_name = sfc_strdup(dp_tx->dp.name);
1936         if (sas->dp_tx_name == NULL) {
1937                 rc = ENOMEM;
1938                 goto fail_dp_tx_name;
1939         }
1940
1941         sfc_notice(sa, "use %s Tx datapath", sas->dp_tx_name);
1942
1943         sa->priv.dp_rx = dp_rx;
1944         sa->priv.dp_tx = dp_tx;
1945
1946         dev->rx_pkt_burst = dp_rx->pkt_burst;
1947         dev->tx_pkt_prepare = dp_tx->pkt_prepare;
1948         dev->tx_pkt_burst = dp_tx->pkt_burst;
1949
1950         dev->dev_ops = &sfc_eth_dev_ops;
1951
1952         return 0;
1953
1954 fail_dp_tx_name:
1955 fail_dp_tx_caps:
1956 fail_dp_tx:
1957 fail_kvarg_tx_datapath:
1958         rte_free(sas->dp_rx_name);
1959         sas->dp_rx_name = NULL;
1960
1961 fail_dp_rx_name:
1962 fail_dp_rx_caps:
1963 fail_dp_rx:
1964 fail_kvarg_rx_datapath:
1965         return rc;
1966 }
1967
1968 static void
1969 sfc_eth_dev_clear_ops(struct rte_eth_dev *dev)
1970 {
1971         struct sfc_adapter *sa = sfc_adapter_by_eth_dev(dev);
1972         struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
1973
1974         dev->dev_ops = NULL;
1975         dev->tx_pkt_prepare = NULL;
1976         dev->rx_pkt_burst = NULL;
1977         dev->tx_pkt_burst = NULL;
1978
1979         rte_free(sas->dp_tx_name);
1980         sas->dp_tx_name = NULL;
1981         sa->priv.dp_tx = NULL;
1982
1983         rte_free(sas->dp_rx_name);
1984         sas->dp_rx_name = NULL;
1985         sa->priv.dp_rx = NULL;
1986 }
1987
1988 static const struct eth_dev_ops sfc_eth_dev_secondary_ops = {
1989         .dev_supported_ptypes_get       = sfc_dev_supported_ptypes_get,
1990         .rx_queue_count                 = sfc_rx_queue_count,
1991         .rx_descriptor_done             = sfc_rx_descriptor_done,
1992         .rx_descriptor_status           = sfc_rx_descriptor_status,
1993         .tx_descriptor_status           = sfc_tx_descriptor_status,
1994         .reta_query                     = sfc_dev_rss_reta_query,
1995         .rss_hash_conf_get              = sfc_dev_rss_hash_conf_get,
1996         .rxq_info_get                   = sfc_rx_queue_info_get,
1997         .txq_info_get                   = sfc_tx_queue_info_get,
1998 };
1999
2000 static int
2001 sfc_eth_dev_secondary_init(struct rte_eth_dev *dev, uint32_t logtype_main)
2002 {
2003         struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
2004         struct sfc_adapter_priv *sap;
2005         const struct sfc_dp_rx *dp_rx;
2006         const struct sfc_dp_tx *dp_tx;
2007         int rc;
2008
2009         /*
2010          * Allocate process private data from heap, since it should not
2011          * be located in shared memory allocated using rte_malloc() API.
2012          */
2013         sap = calloc(1, sizeof(*sap));
2014         if (sap == NULL) {
2015                 rc = ENOMEM;
2016                 goto fail_alloc_priv;
2017         }
2018
2019         sap->logtype_main = logtype_main;
2020
2021         dp_rx = sfc_dp_find_rx_by_name(&sfc_dp_head, sas->dp_rx_name);
2022         if (dp_rx == NULL) {
2023                 SFC_LOG(sas, RTE_LOG_ERR, logtype_main,
2024                         "cannot find %s Rx datapath", sas->dp_rx_name);
2025                 rc = ENOENT;
2026                 goto fail_dp_rx;
2027         }
2028         if (~dp_rx->features & SFC_DP_RX_FEAT_MULTI_PROCESS) {
2029                 SFC_LOG(sas, RTE_LOG_ERR, logtype_main,
2030                         "%s Rx datapath does not support multi-process",
2031                         sas->dp_rx_name);
2032                 rc = EINVAL;
2033                 goto fail_dp_rx_multi_process;
2034         }
2035
2036         dp_tx = sfc_dp_find_tx_by_name(&sfc_dp_head, sas->dp_tx_name);
2037         if (dp_tx == NULL) {
2038                 SFC_LOG(sas, RTE_LOG_ERR, logtype_main,
2039                         "cannot find %s Tx datapath", sas->dp_tx_name);
2040                 rc = ENOENT;
2041                 goto fail_dp_tx;
2042         }
2043         if (~dp_tx->features & SFC_DP_TX_FEAT_MULTI_PROCESS) {
2044                 SFC_LOG(sas, RTE_LOG_ERR, logtype_main,
2045                         "%s Tx datapath does not support multi-process",
2046                         sas->dp_tx_name);
2047                 rc = EINVAL;
2048                 goto fail_dp_tx_multi_process;
2049         }
2050
2051         sap->dp_rx = dp_rx;
2052         sap->dp_tx = dp_tx;
2053
2054         dev->process_private = sap;
2055         dev->rx_pkt_burst = dp_rx->pkt_burst;
2056         dev->tx_pkt_prepare = dp_tx->pkt_prepare;
2057         dev->tx_pkt_burst = dp_tx->pkt_burst;
2058         dev->dev_ops = &sfc_eth_dev_secondary_ops;
2059
2060         return 0;
2061
2062 fail_dp_tx_multi_process:
2063 fail_dp_tx:
2064 fail_dp_rx_multi_process:
2065 fail_dp_rx:
2066         free(sap);
2067
2068 fail_alloc_priv:
2069         return rc;
2070 }
2071
2072 static void
2073 sfc_eth_dev_secondary_clear_ops(struct rte_eth_dev *dev)
2074 {
2075         free(dev->process_private);
2076         dev->process_private = NULL;
2077         dev->dev_ops = NULL;
2078         dev->tx_pkt_prepare = NULL;
2079         dev->tx_pkt_burst = NULL;
2080         dev->rx_pkt_burst = NULL;
2081 }
2082
2083 static void
2084 sfc_register_dp(void)
2085 {
2086         /* Register once */
2087         if (TAILQ_EMPTY(&sfc_dp_head)) {
2088                 /* Prefer EF10 datapath */
2089                 sfc_dp_register(&sfc_dp_head, &sfc_ef10_essb_rx.dp);
2090                 sfc_dp_register(&sfc_dp_head, &sfc_ef10_rx.dp);
2091                 sfc_dp_register(&sfc_dp_head, &sfc_efx_rx.dp);
2092
2093                 sfc_dp_register(&sfc_dp_head, &sfc_ef10_tx.dp);
2094                 sfc_dp_register(&sfc_dp_head, &sfc_efx_tx.dp);
2095                 sfc_dp_register(&sfc_dp_head, &sfc_ef10_simple_tx.dp);
2096         }
2097 }
2098
2099 static int
2100 sfc_eth_dev_init(struct rte_eth_dev *dev)
2101 {
2102         struct sfc_adapter_shared *sas = sfc_adapter_shared_by_eth_dev(dev);
2103         struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(dev);
2104         uint32_t logtype_main;
2105         struct sfc_adapter *sa;
2106         int rc;
2107         const efx_nic_cfg_t *encp;
2108         const struct rte_ether_addr *from;
2109
2110         sfc_register_dp();
2111
2112         logtype_main = sfc_register_logtype(&pci_dev->addr,
2113                                             SFC_LOGTYPE_MAIN_STR,
2114                                             RTE_LOG_NOTICE);
2115
2116         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2117                 return -sfc_eth_dev_secondary_init(dev, logtype_main);
2118
2119         /* Required for logging */
2120         sas->pci_addr = pci_dev->addr;
2121         sas->port_id = dev->data->port_id;
2122
2123         /*
2124          * Allocate process private data from heap, since it should not
2125          * be located in shared memory allocated using rte_malloc() API.
2126          */
2127         sa = calloc(1, sizeof(*sa));
2128         if (sa == NULL) {
2129                 rc = ENOMEM;
2130                 goto fail_alloc_sa;
2131         }
2132
2133         dev->process_private = sa;
2134
2135         /* Required for logging */
2136         sa->priv.shared = sas;
2137         sa->priv.logtype_main = logtype_main;
2138
2139         sa->eth_dev = dev;
2140
2141         /* Copy PCI device info to the dev->data */
2142         rte_eth_copy_pci_info(dev, pci_dev);
2143
2144         rc = sfc_kvargs_parse(sa);
2145         if (rc != 0)
2146                 goto fail_kvargs_parse;
2147
2148         sfc_log_init(sa, "entry");
2149
2150         dev->data->dev_flags |= RTE_ETH_DEV_CLOSE_REMOVE;
2151
2152         dev->data->mac_addrs = rte_zmalloc("sfc", RTE_ETHER_ADDR_LEN, 0);
2153         if (dev->data->mac_addrs == NULL) {
2154                 rc = ENOMEM;
2155                 goto fail_mac_addrs;
2156         }
2157
2158         sfc_adapter_lock_init(sa);
2159         sfc_adapter_lock(sa);
2160
2161         sfc_log_init(sa, "probing");
2162         rc = sfc_probe(sa);
2163         if (rc != 0)
2164                 goto fail_probe;
2165
2166         sfc_log_init(sa, "set device ops");
2167         rc = sfc_eth_dev_set_ops(dev);
2168         if (rc != 0)
2169                 goto fail_set_ops;
2170
2171         sfc_log_init(sa, "attaching");
2172         rc = sfc_attach(sa);
2173         if (rc != 0)
2174                 goto fail_attach;
2175
2176         encp = efx_nic_cfg_get(sa->nic);
2177
2178         /*
2179          * The arguments are really reverse order in comparison to
2180          * Linux kernel. Copy from NIC config to Ethernet device data.
2181          */
2182         from = (const struct rte_ether_addr *)(encp->enc_mac_addr);
2183         rte_ether_addr_copy(from, &dev->data->mac_addrs[0]);
2184
2185         sfc_adapter_unlock(sa);
2186
2187         sfc_log_init(sa, "done");
2188         return 0;
2189
2190 fail_attach:
2191         sfc_eth_dev_clear_ops(dev);
2192
2193 fail_set_ops:
2194         sfc_unprobe(sa);
2195
2196 fail_probe:
2197         sfc_adapter_unlock(sa);
2198         sfc_adapter_lock_fini(sa);
2199         rte_free(dev->data->mac_addrs);
2200         dev->data->mac_addrs = NULL;
2201
2202 fail_mac_addrs:
2203         sfc_kvargs_cleanup(sa);
2204
2205 fail_kvargs_parse:
2206         sfc_log_init(sa, "failed %d", rc);
2207         dev->process_private = NULL;
2208         free(sa);
2209
2210 fail_alloc_sa:
2211         SFC_ASSERT(rc > 0);
2212         return -rc;
2213 }
2214
2215 static int
2216 sfc_eth_dev_uninit(struct rte_eth_dev *dev)
2217 {
2218         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2219                 sfc_eth_dev_secondary_clear_ops(dev);
2220                 return 0;
2221         }
2222
2223         sfc_dev_close(dev);
2224
2225         return 0;
2226 }
2227
2228 static const struct rte_pci_id pci_id_sfc_efx_map[] = {
2229         { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_FARMINGDALE) },
2230         { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_FARMINGDALE_VF) },
2231         { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_GREENPORT) },
2232         { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_GREENPORT_VF) },
2233         { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_MEDFORD) },
2234         { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_MEDFORD_VF) },
2235         { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_MEDFORD2) },
2236         { RTE_PCI_DEVICE(EFX_PCI_VENID_SFC, EFX_PCI_DEVID_MEDFORD2_VF) },
2237         { .vendor_id = 0 /* sentinel */ }
2238 };
2239
2240 static int sfc_eth_dev_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
2241         struct rte_pci_device *pci_dev)
2242 {
2243         return rte_eth_dev_pci_generic_probe(pci_dev,
2244                 sizeof(struct sfc_adapter_shared), sfc_eth_dev_init);
2245 }
2246
2247 static int sfc_eth_dev_pci_remove(struct rte_pci_device *pci_dev)
2248 {
2249         return rte_eth_dev_pci_generic_remove(pci_dev, sfc_eth_dev_uninit);
2250 }
2251
2252 static struct rte_pci_driver sfc_efx_pmd = {
2253         .id_table = pci_id_sfc_efx_map,
2254         .drv_flags =
2255                 RTE_PCI_DRV_INTR_LSC |
2256                 RTE_PCI_DRV_NEED_MAPPING,
2257         .probe = sfc_eth_dev_pci_probe,
2258         .remove = sfc_eth_dev_pci_remove,
2259 };
2260
2261 RTE_PMD_REGISTER_PCI(net_sfc_efx, sfc_efx_pmd);
2262 RTE_PMD_REGISTER_PCI_TABLE(net_sfc_efx, pci_id_sfc_efx_map);
2263 RTE_PMD_REGISTER_KMOD_DEP(net_sfc_efx, "* igb_uio | uio_pci_generic | vfio-pci");
2264 RTE_PMD_REGISTER_PARAM_STRING(net_sfc_efx,
2265         SFC_KVARG_RX_DATAPATH "=" SFC_KVARG_VALUES_RX_DATAPATH " "
2266         SFC_KVARG_TX_DATAPATH "=" SFC_KVARG_VALUES_TX_DATAPATH " "
2267         SFC_KVARG_PERF_PROFILE "=" SFC_KVARG_VALUES_PERF_PROFILE " "
2268         SFC_KVARG_FW_VARIANT "=" SFC_KVARG_VALUES_FW_VARIANT " "
2269         SFC_KVARG_RXD_WAIT_TIMEOUT_NS "=<long> "
2270         SFC_KVARG_STATS_UPDATE_PERIOD_MS "=<long>");
2271
2272 RTE_INIT(sfc_driver_register_logtype)
2273 {
2274         int ret;
2275
2276         ret = rte_log_register_type_and_pick_level(SFC_LOGTYPE_PREFIX "driver",
2277                                                    RTE_LOG_NOTICE);
2278         sfc_logtype_driver = (ret < 0) ? RTE_LOGTYPE_PMD : ret;
2279 }