drivers/net/failsafe/failsafe_ops.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2017 6WIND S.A.
   3  * Copyright 2017 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stdbool.h>
   7 #include <stdint.h>
   8 #include <unistd.h>
   9
  10 #include <rte_debug.h>
  11 #include <rte_atomic.h>
  12 #include <rte_ethdev_driver.h>
  13 #include <rte_malloc.h>
  14 #include <rte_flow.h>
  15 #include <rte_cycles.h>
  16 #include <rte_ethdev.h>
  17
  18 #include "failsafe_private.h"
  19
  20 static struct rte_eth_dev_info default_infos = {
  21         /* Max possible number of elements */
  22         .max_rx_pktlen = UINT32_MAX,
  23         .max_rx_queues = RTE_MAX_QUEUES_PER_PORT,
  24         .max_tx_queues = RTE_MAX_QUEUES_PER_PORT,
  25         .max_mac_addrs = FAILSAFE_MAX_ETHADDR,
  26         .max_hash_mac_addrs = UINT32_MAX,
  27         .max_vfs = UINT16_MAX,
  28         .max_vmdq_pools = UINT16_MAX,
  29         .rx_desc_lim = {
  30                 .nb_max = UINT16_MAX,
  31                 .nb_min = 0,
  32                 .nb_align = 1,
  33                 .nb_seg_max = UINT16_MAX,
  34                 .nb_mtu_seg_max = UINT16_MAX,
  35         },
  36         .tx_desc_lim = {
  37                 .nb_max = UINT16_MAX,
  38                 .nb_min = 0,
  39                 .nb_align = 1,
  40                 .nb_seg_max = UINT16_MAX,
  41                 .nb_mtu_seg_max = UINT16_MAX,
  42         },
  43         /*
  44          * Set of capabilities that can be verified upon
  45          * configuring a sub-device.
  46          */
  47         .rx_offload_capa =
  48                 DEV_RX_OFFLOAD_VLAN_STRIP |
  49                 DEV_RX_OFFLOAD_IPV4_CKSUM |
  50                 DEV_RX_OFFLOAD_UDP_CKSUM |
  51                 DEV_RX_OFFLOAD_TCP_CKSUM |
  52                 DEV_RX_OFFLOAD_TCP_LRO |
  53                 DEV_RX_OFFLOAD_QINQ_STRIP |
  54                 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
  55                 DEV_RX_OFFLOAD_MACSEC_STRIP |
  56                 DEV_RX_OFFLOAD_HEADER_SPLIT |
  57                 DEV_RX_OFFLOAD_VLAN_FILTER |
  58                 DEV_RX_OFFLOAD_VLAN_EXTEND |
  59                 DEV_RX_OFFLOAD_JUMBO_FRAME |
  60                 DEV_RX_OFFLOAD_SCATTER |
  61                 DEV_RX_OFFLOAD_TIMESTAMP |
  62                 DEV_RX_OFFLOAD_SECURITY,
  63         .rx_queue_offload_capa =
  64                 DEV_RX_OFFLOAD_VLAN_STRIP |
  65                 DEV_RX_OFFLOAD_IPV4_CKSUM |
  66                 DEV_RX_OFFLOAD_UDP_CKSUM |
  67                 DEV_RX_OFFLOAD_TCP_CKSUM |
  68                 DEV_RX_OFFLOAD_TCP_LRO |
  69                 DEV_RX_OFFLOAD_QINQ_STRIP |
  70                 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
  71                 DEV_RX_OFFLOAD_MACSEC_STRIP |
  72                 DEV_RX_OFFLOAD_HEADER_SPLIT |
  73                 DEV_RX_OFFLOAD_VLAN_FILTER |
  74                 DEV_RX_OFFLOAD_VLAN_EXTEND |
  75                 DEV_RX_OFFLOAD_JUMBO_FRAME |
  76                 DEV_RX_OFFLOAD_SCATTER |
  77                 DEV_RX_OFFLOAD_TIMESTAMP |
  78                 DEV_RX_OFFLOAD_SECURITY,
  79         .tx_offload_capa =
  80                 DEV_TX_OFFLOAD_MULTI_SEGS |
  81                 DEV_TX_OFFLOAD_IPV4_CKSUM |
  82                 DEV_TX_OFFLOAD_UDP_CKSUM |
  83                 DEV_TX_OFFLOAD_TCP_CKSUM |
  84                 DEV_TX_OFFLOAD_TCP_TSO,
  85         .flow_type_rss_offloads =
  86                         ETH_RSS_IP |
  87                         ETH_RSS_UDP |
  88                         ETH_RSS_TCP,
  89         .dev_capa =
  90                 RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP,
  91 };
  92
  93 static int
  94 fs_dev_configure(struct rte_eth_dev *dev)
  95 {
  96         struct sub_device *sdev;
  97         uint8_t i;
  98         int ret;
  99
 100         fs_lock(dev, 0);
 101         FOREACH_SUBDEV(sdev, i, dev) {
 102                 int rmv_interrupt = 0;
 103                 int lsc_interrupt = 0;
 104                 int lsc_enabled;
 105
 106                 if (sdev->state != DEV_PROBED &&
 107                     !(PRIV(dev)->alarm_lock == 0 && sdev->state == DEV_ACTIVE))
 108                         continue;
 109
 110                 rmv_interrupt = ETH(sdev)->data->dev_flags &
 111                                 RTE_ETH_DEV_INTR_RMV;
 112                 if (rmv_interrupt) {
 113                         DEBUG("Enabling RMV interrupts for sub_device %d", i);
 114                         dev->data->dev_conf.intr_conf.rmv = 1;
 115                 } else {
 116                         DEBUG("sub_device %d does not support RMV event", i);
 117                 }
 118                 lsc_enabled = dev->data->dev_conf.intr_conf.lsc;
 119                 lsc_interrupt = lsc_enabled &&
 120                                 (ETH(sdev)->data->dev_flags &
 121                                  RTE_ETH_DEV_INTR_LSC);
 122                 if (lsc_interrupt) {
 123                         DEBUG("Enabling LSC interrupts for sub_device %d", i);
 124                         dev->data->dev_conf.intr_conf.lsc = 1;
 125                 } else if (lsc_enabled && !lsc_interrupt) {
 126                         DEBUG("Disabling LSC interrupts for sub_device %d", i);
 127                         dev->data->dev_conf.intr_conf.lsc = 0;
 128                 }
 129                 DEBUG("Configuring sub-device %d", i);
 130                 ret = rte_eth_dev_configure(PORT_ID(sdev),
 131                                         dev->data->nb_rx_queues,
 132                                         dev->data->nb_tx_queues,
 133                                         &dev->data->dev_conf);
 134                 if (ret) {
 135                         if (!fs_err(sdev, ret))
 136                                 continue;
 137                         ERROR("Could not configure sub_device %d", i);
 138                         fs_unlock(dev, 0);
 139                         return ret;
 140                 }
 141                 if (rmv_interrupt && sdev->rmv_callback == 0) {
 142                         ret = rte_eth_dev_callback_register(PORT_ID(sdev),
 143                                         RTE_ETH_EVENT_INTR_RMV,
 144                                         failsafe_eth_rmv_event_callback,
 145                                         sdev);
 146                         if (ret)
 147                                 WARN("Failed to register RMV callback for sub_device %d",
 148                                      SUB_ID(sdev));
 149                         else
 150                                 sdev->rmv_callback = 1;
 151                 }
 152                 dev->data->dev_conf.intr_conf.rmv = 0;
 153                 if (lsc_interrupt && sdev->lsc_callback == 0) {
 154                         ret = rte_eth_dev_callback_register(PORT_ID(sdev),
 155                                                 RTE_ETH_EVENT_INTR_LSC,
 156                                                 failsafe_eth_lsc_event_callback,
 157                                                 dev);
 158                         if (ret)
 159                                 WARN("Failed to register LSC callback for sub_device %d",
 160                                      SUB_ID(sdev));
 161                         else
 162                                 sdev->lsc_callback = 1;
 163                 }
 164                 dev->data->dev_conf.intr_conf.lsc = lsc_enabled;
 165                 sdev->state = DEV_ACTIVE;
 166         }
 167         if (PRIV(dev)->state < DEV_ACTIVE)
 168                 PRIV(dev)->state = DEV_ACTIVE;
 169         fs_unlock(dev, 0);
 170         return 0;
 171 }
 172
 173 static void
 174 fs_set_queues_state_start(struct rte_eth_dev *dev)
 175 {
 176         struct rxq *rxq;
 177         struct txq *txq;
 178         uint16_t i;
 179
 180         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 181                 rxq = dev->data->rx_queues[i];
 182                 if (rxq != NULL && !rxq->info.conf.rx_deferred_start)
 183                         dev->data->rx_queue_state[i] =
 184                                                 RTE_ETH_QUEUE_STATE_STARTED;
 185         }
 186         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 187                 txq = dev->data->tx_queues[i];
 188                 if (!txq->info.conf.tx_deferred_start)
 189                         dev->data->tx_queue_state[i] =
 190                                                 RTE_ETH_QUEUE_STATE_STARTED;
 191         }
 192 }
 193
 194 static int
 195 fs_dev_start(struct rte_eth_dev *dev)
 196 {
 197         struct sub_device *sdev;
 198         uint8_t i;
 199         int ret;
 200
 201         fs_lock(dev, 0);
 202         ret = failsafe_rx_intr_install(dev);
 203         if (ret) {
 204                 fs_unlock(dev, 0);
 205                 return ret;
 206         }
 207         FOREACH_SUBDEV(sdev, i, dev) {
 208                 if (sdev->state != DEV_ACTIVE)
 209                         continue;
 210                 DEBUG("Starting sub_device %d", i);
 211                 ret = rte_eth_dev_start(PORT_ID(sdev));
 212                 if (ret) {
 213                         if (!fs_err(sdev, ret))
 214                                 continue;
 215                         fs_unlock(dev, 0);
 216                         return ret;
 217                 }
 218                 ret = failsafe_rx_intr_install_subdevice(sdev);
 219                 if (ret) {
 220                         if (!fs_err(sdev, ret))
 221                                 continue;
 222                         rte_eth_dev_stop(PORT_ID(sdev));
 223                         fs_unlock(dev, 0);
 224                         return ret;
 225                 }
 226                 sdev->state = DEV_STARTED;
 227         }
 228         if (PRIV(dev)->state < DEV_STARTED) {
 229                 PRIV(dev)->state = DEV_STARTED;
 230                 fs_set_queues_state_start(dev);
 231         }
 232         fs_switch_dev(dev, NULL);
 233         fs_unlock(dev, 0);
 234         return 0;
 235 }
 236
 237 static void
 238 fs_set_queues_state_stop(struct rte_eth_dev *dev)
 239 {
 240         uint16_t i;
 241
 242         for (i = 0; i < dev->data->nb_rx_queues; i++)
 243                 if (dev->data->rx_queues[i] != NULL)
 244                         dev->data->rx_queue_state[i] =
 245                                                 RTE_ETH_QUEUE_STATE_STOPPED;
 246         for (i = 0; i < dev->data->nb_tx_queues; i++)
 247                 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
 248 }
 249
 250 static void
 251 fs_dev_stop(struct rte_eth_dev *dev)
 252 {
 253         struct sub_device *sdev;
 254         uint8_t i;
 255
 256         fs_lock(dev, 0);
 257         PRIV(dev)->state = DEV_STARTED - 1;
 258         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_STARTED) {
 259                 rte_eth_dev_stop(PORT_ID(sdev));
 260                 failsafe_rx_intr_uninstall_subdevice(sdev);
 261                 sdev->state = DEV_STARTED - 1;
 262         }
 263         failsafe_rx_intr_uninstall(dev);
 264         fs_set_queues_state_stop(dev);
 265         fs_unlock(dev, 0);
 266 }
 267
 268 static int
 269 fs_dev_set_link_up(struct rte_eth_dev *dev)
 270 {
 271         struct sub_device *sdev;
 272         uint8_t i;
 273         int ret;
 274
 275         fs_lock(dev, 0);
 276         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 277                 DEBUG("Calling rte_eth_dev_set_link_up on sub_device %d", i);
 278                 ret = rte_eth_dev_set_link_up(PORT_ID(sdev));
 279                 if ((ret = fs_err(sdev, ret))) {
 280                         ERROR("Operation rte_eth_dev_set_link_up failed for sub_device %d"
 281                               " with error %d", i, ret);
 282                         fs_unlock(dev, 0);
 283                         return ret;
 284                 }
 285         }
 286         fs_unlock(dev, 0);
 287         return 0;
 288 }
 289
 290 static int
 291 fs_dev_set_link_down(struct rte_eth_dev *dev)
 292 {
 293         struct sub_device *sdev;
 294         uint8_t i;
 295         int ret;
 296
 297         fs_lock(dev, 0);
 298         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 299                 DEBUG("Calling rte_eth_dev_set_link_down on sub_device %d", i);
 300                 ret = rte_eth_dev_set_link_down(PORT_ID(sdev));
 301                 if ((ret = fs_err(sdev, ret))) {
 302                         ERROR("Operation rte_eth_dev_set_link_down failed for sub_device %d"
 303                               " with error %d", i, ret);
 304                         fs_unlock(dev, 0);
 305                         return ret;
 306                 }
 307         }
 308         fs_unlock(dev, 0);
 309         return 0;
 310 }
 311
 312 static void fs_dev_free_queues(struct rte_eth_dev *dev);
 313 static void
 314 fs_dev_close(struct rte_eth_dev *dev)
 315 {
 316         struct sub_device *sdev;
 317         uint8_t i;
 318
 319         fs_lock(dev, 0);
 320         failsafe_hotplug_alarm_cancel(dev);
 321         if (PRIV(dev)->state == DEV_STARTED)
 322                 dev->dev_ops->dev_stop(dev);
 323         PRIV(dev)->state = DEV_ACTIVE - 1;
 324         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 325                 DEBUG("Closing sub_device %d", i);
 326                 failsafe_eth_dev_unregister_callbacks(sdev);
 327                 rte_eth_dev_close(PORT_ID(sdev));
 328                 sdev->state = DEV_ACTIVE - 1;
 329         }
 330         fs_dev_free_queues(dev);
 331         fs_unlock(dev, 0);
 332 }
 333
 334 static int
 335 fs_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 336 {
 337         struct sub_device *sdev;
 338         uint8_t i;
 339         int ret;
 340         int err = 0;
 341         bool failure = true;
 342
 343         fs_lock(dev, 0);
 344         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 345                 uint16_t port_id = ETH(sdev)->data->port_id;
 346
 347                 ret = rte_eth_dev_rx_queue_stop(port_id, rx_queue_id);
 348                 ret = fs_err(sdev, ret);
 349                 if (ret) {
 350                         ERROR("Rx queue stop failed for subdevice %d", i);
 351                         err = ret;
 352                 } else {
 353                         failure = false;
 354                 }
 355         }
 356         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 357         fs_unlock(dev, 0);
 358         /* Return 0 in case of at least one successful queue stop */
 359         return (failure) ? err : 0;
 360 }
 361
 362 static int
 363 fs_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 364 {
 365         struct sub_device *sdev;
 366         uint8_t i;
 367         int ret;
 368
 369         fs_lock(dev, 0);
 370         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 371                 uint16_t port_id = ETH(sdev)->data->port_id;
 372
 373                 ret = rte_eth_dev_rx_queue_start(port_id, rx_queue_id);
 374                 ret = fs_err(sdev, ret);
 375                 if (ret) {
 376                         ERROR("Rx queue start failed for subdevice %d", i);
 377                         fs_rx_queue_stop(dev, rx_queue_id);
 378                         fs_unlock(dev, 0);
 379                         return ret;
 380                 }
 381         }
 382         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 383         fs_unlock(dev, 0);
 384         return 0;
 385 }
 386
 387 static int
 388 fs_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 389 {
 390         struct sub_device *sdev;
 391         uint8_t i;
 392         int ret;
 393         int err = 0;
 394         bool failure = true;
 395
 396         fs_lock(dev, 0);
 397         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 398                 uint16_t port_id = ETH(sdev)->data->port_id;
 399
 400                 ret = rte_eth_dev_tx_queue_stop(port_id, tx_queue_id);
 401                 ret = fs_err(sdev, ret);
 402                 if (ret) {
 403                         ERROR("Tx queue stop failed for subdevice %d", i);
 404                         err = ret;
 405                 } else {
 406                         failure = false;
 407                 }
 408         }
 409         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 410         fs_unlock(dev, 0);
 411         /* Return 0 in case of at least one successful queue stop */
 412         return (failure) ? err : 0;
 413 }
 414
 415 static int
 416 fs_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 417 {
 418         struct sub_device *sdev;
 419         uint8_t i;
 420         int ret;
 421
 422         fs_lock(dev, 0);
 423         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 424                 uint16_t port_id = ETH(sdev)->data->port_id;
 425
 426                 ret = rte_eth_dev_tx_queue_start(port_id, tx_queue_id);
 427                 ret = fs_err(sdev, ret);
 428                 if (ret) {
 429                         ERROR("Tx queue start failed for subdevice %d", i);
 430                         fs_tx_queue_stop(dev, tx_queue_id);
 431                         fs_unlock(dev, 0);
 432                         return ret;
 433                 }
 434         }
 435         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 436         fs_unlock(dev, 0);
 437         return 0;
 438 }
 439
 440 static void
 441 fs_rx_queue_release(void *queue)
 442 {
 443         struct rte_eth_dev *dev;
 444         struct sub_device *sdev;
 445         uint8_t i;
 446         struct rxq *rxq;
 447
 448         if (queue == NULL)
 449                 return;
 450         rxq = queue;
 451         dev = rxq->priv->dev;
 452         fs_lock(dev, 0);
 453         if (rxq->event_fd > 0)
 454                 close(rxq->event_fd);
 455         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 456                 if (ETH(sdev)->data->rx_queues != NULL &&
 457                     ETH(sdev)->data->rx_queues[rxq->qid] != NULL) {
 458                         SUBOPS(sdev, rx_queue_release)
 459                                 (ETH(sdev)->data->rx_queues[rxq->qid]);
 460                 }
 461         }
 462         dev->data->rx_queues[rxq->qid] = NULL;
 463         rte_free(rxq);
 464         fs_unlock(dev, 0);
 465 }
 466
 467 static int
 468 fs_rx_queue_setup(struct rte_eth_dev *dev,
 469                 uint16_t rx_queue_id,
 470                 uint16_t nb_rx_desc,
 471                 unsigned int socket_id,
 472                 const struct rte_eth_rxconf *rx_conf,
 473                 struct rte_mempool *mb_pool)
 474 {
 475         /*
 476          * FIXME: Add a proper interface in rte_eal_interrupts for
 477          * allocating eventfd as an interrupt vector.
 478          * For the time being, fake as if we are using MSIX interrupts,
 479          * this will cause rte_intr_efd_enable to allocate an eventfd for us.
 480          */
 481         struct rte_intr_handle intr_handle = {
 482                 .type = RTE_INTR_HANDLE_VFIO_MSIX,
 483                 .efds = { -1, },
 484         };
 485         struct sub_device *sdev;
 486         struct rxq *rxq;
 487         uint8_t i;
 488         int ret;
 489
 490         fs_lock(dev, 0);
 491         if (rx_conf->rx_deferred_start) {
 492                 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) {
 493                         if (SUBOPS(sdev, rx_queue_start) == NULL) {
 494                                 ERROR("Rx queue deferred start is not "
 495                                         "supported for subdevice %d", i);
 496                                 fs_unlock(dev, 0);
 497                                 return -EINVAL;
 498                         }
 499                 }
 500         }
 501         rxq = dev->data->rx_queues[rx_queue_id];
 502         if (rxq != NULL) {
 503                 fs_rx_queue_release(rxq);
 504                 dev->data->rx_queues[rx_queue_id] = NULL;
 505         }
 506         rxq = rte_zmalloc(NULL,
 507                           sizeof(*rxq) +
 508                           sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail,
 509                           RTE_CACHE_LINE_SIZE);
 510         if (rxq == NULL) {
 511                 fs_unlock(dev, 0);
 512                 return -ENOMEM;
 513         }
 514         FOREACH_SUBDEV(sdev, i, dev)
 515                 rte_atomic64_init(&rxq->refcnt[i]);
 516         rxq->qid = rx_queue_id;
 517         rxq->socket_id = socket_id;
 518         rxq->info.mp = mb_pool;
 519         rxq->info.conf = *rx_conf;
 520         rxq->info.nb_desc = nb_rx_desc;
 521         rxq->priv = PRIV(dev);
 522         rxq->sdev = PRIV(dev)->subs;
 523         ret = rte_intr_efd_enable(&intr_handle, 1);
 524         if (ret < 0) {
 525                 fs_unlock(dev, 0);
 526                 return ret;
 527         }
 528         rxq->event_fd = intr_handle.efds[0];
 529         dev->data->rx_queues[rx_queue_id] = rxq;
 530         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 531                 ret = rte_eth_rx_queue_setup(PORT_ID(sdev),
 532                                 rx_queue_id,
 533                                 nb_rx_desc, socket_id,
 534                                 rx_conf, mb_pool);
 535                 if ((ret = fs_err(sdev, ret))) {
 536                         ERROR("RX queue setup failed for sub_device %d", i);
 537                         goto free_rxq;
 538                 }
 539         }
 540         fs_unlock(dev, 0);
 541         return 0;
 542 free_rxq:
 543         fs_rx_queue_release(rxq);
 544         fs_unlock(dev, 0);
 545         return ret;
 546 }
 547
 548 static int
 549 fs_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
 550 {
 551         struct rxq *rxq;
 552         struct sub_device *sdev;
 553         uint8_t i;
 554         int ret;
 555         int rc = 0;
 556
 557         fs_lock(dev, 0);
 558         if (idx >= dev->data->nb_rx_queues) {
 559                 rc = -EINVAL;
 560                 goto unlock;
 561         }
 562         rxq = dev->data->rx_queues[idx];
 563         if (rxq == NULL || rxq->event_fd <= 0) {
 564                 rc = -EINVAL;
 565                 goto unlock;
 566         }
 567         /* Fail if proxy service is nor running. */
 568         if (PRIV(dev)->rxp.sstate != SS_RUNNING) {
 569                 ERROR("failsafe interrupt services are not running");
 570                 rc = -EAGAIN;
 571                 goto unlock;
 572         }
 573         rxq->enable_events = 1;
 574         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 575                 ret = rte_eth_dev_rx_intr_enable(PORT_ID(sdev), idx);
 576                 ret = fs_err(sdev, ret);
 577                 if (ret)
 578                         rc = ret;
 579         }
 580 unlock:
 581         fs_unlock(dev, 0);
 582         if (rc)
 583                 rte_errno = -rc;
 584         return rc;
 585 }
 586
 587 static int
 588 fs_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
 589 {
 590         struct rxq *rxq;
 591         struct sub_device *sdev;
 592         uint64_t u64;
 593         uint8_t i;
 594         int rc = 0;
 595         int ret;
 596
 597         fs_lock(dev, 0);
 598         if (idx >= dev->data->nb_rx_queues) {
 599                 rc = -EINVAL;
 600                 goto unlock;
 601         }
 602         rxq = dev->data->rx_queues[idx];
 603         if (rxq == NULL || rxq->event_fd <= 0) {
 604                 rc = -EINVAL;
 605                 goto unlock;
 606         }
 607         rxq->enable_events = 0;
 608         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 609                 ret = rte_eth_dev_rx_intr_disable(PORT_ID(sdev), idx);
 610                 ret = fs_err(sdev, ret);
 611                 if (ret)
 612                         rc = ret;
 613         }
 614         /* Clear pending events */
 615         while (read(rxq->event_fd, &u64, sizeof(uint64_t)) >  0)
 616                 ;
 617 unlock:
 618         fs_unlock(dev, 0);
 619         if (rc)
 620                 rte_errno = -rc;
 621         return rc;
 622 }
 623
 624 static void
 625 fs_tx_queue_release(void *queue)
 626 {
 627         struct rte_eth_dev *dev;
 628         struct sub_device *sdev;
 629         uint8_t i;
 630         struct txq *txq;
 631
 632         if (queue == NULL)
 633                 return;
 634         txq = queue;
 635         dev = txq->priv->dev;
 636         fs_lock(dev, 0);
 637         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 638                 if (ETH(sdev)->data->tx_queues != NULL &&
 639                     ETH(sdev)->data->tx_queues[txq->qid] != NULL) {
 640                         SUBOPS(sdev, tx_queue_release)
 641                                 (ETH(sdev)->data->tx_queues[txq->qid]);
 642                 }
 643         }
 644         dev->data->tx_queues[txq->qid] = NULL;
 645         rte_free(txq);
 646         fs_unlock(dev, 0);
 647 }
 648
 649 static int
 650 fs_tx_queue_setup(struct rte_eth_dev *dev,
 651                 uint16_t tx_queue_id,
 652                 uint16_t nb_tx_desc,
 653                 unsigned int socket_id,
 654                 const struct rte_eth_txconf *tx_conf)
 655 {
 656         struct sub_device *sdev;
 657         struct txq *txq;
 658         uint8_t i;
 659         int ret;
 660
 661         fs_lock(dev, 0);
 662         if (tx_conf->tx_deferred_start) {
 663                 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) {
 664                         if (SUBOPS(sdev, tx_queue_start) == NULL) {
 665                                 ERROR("Tx queue deferred start is not "
 666                                         "supported for subdevice %d", i);
 667                                 fs_unlock(dev, 0);
 668                                 return -EINVAL;
 669                         }
 670                 }
 671         }
 672         txq = dev->data->tx_queues[tx_queue_id];
 673         if (txq != NULL) {
 674                 fs_tx_queue_release(txq);
 675                 dev->data->tx_queues[tx_queue_id] = NULL;
 676         }
 677         txq = rte_zmalloc("ethdev TX queue",
 678                           sizeof(*txq) +
 679                           sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail,
 680                           RTE_CACHE_LINE_SIZE);
 681         if (txq == NULL) {
 682                 fs_unlock(dev, 0);
 683                 return -ENOMEM;
 684         }
 685         FOREACH_SUBDEV(sdev, i, dev)
 686                 rte_atomic64_init(&txq->refcnt[i]);
 687         txq->qid = tx_queue_id;
 688         txq->socket_id = socket_id;
 689         txq->info.conf = *tx_conf;
 690         txq->info.nb_desc = nb_tx_desc;
 691         txq->priv = PRIV(dev);
 692         dev->data->tx_queues[tx_queue_id] = txq;
 693         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 694                 ret = rte_eth_tx_queue_setup(PORT_ID(sdev),
 695                                 tx_queue_id,
 696                                 nb_tx_desc, socket_id,
 697                                 tx_conf);
 698                 if ((ret = fs_err(sdev, ret))) {
 699                         ERROR("TX queue setup failed for sub_device %d", i);
 700                         goto free_txq;
 701                 }
 702         }
 703         fs_unlock(dev, 0);
 704         return 0;
 705 free_txq:
 706         fs_tx_queue_release(txq);
 707         fs_unlock(dev, 0);
 708         return ret;
 709 }
 710
 711 static void
 712 fs_dev_free_queues(struct rte_eth_dev *dev)
 713 {
 714         uint16_t i;
 715
 716         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 717                 fs_rx_queue_release(dev->data->rx_queues[i]);
 718                 dev->data->rx_queues[i] = NULL;
 719         }
 720         dev->data->nb_rx_queues = 0;
 721         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 722                 fs_tx_queue_release(dev->data->tx_queues[i]);
 723                 dev->data->tx_queues[i] = NULL;
 724         }
 725         dev->data->nb_tx_queues = 0;
 726 }
 727
 728 static void
 729 fs_promiscuous_enable(struct rte_eth_dev *dev)
 730 {
 731         struct sub_device *sdev;
 732         uint8_t i;
 733
 734         fs_lock(dev, 0);
 735         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 736                 rte_eth_promiscuous_enable(PORT_ID(sdev));
 737         fs_unlock(dev, 0);
 738 }
 739
 740 static void
 741 fs_promiscuous_disable(struct rte_eth_dev *dev)
 742 {
 743         struct sub_device *sdev;
 744         uint8_t i;
 745
 746         fs_lock(dev, 0);
 747         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 748                 rte_eth_promiscuous_disable(PORT_ID(sdev));
 749         fs_unlock(dev, 0);
 750 }
 751
 752 static void
 753 fs_allmulticast_enable(struct rte_eth_dev *dev)
 754 {
 755         struct sub_device *sdev;
 756         uint8_t i;
 757
 758         fs_lock(dev, 0);
 759         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 760                 rte_eth_allmulticast_enable(PORT_ID(sdev));
 761         fs_unlock(dev, 0);
 762 }
 763
 764 static void
 765 fs_allmulticast_disable(struct rte_eth_dev *dev)
 766 {
 767         struct sub_device *sdev;
 768         uint8_t i;
 769
 770         fs_lock(dev, 0);
 771         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 772                 rte_eth_allmulticast_disable(PORT_ID(sdev));
 773         fs_unlock(dev, 0);
 774 }
 775
 776 static int
 777 fs_link_update(struct rte_eth_dev *dev,
 778                 int wait_to_complete)
 779 {
 780         struct sub_device *sdev;
 781         uint8_t i;
 782         int ret;
 783
 784         fs_lock(dev, 0);
 785         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 786                 DEBUG("Calling link_update on sub_device %d", i);
 787                 ret = (SUBOPS(sdev, link_update))(ETH(sdev), wait_to_complete);
 788                 if (ret && ret != -1 && sdev->remove == 0 &&
 789                     rte_eth_dev_is_removed(PORT_ID(sdev)) == 0) {
 790                         ERROR("Link update failed for sub_device %d with error %d",
 791                               i, ret);
 792                         fs_unlock(dev, 0);
 793                         return ret;
 794                 }
 795         }
 796         if (TX_SUBDEV(dev)) {
 797                 struct rte_eth_link *l1;
 798                 struct rte_eth_link *l2;
 799
 800                 l1 = &dev->data->dev_link;
 801                 l2 = &ETH(TX_SUBDEV(dev))->data->dev_link;
 802                 if (memcmp(l1, l2, sizeof(*l1))) {
 803                         *l1 = *l2;
 804                         fs_unlock(dev, 0);
 805                         return 0;
 806                 }
 807         }
 808         fs_unlock(dev, 0);
 809         return -1;
 810 }
 811
 812 static int
 813 fs_stats_get(struct rte_eth_dev *dev,
 814              struct rte_eth_stats *stats)
 815 {
 816         struct rte_eth_stats backup;
 817         struct sub_device *sdev;
 818         uint8_t i;
 819         int ret;
 820
 821         fs_lock(dev, 0);
 822         rte_memcpy(stats, &PRIV(dev)->stats_accumulator, sizeof(*stats));
 823         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 824                 struct rte_eth_stats *snapshot = &sdev->stats_snapshot.stats;
 825                 uint64_t *timestamp = &sdev->stats_snapshot.timestamp;
 826
 827                 rte_memcpy(&backup, snapshot, sizeof(backup));
 828                 ret = rte_eth_stats_get(PORT_ID(sdev), snapshot);
 829                 if (ret) {
 830                         if (!fs_err(sdev, ret)) {
 831                                 rte_memcpy(snapshot, &backup, sizeof(backup));
 832                                 goto inc;
 833                         }
 834                         ERROR("Operation rte_eth_stats_get failed for sub_device %d with error %d",
 835                                   i, ret);
 836                         *timestamp = 0;
 837                         fs_unlock(dev, 0);
 838                         return ret;
 839                 }
 840                 *timestamp = rte_rdtsc();
 841 inc:
 842                 failsafe_stats_increment(stats, snapshot);
 843         }
 844         fs_unlock(dev, 0);
 845         return 0;
 846 }
 847
 848 static void
 849 fs_stats_reset(struct rte_eth_dev *dev)
 850 {
 851         struct sub_device *sdev;
 852         uint8_t i;
 853
 854         fs_lock(dev, 0);
 855         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 856                 rte_eth_stats_reset(PORT_ID(sdev));
 857                 memset(&sdev->stats_snapshot, 0, sizeof(struct rte_eth_stats));
 858         }
 859         memset(&PRIV(dev)->stats_accumulator, 0, sizeof(struct rte_eth_stats));
 860         fs_unlock(dev, 0);
 861 }
 862
 863 /**
 864  * Fail-safe dev_infos_get rules:
 865  *
 866  * No sub_device:
 867  *   Numerables:
 868  *      Use the maximum possible values for any field, so as not
 869  *      to impede any further configuration effort.
 870  *   Capabilities:
 871  *      Limits capabilities to those that are understood by the
 872  *      fail-safe PMD. This understanding stems from the fail-safe
 873  *      being capable of verifying that the related capability is
 874  *      expressed within the device configuration (struct rte_eth_conf).
 875  *
 876  * At least one probed sub_device:
 877  *   Numerables:
 878  *      Uses values from the active probed sub_device
 879  *      The rationale here is that if any sub_device is less capable
 880  *      (for example concerning the number of queues) than the active
 881  *      sub_device, then its subsequent configuration will fail.
 882  *      It is impossible to foresee this failure when the failing sub_device
 883  *      is supposed to be plugged-in later on, so the configuration process
 884  *      is the single point of failure and error reporting.
 885  *   Capabilities:
 886  *      Uses a logical AND of RX capabilities among
 887  *      all sub_devices and the default capabilities.
 888  *      Uses a logical AND of TX capabilities among
 889  *      the active probed sub_device and the default capabilities.
 890  *
 891  */
 892 static void
 893 fs_dev_infos_get(struct rte_eth_dev *dev,
 894                   struct rte_eth_dev_info *infos)
 895 {
 896         struct sub_device *sdev;
 897         uint8_t i;
 898
 899         sdev = TX_SUBDEV(dev);
 900         if (sdev == NULL) {
 901                 DEBUG("No probed device, using default infos");
 902                 rte_memcpy(&PRIV(dev)->infos, &default_infos,
 903                            sizeof(default_infos));
 904         } else {
 905                 uint64_t rx_offload_capa;
 906                 uint64_t rxq_offload_capa;
 907                 uint64_t rss_hf_offload_capa;
 908
 909                 rx_offload_capa = default_infos.rx_offload_capa;
 910                 rxq_offload_capa = default_infos.rx_queue_offload_capa;
 911                 rss_hf_offload_capa = default_infos.flow_type_rss_offloads;
 912                 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) {
 913                         rte_eth_dev_info_get(PORT_ID(sdev),
 914                                         &PRIV(dev)->infos);
 915                         rx_offload_capa &= PRIV(dev)->infos.rx_offload_capa;
 916                         rxq_offload_capa &=
 917                                         PRIV(dev)->infos.rx_queue_offload_capa;
 918                         rss_hf_offload_capa &=
 919                                         PRIV(dev)->infos.flow_type_rss_offloads;
 920                 }
 921                 sdev = TX_SUBDEV(dev);
 922                 rte_eth_dev_info_get(PORT_ID(sdev), &PRIV(dev)->infos);
 923                 PRIV(dev)->infos.rx_offload_capa = rx_offload_capa;
 924                 PRIV(dev)->infos.rx_queue_offload_capa = rxq_offload_capa;
 925                 PRIV(dev)->infos.flow_type_rss_offloads = rss_hf_offload_capa;
 926                 PRIV(dev)->infos.tx_offload_capa &=
 927                                         default_infos.tx_offload_capa;
 928                 PRIV(dev)->infos.tx_queue_offload_capa &=
 929                                         default_infos.tx_queue_offload_capa;
 930         }
 931         rte_memcpy(infos, &PRIV(dev)->infos, sizeof(*infos));
 932 }
 933
 934 static const uint32_t *
 935 fs_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 936 {
 937         struct sub_device *sdev;
 938         struct rte_eth_dev *edev;
 939         const uint32_t *ret;
 940
 941         fs_lock(dev, 0);
 942         sdev = TX_SUBDEV(dev);
 943         if (sdev == NULL) {
 944                 ret = NULL;
 945                 goto unlock;
 946         }
 947         edev = ETH(sdev);
 948         /* ENOTSUP: counts as no supported ptypes */
 949         if (SUBOPS(sdev, dev_supported_ptypes_get) == NULL) {
 950                 ret = NULL;
 951                 goto unlock;
 952         }
 953         /*
 954          * The API does not permit to do a clean AND of all ptypes,
 955          * It is also incomplete by design and we do not really care
 956          * to have a best possible value in this context.
 957          * We just return the ptypes of the device of highest
 958          * priority, usually the PREFERRED device.
 959          */
 960         ret = SUBOPS(sdev, dev_supported_ptypes_get)(edev);
 961 unlock:
 962         fs_unlock(dev, 0);
 963         return ret;
 964 }
 965
 966 static int
 967 fs_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 968 {
 969         struct sub_device *sdev;
 970         uint8_t i;
 971         int ret;
 972
 973         fs_lock(dev, 0);
 974         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 975                 DEBUG("Calling rte_eth_dev_set_mtu on sub_device %d", i);
 976                 ret = rte_eth_dev_set_mtu(PORT_ID(sdev), mtu);
 977                 if ((ret = fs_err(sdev, ret))) {
 978                         ERROR("Operation rte_eth_dev_set_mtu failed for sub_device %d with error %d",
 979                               i, ret);
 980                         fs_unlock(dev, 0);
 981                         return ret;
 982                 }
 983         }
 984         fs_unlock(dev, 0);
 985         return 0;
 986 }
 987
 988 static int
 989 fs_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 990 {
 991         struct sub_device *sdev;
 992         uint8_t i;
 993         int ret;
 994
 995         fs_lock(dev, 0);
 996         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 997                 DEBUG("Calling rte_eth_dev_vlan_filter on sub_device %d", i);
 998                 ret = rte_eth_dev_vlan_filter(PORT_ID(sdev), vlan_id, on);
 999                 if ((ret = fs_err(sdev, ret))) {
1000                         ERROR("Operation rte_eth_dev_vlan_filter failed for sub_device %d"
1001                               " with error %d", i, ret);
1002                         fs_unlock(dev, 0);
1003                         return ret;
1004                 }
1005         }
1006         fs_unlock(dev, 0);
1007         return 0;
1008 }
1009
1010 static int
1011 fs_flow_ctrl_get(struct rte_eth_dev *dev,
1012                 struct rte_eth_fc_conf *fc_conf)
1013 {
1014         struct sub_device *sdev;
1015         int ret;
1016
1017         fs_lock(dev, 0);
1018         sdev = TX_SUBDEV(dev);
1019         if (sdev == NULL) {
1020                 ret = 0;
1021                 goto unlock;
1022         }
1023         if (SUBOPS(sdev, flow_ctrl_get) == NULL) {
1024                 ret = -ENOTSUP;
1025                 goto unlock;
1026         }
1027         ret = SUBOPS(sdev, flow_ctrl_get)(ETH(sdev), fc_conf);
1028 unlock:
1029         fs_unlock(dev, 0);
1030         return ret;
1031 }
1032
1033 static int
1034 fs_flow_ctrl_set(struct rte_eth_dev *dev,
1035                 struct rte_eth_fc_conf *fc_conf)
1036 {
1037         struct sub_device *sdev;
1038         uint8_t i;
1039         int ret;
1040
1041         fs_lock(dev, 0);
1042         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
1043                 DEBUG("Calling rte_eth_dev_flow_ctrl_set on sub_device %d", i);
1044                 ret = rte_eth_dev_flow_ctrl_set(PORT_ID(sdev), fc_conf);
1045                 if ((ret = fs_err(sdev, ret))) {
1046                         ERROR("Operation rte_eth_dev_flow_ctrl_set failed for sub_device %d"
1047                               " with error %d", i, ret);
1048                         fs_unlock(dev, 0);
1049                         return ret;
1050                 }
1051         }
1052         fs_unlock(dev, 0);
1053         return 0;
1054 }
1055
1056 static void
1057 fs_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
1058 {
1059         struct sub_device *sdev;
1060         uint8_t i;
1061
1062         fs_lock(dev, 0);
1063         /* No check: already done within the rte_eth_dev_mac_addr_remove
1064          * call for the fail-safe device.
1065          */
1066         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
1067                 rte_eth_dev_mac_addr_remove(PORT_ID(sdev),
1068                                 &dev->data->mac_addrs[index]);
1069         PRIV(dev)->mac_addr_pool[index] = 0;
1070         fs_unlock(dev, 0);
1071 }
1072
1073 static int
1074 fs_mac_addr_add(struct rte_eth_dev *dev,
1075                 struct ether_addr *mac_addr,
1076                 uint32_t index,
1077                 uint32_t vmdq)
1078 {
1079         struct sub_device *sdev;
1080         int ret;
1081         uint8_t i;
1082
1083         RTE_ASSERT(index < FAILSAFE_MAX_ETHADDR);
1084         fs_lock(dev, 0);
1085         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
1086                 ret = rte_eth_dev_mac_addr_add(PORT_ID(sdev), mac_addr, vmdq);
1087                 if ((ret = fs_err(sdev, ret))) {
1088                         ERROR("Operation rte_eth_dev_mac_addr_add failed for sub_device %"
1089                               PRIu8 " with error %d", i, ret);
1090                         fs_unlock(dev, 0);
1091                         return ret;
1092                 }
1093         }
1094         if (index >= PRIV(dev)->nb_mac_addr) {
1095                 DEBUG("Growing mac_addrs array");
1096                 PRIV(dev)->nb_mac_addr = index;
1097         }
1098         PRIV(dev)->mac_addr_pool[index] = vmdq;
1099         fs_unlock(dev, 0);
1100         return 0;
1101 }
1102
1103 static int
1104 fs_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
1105 {
1106         struct sub_device *sdev;
1107         uint8_t i;
1108         int ret;
1109
1110         fs_lock(dev, 0);
1111         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
1112                 ret = rte_eth_dev_default_mac_addr_set(PORT_ID(sdev), mac_addr);
1113                 ret = fs_err(sdev, ret);
1114                 if (ret) {
1115                         ERROR("Operation rte_eth_dev_mac_addr_set failed for sub_device %d with error %d",
1116                                 i, ret);
1117                         fs_unlock(dev, 0);
1118                         return ret;
1119                 }
1120         }
1121         fs_unlock(dev, 0);
1122
1123         return 0;
1124 }
1125
1126 static int
1127 fs_rss_hash_update(struct rte_eth_dev *dev,
1128                         struct rte_eth_rss_conf *rss_conf)
1129 {
1130         struct sub_device *sdev;
1131         uint8_t i;
1132         int ret;
1133
1134         fs_lock(dev, 0);
1135         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
1136                 ret = rte_eth_dev_rss_hash_update(PORT_ID(sdev), rss_conf);
1137                 ret = fs_err(sdev, ret);
1138                 if (ret) {
1139                         ERROR("Operation rte_eth_dev_rss_hash_update"
1140                                 " failed for sub_device %d with error %d",
1141                                 i, ret);
1142                         fs_unlock(dev, 0);
1143                         return ret;
1144                 }
1145         }
1146         fs_unlock(dev, 0);
1147
1148         return 0;
1149 }
1150
1151 static int
1152 fs_filter_ctrl(struct rte_eth_dev *dev,
1153                 enum rte_filter_type type,
1154                 enum rte_filter_op op,
1155                 void *arg)
1156 {
1157         struct sub_device *sdev;
1158         uint8_t i;
1159         int ret;
1160
1161         if (type == RTE_ETH_FILTER_GENERIC &&
1162             op == RTE_ETH_FILTER_GET) {
1163                 *(const void **)arg = &fs_flow_ops;
1164                 return 0;
1165         }
1166         fs_lock(dev, 0);
1167         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
1168                 DEBUG("Calling rte_eth_dev_filter_ctrl on sub_device %d", i);
1169                 ret = rte_eth_dev_filter_ctrl(PORT_ID(sdev), type, op, arg);
1170                 if ((ret = fs_err(sdev, ret))) {
1171                         ERROR("Operation rte_eth_dev_filter_ctrl failed for sub_device %d"
1172                               " with error %d", i, ret);
1173                         fs_unlock(dev, 0);
1174                         return ret;
1175                 }
1176         }
1177         fs_unlock(dev, 0);
1178         return 0;
1179 }
1180
1181 const struct eth_dev_ops failsafe_ops = {
1182         .dev_configure = fs_dev_configure,
1183         .dev_start = fs_dev_start,
1184         .dev_stop = fs_dev_stop,
1185         .dev_set_link_down = fs_dev_set_link_down,
1186         .dev_set_link_up = fs_dev_set_link_up,
1187         .dev_close = fs_dev_close,
1188         .promiscuous_enable = fs_promiscuous_enable,
1189         .promiscuous_disable = fs_promiscuous_disable,
1190         .allmulticast_enable = fs_allmulticast_enable,
1191         .allmulticast_disable = fs_allmulticast_disable,
1192         .link_update = fs_link_update,
1193         .stats_get = fs_stats_get,
1194         .stats_reset = fs_stats_reset,
1195         .dev_infos_get = fs_dev_infos_get,
1196         .dev_supported_ptypes_get = fs_dev_supported_ptypes_get,
1197         .mtu_set = fs_mtu_set,
1198         .vlan_filter_set = fs_vlan_filter_set,
1199         .rx_queue_start = fs_rx_queue_start,
1200         .rx_queue_stop = fs_rx_queue_stop,
1201         .tx_queue_start = fs_tx_queue_start,
1202         .tx_queue_stop = fs_tx_queue_stop,
1203         .rx_queue_setup = fs_rx_queue_setup,
1204         .tx_queue_setup = fs_tx_queue_setup,
1205         .rx_queue_release = fs_rx_queue_release,
1206         .tx_queue_release = fs_tx_queue_release,
1207         .rx_queue_intr_enable = fs_rx_intr_enable,
1208         .rx_queue_intr_disable = fs_rx_intr_disable,
1209         .flow_ctrl_get = fs_flow_ctrl_get,
1210         .flow_ctrl_set = fs_flow_ctrl_set,
1211         .mac_addr_remove = fs_mac_addr_remove,
1212         .mac_addr_add = fs_mac_addr_add,
1213         .mac_addr_set = fs_mac_addr_set,
1214         .rss_hash_update = fs_rss_hash_update,
1215         .filter_ctrl = fs_filter_ctrl,
1216 };