drivers/net/failsafe/failsafe_ops.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2017 6WIND S.A.
   3  * Copyright 2017 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stdbool.h>
   7 #include <stdint.h>
   8 #include <unistd.h>
   9
  10 #include <rte_debug.h>
  11 #include <rte_atomic.h>
  12 #include <rte_ethdev_driver.h>
  13 #include <rte_malloc.h>
  14 #include <rte_flow.h>
  15 #include <rte_cycles.h>
  16 #include <rte_ethdev.h>
  17
  18 #include "failsafe_private.h"
  19
  20 static struct rte_eth_dev_info default_infos = {
  21         /* Max possible number of elements */
  22         .max_rx_pktlen = UINT32_MAX,
  23         .max_rx_queues = RTE_MAX_QUEUES_PER_PORT,
  24         .max_tx_queues = RTE_MAX_QUEUES_PER_PORT,
  25         .max_mac_addrs = FAILSAFE_MAX_ETHADDR,
  26         .max_hash_mac_addrs = UINT32_MAX,
  27         .max_vfs = UINT16_MAX,
  28         .max_vmdq_pools = UINT16_MAX,
  29         .rx_desc_lim = {
  30                 .nb_max = UINT16_MAX,
  31                 .nb_min = 0,
  32                 .nb_align = 1,
  33                 .nb_seg_max = UINT16_MAX,
  34                 .nb_mtu_seg_max = UINT16_MAX,
  35         },
  36         .tx_desc_lim = {
  37                 .nb_max = UINT16_MAX,
  38                 .nb_min = 0,
  39                 .nb_align = 1,
  40                 .nb_seg_max = UINT16_MAX,
  41                 .nb_mtu_seg_max = UINT16_MAX,
  42         },
  43         /*
  44          * Set of capabilities that can be verified upon
  45          * configuring a sub-device.
  46          */
  47         .rx_offload_capa =
  48                 DEV_RX_OFFLOAD_VLAN_STRIP |
  49                 DEV_RX_OFFLOAD_IPV4_CKSUM |
  50                 DEV_RX_OFFLOAD_UDP_CKSUM |
  51                 DEV_RX_OFFLOAD_TCP_CKSUM |
  52                 DEV_RX_OFFLOAD_TCP_LRO |
  53                 DEV_RX_OFFLOAD_QINQ_STRIP |
  54                 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
  55                 DEV_RX_OFFLOAD_MACSEC_STRIP |
  56                 DEV_RX_OFFLOAD_HEADER_SPLIT |
  57                 DEV_RX_OFFLOAD_VLAN_FILTER |
  58                 DEV_RX_OFFLOAD_VLAN_EXTEND |
  59                 DEV_RX_OFFLOAD_JUMBO_FRAME |
  60                 DEV_RX_OFFLOAD_SCATTER |
  61                 DEV_RX_OFFLOAD_TIMESTAMP |
  62                 DEV_RX_OFFLOAD_SECURITY,
  63         .rx_queue_offload_capa =
  64                 DEV_RX_OFFLOAD_VLAN_STRIP |
  65                 DEV_RX_OFFLOAD_IPV4_CKSUM |
  66                 DEV_RX_OFFLOAD_UDP_CKSUM |
  67                 DEV_RX_OFFLOAD_TCP_CKSUM |
  68                 DEV_RX_OFFLOAD_TCP_LRO |
  69                 DEV_RX_OFFLOAD_QINQ_STRIP |
  70                 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
  71                 DEV_RX_OFFLOAD_MACSEC_STRIP |
  72                 DEV_RX_OFFLOAD_HEADER_SPLIT |
  73                 DEV_RX_OFFLOAD_VLAN_FILTER |
  74                 DEV_RX_OFFLOAD_VLAN_EXTEND |
  75                 DEV_RX_OFFLOAD_JUMBO_FRAME |
  76                 DEV_RX_OFFLOAD_SCATTER |
  77                 DEV_RX_OFFLOAD_TIMESTAMP |
  78                 DEV_RX_OFFLOAD_SECURITY,
  79         .tx_offload_capa =
  80                 DEV_TX_OFFLOAD_MULTI_SEGS |
  81                 DEV_TX_OFFLOAD_IPV4_CKSUM |
  82                 DEV_TX_OFFLOAD_UDP_CKSUM |
  83                 DEV_TX_OFFLOAD_TCP_CKSUM |
  84                 DEV_TX_OFFLOAD_TCP_TSO,
  85         .flow_type_rss_offloads =
  86                         ETH_RSS_IP |
  87                         ETH_RSS_UDP |
  88                         ETH_RSS_TCP,
  89 };
  90
  91 static int
  92 fs_dev_configure(struct rte_eth_dev *dev)
  93 {
  94         struct sub_device *sdev;
  95         uint8_t i;
  96         int ret;
  97
  98         fs_lock(dev, 0);
  99         FOREACH_SUBDEV(sdev, i, dev) {
 100                 int rmv_interrupt = 0;
 101                 int lsc_interrupt = 0;
 102                 int lsc_enabled;
 103
 104                 if (sdev->state != DEV_PROBED &&
 105                     !(PRIV(dev)->alarm_lock == 0 && sdev->state == DEV_ACTIVE))
 106                         continue;
 107
 108                 rmv_interrupt = ETH(sdev)->data->dev_flags &
 109                                 RTE_ETH_DEV_INTR_RMV;
 110                 if (rmv_interrupt) {
 111                         DEBUG("Enabling RMV interrupts for sub_device %d", i);
 112                         dev->data->dev_conf.intr_conf.rmv = 1;
 113                 } else {
 114                         DEBUG("sub_device %d does not support RMV event", i);
 115                 }
 116                 lsc_enabled = dev->data->dev_conf.intr_conf.lsc;
 117                 lsc_interrupt = lsc_enabled &&
 118                                 (ETH(sdev)->data->dev_flags &
 119                                  RTE_ETH_DEV_INTR_LSC);
 120                 if (lsc_interrupt) {
 121                         DEBUG("Enabling LSC interrupts for sub_device %d", i);
 122                         dev->data->dev_conf.intr_conf.lsc = 1;
 123                 } else if (lsc_enabled && !lsc_interrupt) {
 124                         DEBUG("Disabling LSC interrupts for sub_device %d", i);
 125                         dev->data->dev_conf.intr_conf.lsc = 0;
 126                 }
 127                 DEBUG("Configuring sub-device %d", i);
 128                 ret = rte_eth_dev_configure(PORT_ID(sdev),
 129                                         dev->data->nb_rx_queues,
 130                                         dev->data->nb_tx_queues,
 131                                         &dev->data->dev_conf);
 132                 if (ret) {
 133                         if (!fs_err(sdev, ret))
 134                                 continue;
 135                         ERROR("Could not configure sub_device %d", i);
 136                         fs_unlock(dev, 0);
 137                         return ret;
 138                 }
 139                 if (rmv_interrupt && sdev->rmv_callback == 0) {
 140                         ret = rte_eth_dev_callback_register(PORT_ID(sdev),
 141                                         RTE_ETH_EVENT_INTR_RMV,
 142                                         failsafe_eth_rmv_event_callback,
 143                                         sdev);
 144                         if (ret)
 145                                 WARN("Failed to register RMV callback for sub_device %d",
 146                                      SUB_ID(sdev));
 147                         else
 148                                 sdev->rmv_callback = 1;
 149                 }
 150                 dev->data->dev_conf.intr_conf.rmv = 0;
 151                 if (lsc_interrupt && sdev->lsc_callback == 0) {
 152                         ret = rte_eth_dev_callback_register(PORT_ID(sdev),
 153                                                 RTE_ETH_EVENT_INTR_LSC,
 154                                                 failsafe_eth_lsc_event_callback,
 155                                                 dev);
 156                         if (ret)
 157                                 WARN("Failed to register LSC callback for sub_device %d",
 158                                      SUB_ID(sdev));
 159                         else
 160                                 sdev->lsc_callback = 1;
 161                 }
 162                 dev->data->dev_conf.intr_conf.lsc = lsc_enabled;
 163                 sdev->state = DEV_ACTIVE;
 164         }
 165         if (PRIV(dev)->state < DEV_ACTIVE)
 166                 PRIV(dev)->state = DEV_ACTIVE;
 167         fs_unlock(dev, 0);
 168         return 0;
 169 }
 170
 171 static void
 172 fs_set_queues_state_start(struct rte_eth_dev *dev)
 173 {
 174         struct rxq *rxq;
 175         struct txq *txq;
 176         uint16_t i;
 177
 178         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 179                 rxq = dev->data->rx_queues[i];
 180                 if (!rxq->info.conf.rx_deferred_start)
 181                         dev->data->rx_queue_state[i] =
 182                                                 RTE_ETH_QUEUE_STATE_STARTED;
 183         }
 184         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 185                 txq = dev->data->tx_queues[i];
 186                 if (!txq->info.conf.tx_deferred_start)
 187                         dev->data->tx_queue_state[i] =
 188                                                 RTE_ETH_QUEUE_STATE_STARTED;
 189         }
 190 }
 191
 192 static int
 193 fs_dev_start(struct rte_eth_dev *dev)
 194 {
 195         struct sub_device *sdev;
 196         uint8_t i;
 197         int ret;
 198
 199         fs_lock(dev, 0);
 200         ret = failsafe_rx_intr_install(dev);
 201         if (ret) {
 202                 fs_unlock(dev, 0);
 203                 return ret;
 204         }
 205         FOREACH_SUBDEV(sdev, i, dev) {
 206                 if (sdev->state != DEV_ACTIVE)
 207                         continue;
 208                 DEBUG("Starting sub_device %d", i);
 209                 ret = rte_eth_dev_start(PORT_ID(sdev));
 210                 if (ret) {
 211                         if (!fs_err(sdev, ret))
 212                                 continue;
 213                         fs_unlock(dev, 0);
 214                         return ret;
 215                 }
 216                 ret = failsafe_rx_intr_install_subdevice(sdev);
 217                 if (ret) {
 218                         if (!fs_err(sdev, ret))
 219                                 continue;
 220                         rte_eth_dev_stop(PORT_ID(sdev));
 221                         fs_unlock(dev, 0);
 222                         return ret;
 223                 }
 224                 sdev->state = DEV_STARTED;
 225         }
 226         if (PRIV(dev)->state < DEV_STARTED) {
 227                 PRIV(dev)->state = DEV_STARTED;
 228                 fs_set_queues_state_start(dev);
 229         }
 230         fs_switch_dev(dev, NULL);
 231         fs_unlock(dev, 0);
 232         return 0;
 233 }
 234
 235 static void
 236 fs_set_queues_state_stop(struct rte_eth_dev *dev)
 237 {
 238         uint16_t i;
 239
 240         for (i = 0; i < dev->data->nb_rx_queues; i++)
 241                 dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
 242         for (i = 0; i < dev->data->nb_tx_queues; i++)
 243                 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
 244 }
 245
 246 static void
 247 fs_dev_stop(struct rte_eth_dev *dev)
 248 {
 249         struct sub_device *sdev;
 250         uint8_t i;
 251
 252         fs_lock(dev, 0);
 253         PRIV(dev)->state = DEV_STARTED - 1;
 254         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_STARTED) {
 255                 rte_eth_dev_stop(PORT_ID(sdev));
 256                 failsafe_rx_intr_uninstall_subdevice(sdev);
 257                 sdev->state = DEV_STARTED - 1;
 258         }
 259         failsafe_rx_intr_uninstall(dev);
 260         fs_set_queues_state_stop(dev);
 261         fs_unlock(dev, 0);
 262 }
 263
 264 static int
 265 fs_dev_set_link_up(struct rte_eth_dev *dev)
 266 {
 267         struct sub_device *sdev;
 268         uint8_t i;
 269         int ret;
 270
 271         fs_lock(dev, 0);
 272         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 273                 DEBUG("Calling rte_eth_dev_set_link_up on sub_device %d", i);
 274                 ret = rte_eth_dev_set_link_up(PORT_ID(sdev));
 275                 if ((ret = fs_err(sdev, ret))) {
 276                         ERROR("Operation rte_eth_dev_set_link_up failed for sub_device %d"
 277                               " with error %d", i, ret);
 278                         fs_unlock(dev, 0);
 279                         return ret;
 280                 }
 281         }
 282         fs_unlock(dev, 0);
 283         return 0;
 284 }
 285
 286 static int
 287 fs_dev_set_link_down(struct rte_eth_dev *dev)
 288 {
 289         struct sub_device *sdev;
 290         uint8_t i;
 291         int ret;
 292
 293         fs_lock(dev, 0);
 294         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 295                 DEBUG("Calling rte_eth_dev_set_link_down on sub_device %d", i);
 296                 ret = rte_eth_dev_set_link_down(PORT_ID(sdev));
 297                 if ((ret = fs_err(sdev, ret))) {
 298                         ERROR("Operation rte_eth_dev_set_link_down failed for sub_device %d"
 299                               " with error %d", i, ret);
 300                         fs_unlock(dev, 0);
 301                         return ret;
 302                 }
 303         }
 304         fs_unlock(dev, 0);
 305         return 0;
 306 }
 307
 308 static void fs_dev_free_queues(struct rte_eth_dev *dev);
 309 static void
 310 fs_dev_close(struct rte_eth_dev *dev)
 311 {
 312         struct sub_device *sdev;
 313         uint8_t i;
 314
 315         fs_lock(dev, 0);
 316         failsafe_hotplug_alarm_cancel(dev);
 317         if (PRIV(dev)->state == DEV_STARTED)
 318                 dev->dev_ops->dev_stop(dev);
 319         PRIV(dev)->state = DEV_ACTIVE - 1;
 320         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 321                 DEBUG("Closing sub_device %d", i);
 322                 failsafe_eth_dev_unregister_callbacks(sdev);
 323                 rte_eth_dev_close(PORT_ID(sdev));
 324                 sdev->state = DEV_ACTIVE - 1;
 325         }
 326         fs_dev_free_queues(dev);
 327         fs_unlock(dev, 0);
 328 }
 329
 330 static int
 331 fs_rx_queue_stop(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 332 {
 333         struct sub_device *sdev;
 334         uint8_t i;
 335         int ret;
 336         int err = 0;
 337         bool failure = true;
 338
 339         fs_lock(dev, 0);
 340         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 341                 uint16_t port_id = ETH(sdev)->data->port_id;
 342
 343                 ret = rte_eth_dev_rx_queue_stop(port_id, rx_queue_id);
 344                 ret = fs_err(sdev, ret);
 345                 if (ret) {
 346                         ERROR("Rx queue stop failed for subdevice %d", i);
 347                         err = ret;
 348                 } else {
 349                         failure = false;
 350                 }
 351         }
 352         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 353         fs_unlock(dev, 0);
 354         /* Return 0 in case of at least one successful queue stop */
 355         return (failure) ? err : 0;
 356 }
 357
 358 static int
 359 fs_rx_queue_start(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 360 {
 361         struct sub_device *sdev;
 362         uint8_t i;
 363         int ret;
 364
 365         fs_lock(dev, 0);
 366         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 367                 uint16_t port_id = ETH(sdev)->data->port_id;
 368
 369                 ret = rte_eth_dev_rx_queue_start(port_id, rx_queue_id);
 370                 ret = fs_err(sdev, ret);
 371                 if (ret) {
 372                         ERROR("Rx queue start failed for subdevice %d", i);
 373                         fs_rx_queue_stop(dev, rx_queue_id);
 374                         fs_unlock(dev, 0);
 375                         return ret;
 376                 }
 377         }
 378         dev->data->rx_queue_state[rx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 379         fs_unlock(dev, 0);
 380         return 0;
 381 }
 382
 383 static int
 384 fs_tx_queue_stop(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 385 {
 386         struct sub_device *sdev;
 387         uint8_t i;
 388         int ret;
 389         int err = 0;
 390         bool failure = true;
 391
 392         fs_lock(dev, 0);
 393         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 394                 uint16_t port_id = ETH(sdev)->data->port_id;
 395
 396                 ret = rte_eth_dev_tx_queue_stop(port_id, tx_queue_id);
 397                 ret = fs_err(sdev, ret);
 398                 if (ret) {
 399                         ERROR("Tx queue stop failed for subdevice %d", i);
 400                         err = ret;
 401                 } else {
 402                         failure = false;
 403                 }
 404         }
 405         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 406         fs_unlock(dev, 0);
 407         /* Return 0 in case of at least one successful queue stop */
 408         return (failure) ? err : 0;
 409 }
 410
 411 static int
 412 fs_tx_queue_start(struct rte_eth_dev *dev, uint16_t tx_queue_id)
 413 {
 414         struct sub_device *sdev;
 415         uint8_t i;
 416         int ret;
 417
 418         fs_lock(dev, 0);
 419         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 420                 uint16_t port_id = ETH(sdev)->data->port_id;
 421
 422                 ret = rte_eth_dev_tx_queue_start(port_id, tx_queue_id);
 423                 ret = fs_err(sdev, ret);
 424                 if (ret) {
 425                         ERROR("Tx queue start failed for subdevice %d", i);
 426                         fs_tx_queue_stop(dev, tx_queue_id);
 427                         fs_unlock(dev, 0);
 428                         return ret;
 429                 }
 430         }
 431         dev->data->tx_queue_state[tx_queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 432         fs_unlock(dev, 0);
 433         return 0;
 434 }
 435
 436 static void
 437 fs_rx_queue_release(void *queue)
 438 {
 439         struct rte_eth_dev *dev;
 440         struct sub_device *sdev;
 441         uint8_t i;
 442         struct rxq *rxq;
 443
 444         if (queue == NULL)
 445                 return;
 446         rxq = queue;
 447         dev = rxq->priv->dev;
 448         fs_lock(dev, 0);
 449         if (rxq->event_fd > 0)
 450                 close(rxq->event_fd);
 451         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 452                 if (ETH(sdev)->data->rx_queues != NULL &&
 453                     ETH(sdev)->data->rx_queues[rxq->qid] != NULL) {
 454                         SUBOPS(sdev, rx_queue_release)
 455                                 (ETH(sdev)->data->rx_queues[rxq->qid]);
 456                 }
 457         }
 458         dev->data->rx_queues[rxq->qid] = NULL;
 459         rte_free(rxq);
 460         fs_unlock(dev, 0);
 461 }
 462
 463 static int
 464 fs_rx_queue_setup(struct rte_eth_dev *dev,
 465                 uint16_t rx_queue_id,
 466                 uint16_t nb_rx_desc,
 467                 unsigned int socket_id,
 468                 const struct rte_eth_rxconf *rx_conf,
 469                 struct rte_mempool *mb_pool)
 470 {
 471         /*
 472          * FIXME: Add a proper interface in rte_eal_interrupts for
 473          * allocating eventfd as an interrupt vector.
 474          * For the time being, fake as if we are using MSIX interrupts,
 475          * this will cause rte_intr_efd_enable to allocate an eventfd for us.
 476          */
 477         struct rte_intr_handle intr_handle = {
 478                 .type = RTE_INTR_HANDLE_VFIO_MSIX,
 479                 .efds = { -1, },
 480         };
 481         struct sub_device *sdev;
 482         struct rxq *rxq;
 483         uint8_t i;
 484         int ret;
 485
 486         fs_lock(dev, 0);
 487         if (rx_conf->rx_deferred_start) {
 488                 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) {
 489                         if (SUBOPS(sdev, rx_queue_start) == NULL) {
 490                                 ERROR("Rx queue deferred start is not "
 491                                         "supported for subdevice %d", i);
 492                                 fs_unlock(dev, 0);
 493                                 return -EINVAL;
 494                         }
 495                 }
 496         }
 497         rxq = dev->data->rx_queues[rx_queue_id];
 498         if (rxq != NULL) {
 499                 fs_rx_queue_release(rxq);
 500                 dev->data->rx_queues[rx_queue_id] = NULL;
 501         }
 502         rxq = rte_zmalloc(NULL,
 503                           sizeof(*rxq) +
 504                           sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail,
 505                           RTE_CACHE_LINE_SIZE);
 506         if (rxq == NULL) {
 507                 fs_unlock(dev, 0);
 508                 return -ENOMEM;
 509         }
 510         FOREACH_SUBDEV(sdev, i, dev)
 511                 rte_atomic64_init(&rxq->refcnt[i]);
 512         rxq->qid = rx_queue_id;
 513         rxq->socket_id = socket_id;
 514         rxq->info.mp = mb_pool;
 515         rxq->info.conf = *rx_conf;
 516         rxq->info.nb_desc = nb_rx_desc;
 517         rxq->priv = PRIV(dev);
 518         rxq->sdev = PRIV(dev)->subs;
 519         ret = rte_intr_efd_enable(&intr_handle, 1);
 520         if (ret < 0) {
 521                 fs_unlock(dev, 0);
 522                 return ret;
 523         }
 524         rxq->event_fd = intr_handle.efds[0];
 525         dev->data->rx_queues[rx_queue_id] = rxq;
 526         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 527                 ret = rte_eth_rx_queue_setup(PORT_ID(sdev),
 528                                 rx_queue_id,
 529                                 nb_rx_desc, socket_id,
 530                                 rx_conf, mb_pool);
 531                 if ((ret = fs_err(sdev, ret))) {
 532                         ERROR("RX queue setup failed for sub_device %d", i);
 533                         goto free_rxq;
 534                 }
 535         }
 536         fs_unlock(dev, 0);
 537         return 0;
 538 free_rxq:
 539         fs_rx_queue_release(rxq);
 540         fs_unlock(dev, 0);
 541         return ret;
 542 }
 543
 544 static int
 545 fs_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
 546 {
 547         struct rxq *rxq;
 548         struct sub_device *sdev;
 549         uint8_t i;
 550         int ret;
 551         int rc = 0;
 552
 553         fs_lock(dev, 0);
 554         if (idx >= dev->data->nb_rx_queues) {
 555                 rc = -EINVAL;
 556                 goto unlock;
 557         }
 558         rxq = dev->data->rx_queues[idx];
 559         if (rxq == NULL || rxq->event_fd <= 0) {
 560                 rc = -EINVAL;
 561                 goto unlock;
 562         }
 563         /* Fail if proxy service is nor running. */
 564         if (PRIV(dev)->rxp.sstate != SS_RUNNING) {
 565                 ERROR("failsafe interrupt services are not running");
 566                 rc = -EAGAIN;
 567                 goto unlock;
 568         }
 569         rxq->enable_events = 1;
 570         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 571                 ret = rte_eth_dev_rx_intr_enable(PORT_ID(sdev), idx);
 572                 ret = fs_err(sdev, ret);
 573                 if (ret)
 574                         rc = ret;
 575         }
 576 unlock:
 577         fs_unlock(dev, 0);
 578         if (rc)
 579                 rte_errno = -rc;
 580         return rc;
 581 }
 582
 583 static int
 584 fs_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
 585 {
 586         struct rxq *rxq;
 587         struct sub_device *sdev;
 588         uint64_t u64;
 589         uint8_t i;
 590         int rc = 0;
 591         int ret;
 592
 593         fs_lock(dev, 0);
 594         if (idx >= dev->data->nb_rx_queues) {
 595                 rc = -EINVAL;
 596                 goto unlock;
 597         }
 598         rxq = dev->data->rx_queues[idx];
 599         if (rxq == NULL || rxq->event_fd <= 0) {
 600                 rc = -EINVAL;
 601                 goto unlock;
 602         }
 603         rxq->enable_events = 0;
 604         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 605                 ret = rte_eth_dev_rx_intr_disable(PORT_ID(sdev), idx);
 606                 ret = fs_err(sdev, ret);
 607                 if (ret)
 608                         rc = ret;
 609         }
 610         /* Clear pending events */
 611         while (read(rxq->event_fd, &u64, sizeof(uint64_t)) >  0)
 612                 ;
 613 unlock:
 614         fs_unlock(dev, 0);
 615         if (rc)
 616                 rte_errno = -rc;
 617         return rc;
 618 }
 619
 620 static void
 621 fs_tx_queue_release(void *queue)
 622 {
 623         struct rte_eth_dev *dev;
 624         struct sub_device *sdev;
 625         uint8_t i;
 626         struct txq *txq;
 627
 628         if (queue == NULL)
 629                 return;
 630         txq = queue;
 631         dev = txq->priv->dev;
 632         fs_lock(dev, 0);
 633         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 634                 if (ETH(sdev)->data->tx_queues != NULL &&
 635                     ETH(sdev)->data->tx_queues[txq->qid] != NULL) {
 636                         SUBOPS(sdev, tx_queue_release)
 637                                 (ETH(sdev)->data->tx_queues[txq->qid]);
 638                 }
 639         }
 640         dev->data->tx_queues[txq->qid] = NULL;
 641         rte_free(txq);
 642         fs_unlock(dev, 0);
 643 }
 644
 645 static int
 646 fs_tx_queue_setup(struct rte_eth_dev *dev,
 647                 uint16_t tx_queue_id,
 648                 uint16_t nb_tx_desc,
 649                 unsigned int socket_id,
 650                 const struct rte_eth_txconf *tx_conf)
 651 {
 652         struct sub_device *sdev;
 653         struct txq *txq;
 654         uint8_t i;
 655         int ret;
 656
 657         fs_lock(dev, 0);
 658         if (tx_conf->tx_deferred_start) {
 659                 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) {
 660                         if (SUBOPS(sdev, tx_queue_start) == NULL) {
 661                                 ERROR("Tx queue deferred start is not "
 662                                         "supported for subdevice %d", i);
 663                                 fs_unlock(dev, 0);
 664                                 return -EINVAL;
 665                         }
 666                 }
 667         }
 668         txq = dev->data->tx_queues[tx_queue_id];
 669         if (txq != NULL) {
 670                 fs_tx_queue_release(txq);
 671                 dev->data->tx_queues[tx_queue_id] = NULL;
 672         }
 673         txq = rte_zmalloc("ethdev TX queue",
 674                           sizeof(*txq) +
 675                           sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail,
 676                           RTE_CACHE_LINE_SIZE);
 677         if (txq == NULL) {
 678                 fs_unlock(dev, 0);
 679                 return -ENOMEM;
 680         }
 681         FOREACH_SUBDEV(sdev, i, dev)
 682                 rte_atomic64_init(&txq->refcnt[i]);
 683         txq->qid = tx_queue_id;
 684         txq->socket_id = socket_id;
 685         txq->info.conf = *tx_conf;
 686         txq->info.nb_desc = nb_tx_desc;
 687         txq->priv = PRIV(dev);
 688         dev->data->tx_queues[tx_queue_id] = txq;
 689         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 690                 ret = rte_eth_tx_queue_setup(PORT_ID(sdev),
 691                                 tx_queue_id,
 692                                 nb_tx_desc, socket_id,
 693                                 tx_conf);
 694                 if ((ret = fs_err(sdev, ret))) {
 695                         ERROR("TX queue setup failed for sub_device %d", i);
 696                         goto free_txq;
 697                 }
 698         }
 699         fs_unlock(dev, 0);
 700         return 0;
 701 free_txq:
 702         fs_tx_queue_release(txq);
 703         fs_unlock(dev, 0);
 704         return ret;
 705 }
 706
 707 static void
 708 fs_dev_free_queues(struct rte_eth_dev *dev)
 709 {
 710         uint16_t i;
 711
 712         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 713                 fs_rx_queue_release(dev->data->rx_queues[i]);
 714                 dev->data->rx_queues[i] = NULL;
 715         }
 716         dev->data->nb_rx_queues = 0;
 717         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 718                 fs_tx_queue_release(dev->data->tx_queues[i]);
 719                 dev->data->tx_queues[i] = NULL;
 720         }
 721         dev->data->nb_tx_queues = 0;
 722 }
 723
 724 static void
 725 fs_promiscuous_enable(struct rte_eth_dev *dev)
 726 {
 727         struct sub_device *sdev;
 728         uint8_t i;
 729
 730         fs_lock(dev, 0);
 731         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 732                 rte_eth_promiscuous_enable(PORT_ID(sdev));
 733         fs_unlock(dev, 0);
 734 }
 735
 736 static void
 737 fs_promiscuous_disable(struct rte_eth_dev *dev)
 738 {
 739         struct sub_device *sdev;
 740         uint8_t i;
 741
 742         fs_lock(dev, 0);
 743         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 744                 rte_eth_promiscuous_disable(PORT_ID(sdev));
 745         fs_unlock(dev, 0);
 746 }
 747
 748 static void
 749 fs_allmulticast_enable(struct rte_eth_dev *dev)
 750 {
 751         struct sub_device *sdev;
 752         uint8_t i;
 753
 754         fs_lock(dev, 0);
 755         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 756                 rte_eth_allmulticast_enable(PORT_ID(sdev));
 757         fs_unlock(dev, 0);
 758 }
 759
 760 static void
 761 fs_allmulticast_disable(struct rte_eth_dev *dev)
 762 {
 763         struct sub_device *sdev;
 764         uint8_t i;
 765
 766         fs_lock(dev, 0);
 767         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 768                 rte_eth_allmulticast_disable(PORT_ID(sdev));
 769         fs_unlock(dev, 0);
 770 }
 771
 772 static int
 773 fs_link_update(struct rte_eth_dev *dev,
 774                 int wait_to_complete)
 775 {
 776         struct sub_device *sdev;
 777         uint8_t i;
 778         int ret;
 779
 780         fs_lock(dev, 0);
 781         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 782                 DEBUG("Calling link_update on sub_device %d", i);
 783                 ret = (SUBOPS(sdev, link_update))(ETH(sdev), wait_to_complete);
 784                 if (ret && ret != -1 && sdev->remove == 0 &&
 785                     rte_eth_dev_is_removed(PORT_ID(sdev)) == 0) {
 786                         ERROR("Link update failed for sub_device %d with error %d",
 787                               i, ret);
 788                         fs_unlock(dev, 0);
 789                         return ret;
 790                 }
 791         }
 792         if (TX_SUBDEV(dev)) {
 793                 struct rte_eth_link *l1;
 794                 struct rte_eth_link *l2;
 795
 796                 l1 = &dev->data->dev_link;
 797                 l2 = &ETH(TX_SUBDEV(dev))->data->dev_link;
 798                 if (memcmp(l1, l2, sizeof(*l1))) {
 799                         *l1 = *l2;
 800                         fs_unlock(dev, 0);
 801                         return 0;
 802                 }
 803         }
 804         fs_unlock(dev, 0);
 805         return -1;
 806 }
 807
 808 static int
 809 fs_stats_get(struct rte_eth_dev *dev,
 810              struct rte_eth_stats *stats)
 811 {
 812         struct rte_eth_stats backup;
 813         struct sub_device *sdev;
 814         uint8_t i;
 815         int ret;
 816
 817         fs_lock(dev, 0);
 818         rte_memcpy(stats, &PRIV(dev)->stats_accumulator, sizeof(*stats));
 819         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 820                 struct rte_eth_stats *snapshot = &sdev->stats_snapshot.stats;
 821                 uint64_t *timestamp = &sdev->stats_snapshot.timestamp;
 822
 823                 rte_memcpy(&backup, snapshot, sizeof(backup));
 824                 ret = rte_eth_stats_get(PORT_ID(sdev), snapshot);
 825                 if (ret) {
 826                         if (!fs_err(sdev, ret)) {
 827                                 rte_memcpy(snapshot, &backup, sizeof(backup));
 828                                 goto inc;
 829                         }
 830                         ERROR("Operation rte_eth_stats_get failed for sub_device %d with error %d",
 831                                   i, ret);
 832                         *timestamp = 0;
 833                         fs_unlock(dev, 0);
 834                         return ret;
 835                 }
 836                 *timestamp = rte_rdtsc();
 837 inc:
 838                 failsafe_stats_increment(stats, snapshot);
 839         }
 840         fs_unlock(dev, 0);
 841         return 0;
 842 }
 843
 844 static void
 845 fs_stats_reset(struct rte_eth_dev *dev)
 846 {
 847         struct sub_device *sdev;
 848         uint8_t i;
 849
 850         fs_lock(dev, 0);
 851         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 852                 rte_eth_stats_reset(PORT_ID(sdev));
 853                 memset(&sdev->stats_snapshot, 0, sizeof(struct rte_eth_stats));
 854         }
 855         memset(&PRIV(dev)->stats_accumulator, 0, sizeof(struct rte_eth_stats));
 856         fs_unlock(dev, 0);
 857 }
 858
 859 /**
 860  * Fail-safe dev_infos_get rules:
 861  *
 862  * No sub_device:
 863  *   Numerables:
 864  *      Use the maximum possible values for any field, so as not
 865  *      to impede any further configuration effort.
 866  *   Capabilities:
 867  *      Limits capabilities to those that are understood by the
 868  *      fail-safe PMD. This understanding stems from the fail-safe
 869  *      being capable of verifying that the related capability is
 870  *      expressed within the device configuration (struct rte_eth_conf).
 871  *
 872  * At least one probed sub_device:
 873  *   Numerables:
 874  *      Uses values from the active probed sub_device
 875  *      The rationale here is that if any sub_device is less capable
 876  *      (for example concerning the number of queues) than the active
 877  *      sub_device, then its subsequent configuration will fail.
 878  *      It is impossible to foresee this failure when the failing sub_device
 879  *      is supposed to be plugged-in later on, so the configuration process
 880  *      is the single point of failure and error reporting.
 881  *   Capabilities:
 882  *      Uses a logical AND of RX capabilities among
 883  *      all sub_devices and the default capabilities.
 884  *      Uses a logical AND of TX capabilities among
 885  *      the active probed sub_device and the default capabilities.
 886  *
 887  */
 888 static void
 889 fs_dev_infos_get(struct rte_eth_dev *dev,
 890                   struct rte_eth_dev_info *infos)
 891 {
 892         struct sub_device *sdev;
 893         uint8_t i;
 894
 895         sdev = TX_SUBDEV(dev);
 896         if (sdev == NULL) {
 897                 DEBUG("No probed device, using default infos");
 898                 rte_memcpy(&PRIV(dev)->infos, &default_infos,
 899                            sizeof(default_infos));
 900         } else {
 901                 uint64_t rx_offload_capa;
 902                 uint64_t rxq_offload_capa;
 903                 uint64_t rss_hf_offload_capa;
 904
 905                 rx_offload_capa = default_infos.rx_offload_capa;
 906                 rxq_offload_capa = default_infos.rx_queue_offload_capa;
 907                 rss_hf_offload_capa = default_infos.flow_type_rss_offloads;
 908                 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) {
 909                         rte_eth_dev_info_get(PORT_ID(sdev),
 910                                         &PRIV(dev)->infos);
 911                         rx_offload_capa &= PRIV(dev)->infos.rx_offload_capa;
 912                         rxq_offload_capa &=
 913                                         PRIV(dev)->infos.rx_queue_offload_capa;
 914                         rss_hf_offload_capa &=
 915                                         PRIV(dev)->infos.flow_type_rss_offloads;
 916                 }
 917                 sdev = TX_SUBDEV(dev);
 918                 rte_eth_dev_info_get(PORT_ID(sdev), &PRIV(dev)->infos);
 919                 PRIV(dev)->infos.rx_offload_capa = rx_offload_capa;
 920                 PRIV(dev)->infos.rx_queue_offload_capa = rxq_offload_capa;
 921                 PRIV(dev)->infos.flow_type_rss_offloads = rss_hf_offload_capa;
 922                 PRIV(dev)->infos.tx_offload_capa &=
 923                                         default_infos.tx_offload_capa;
 924                 PRIV(dev)->infos.tx_queue_offload_capa &=
 925                                         default_infos.tx_queue_offload_capa;
 926         }
 927         rte_memcpy(infos, &PRIV(dev)->infos, sizeof(*infos));
 928 }
 929
 930 static const uint32_t *
 931 fs_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 932 {
 933         struct sub_device *sdev;
 934         struct rte_eth_dev *edev;
 935         const uint32_t *ret;
 936
 937         fs_lock(dev, 0);
 938         sdev = TX_SUBDEV(dev);
 939         if (sdev == NULL) {
 940                 ret = NULL;
 941                 goto unlock;
 942         }
 943         edev = ETH(sdev);
 944         /* ENOTSUP: counts as no supported ptypes */
 945         if (SUBOPS(sdev, dev_supported_ptypes_get) == NULL) {
 946                 ret = NULL;
 947                 goto unlock;
 948         }
 949         /*
 950          * The API does not permit to do a clean AND of all ptypes,
 951          * It is also incomplete by design and we do not really care
 952          * to have a best possible value in this context.
 953          * We just return the ptypes of the device of highest
 954          * priority, usually the PREFERRED device.
 955          */
 956         ret = SUBOPS(sdev, dev_supported_ptypes_get)(edev);
 957 unlock:
 958         fs_unlock(dev, 0);
 959         return ret;
 960 }
 961
 962 static int
 963 fs_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 964 {
 965         struct sub_device *sdev;
 966         uint8_t i;
 967         int ret;
 968
 969         fs_lock(dev, 0);
 970         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 971                 DEBUG("Calling rte_eth_dev_set_mtu on sub_device %d", i);
 972                 ret = rte_eth_dev_set_mtu(PORT_ID(sdev), mtu);
 973                 if ((ret = fs_err(sdev, ret))) {
 974                         ERROR("Operation rte_eth_dev_set_mtu failed for sub_device %d with error %d",
 975                               i, ret);
 976                         fs_unlock(dev, 0);
 977                         return ret;
 978                 }
 979         }
 980         fs_unlock(dev, 0);
 981         return 0;
 982 }
 983
 984 static int
 985 fs_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 986 {
 987         struct sub_device *sdev;
 988         uint8_t i;
 989         int ret;
 990
 991         fs_lock(dev, 0);
 992         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 993                 DEBUG("Calling rte_eth_dev_vlan_filter on sub_device %d", i);
 994                 ret = rte_eth_dev_vlan_filter(PORT_ID(sdev), vlan_id, on);
 995                 if ((ret = fs_err(sdev, ret))) {
 996                         ERROR("Operation rte_eth_dev_vlan_filter failed for sub_device %d"
 997                               " with error %d", i, ret);
 998                         fs_unlock(dev, 0);
 999                         return ret;
1000                 }
1001         }
1002         fs_unlock(dev, 0);
1003         return 0;
1004 }
1005
1006 static int
1007 fs_flow_ctrl_get(struct rte_eth_dev *dev,
1008                 struct rte_eth_fc_conf *fc_conf)
1009 {
1010         struct sub_device *sdev;
1011         int ret;
1012
1013         fs_lock(dev, 0);
1014         sdev = TX_SUBDEV(dev);
1015         if (sdev == NULL) {
1016                 ret = 0;
1017                 goto unlock;
1018         }
1019         if (SUBOPS(sdev, flow_ctrl_get) == NULL) {
1020                 ret = -ENOTSUP;
1021                 goto unlock;
1022         }
1023         ret = SUBOPS(sdev, flow_ctrl_get)(ETH(sdev), fc_conf);
1024 unlock:
1025         fs_unlock(dev, 0);
1026         return ret;
1027 }
1028
1029 static int
1030 fs_flow_ctrl_set(struct rte_eth_dev *dev,
1031                 struct rte_eth_fc_conf *fc_conf)
1032 {
1033         struct sub_device *sdev;
1034         uint8_t i;
1035         int ret;
1036
1037         fs_lock(dev, 0);
1038         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
1039                 DEBUG("Calling rte_eth_dev_flow_ctrl_set on sub_device %d", i);
1040                 ret = rte_eth_dev_flow_ctrl_set(PORT_ID(sdev), fc_conf);
1041                 if ((ret = fs_err(sdev, ret))) {
1042                         ERROR("Operation rte_eth_dev_flow_ctrl_set failed for sub_device %d"
1043                               " with error %d", i, ret);
1044                         fs_unlock(dev, 0);
1045                         return ret;
1046                 }
1047         }
1048         fs_unlock(dev, 0);
1049         return 0;
1050 }
1051
1052 static void
1053 fs_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
1054 {
1055         struct sub_device *sdev;
1056         uint8_t i;
1057
1058         fs_lock(dev, 0);
1059         /* No check: already done within the rte_eth_dev_mac_addr_remove
1060          * call for the fail-safe device.
1061          */
1062         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
1063                 rte_eth_dev_mac_addr_remove(PORT_ID(sdev),
1064                                 &dev->data->mac_addrs[index]);
1065         PRIV(dev)->mac_addr_pool[index] = 0;
1066         fs_unlock(dev, 0);
1067 }
1068
1069 static int
1070 fs_mac_addr_add(struct rte_eth_dev *dev,
1071                 struct ether_addr *mac_addr,
1072                 uint32_t index,
1073                 uint32_t vmdq)
1074 {
1075         struct sub_device *sdev;
1076         int ret;
1077         uint8_t i;
1078
1079         RTE_ASSERT(index < FAILSAFE_MAX_ETHADDR);
1080         fs_lock(dev, 0);
1081         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
1082                 ret = rte_eth_dev_mac_addr_add(PORT_ID(sdev), mac_addr, vmdq);
1083                 if ((ret = fs_err(sdev, ret))) {
1084                         ERROR("Operation rte_eth_dev_mac_addr_add failed for sub_device %"
1085                               PRIu8 " with error %d", i, ret);
1086                         fs_unlock(dev, 0);
1087                         return ret;
1088                 }
1089         }
1090         if (index >= PRIV(dev)->nb_mac_addr) {
1091                 DEBUG("Growing mac_addrs array");
1092                 PRIV(dev)->nb_mac_addr = index;
1093         }
1094         PRIV(dev)->mac_addr_pool[index] = vmdq;
1095         fs_unlock(dev, 0);
1096         return 0;
1097 }
1098
1099 static int
1100 fs_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
1101 {
1102         struct sub_device *sdev;
1103         uint8_t i;
1104         int ret;
1105
1106         fs_lock(dev, 0);
1107         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
1108                 ret = rte_eth_dev_default_mac_addr_set(PORT_ID(sdev), mac_addr);
1109                 ret = fs_err(sdev, ret);
1110                 if (ret) {
1111                         ERROR("Operation rte_eth_dev_mac_addr_set failed for sub_device %d with error %d",
1112                                 i, ret);
1113                         fs_unlock(dev, 0);
1114                         return ret;
1115                 }
1116         }
1117         fs_unlock(dev, 0);
1118
1119         return 0;
1120 }
1121
1122 static int
1123 fs_rss_hash_update(struct rte_eth_dev *dev,
1124                         struct rte_eth_rss_conf *rss_conf)
1125 {
1126         struct sub_device *sdev;
1127         uint8_t i;
1128         int ret;
1129
1130         fs_lock(dev, 0);
1131         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
1132                 ret = rte_eth_dev_rss_hash_update(PORT_ID(sdev), rss_conf);
1133                 ret = fs_err(sdev, ret);
1134                 if (ret) {
1135                         ERROR("Operation rte_eth_dev_rss_hash_update"
1136                                 " failed for sub_device %d with error %d",
1137                                 i, ret);
1138                         fs_unlock(dev, 0);
1139                         return ret;
1140                 }
1141         }
1142         fs_unlock(dev, 0);
1143
1144         return 0;
1145 }
1146
1147 static int
1148 fs_filter_ctrl(struct rte_eth_dev *dev,
1149                 enum rte_filter_type type,
1150                 enum rte_filter_op op,
1151                 void *arg)
1152 {
1153         struct sub_device *sdev;
1154         uint8_t i;
1155         int ret;
1156
1157         if (type == RTE_ETH_FILTER_GENERIC &&
1158             op == RTE_ETH_FILTER_GET) {
1159                 *(const void **)arg = &fs_flow_ops;
1160                 return 0;
1161         }
1162         fs_lock(dev, 0);
1163         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
1164                 DEBUG("Calling rte_eth_dev_filter_ctrl on sub_device %d", i);
1165                 ret = rte_eth_dev_filter_ctrl(PORT_ID(sdev), type, op, arg);
1166                 if ((ret = fs_err(sdev, ret))) {
1167                         ERROR("Operation rte_eth_dev_filter_ctrl failed for sub_device %d"
1168                               " with error %d", i, ret);
1169                         fs_unlock(dev, 0);
1170                         return ret;
1171                 }
1172         }
1173         fs_unlock(dev, 0);
1174         return 0;
1175 }
1176
1177 const struct eth_dev_ops failsafe_ops = {
1178         .dev_configure = fs_dev_configure,
1179         .dev_start = fs_dev_start,
1180         .dev_stop = fs_dev_stop,
1181         .dev_set_link_down = fs_dev_set_link_down,
1182         .dev_set_link_up = fs_dev_set_link_up,
1183         .dev_close = fs_dev_close,
1184         .promiscuous_enable = fs_promiscuous_enable,
1185         .promiscuous_disable = fs_promiscuous_disable,
1186         .allmulticast_enable = fs_allmulticast_enable,
1187         .allmulticast_disable = fs_allmulticast_disable,
1188         .link_update = fs_link_update,
1189         .stats_get = fs_stats_get,
1190         .stats_reset = fs_stats_reset,
1191         .dev_infos_get = fs_dev_infos_get,
1192         .dev_supported_ptypes_get = fs_dev_supported_ptypes_get,
1193         .mtu_set = fs_mtu_set,
1194         .vlan_filter_set = fs_vlan_filter_set,
1195         .rx_queue_start = fs_rx_queue_start,
1196         .rx_queue_stop = fs_rx_queue_stop,
1197         .tx_queue_start = fs_tx_queue_start,
1198         .tx_queue_stop = fs_tx_queue_stop,
1199         .rx_queue_setup = fs_rx_queue_setup,
1200         .tx_queue_setup = fs_tx_queue_setup,
1201         .rx_queue_release = fs_rx_queue_release,
1202         .tx_queue_release = fs_tx_queue_release,
1203         .rx_queue_intr_enable = fs_rx_intr_enable,
1204         .rx_queue_intr_disable = fs_rx_intr_disable,
1205         .flow_ctrl_get = fs_flow_ctrl_get,
1206         .flow_ctrl_set = fs_flow_ctrl_set,
1207         .mac_addr_remove = fs_mac_addr_remove,
1208         .mac_addr_add = fs_mac_addr_add,
1209         .mac_addr_set = fs_mac_addr_set,
1210         .rss_hash_update = fs_rss_hash_update,
1211         .filter_ctrl = fs_filter_ctrl,
1212 };