drivers/net/failsafe/failsafe_ops.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2017 6WIND S.A.
   3  * Copyright 2017 Mellanox.
   4  */
   5
   6 #include <stdbool.h>
   7 #include <stdint.h>
   8 #include <unistd.h>
   9
  10 #include <rte_debug.h>
  11 #include <rte_atomic.h>
  12 #include <rte_ethdev_driver.h>
  13 #include <rte_malloc.h>
  14 #include <rte_flow.h>
  15 #include <rte_cycles.h>
  16
  17 #include "failsafe_private.h"
  18
  19 static struct rte_eth_dev_info default_infos = {
  20         /* Max possible number of elements */
  21         .max_rx_pktlen = UINT32_MAX,
  22         .max_rx_queues = RTE_MAX_QUEUES_PER_PORT,
  23         .max_tx_queues = RTE_MAX_QUEUES_PER_PORT,
  24         .max_mac_addrs = FAILSAFE_MAX_ETHADDR,
  25         .max_hash_mac_addrs = UINT32_MAX,
  26         .max_vfs = UINT16_MAX,
  27         .max_vmdq_pools = UINT16_MAX,
  28         .rx_desc_lim = {
  29                 .nb_max = UINT16_MAX,
  30                 .nb_min = 0,
  31                 .nb_align = 1,
  32                 .nb_seg_max = UINT16_MAX,
  33                 .nb_mtu_seg_max = UINT16_MAX,
  34         },
  35         .tx_desc_lim = {
  36                 .nb_max = UINT16_MAX,
  37                 .nb_min = 0,
  38                 .nb_align = 1,
  39                 .nb_seg_max = UINT16_MAX,
  40                 .nb_mtu_seg_max = UINT16_MAX,
  41         },
  42         /*
  43          * Set of capabilities that can be verified upon
  44          * configuring a sub-device.
  45          */
  46         .rx_offload_capa =
  47                 DEV_RX_OFFLOAD_VLAN_STRIP |
  48                 DEV_RX_OFFLOAD_IPV4_CKSUM |
  49                 DEV_RX_OFFLOAD_UDP_CKSUM |
  50                 DEV_RX_OFFLOAD_TCP_CKSUM |
  51                 DEV_RX_OFFLOAD_TCP_LRO |
  52                 DEV_RX_OFFLOAD_QINQ_STRIP |
  53                 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
  54                 DEV_RX_OFFLOAD_MACSEC_STRIP |
  55                 DEV_RX_OFFLOAD_HEADER_SPLIT |
  56                 DEV_RX_OFFLOAD_VLAN_FILTER |
  57                 DEV_RX_OFFLOAD_VLAN_EXTEND |
  58                 DEV_RX_OFFLOAD_JUMBO_FRAME |
  59                 DEV_RX_OFFLOAD_CRC_STRIP |
  60                 DEV_RX_OFFLOAD_SCATTER |
  61                 DEV_RX_OFFLOAD_TIMESTAMP |
  62                 DEV_RX_OFFLOAD_SECURITY,
  63         .rx_queue_offload_capa =
  64                 DEV_RX_OFFLOAD_VLAN_STRIP |
  65                 DEV_RX_OFFLOAD_IPV4_CKSUM |
  66                 DEV_RX_OFFLOAD_UDP_CKSUM |
  67                 DEV_RX_OFFLOAD_TCP_CKSUM |
  68                 DEV_RX_OFFLOAD_TCP_LRO |
  69                 DEV_RX_OFFLOAD_QINQ_STRIP |
  70                 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
  71                 DEV_RX_OFFLOAD_MACSEC_STRIP |
  72                 DEV_RX_OFFLOAD_HEADER_SPLIT |
  73                 DEV_RX_OFFLOAD_VLAN_FILTER |
  74                 DEV_RX_OFFLOAD_VLAN_EXTEND |
  75                 DEV_RX_OFFLOAD_JUMBO_FRAME |
  76                 DEV_RX_OFFLOAD_CRC_STRIP |
  77                 DEV_RX_OFFLOAD_SCATTER |
  78                 DEV_RX_OFFLOAD_TIMESTAMP |
  79                 DEV_RX_OFFLOAD_SECURITY,
  80         .tx_offload_capa = 0x0,
  81         .flow_type_rss_offloads = 0x0,
  82 };
  83
  84 static int
  85 fs_dev_configure(struct rte_eth_dev *dev)
  86 {
  87         struct sub_device *sdev;
  88         uint64_t supp_tx_offloads;
  89         uint64_t tx_offloads;
  90         uint8_t i;
  91         int ret;
  92
  93         supp_tx_offloads = PRIV(dev)->infos.tx_offload_capa;
  94         tx_offloads = dev->data->dev_conf.txmode.offloads;
  95         if ((tx_offloads & supp_tx_offloads) != tx_offloads) {
  96                 rte_errno = ENOTSUP;
  97                 ERROR("Some Tx offloads are not supported, "
  98                       "requested 0x%" PRIx64 " supported 0x%" PRIx64,
  99                       tx_offloads, supp_tx_offloads);
 100                 return -rte_errno;
 101         }
 102         FOREACH_SUBDEV(sdev, i, dev) {
 103                 int rmv_interrupt = 0;
 104                 int lsc_interrupt = 0;
 105                 int lsc_enabled;
 106
 107                 if (sdev->state != DEV_PROBED)
 108                         continue;
 109
 110                 rmv_interrupt = ETH(sdev)->data->dev_flags &
 111                                 RTE_ETH_DEV_INTR_RMV;
 112                 if (rmv_interrupt) {
 113                         DEBUG("Enabling RMV interrupts for sub_device %d", i);
 114                         dev->data->dev_conf.intr_conf.rmv = 1;
 115                 } else {
 116                         DEBUG("sub_device %d does not support RMV event", i);
 117                 }
 118                 lsc_enabled = dev->data->dev_conf.intr_conf.lsc;
 119                 lsc_interrupt = lsc_enabled &&
 120                                 (ETH(sdev)->data->dev_flags &
 121                                  RTE_ETH_DEV_INTR_LSC);
 122                 if (lsc_interrupt) {
 123                         DEBUG("Enabling LSC interrupts for sub_device %d", i);
 124                         dev->data->dev_conf.intr_conf.lsc = 1;
 125                 } else if (lsc_enabled && !lsc_interrupt) {
 126                         DEBUG("Disabling LSC interrupts for sub_device %d", i);
 127                         dev->data->dev_conf.intr_conf.lsc = 0;
 128                 }
 129                 DEBUG("Configuring sub-device %d", i);
 130                 sdev->remove = 0;
 131                 ret = rte_eth_dev_configure(PORT_ID(sdev),
 132                                         dev->data->nb_rx_queues,
 133                                         dev->data->nb_tx_queues,
 134                                         &dev->data->dev_conf);
 135                 if (ret) {
 136                         if (!fs_err(sdev, ret))
 137                                 continue;
 138                         ERROR("Could not configure sub_device %d", i);
 139                         return ret;
 140                 }
 141                 if (rmv_interrupt) {
 142                         ret = rte_eth_dev_callback_register(PORT_ID(sdev),
 143                                         RTE_ETH_EVENT_INTR_RMV,
 144                                         failsafe_eth_rmv_event_callback,
 145                                         sdev);
 146                         if (ret)
 147                                 WARN("Failed to register RMV callback for sub_device %d",
 148                                      SUB_ID(sdev));
 149                 }
 150                 dev->data->dev_conf.intr_conf.rmv = 0;
 151                 if (lsc_interrupt) {
 152                         ret = rte_eth_dev_callback_register(PORT_ID(sdev),
 153                                                 RTE_ETH_EVENT_INTR_LSC,
 154                                                 failsafe_eth_lsc_event_callback,
 155                                                 dev);
 156                         if (ret)
 157                                 WARN("Failed to register LSC callback for sub_device %d",
 158                                      SUB_ID(sdev));
 159                 }
 160                 dev->data->dev_conf.intr_conf.lsc = lsc_enabled;
 161                 sdev->state = DEV_ACTIVE;
 162         }
 163         if (PRIV(dev)->state < DEV_ACTIVE)
 164                 PRIV(dev)->state = DEV_ACTIVE;
 165         return 0;
 166 }
 167
 168 static int
 169 fs_dev_start(struct rte_eth_dev *dev)
 170 {
 171         struct sub_device *sdev;
 172         uint8_t i;
 173         int ret;
 174
 175         ret = failsafe_rx_intr_install(dev);
 176         if (ret)
 177                 return ret;
 178         FOREACH_SUBDEV(sdev, i, dev) {
 179                 if (sdev->state != DEV_ACTIVE)
 180                         continue;
 181                 DEBUG("Starting sub_device %d", i);
 182                 ret = rte_eth_dev_start(PORT_ID(sdev));
 183                 if (ret) {
 184                         if (!fs_err(sdev, ret))
 185                                 continue;
 186                         return ret;
 187                 }
 188                 ret = failsafe_rx_intr_install_subdevice(sdev);
 189                 if (ret) {
 190                         if (!fs_err(sdev, ret))
 191                                 continue;
 192                         rte_eth_dev_stop(PORT_ID(sdev));
 193                         return ret;
 194                 }
 195                 sdev->state = DEV_STARTED;
 196         }
 197         if (PRIV(dev)->state < DEV_STARTED)
 198                 PRIV(dev)->state = DEV_STARTED;
 199         fs_switch_dev(dev, NULL);
 200         return 0;
 201 }
 202
 203 static void
 204 fs_dev_stop(struct rte_eth_dev *dev)
 205 {
 206         struct sub_device *sdev;
 207         uint8_t i;
 208
 209         PRIV(dev)->state = DEV_STARTED - 1;
 210         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_STARTED) {
 211                 rte_eth_dev_stop(PORT_ID(sdev));
 212                 failsafe_rx_intr_uninstall_subdevice(sdev);
 213                 sdev->state = DEV_STARTED - 1;
 214         }
 215         failsafe_rx_intr_uninstall(dev);
 216 }
 217
 218 static int
 219 fs_dev_set_link_up(struct rte_eth_dev *dev)
 220 {
 221         struct sub_device *sdev;
 222         uint8_t i;
 223         int ret;
 224
 225         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 226                 DEBUG("Calling rte_eth_dev_set_link_up on sub_device %d", i);
 227                 ret = rte_eth_dev_set_link_up(PORT_ID(sdev));
 228                 if ((ret = fs_err(sdev, ret))) {
 229                         ERROR("Operation rte_eth_dev_set_link_up failed for sub_device %d"
 230                               " with error %d", i, ret);
 231                         return ret;
 232                 }
 233         }
 234         return 0;
 235 }
 236
 237 static int
 238 fs_dev_set_link_down(struct rte_eth_dev *dev)
 239 {
 240         struct sub_device *sdev;
 241         uint8_t i;
 242         int ret;
 243
 244         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 245                 DEBUG("Calling rte_eth_dev_set_link_down on sub_device %d", i);
 246                 ret = rte_eth_dev_set_link_down(PORT_ID(sdev));
 247                 if ((ret = fs_err(sdev, ret))) {
 248                         ERROR("Operation rte_eth_dev_set_link_down failed for sub_device %d"
 249                               " with error %d", i, ret);
 250                         return ret;
 251                 }
 252         }
 253         return 0;
 254 }
 255
 256 static void fs_dev_free_queues(struct rte_eth_dev *dev);
 257 static void
 258 fs_dev_close(struct rte_eth_dev *dev)
 259 {
 260         struct sub_device *sdev;
 261         uint8_t i;
 262
 263         failsafe_hotplug_alarm_cancel(dev);
 264         if (PRIV(dev)->state == DEV_STARTED)
 265                 dev->dev_ops->dev_stop(dev);
 266         PRIV(dev)->state = DEV_ACTIVE - 1;
 267         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 268                 DEBUG("Closing sub_device %d", i);
 269                 rte_eth_dev_close(PORT_ID(sdev));
 270                 sdev->state = DEV_ACTIVE - 1;
 271         }
 272         fs_dev_free_queues(dev);
 273 }
 274
 275 static bool
 276 fs_rxq_offloads_valid(struct rte_eth_dev *dev, uint64_t offloads)
 277 {
 278         uint64_t port_offloads;
 279         uint64_t queue_supp_offloads;
 280         uint64_t port_supp_offloads;
 281
 282         port_offloads = dev->data->dev_conf.rxmode.offloads;
 283         queue_supp_offloads = PRIV(dev)->infos.rx_queue_offload_capa;
 284         port_supp_offloads = PRIV(dev)->infos.rx_offload_capa;
 285         if ((offloads & (queue_supp_offloads | port_supp_offloads)) !=
 286              offloads)
 287                 return false;
 288         /* Verify we have no conflict with port offloads */
 289         if ((port_offloads ^ offloads) & port_supp_offloads)
 290                 return false;
 291         return true;
 292 }
 293
 294 static void
 295 fs_rx_queue_release(void *queue)
 296 {
 297         struct rte_eth_dev *dev;
 298         struct sub_device *sdev;
 299         uint8_t i;
 300         struct rxq *rxq;
 301
 302         if (queue == NULL)
 303                 return;
 304         rxq = queue;
 305         if (rxq->event_fd > 0)
 306                 close(rxq->event_fd);
 307         dev = rxq->priv->dev;
 308         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 309                 SUBOPS(sdev, rx_queue_release)
 310                         (ETH(sdev)->data->rx_queues[rxq->qid]);
 311         dev->data->rx_queues[rxq->qid] = NULL;
 312         rte_free(rxq);
 313 }
 314
 315 static int
 316 fs_rx_queue_setup(struct rte_eth_dev *dev,
 317                 uint16_t rx_queue_id,
 318                 uint16_t nb_rx_desc,
 319                 unsigned int socket_id,
 320                 const struct rte_eth_rxconf *rx_conf,
 321                 struct rte_mempool *mb_pool)
 322 {
 323         /*
 324          * FIXME: Add a proper interface in rte_eal_interrupts for
 325          * allocating eventfd as an interrupt vector.
 326          * For the time being, fake as if we are using MSIX interrupts,
 327          * this will cause rte_intr_efd_enable to allocate an eventfd for us.
 328          */
 329         struct rte_intr_handle intr_handle = {
 330                 .type = RTE_INTR_HANDLE_VFIO_MSIX,
 331                 .efds = { -1, },
 332         };
 333         struct sub_device *sdev;
 334         struct rxq *rxq;
 335         uint8_t i;
 336         int ret;
 337
 338         rxq = dev->data->rx_queues[rx_queue_id];
 339         if (rxq != NULL) {
 340                 fs_rx_queue_release(rxq);
 341                 dev->data->rx_queues[rx_queue_id] = NULL;
 342         }
 343         /* Verify application offloads are valid for our port and queue. */
 344         if (fs_rxq_offloads_valid(dev, rx_conf->offloads) == false) {
 345                 rte_errno = ENOTSUP;
 346                 ERROR("Rx queue offloads 0x%" PRIx64
 347                       " don't match port offloads 0x%" PRIx64
 348                       " or supported offloads 0x%" PRIx64,
 349                       rx_conf->offloads,
 350                       dev->data->dev_conf.rxmode.offloads,
 351                       PRIV(dev)->infos.rx_offload_capa |
 352                       PRIV(dev)->infos.rx_queue_offload_capa);
 353                 return -rte_errno;
 354         }
 355         rxq = rte_zmalloc(NULL,
 356                           sizeof(*rxq) +
 357                           sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail,
 358                           RTE_CACHE_LINE_SIZE);
 359         if (rxq == NULL)
 360                 return -ENOMEM;
 361         FOREACH_SUBDEV(sdev, i, dev)
 362                 rte_atomic64_init(&rxq->refcnt[i]);
 363         rxq->qid = rx_queue_id;
 364         rxq->socket_id = socket_id;
 365         rxq->info.mp = mb_pool;
 366         rxq->info.conf = *rx_conf;
 367         rxq->info.nb_desc = nb_rx_desc;
 368         rxq->priv = PRIV(dev);
 369         rxq->sdev = PRIV(dev)->subs;
 370         ret = rte_intr_efd_enable(&intr_handle, 1);
 371         if (ret < 0)
 372                 return ret;
 373         rxq->event_fd = intr_handle.efds[0];
 374         dev->data->rx_queues[rx_queue_id] = rxq;
 375         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 376                 ret = rte_eth_rx_queue_setup(PORT_ID(sdev),
 377                                 rx_queue_id,
 378                                 nb_rx_desc, socket_id,
 379                                 rx_conf, mb_pool);
 380                 if ((ret = fs_err(sdev, ret))) {
 381                         ERROR("RX queue setup failed for sub_device %d", i);
 382                         goto free_rxq;
 383                 }
 384         }
 385         return 0;
 386 free_rxq:
 387         fs_rx_queue_release(rxq);
 388         return ret;
 389 }
 390
 391 static int
 392 fs_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
 393 {
 394         struct rxq *rxq;
 395         struct sub_device *sdev;
 396         uint8_t i;
 397         int ret;
 398         int rc = 0;
 399
 400         if (idx >= dev->data->nb_rx_queues) {
 401                 rte_errno = EINVAL;
 402                 return -rte_errno;
 403         }
 404         rxq = dev->data->rx_queues[idx];
 405         if (rxq == NULL || rxq->event_fd <= 0) {
 406                 rte_errno = EINVAL;
 407                 return -rte_errno;
 408         }
 409         /* Fail if proxy service is nor running. */
 410         if (PRIV(dev)->rxp.sstate != SS_RUNNING) {
 411                 ERROR("failsafe interrupt services are not running");
 412                 rte_errno = EAGAIN;
 413                 return -rte_errno;
 414         }
 415         rxq->enable_events = 1;
 416         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 417                 ret = rte_eth_dev_rx_intr_enable(PORT_ID(sdev), idx);
 418                 ret = fs_err(sdev, ret);
 419                 if (ret)
 420                         rc = ret;
 421         }
 422         if (rc)
 423                 rte_errno = -rc;
 424         return rc;
 425 }
 426
 427 static int
 428 fs_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
 429 {
 430         struct rxq *rxq;
 431         struct sub_device *sdev;
 432         uint64_t u64;
 433         uint8_t i;
 434         int rc = 0;
 435         int ret;
 436
 437         if (idx >= dev->data->nb_rx_queues) {
 438                 rte_errno = EINVAL;
 439                 return -rte_errno;
 440         }
 441         rxq = dev->data->rx_queues[idx];
 442         if (rxq == NULL || rxq->event_fd <= 0) {
 443                 rte_errno = EINVAL;
 444                 return -rte_errno;
 445         }
 446         rxq->enable_events = 0;
 447         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 448                 ret = rte_eth_dev_rx_intr_disable(PORT_ID(sdev), idx);
 449                 ret = fs_err(sdev, ret);
 450                 if (ret)
 451                         rc = ret;
 452         }
 453         /* Clear pending events */
 454         while (read(rxq->event_fd, &u64, sizeof(uint64_t)) >  0)
 455                 ;
 456         if (rc)
 457                 rte_errno = -rc;
 458         return rc;
 459 }
 460
 461 static bool
 462 fs_txq_offloads_valid(struct rte_eth_dev *dev, uint64_t offloads)
 463 {
 464         uint64_t port_offloads;
 465         uint64_t queue_supp_offloads;
 466         uint64_t port_supp_offloads;
 467
 468         port_offloads = dev->data->dev_conf.txmode.offloads;
 469         queue_supp_offloads = PRIV(dev)->infos.tx_queue_offload_capa;
 470         port_supp_offloads = PRIV(dev)->infos.tx_offload_capa;
 471         if ((offloads & (queue_supp_offloads | port_supp_offloads)) !=
 472              offloads)
 473                 return false;
 474         /* Verify we have no conflict with port offloads */
 475         if ((port_offloads ^ offloads) & port_supp_offloads)
 476                 return false;
 477         return true;
 478 }
 479
 480 static void
 481 fs_tx_queue_release(void *queue)
 482 {
 483         struct rte_eth_dev *dev;
 484         struct sub_device *sdev;
 485         uint8_t i;
 486         struct txq *txq;
 487
 488         if (queue == NULL)
 489                 return;
 490         txq = queue;
 491         dev = txq->priv->dev;
 492         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 493                 SUBOPS(sdev, tx_queue_release)
 494                         (ETH(sdev)->data->tx_queues[txq->qid]);
 495         dev->data->tx_queues[txq->qid] = NULL;
 496         rte_free(txq);
 497 }
 498
 499 static int
 500 fs_tx_queue_setup(struct rte_eth_dev *dev,
 501                 uint16_t tx_queue_id,
 502                 uint16_t nb_tx_desc,
 503                 unsigned int socket_id,
 504                 const struct rte_eth_txconf *tx_conf)
 505 {
 506         struct sub_device *sdev;
 507         struct txq *txq;
 508         uint8_t i;
 509         int ret;
 510
 511         txq = dev->data->tx_queues[tx_queue_id];
 512         if (txq != NULL) {
 513                 fs_tx_queue_release(txq);
 514                 dev->data->tx_queues[tx_queue_id] = NULL;
 515         }
 516         /*
 517          * Don't verify queue offloads for applications which
 518          * use the old API.
 519          */
 520         if (tx_conf != NULL &&
 521             (tx_conf->txq_flags & ETH_TXQ_FLAGS_IGNORE) &&
 522             fs_txq_offloads_valid(dev, tx_conf->offloads) == false) {
 523                 rte_errno = ENOTSUP;
 524                 ERROR("Tx queue offloads 0x%" PRIx64
 525                       " don't match port offloads 0x%" PRIx64
 526                       " or supported offloads 0x%" PRIx64,
 527                       tx_conf->offloads,
 528                       dev->data->dev_conf.txmode.offloads,
 529                       PRIV(dev)->infos.tx_offload_capa |
 530                       PRIV(dev)->infos.tx_queue_offload_capa);
 531                 return -rte_errno;
 532         }
 533         txq = rte_zmalloc("ethdev TX queue",
 534                           sizeof(*txq) +
 535                           sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail,
 536                           RTE_CACHE_LINE_SIZE);
 537         if (txq == NULL)
 538                 return -ENOMEM;
 539         FOREACH_SUBDEV(sdev, i, dev)
 540                 rte_atomic64_init(&txq->refcnt[i]);
 541         txq->qid = tx_queue_id;
 542         txq->socket_id = socket_id;
 543         txq->info.conf = *tx_conf;
 544         txq->info.nb_desc = nb_tx_desc;
 545         txq->priv = PRIV(dev);
 546         dev->data->tx_queues[tx_queue_id] = txq;
 547         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 548                 ret = rte_eth_tx_queue_setup(PORT_ID(sdev),
 549                                 tx_queue_id,
 550                                 nb_tx_desc, socket_id,
 551                                 tx_conf);
 552                 if ((ret = fs_err(sdev, ret))) {
 553                         ERROR("TX queue setup failed for sub_device %d", i);
 554                         goto free_txq;
 555                 }
 556         }
 557         return 0;
 558 free_txq:
 559         fs_tx_queue_release(txq);
 560         return ret;
 561 }
 562
 563 static void
 564 fs_dev_free_queues(struct rte_eth_dev *dev)
 565 {
 566         uint16_t i;
 567
 568         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 569                 fs_rx_queue_release(dev->data->rx_queues[i]);
 570                 dev->data->rx_queues[i] = NULL;
 571         }
 572         dev->data->nb_rx_queues = 0;
 573         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 574                 fs_tx_queue_release(dev->data->tx_queues[i]);
 575                 dev->data->tx_queues[i] = NULL;
 576         }
 577         dev->data->nb_tx_queues = 0;
 578 }
 579
 580 static void
 581 fs_promiscuous_enable(struct rte_eth_dev *dev)
 582 {
 583         struct sub_device *sdev;
 584         uint8_t i;
 585
 586         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 587                 rte_eth_promiscuous_enable(PORT_ID(sdev));
 588 }
 589
 590 static void
 591 fs_promiscuous_disable(struct rte_eth_dev *dev)
 592 {
 593         struct sub_device *sdev;
 594         uint8_t i;
 595
 596         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 597                 rte_eth_promiscuous_disable(PORT_ID(sdev));
 598 }
 599
 600 static void
 601 fs_allmulticast_enable(struct rte_eth_dev *dev)
 602 {
 603         struct sub_device *sdev;
 604         uint8_t i;
 605
 606         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 607                 rte_eth_allmulticast_enable(PORT_ID(sdev));
 608 }
 609
 610 static void
 611 fs_allmulticast_disable(struct rte_eth_dev *dev)
 612 {
 613         struct sub_device *sdev;
 614         uint8_t i;
 615
 616         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 617                 rte_eth_allmulticast_disable(PORT_ID(sdev));
 618 }
 619
 620 static int
 621 fs_link_update(struct rte_eth_dev *dev,
 622                 int wait_to_complete)
 623 {
 624         struct sub_device *sdev;
 625         uint8_t i;
 626         int ret;
 627
 628         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 629                 DEBUG("Calling link_update on sub_device %d", i);
 630                 ret = (SUBOPS(sdev, link_update))(ETH(sdev), wait_to_complete);
 631                 if (ret && ret != -1 && sdev->remove == 0 &&
 632                     rte_eth_dev_is_removed(PORT_ID(sdev)) == 0) {
 633                         ERROR("Link update failed for sub_device %d with error %d",
 634                               i, ret);
 635                         return ret;
 636                 }
 637         }
 638         if (TX_SUBDEV(dev)) {
 639                 struct rte_eth_link *l1;
 640                 struct rte_eth_link *l2;
 641
 642                 l1 = &dev->data->dev_link;
 643                 l2 = &ETH(TX_SUBDEV(dev))->data->dev_link;
 644                 if (memcmp(l1, l2, sizeof(*l1))) {
 645                         *l1 = *l2;
 646                         return 0;
 647                 }
 648         }
 649         return -1;
 650 }
 651
 652 static int
 653 fs_stats_get(struct rte_eth_dev *dev,
 654              struct rte_eth_stats *stats)
 655 {
 656         struct rte_eth_stats backup;
 657         struct sub_device *sdev;
 658         uint8_t i;
 659         int ret;
 660
 661         rte_memcpy(stats, &PRIV(dev)->stats_accumulator, sizeof(*stats));
 662         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 663                 struct rte_eth_stats *snapshot = &sdev->stats_snapshot.stats;
 664                 uint64_t *timestamp = &sdev->stats_snapshot.timestamp;
 665
 666                 rte_memcpy(&backup, snapshot, sizeof(backup));
 667                 ret = rte_eth_stats_get(PORT_ID(sdev), snapshot);
 668                 if (ret) {
 669                         if (!fs_err(sdev, ret)) {
 670                                 rte_memcpy(snapshot, &backup, sizeof(backup));
 671                                 goto inc;
 672                         }
 673                         ERROR("Operation rte_eth_stats_get failed for sub_device %d with error %d",
 674                                   i, ret);
 675                         *timestamp = 0;
 676                         return ret;
 677                 }
 678                 *timestamp = rte_rdtsc();
 679 inc:
 680                 failsafe_stats_increment(stats, snapshot);
 681         }
 682         return 0;
 683 }
 684
 685 static void
 686 fs_stats_reset(struct rte_eth_dev *dev)
 687 {
 688         struct sub_device *sdev;
 689         uint8_t i;
 690
 691         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 692                 rte_eth_stats_reset(PORT_ID(sdev));
 693                 memset(&sdev->stats_snapshot, 0, sizeof(struct rte_eth_stats));
 694         }
 695         memset(&PRIV(dev)->stats_accumulator, 0, sizeof(struct rte_eth_stats));
 696 }
 697
 698 /**
 699  * Fail-safe dev_infos_get rules:
 700  *
 701  * No sub_device:
 702  *   Numerables:
 703  *      Use the maximum possible values for any field, so as not
 704  *      to impede any further configuration effort.
 705  *   Capabilities:
 706  *      Limits capabilities to those that are understood by the
 707  *      fail-safe PMD. This understanding stems from the fail-safe
 708  *      being capable of verifying that the related capability is
 709  *      expressed within the device configuration (struct rte_eth_conf).
 710  *
 711  * At least one probed sub_device:
 712  *   Numerables:
 713  *      Uses values from the active probed sub_device
 714  *      The rationale here is that if any sub_device is less capable
 715  *      (for example concerning the number of queues) than the active
 716  *      sub_device, then its subsequent configuration will fail.
 717  *      It is impossible to foresee this failure when the failing sub_device
 718  *      is supposed to be plugged-in later on, so the configuration process
 719  *      is the single point of failure and error reporting.
 720  *   Capabilities:
 721  *      Uses a logical AND of RX capabilities among
 722  *      all sub_devices and the default capabilities.
 723  *      Uses a logical AND of TX capabilities among
 724  *      the active probed sub_device and the default capabilities.
 725  *
 726  */
 727 static void
 728 fs_dev_infos_get(struct rte_eth_dev *dev,
 729                   struct rte_eth_dev_info *infos)
 730 {
 731         struct sub_device *sdev;
 732         uint8_t i;
 733
 734         sdev = TX_SUBDEV(dev);
 735         if (sdev == NULL) {
 736                 DEBUG("No probed device, using default infos");
 737                 rte_memcpy(&PRIV(dev)->infos, &default_infos,
 738                            sizeof(default_infos));
 739         } else {
 740                 uint64_t rx_offload_capa;
 741                 uint64_t rxq_offload_capa;
 742
 743                 rx_offload_capa = default_infos.rx_offload_capa;
 744                 rxq_offload_capa = default_infos.rx_queue_offload_capa;
 745                 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) {
 746                         rte_eth_dev_info_get(PORT_ID(sdev),
 747                                         &PRIV(dev)->infos);
 748                         rx_offload_capa &= PRIV(dev)->infos.rx_offload_capa;
 749                         rxq_offload_capa &=
 750                                         PRIV(dev)->infos.rx_queue_offload_capa;
 751                 }
 752                 sdev = TX_SUBDEV(dev);
 753                 rte_eth_dev_info_get(PORT_ID(sdev), &PRIV(dev)->infos);
 754                 PRIV(dev)->infos.rx_offload_capa = rx_offload_capa;
 755                 PRIV(dev)->infos.rx_queue_offload_capa = rxq_offload_capa;
 756                 PRIV(dev)->infos.tx_offload_capa &=
 757                                         default_infos.tx_offload_capa;
 758                 PRIV(dev)->infos.tx_queue_offload_capa &=
 759                                         default_infos.tx_queue_offload_capa;
 760                 PRIV(dev)->infos.flow_type_rss_offloads &=
 761                                         default_infos.flow_type_rss_offloads;
 762         }
 763         rte_memcpy(infos, &PRIV(dev)->infos, sizeof(*infos));
 764 }
 765
 766 static const uint32_t *
 767 fs_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 768 {
 769         struct sub_device *sdev;
 770         struct rte_eth_dev *edev;
 771
 772         sdev = TX_SUBDEV(dev);
 773         if (sdev == NULL)
 774                 return NULL;
 775         edev = ETH(sdev);
 776         /* ENOTSUP: counts as no supported ptypes */
 777         if (SUBOPS(sdev, dev_supported_ptypes_get) == NULL)
 778                 return NULL;
 779         /*
 780          * The API does not permit to do a clean AND of all ptypes,
 781          * It is also incomplete by design and we do not really care
 782          * to have a best possible value in this context.
 783          * We just return the ptypes of the device of highest
 784          * priority, usually the PREFERRED device.
 785          */
 786         return SUBOPS(sdev, dev_supported_ptypes_get)(edev);
 787 }
 788
 789 static int
 790 fs_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 791 {
 792         struct sub_device *sdev;
 793         uint8_t i;
 794         int ret;
 795
 796         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 797                 DEBUG("Calling rte_eth_dev_set_mtu on sub_device %d", i);
 798                 ret = rte_eth_dev_set_mtu(PORT_ID(sdev), mtu);
 799                 if ((ret = fs_err(sdev, ret))) {
 800                         ERROR("Operation rte_eth_dev_set_mtu failed for sub_device %d with error %d",
 801                               i, ret);
 802                         return ret;
 803                 }
 804         }
 805         return 0;
 806 }
 807
 808 static int
 809 fs_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 810 {
 811         struct sub_device *sdev;
 812         uint8_t i;
 813         int ret;
 814
 815         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 816                 DEBUG("Calling rte_eth_dev_vlan_filter on sub_device %d", i);
 817                 ret = rte_eth_dev_vlan_filter(PORT_ID(sdev), vlan_id, on);
 818                 if ((ret = fs_err(sdev, ret))) {
 819                         ERROR("Operation rte_eth_dev_vlan_filter failed for sub_device %d"
 820                               " with error %d", i, ret);
 821                         return ret;
 822                 }
 823         }
 824         return 0;
 825 }
 826
 827 static int
 828 fs_flow_ctrl_get(struct rte_eth_dev *dev,
 829                 struct rte_eth_fc_conf *fc_conf)
 830 {
 831         struct sub_device *sdev;
 832
 833         sdev = TX_SUBDEV(dev);
 834         if (sdev == NULL)
 835                 return 0;
 836         if (SUBOPS(sdev, flow_ctrl_get) == NULL)
 837                 return -ENOTSUP;
 838         return SUBOPS(sdev, flow_ctrl_get)(ETH(sdev), fc_conf);
 839 }
 840
 841 static int
 842 fs_flow_ctrl_set(struct rte_eth_dev *dev,
 843                 struct rte_eth_fc_conf *fc_conf)
 844 {
 845         struct sub_device *sdev;
 846         uint8_t i;
 847         int ret;
 848
 849         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 850                 DEBUG("Calling rte_eth_dev_flow_ctrl_set on sub_device %d", i);
 851                 ret = rte_eth_dev_flow_ctrl_set(PORT_ID(sdev), fc_conf);
 852                 if ((ret = fs_err(sdev, ret))) {
 853                         ERROR("Operation rte_eth_dev_flow_ctrl_set failed for sub_device %d"
 854                               " with error %d", i, ret);
 855                         return ret;
 856                 }
 857         }
 858         return 0;
 859 }
 860
 861 static void
 862 fs_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
 863 {
 864         struct sub_device *sdev;
 865         uint8_t i;
 866
 867         /* No check: already done within the rte_eth_dev_mac_addr_remove
 868          * call for the fail-safe device.
 869          */
 870         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 871                 rte_eth_dev_mac_addr_remove(PORT_ID(sdev),
 872                                 &dev->data->mac_addrs[index]);
 873         PRIV(dev)->mac_addr_pool[index] = 0;
 874 }
 875
 876 static int
 877 fs_mac_addr_add(struct rte_eth_dev *dev,
 878                 struct ether_addr *mac_addr,
 879                 uint32_t index,
 880                 uint32_t vmdq)
 881 {
 882         struct sub_device *sdev;
 883         int ret;
 884         uint8_t i;
 885
 886         RTE_ASSERT(index < FAILSAFE_MAX_ETHADDR);
 887         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 888                 ret = rte_eth_dev_mac_addr_add(PORT_ID(sdev), mac_addr, vmdq);
 889                 if ((ret = fs_err(sdev, ret))) {
 890                         ERROR("Operation rte_eth_dev_mac_addr_add failed for sub_device %"
 891                               PRIu8 " with error %d", i, ret);
 892                         return ret;
 893                 }
 894         }
 895         if (index >= PRIV(dev)->nb_mac_addr) {
 896                 DEBUG("Growing mac_addrs array");
 897                 PRIV(dev)->nb_mac_addr = index;
 898         }
 899         PRIV(dev)->mac_addr_pool[index] = vmdq;
 900         return 0;
 901 }
 902
 903 static void
 904 fs_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
 905 {
 906         struct sub_device *sdev;
 907         uint8_t i;
 908
 909         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 910                 rte_eth_dev_default_mac_addr_set(PORT_ID(sdev), mac_addr);
 911 }
 912
 913 static int
 914 fs_filter_ctrl(struct rte_eth_dev *dev,
 915                 enum rte_filter_type type,
 916                 enum rte_filter_op op,
 917                 void *arg)
 918 {
 919         struct sub_device *sdev;
 920         uint8_t i;
 921         int ret;
 922
 923         if (type == RTE_ETH_FILTER_GENERIC &&
 924             op == RTE_ETH_FILTER_GET) {
 925                 *(const void **)arg = &fs_flow_ops;
 926                 return 0;
 927         }
 928         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 929                 DEBUG("Calling rte_eth_dev_filter_ctrl on sub_device %d", i);
 930                 ret = rte_eth_dev_filter_ctrl(PORT_ID(sdev), type, op, arg);
 931                 if ((ret = fs_err(sdev, ret))) {
 932                         ERROR("Operation rte_eth_dev_filter_ctrl failed for sub_device %d"
 933                               " with error %d", i, ret);
 934                         return ret;
 935                 }
 936         }
 937         return 0;
 938 }
 939
 940 const struct eth_dev_ops failsafe_ops = {
 941         .dev_configure = fs_dev_configure,
 942         .dev_start = fs_dev_start,
 943         .dev_stop = fs_dev_stop,
 944         .dev_set_link_down = fs_dev_set_link_down,
 945         .dev_set_link_up = fs_dev_set_link_up,
 946         .dev_close = fs_dev_close,
 947         .promiscuous_enable = fs_promiscuous_enable,
 948         .promiscuous_disable = fs_promiscuous_disable,
 949         .allmulticast_enable = fs_allmulticast_enable,
 950         .allmulticast_disable = fs_allmulticast_disable,
 951         .link_update = fs_link_update,
 952         .stats_get = fs_stats_get,
 953         .stats_reset = fs_stats_reset,
 954         .dev_infos_get = fs_dev_infos_get,
 955         .dev_supported_ptypes_get = fs_dev_supported_ptypes_get,
 956         .mtu_set = fs_mtu_set,
 957         .vlan_filter_set = fs_vlan_filter_set,
 958         .rx_queue_setup = fs_rx_queue_setup,
 959         .tx_queue_setup = fs_tx_queue_setup,
 960         .rx_queue_release = fs_rx_queue_release,
 961         .tx_queue_release = fs_tx_queue_release,
 962         .rx_queue_intr_enable = fs_rx_intr_enable,
 963         .rx_queue_intr_disable = fs_rx_intr_disable,
 964         .flow_ctrl_get = fs_flow_ctrl_get,
 965         .flow_ctrl_set = fs_flow_ctrl_set,
 966         .mac_addr_remove = fs_mac_addr_remove,
 967         .mac_addr_add = fs_mac_addr_add,
 968         .mac_addr_set = fs_mac_addr_set,
 969         .filter_ctrl = fs_filter_ctrl,
 970 };