drivers/net/failsafe/failsafe_ops.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2017 6WIND S.A.
   3  * Copyright 2017 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stdbool.h>
   7 #include <stdint.h>
   8 #include <unistd.h>
   9
  10 #include <rte_debug.h>
  11 #include <rte_atomic.h>
  12 #include <rte_ethdev_driver.h>
  13 #include <rte_malloc.h>
  14 #include <rte_flow.h>
  15 #include <rte_cycles.h>
  16 #include <rte_ethdev.h>
  17
  18 #include "failsafe_private.h"
  19
  20 static struct rte_eth_dev_info default_infos = {
  21         /* Max possible number of elements */
  22         .max_rx_pktlen = UINT32_MAX,
  23         .max_rx_queues = RTE_MAX_QUEUES_PER_PORT,
  24         .max_tx_queues = RTE_MAX_QUEUES_PER_PORT,
  25         .max_mac_addrs = FAILSAFE_MAX_ETHADDR,
  26         .max_hash_mac_addrs = UINT32_MAX,
  27         .max_vfs = UINT16_MAX,
  28         .max_vmdq_pools = UINT16_MAX,
  29         .rx_desc_lim = {
  30                 .nb_max = UINT16_MAX,
  31                 .nb_min = 0,
  32                 .nb_align = 1,
  33                 .nb_seg_max = UINT16_MAX,
  34                 .nb_mtu_seg_max = UINT16_MAX,
  35         },
  36         .tx_desc_lim = {
  37                 .nb_max = UINT16_MAX,
  38                 .nb_min = 0,
  39                 .nb_align = 1,
  40                 .nb_seg_max = UINT16_MAX,
  41                 .nb_mtu_seg_max = UINT16_MAX,
  42         },
  43         /*
  44          * Set of capabilities that can be verified upon
  45          * configuring a sub-device.
  46          */
  47         .rx_offload_capa =
  48                 DEV_RX_OFFLOAD_VLAN_STRIP |
  49                 DEV_RX_OFFLOAD_IPV4_CKSUM |
  50                 DEV_RX_OFFLOAD_UDP_CKSUM |
  51                 DEV_RX_OFFLOAD_TCP_CKSUM |
  52                 DEV_RX_OFFLOAD_TCP_LRO |
  53                 DEV_RX_OFFLOAD_QINQ_STRIP |
  54                 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
  55                 DEV_RX_OFFLOAD_MACSEC_STRIP |
  56                 DEV_RX_OFFLOAD_HEADER_SPLIT |
  57                 DEV_RX_OFFLOAD_VLAN_FILTER |
  58                 DEV_RX_OFFLOAD_VLAN_EXTEND |
  59                 DEV_RX_OFFLOAD_JUMBO_FRAME |
  60                 DEV_RX_OFFLOAD_SCATTER |
  61                 DEV_RX_OFFLOAD_TIMESTAMP |
  62                 DEV_RX_OFFLOAD_SECURITY,
  63         .rx_queue_offload_capa =
  64                 DEV_RX_OFFLOAD_VLAN_STRIP |
  65                 DEV_RX_OFFLOAD_IPV4_CKSUM |
  66                 DEV_RX_OFFLOAD_UDP_CKSUM |
  67                 DEV_RX_OFFLOAD_TCP_CKSUM |
  68                 DEV_RX_OFFLOAD_TCP_LRO |
  69                 DEV_RX_OFFLOAD_QINQ_STRIP |
  70                 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
  71                 DEV_RX_OFFLOAD_MACSEC_STRIP |
  72                 DEV_RX_OFFLOAD_HEADER_SPLIT |
  73                 DEV_RX_OFFLOAD_VLAN_FILTER |
  74                 DEV_RX_OFFLOAD_VLAN_EXTEND |
  75                 DEV_RX_OFFLOAD_JUMBO_FRAME |
  76                 DEV_RX_OFFLOAD_SCATTER |
  77                 DEV_RX_OFFLOAD_TIMESTAMP |
  78                 DEV_RX_OFFLOAD_SECURITY,
  79         .tx_offload_capa =
  80                 DEV_TX_OFFLOAD_MULTI_SEGS |
  81                 DEV_TX_OFFLOAD_IPV4_CKSUM |
  82                 DEV_TX_OFFLOAD_UDP_CKSUM |
  83                 DEV_TX_OFFLOAD_TCP_CKSUM |
  84                 DEV_TX_OFFLOAD_TCP_TSO,
  85         .flow_type_rss_offloads =
  86                         ETH_RSS_IP |
  87                         ETH_RSS_UDP |
  88                         ETH_RSS_TCP,
  89 };
  90
  91 static int
  92 fs_dev_configure(struct rte_eth_dev *dev)
  93 {
  94         struct sub_device *sdev;
  95         uint8_t i;
  96         int ret;
  97
  98         fs_lock(dev, 0);
  99         FOREACH_SUBDEV(sdev, i, dev) {
 100                 int rmv_interrupt = 0;
 101                 int lsc_interrupt = 0;
 102                 int lsc_enabled;
 103
 104                 if (sdev->state != DEV_PROBED &&
 105                     !(PRIV(dev)->alarm_lock == 0 && sdev->state == DEV_ACTIVE))
 106                         continue;
 107
 108                 rmv_interrupt = ETH(sdev)->data->dev_flags &
 109                                 RTE_ETH_DEV_INTR_RMV;
 110                 if (rmv_interrupt) {
 111                         DEBUG("Enabling RMV interrupts for sub_device %d", i);
 112                         dev->data->dev_conf.intr_conf.rmv = 1;
 113                 } else {
 114                         DEBUG("sub_device %d does not support RMV event", i);
 115                 }
 116                 lsc_enabled = dev->data->dev_conf.intr_conf.lsc;
 117                 lsc_interrupt = lsc_enabled &&
 118                                 (ETH(sdev)->data->dev_flags &
 119                                  RTE_ETH_DEV_INTR_LSC);
 120                 if (lsc_interrupt) {
 121                         DEBUG("Enabling LSC interrupts for sub_device %d", i);
 122                         dev->data->dev_conf.intr_conf.lsc = 1;
 123                 } else if (lsc_enabled && !lsc_interrupt) {
 124                         DEBUG("Disabling LSC interrupts for sub_device %d", i);
 125                         dev->data->dev_conf.intr_conf.lsc = 0;
 126                 }
 127                 DEBUG("Configuring sub-device %d", i);
 128                 ret = rte_eth_dev_configure(PORT_ID(sdev),
 129                                         dev->data->nb_rx_queues,
 130                                         dev->data->nb_tx_queues,
 131                                         &dev->data->dev_conf);
 132                 if (ret) {
 133                         if (!fs_err(sdev, ret))
 134                                 continue;
 135                         ERROR("Could not configure sub_device %d", i);
 136                         fs_unlock(dev, 0);
 137                         return ret;
 138                 }
 139                 if (rmv_interrupt && sdev->rmv_callback == 0) {
 140                         ret = rte_eth_dev_callback_register(PORT_ID(sdev),
 141                                         RTE_ETH_EVENT_INTR_RMV,
 142                                         failsafe_eth_rmv_event_callback,
 143                                         sdev);
 144                         if (ret)
 145                                 WARN("Failed to register RMV callback for sub_device %d",
 146                                      SUB_ID(sdev));
 147                         else
 148                                 sdev->rmv_callback = 1;
 149                 }
 150                 dev->data->dev_conf.intr_conf.rmv = 0;
 151                 if (lsc_interrupt && sdev->lsc_callback == 0) {
 152                         ret = rte_eth_dev_callback_register(PORT_ID(sdev),
 153                                                 RTE_ETH_EVENT_INTR_LSC,
 154                                                 failsafe_eth_lsc_event_callback,
 155                                                 dev);
 156                         if (ret)
 157                                 WARN("Failed to register LSC callback for sub_device %d",
 158                                      SUB_ID(sdev));
 159                         else
 160                                 sdev->lsc_callback = 1;
 161                 }
 162                 dev->data->dev_conf.intr_conf.lsc = lsc_enabled;
 163                 sdev->state = DEV_ACTIVE;
 164         }
 165         if (PRIV(dev)->state < DEV_ACTIVE)
 166                 PRIV(dev)->state = DEV_ACTIVE;
 167         fs_unlock(dev, 0);
 168         return 0;
 169 }
 170
 171 static int
 172 fs_dev_start(struct rte_eth_dev *dev)
 173 {
 174         struct sub_device *sdev;
 175         uint8_t i;
 176         int ret;
 177
 178         fs_lock(dev, 0);
 179         ret = failsafe_rx_intr_install(dev);
 180         if (ret) {
 181                 fs_unlock(dev, 0);
 182                 return ret;
 183         }
 184         FOREACH_SUBDEV(sdev, i, dev) {
 185                 if (sdev->state != DEV_ACTIVE)
 186                         continue;
 187                 DEBUG("Starting sub_device %d", i);
 188                 ret = rte_eth_dev_start(PORT_ID(sdev));
 189                 if (ret) {
 190                         if (!fs_err(sdev, ret))
 191                                 continue;
 192                         fs_unlock(dev, 0);
 193                         return ret;
 194                 }
 195                 ret = failsafe_rx_intr_install_subdevice(sdev);
 196                 if (ret) {
 197                         if (!fs_err(sdev, ret))
 198                                 continue;
 199                         rte_eth_dev_stop(PORT_ID(sdev));
 200                         fs_unlock(dev, 0);
 201                         return ret;
 202                 }
 203                 sdev->state = DEV_STARTED;
 204         }
 205         if (PRIV(dev)->state < DEV_STARTED)
 206                 PRIV(dev)->state = DEV_STARTED;
 207         fs_switch_dev(dev, NULL);
 208         fs_unlock(dev, 0);
 209         return 0;
 210 }
 211
 212 static void
 213 fs_dev_stop(struct rte_eth_dev *dev)
 214 {
 215         struct sub_device *sdev;
 216         uint8_t i;
 217
 218         fs_lock(dev, 0);
 219         PRIV(dev)->state = DEV_STARTED - 1;
 220         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_STARTED) {
 221                 rte_eth_dev_stop(PORT_ID(sdev));
 222                 failsafe_rx_intr_uninstall_subdevice(sdev);
 223                 sdev->state = DEV_STARTED - 1;
 224         }
 225         failsafe_rx_intr_uninstall(dev);
 226         fs_unlock(dev, 0);
 227 }
 228
 229 static int
 230 fs_dev_set_link_up(struct rte_eth_dev *dev)
 231 {
 232         struct sub_device *sdev;
 233         uint8_t i;
 234         int ret;
 235
 236         fs_lock(dev, 0);
 237         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 238                 DEBUG("Calling rte_eth_dev_set_link_up on sub_device %d", i);
 239                 ret = rte_eth_dev_set_link_up(PORT_ID(sdev));
 240                 if ((ret = fs_err(sdev, ret))) {
 241                         ERROR("Operation rte_eth_dev_set_link_up failed for sub_device %d"
 242                               " with error %d", i, ret);
 243                         fs_unlock(dev, 0);
 244                         return ret;
 245                 }
 246         }
 247         fs_unlock(dev, 0);
 248         return 0;
 249 }
 250
 251 static int
 252 fs_dev_set_link_down(struct rte_eth_dev *dev)
 253 {
 254         struct sub_device *sdev;
 255         uint8_t i;
 256         int ret;
 257
 258         fs_lock(dev, 0);
 259         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 260                 DEBUG("Calling rte_eth_dev_set_link_down on sub_device %d", i);
 261                 ret = rte_eth_dev_set_link_down(PORT_ID(sdev));
 262                 if ((ret = fs_err(sdev, ret))) {
 263                         ERROR("Operation rte_eth_dev_set_link_down failed for sub_device %d"
 264                               " with error %d", i, ret);
 265                         fs_unlock(dev, 0);
 266                         return ret;
 267                 }
 268         }
 269         fs_unlock(dev, 0);
 270         return 0;
 271 }
 272
 273 static void fs_dev_free_queues(struct rte_eth_dev *dev);
 274 static void
 275 fs_dev_close(struct rte_eth_dev *dev)
 276 {
 277         struct sub_device *sdev;
 278         uint8_t i;
 279
 280         fs_lock(dev, 0);
 281         failsafe_hotplug_alarm_cancel(dev);
 282         if (PRIV(dev)->state == DEV_STARTED)
 283                 dev->dev_ops->dev_stop(dev);
 284         PRIV(dev)->state = DEV_ACTIVE - 1;
 285         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 286                 DEBUG("Closing sub_device %d", i);
 287                 failsafe_eth_dev_unregister_callbacks(sdev);
 288                 rte_eth_dev_close(PORT_ID(sdev));
 289                 sdev->state = DEV_ACTIVE - 1;
 290         }
 291         fs_dev_free_queues(dev);
 292         fs_unlock(dev, 0);
 293 }
 294
 295 static void
 296 fs_rx_queue_release(void *queue)
 297 {
 298         struct rte_eth_dev *dev;
 299         struct sub_device *sdev;
 300         uint8_t i;
 301         struct rxq *rxq;
 302
 303         if (queue == NULL)
 304                 return;
 305         rxq = queue;
 306         dev = rxq->priv->dev;
 307         fs_lock(dev, 0);
 308         if (rxq->event_fd > 0)
 309                 close(rxq->event_fd);
 310         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 311                 if (ETH(sdev)->data->rx_queues != NULL &&
 312                     ETH(sdev)->data->rx_queues[rxq->qid] != NULL) {
 313                         SUBOPS(sdev, rx_queue_release)
 314                                 (ETH(sdev)->data->rx_queues[rxq->qid]);
 315                 }
 316         }
 317         dev->data->rx_queues[rxq->qid] = NULL;
 318         rte_free(rxq);
 319         fs_unlock(dev, 0);
 320 }
 321
 322 static int
 323 fs_rx_queue_setup(struct rte_eth_dev *dev,
 324                 uint16_t rx_queue_id,
 325                 uint16_t nb_rx_desc,
 326                 unsigned int socket_id,
 327                 const struct rte_eth_rxconf *rx_conf,
 328                 struct rte_mempool *mb_pool)
 329 {
 330         /*
 331          * FIXME: Add a proper interface in rte_eal_interrupts for
 332          * allocating eventfd as an interrupt vector.
 333          * For the time being, fake as if we are using MSIX interrupts,
 334          * this will cause rte_intr_efd_enable to allocate an eventfd for us.
 335          */
 336         struct rte_intr_handle intr_handle = {
 337                 .type = RTE_INTR_HANDLE_VFIO_MSIX,
 338                 .efds = { -1, },
 339         };
 340         struct sub_device *sdev;
 341         struct rxq *rxq;
 342         uint8_t i;
 343         int ret;
 344
 345         if (rx_conf->rx_deferred_start) {
 346                 ERROR("Rx queue deferred start is not supported");
 347                 return -EINVAL;
 348         }
 349
 350         fs_lock(dev, 0);
 351         rxq = dev->data->rx_queues[rx_queue_id];
 352         if (rxq != NULL) {
 353                 fs_rx_queue_release(rxq);
 354                 dev->data->rx_queues[rx_queue_id] = NULL;
 355         }
 356         rxq = rte_zmalloc(NULL,
 357                           sizeof(*rxq) +
 358                           sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail,
 359                           RTE_CACHE_LINE_SIZE);
 360         if (rxq == NULL) {
 361                 fs_unlock(dev, 0);
 362                 return -ENOMEM;
 363         }
 364         FOREACH_SUBDEV(sdev, i, dev)
 365                 rte_atomic64_init(&rxq->refcnt[i]);
 366         rxq->qid = rx_queue_id;
 367         rxq->socket_id = socket_id;
 368         rxq->info.mp = mb_pool;
 369         rxq->info.conf = *rx_conf;
 370         rxq->info.nb_desc = nb_rx_desc;
 371         rxq->priv = PRIV(dev);
 372         rxq->sdev = PRIV(dev)->subs;
 373         ret = rte_intr_efd_enable(&intr_handle, 1);
 374         if (ret < 0) {
 375                 fs_unlock(dev, 0);
 376                 return ret;
 377         }
 378         rxq->event_fd = intr_handle.efds[0];
 379         dev->data->rx_queues[rx_queue_id] = rxq;
 380         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 381                 ret = rte_eth_rx_queue_setup(PORT_ID(sdev),
 382                                 rx_queue_id,
 383                                 nb_rx_desc, socket_id,
 384                                 rx_conf, mb_pool);
 385                 if ((ret = fs_err(sdev, ret))) {
 386                         ERROR("RX queue setup failed for sub_device %d", i);
 387                         goto free_rxq;
 388                 }
 389         }
 390         fs_unlock(dev, 0);
 391         return 0;
 392 free_rxq:
 393         fs_rx_queue_release(rxq);
 394         fs_unlock(dev, 0);
 395         return ret;
 396 }
 397
 398 static int
 399 fs_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
 400 {
 401         struct rxq *rxq;
 402         struct sub_device *sdev;
 403         uint8_t i;
 404         int ret;
 405         int rc = 0;
 406
 407         fs_lock(dev, 0);
 408         if (idx >= dev->data->nb_rx_queues) {
 409                 rc = -EINVAL;
 410                 goto unlock;
 411         }
 412         rxq = dev->data->rx_queues[idx];
 413         if (rxq == NULL || rxq->event_fd <= 0) {
 414                 rc = -EINVAL;
 415                 goto unlock;
 416         }
 417         /* Fail if proxy service is nor running. */
 418         if (PRIV(dev)->rxp.sstate != SS_RUNNING) {
 419                 ERROR("failsafe interrupt services are not running");
 420                 rc = -EAGAIN;
 421                 goto unlock;
 422         }
 423         rxq->enable_events = 1;
 424         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 425                 ret = rte_eth_dev_rx_intr_enable(PORT_ID(sdev), idx);
 426                 ret = fs_err(sdev, ret);
 427                 if (ret)
 428                         rc = ret;
 429         }
 430 unlock:
 431         fs_unlock(dev, 0);
 432         if (rc)
 433                 rte_errno = -rc;
 434         return rc;
 435 }
 436
 437 static int
 438 fs_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
 439 {
 440         struct rxq *rxq;
 441         struct sub_device *sdev;
 442         uint64_t u64;
 443         uint8_t i;
 444         int rc = 0;
 445         int ret;
 446
 447         fs_lock(dev, 0);
 448         if (idx >= dev->data->nb_rx_queues) {
 449                 rc = -EINVAL;
 450                 goto unlock;
 451         }
 452         rxq = dev->data->rx_queues[idx];
 453         if (rxq == NULL || rxq->event_fd <= 0) {
 454                 rc = -EINVAL;
 455                 goto unlock;
 456         }
 457         rxq->enable_events = 0;
 458         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 459                 ret = rte_eth_dev_rx_intr_disable(PORT_ID(sdev), idx);
 460                 ret = fs_err(sdev, ret);
 461                 if (ret)
 462                         rc = ret;
 463         }
 464         /* Clear pending events */
 465         while (read(rxq->event_fd, &u64, sizeof(uint64_t)) >  0)
 466                 ;
 467 unlock:
 468         fs_unlock(dev, 0);
 469         if (rc)
 470                 rte_errno = -rc;
 471         return rc;
 472 }
 473
 474 static void
 475 fs_tx_queue_release(void *queue)
 476 {
 477         struct rte_eth_dev *dev;
 478         struct sub_device *sdev;
 479         uint8_t i;
 480         struct txq *txq;
 481
 482         if (queue == NULL)
 483                 return;
 484         txq = queue;
 485         dev = txq->priv->dev;
 486         fs_lock(dev, 0);
 487         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 488                 if (ETH(sdev)->data->tx_queues != NULL &&
 489                     ETH(sdev)->data->tx_queues[txq->qid] != NULL) {
 490                         SUBOPS(sdev, tx_queue_release)
 491                                 (ETH(sdev)->data->tx_queues[txq->qid]);
 492                 }
 493         }
 494         dev->data->tx_queues[txq->qid] = NULL;
 495         rte_free(txq);
 496         fs_unlock(dev, 0);
 497 }
 498
 499 static int
 500 fs_tx_queue_setup(struct rte_eth_dev *dev,
 501                 uint16_t tx_queue_id,
 502                 uint16_t nb_tx_desc,
 503                 unsigned int socket_id,
 504                 const struct rte_eth_txconf *tx_conf)
 505 {
 506         struct sub_device *sdev;
 507         struct txq *txq;
 508         uint8_t i;
 509         int ret;
 510
 511         if (tx_conf->tx_deferred_start) {
 512                 ERROR("Tx queue deferred start is not supported");
 513                 return -EINVAL;
 514         }
 515
 516         fs_lock(dev, 0);
 517         txq = dev->data->tx_queues[tx_queue_id];
 518         if (txq != NULL) {
 519                 fs_tx_queue_release(txq);
 520                 dev->data->tx_queues[tx_queue_id] = NULL;
 521         }
 522         txq = rte_zmalloc("ethdev TX queue",
 523                           sizeof(*txq) +
 524                           sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail,
 525                           RTE_CACHE_LINE_SIZE);
 526         if (txq == NULL) {
 527                 fs_unlock(dev, 0);
 528                 return -ENOMEM;
 529         }
 530         FOREACH_SUBDEV(sdev, i, dev)
 531                 rte_atomic64_init(&txq->refcnt[i]);
 532         txq->qid = tx_queue_id;
 533         txq->socket_id = socket_id;
 534         txq->info.conf = *tx_conf;
 535         txq->info.nb_desc = nb_tx_desc;
 536         txq->priv = PRIV(dev);
 537         dev->data->tx_queues[tx_queue_id] = txq;
 538         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 539                 ret = rte_eth_tx_queue_setup(PORT_ID(sdev),
 540                                 tx_queue_id,
 541                                 nb_tx_desc, socket_id,
 542                                 tx_conf);
 543                 if ((ret = fs_err(sdev, ret))) {
 544                         ERROR("TX queue setup failed for sub_device %d", i);
 545                         goto free_txq;
 546                 }
 547         }
 548         fs_unlock(dev, 0);
 549         return 0;
 550 free_txq:
 551         fs_tx_queue_release(txq);
 552         fs_unlock(dev, 0);
 553         return ret;
 554 }
 555
 556 static void
 557 fs_dev_free_queues(struct rte_eth_dev *dev)
 558 {
 559         uint16_t i;
 560
 561         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 562                 fs_rx_queue_release(dev->data->rx_queues[i]);
 563                 dev->data->rx_queues[i] = NULL;
 564         }
 565         dev->data->nb_rx_queues = 0;
 566         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 567                 fs_tx_queue_release(dev->data->tx_queues[i]);
 568                 dev->data->tx_queues[i] = NULL;
 569         }
 570         dev->data->nb_tx_queues = 0;
 571 }
 572
 573 static void
 574 fs_promiscuous_enable(struct rte_eth_dev *dev)
 575 {
 576         struct sub_device *sdev;
 577         uint8_t i;
 578
 579         fs_lock(dev, 0);
 580         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 581                 rte_eth_promiscuous_enable(PORT_ID(sdev));
 582         fs_unlock(dev, 0);
 583 }
 584
 585 static void
 586 fs_promiscuous_disable(struct rte_eth_dev *dev)
 587 {
 588         struct sub_device *sdev;
 589         uint8_t i;
 590
 591         fs_lock(dev, 0);
 592         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 593                 rte_eth_promiscuous_disable(PORT_ID(sdev));
 594         fs_unlock(dev, 0);
 595 }
 596
 597 static void
 598 fs_allmulticast_enable(struct rte_eth_dev *dev)
 599 {
 600         struct sub_device *sdev;
 601         uint8_t i;
 602
 603         fs_lock(dev, 0);
 604         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 605                 rte_eth_allmulticast_enable(PORT_ID(sdev));
 606         fs_unlock(dev, 0);
 607 }
 608
 609 static void
 610 fs_allmulticast_disable(struct rte_eth_dev *dev)
 611 {
 612         struct sub_device *sdev;
 613         uint8_t i;
 614
 615         fs_lock(dev, 0);
 616         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 617                 rte_eth_allmulticast_disable(PORT_ID(sdev));
 618         fs_unlock(dev, 0);
 619 }
 620
 621 static int
 622 fs_link_update(struct rte_eth_dev *dev,
 623                 int wait_to_complete)
 624 {
 625         struct sub_device *sdev;
 626         uint8_t i;
 627         int ret;
 628
 629         fs_lock(dev, 0);
 630         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 631                 DEBUG("Calling link_update on sub_device %d", i);
 632                 ret = (SUBOPS(sdev, link_update))(ETH(sdev), wait_to_complete);
 633                 if (ret && ret != -1 && sdev->remove == 0 &&
 634                     rte_eth_dev_is_removed(PORT_ID(sdev)) == 0) {
 635                         ERROR("Link update failed for sub_device %d with error %d",
 636                               i, ret);
 637                         fs_unlock(dev, 0);
 638                         return ret;
 639                 }
 640         }
 641         if (TX_SUBDEV(dev)) {
 642                 struct rte_eth_link *l1;
 643                 struct rte_eth_link *l2;
 644
 645                 l1 = &dev->data->dev_link;
 646                 l2 = &ETH(TX_SUBDEV(dev))->data->dev_link;
 647                 if (memcmp(l1, l2, sizeof(*l1))) {
 648                         *l1 = *l2;
 649                         fs_unlock(dev, 0);
 650                         return 0;
 651                 }
 652         }
 653         fs_unlock(dev, 0);
 654         return -1;
 655 }
 656
 657 static int
 658 fs_stats_get(struct rte_eth_dev *dev,
 659              struct rte_eth_stats *stats)
 660 {
 661         struct rte_eth_stats backup;
 662         struct sub_device *sdev;
 663         uint8_t i;
 664         int ret;
 665
 666         fs_lock(dev, 0);
 667         rte_memcpy(stats, &PRIV(dev)->stats_accumulator, sizeof(*stats));
 668         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 669                 struct rte_eth_stats *snapshot = &sdev->stats_snapshot.stats;
 670                 uint64_t *timestamp = &sdev->stats_snapshot.timestamp;
 671
 672                 rte_memcpy(&backup, snapshot, sizeof(backup));
 673                 ret = rte_eth_stats_get(PORT_ID(sdev), snapshot);
 674                 if (ret) {
 675                         if (!fs_err(sdev, ret)) {
 676                                 rte_memcpy(snapshot, &backup, sizeof(backup));
 677                                 goto inc;
 678                         }
 679                         ERROR("Operation rte_eth_stats_get failed for sub_device %d with error %d",
 680                                   i, ret);
 681                         *timestamp = 0;
 682                         fs_unlock(dev, 0);
 683                         return ret;
 684                 }
 685                 *timestamp = rte_rdtsc();
 686 inc:
 687                 failsafe_stats_increment(stats, snapshot);
 688         }
 689         fs_unlock(dev, 0);
 690         return 0;
 691 }
 692
 693 static void
 694 fs_stats_reset(struct rte_eth_dev *dev)
 695 {
 696         struct sub_device *sdev;
 697         uint8_t i;
 698
 699         fs_lock(dev, 0);
 700         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 701                 rte_eth_stats_reset(PORT_ID(sdev));
 702                 memset(&sdev->stats_snapshot, 0, sizeof(struct rte_eth_stats));
 703         }
 704         memset(&PRIV(dev)->stats_accumulator, 0, sizeof(struct rte_eth_stats));
 705         fs_unlock(dev, 0);
 706 }
 707
 708 /**
 709  * Fail-safe dev_infos_get rules:
 710  *
 711  * No sub_device:
 712  *   Numerables:
 713  *      Use the maximum possible values for any field, so as not
 714  *      to impede any further configuration effort.
 715  *   Capabilities:
 716  *      Limits capabilities to those that are understood by the
 717  *      fail-safe PMD. This understanding stems from the fail-safe
 718  *      being capable of verifying that the related capability is
 719  *      expressed within the device configuration (struct rte_eth_conf).
 720  *
 721  * At least one probed sub_device:
 722  *   Numerables:
 723  *      Uses values from the active probed sub_device
 724  *      The rationale here is that if any sub_device is less capable
 725  *      (for example concerning the number of queues) than the active
 726  *      sub_device, then its subsequent configuration will fail.
 727  *      It is impossible to foresee this failure when the failing sub_device
 728  *      is supposed to be plugged-in later on, so the configuration process
 729  *      is the single point of failure and error reporting.
 730  *   Capabilities:
 731  *      Uses a logical AND of RX capabilities among
 732  *      all sub_devices and the default capabilities.
 733  *      Uses a logical AND of TX capabilities among
 734  *      the active probed sub_device and the default capabilities.
 735  *
 736  */
 737 static void
 738 fs_dev_infos_get(struct rte_eth_dev *dev,
 739                   struct rte_eth_dev_info *infos)
 740 {
 741         struct sub_device *sdev;
 742         uint8_t i;
 743
 744         sdev = TX_SUBDEV(dev);
 745         if (sdev == NULL) {
 746                 DEBUG("No probed device, using default infos");
 747                 rte_memcpy(&PRIV(dev)->infos, &default_infos,
 748                            sizeof(default_infos));
 749         } else {
 750                 uint64_t rx_offload_capa;
 751                 uint64_t rxq_offload_capa;
 752                 uint64_t rss_hf_offload_capa;
 753
 754                 rx_offload_capa = default_infos.rx_offload_capa;
 755                 rxq_offload_capa = default_infos.rx_queue_offload_capa;
 756                 rss_hf_offload_capa = default_infos.flow_type_rss_offloads;
 757                 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) {
 758                         rte_eth_dev_info_get(PORT_ID(sdev),
 759                                         &PRIV(dev)->infos);
 760                         rx_offload_capa &= PRIV(dev)->infos.rx_offload_capa;
 761                         rxq_offload_capa &=
 762                                         PRIV(dev)->infos.rx_queue_offload_capa;
 763                         rss_hf_offload_capa &=
 764                                         PRIV(dev)->infos.flow_type_rss_offloads;
 765                 }
 766                 sdev = TX_SUBDEV(dev);
 767                 rte_eth_dev_info_get(PORT_ID(sdev), &PRIV(dev)->infos);
 768                 PRIV(dev)->infos.rx_offload_capa = rx_offload_capa;
 769                 PRIV(dev)->infos.rx_queue_offload_capa = rxq_offload_capa;
 770                 PRIV(dev)->infos.flow_type_rss_offloads = rss_hf_offload_capa;
 771                 PRIV(dev)->infos.tx_offload_capa &=
 772                                         default_infos.tx_offload_capa;
 773                 PRIV(dev)->infos.tx_queue_offload_capa &=
 774                                         default_infos.tx_queue_offload_capa;
 775         }
 776         rte_memcpy(infos, &PRIV(dev)->infos, sizeof(*infos));
 777 }
 778
 779 static const uint32_t *
 780 fs_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 781 {
 782         struct sub_device *sdev;
 783         struct rte_eth_dev *edev;
 784         const uint32_t *ret;
 785
 786         fs_lock(dev, 0);
 787         sdev = TX_SUBDEV(dev);
 788         if (sdev == NULL) {
 789                 ret = NULL;
 790                 goto unlock;
 791         }
 792         edev = ETH(sdev);
 793         /* ENOTSUP: counts as no supported ptypes */
 794         if (SUBOPS(sdev, dev_supported_ptypes_get) == NULL) {
 795                 ret = NULL;
 796                 goto unlock;
 797         }
 798         /*
 799          * The API does not permit to do a clean AND of all ptypes,
 800          * It is also incomplete by design and we do not really care
 801          * to have a best possible value in this context.
 802          * We just return the ptypes of the device of highest
 803          * priority, usually the PREFERRED device.
 804          */
 805         ret = SUBOPS(sdev, dev_supported_ptypes_get)(edev);
 806 unlock:
 807         fs_unlock(dev, 0);
 808         return ret;
 809 }
 810
 811 static int
 812 fs_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 813 {
 814         struct sub_device *sdev;
 815         uint8_t i;
 816         int ret;
 817
 818         fs_lock(dev, 0);
 819         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 820                 DEBUG("Calling rte_eth_dev_set_mtu on sub_device %d", i);
 821                 ret = rte_eth_dev_set_mtu(PORT_ID(sdev), mtu);
 822                 if ((ret = fs_err(sdev, ret))) {
 823                         ERROR("Operation rte_eth_dev_set_mtu failed for sub_device %d with error %d",
 824                               i, ret);
 825                         fs_unlock(dev, 0);
 826                         return ret;
 827                 }
 828         }
 829         fs_unlock(dev, 0);
 830         return 0;
 831 }
 832
 833 static int
 834 fs_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 835 {
 836         struct sub_device *sdev;
 837         uint8_t i;
 838         int ret;
 839
 840         fs_lock(dev, 0);
 841         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 842                 DEBUG("Calling rte_eth_dev_vlan_filter on sub_device %d", i);
 843                 ret = rte_eth_dev_vlan_filter(PORT_ID(sdev), vlan_id, on);
 844                 if ((ret = fs_err(sdev, ret))) {
 845                         ERROR("Operation rte_eth_dev_vlan_filter failed for sub_device %d"
 846                               " with error %d", i, ret);
 847                         fs_unlock(dev, 0);
 848                         return ret;
 849                 }
 850         }
 851         fs_unlock(dev, 0);
 852         return 0;
 853 }
 854
 855 static int
 856 fs_flow_ctrl_get(struct rte_eth_dev *dev,
 857                 struct rte_eth_fc_conf *fc_conf)
 858 {
 859         struct sub_device *sdev;
 860         int ret;
 861
 862         fs_lock(dev, 0);
 863         sdev = TX_SUBDEV(dev);
 864         if (sdev == NULL) {
 865                 ret = 0;
 866                 goto unlock;
 867         }
 868         if (SUBOPS(sdev, flow_ctrl_get) == NULL) {
 869                 ret = -ENOTSUP;
 870                 goto unlock;
 871         }
 872         ret = SUBOPS(sdev, flow_ctrl_get)(ETH(sdev), fc_conf);
 873 unlock:
 874         fs_unlock(dev, 0);
 875         return ret;
 876 }
 877
 878 static int
 879 fs_flow_ctrl_set(struct rte_eth_dev *dev,
 880                 struct rte_eth_fc_conf *fc_conf)
 881 {
 882         struct sub_device *sdev;
 883         uint8_t i;
 884         int ret;
 885
 886         fs_lock(dev, 0);
 887         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 888                 DEBUG("Calling rte_eth_dev_flow_ctrl_set on sub_device %d", i);
 889                 ret = rte_eth_dev_flow_ctrl_set(PORT_ID(sdev), fc_conf);
 890                 if ((ret = fs_err(sdev, ret))) {
 891                         ERROR("Operation rte_eth_dev_flow_ctrl_set failed for sub_device %d"
 892                               " with error %d", i, ret);
 893                         fs_unlock(dev, 0);
 894                         return ret;
 895                 }
 896         }
 897         fs_unlock(dev, 0);
 898         return 0;
 899 }
 900
 901 static void
 902 fs_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
 903 {
 904         struct sub_device *sdev;
 905         uint8_t i;
 906
 907         fs_lock(dev, 0);
 908         /* No check: already done within the rte_eth_dev_mac_addr_remove
 909          * call for the fail-safe device.
 910          */
 911         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 912                 rte_eth_dev_mac_addr_remove(PORT_ID(sdev),
 913                                 &dev->data->mac_addrs[index]);
 914         PRIV(dev)->mac_addr_pool[index] = 0;
 915         fs_unlock(dev, 0);
 916 }
 917
 918 static int
 919 fs_mac_addr_add(struct rte_eth_dev *dev,
 920                 struct ether_addr *mac_addr,
 921                 uint32_t index,
 922                 uint32_t vmdq)
 923 {
 924         struct sub_device *sdev;
 925         int ret;
 926         uint8_t i;
 927
 928         RTE_ASSERT(index < FAILSAFE_MAX_ETHADDR);
 929         fs_lock(dev, 0);
 930         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 931                 ret = rte_eth_dev_mac_addr_add(PORT_ID(sdev), mac_addr, vmdq);
 932                 if ((ret = fs_err(sdev, ret))) {
 933                         ERROR("Operation rte_eth_dev_mac_addr_add failed for sub_device %"
 934                               PRIu8 " with error %d", i, ret);
 935                         fs_unlock(dev, 0);
 936                         return ret;
 937                 }
 938         }
 939         if (index >= PRIV(dev)->nb_mac_addr) {
 940                 DEBUG("Growing mac_addrs array");
 941                 PRIV(dev)->nb_mac_addr = index;
 942         }
 943         PRIV(dev)->mac_addr_pool[index] = vmdq;
 944         fs_unlock(dev, 0);
 945         return 0;
 946 }
 947
 948 static int
 949 fs_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
 950 {
 951         struct sub_device *sdev;
 952         uint8_t i;
 953         int ret;
 954
 955         fs_lock(dev, 0);
 956         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 957                 ret = rte_eth_dev_default_mac_addr_set(PORT_ID(sdev), mac_addr);
 958                 ret = fs_err(sdev, ret);
 959                 if (ret) {
 960                         ERROR("Operation rte_eth_dev_mac_addr_set failed for sub_device %d with error %d",
 961                                 i, ret);
 962                         fs_unlock(dev, 0);
 963                         return ret;
 964                 }
 965         }
 966         fs_unlock(dev, 0);
 967
 968         return 0;
 969 }
 970
 971 static int
 972 fs_rss_hash_update(struct rte_eth_dev *dev,
 973                         struct rte_eth_rss_conf *rss_conf)
 974 {
 975         struct sub_device *sdev;
 976         uint8_t i;
 977         int ret;
 978
 979         fs_lock(dev, 0);
 980         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 981                 ret = rte_eth_dev_rss_hash_update(PORT_ID(sdev), rss_conf);
 982                 ret = fs_err(sdev, ret);
 983                 if (ret) {
 984                         ERROR("Operation rte_eth_dev_rss_hash_update"
 985                                 " failed for sub_device %d with error %d",
 986                                 i, ret);
 987                         fs_unlock(dev, 0);
 988                         return ret;
 989                 }
 990         }
 991         fs_unlock(dev, 0);
 992
 993         return 0;
 994 }
 995
 996 static int
 997 fs_filter_ctrl(struct rte_eth_dev *dev,
 998                 enum rte_filter_type type,
 999                 enum rte_filter_op op,
1000                 void *arg)
1001 {
1002         struct sub_device *sdev;
1003         uint8_t i;
1004         int ret;
1005
1006         if (type == RTE_ETH_FILTER_GENERIC &&
1007             op == RTE_ETH_FILTER_GET) {
1008                 *(const void **)arg = &fs_flow_ops;
1009                 return 0;
1010         }
1011         fs_lock(dev, 0);
1012         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
1013                 DEBUG("Calling rte_eth_dev_filter_ctrl on sub_device %d", i);
1014                 ret = rte_eth_dev_filter_ctrl(PORT_ID(sdev), type, op, arg);
1015                 if ((ret = fs_err(sdev, ret))) {
1016                         ERROR("Operation rte_eth_dev_filter_ctrl failed for sub_device %d"
1017                               " with error %d", i, ret);
1018                         fs_unlock(dev, 0);
1019                         return ret;
1020                 }
1021         }
1022         fs_unlock(dev, 0);
1023         return 0;
1024 }
1025
1026 const struct eth_dev_ops failsafe_ops = {
1027         .dev_configure = fs_dev_configure,
1028         .dev_start = fs_dev_start,
1029         .dev_stop = fs_dev_stop,
1030         .dev_set_link_down = fs_dev_set_link_down,
1031         .dev_set_link_up = fs_dev_set_link_up,
1032         .dev_close = fs_dev_close,
1033         .promiscuous_enable = fs_promiscuous_enable,
1034         .promiscuous_disable = fs_promiscuous_disable,
1035         .allmulticast_enable = fs_allmulticast_enable,
1036         .allmulticast_disable = fs_allmulticast_disable,
1037         .link_update = fs_link_update,
1038         .stats_get = fs_stats_get,
1039         .stats_reset = fs_stats_reset,
1040         .dev_infos_get = fs_dev_infos_get,
1041         .dev_supported_ptypes_get = fs_dev_supported_ptypes_get,
1042         .mtu_set = fs_mtu_set,
1043         .vlan_filter_set = fs_vlan_filter_set,
1044         .rx_queue_setup = fs_rx_queue_setup,
1045         .tx_queue_setup = fs_tx_queue_setup,
1046         .rx_queue_release = fs_rx_queue_release,
1047         .tx_queue_release = fs_tx_queue_release,
1048         .rx_queue_intr_enable = fs_rx_intr_enable,
1049         .rx_queue_intr_disable = fs_rx_intr_disable,
1050         .flow_ctrl_get = fs_flow_ctrl_get,
1051         .flow_ctrl_set = fs_flow_ctrl_set,
1052         .mac_addr_remove = fs_mac_addr_remove,
1053         .mac_addr_add = fs_mac_addr_add,
1054         .mac_addr_set = fs_mac_addr_set,
1055         .rss_hash_update = fs_rss_hash_update,
1056         .filter_ctrl = fs_filter_ctrl,
1057 };