drivers/net/failsafe/failsafe_ops.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2017 6WIND S.A.
   3  * Copyright 2017 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stdbool.h>
   7 #include <stdint.h>
   8 #include <unistd.h>
   9
  10 #include <rte_debug.h>
  11 #include <rte_atomic.h>
  12 #include <rte_ethdev_driver.h>
  13 #include <rte_malloc.h>
  14 #include <rte_flow.h>
  15 #include <rte_cycles.h>
  16 #include <rte_ethdev.h>
  17
  18 #include "failsafe_private.h"
  19
  20 static struct rte_eth_dev_info default_infos = {
  21         /* Max possible number of elements */
  22         .max_rx_pktlen = UINT32_MAX,
  23         .max_rx_queues = RTE_MAX_QUEUES_PER_PORT,
  24         .max_tx_queues = RTE_MAX_QUEUES_PER_PORT,
  25         .max_mac_addrs = FAILSAFE_MAX_ETHADDR,
  26         .max_hash_mac_addrs = UINT32_MAX,
  27         .max_vfs = UINT16_MAX,
  28         .max_vmdq_pools = UINT16_MAX,
  29         .rx_desc_lim = {
  30                 .nb_max = UINT16_MAX,
  31                 .nb_min = 0,
  32                 .nb_align = 1,
  33                 .nb_seg_max = UINT16_MAX,
  34                 .nb_mtu_seg_max = UINT16_MAX,
  35         },
  36         .tx_desc_lim = {
  37                 .nb_max = UINT16_MAX,
  38                 .nb_min = 0,
  39                 .nb_align = 1,
  40                 .nb_seg_max = UINT16_MAX,
  41                 .nb_mtu_seg_max = UINT16_MAX,
  42         },
  43         /*
  44          * Set of capabilities that can be verified upon
  45          * configuring a sub-device.
  46          */
  47         .rx_offload_capa =
  48                 DEV_RX_OFFLOAD_VLAN_STRIP |
  49                 DEV_RX_OFFLOAD_IPV4_CKSUM |
  50                 DEV_RX_OFFLOAD_UDP_CKSUM |
  51                 DEV_RX_OFFLOAD_TCP_CKSUM |
  52                 DEV_RX_OFFLOAD_TCP_LRO |
  53                 DEV_RX_OFFLOAD_QINQ_STRIP |
  54                 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
  55                 DEV_RX_OFFLOAD_MACSEC_STRIP |
  56                 DEV_RX_OFFLOAD_HEADER_SPLIT |
  57                 DEV_RX_OFFLOAD_VLAN_FILTER |
  58                 DEV_RX_OFFLOAD_VLAN_EXTEND |
  59                 DEV_RX_OFFLOAD_JUMBO_FRAME |
  60                 DEV_RX_OFFLOAD_SCATTER |
  61                 DEV_RX_OFFLOAD_TIMESTAMP |
  62                 DEV_RX_OFFLOAD_SECURITY,
  63         .rx_queue_offload_capa =
  64                 DEV_RX_OFFLOAD_VLAN_STRIP |
  65                 DEV_RX_OFFLOAD_IPV4_CKSUM |
  66                 DEV_RX_OFFLOAD_UDP_CKSUM |
  67                 DEV_RX_OFFLOAD_TCP_CKSUM |
  68                 DEV_RX_OFFLOAD_TCP_LRO |
  69                 DEV_RX_OFFLOAD_QINQ_STRIP |
  70                 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
  71                 DEV_RX_OFFLOAD_MACSEC_STRIP |
  72                 DEV_RX_OFFLOAD_HEADER_SPLIT |
  73                 DEV_RX_OFFLOAD_VLAN_FILTER |
  74                 DEV_RX_OFFLOAD_VLAN_EXTEND |
  75                 DEV_RX_OFFLOAD_JUMBO_FRAME |
  76                 DEV_RX_OFFLOAD_SCATTER |
  77                 DEV_RX_OFFLOAD_TIMESTAMP |
  78                 DEV_RX_OFFLOAD_SECURITY,
  79         .tx_offload_capa =
  80                 DEV_TX_OFFLOAD_MULTI_SEGS |
  81                 DEV_TX_OFFLOAD_IPV4_CKSUM |
  82                 DEV_TX_OFFLOAD_UDP_CKSUM |
  83                 DEV_TX_OFFLOAD_TCP_CKSUM |
  84                 DEV_TX_OFFLOAD_TCP_TSO,
  85         .flow_type_rss_offloads =
  86                         ETH_RSS_IP |
  87                         ETH_RSS_UDP |
  88                         ETH_RSS_TCP,
  89 };
  90
  91 static int
  92 fs_dev_configure(struct rte_eth_dev *dev)
  93 {
  94         struct sub_device *sdev;
  95         uint8_t i;
  96         int ret;
  97
  98         fs_lock(dev, 0);
  99         FOREACH_SUBDEV(sdev, i, dev) {
 100                 int rmv_interrupt = 0;
 101                 int lsc_interrupt = 0;
 102                 int lsc_enabled;
 103
 104                 if (sdev->state != DEV_PROBED &&
 105                     !(PRIV(dev)->alarm_lock == 0 && sdev->state == DEV_ACTIVE))
 106                         continue;
 107
 108                 rmv_interrupt = ETH(sdev)->data->dev_flags &
 109                                 RTE_ETH_DEV_INTR_RMV;
 110                 if (rmv_interrupt) {
 111                         DEBUG("Enabling RMV interrupts for sub_device %d", i);
 112                         dev->data->dev_conf.intr_conf.rmv = 1;
 113                 } else {
 114                         DEBUG("sub_device %d does not support RMV event", i);
 115                 }
 116                 lsc_enabled = dev->data->dev_conf.intr_conf.lsc;
 117                 lsc_interrupt = lsc_enabled &&
 118                                 (ETH(sdev)->data->dev_flags &
 119                                  RTE_ETH_DEV_INTR_LSC);
 120                 if (lsc_interrupt) {
 121                         DEBUG("Enabling LSC interrupts for sub_device %d", i);
 122                         dev->data->dev_conf.intr_conf.lsc = 1;
 123                 } else if (lsc_enabled && !lsc_interrupt) {
 124                         DEBUG("Disabling LSC interrupts for sub_device %d", i);
 125                         dev->data->dev_conf.intr_conf.lsc = 0;
 126                 }
 127                 DEBUG("Configuring sub-device %d", i);
 128                 ret = rte_eth_dev_configure(PORT_ID(sdev),
 129                                         dev->data->nb_rx_queues,
 130                                         dev->data->nb_tx_queues,
 131                                         &dev->data->dev_conf);
 132                 if (ret) {
 133                         if (!fs_err(sdev, ret))
 134                                 continue;
 135                         ERROR("Could not configure sub_device %d", i);
 136                         fs_unlock(dev, 0);
 137                         return ret;
 138                 }
 139                 if (rmv_interrupt && sdev->rmv_callback == 0) {
 140                         ret = rte_eth_dev_callback_register(PORT_ID(sdev),
 141                                         RTE_ETH_EVENT_INTR_RMV,
 142                                         failsafe_eth_rmv_event_callback,
 143                                         sdev);
 144                         if (ret)
 145                                 WARN("Failed to register RMV callback for sub_device %d",
 146                                      SUB_ID(sdev));
 147                         else
 148                                 sdev->rmv_callback = 1;
 149                 }
 150                 dev->data->dev_conf.intr_conf.rmv = 0;
 151                 if (lsc_interrupt && sdev->lsc_callback == 0) {
 152                         ret = rte_eth_dev_callback_register(PORT_ID(sdev),
 153                                                 RTE_ETH_EVENT_INTR_LSC,
 154                                                 failsafe_eth_lsc_event_callback,
 155                                                 dev);
 156                         if (ret)
 157                                 WARN("Failed to register LSC callback for sub_device %d",
 158                                      SUB_ID(sdev));
 159                         else
 160                                 sdev->lsc_callback = 1;
 161                 }
 162                 dev->data->dev_conf.intr_conf.lsc = lsc_enabled;
 163                 sdev->state = DEV_ACTIVE;
 164         }
 165         if (PRIV(dev)->state < DEV_ACTIVE)
 166                 PRIV(dev)->state = DEV_ACTIVE;
 167         fs_unlock(dev, 0);
 168         return 0;
 169 }
 170
 171 static int
 172 fs_dev_start(struct rte_eth_dev *dev)
 173 {
 174         struct sub_device *sdev;
 175         uint8_t i;
 176         int ret;
 177
 178         fs_lock(dev, 0);
 179         ret = failsafe_rx_intr_install(dev);
 180         if (ret) {
 181                 fs_unlock(dev, 0);
 182                 return ret;
 183         }
 184         FOREACH_SUBDEV(sdev, i, dev) {
 185                 if (sdev->state != DEV_ACTIVE)
 186                         continue;
 187                 DEBUG("Starting sub_device %d", i);
 188                 ret = rte_eth_dev_start(PORT_ID(sdev));
 189                 if (ret) {
 190                         if (!fs_err(sdev, ret))
 191                                 continue;
 192                         fs_unlock(dev, 0);
 193                         return ret;
 194                 }
 195                 ret = failsafe_rx_intr_install_subdevice(sdev);
 196                 if (ret) {
 197                         if (!fs_err(sdev, ret))
 198                                 continue;
 199                         rte_eth_dev_stop(PORT_ID(sdev));
 200                         fs_unlock(dev, 0);
 201                         return ret;
 202                 }
 203                 sdev->state = DEV_STARTED;
 204         }
 205         if (PRIV(dev)->state < DEV_STARTED)
 206                 PRIV(dev)->state = DEV_STARTED;
 207         fs_switch_dev(dev, NULL);
 208         fs_unlock(dev, 0);
 209         return 0;
 210 }
 211
 212 static void
 213 fs_dev_stop(struct rte_eth_dev *dev)
 214 {
 215         struct sub_device *sdev;
 216         uint8_t i;
 217
 218         fs_lock(dev, 0);
 219         PRIV(dev)->state = DEV_STARTED - 1;
 220         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_STARTED) {
 221                 rte_eth_dev_stop(PORT_ID(sdev));
 222                 failsafe_rx_intr_uninstall_subdevice(sdev);
 223                 sdev->state = DEV_STARTED - 1;
 224         }
 225         failsafe_rx_intr_uninstall(dev);
 226         fs_unlock(dev, 0);
 227 }
 228
 229 static int
 230 fs_dev_set_link_up(struct rte_eth_dev *dev)
 231 {
 232         struct sub_device *sdev;
 233         uint8_t i;
 234         int ret;
 235
 236         fs_lock(dev, 0);
 237         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 238                 DEBUG("Calling rte_eth_dev_set_link_up on sub_device %d", i);
 239                 ret = rte_eth_dev_set_link_up(PORT_ID(sdev));
 240                 if ((ret = fs_err(sdev, ret))) {
 241                         ERROR("Operation rte_eth_dev_set_link_up failed for sub_device %d"
 242                               " with error %d", i, ret);
 243                         fs_unlock(dev, 0);
 244                         return ret;
 245                 }
 246         }
 247         fs_unlock(dev, 0);
 248         return 0;
 249 }
 250
 251 static int
 252 fs_dev_set_link_down(struct rte_eth_dev *dev)
 253 {
 254         struct sub_device *sdev;
 255         uint8_t i;
 256         int ret;
 257
 258         fs_lock(dev, 0);
 259         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 260                 DEBUG("Calling rte_eth_dev_set_link_down on sub_device %d", i);
 261                 ret = rte_eth_dev_set_link_down(PORT_ID(sdev));
 262                 if ((ret = fs_err(sdev, ret))) {
 263                         ERROR("Operation rte_eth_dev_set_link_down failed for sub_device %d"
 264                               " with error %d", i, ret);
 265                         fs_unlock(dev, 0);
 266                         return ret;
 267                 }
 268         }
 269         fs_unlock(dev, 0);
 270         return 0;
 271 }
 272
 273 static void fs_dev_free_queues(struct rte_eth_dev *dev);
 274 static void
 275 fs_dev_close(struct rte_eth_dev *dev)
 276 {
 277         struct sub_device *sdev;
 278         uint8_t i;
 279
 280         fs_lock(dev, 0);
 281         failsafe_hotplug_alarm_cancel(dev);
 282         if (PRIV(dev)->state == DEV_STARTED)
 283                 dev->dev_ops->dev_stop(dev);
 284         PRIV(dev)->state = DEV_ACTIVE - 1;
 285         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 286                 DEBUG("Closing sub_device %d", i);
 287                 failsafe_eth_dev_unregister_callbacks(sdev);
 288                 rte_eth_dev_close(PORT_ID(sdev));
 289                 sdev->state = DEV_ACTIVE - 1;
 290         }
 291         fs_dev_free_queues(dev);
 292         fs_unlock(dev, 0);
 293 }
 294
 295 static void
 296 fs_rx_queue_release(void *queue)
 297 {
 298         struct rte_eth_dev *dev;
 299         struct sub_device *sdev;
 300         uint8_t i;
 301         struct rxq *rxq;
 302
 303         if (queue == NULL)
 304                 return;
 305         rxq = queue;
 306         dev = rxq->priv->dev;
 307         fs_lock(dev, 0);
 308         if (rxq->event_fd > 0)
 309                 close(rxq->event_fd);
 310         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 311                 if (ETH(sdev)->data->rx_queues != NULL &&
 312                     ETH(sdev)->data->rx_queues[rxq->qid] != NULL) {
 313                         SUBOPS(sdev, rx_queue_release)
 314                                 (ETH(sdev)->data->rx_queues[rxq->qid]);
 315                 }
 316         }
 317         dev->data->rx_queues[rxq->qid] = NULL;
 318         rte_free(rxq);
 319         fs_unlock(dev, 0);
 320 }
 321
 322 static int
 323 fs_rx_queue_setup(struct rte_eth_dev *dev,
 324                 uint16_t rx_queue_id,
 325                 uint16_t nb_rx_desc,
 326                 unsigned int socket_id,
 327                 const struct rte_eth_rxconf *rx_conf,
 328                 struct rte_mempool *mb_pool)
 329 {
 330         /*
 331          * FIXME: Add a proper interface in rte_eal_interrupts for
 332          * allocating eventfd as an interrupt vector.
 333          * For the time being, fake as if we are using MSIX interrupts,
 334          * this will cause rte_intr_efd_enable to allocate an eventfd for us.
 335          */
 336         struct rte_intr_handle intr_handle = {
 337                 .type = RTE_INTR_HANDLE_VFIO_MSIX,
 338                 .efds = { -1, },
 339         };
 340         struct sub_device *sdev;
 341         struct rxq *rxq;
 342         uint8_t i;
 343         int ret;
 344
 345         fs_lock(dev, 0);
 346         rxq = dev->data->rx_queues[rx_queue_id];
 347         if (rxq != NULL) {
 348                 fs_rx_queue_release(rxq);
 349                 dev->data->rx_queues[rx_queue_id] = NULL;
 350         }
 351         rxq = rte_zmalloc(NULL,
 352                           sizeof(*rxq) +
 353                           sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail,
 354                           RTE_CACHE_LINE_SIZE);
 355         if (rxq == NULL) {
 356                 fs_unlock(dev, 0);
 357                 return -ENOMEM;
 358         }
 359         FOREACH_SUBDEV(sdev, i, dev)
 360                 rte_atomic64_init(&rxq->refcnt[i]);
 361         rxq->qid = rx_queue_id;
 362         rxq->socket_id = socket_id;
 363         rxq->info.mp = mb_pool;
 364         rxq->info.conf = *rx_conf;
 365         rxq->info.nb_desc = nb_rx_desc;
 366         rxq->priv = PRIV(dev);
 367         rxq->sdev = PRIV(dev)->subs;
 368         ret = rte_intr_efd_enable(&intr_handle, 1);
 369         if (ret < 0) {
 370                 fs_unlock(dev, 0);
 371                 return ret;
 372         }
 373         rxq->event_fd = intr_handle.efds[0];
 374         dev->data->rx_queues[rx_queue_id] = rxq;
 375         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 376                 ret = rte_eth_rx_queue_setup(PORT_ID(sdev),
 377                                 rx_queue_id,
 378                                 nb_rx_desc, socket_id,
 379                                 rx_conf, mb_pool);
 380                 if ((ret = fs_err(sdev, ret))) {
 381                         ERROR("RX queue setup failed for sub_device %d", i);
 382                         goto free_rxq;
 383                 }
 384         }
 385         fs_unlock(dev, 0);
 386         return 0;
 387 free_rxq:
 388         fs_rx_queue_release(rxq);
 389         fs_unlock(dev, 0);
 390         return ret;
 391 }
 392
 393 static int
 394 fs_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
 395 {
 396         struct rxq *rxq;
 397         struct sub_device *sdev;
 398         uint8_t i;
 399         int ret;
 400         int rc = 0;
 401
 402         fs_lock(dev, 0);
 403         if (idx >= dev->data->nb_rx_queues) {
 404                 rc = -EINVAL;
 405                 goto unlock;
 406         }
 407         rxq = dev->data->rx_queues[idx];
 408         if (rxq == NULL || rxq->event_fd <= 0) {
 409                 rc = -EINVAL;
 410                 goto unlock;
 411         }
 412         /* Fail if proxy service is nor running. */
 413         if (PRIV(dev)->rxp.sstate != SS_RUNNING) {
 414                 ERROR("failsafe interrupt services are not running");
 415                 rc = -EAGAIN;
 416                 goto unlock;
 417         }
 418         rxq->enable_events = 1;
 419         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 420                 ret = rte_eth_dev_rx_intr_enable(PORT_ID(sdev), idx);
 421                 ret = fs_err(sdev, ret);
 422                 if (ret)
 423                         rc = ret;
 424         }
 425 unlock:
 426         fs_unlock(dev, 0);
 427         if (rc)
 428                 rte_errno = -rc;
 429         return rc;
 430 }
 431
 432 static int
 433 fs_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
 434 {
 435         struct rxq *rxq;
 436         struct sub_device *sdev;
 437         uint64_t u64;
 438         uint8_t i;
 439         int rc = 0;
 440         int ret;
 441
 442         fs_lock(dev, 0);
 443         if (idx >= dev->data->nb_rx_queues) {
 444                 rc = -EINVAL;
 445                 goto unlock;
 446         }
 447         rxq = dev->data->rx_queues[idx];
 448         if (rxq == NULL || rxq->event_fd <= 0) {
 449                 rc = -EINVAL;
 450                 goto unlock;
 451         }
 452         rxq->enable_events = 0;
 453         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 454                 ret = rte_eth_dev_rx_intr_disable(PORT_ID(sdev), idx);
 455                 ret = fs_err(sdev, ret);
 456                 if (ret)
 457                         rc = ret;
 458         }
 459         /* Clear pending events */
 460         while (read(rxq->event_fd, &u64, sizeof(uint64_t)) >  0)
 461                 ;
 462 unlock:
 463         fs_unlock(dev, 0);
 464         if (rc)
 465                 rte_errno = -rc;
 466         return rc;
 467 }
 468
 469 static void
 470 fs_tx_queue_release(void *queue)
 471 {
 472         struct rte_eth_dev *dev;
 473         struct sub_device *sdev;
 474         uint8_t i;
 475         struct txq *txq;
 476
 477         if (queue == NULL)
 478                 return;
 479         txq = queue;
 480         dev = txq->priv->dev;
 481         fs_lock(dev, 0);
 482         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 483                 if (ETH(sdev)->data->tx_queues != NULL &&
 484                     ETH(sdev)->data->tx_queues[txq->qid] != NULL) {
 485                         SUBOPS(sdev, tx_queue_release)
 486                                 (ETH(sdev)->data->tx_queues[txq->qid]);
 487                 }
 488         }
 489         dev->data->tx_queues[txq->qid] = NULL;
 490         rte_free(txq);
 491         fs_unlock(dev, 0);
 492 }
 493
 494 static int
 495 fs_tx_queue_setup(struct rte_eth_dev *dev,
 496                 uint16_t tx_queue_id,
 497                 uint16_t nb_tx_desc,
 498                 unsigned int socket_id,
 499                 const struct rte_eth_txconf *tx_conf)
 500 {
 501         struct sub_device *sdev;
 502         struct txq *txq;
 503         uint8_t i;
 504         int ret;
 505
 506         fs_lock(dev, 0);
 507         txq = dev->data->tx_queues[tx_queue_id];
 508         if (txq != NULL) {
 509                 fs_tx_queue_release(txq);
 510                 dev->data->tx_queues[tx_queue_id] = NULL;
 511         }
 512         txq = rte_zmalloc("ethdev TX queue",
 513                           sizeof(*txq) +
 514                           sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail,
 515                           RTE_CACHE_LINE_SIZE);
 516         if (txq == NULL) {
 517                 fs_unlock(dev, 0);
 518                 return -ENOMEM;
 519         }
 520         FOREACH_SUBDEV(sdev, i, dev)
 521                 rte_atomic64_init(&txq->refcnt[i]);
 522         txq->qid = tx_queue_id;
 523         txq->socket_id = socket_id;
 524         txq->info.conf = *tx_conf;
 525         txq->info.nb_desc = nb_tx_desc;
 526         txq->priv = PRIV(dev);
 527         dev->data->tx_queues[tx_queue_id] = txq;
 528         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 529                 ret = rte_eth_tx_queue_setup(PORT_ID(sdev),
 530                                 tx_queue_id,
 531                                 nb_tx_desc, socket_id,
 532                                 tx_conf);
 533                 if ((ret = fs_err(sdev, ret))) {
 534                         ERROR("TX queue setup failed for sub_device %d", i);
 535                         goto free_txq;
 536                 }
 537         }
 538         fs_unlock(dev, 0);
 539         return 0;
 540 free_txq:
 541         fs_tx_queue_release(txq);
 542         fs_unlock(dev, 0);
 543         return ret;
 544 }
 545
 546 static void
 547 fs_dev_free_queues(struct rte_eth_dev *dev)
 548 {
 549         uint16_t i;
 550
 551         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 552                 fs_rx_queue_release(dev->data->rx_queues[i]);
 553                 dev->data->rx_queues[i] = NULL;
 554         }
 555         dev->data->nb_rx_queues = 0;
 556         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 557                 fs_tx_queue_release(dev->data->tx_queues[i]);
 558                 dev->data->tx_queues[i] = NULL;
 559         }
 560         dev->data->nb_tx_queues = 0;
 561 }
 562
 563 static void
 564 fs_promiscuous_enable(struct rte_eth_dev *dev)
 565 {
 566         struct sub_device *sdev;
 567         uint8_t i;
 568
 569         fs_lock(dev, 0);
 570         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 571                 rte_eth_promiscuous_enable(PORT_ID(sdev));
 572         fs_unlock(dev, 0);
 573 }
 574
 575 static void
 576 fs_promiscuous_disable(struct rte_eth_dev *dev)
 577 {
 578         struct sub_device *sdev;
 579         uint8_t i;
 580
 581         fs_lock(dev, 0);
 582         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 583                 rte_eth_promiscuous_disable(PORT_ID(sdev));
 584         fs_unlock(dev, 0);
 585 }
 586
 587 static void
 588 fs_allmulticast_enable(struct rte_eth_dev *dev)
 589 {
 590         struct sub_device *sdev;
 591         uint8_t i;
 592
 593         fs_lock(dev, 0);
 594         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 595                 rte_eth_allmulticast_enable(PORT_ID(sdev));
 596         fs_unlock(dev, 0);
 597 }
 598
 599 static void
 600 fs_allmulticast_disable(struct rte_eth_dev *dev)
 601 {
 602         struct sub_device *sdev;
 603         uint8_t i;
 604
 605         fs_lock(dev, 0);
 606         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 607                 rte_eth_allmulticast_disable(PORT_ID(sdev));
 608         fs_unlock(dev, 0);
 609 }
 610
 611 static int
 612 fs_link_update(struct rte_eth_dev *dev,
 613                 int wait_to_complete)
 614 {
 615         struct sub_device *sdev;
 616         uint8_t i;
 617         int ret;
 618
 619         fs_lock(dev, 0);
 620         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 621                 DEBUG("Calling link_update on sub_device %d", i);
 622                 ret = (SUBOPS(sdev, link_update))(ETH(sdev), wait_to_complete);
 623                 if (ret && ret != -1 && sdev->remove == 0 &&
 624                     rte_eth_dev_is_removed(PORT_ID(sdev)) == 0) {
 625                         ERROR("Link update failed for sub_device %d with error %d",
 626                               i, ret);
 627                         fs_unlock(dev, 0);
 628                         return ret;
 629                 }
 630         }
 631         if (TX_SUBDEV(dev)) {
 632                 struct rte_eth_link *l1;
 633                 struct rte_eth_link *l2;
 634
 635                 l1 = &dev->data->dev_link;
 636                 l2 = &ETH(TX_SUBDEV(dev))->data->dev_link;
 637                 if (memcmp(l1, l2, sizeof(*l1))) {
 638                         *l1 = *l2;
 639                         fs_unlock(dev, 0);
 640                         return 0;
 641                 }
 642         }
 643         fs_unlock(dev, 0);
 644         return -1;
 645 }
 646
 647 static int
 648 fs_stats_get(struct rte_eth_dev *dev,
 649              struct rte_eth_stats *stats)
 650 {
 651         struct rte_eth_stats backup;
 652         struct sub_device *sdev;
 653         uint8_t i;
 654         int ret;
 655
 656         fs_lock(dev, 0);
 657         rte_memcpy(stats, &PRIV(dev)->stats_accumulator, sizeof(*stats));
 658         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 659                 struct rte_eth_stats *snapshot = &sdev->stats_snapshot.stats;
 660                 uint64_t *timestamp = &sdev->stats_snapshot.timestamp;
 661
 662                 rte_memcpy(&backup, snapshot, sizeof(backup));
 663                 ret = rte_eth_stats_get(PORT_ID(sdev), snapshot);
 664                 if (ret) {
 665                         if (!fs_err(sdev, ret)) {
 666                                 rte_memcpy(snapshot, &backup, sizeof(backup));
 667                                 goto inc;
 668                         }
 669                         ERROR("Operation rte_eth_stats_get failed for sub_device %d with error %d",
 670                                   i, ret);
 671                         *timestamp = 0;
 672                         fs_unlock(dev, 0);
 673                         return ret;
 674                 }
 675                 *timestamp = rte_rdtsc();
 676 inc:
 677                 failsafe_stats_increment(stats, snapshot);
 678         }
 679         fs_unlock(dev, 0);
 680         return 0;
 681 }
 682
 683 static void
 684 fs_stats_reset(struct rte_eth_dev *dev)
 685 {
 686         struct sub_device *sdev;
 687         uint8_t i;
 688
 689         fs_lock(dev, 0);
 690         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 691                 rte_eth_stats_reset(PORT_ID(sdev));
 692                 memset(&sdev->stats_snapshot, 0, sizeof(struct rte_eth_stats));
 693         }
 694         memset(&PRIV(dev)->stats_accumulator, 0, sizeof(struct rte_eth_stats));
 695         fs_unlock(dev, 0);
 696 }
 697
 698 /**
 699  * Fail-safe dev_infos_get rules:
 700  *
 701  * No sub_device:
 702  *   Numerables:
 703  *      Use the maximum possible values for any field, so as not
 704  *      to impede any further configuration effort.
 705  *   Capabilities:
 706  *      Limits capabilities to those that are understood by the
 707  *      fail-safe PMD. This understanding stems from the fail-safe
 708  *      being capable of verifying that the related capability is
 709  *      expressed within the device configuration (struct rte_eth_conf).
 710  *
 711  * At least one probed sub_device:
 712  *   Numerables:
 713  *      Uses values from the active probed sub_device
 714  *      The rationale here is that if any sub_device is less capable
 715  *      (for example concerning the number of queues) than the active
 716  *      sub_device, then its subsequent configuration will fail.
 717  *      It is impossible to foresee this failure when the failing sub_device
 718  *      is supposed to be plugged-in later on, so the configuration process
 719  *      is the single point of failure and error reporting.
 720  *   Capabilities:
 721  *      Uses a logical AND of RX capabilities among
 722  *      all sub_devices and the default capabilities.
 723  *      Uses a logical AND of TX capabilities among
 724  *      the active probed sub_device and the default capabilities.
 725  *
 726  */
 727 static void
 728 fs_dev_infos_get(struct rte_eth_dev *dev,
 729                   struct rte_eth_dev_info *infos)
 730 {
 731         struct sub_device *sdev;
 732         uint8_t i;
 733
 734         sdev = TX_SUBDEV(dev);
 735         if (sdev == NULL) {
 736                 DEBUG("No probed device, using default infos");
 737                 rte_memcpy(&PRIV(dev)->infos, &default_infos,
 738                            sizeof(default_infos));
 739         } else {
 740                 uint64_t rx_offload_capa;
 741                 uint64_t rxq_offload_capa;
 742                 uint64_t rss_hf_offload_capa;
 743
 744                 rx_offload_capa = default_infos.rx_offload_capa;
 745                 rxq_offload_capa = default_infos.rx_queue_offload_capa;
 746                 rss_hf_offload_capa = default_infos.flow_type_rss_offloads;
 747                 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) {
 748                         rte_eth_dev_info_get(PORT_ID(sdev),
 749                                         &PRIV(dev)->infos);
 750                         rx_offload_capa &= PRIV(dev)->infos.rx_offload_capa;
 751                         rxq_offload_capa &=
 752                                         PRIV(dev)->infos.rx_queue_offload_capa;
 753                         rss_hf_offload_capa &=
 754                                         PRIV(dev)->infos.flow_type_rss_offloads;
 755                 }
 756                 sdev = TX_SUBDEV(dev);
 757                 rte_eth_dev_info_get(PORT_ID(sdev), &PRIV(dev)->infos);
 758                 PRIV(dev)->infos.rx_offload_capa = rx_offload_capa;
 759                 PRIV(dev)->infos.rx_queue_offload_capa = rxq_offload_capa;
 760                 PRIV(dev)->infos.flow_type_rss_offloads = rss_hf_offload_capa;
 761                 PRIV(dev)->infos.tx_offload_capa &=
 762                                         default_infos.tx_offload_capa;
 763                 PRIV(dev)->infos.tx_queue_offload_capa &=
 764                                         default_infos.tx_queue_offload_capa;
 765         }
 766         rte_memcpy(infos, &PRIV(dev)->infos, sizeof(*infos));
 767 }
 768
 769 static const uint32_t *
 770 fs_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 771 {
 772         struct sub_device *sdev;
 773         struct rte_eth_dev *edev;
 774         const uint32_t *ret;
 775
 776         fs_lock(dev, 0);
 777         sdev = TX_SUBDEV(dev);
 778         if (sdev == NULL) {
 779                 ret = NULL;
 780                 goto unlock;
 781         }
 782         edev = ETH(sdev);
 783         /* ENOTSUP: counts as no supported ptypes */
 784         if (SUBOPS(sdev, dev_supported_ptypes_get) == NULL) {
 785                 ret = NULL;
 786                 goto unlock;
 787         }
 788         /*
 789          * The API does not permit to do a clean AND of all ptypes,
 790          * It is also incomplete by design and we do not really care
 791          * to have a best possible value in this context.
 792          * We just return the ptypes of the device of highest
 793          * priority, usually the PREFERRED device.
 794          */
 795         ret = SUBOPS(sdev, dev_supported_ptypes_get)(edev);
 796 unlock:
 797         fs_unlock(dev, 0);
 798         return ret;
 799 }
 800
 801 static int
 802 fs_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 803 {
 804         struct sub_device *sdev;
 805         uint8_t i;
 806         int ret;
 807
 808         fs_lock(dev, 0);
 809         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 810                 DEBUG("Calling rte_eth_dev_set_mtu on sub_device %d", i);
 811                 ret = rte_eth_dev_set_mtu(PORT_ID(sdev), mtu);
 812                 if ((ret = fs_err(sdev, ret))) {
 813                         ERROR("Operation rte_eth_dev_set_mtu failed for sub_device %d with error %d",
 814                               i, ret);
 815                         fs_unlock(dev, 0);
 816                         return ret;
 817                 }
 818         }
 819         fs_unlock(dev, 0);
 820         return 0;
 821 }
 822
 823 static int
 824 fs_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 825 {
 826         struct sub_device *sdev;
 827         uint8_t i;
 828         int ret;
 829
 830         fs_lock(dev, 0);
 831         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 832                 DEBUG("Calling rte_eth_dev_vlan_filter on sub_device %d", i);
 833                 ret = rte_eth_dev_vlan_filter(PORT_ID(sdev), vlan_id, on);
 834                 if ((ret = fs_err(sdev, ret))) {
 835                         ERROR("Operation rte_eth_dev_vlan_filter failed for sub_device %d"
 836                               " with error %d", i, ret);
 837                         fs_unlock(dev, 0);
 838                         return ret;
 839                 }
 840         }
 841         fs_unlock(dev, 0);
 842         return 0;
 843 }
 844
 845 static int
 846 fs_flow_ctrl_get(struct rte_eth_dev *dev,
 847                 struct rte_eth_fc_conf *fc_conf)
 848 {
 849         struct sub_device *sdev;
 850         int ret;
 851
 852         fs_lock(dev, 0);
 853         sdev = TX_SUBDEV(dev);
 854         if (sdev == NULL) {
 855                 ret = 0;
 856                 goto unlock;
 857         }
 858         if (SUBOPS(sdev, flow_ctrl_get) == NULL) {
 859                 ret = -ENOTSUP;
 860                 goto unlock;
 861         }
 862         ret = SUBOPS(sdev, flow_ctrl_get)(ETH(sdev), fc_conf);
 863 unlock:
 864         fs_unlock(dev, 0);
 865         return ret;
 866 }
 867
 868 static int
 869 fs_flow_ctrl_set(struct rte_eth_dev *dev,
 870                 struct rte_eth_fc_conf *fc_conf)
 871 {
 872         struct sub_device *sdev;
 873         uint8_t i;
 874         int ret;
 875
 876         fs_lock(dev, 0);
 877         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 878                 DEBUG("Calling rte_eth_dev_flow_ctrl_set on sub_device %d", i);
 879                 ret = rte_eth_dev_flow_ctrl_set(PORT_ID(sdev), fc_conf);
 880                 if ((ret = fs_err(sdev, ret))) {
 881                         ERROR("Operation rte_eth_dev_flow_ctrl_set failed for sub_device %d"
 882                               " with error %d", i, ret);
 883                         fs_unlock(dev, 0);
 884                         return ret;
 885                 }
 886         }
 887         fs_unlock(dev, 0);
 888         return 0;
 889 }
 890
 891 static void
 892 fs_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
 893 {
 894         struct sub_device *sdev;
 895         uint8_t i;
 896
 897         fs_lock(dev, 0);
 898         /* No check: already done within the rte_eth_dev_mac_addr_remove
 899          * call for the fail-safe device.
 900          */
 901         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 902                 rte_eth_dev_mac_addr_remove(PORT_ID(sdev),
 903                                 &dev->data->mac_addrs[index]);
 904         PRIV(dev)->mac_addr_pool[index] = 0;
 905         fs_unlock(dev, 0);
 906 }
 907
 908 static int
 909 fs_mac_addr_add(struct rte_eth_dev *dev,
 910                 struct ether_addr *mac_addr,
 911                 uint32_t index,
 912                 uint32_t vmdq)
 913 {
 914         struct sub_device *sdev;
 915         int ret;
 916         uint8_t i;
 917
 918         RTE_ASSERT(index < FAILSAFE_MAX_ETHADDR);
 919         fs_lock(dev, 0);
 920         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 921                 ret = rte_eth_dev_mac_addr_add(PORT_ID(sdev), mac_addr, vmdq);
 922                 if ((ret = fs_err(sdev, ret))) {
 923                         ERROR("Operation rte_eth_dev_mac_addr_add failed for sub_device %"
 924                               PRIu8 " with error %d", i, ret);
 925                         fs_unlock(dev, 0);
 926                         return ret;
 927                 }
 928         }
 929         if (index >= PRIV(dev)->nb_mac_addr) {
 930                 DEBUG("Growing mac_addrs array");
 931                 PRIV(dev)->nb_mac_addr = index;
 932         }
 933         PRIV(dev)->mac_addr_pool[index] = vmdq;
 934         fs_unlock(dev, 0);
 935         return 0;
 936 }
 937
 938 static int
 939 fs_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
 940 {
 941         struct sub_device *sdev;
 942         uint8_t i;
 943         int ret;
 944
 945         fs_lock(dev, 0);
 946         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 947                 ret = rte_eth_dev_default_mac_addr_set(PORT_ID(sdev), mac_addr);
 948                 ret = fs_err(sdev, ret);
 949                 if (ret) {
 950                         ERROR("Operation rte_eth_dev_mac_addr_set failed for sub_device %d with error %d",
 951                                 i, ret);
 952                         fs_unlock(dev, 0);
 953                         return ret;
 954                 }
 955         }
 956         fs_unlock(dev, 0);
 957
 958         return 0;
 959 }
 960
 961 static int
 962 fs_rss_hash_update(struct rte_eth_dev *dev,
 963                         struct rte_eth_rss_conf *rss_conf)
 964 {
 965         struct sub_device *sdev;
 966         uint8_t i;
 967         int ret;
 968
 969         fs_lock(dev, 0);
 970         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 971                 ret = rte_eth_dev_rss_hash_update(PORT_ID(sdev), rss_conf);
 972                 ret = fs_err(sdev, ret);
 973                 if (ret) {
 974                         ERROR("Operation rte_eth_dev_rss_hash_update"
 975                                 " failed for sub_device %d with error %d",
 976                                 i, ret);
 977                         fs_unlock(dev, 0);
 978                         return ret;
 979                 }
 980         }
 981         fs_unlock(dev, 0);
 982
 983         return 0;
 984 }
 985
 986 static int
 987 fs_filter_ctrl(struct rte_eth_dev *dev,
 988                 enum rte_filter_type type,
 989                 enum rte_filter_op op,
 990                 void *arg)
 991 {
 992         struct sub_device *sdev;
 993         uint8_t i;
 994         int ret;
 995
 996         if (type == RTE_ETH_FILTER_GENERIC &&
 997             op == RTE_ETH_FILTER_GET) {
 998                 *(const void **)arg = &fs_flow_ops;
 999                 return 0;
1000         }
1001         fs_lock(dev, 0);
1002         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
1003                 DEBUG("Calling rte_eth_dev_filter_ctrl on sub_device %d", i);
1004                 ret = rte_eth_dev_filter_ctrl(PORT_ID(sdev), type, op, arg);
1005                 if ((ret = fs_err(sdev, ret))) {
1006                         ERROR("Operation rte_eth_dev_filter_ctrl failed for sub_device %d"
1007                               " with error %d", i, ret);
1008                         fs_unlock(dev, 0);
1009                         return ret;
1010                 }
1011         }
1012         fs_unlock(dev, 0);
1013         return 0;
1014 }
1015
1016 const struct eth_dev_ops failsafe_ops = {
1017         .dev_configure = fs_dev_configure,
1018         .dev_start = fs_dev_start,
1019         .dev_stop = fs_dev_stop,
1020         .dev_set_link_down = fs_dev_set_link_down,
1021         .dev_set_link_up = fs_dev_set_link_up,
1022         .dev_close = fs_dev_close,
1023         .promiscuous_enable = fs_promiscuous_enable,
1024         .promiscuous_disable = fs_promiscuous_disable,
1025         .allmulticast_enable = fs_allmulticast_enable,
1026         .allmulticast_disable = fs_allmulticast_disable,
1027         .link_update = fs_link_update,
1028         .stats_get = fs_stats_get,
1029         .stats_reset = fs_stats_reset,
1030         .dev_infos_get = fs_dev_infos_get,
1031         .dev_supported_ptypes_get = fs_dev_supported_ptypes_get,
1032         .mtu_set = fs_mtu_set,
1033         .vlan_filter_set = fs_vlan_filter_set,
1034         .rx_queue_setup = fs_rx_queue_setup,
1035         .tx_queue_setup = fs_tx_queue_setup,
1036         .rx_queue_release = fs_rx_queue_release,
1037         .tx_queue_release = fs_tx_queue_release,
1038         .rx_queue_intr_enable = fs_rx_intr_enable,
1039         .rx_queue_intr_disable = fs_rx_intr_disable,
1040         .flow_ctrl_get = fs_flow_ctrl_get,
1041         .flow_ctrl_set = fs_flow_ctrl_set,
1042         .mac_addr_remove = fs_mac_addr_remove,
1043         .mac_addr_add = fs_mac_addr_add,
1044         .mac_addr_set = fs_mac_addr_set,
1045         .rss_hash_update = fs_rss_hash_update,
1046         .filter_ctrl = fs_filter_ctrl,
1047 };