drivers/net/failsafe/failsafe_ops.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2017 6WIND S.A.
   3  * Copyright 2017 Mellanox.
   4  */
   5
   6 #include <stdbool.h>
   7 #include <stdint.h>
   8 #include <unistd.h>
   9
  10 #include <rte_debug.h>
  11 #include <rte_atomic.h>
  12 #include <rte_ethdev_driver.h>
  13 #include <rte_malloc.h>
  14 #include <rte_flow.h>
  15 #include <rte_cycles.h>
  16
  17 #include "failsafe_private.h"
  18
  19 static struct rte_eth_dev_info default_infos = {
  20         /* Max possible number of elements */
  21         .max_rx_pktlen = UINT32_MAX,
  22         .max_rx_queues = RTE_MAX_QUEUES_PER_PORT,
  23         .max_tx_queues = RTE_MAX_QUEUES_PER_PORT,
  24         .max_mac_addrs = FAILSAFE_MAX_ETHADDR,
  25         .max_hash_mac_addrs = UINT32_MAX,
  26         .max_vfs = UINT16_MAX,
  27         .max_vmdq_pools = UINT16_MAX,
  28         .rx_desc_lim = {
  29                 .nb_max = UINT16_MAX,
  30                 .nb_min = 0,
  31                 .nb_align = 1,
  32                 .nb_seg_max = UINT16_MAX,
  33                 .nb_mtu_seg_max = UINT16_MAX,
  34         },
  35         .tx_desc_lim = {
  36                 .nb_max = UINT16_MAX,
  37                 .nb_min = 0,
  38                 .nb_align = 1,
  39                 .nb_seg_max = UINT16_MAX,
  40                 .nb_mtu_seg_max = UINT16_MAX,
  41         },
  42         /*
  43          * Set of capabilities that can be verified upon
  44          * configuring a sub-device.
  45          */
  46         .rx_offload_capa =
  47                 DEV_RX_OFFLOAD_VLAN_STRIP |
  48                 DEV_RX_OFFLOAD_IPV4_CKSUM |
  49                 DEV_RX_OFFLOAD_UDP_CKSUM |
  50                 DEV_RX_OFFLOAD_TCP_CKSUM |
  51                 DEV_RX_OFFLOAD_TCP_LRO |
  52                 DEV_RX_OFFLOAD_QINQ_STRIP |
  53                 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
  54                 DEV_RX_OFFLOAD_MACSEC_STRIP |
  55                 DEV_RX_OFFLOAD_HEADER_SPLIT |
  56                 DEV_RX_OFFLOAD_VLAN_FILTER |
  57                 DEV_RX_OFFLOAD_VLAN_EXTEND |
  58                 DEV_RX_OFFLOAD_JUMBO_FRAME |
  59                 DEV_RX_OFFLOAD_CRC_STRIP |
  60                 DEV_RX_OFFLOAD_SCATTER |
  61                 DEV_RX_OFFLOAD_TIMESTAMP |
  62                 DEV_RX_OFFLOAD_SECURITY,
  63         .rx_queue_offload_capa =
  64                 DEV_RX_OFFLOAD_VLAN_STRIP |
  65                 DEV_RX_OFFLOAD_IPV4_CKSUM |
  66                 DEV_RX_OFFLOAD_UDP_CKSUM |
  67                 DEV_RX_OFFLOAD_TCP_CKSUM |
  68                 DEV_RX_OFFLOAD_TCP_LRO |
  69                 DEV_RX_OFFLOAD_QINQ_STRIP |
  70                 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
  71                 DEV_RX_OFFLOAD_MACSEC_STRIP |
  72                 DEV_RX_OFFLOAD_HEADER_SPLIT |
  73                 DEV_RX_OFFLOAD_VLAN_FILTER |
  74                 DEV_RX_OFFLOAD_VLAN_EXTEND |
  75                 DEV_RX_OFFLOAD_JUMBO_FRAME |
  76                 DEV_RX_OFFLOAD_CRC_STRIP |
  77                 DEV_RX_OFFLOAD_SCATTER |
  78                 DEV_RX_OFFLOAD_TIMESTAMP |
  79                 DEV_RX_OFFLOAD_SECURITY,
  80         .tx_offload_capa =
  81                 DEV_TX_OFFLOAD_MULTI_SEGS |
  82                 DEV_TX_OFFLOAD_IPV4_CKSUM |
  83                 DEV_TX_OFFLOAD_UDP_CKSUM |
  84                 DEV_TX_OFFLOAD_TCP_CKSUM,
  85         .flow_type_rss_offloads = 0x0,
  86 };
  87
  88 static int
  89 fs_dev_configure(struct rte_eth_dev *dev)
  90 {
  91         struct sub_device *sdev;
  92         uint64_t supp_tx_offloads;
  93         uint64_t tx_offloads;
  94         uint8_t i;
  95         int ret;
  96
  97         fs_lock(dev, 0);
  98         supp_tx_offloads = PRIV(dev)->infos.tx_offload_capa;
  99         tx_offloads = dev->data->dev_conf.txmode.offloads;
 100         if ((tx_offloads & supp_tx_offloads) != tx_offloads) {
 101                 rte_errno = ENOTSUP;
 102                 ERROR("Some Tx offloads are not supported, "
 103                       "requested 0x%" PRIx64 " supported 0x%" PRIx64,
 104                       tx_offloads, supp_tx_offloads);
 105                 fs_unlock(dev, 0);
 106                 return -rte_errno;
 107         }
 108         FOREACH_SUBDEV(sdev, i, dev) {
 109                 int rmv_interrupt = 0;
 110                 int lsc_interrupt = 0;
 111                 int lsc_enabled;
 112
 113                 if (sdev->state != DEV_PROBED)
 114                         continue;
 115
 116                 rmv_interrupt = ETH(sdev)->data->dev_flags &
 117                                 RTE_ETH_DEV_INTR_RMV;
 118                 if (rmv_interrupt) {
 119                         DEBUG("Enabling RMV interrupts for sub_device %d", i);
 120                         dev->data->dev_conf.intr_conf.rmv = 1;
 121                 } else {
 122                         DEBUG("sub_device %d does not support RMV event", i);
 123                 }
 124                 lsc_enabled = dev->data->dev_conf.intr_conf.lsc;
 125                 lsc_interrupt = lsc_enabled &&
 126                                 (ETH(sdev)->data->dev_flags &
 127                                  RTE_ETH_DEV_INTR_LSC);
 128                 if (lsc_interrupt) {
 129                         DEBUG("Enabling LSC interrupts for sub_device %d", i);
 130                         dev->data->dev_conf.intr_conf.lsc = 1;
 131                 } else if (lsc_enabled && !lsc_interrupt) {
 132                         DEBUG("Disabling LSC interrupts for sub_device %d", i);
 133                         dev->data->dev_conf.intr_conf.lsc = 0;
 134                 }
 135                 DEBUG("Configuring sub-device %d", i);
 136                 ret = rte_eth_dev_configure(PORT_ID(sdev),
 137                                         dev->data->nb_rx_queues,
 138                                         dev->data->nb_tx_queues,
 139                                         &dev->data->dev_conf);
 140                 if (ret) {
 141                         if (!fs_err(sdev, ret))
 142                                 continue;
 143                         ERROR("Could not configure sub_device %d", i);
 144                         fs_unlock(dev, 0);
 145                         return ret;
 146                 }
 147                 if (rmv_interrupt) {
 148                         ret = rte_eth_dev_callback_register(PORT_ID(sdev),
 149                                         RTE_ETH_EVENT_INTR_RMV,
 150                                         failsafe_eth_rmv_event_callback,
 151                                         sdev);
 152                         if (ret)
 153                                 WARN("Failed to register RMV callback for sub_device %d",
 154                                      SUB_ID(sdev));
 155                 }
 156                 dev->data->dev_conf.intr_conf.rmv = 0;
 157                 if (lsc_interrupt) {
 158                         ret = rte_eth_dev_callback_register(PORT_ID(sdev),
 159                                                 RTE_ETH_EVENT_INTR_LSC,
 160                                                 failsafe_eth_lsc_event_callback,
 161                                                 dev);
 162                         if (ret)
 163                                 WARN("Failed to register LSC callback for sub_device %d",
 164                                      SUB_ID(sdev));
 165                 }
 166                 dev->data->dev_conf.intr_conf.lsc = lsc_enabled;
 167                 sdev->state = DEV_ACTIVE;
 168         }
 169         if (PRIV(dev)->state < DEV_ACTIVE)
 170                 PRIV(dev)->state = DEV_ACTIVE;
 171         fs_unlock(dev, 0);
 172         return 0;
 173 }
 174
 175 static int
 176 fs_dev_start(struct rte_eth_dev *dev)
 177 {
 178         struct sub_device *sdev;
 179         uint8_t i;
 180         int ret;
 181
 182         fs_lock(dev, 0);
 183         ret = failsafe_rx_intr_install(dev);
 184         if (ret) {
 185                 fs_unlock(dev, 0);
 186                 return ret;
 187         }
 188         FOREACH_SUBDEV(sdev, i, dev) {
 189                 if (sdev->state != DEV_ACTIVE)
 190                         continue;
 191                 DEBUG("Starting sub_device %d", i);
 192                 ret = rte_eth_dev_start(PORT_ID(sdev));
 193                 if (ret) {
 194                         if (!fs_err(sdev, ret))
 195                                 continue;
 196                         fs_unlock(dev, 0);
 197                         return ret;
 198                 }
 199                 ret = failsafe_rx_intr_install_subdevice(sdev);
 200                 if (ret) {
 201                         if (!fs_err(sdev, ret))
 202                                 continue;
 203                         rte_eth_dev_stop(PORT_ID(sdev));
 204                         fs_unlock(dev, 0);
 205                         return ret;
 206                 }
 207                 sdev->state = DEV_STARTED;
 208         }
 209         if (PRIV(dev)->state < DEV_STARTED)
 210                 PRIV(dev)->state = DEV_STARTED;
 211         fs_switch_dev(dev, NULL);
 212         fs_unlock(dev, 0);
 213         return 0;
 214 }
 215
 216 static void
 217 fs_dev_stop(struct rte_eth_dev *dev)
 218 {
 219         struct sub_device *sdev;
 220         uint8_t i;
 221
 222         fs_lock(dev, 0);
 223         PRIV(dev)->state = DEV_STARTED - 1;
 224         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_STARTED) {
 225                 rte_eth_dev_stop(PORT_ID(sdev));
 226                 failsafe_rx_intr_uninstall_subdevice(sdev);
 227                 sdev->state = DEV_STARTED - 1;
 228         }
 229         failsafe_rx_intr_uninstall(dev);
 230         fs_unlock(dev, 0);
 231 }
 232
 233 static int
 234 fs_dev_set_link_up(struct rte_eth_dev *dev)
 235 {
 236         struct sub_device *sdev;
 237         uint8_t i;
 238         int ret;
 239
 240         fs_lock(dev, 0);
 241         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 242                 DEBUG("Calling rte_eth_dev_set_link_up on sub_device %d", i);
 243                 ret = rte_eth_dev_set_link_up(PORT_ID(sdev));
 244                 if ((ret = fs_err(sdev, ret))) {
 245                         ERROR("Operation rte_eth_dev_set_link_up failed for sub_device %d"
 246                               " with error %d", i, ret);
 247                         fs_unlock(dev, 0);
 248                         return ret;
 249                 }
 250         }
 251         fs_unlock(dev, 0);
 252         return 0;
 253 }
 254
 255 static int
 256 fs_dev_set_link_down(struct rte_eth_dev *dev)
 257 {
 258         struct sub_device *sdev;
 259         uint8_t i;
 260         int ret;
 261
 262         fs_lock(dev, 0);
 263         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 264                 DEBUG("Calling rte_eth_dev_set_link_down on sub_device %d", i);
 265                 ret = rte_eth_dev_set_link_down(PORT_ID(sdev));
 266                 if ((ret = fs_err(sdev, ret))) {
 267                         ERROR("Operation rte_eth_dev_set_link_down failed for sub_device %d"
 268                               " with error %d", i, ret);
 269                         fs_unlock(dev, 0);
 270                         return ret;
 271                 }
 272         }
 273         fs_unlock(dev, 0);
 274         return 0;
 275 }
 276
 277 static void fs_dev_free_queues(struct rte_eth_dev *dev);
 278 static void
 279 fs_dev_close(struct rte_eth_dev *dev)
 280 {
 281         struct sub_device *sdev;
 282         uint8_t i;
 283
 284         fs_lock(dev, 0);
 285         failsafe_hotplug_alarm_cancel(dev);
 286         if (PRIV(dev)->state == DEV_STARTED)
 287                 dev->dev_ops->dev_stop(dev);
 288         PRIV(dev)->state = DEV_ACTIVE - 1;
 289         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 290                 DEBUG("Closing sub_device %d", i);
 291                 rte_eth_dev_close(PORT_ID(sdev));
 292                 sdev->state = DEV_ACTIVE - 1;
 293         }
 294         fs_dev_free_queues(dev);
 295         fs_unlock(dev, 0);
 296 }
 297
 298 static bool
 299 fs_rxq_offloads_valid(struct rte_eth_dev *dev, uint64_t offloads)
 300 {
 301         uint64_t port_offloads;
 302         uint64_t queue_supp_offloads;
 303         uint64_t port_supp_offloads;
 304
 305         port_offloads = dev->data->dev_conf.rxmode.offloads;
 306         queue_supp_offloads = PRIV(dev)->infos.rx_queue_offload_capa;
 307         port_supp_offloads = PRIV(dev)->infos.rx_offload_capa;
 308         if ((offloads & (queue_supp_offloads | port_supp_offloads)) !=
 309              offloads)
 310                 return false;
 311         /* Verify we have no conflict with port offloads */
 312         if ((port_offloads ^ offloads) & port_supp_offloads)
 313                 return false;
 314         return true;
 315 }
 316
 317 static void
 318 fs_rx_queue_release(void *queue)
 319 {
 320         struct rte_eth_dev *dev;
 321         struct sub_device *sdev;
 322         uint8_t i;
 323         struct rxq *rxq;
 324
 325         if (queue == NULL)
 326                 return;
 327         rxq = queue;
 328         dev = rxq->priv->dev;
 329         fs_lock(dev, 0);
 330         if (rxq->event_fd > 0)
 331                 close(rxq->event_fd);
 332         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 333                 SUBOPS(sdev, rx_queue_release)
 334                         (ETH(sdev)->data->rx_queues[rxq->qid]);
 335         dev->data->rx_queues[rxq->qid] = NULL;
 336         rte_free(rxq);
 337         fs_unlock(dev, 0);
 338 }
 339
 340 static int
 341 fs_rx_queue_setup(struct rte_eth_dev *dev,
 342                 uint16_t rx_queue_id,
 343                 uint16_t nb_rx_desc,
 344                 unsigned int socket_id,
 345                 const struct rte_eth_rxconf *rx_conf,
 346                 struct rte_mempool *mb_pool)
 347 {
 348         /*
 349          * FIXME: Add a proper interface in rte_eal_interrupts for
 350          * allocating eventfd as an interrupt vector.
 351          * For the time being, fake as if we are using MSIX interrupts,
 352          * this will cause rte_intr_efd_enable to allocate an eventfd for us.
 353          */
 354         struct rte_intr_handle intr_handle = {
 355                 .type = RTE_INTR_HANDLE_VFIO_MSIX,
 356                 .efds = { -1, },
 357         };
 358         struct sub_device *sdev;
 359         struct rxq *rxq;
 360         uint8_t i;
 361         int ret;
 362
 363         fs_lock(dev, 0);
 364         rxq = dev->data->rx_queues[rx_queue_id];
 365         if (rxq != NULL) {
 366                 fs_rx_queue_release(rxq);
 367                 dev->data->rx_queues[rx_queue_id] = NULL;
 368         }
 369         /* Verify application offloads are valid for our port and queue. */
 370         if (fs_rxq_offloads_valid(dev, rx_conf->offloads) == false) {
 371                 rte_errno = ENOTSUP;
 372                 ERROR("Rx queue offloads 0x%" PRIx64
 373                       " don't match port offloads 0x%" PRIx64
 374                       " or supported offloads 0x%" PRIx64,
 375                       rx_conf->offloads,
 376                       dev->data->dev_conf.rxmode.offloads,
 377                       PRIV(dev)->infos.rx_offload_capa |
 378                       PRIV(dev)->infos.rx_queue_offload_capa);
 379                 fs_unlock(dev, 0);
 380                 return -rte_errno;
 381         }
 382         rxq = rte_zmalloc(NULL,
 383                           sizeof(*rxq) +
 384                           sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail,
 385                           RTE_CACHE_LINE_SIZE);
 386         if (rxq == NULL) {
 387                 fs_unlock(dev, 0);
 388                 return -ENOMEM;
 389         }
 390         FOREACH_SUBDEV(sdev, i, dev)
 391                 rte_atomic64_init(&rxq->refcnt[i]);
 392         rxq->qid = rx_queue_id;
 393         rxq->socket_id = socket_id;
 394         rxq->info.mp = mb_pool;
 395         rxq->info.conf = *rx_conf;
 396         rxq->info.nb_desc = nb_rx_desc;
 397         rxq->priv = PRIV(dev);
 398         rxq->sdev = PRIV(dev)->subs;
 399         ret = rte_intr_efd_enable(&intr_handle, 1);
 400         if (ret < 0) {
 401                 fs_unlock(dev, 0);
 402                 return ret;
 403         }
 404         rxq->event_fd = intr_handle.efds[0];
 405         dev->data->rx_queues[rx_queue_id] = rxq;
 406         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 407                 ret = rte_eth_rx_queue_setup(PORT_ID(sdev),
 408                                 rx_queue_id,
 409                                 nb_rx_desc, socket_id,
 410                                 rx_conf, mb_pool);
 411                 if ((ret = fs_err(sdev, ret))) {
 412                         ERROR("RX queue setup failed for sub_device %d", i);
 413                         goto free_rxq;
 414                 }
 415         }
 416         fs_unlock(dev, 0);
 417         return 0;
 418 free_rxq:
 419         fs_rx_queue_release(rxq);
 420         fs_unlock(dev, 0);
 421         return ret;
 422 }
 423
 424 static int
 425 fs_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
 426 {
 427         struct rxq *rxq;
 428         struct sub_device *sdev;
 429         uint8_t i;
 430         int ret;
 431         int rc = 0;
 432
 433         fs_lock(dev, 0);
 434         if (idx >= dev->data->nb_rx_queues) {
 435                 rc = -EINVAL;
 436                 goto unlock;
 437         }
 438         rxq = dev->data->rx_queues[idx];
 439         if (rxq == NULL || rxq->event_fd <= 0) {
 440                 rc = -EINVAL;
 441                 goto unlock;
 442         }
 443         /* Fail if proxy service is nor running. */
 444         if (PRIV(dev)->rxp.sstate != SS_RUNNING) {
 445                 ERROR("failsafe interrupt services are not running");
 446                 rc = -EAGAIN;
 447                 goto unlock;
 448         }
 449         rxq->enable_events = 1;
 450         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 451                 ret = rte_eth_dev_rx_intr_enable(PORT_ID(sdev), idx);
 452                 ret = fs_err(sdev, ret);
 453                 if (ret)
 454                         rc = ret;
 455         }
 456 unlock:
 457         fs_unlock(dev, 0);
 458         if (rc)
 459                 rte_errno = -rc;
 460         return rc;
 461 }
 462
 463 static int
 464 fs_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
 465 {
 466         struct rxq *rxq;
 467         struct sub_device *sdev;
 468         uint64_t u64;
 469         uint8_t i;
 470         int rc = 0;
 471         int ret;
 472
 473         fs_lock(dev, 0);
 474         if (idx >= dev->data->nb_rx_queues) {
 475                 rc = -EINVAL;
 476                 goto unlock;
 477         }
 478         rxq = dev->data->rx_queues[idx];
 479         if (rxq == NULL || rxq->event_fd <= 0) {
 480                 rc = -EINVAL;
 481                 goto unlock;
 482         }
 483         rxq->enable_events = 0;
 484         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 485                 ret = rte_eth_dev_rx_intr_disable(PORT_ID(sdev), idx);
 486                 ret = fs_err(sdev, ret);
 487                 if (ret)
 488                         rc = ret;
 489         }
 490         /* Clear pending events */
 491         while (read(rxq->event_fd, &u64, sizeof(uint64_t)) >  0)
 492                 ;
 493 unlock:
 494         fs_unlock(dev, 0);
 495         if (rc)
 496                 rte_errno = -rc;
 497         return rc;
 498 }
 499
 500 static bool
 501 fs_txq_offloads_valid(struct rte_eth_dev *dev, uint64_t offloads)
 502 {
 503         uint64_t port_offloads;
 504         uint64_t queue_supp_offloads;
 505         uint64_t port_supp_offloads;
 506
 507         port_offloads = dev->data->dev_conf.txmode.offloads;
 508         queue_supp_offloads = PRIV(dev)->infos.tx_queue_offload_capa;
 509         port_supp_offloads = PRIV(dev)->infos.tx_offload_capa;
 510         if ((offloads & (queue_supp_offloads | port_supp_offloads)) !=
 511              offloads)
 512                 return false;
 513         /* Verify we have no conflict with port offloads */
 514         if ((port_offloads ^ offloads) & port_supp_offloads)
 515                 return false;
 516         return true;
 517 }
 518
 519 static void
 520 fs_tx_queue_release(void *queue)
 521 {
 522         struct rte_eth_dev *dev;
 523         struct sub_device *sdev;
 524         uint8_t i;
 525         struct txq *txq;
 526
 527         if (queue == NULL)
 528                 return;
 529         txq = queue;
 530         dev = txq->priv->dev;
 531         fs_lock(dev, 0);
 532         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 533                 SUBOPS(sdev, tx_queue_release)
 534                         (ETH(sdev)->data->tx_queues[txq->qid]);
 535         dev->data->tx_queues[txq->qid] = NULL;
 536         rte_free(txq);
 537         fs_unlock(dev, 0);
 538 }
 539
 540 static int
 541 fs_tx_queue_setup(struct rte_eth_dev *dev,
 542                 uint16_t tx_queue_id,
 543                 uint16_t nb_tx_desc,
 544                 unsigned int socket_id,
 545                 const struct rte_eth_txconf *tx_conf)
 546 {
 547         struct sub_device *sdev;
 548         struct txq *txq;
 549         uint8_t i;
 550         int ret;
 551
 552         fs_lock(dev, 0);
 553         txq = dev->data->tx_queues[tx_queue_id];
 554         if (txq != NULL) {
 555                 fs_tx_queue_release(txq);
 556                 dev->data->tx_queues[tx_queue_id] = NULL;
 557         }
 558         /*
 559          * Don't verify queue offloads for applications which
 560          * use the old API.
 561          */
 562         if (tx_conf != NULL &&
 563             (tx_conf->txq_flags & ETH_TXQ_FLAGS_IGNORE) &&
 564             fs_txq_offloads_valid(dev, tx_conf->offloads) == false) {
 565                 rte_errno = ENOTSUP;
 566                 ERROR("Tx queue offloads 0x%" PRIx64
 567                       " don't match port offloads 0x%" PRIx64
 568                       " or supported offloads 0x%" PRIx64,
 569                       tx_conf->offloads,
 570                       dev->data->dev_conf.txmode.offloads,
 571                       PRIV(dev)->infos.tx_offload_capa |
 572                       PRIV(dev)->infos.tx_queue_offload_capa);
 573                 fs_unlock(dev, 0);
 574                 return -rte_errno;
 575         }
 576         txq = rte_zmalloc("ethdev TX queue",
 577                           sizeof(*txq) +
 578                           sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail,
 579                           RTE_CACHE_LINE_SIZE);
 580         if (txq == NULL) {
 581                 fs_unlock(dev, 0);
 582                 return -ENOMEM;
 583         }
 584         FOREACH_SUBDEV(sdev, i, dev)
 585                 rte_atomic64_init(&txq->refcnt[i]);
 586         txq->qid = tx_queue_id;
 587         txq->socket_id = socket_id;
 588         txq->info.conf = *tx_conf;
 589         txq->info.nb_desc = nb_tx_desc;
 590         txq->priv = PRIV(dev);
 591         dev->data->tx_queues[tx_queue_id] = txq;
 592         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 593                 ret = rte_eth_tx_queue_setup(PORT_ID(sdev),
 594                                 tx_queue_id,
 595                                 nb_tx_desc, socket_id,
 596                                 tx_conf);
 597                 if ((ret = fs_err(sdev, ret))) {
 598                         ERROR("TX queue setup failed for sub_device %d", i);
 599                         goto free_txq;
 600                 }
 601         }
 602         fs_unlock(dev, 0);
 603         return 0;
 604 free_txq:
 605         fs_tx_queue_release(txq);
 606         fs_unlock(dev, 0);
 607         return ret;
 608 }
 609
 610 static void
 611 fs_dev_free_queues(struct rte_eth_dev *dev)
 612 {
 613         uint16_t i;
 614
 615         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 616                 fs_rx_queue_release(dev->data->rx_queues[i]);
 617                 dev->data->rx_queues[i] = NULL;
 618         }
 619         dev->data->nb_rx_queues = 0;
 620         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 621                 fs_tx_queue_release(dev->data->tx_queues[i]);
 622                 dev->data->tx_queues[i] = NULL;
 623         }
 624         dev->data->nb_tx_queues = 0;
 625 }
 626
 627 static void
 628 fs_promiscuous_enable(struct rte_eth_dev *dev)
 629 {
 630         struct sub_device *sdev;
 631         uint8_t i;
 632
 633         fs_lock(dev, 0);
 634         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 635                 rte_eth_promiscuous_enable(PORT_ID(sdev));
 636         fs_unlock(dev, 0);
 637 }
 638
 639 static void
 640 fs_promiscuous_disable(struct rte_eth_dev *dev)
 641 {
 642         struct sub_device *sdev;
 643         uint8_t i;
 644
 645         fs_lock(dev, 0);
 646         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 647                 rte_eth_promiscuous_disable(PORT_ID(sdev));
 648         fs_unlock(dev, 0);
 649 }
 650
 651 static void
 652 fs_allmulticast_enable(struct rte_eth_dev *dev)
 653 {
 654         struct sub_device *sdev;
 655         uint8_t i;
 656
 657         fs_lock(dev, 0);
 658         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 659                 rte_eth_allmulticast_enable(PORT_ID(sdev));
 660         fs_unlock(dev, 0);
 661 }
 662
 663 static void
 664 fs_allmulticast_disable(struct rte_eth_dev *dev)
 665 {
 666         struct sub_device *sdev;
 667         uint8_t i;
 668
 669         fs_lock(dev, 0);
 670         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 671                 rte_eth_allmulticast_disable(PORT_ID(sdev));
 672         fs_unlock(dev, 0);
 673 }
 674
 675 static int
 676 fs_link_update(struct rte_eth_dev *dev,
 677                 int wait_to_complete)
 678 {
 679         struct sub_device *sdev;
 680         uint8_t i;
 681         int ret;
 682
 683         fs_lock(dev, 0);
 684         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 685                 DEBUG("Calling link_update on sub_device %d", i);
 686                 ret = (SUBOPS(sdev, link_update))(ETH(sdev), wait_to_complete);
 687                 if (ret && ret != -1 && sdev->remove == 0 &&
 688                     rte_eth_dev_is_removed(PORT_ID(sdev)) == 0) {
 689                         ERROR("Link update failed for sub_device %d with error %d",
 690                               i, ret);
 691                         fs_unlock(dev, 0);
 692                         return ret;
 693                 }
 694         }
 695         if (TX_SUBDEV(dev)) {
 696                 struct rte_eth_link *l1;
 697                 struct rte_eth_link *l2;
 698
 699                 l1 = &dev->data->dev_link;
 700                 l2 = &ETH(TX_SUBDEV(dev))->data->dev_link;
 701                 if (memcmp(l1, l2, sizeof(*l1))) {
 702                         *l1 = *l2;
 703                         fs_unlock(dev, 0);
 704                         return 0;
 705                 }
 706         }
 707         fs_unlock(dev, 0);
 708         return -1;
 709 }
 710
 711 static int
 712 fs_stats_get(struct rte_eth_dev *dev,
 713              struct rte_eth_stats *stats)
 714 {
 715         struct rte_eth_stats backup;
 716         struct sub_device *sdev;
 717         uint8_t i;
 718         int ret;
 719
 720         fs_lock(dev, 0);
 721         rte_memcpy(stats, &PRIV(dev)->stats_accumulator, sizeof(*stats));
 722         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 723                 struct rte_eth_stats *snapshot = &sdev->stats_snapshot.stats;
 724                 uint64_t *timestamp = &sdev->stats_snapshot.timestamp;
 725
 726                 rte_memcpy(&backup, snapshot, sizeof(backup));
 727                 ret = rte_eth_stats_get(PORT_ID(sdev), snapshot);
 728                 if (ret) {
 729                         if (!fs_err(sdev, ret)) {
 730                                 rte_memcpy(snapshot, &backup, sizeof(backup));
 731                                 goto inc;
 732                         }
 733                         ERROR("Operation rte_eth_stats_get failed for sub_device %d with error %d",
 734                                   i, ret);
 735                         *timestamp = 0;
 736                         fs_unlock(dev, 0);
 737                         return ret;
 738                 }
 739                 *timestamp = rte_rdtsc();
 740 inc:
 741                 failsafe_stats_increment(stats, snapshot);
 742         }
 743         fs_unlock(dev, 0);
 744         return 0;
 745 }
 746
 747 static void
 748 fs_stats_reset(struct rte_eth_dev *dev)
 749 {
 750         struct sub_device *sdev;
 751         uint8_t i;
 752
 753         fs_lock(dev, 0);
 754         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 755                 rte_eth_stats_reset(PORT_ID(sdev));
 756                 memset(&sdev->stats_snapshot, 0, sizeof(struct rte_eth_stats));
 757         }
 758         memset(&PRIV(dev)->stats_accumulator, 0, sizeof(struct rte_eth_stats));
 759         fs_unlock(dev, 0);
 760 }
 761
 762 /**
 763  * Fail-safe dev_infos_get rules:
 764  *
 765  * No sub_device:
 766  *   Numerables:
 767  *      Use the maximum possible values for any field, so as not
 768  *      to impede any further configuration effort.
 769  *   Capabilities:
 770  *      Limits capabilities to those that are understood by the
 771  *      fail-safe PMD. This understanding stems from the fail-safe
 772  *      being capable of verifying that the related capability is
 773  *      expressed within the device configuration (struct rte_eth_conf).
 774  *
 775  * At least one probed sub_device:
 776  *   Numerables:
 777  *      Uses values from the active probed sub_device
 778  *      The rationale here is that if any sub_device is less capable
 779  *      (for example concerning the number of queues) than the active
 780  *      sub_device, then its subsequent configuration will fail.
 781  *      It is impossible to foresee this failure when the failing sub_device
 782  *      is supposed to be plugged-in later on, so the configuration process
 783  *      is the single point of failure and error reporting.
 784  *   Capabilities:
 785  *      Uses a logical AND of RX capabilities among
 786  *      all sub_devices and the default capabilities.
 787  *      Uses a logical AND of TX capabilities among
 788  *      the active probed sub_device and the default capabilities.
 789  *
 790  */
 791 static void
 792 fs_dev_infos_get(struct rte_eth_dev *dev,
 793                   struct rte_eth_dev_info *infos)
 794 {
 795         struct sub_device *sdev;
 796         uint8_t i;
 797
 798         sdev = TX_SUBDEV(dev);
 799         if (sdev == NULL) {
 800                 DEBUG("No probed device, using default infos");
 801                 rte_memcpy(&PRIV(dev)->infos, &default_infos,
 802                            sizeof(default_infos));
 803         } else {
 804                 uint64_t rx_offload_capa;
 805                 uint64_t rxq_offload_capa;
 806
 807                 rx_offload_capa = default_infos.rx_offload_capa;
 808                 rxq_offload_capa = default_infos.rx_queue_offload_capa;
 809                 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) {
 810                         rte_eth_dev_info_get(PORT_ID(sdev),
 811                                         &PRIV(dev)->infos);
 812                         rx_offload_capa &= PRIV(dev)->infos.rx_offload_capa;
 813                         rxq_offload_capa &=
 814                                         PRIV(dev)->infos.rx_queue_offload_capa;
 815                 }
 816                 sdev = TX_SUBDEV(dev);
 817                 rte_eth_dev_info_get(PORT_ID(sdev), &PRIV(dev)->infos);
 818                 PRIV(dev)->infos.rx_offload_capa = rx_offload_capa;
 819                 PRIV(dev)->infos.rx_queue_offload_capa = rxq_offload_capa;
 820                 PRIV(dev)->infos.tx_offload_capa &=
 821                                         default_infos.tx_offload_capa;
 822                 PRIV(dev)->infos.tx_queue_offload_capa &=
 823                                         default_infos.tx_queue_offload_capa;
 824                 PRIV(dev)->infos.flow_type_rss_offloads &=
 825                                         default_infos.flow_type_rss_offloads;
 826         }
 827         rte_memcpy(infos, &PRIV(dev)->infos, sizeof(*infos));
 828 }
 829
 830 static const uint32_t *
 831 fs_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 832 {
 833         struct sub_device *sdev;
 834         struct rte_eth_dev *edev;
 835         const uint32_t *ret;
 836
 837         fs_lock(dev, 0);
 838         sdev = TX_SUBDEV(dev);
 839         if (sdev == NULL) {
 840                 ret = NULL;
 841                 goto unlock;
 842         }
 843         edev = ETH(sdev);
 844         /* ENOTSUP: counts as no supported ptypes */
 845         if (SUBOPS(sdev, dev_supported_ptypes_get) == NULL) {
 846                 ret = NULL;
 847                 goto unlock;
 848         }
 849         /*
 850          * The API does not permit to do a clean AND of all ptypes,
 851          * It is also incomplete by design and we do not really care
 852          * to have a best possible value in this context.
 853          * We just return the ptypes of the device of highest
 854          * priority, usually the PREFERRED device.
 855          */
 856         ret = SUBOPS(sdev, dev_supported_ptypes_get)(edev);
 857 unlock:
 858         fs_unlock(dev, 0);
 859         return ret;
 860 }
 861
 862 static int
 863 fs_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 864 {
 865         struct sub_device *sdev;
 866         uint8_t i;
 867         int ret;
 868
 869         fs_lock(dev, 0);
 870         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 871                 DEBUG("Calling rte_eth_dev_set_mtu on sub_device %d", i);
 872                 ret = rte_eth_dev_set_mtu(PORT_ID(sdev), mtu);
 873                 if ((ret = fs_err(sdev, ret))) {
 874                         ERROR("Operation rte_eth_dev_set_mtu failed for sub_device %d with error %d",
 875                               i, ret);
 876                         fs_unlock(dev, 0);
 877                         return ret;
 878                 }
 879         }
 880         fs_unlock(dev, 0);
 881         return 0;
 882 }
 883
 884 static int
 885 fs_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 886 {
 887         struct sub_device *sdev;
 888         uint8_t i;
 889         int ret;
 890
 891         fs_lock(dev, 0);
 892         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 893                 DEBUG("Calling rte_eth_dev_vlan_filter on sub_device %d", i);
 894                 ret = rte_eth_dev_vlan_filter(PORT_ID(sdev), vlan_id, on);
 895                 if ((ret = fs_err(sdev, ret))) {
 896                         ERROR("Operation rte_eth_dev_vlan_filter failed for sub_device %d"
 897                               " with error %d", i, ret);
 898                         fs_unlock(dev, 0);
 899                         return ret;
 900                 }
 901         }
 902         fs_unlock(dev, 0);
 903         return 0;
 904 }
 905
 906 static int
 907 fs_flow_ctrl_get(struct rte_eth_dev *dev,
 908                 struct rte_eth_fc_conf *fc_conf)
 909 {
 910         struct sub_device *sdev;
 911         int ret;
 912
 913         fs_lock(dev, 0);
 914         sdev = TX_SUBDEV(dev);
 915         if (sdev == NULL) {
 916                 ret = 0;
 917                 goto unlock;
 918         }
 919         if (SUBOPS(sdev, flow_ctrl_get) == NULL) {
 920                 ret = -ENOTSUP;
 921                 goto unlock;
 922         }
 923         ret = SUBOPS(sdev, flow_ctrl_get)(ETH(sdev), fc_conf);
 924 unlock:
 925         fs_unlock(dev, 0);
 926         return ret;
 927 }
 928
 929 static int
 930 fs_flow_ctrl_set(struct rte_eth_dev *dev,
 931                 struct rte_eth_fc_conf *fc_conf)
 932 {
 933         struct sub_device *sdev;
 934         uint8_t i;
 935         int ret;
 936
 937         fs_lock(dev, 0);
 938         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 939                 DEBUG("Calling rte_eth_dev_flow_ctrl_set on sub_device %d", i);
 940                 ret = rte_eth_dev_flow_ctrl_set(PORT_ID(sdev), fc_conf);
 941                 if ((ret = fs_err(sdev, ret))) {
 942                         ERROR("Operation rte_eth_dev_flow_ctrl_set failed for sub_device %d"
 943                               " with error %d", i, ret);
 944                         fs_unlock(dev, 0);
 945                         return ret;
 946                 }
 947         }
 948         fs_unlock(dev, 0);
 949         return 0;
 950 }
 951
 952 static void
 953 fs_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
 954 {
 955         struct sub_device *sdev;
 956         uint8_t i;
 957
 958         fs_lock(dev, 0);
 959         /* No check: already done within the rte_eth_dev_mac_addr_remove
 960          * call for the fail-safe device.
 961          */
 962         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 963                 rte_eth_dev_mac_addr_remove(PORT_ID(sdev),
 964                                 &dev->data->mac_addrs[index]);
 965         PRIV(dev)->mac_addr_pool[index] = 0;
 966         fs_unlock(dev, 0);
 967 }
 968
 969 static int
 970 fs_mac_addr_add(struct rte_eth_dev *dev,
 971                 struct ether_addr *mac_addr,
 972                 uint32_t index,
 973                 uint32_t vmdq)
 974 {
 975         struct sub_device *sdev;
 976         int ret;
 977         uint8_t i;
 978
 979         RTE_ASSERT(index < FAILSAFE_MAX_ETHADDR);
 980         fs_lock(dev, 0);
 981         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 982                 ret = rte_eth_dev_mac_addr_add(PORT_ID(sdev), mac_addr, vmdq);
 983                 if ((ret = fs_err(sdev, ret))) {
 984                         ERROR("Operation rte_eth_dev_mac_addr_add failed for sub_device %"
 985                               PRIu8 " with error %d", i, ret);
 986                         fs_unlock(dev, 0);
 987                         return ret;
 988                 }
 989         }
 990         if (index >= PRIV(dev)->nb_mac_addr) {
 991                 DEBUG("Growing mac_addrs array");
 992                 PRIV(dev)->nb_mac_addr = index;
 993         }
 994         PRIV(dev)->mac_addr_pool[index] = vmdq;
 995         fs_unlock(dev, 0);
 996         return 0;
 997 }
 998
 999 static void
1000 fs_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
1001 {
1002         struct sub_device *sdev;
1003         uint8_t i;
1004
1005         fs_lock(dev, 0);
1006         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
1007                 rte_eth_dev_default_mac_addr_set(PORT_ID(sdev), mac_addr);
1008         fs_unlock(dev, 0);
1009 }
1010
1011 static int
1012 fs_filter_ctrl(struct rte_eth_dev *dev,
1013                 enum rte_filter_type type,
1014                 enum rte_filter_op op,
1015                 void *arg)
1016 {
1017         struct sub_device *sdev;
1018         uint8_t i;
1019         int ret;
1020
1021         if (type == RTE_ETH_FILTER_GENERIC &&
1022             op == RTE_ETH_FILTER_GET) {
1023                 *(const void **)arg = &fs_flow_ops;
1024                 return 0;
1025         }
1026         fs_lock(dev, 0);
1027         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
1028                 DEBUG("Calling rte_eth_dev_filter_ctrl on sub_device %d", i);
1029                 ret = rte_eth_dev_filter_ctrl(PORT_ID(sdev), type, op, arg);
1030                 if ((ret = fs_err(sdev, ret))) {
1031                         ERROR("Operation rte_eth_dev_filter_ctrl failed for sub_device %d"
1032                               " with error %d", i, ret);
1033                         fs_unlock(dev, 0);
1034                         return ret;
1035                 }
1036         }
1037         fs_unlock(dev, 0);
1038         return 0;
1039 }
1040
1041 const struct eth_dev_ops failsafe_ops = {
1042         .dev_configure = fs_dev_configure,
1043         .dev_start = fs_dev_start,
1044         .dev_stop = fs_dev_stop,
1045         .dev_set_link_down = fs_dev_set_link_down,
1046         .dev_set_link_up = fs_dev_set_link_up,
1047         .dev_close = fs_dev_close,
1048         .promiscuous_enable = fs_promiscuous_enable,
1049         .promiscuous_disable = fs_promiscuous_disable,
1050         .allmulticast_enable = fs_allmulticast_enable,
1051         .allmulticast_disable = fs_allmulticast_disable,
1052         .link_update = fs_link_update,
1053         .stats_get = fs_stats_get,
1054         .stats_reset = fs_stats_reset,
1055         .dev_infos_get = fs_dev_infos_get,
1056         .dev_supported_ptypes_get = fs_dev_supported_ptypes_get,
1057         .mtu_set = fs_mtu_set,
1058         .vlan_filter_set = fs_vlan_filter_set,
1059         .rx_queue_setup = fs_rx_queue_setup,
1060         .tx_queue_setup = fs_tx_queue_setup,
1061         .rx_queue_release = fs_rx_queue_release,
1062         .tx_queue_release = fs_tx_queue_release,
1063         .rx_queue_intr_enable = fs_rx_intr_enable,
1064         .rx_queue_intr_disable = fs_rx_intr_disable,
1065         .flow_ctrl_get = fs_flow_ctrl_get,
1066         .flow_ctrl_set = fs_flow_ctrl_set,
1067         .mac_addr_remove = fs_mac_addr_remove,
1068         .mac_addr_add = fs_mac_addr_add,
1069         .mac_addr_set = fs_mac_addr_set,
1070         .filter_ctrl = fs_filter_ctrl,
1071 };