drivers/net/failsafe/failsafe_ops.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2017 6WIND S.A.
   3  * Copyright 2017 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <stdbool.h>
   7 #include <stdint.h>
   8 #include <unistd.h>
   9
  10 #include <rte_debug.h>
  11 #include <rte_atomic.h>
  12 #include <rte_ethdev_driver.h>
  13 #include <rte_malloc.h>
  14 #include <rte_flow.h>
  15 #include <rte_cycles.h>
  16
  17 #include "failsafe_private.h"
  18
  19 static struct rte_eth_dev_info default_infos = {
  20         /* Max possible number of elements */
  21         .max_rx_pktlen = UINT32_MAX,
  22         .max_rx_queues = RTE_MAX_QUEUES_PER_PORT,
  23         .max_tx_queues = RTE_MAX_QUEUES_PER_PORT,
  24         .max_mac_addrs = FAILSAFE_MAX_ETHADDR,
  25         .max_hash_mac_addrs = UINT32_MAX,
  26         .max_vfs = UINT16_MAX,
  27         .max_vmdq_pools = UINT16_MAX,
  28         .rx_desc_lim = {
  29                 .nb_max = UINT16_MAX,
  30                 .nb_min = 0,
  31                 .nb_align = 1,
  32                 .nb_seg_max = UINT16_MAX,
  33                 .nb_mtu_seg_max = UINT16_MAX,
  34         },
  35         .tx_desc_lim = {
  36                 .nb_max = UINT16_MAX,
  37                 .nb_min = 0,
  38                 .nb_align = 1,
  39                 .nb_seg_max = UINT16_MAX,
  40                 .nb_mtu_seg_max = UINT16_MAX,
  41         },
  42         /*
  43          * Set of capabilities that can be verified upon
  44          * configuring a sub-device.
  45          */
  46         .rx_offload_capa =
  47                 DEV_RX_OFFLOAD_VLAN_STRIP |
  48                 DEV_RX_OFFLOAD_IPV4_CKSUM |
  49                 DEV_RX_OFFLOAD_UDP_CKSUM |
  50                 DEV_RX_OFFLOAD_TCP_CKSUM |
  51                 DEV_RX_OFFLOAD_TCP_LRO |
  52                 DEV_RX_OFFLOAD_QINQ_STRIP |
  53                 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
  54                 DEV_RX_OFFLOAD_MACSEC_STRIP |
  55                 DEV_RX_OFFLOAD_HEADER_SPLIT |
  56                 DEV_RX_OFFLOAD_VLAN_FILTER |
  57                 DEV_RX_OFFLOAD_VLAN_EXTEND |
  58                 DEV_RX_OFFLOAD_JUMBO_FRAME |
  59                 DEV_RX_OFFLOAD_CRC_STRIP |
  60                 DEV_RX_OFFLOAD_SCATTER |
  61                 DEV_RX_OFFLOAD_TIMESTAMP |
  62                 DEV_RX_OFFLOAD_SECURITY,
  63         .rx_queue_offload_capa =
  64                 DEV_RX_OFFLOAD_VLAN_STRIP |
  65                 DEV_RX_OFFLOAD_IPV4_CKSUM |
  66                 DEV_RX_OFFLOAD_UDP_CKSUM |
  67                 DEV_RX_OFFLOAD_TCP_CKSUM |
  68                 DEV_RX_OFFLOAD_TCP_LRO |
  69                 DEV_RX_OFFLOAD_QINQ_STRIP |
  70                 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
  71                 DEV_RX_OFFLOAD_MACSEC_STRIP |
  72                 DEV_RX_OFFLOAD_HEADER_SPLIT |
  73                 DEV_RX_OFFLOAD_VLAN_FILTER |
  74                 DEV_RX_OFFLOAD_VLAN_EXTEND |
  75                 DEV_RX_OFFLOAD_JUMBO_FRAME |
  76                 DEV_RX_OFFLOAD_CRC_STRIP |
  77                 DEV_RX_OFFLOAD_SCATTER |
  78                 DEV_RX_OFFLOAD_TIMESTAMP |
  79                 DEV_RX_OFFLOAD_SECURITY,
  80         .tx_offload_capa =
  81                 DEV_TX_OFFLOAD_MULTI_SEGS |
  82                 DEV_TX_OFFLOAD_IPV4_CKSUM |
  83                 DEV_TX_OFFLOAD_UDP_CKSUM |
  84                 DEV_TX_OFFLOAD_TCP_CKSUM |
  85                 DEV_TX_OFFLOAD_TCP_TSO,
  86         .flow_type_rss_offloads =
  87                         ETH_RSS_IP |
  88                         ETH_RSS_UDP |
  89                         ETH_RSS_TCP,
  90 };
  91
  92 static int
  93 fs_dev_configure(struct rte_eth_dev *dev)
  94 {
  95         struct sub_device *sdev;
  96         uint64_t supp_tx_offloads;
  97         uint64_t tx_offloads;
  98         uint8_t i;
  99         int ret;
 100
 101         fs_lock(dev, 0);
 102         supp_tx_offloads = PRIV(dev)->infos.tx_offload_capa;
 103         tx_offloads = dev->data->dev_conf.txmode.offloads;
 104         if ((tx_offloads & supp_tx_offloads) != tx_offloads) {
 105                 rte_errno = ENOTSUP;
 106                 ERROR("Some Tx offloads are not supported, "
 107                       "requested 0x%" PRIx64 " supported 0x%" PRIx64,
 108                       tx_offloads, supp_tx_offloads);
 109                 fs_unlock(dev, 0);
 110                 return -rte_errno;
 111         }
 112         FOREACH_SUBDEV(sdev, i, dev) {
 113                 int rmv_interrupt = 0;
 114                 int lsc_interrupt = 0;
 115                 int lsc_enabled;
 116
 117                 if (sdev->state != DEV_PROBED &&
 118                     !(PRIV(dev)->alarm_lock == 0 && sdev->state == DEV_ACTIVE))
 119                         continue;
 120
 121                 rmv_interrupt = ETH(sdev)->data->dev_flags &
 122                                 RTE_ETH_DEV_INTR_RMV;
 123                 if (rmv_interrupt) {
 124                         DEBUG("Enabling RMV interrupts for sub_device %d", i);
 125                         dev->data->dev_conf.intr_conf.rmv = 1;
 126                 } else {
 127                         DEBUG("sub_device %d does not support RMV event", i);
 128                 }
 129                 lsc_enabled = dev->data->dev_conf.intr_conf.lsc;
 130                 lsc_interrupt = lsc_enabled &&
 131                                 (ETH(sdev)->data->dev_flags &
 132                                  RTE_ETH_DEV_INTR_LSC);
 133                 if (lsc_interrupt) {
 134                         DEBUG("Enabling LSC interrupts for sub_device %d", i);
 135                         dev->data->dev_conf.intr_conf.lsc = 1;
 136                 } else if (lsc_enabled && !lsc_interrupt) {
 137                         DEBUG("Disabling LSC interrupts for sub_device %d", i);
 138                         dev->data->dev_conf.intr_conf.lsc = 0;
 139                 }
 140                 DEBUG("Configuring sub-device %d", i);
 141                 ret = rte_eth_dev_configure(PORT_ID(sdev),
 142                                         dev->data->nb_rx_queues,
 143                                         dev->data->nb_tx_queues,
 144                                         &dev->data->dev_conf);
 145                 if (ret) {
 146                         if (!fs_err(sdev, ret))
 147                                 continue;
 148                         ERROR("Could not configure sub_device %d", i);
 149                         fs_unlock(dev, 0);
 150                         return ret;
 151                 }
 152                 if (rmv_interrupt) {
 153                         ret = rte_eth_dev_callback_register(PORT_ID(sdev),
 154                                         RTE_ETH_EVENT_INTR_RMV,
 155                                         failsafe_eth_rmv_event_callback,
 156                                         sdev);
 157                         if (ret)
 158                                 WARN("Failed to register RMV callback for sub_device %d",
 159                                      SUB_ID(sdev));
 160                 }
 161                 dev->data->dev_conf.intr_conf.rmv = 0;
 162                 if (lsc_interrupt) {
 163                         ret = rte_eth_dev_callback_register(PORT_ID(sdev),
 164                                                 RTE_ETH_EVENT_INTR_LSC,
 165                                                 failsafe_eth_lsc_event_callback,
 166                                                 dev);
 167                         if (ret)
 168                                 WARN("Failed to register LSC callback for sub_device %d",
 169                                      SUB_ID(sdev));
 170                 }
 171                 dev->data->dev_conf.intr_conf.lsc = lsc_enabled;
 172                 sdev->state = DEV_ACTIVE;
 173         }
 174         if (PRIV(dev)->state < DEV_ACTIVE)
 175                 PRIV(dev)->state = DEV_ACTIVE;
 176         fs_unlock(dev, 0);
 177         return 0;
 178 }
 179
 180 static int
 181 fs_dev_start(struct rte_eth_dev *dev)
 182 {
 183         struct sub_device *sdev;
 184         uint8_t i;
 185         int ret;
 186
 187         fs_lock(dev, 0);
 188         ret = failsafe_rx_intr_install(dev);
 189         if (ret) {
 190                 fs_unlock(dev, 0);
 191                 return ret;
 192         }
 193         FOREACH_SUBDEV(sdev, i, dev) {
 194                 if (sdev->state != DEV_ACTIVE)
 195                         continue;
 196                 DEBUG("Starting sub_device %d", i);
 197                 ret = rte_eth_dev_start(PORT_ID(sdev));
 198                 if (ret) {
 199                         if (!fs_err(sdev, ret))
 200                                 continue;
 201                         fs_unlock(dev, 0);
 202                         return ret;
 203                 }
 204                 ret = failsafe_rx_intr_install_subdevice(sdev);
 205                 if (ret) {
 206                         if (!fs_err(sdev, ret))
 207                                 continue;
 208                         rte_eth_dev_stop(PORT_ID(sdev));
 209                         fs_unlock(dev, 0);
 210                         return ret;
 211                 }
 212                 sdev->state = DEV_STARTED;
 213         }
 214         if (PRIV(dev)->state < DEV_STARTED)
 215                 PRIV(dev)->state = DEV_STARTED;
 216         fs_switch_dev(dev, NULL);
 217         fs_unlock(dev, 0);
 218         return 0;
 219 }
 220
 221 static void
 222 fs_dev_stop(struct rte_eth_dev *dev)
 223 {
 224         struct sub_device *sdev;
 225         uint8_t i;
 226
 227         fs_lock(dev, 0);
 228         PRIV(dev)->state = DEV_STARTED - 1;
 229         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_STARTED) {
 230                 rte_eth_dev_stop(PORT_ID(sdev));
 231                 failsafe_rx_intr_uninstall_subdevice(sdev);
 232                 sdev->state = DEV_STARTED - 1;
 233         }
 234         failsafe_rx_intr_uninstall(dev);
 235         fs_unlock(dev, 0);
 236 }
 237
 238 static int
 239 fs_dev_set_link_up(struct rte_eth_dev *dev)
 240 {
 241         struct sub_device *sdev;
 242         uint8_t i;
 243         int ret;
 244
 245         fs_lock(dev, 0);
 246         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 247                 DEBUG("Calling rte_eth_dev_set_link_up on sub_device %d", i);
 248                 ret = rte_eth_dev_set_link_up(PORT_ID(sdev));
 249                 if ((ret = fs_err(sdev, ret))) {
 250                         ERROR("Operation rte_eth_dev_set_link_up failed for sub_device %d"
 251                               " with error %d", i, ret);
 252                         fs_unlock(dev, 0);
 253                         return ret;
 254                 }
 255         }
 256         fs_unlock(dev, 0);
 257         return 0;
 258 }
 259
 260 static int
 261 fs_dev_set_link_down(struct rte_eth_dev *dev)
 262 {
 263         struct sub_device *sdev;
 264         uint8_t i;
 265         int ret;
 266
 267         fs_lock(dev, 0);
 268         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 269                 DEBUG("Calling rte_eth_dev_set_link_down on sub_device %d", i);
 270                 ret = rte_eth_dev_set_link_down(PORT_ID(sdev));
 271                 if ((ret = fs_err(sdev, ret))) {
 272                         ERROR("Operation rte_eth_dev_set_link_down failed for sub_device %d"
 273                               " with error %d", i, ret);
 274                         fs_unlock(dev, 0);
 275                         return ret;
 276                 }
 277         }
 278         fs_unlock(dev, 0);
 279         return 0;
 280 }
 281
 282 static void fs_dev_free_queues(struct rte_eth_dev *dev);
 283 static void
 284 fs_dev_close(struct rte_eth_dev *dev)
 285 {
 286         struct sub_device *sdev;
 287         uint8_t i;
 288
 289         fs_lock(dev, 0);
 290         failsafe_hotplug_alarm_cancel(dev);
 291         if (PRIV(dev)->state == DEV_STARTED)
 292                 dev->dev_ops->dev_stop(dev);
 293         PRIV(dev)->state = DEV_ACTIVE - 1;
 294         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 295                 DEBUG("Closing sub_device %d", i);
 296                 rte_eth_dev_close(PORT_ID(sdev));
 297                 sdev->state = DEV_ACTIVE - 1;
 298         }
 299         fs_dev_free_queues(dev);
 300         fs_unlock(dev, 0);
 301 }
 302
 303 static bool
 304 fs_rxq_offloads_valid(struct rte_eth_dev *dev, uint64_t offloads)
 305 {
 306         uint64_t port_offloads;
 307         uint64_t queue_supp_offloads;
 308         uint64_t port_supp_offloads;
 309
 310         port_offloads = dev->data->dev_conf.rxmode.offloads;
 311         queue_supp_offloads = PRIV(dev)->infos.rx_queue_offload_capa;
 312         port_supp_offloads = PRIV(dev)->infos.rx_offload_capa;
 313         if ((offloads & (queue_supp_offloads | port_supp_offloads)) !=
 314              offloads)
 315                 return false;
 316         /* Verify we have no conflict with port offloads */
 317         if ((port_offloads ^ offloads) & port_supp_offloads)
 318                 return false;
 319         return true;
 320 }
 321
 322 static void
 323 fs_rx_queue_release(void *queue)
 324 {
 325         struct rte_eth_dev *dev;
 326         struct sub_device *sdev;
 327         uint8_t i;
 328         struct rxq *rxq;
 329
 330         if (queue == NULL)
 331                 return;
 332         rxq = queue;
 333         dev = rxq->priv->dev;
 334         fs_lock(dev, 0);
 335         if (rxq->event_fd > 0)
 336                 close(rxq->event_fd);
 337         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 338                 SUBOPS(sdev, rx_queue_release)
 339                         (ETH(sdev)->data->rx_queues[rxq->qid]);
 340         dev->data->rx_queues[rxq->qid] = NULL;
 341         rte_free(rxq);
 342         fs_unlock(dev, 0);
 343 }
 344
 345 static int
 346 fs_rx_queue_setup(struct rte_eth_dev *dev,
 347                 uint16_t rx_queue_id,
 348                 uint16_t nb_rx_desc,
 349                 unsigned int socket_id,
 350                 const struct rte_eth_rxconf *rx_conf,
 351                 struct rte_mempool *mb_pool)
 352 {
 353         /*
 354          * FIXME: Add a proper interface in rte_eal_interrupts for
 355          * allocating eventfd as an interrupt vector.
 356          * For the time being, fake as if we are using MSIX interrupts,
 357          * this will cause rte_intr_efd_enable to allocate an eventfd for us.
 358          */
 359         struct rte_intr_handle intr_handle = {
 360                 .type = RTE_INTR_HANDLE_VFIO_MSIX,
 361                 .efds = { -1, },
 362         };
 363         struct sub_device *sdev;
 364         struct rxq *rxq;
 365         uint8_t i;
 366         int ret;
 367
 368         fs_lock(dev, 0);
 369         rxq = dev->data->rx_queues[rx_queue_id];
 370         if (rxq != NULL) {
 371                 fs_rx_queue_release(rxq);
 372                 dev->data->rx_queues[rx_queue_id] = NULL;
 373         }
 374         /* Verify application offloads are valid for our port and queue. */
 375         if (fs_rxq_offloads_valid(dev, rx_conf->offloads) == false) {
 376                 rte_errno = ENOTSUP;
 377                 ERROR("Rx queue offloads 0x%" PRIx64
 378                       " don't match port offloads 0x%" PRIx64
 379                       " or supported offloads 0x%" PRIx64,
 380                       rx_conf->offloads,
 381                       dev->data->dev_conf.rxmode.offloads,
 382                       PRIV(dev)->infos.rx_offload_capa |
 383                       PRIV(dev)->infos.rx_queue_offload_capa);
 384                 fs_unlock(dev, 0);
 385                 return -rte_errno;
 386         }
 387         rxq = rte_zmalloc(NULL,
 388                           sizeof(*rxq) +
 389                           sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail,
 390                           RTE_CACHE_LINE_SIZE);
 391         if (rxq == NULL) {
 392                 fs_unlock(dev, 0);
 393                 return -ENOMEM;
 394         }
 395         FOREACH_SUBDEV(sdev, i, dev)
 396                 rte_atomic64_init(&rxq->refcnt[i]);
 397         rxq->qid = rx_queue_id;
 398         rxq->socket_id = socket_id;
 399         rxq->info.mp = mb_pool;
 400         rxq->info.conf = *rx_conf;
 401         rxq->info.nb_desc = nb_rx_desc;
 402         rxq->priv = PRIV(dev);
 403         rxq->sdev = PRIV(dev)->subs;
 404         ret = rte_intr_efd_enable(&intr_handle, 1);
 405         if (ret < 0) {
 406                 fs_unlock(dev, 0);
 407                 return ret;
 408         }
 409         rxq->event_fd = intr_handle.efds[0];
 410         dev->data->rx_queues[rx_queue_id] = rxq;
 411         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 412                 ret = rte_eth_rx_queue_setup(PORT_ID(sdev),
 413                                 rx_queue_id,
 414                                 nb_rx_desc, socket_id,
 415                                 rx_conf, mb_pool);
 416                 if ((ret = fs_err(sdev, ret))) {
 417                         ERROR("RX queue setup failed for sub_device %d", i);
 418                         goto free_rxq;
 419                 }
 420         }
 421         fs_unlock(dev, 0);
 422         return 0;
 423 free_rxq:
 424         fs_rx_queue_release(rxq);
 425         fs_unlock(dev, 0);
 426         return ret;
 427 }
 428
 429 static int
 430 fs_rx_intr_enable(struct rte_eth_dev *dev, uint16_t idx)
 431 {
 432         struct rxq *rxq;
 433         struct sub_device *sdev;
 434         uint8_t i;
 435         int ret;
 436         int rc = 0;
 437
 438         fs_lock(dev, 0);
 439         if (idx >= dev->data->nb_rx_queues) {
 440                 rc = -EINVAL;
 441                 goto unlock;
 442         }
 443         rxq = dev->data->rx_queues[idx];
 444         if (rxq == NULL || rxq->event_fd <= 0) {
 445                 rc = -EINVAL;
 446                 goto unlock;
 447         }
 448         /* Fail if proxy service is nor running. */
 449         if (PRIV(dev)->rxp.sstate != SS_RUNNING) {
 450                 ERROR("failsafe interrupt services are not running");
 451                 rc = -EAGAIN;
 452                 goto unlock;
 453         }
 454         rxq->enable_events = 1;
 455         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 456                 ret = rte_eth_dev_rx_intr_enable(PORT_ID(sdev), idx);
 457                 ret = fs_err(sdev, ret);
 458                 if (ret)
 459                         rc = ret;
 460         }
 461 unlock:
 462         fs_unlock(dev, 0);
 463         if (rc)
 464                 rte_errno = -rc;
 465         return rc;
 466 }
 467
 468 static int
 469 fs_rx_intr_disable(struct rte_eth_dev *dev, uint16_t idx)
 470 {
 471         struct rxq *rxq;
 472         struct sub_device *sdev;
 473         uint64_t u64;
 474         uint8_t i;
 475         int rc = 0;
 476         int ret;
 477
 478         fs_lock(dev, 0);
 479         if (idx >= dev->data->nb_rx_queues) {
 480                 rc = -EINVAL;
 481                 goto unlock;
 482         }
 483         rxq = dev->data->rx_queues[idx];
 484         if (rxq == NULL || rxq->event_fd <= 0) {
 485                 rc = -EINVAL;
 486                 goto unlock;
 487         }
 488         rxq->enable_events = 0;
 489         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 490                 ret = rte_eth_dev_rx_intr_disable(PORT_ID(sdev), idx);
 491                 ret = fs_err(sdev, ret);
 492                 if (ret)
 493                         rc = ret;
 494         }
 495         /* Clear pending events */
 496         while (read(rxq->event_fd, &u64, sizeof(uint64_t)) >  0)
 497                 ;
 498 unlock:
 499         fs_unlock(dev, 0);
 500         if (rc)
 501                 rte_errno = -rc;
 502         return rc;
 503 }
 504
 505 static bool
 506 fs_txq_offloads_valid(struct rte_eth_dev *dev, uint64_t offloads)
 507 {
 508         uint64_t port_offloads;
 509         uint64_t queue_supp_offloads;
 510         uint64_t port_supp_offloads;
 511
 512         port_offloads = dev->data->dev_conf.txmode.offloads;
 513         queue_supp_offloads = PRIV(dev)->infos.tx_queue_offload_capa;
 514         port_supp_offloads = PRIV(dev)->infos.tx_offload_capa;
 515         if ((offloads & (queue_supp_offloads | port_supp_offloads)) !=
 516              offloads)
 517                 return false;
 518         /* Verify we have no conflict with port offloads */
 519         if ((port_offloads ^ offloads) & port_supp_offloads)
 520                 return false;
 521         return true;
 522 }
 523
 524 static void
 525 fs_tx_queue_release(void *queue)
 526 {
 527         struct rte_eth_dev *dev;
 528         struct sub_device *sdev;
 529         uint8_t i;
 530         struct txq *txq;
 531
 532         if (queue == NULL)
 533                 return;
 534         txq = queue;
 535         dev = txq->priv->dev;
 536         fs_lock(dev, 0);
 537         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 538                 SUBOPS(sdev, tx_queue_release)
 539                         (ETH(sdev)->data->tx_queues[txq->qid]);
 540         dev->data->tx_queues[txq->qid] = NULL;
 541         rte_free(txq);
 542         fs_unlock(dev, 0);
 543 }
 544
 545 static int
 546 fs_tx_queue_setup(struct rte_eth_dev *dev,
 547                 uint16_t tx_queue_id,
 548                 uint16_t nb_tx_desc,
 549                 unsigned int socket_id,
 550                 const struct rte_eth_txconf *tx_conf)
 551 {
 552         struct sub_device *sdev;
 553         struct txq *txq;
 554         uint8_t i;
 555         int ret;
 556
 557         fs_lock(dev, 0);
 558         txq = dev->data->tx_queues[tx_queue_id];
 559         if (txq != NULL) {
 560                 fs_tx_queue_release(txq);
 561                 dev->data->tx_queues[tx_queue_id] = NULL;
 562         }
 563         /*
 564          * Don't verify queue offloads for applications which
 565          * use the old API.
 566          */
 567         if (tx_conf != NULL &&
 568             (tx_conf->txq_flags & ETH_TXQ_FLAGS_IGNORE) &&
 569             fs_txq_offloads_valid(dev, tx_conf->offloads) == false) {
 570                 rte_errno = ENOTSUP;
 571                 ERROR("Tx queue offloads 0x%" PRIx64
 572                       " don't match port offloads 0x%" PRIx64
 573                       " or supported offloads 0x%" PRIx64,
 574                       tx_conf->offloads,
 575                       dev->data->dev_conf.txmode.offloads,
 576                       PRIV(dev)->infos.tx_offload_capa |
 577                       PRIV(dev)->infos.tx_queue_offload_capa);
 578                 fs_unlock(dev, 0);
 579                 return -rte_errno;
 580         }
 581         txq = rte_zmalloc("ethdev TX queue",
 582                           sizeof(*txq) +
 583                           sizeof(rte_atomic64_t) * PRIV(dev)->subs_tail,
 584                           RTE_CACHE_LINE_SIZE);
 585         if (txq == NULL) {
 586                 fs_unlock(dev, 0);
 587                 return -ENOMEM;
 588         }
 589         FOREACH_SUBDEV(sdev, i, dev)
 590                 rte_atomic64_init(&txq->refcnt[i]);
 591         txq->qid = tx_queue_id;
 592         txq->socket_id = socket_id;
 593         txq->info.conf = *tx_conf;
 594         txq->info.nb_desc = nb_tx_desc;
 595         txq->priv = PRIV(dev);
 596         dev->data->tx_queues[tx_queue_id] = txq;
 597         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 598                 ret = rte_eth_tx_queue_setup(PORT_ID(sdev),
 599                                 tx_queue_id,
 600                                 nb_tx_desc, socket_id,
 601                                 tx_conf);
 602                 if ((ret = fs_err(sdev, ret))) {
 603                         ERROR("TX queue setup failed for sub_device %d", i);
 604                         goto free_txq;
 605                 }
 606         }
 607         fs_unlock(dev, 0);
 608         return 0;
 609 free_txq:
 610         fs_tx_queue_release(txq);
 611         fs_unlock(dev, 0);
 612         return ret;
 613 }
 614
 615 static void
 616 fs_dev_free_queues(struct rte_eth_dev *dev)
 617 {
 618         uint16_t i;
 619
 620         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 621                 fs_rx_queue_release(dev->data->rx_queues[i]);
 622                 dev->data->rx_queues[i] = NULL;
 623         }
 624         dev->data->nb_rx_queues = 0;
 625         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 626                 fs_tx_queue_release(dev->data->tx_queues[i]);
 627                 dev->data->tx_queues[i] = NULL;
 628         }
 629         dev->data->nb_tx_queues = 0;
 630 }
 631
 632 static void
 633 fs_promiscuous_enable(struct rte_eth_dev *dev)
 634 {
 635         struct sub_device *sdev;
 636         uint8_t i;
 637
 638         fs_lock(dev, 0);
 639         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 640                 rte_eth_promiscuous_enable(PORT_ID(sdev));
 641         fs_unlock(dev, 0);
 642 }
 643
 644 static void
 645 fs_promiscuous_disable(struct rte_eth_dev *dev)
 646 {
 647         struct sub_device *sdev;
 648         uint8_t i;
 649
 650         fs_lock(dev, 0);
 651         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 652                 rte_eth_promiscuous_disable(PORT_ID(sdev));
 653         fs_unlock(dev, 0);
 654 }
 655
 656 static void
 657 fs_allmulticast_enable(struct rte_eth_dev *dev)
 658 {
 659         struct sub_device *sdev;
 660         uint8_t i;
 661
 662         fs_lock(dev, 0);
 663         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 664                 rte_eth_allmulticast_enable(PORT_ID(sdev));
 665         fs_unlock(dev, 0);
 666 }
 667
 668 static void
 669 fs_allmulticast_disable(struct rte_eth_dev *dev)
 670 {
 671         struct sub_device *sdev;
 672         uint8_t i;
 673
 674         fs_lock(dev, 0);
 675         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 676                 rte_eth_allmulticast_disable(PORT_ID(sdev));
 677         fs_unlock(dev, 0);
 678 }
 679
 680 static int
 681 fs_link_update(struct rte_eth_dev *dev,
 682                 int wait_to_complete)
 683 {
 684         struct sub_device *sdev;
 685         uint8_t i;
 686         int ret;
 687
 688         fs_lock(dev, 0);
 689         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 690                 DEBUG("Calling link_update on sub_device %d", i);
 691                 ret = (SUBOPS(sdev, link_update))(ETH(sdev), wait_to_complete);
 692                 if (ret && ret != -1 && sdev->remove == 0 &&
 693                     rte_eth_dev_is_removed(PORT_ID(sdev)) == 0) {
 694                         ERROR("Link update failed for sub_device %d with error %d",
 695                               i, ret);
 696                         fs_unlock(dev, 0);
 697                         return ret;
 698                 }
 699         }
 700         if (TX_SUBDEV(dev)) {
 701                 struct rte_eth_link *l1;
 702                 struct rte_eth_link *l2;
 703
 704                 l1 = &dev->data->dev_link;
 705                 l2 = &ETH(TX_SUBDEV(dev))->data->dev_link;
 706                 if (memcmp(l1, l2, sizeof(*l1))) {
 707                         *l1 = *l2;
 708                         fs_unlock(dev, 0);
 709                         return 0;
 710                 }
 711         }
 712         fs_unlock(dev, 0);
 713         return -1;
 714 }
 715
 716 static int
 717 fs_stats_get(struct rte_eth_dev *dev,
 718              struct rte_eth_stats *stats)
 719 {
 720         struct rte_eth_stats backup;
 721         struct sub_device *sdev;
 722         uint8_t i;
 723         int ret;
 724
 725         fs_lock(dev, 0);
 726         rte_memcpy(stats, &PRIV(dev)->stats_accumulator, sizeof(*stats));
 727         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 728                 struct rte_eth_stats *snapshot = &sdev->stats_snapshot.stats;
 729                 uint64_t *timestamp = &sdev->stats_snapshot.timestamp;
 730
 731                 rte_memcpy(&backup, snapshot, sizeof(backup));
 732                 ret = rte_eth_stats_get(PORT_ID(sdev), snapshot);
 733                 if (ret) {
 734                         if (!fs_err(sdev, ret)) {
 735                                 rte_memcpy(snapshot, &backup, sizeof(backup));
 736                                 goto inc;
 737                         }
 738                         ERROR("Operation rte_eth_stats_get failed for sub_device %d with error %d",
 739                                   i, ret);
 740                         *timestamp = 0;
 741                         fs_unlock(dev, 0);
 742                         return ret;
 743                 }
 744                 *timestamp = rte_rdtsc();
 745 inc:
 746                 failsafe_stats_increment(stats, snapshot);
 747         }
 748         fs_unlock(dev, 0);
 749         return 0;
 750 }
 751
 752 static void
 753 fs_stats_reset(struct rte_eth_dev *dev)
 754 {
 755         struct sub_device *sdev;
 756         uint8_t i;
 757
 758         fs_lock(dev, 0);
 759         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 760                 rte_eth_stats_reset(PORT_ID(sdev));
 761                 memset(&sdev->stats_snapshot, 0, sizeof(struct rte_eth_stats));
 762         }
 763         memset(&PRIV(dev)->stats_accumulator, 0, sizeof(struct rte_eth_stats));
 764         fs_unlock(dev, 0);
 765 }
 766
 767 /**
 768  * Fail-safe dev_infos_get rules:
 769  *
 770  * No sub_device:
 771  *   Numerables:
 772  *      Use the maximum possible values for any field, so as not
 773  *      to impede any further configuration effort.
 774  *   Capabilities:
 775  *      Limits capabilities to those that are understood by the
 776  *      fail-safe PMD. This understanding stems from the fail-safe
 777  *      being capable of verifying that the related capability is
 778  *      expressed within the device configuration (struct rte_eth_conf).
 779  *
 780  * At least one probed sub_device:
 781  *   Numerables:
 782  *      Uses values from the active probed sub_device
 783  *      The rationale here is that if any sub_device is less capable
 784  *      (for example concerning the number of queues) than the active
 785  *      sub_device, then its subsequent configuration will fail.
 786  *      It is impossible to foresee this failure when the failing sub_device
 787  *      is supposed to be plugged-in later on, so the configuration process
 788  *      is the single point of failure and error reporting.
 789  *   Capabilities:
 790  *      Uses a logical AND of RX capabilities among
 791  *      all sub_devices and the default capabilities.
 792  *      Uses a logical AND of TX capabilities among
 793  *      the active probed sub_device and the default capabilities.
 794  *
 795  */
 796 static void
 797 fs_dev_infos_get(struct rte_eth_dev *dev,
 798                   struct rte_eth_dev_info *infos)
 799 {
 800         struct sub_device *sdev;
 801         uint8_t i;
 802
 803         sdev = TX_SUBDEV(dev);
 804         if (sdev == NULL) {
 805                 DEBUG("No probed device, using default infos");
 806                 rte_memcpy(&PRIV(dev)->infos, &default_infos,
 807                            sizeof(default_infos));
 808         } else {
 809                 uint64_t rx_offload_capa;
 810                 uint64_t rxq_offload_capa;
 811                 uint64_t rss_hf_offload_capa;
 812
 813                 rx_offload_capa = default_infos.rx_offload_capa;
 814                 rxq_offload_capa = default_infos.rx_queue_offload_capa;
 815                 rss_hf_offload_capa = default_infos.flow_type_rss_offloads;
 816                 FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_PROBED) {
 817                         rte_eth_dev_info_get(PORT_ID(sdev),
 818                                         &PRIV(dev)->infos);
 819                         rx_offload_capa &= PRIV(dev)->infos.rx_offload_capa;
 820                         rxq_offload_capa &=
 821                                         PRIV(dev)->infos.rx_queue_offload_capa;
 822                         rss_hf_offload_capa &=
 823                                         PRIV(dev)->infos.flow_type_rss_offloads;
 824                 }
 825                 sdev = TX_SUBDEV(dev);
 826                 rte_eth_dev_info_get(PORT_ID(sdev), &PRIV(dev)->infos);
 827                 PRIV(dev)->infos.rx_offload_capa = rx_offload_capa;
 828                 PRIV(dev)->infos.rx_queue_offload_capa = rxq_offload_capa;
 829                 PRIV(dev)->infos.flow_type_rss_offloads = rss_hf_offload_capa;
 830                 PRIV(dev)->infos.tx_offload_capa &=
 831                                         default_infos.tx_offload_capa;
 832                 PRIV(dev)->infos.tx_queue_offload_capa &=
 833                                         default_infos.tx_queue_offload_capa;
 834         }
 835         rte_memcpy(infos, &PRIV(dev)->infos, sizeof(*infos));
 836 }
 837
 838 static const uint32_t *
 839 fs_dev_supported_ptypes_get(struct rte_eth_dev *dev)
 840 {
 841         struct sub_device *sdev;
 842         struct rte_eth_dev *edev;
 843         const uint32_t *ret;
 844
 845         fs_lock(dev, 0);
 846         sdev = TX_SUBDEV(dev);
 847         if (sdev == NULL) {
 848                 ret = NULL;
 849                 goto unlock;
 850         }
 851         edev = ETH(sdev);
 852         /* ENOTSUP: counts as no supported ptypes */
 853         if (SUBOPS(sdev, dev_supported_ptypes_get) == NULL) {
 854                 ret = NULL;
 855                 goto unlock;
 856         }
 857         /*
 858          * The API does not permit to do a clean AND of all ptypes,
 859          * It is also incomplete by design and we do not really care
 860          * to have a best possible value in this context.
 861          * We just return the ptypes of the device of highest
 862          * priority, usually the PREFERRED device.
 863          */
 864         ret = SUBOPS(sdev, dev_supported_ptypes_get)(edev);
 865 unlock:
 866         fs_unlock(dev, 0);
 867         return ret;
 868 }
 869
 870 static int
 871 fs_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 872 {
 873         struct sub_device *sdev;
 874         uint8_t i;
 875         int ret;
 876
 877         fs_lock(dev, 0);
 878         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 879                 DEBUG("Calling rte_eth_dev_set_mtu on sub_device %d", i);
 880                 ret = rte_eth_dev_set_mtu(PORT_ID(sdev), mtu);
 881                 if ((ret = fs_err(sdev, ret))) {
 882                         ERROR("Operation rte_eth_dev_set_mtu failed for sub_device %d with error %d",
 883                               i, ret);
 884                         fs_unlock(dev, 0);
 885                         return ret;
 886                 }
 887         }
 888         fs_unlock(dev, 0);
 889         return 0;
 890 }
 891
 892 static int
 893 fs_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 894 {
 895         struct sub_device *sdev;
 896         uint8_t i;
 897         int ret;
 898
 899         fs_lock(dev, 0);
 900         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 901                 DEBUG("Calling rte_eth_dev_vlan_filter on sub_device %d", i);
 902                 ret = rte_eth_dev_vlan_filter(PORT_ID(sdev), vlan_id, on);
 903                 if ((ret = fs_err(sdev, ret))) {
 904                         ERROR("Operation rte_eth_dev_vlan_filter failed for sub_device %d"
 905                               " with error %d", i, ret);
 906                         fs_unlock(dev, 0);
 907                         return ret;
 908                 }
 909         }
 910         fs_unlock(dev, 0);
 911         return 0;
 912 }
 913
 914 static int
 915 fs_flow_ctrl_get(struct rte_eth_dev *dev,
 916                 struct rte_eth_fc_conf *fc_conf)
 917 {
 918         struct sub_device *sdev;
 919         int ret;
 920
 921         fs_lock(dev, 0);
 922         sdev = TX_SUBDEV(dev);
 923         if (sdev == NULL) {
 924                 ret = 0;
 925                 goto unlock;
 926         }
 927         if (SUBOPS(sdev, flow_ctrl_get) == NULL) {
 928                 ret = -ENOTSUP;
 929                 goto unlock;
 930         }
 931         ret = SUBOPS(sdev, flow_ctrl_get)(ETH(sdev), fc_conf);
 932 unlock:
 933         fs_unlock(dev, 0);
 934         return ret;
 935 }
 936
 937 static int
 938 fs_flow_ctrl_set(struct rte_eth_dev *dev,
 939                 struct rte_eth_fc_conf *fc_conf)
 940 {
 941         struct sub_device *sdev;
 942         uint8_t i;
 943         int ret;
 944
 945         fs_lock(dev, 0);
 946         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 947                 DEBUG("Calling rte_eth_dev_flow_ctrl_set on sub_device %d", i);
 948                 ret = rte_eth_dev_flow_ctrl_set(PORT_ID(sdev), fc_conf);
 949                 if ((ret = fs_err(sdev, ret))) {
 950                         ERROR("Operation rte_eth_dev_flow_ctrl_set failed for sub_device %d"
 951                               " with error %d", i, ret);
 952                         fs_unlock(dev, 0);
 953                         return ret;
 954                 }
 955         }
 956         fs_unlock(dev, 0);
 957         return 0;
 958 }
 959
 960 static void
 961 fs_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
 962 {
 963         struct sub_device *sdev;
 964         uint8_t i;
 965
 966         fs_lock(dev, 0);
 967         /* No check: already done within the rte_eth_dev_mac_addr_remove
 968          * call for the fail-safe device.
 969          */
 970         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 971                 rte_eth_dev_mac_addr_remove(PORT_ID(sdev),
 972                                 &dev->data->mac_addrs[index]);
 973         PRIV(dev)->mac_addr_pool[index] = 0;
 974         fs_unlock(dev, 0);
 975 }
 976
 977 static int
 978 fs_mac_addr_add(struct rte_eth_dev *dev,
 979                 struct ether_addr *mac_addr,
 980                 uint32_t index,
 981                 uint32_t vmdq)
 982 {
 983         struct sub_device *sdev;
 984         int ret;
 985         uint8_t i;
 986
 987         RTE_ASSERT(index < FAILSAFE_MAX_ETHADDR);
 988         fs_lock(dev, 0);
 989         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
 990                 ret = rte_eth_dev_mac_addr_add(PORT_ID(sdev), mac_addr, vmdq);
 991                 if ((ret = fs_err(sdev, ret))) {
 992                         ERROR("Operation rte_eth_dev_mac_addr_add failed for sub_device %"
 993                               PRIu8 " with error %d", i, ret);
 994                         fs_unlock(dev, 0);
 995                         return ret;
 996                 }
 997         }
 998         if (index >= PRIV(dev)->nb_mac_addr) {
 999                 DEBUG("Growing mac_addrs array");
1000                 PRIV(dev)->nb_mac_addr = index;
1001         }
1002         PRIV(dev)->mac_addr_pool[index] = vmdq;
1003         fs_unlock(dev, 0);
1004         return 0;
1005 }
1006
1007 static int
1008 fs_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
1009 {
1010         struct sub_device *sdev;
1011         uint8_t i;
1012         int ret;
1013
1014         fs_lock(dev, 0);
1015         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
1016                 ret = rte_eth_dev_default_mac_addr_set(PORT_ID(sdev), mac_addr);
1017                 ret = fs_err(sdev, ret);
1018                 if (ret) {
1019                         ERROR("Operation rte_eth_dev_mac_addr_set failed for sub_device %d with error %d",
1020                                 i, ret);
1021                         fs_unlock(dev, 0);
1022                         return ret;
1023                 }
1024         }
1025         fs_unlock(dev, 0);
1026
1027         return 0;
1028 }
1029
1030 static int
1031 fs_filter_ctrl(struct rte_eth_dev *dev,
1032                 enum rte_filter_type type,
1033                 enum rte_filter_op op,
1034                 void *arg)
1035 {
1036         struct sub_device *sdev;
1037         uint8_t i;
1038         int ret;
1039
1040         if (type == RTE_ETH_FILTER_GENERIC &&
1041             op == RTE_ETH_FILTER_GET) {
1042                 *(const void **)arg = &fs_flow_ops;
1043                 return 0;
1044         }
1045         fs_lock(dev, 0);
1046         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE) {
1047                 DEBUG("Calling rte_eth_dev_filter_ctrl on sub_device %d", i);
1048                 ret = rte_eth_dev_filter_ctrl(PORT_ID(sdev), type, op, arg);
1049                 if ((ret = fs_err(sdev, ret))) {
1050                         ERROR("Operation rte_eth_dev_filter_ctrl failed for sub_device %d"
1051                               " with error %d", i, ret);
1052                         fs_unlock(dev, 0);
1053                         return ret;
1054                 }
1055         }
1056         fs_unlock(dev, 0);
1057         return 0;
1058 }
1059
1060 const struct eth_dev_ops failsafe_ops = {
1061         .dev_configure = fs_dev_configure,
1062         .dev_start = fs_dev_start,
1063         .dev_stop = fs_dev_stop,
1064         .dev_set_link_down = fs_dev_set_link_down,
1065         .dev_set_link_up = fs_dev_set_link_up,
1066         .dev_close = fs_dev_close,
1067         .promiscuous_enable = fs_promiscuous_enable,
1068         .promiscuous_disable = fs_promiscuous_disable,
1069         .allmulticast_enable = fs_allmulticast_enable,
1070         .allmulticast_disable = fs_allmulticast_disable,
1071         .link_update = fs_link_update,
1072         .stats_get = fs_stats_get,
1073         .stats_reset = fs_stats_reset,
1074         .dev_infos_get = fs_dev_infos_get,
1075         .dev_supported_ptypes_get = fs_dev_supported_ptypes_get,
1076         .mtu_set = fs_mtu_set,
1077         .vlan_filter_set = fs_vlan_filter_set,
1078         .rx_queue_setup = fs_rx_queue_setup,
1079         .tx_queue_setup = fs_tx_queue_setup,
1080         .rx_queue_release = fs_rx_queue_release,
1081         .tx_queue_release = fs_tx_queue_release,
1082         .rx_queue_intr_enable = fs_rx_intr_enable,
1083         .rx_queue_intr_disable = fs_rx_intr_disable,
1084         .flow_ctrl_get = fs_flow_ctrl_get,
1085         .flow_ctrl_set = fs_flow_ctrl_set,
1086         .mac_addr_remove = fs_mac_addr_remove,
1087         .mac_addr_add = fs_mac_addr_add,
1088         .mac_addr_set = fs_mac_addr_set,
1089         .filter_ctrl = fs_filter_ctrl,
1090 };