drivers/net/failsafe/failsafe_ether.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2017 6WIND S.A.
   3  * Copyright 2017 Mellanox Technologies, Ltd
   4  */
   5
   6 #include <unistd.h>
   7
   8 #include <rte_flow.h>
   9 #include <rte_flow_driver.h>
  10 #include <rte_cycles.h>
  11
  12 #include "failsafe_private.h"
  13
  14 /** Print a message out of a flow error. */
  15 static int
  16 fs_flow_complain(struct rte_flow_error *error)
  17 {
  18         static const char *const errstrlist[] = {
  19                 [RTE_FLOW_ERROR_TYPE_NONE] = "no error",
  20                 [RTE_FLOW_ERROR_TYPE_UNSPECIFIED] = "cause unspecified",
  21                 [RTE_FLOW_ERROR_TYPE_HANDLE] = "flow rule (handle)",
  22                 [RTE_FLOW_ERROR_TYPE_ATTR_GROUP] = "group field",
  23                 [RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY] = "priority field",
  24                 [RTE_FLOW_ERROR_TYPE_ATTR_INGRESS] = "ingress field",
  25                 [RTE_FLOW_ERROR_TYPE_ATTR_EGRESS] = "egress field",
  26                 [RTE_FLOW_ERROR_TYPE_ATTR] = "attributes structure",
  27                 [RTE_FLOW_ERROR_TYPE_ITEM_NUM] = "pattern length",
  28                 [RTE_FLOW_ERROR_TYPE_ITEM] = "specific pattern item",
  29                 [RTE_FLOW_ERROR_TYPE_ACTION_NUM] = "number of actions",
  30                 [RTE_FLOW_ERROR_TYPE_ACTION] = "specific action",
  31         };
  32         const char *errstr;
  33         char buf[32];
  34         int err = rte_errno;
  35
  36         if ((unsigned int)error->type >= RTE_DIM(errstrlist) ||
  37                         !errstrlist[error->type])
  38                 errstr = "unknown type";
  39         else
  40                 errstr = errstrlist[error->type];
  41         ERROR("Caught error type %d (%s): %s%s\n",
  42                 error->type, errstr,
  43                 error->cause ? (snprintf(buf, sizeof(buf), "cause: %p, ",
  44                                 error->cause), buf) : "",
  45                 error->message ? error->message : "(no stated reason)");
  46         return -err;
  47 }
  48
  49 static int
  50 eth_dev_flow_isolate_set(struct rte_eth_dev *dev,
  51                          struct sub_device *sdev)
  52 {
  53         struct rte_flow_error ferror;
  54         int ret;
  55
  56         if (!PRIV(dev)->flow_isolated) {
  57                 DEBUG("Flow isolation already disabled");
  58         } else {
  59                 DEBUG("Enabling flow isolation");
  60                 ret = rte_flow_isolate(PORT_ID(sdev),
  61                                        PRIV(dev)->flow_isolated,
  62                                        &ferror);
  63                 if (ret) {
  64                         fs_flow_complain(&ferror);
  65                         return ret;
  66                 }
  67         }
  68         return 0;
  69 }
  70
  71 static int
  72 fs_eth_dev_conf_apply(struct rte_eth_dev *dev,
  73                 struct sub_device *sdev)
  74 {
  75         struct rte_eth_dev *edev;
  76         struct rte_vlan_filter_conf *vfc1;
  77         struct rte_vlan_filter_conf *vfc2;
  78         struct rte_flow *flow;
  79         struct rte_flow_error ferror;
  80         uint32_t i;
  81         int ret;
  82
  83         edev = ETH(sdev);
  84         /* RX queue setup */
  85         for (i = 0; i < dev->data->nb_rx_queues; i++) {
  86                 struct rxq *rxq;
  87
  88                 rxq = dev->data->rx_queues[i];
  89                 ret = rte_eth_rx_queue_setup(PORT_ID(sdev), i,
  90                                 rxq->info.nb_desc, rxq->socket_id,
  91                                 &rxq->info.conf, rxq->info.mp);
  92                 if (ret) {
  93                         ERROR("rx_queue_setup failed");
  94                         return ret;
  95                 }
  96         }
  97         /* TX queue setup */
  98         for (i = 0; i < dev->data->nb_tx_queues; i++) {
  99                 struct txq *txq;
 100
 101                 txq = dev->data->tx_queues[i];
 102                 ret = rte_eth_tx_queue_setup(PORT_ID(sdev), i,
 103                                 txq->info.nb_desc, txq->socket_id,
 104                                 &txq->info.conf);
 105                 if (ret) {
 106                         ERROR("tx_queue_setup failed");
 107                         return ret;
 108                 }
 109         }
 110         /* dev_link.link_status */
 111         if (dev->data->dev_link.link_status !=
 112             edev->data->dev_link.link_status) {
 113                 DEBUG("Configuring link_status");
 114                 if (dev->data->dev_link.link_status)
 115                         ret = rte_eth_dev_set_link_up(PORT_ID(sdev));
 116                 else
 117                         ret = rte_eth_dev_set_link_down(PORT_ID(sdev));
 118                 if (ret) {
 119                         ERROR("Failed to apply link_status");
 120                         return ret;
 121                 }
 122         } else {
 123                 DEBUG("link_status already set");
 124         }
 125         /* promiscuous */
 126         if (dev->data->promiscuous != edev->data->promiscuous) {
 127                 DEBUG("Configuring promiscuous");
 128                 if (dev->data->promiscuous)
 129                         rte_eth_promiscuous_enable(PORT_ID(sdev));
 130                 else
 131                         rte_eth_promiscuous_disable(PORT_ID(sdev));
 132         } else {
 133                 DEBUG("promiscuous already set");
 134         }
 135         /* all_multicast */
 136         if (dev->data->all_multicast != edev->data->all_multicast) {
 137                 DEBUG("Configuring all_multicast");
 138                 if (dev->data->all_multicast)
 139                         rte_eth_allmulticast_enable(PORT_ID(sdev));
 140                 else
 141                         rte_eth_allmulticast_disable(PORT_ID(sdev));
 142         } else {
 143                 DEBUG("all_multicast already set");
 144         }
 145         /* MTU */
 146         if (dev->data->mtu != edev->data->mtu) {
 147                 DEBUG("Configuring MTU");
 148                 ret = rte_eth_dev_set_mtu(PORT_ID(sdev), dev->data->mtu);
 149                 if (ret) {
 150                         ERROR("Failed to apply MTU");
 151                         return ret;
 152                 }
 153         } else {
 154                 DEBUG("MTU already set");
 155         }
 156         /* default MAC */
 157         DEBUG("Configuring default MAC address");
 158         ret = rte_eth_dev_default_mac_addr_set(PORT_ID(sdev),
 159                         &dev->data->mac_addrs[0]);
 160         if (ret) {
 161                 ERROR("Setting default MAC address failed");
 162                 return ret;
 163         }
 164         /* additional MAC */
 165         if (PRIV(dev)->nb_mac_addr > 1)
 166                 DEBUG("Configure additional MAC address%s",
 167                         (PRIV(dev)->nb_mac_addr > 2 ? "es" : ""));
 168         for (i = 1; i < PRIV(dev)->nb_mac_addr; i++) {
 169                 struct ether_addr *ea;
 170
 171                 ea = &dev->data->mac_addrs[i];
 172                 ret = rte_eth_dev_mac_addr_add(PORT_ID(sdev), ea,
 173                                 PRIV(dev)->mac_addr_pool[i]);
 174                 if (ret) {
 175                         char ea_fmt[ETHER_ADDR_FMT_SIZE];
 176
 177                         ether_format_addr(ea_fmt, ETHER_ADDR_FMT_SIZE, ea);
 178                         ERROR("Adding MAC address %s failed", ea_fmt);
 179                         return ret;
 180                 }
 181         }
 182         /* VLAN filter */
 183         vfc1 = &dev->data->vlan_filter_conf;
 184         vfc2 = &edev->data->vlan_filter_conf;
 185         if (memcmp(vfc1, vfc2, sizeof(struct rte_vlan_filter_conf))) {
 186                 uint64_t vbit;
 187                 uint64_t ids;
 188                 size_t i;
 189                 uint16_t vlan_id;
 190
 191                 DEBUG("Configuring VLAN filter");
 192                 for (i = 0; i < RTE_DIM(vfc1->ids); i++) {
 193                         if (vfc1->ids[i] == 0)
 194                                 continue;
 195                         ids = vfc1->ids[i];
 196                         while (ids) {
 197                                 vlan_id = 64 * i;
 198                                 /* count trailing zeroes */
 199                                 vbit = ~ids & (ids - 1);
 200                                 /* clear least significant bit set */
 201                                 ids ^= (ids ^ (ids - 1)) ^ vbit;
 202                                 for (; vbit; vlan_id++)
 203                                         vbit >>= 1;
 204                                 ret = rte_eth_dev_vlan_filter(
 205                                         PORT_ID(sdev), vlan_id, 1);
 206                                 if (ret) {
 207                                         ERROR("Failed to apply VLAN filter %hu",
 208                                                 vlan_id);
 209                                         return ret;
 210                                 }
 211                         }
 212                 }
 213         } else {
 214                 DEBUG("VLAN filter already set");
 215         }
 216         /* rte_flow */
 217         if (TAILQ_EMPTY(&PRIV(dev)->flow_list)) {
 218                 DEBUG("rte_flow already set");
 219         } else {
 220                 DEBUG("Resetting rte_flow configuration");
 221                 ret = rte_flow_flush(PORT_ID(sdev), &ferror);
 222                 if (ret) {
 223                         fs_flow_complain(&ferror);
 224                         return ret;
 225                 }
 226                 i = 0;
 227                 rte_errno = 0;
 228                 DEBUG("Configuring rte_flow");
 229                 TAILQ_FOREACH(flow, &PRIV(dev)->flow_list, next) {
 230                         DEBUG("Creating flow #%" PRIu32, i++);
 231                         flow->flows[SUB_ID(sdev)] =
 232                                 rte_flow_create(PORT_ID(sdev),
 233                                                 &flow->fd->attr,
 234                                                 flow->fd->items,
 235                                                 flow->fd->actions,
 236                                                 &ferror);
 237                         ret = rte_errno;
 238                         if (ret)
 239                                 break;
 240                 }
 241                 if (ret) {
 242                         fs_flow_complain(&ferror);
 243                         return ret;
 244                 }
 245         }
 246         return 0;
 247 }
 248
 249 static void
 250 fs_dev_remove(struct sub_device *sdev)
 251 {
 252         int ret;
 253
 254         if (sdev == NULL)
 255                 return;
 256         switch (sdev->state) {
 257         case DEV_STARTED:
 258                 failsafe_rx_intr_uninstall_subdevice(sdev);
 259                 rte_eth_dev_stop(PORT_ID(sdev));
 260                 sdev->state = DEV_ACTIVE;
 261                 /* fallthrough */
 262         case DEV_ACTIVE:
 263                 failsafe_eth_dev_unregister_callbacks(sdev);
 264                 rte_eth_dev_close(PORT_ID(sdev));
 265                 sdev->state = DEV_PROBED;
 266                 /* fallthrough */
 267         case DEV_PROBED:
 268                 ret = rte_eal_hotplug_remove(sdev->bus->name,
 269                                              sdev->dev->name);
 270                 if (ret) {
 271                         ERROR("Bus detach failed for sub_device %u",
 272                               SUB_ID(sdev));
 273                 } else {
 274                         rte_eth_dev_release_port(ETH(sdev));
 275                 }
 276                 sdev->state = DEV_PARSED;
 277                 /* fallthrough */
 278         case DEV_PARSED:
 279         case DEV_UNDEFINED:
 280                 sdev->state = DEV_UNDEFINED;
 281                 /* the end */
 282                 break;
 283         }
 284         sdev->remove = 0;
 285         failsafe_hotplug_alarm_install(sdev->fs_dev);
 286 }
 287
 288 static void
 289 fs_dev_stats_save(struct sub_device *sdev)
 290 {
 291         struct rte_eth_stats stats;
 292         int err;
 293
 294         /* Attempt to read current stats. */
 295         err = rte_eth_stats_get(PORT_ID(sdev), &stats);
 296         if (err) {
 297                 uint64_t timestamp = sdev->stats_snapshot.timestamp;
 298
 299                 WARN("Could not access latest statistics from sub-device %d.\n",
 300                          SUB_ID(sdev));
 301                 if (timestamp != 0)
 302                         WARN("Using latest snapshot taken before %"PRIu64" seconds.\n",
 303                                  (rte_rdtsc() - timestamp) / rte_get_tsc_hz());
 304         }
 305         failsafe_stats_increment(&PRIV(sdev->fs_dev)->stats_accumulator,
 306                         err ? &sdev->stats_snapshot.stats : &stats);
 307         memset(&sdev->stats_snapshot, 0, sizeof(sdev->stats_snapshot));
 308 }
 309
 310 static inline int
 311 fs_rxtx_clean(struct sub_device *sdev)
 312 {
 313         uint16_t i;
 314
 315         for (i = 0; i < ETH(sdev)->data->nb_rx_queues; i++)
 316                 if (FS_ATOMIC_RX(sdev, i))
 317                         return 0;
 318         for (i = 0; i < ETH(sdev)->data->nb_tx_queues; i++)
 319                 if (FS_ATOMIC_TX(sdev, i))
 320                         return 0;
 321         return 1;
 322 }
 323
 324 void
 325 failsafe_eth_dev_unregister_callbacks(struct sub_device *sdev)
 326 {
 327         int ret;
 328
 329         if (sdev == NULL)
 330                 return;
 331         if (sdev->rmv_callback) {
 332                 ret = rte_eth_dev_callback_unregister(PORT_ID(sdev),
 333                                                 RTE_ETH_EVENT_INTR_RMV,
 334                                                 failsafe_eth_rmv_event_callback,
 335                                                 sdev);
 336                 if (ret)
 337                         WARN("Failed to unregister RMV callback for sub_device"
 338                              " %d", SUB_ID(sdev));
 339                 sdev->rmv_callback = 0;
 340         }
 341         if (sdev->lsc_callback) {
 342                 ret = rte_eth_dev_callback_unregister(PORT_ID(sdev),
 343                                                 RTE_ETH_EVENT_INTR_LSC,
 344                                                 failsafe_eth_lsc_event_callback,
 345                                                 sdev);
 346                 if (ret)
 347                         WARN("Failed to unregister LSC callback for sub_device"
 348                              " %d", SUB_ID(sdev));
 349                 sdev->lsc_callback = 0;
 350         }
 351 }
 352
 353 void
 354 failsafe_dev_remove(struct rte_eth_dev *dev)
 355 {
 356         struct sub_device *sdev;
 357         uint8_t i;
 358
 359         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
 360                 if (sdev->remove && fs_rxtx_clean(sdev)) {
 361                         if (fs_lock(dev, 1) != 0)
 362                                 return;
 363                         fs_dev_stats_save(sdev);
 364                         fs_dev_remove(sdev);
 365                         fs_unlock(dev, 1);
 366                 }
 367 }
 368
 369 static int
 370 failsafe_eth_dev_rx_queues_sync(struct rte_eth_dev *dev)
 371 {
 372         struct rxq *rxq;
 373         int ret;
 374         uint16_t i;
 375
 376         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 377                 rxq = dev->data->rx_queues[i];
 378
 379                 if (rxq->info.conf.rx_deferred_start &&
 380                     dev->data->rx_queue_state[i] ==
 381                                                 RTE_ETH_QUEUE_STATE_STARTED) {
 382                         /*
 383                          * The subdevice Rx queue does not launch on device
 384                          * start if deferred start flag is set. It needs to be
 385                          * started manually in case an appropriate failsafe Rx
 386                          * queue has been started earlier.
 387                          */
 388                         ret = dev->dev_ops->rx_queue_start(dev, i);
 389                         if (ret) {
 390                                 ERROR("Could not synchronize Rx queue %d", i);
 391                                 return ret;
 392                         }
 393                 } else if (dev->data->rx_queue_state[i] ==
 394                                                 RTE_ETH_QUEUE_STATE_STOPPED) {
 395                         /*
 396                          * The subdevice Rx queue needs to be stopped manually
 397                          * in case an appropriate failsafe Rx queue has been
 398                          * stopped earlier.
 399                          */
 400                         ret = dev->dev_ops->rx_queue_stop(dev, i);
 401                         if (ret) {
 402                                 ERROR("Could not synchronize Rx queue %d", i);
 403                                 return ret;
 404                         }
 405                 }
 406         }
 407         return 0;
 408 }
 409
 410 static int
 411 failsafe_eth_dev_tx_queues_sync(struct rte_eth_dev *dev)
 412 {
 413         struct txq *txq;
 414         int ret;
 415         uint16_t i;
 416
 417         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 418                 txq = dev->data->tx_queues[i];
 419
 420                 if (txq->info.conf.tx_deferred_start &&
 421                     dev->data->tx_queue_state[i] ==
 422                                                 RTE_ETH_QUEUE_STATE_STARTED) {
 423                         /*
 424                          * The subdevice Tx queue does not launch on device
 425                          * start if deferred start flag is set. It needs to be
 426                          * started manually in case an appropriate failsafe Tx
 427                          * queue has been started earlier.
 428                          */
 429                         ret = dev->dev_ops->tx_queue_start(dev, i);
 430                         if (ret) {
 431                                 ERROR("Could not synchronize Tx queue %d", i);
 432                                 return ret;
 433                         }
 434                 } else if (dev->data->tx_queue_state[i] ==
 435                                                 RTE_ETH_QUEUE_STATE_STOPPED) {
 436                         /*
 437                          * The subdevice Tx queue needs to be stopped manually
 438                          * in case an appropriate failsafe Tx queue has been
 439                          * stopped earlier.
 440                          */
 441                         ret = dev->dev_ops->tx_queue_stop(dev, i);
 442                         if (ret) {
 443                                 ERROR("Could not synchronize Tx queue %d", i);
 444                                 return ret;
 445                         }
 446                 }
 447         }
 448         return 0;
 449 }
 450
 451 int
 452 failsafe_eth_dev_state_sync(struct rte_eth_dev *dev)
 453 {
 454         struct sub_device *sdev;
 455         uint32_t inactive;
 456         int ret;
 457         uint8_t i;
 458
 459         if (PRIV(dev)->state < DEV_PARSED)
 460                 return 0;
 461
 462         ret = failsafe_args_parse_subs(dev);
 463         if (ret)
 464                 goto err_remove;
 465
 466         if (PRIV(dev)->state < DEV_PROBED)
 467                 return 0;
 468         ret = failsafe_eal_init(dev);
 469         if (ret)
 470                 goto err_remove;
 471         if (PRIV(dev)->state < DEV_ACTIVE)
 472                 return 0;
 473         inactive = 0;
 474         FOREACH_SUBDEV(sdev, i, dev) {
 475                 if (sdev->state == DEV_PROBED) {
 476                         inactive |= UINT32_C(1) << i;
 477                         ret = eth_dev_flow_isolate_set(dev, sdev);
 478                         if (ret) {
 479                                 ERROR("Could not apply configuration to sub_device %d",
 480                                       i);
 481                                 goto err_remove;
 482                         }
 483                 }
 484         }
 485         ret = dev->dev_ops->dev_configure(dev);
 486         if (ret)
 487                 goto err_remove;
 488         FOREACH_SUBDEV(sdev, i, dev) {
 489                 if (inactive & (UINT32_C(1) << i)) {
 490                         ret = fs_eth_dev_conf_apply(dev, sdev);
 491                         if (ret) {
 492                                 ERROR("Could not apply configuration to sub_device %d",
 493                                       i);
 494                                 goto err_remove;
 495                         }
 496                 }
 497         }
 498         /*
 499          * If new devices have been configured, check if
 500          * the link state has changed.
 501          */
 502         if (inactive)
 503                 dev->dev_ops->link_update(dev, 1);
 504         if (PRIV(dev)->state < DEV_STARTED)
 505                 return 0;
 506         ret = dev->dev_ops->dev_start(dev);
 507         if (ret)
 508                 goto err_remove;
 509         ret = failsafe_eth_dev_rx_queues_sync(dev);
 510         if (ret)
 511                 goto err_remove;
 512         ret = failsafe_eth_dev_tx_queues_sync(dev);
 513         if (ret)
 514                 goto err_remove;
 515         return 0;
 516 err_remove:
 517         FOREACH_SUBDEV(sdev, i, dev)
 518                 if (sdev->state != PRIV(dev)->state)
 519                         sdev->remove = 1;
 520         return ret;
 521 }
 522
 523 void
 524 failsafe_stats_increment(struct rte_eth_stats *to, struct rte_eth_stats *from)
 525 {
 526         uint32_t i;
 527
 528         RTE_ASSERT(to != NULL && from != NULL);
 529         to->ipackets += from->ipackets;
 530         to->opackets += from->opackets;
 531         to->ibytes += from->ibytes;
 532         to->obytes += from->obytes;
 533         to->imissed += from->imissed;
 534         to->ierrors += from->ierrors;
 535         to->oerrors += from->oerrors;
 536         to->rx_nombuf += from->rx_nombuf;
 537         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
 538                 to->q_ipackets[i] += from->q_ipackets[i];
 539                 to->q_opackets[i] += from->q_opackets[i];
 540                 to->q_ibytes[i] += from->q_ibytes[i];
 541                 to->q_obytes[i] += from->q_obytes[i];
 542                 to->q_errors[i] += from->q_errors[i];
 543         }
 544 }
 545
 546 int
 547 failsafe_eth_rmv_event_callback(uint16_t port_id __rte_unused,
 548                                 enum rte_eth_event_type event __rte_unused,
 549                                 void *cb_arg, void *out __rte_unused)
 550 {
 551         struct sub_device *sdev = cb_arg;
 552
 553         fs_lock(sdev->fs_dev, 0);
 554         /* Switch as soon as possible tx_dev. */
 555         fs_switch_dev(sdev->fs_dev, sdev);
 556         /* Use safe bursts in any case. */
 557         set_burst_fn(sdev->fs_dev, 1);
 558         /*
 559          * Async removal, the sub-PMD will try to unregister
 560          * the callback at the source of the current thread context.
 561          */
 562         sdev->remove = 1;
 563         fs_unlock(sdev->fs_dev, 0);
 564         return 0;
 565 }
 566
 567 int
 568 failsafe_eth_lsc_event_callback(uint16_t port_id __rte_unused,
 569                                 enum rte_eth_event_type event __rte_unused,
 570                                 void *cb_arg, void *out __rte_unused)
 571 {
 572         struct rte_eth_dev *dev = cb_arg;
 573         int ret;
 574
 575         ret = dev->dev_ops->link_update(dev, 0);
 576         /* We must pass on the LSC event */
 577         if (ret)
 578                 return _rte_eth_dev_callback_process(dev,
 579                                                      RTE_ETH_EVENT_INTR_LSC,
 580                                                      NULL);
 581         else
 582                 return 0;
 583 }
 584
 585 /* Take sub-device ownership before it becomes exposed to the application. */
 586 int
 587 failsafe_eth_new_event_callback(uint16_t port_id,
 588                                 enum rte_eth_event_type event __rte_unused,
 589                                 void *cb_arg, void *out __rte_unused)
 590 {
 591         struct rte_eth_dev *fs_dev = cb_arg;
 592         struct sub_device *sdev;
 593         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 594         uint8_t i;
 595
 596         FOREACH_SUBDEV_STATE(sdev, i, fs_dev, DEV_PARSED) {
 597                 if (sdev->state >= DEV_PROBED)
 598                         continue;
 599                 if (strcmp(sdev->devargs.name, dev->device->name) != 0)
 600                         continue;
 601                 rte_eth_dev_owner_set(port_id, &PRIV(fs_dev)->my_owner);
 602                 /* The actual owner will be checked after the port probing. */
 603                 break;
 604         }
 605         return 0;
 606 }