net/failsafe: support multicast address list set
[dpdk.git] / drivers / net / failsafe / failsafe_ether.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright 2017 6WIND S.A.
3  * Copyright 2017 Mellanox Technologies, Ltd
4  */
5
6 #include <unistd.h>
7
8 #include <rte_flow.h>
9 #include <rte_flow_driver.h>
10 #include <rte_cycles.h>
11
12 #include "failsafe_private.h"
13
14 /** Print a message out of a flow error. */
15 static int
16 fs_flow_complain(struct rte_flow_error *error)
17 {
18         static const char *const errstrlist[] = {
19                 [RTE_FLOW_ERROR_TYPE_NONE] = "no error",
20                 [RTE_FLOW_ERROR_TYPE_UNSPECIFIED] = "cause unspecified",
21                 [RTE_FLOW_ERROR_TYPE_HANDLE] = "flow rule (handle)",
22                 [RTE_FLOW_ERROR_TYPE_ATTR_GROUP] = "group field",
23                 [RTE_FLOW_ERROR_TYPE_ATTR_PRIORITY] = "priority field",
24                 [RTE_FLOW_ERROR_TYPE_ATTR_INGRESS] = "ingress field",
25                 [RTE_FLOW_ERROR_TYPE_ATTR_EGRESS] = "egress field",
26                 [RTE_FLOW_ERROR_TYPE_ATTR] = "attributes structure",
27                 [RTE_FLOW_ERROR_TYPE_ITEM_NUM] = "pattern length",
28                 [RTE_FLOW_ERROR_TYPE_ITEM] = "specific pattern item",
29                 [RTE_FLOW_ERROR_TYPE_ACTION_NUM] = "number of actions",
30                 [RTE_FLOW_ERROR_TYPE_ACTION] = "specific action",
31         };
32         const char *errstr;
33         char buf[32];
34         int err = rte_errno;
35
36         if ((unsigned int)error->type >= RTE_DIM(errstrlist) ||
37                         !errstrlist[error->type])
38                 errstr = "unknown type";
39         else
40                 errstr = errstrlist[error->type];
41         ERROR("Caught error type %d (%s): %s%s\n",
42                 error->type, errstr,
43                 error->cause ? (snprintf(buf, sizeof(buf), "cause: %p, ",
44                                 error->cause), buf) : "",
45                 error->message ? error->message : "(no stated reason)");
46         return -err;
47 }
48
49 static int
50 eth_dev_flow_isolate_set(struct rte_eth_dev *dev,
51                          struct sub_device *sdev)
52 {
53         struct rte_flow_error ferror;
54         int ret;
55
56         if (!PRIV(dev)->flow_isolated) {
57                 DEBUG("Flow isolation already disabled");
58         } else {
59                 DEBUG("Enabling flow isolation");
60                 ret = rte_flow_isolate(PORT_ID(sdev),
61                                        PRIV(dev)->flow_isolated,
62                                        &ferror);
63                 if (ret) {
64                         fs_flow_complain(&ferror);
65                         return ret;
66                 }
67         }
68         return 0;
69 }
70
71 static int
72 fs_eth_dev_conf_apply(struct rte_eth_dev *dev,
73                 struct sub_device *sdev)
74 {
75         struct rte_eth_dev *edev;
76         struct rte_vlan_filter_conf *vfc1;
77         struct rte_vlan_filter_conf *vfc2;
78         struct rte_flow *flow;
79         struct rte_flow_error ferror;
80         uint32_t i;
81         int ret;
82
83         edev = ETH(sdev);
84         /* RX queue setup */
85         for (i = 0; i < dev->data->nb_rx_queues; i++) {
86                 struct rxq *rxq;
87
88                 rxq = dev->data->rx_queues[i];
89                 ret = rte_eth_rx_queue_setup(PORT_ID(sdev), i,
90                                 rxq->info.nb_desc, rxq->socket_id,
91                                 &rxq->info.conf, rxq->info.mp);
92                 if (ret) {
93                         ERROR("rx_queue_setup failed");
94                         return ret;
95                 }
96         }
97         /* TX queue setup */
98         for (i = 0; i < dev->data->nb_tx_queues; i++) {
99                 struct txq *txq;
100
101                 txq = dev->data->tx_queues[i];
102                 ret = rte_eth_tx_queue_setup(PORT_ID(sdev), i,
103                                 txq->info.nb_desc, txq->socket_id,
104                                 &txq->info.conf);
105                 if (ret) {
106                         ERROR("tx_queue_setup failed");
107                         return ret;
108                 }
109         }
110         /* dev_link.link_status */
111         if (dev->data->dev_link.link_status !=
112             edev->data->dev_link.link_status) {
113                 DEBUG("Configuring link_status");
114                 if (dev->data->dev_link.link_status)
115                         ret = rte_eth_dev_set_link_up(PORT_ID(sdev));
116                 else
117                         ret = rte_eth_dev_set_link_down(PORT_ID(sdev));
118                 if (ret) {
119                         ERROR("Failed to apply link_status");
120                         return ret;
121                 }
122         } else {
123                 DEBUG("link_status already set");
124         }
125         /* promiscuous */
126         if (dev->data->promiscuous != edev->data->promiscuous) {
127                 DEBUG("Configuring promiscuous");
128                 if (dev->data->promiscuous)
129                         rte_eth_promiscuous_enable(PORT_ID(sdev));
130                 else
131                         rte_eth_promiscuous_disable(PORT_ID(sdev));
132         } else {
133                 DEBUG("promiscuous already set");
134         }
135         /* all_multicast */
136         if (dev->data->all_multicast != edev->data->all_multicast) {
137                 DEBUG("Configuring all_multicast");
138                 if (dev->data->all_multicast)
139                         rte_eth_allmulticast_enable(PORT_ID(sdev));
140                 else
141                         rte_eth_allmulticast_disable(PORT_ID(sdev));
142         } else {
143                 DEBUG("all_multicast already set");
144         }
145         /* MTU */
146         if (dev->data->mtu != edev->data->mtu) {
147                 DEBUG("Configuring MTU");
148                 ret = rte_eth_dev_set_mtu(PORT_ID(sdev), dev->data->mtu);
149                 if (ret) {
150                         ERROR("Failed to apply MTU");
151                         return ret;
152                 }
153         } else {
154                 DEBUG("MTU already set");
155         }
156         /* default MAC */
157         DEBUG("Configuring default MAC address");
158         ret = rte_eth_dev_default_mac_addr_set(PORT_ID(sdev),
159                         &dev->data->mac_addrs[0]);
160         if (ret) {
161                 ERROR("Setting default MAC address failed");
162                 return ret;
163         }
164         /* additional MAC */
165         if (PRIV(dev)->nb_mac_addr > 1)
166                 DEBUG("Configure additional MAC address%s",
167                         (PRIV(dev)->nb_mac_addr > 2 ? "es" : ""));
168         for (i = 1; i < PRIV(dev)->nb_mac_addr; i++) {
169                 struct ether_addr *ea;
170
171                 ea = &dev->data->mac_addrs[i];
172                 ret = rte_eth_dev_mac_addr_add(PORT_ID(sdev), ea,
173                                 PRIV(dev)->mac_addr_pool[i]);
174                 if (ret) {
175                         char ea_fmt[ETHER_ADDR_FMT_SIZE];
176
177                         ether_format_addr(ea_fmt, ETHER_ADDR_FMT_SIZE, ea);
178                         ERROR("Adding MAC address %s failed", ea_fmt);
179                         return ret;
180                 }
181         }
182         /*
183          * Propagate multicast MAC addresses to sub-devices,
184          * if non zero number of addresses is set.
185          * The condition is required to avoid breakage of failsafe
186          * for sub-devices which do not support the operation
187          * if the feature is really not used.
188          */
189         if (PRIV(dev)->nb_mcast_addr > 0) {
190                 DEBUG("Configuring multicast MAC addresses");
191                 ret = rte_eth_dev_set_mc_addr_list(PORT_ID(sdev),
192                                                    PRIV(dev)->mcast_addrs,
193                                                    PRIV(dev)->nb_mcast_addr);
194                 if (ret) {
195                         ERROR("Failed to apply multicast MAC addresses");
196                         return ret;
197                 }
198         }
199         /* VLAN filter */
200         vfc1 = &dev->data->vlan_filter_conf;
201         vfc2 = &edev->data->vlan_filter_conf;
202         if (memcmp(vfc1, vfc2, sizeof(struct rte_vlan_filter_conf))) {
203                 uint64_t vbit;
204                 uint64_t ids;
205                 size_t i;
206                 uint16_t vlan_id;
207
208                 DEBUG("Configuring VLAN filter");
209                 for (i = 0; i < RTE_DIM(vfc1->ids); i++) {
210                         if (vfc1->ids[i] == 0)
211                                 continue;
212                         ids = vfc1->ids[i];
213                         while (ids) {
214                                 vlan_id = 64 * i;
215                                 /* count trailing zeroes */
216                                 vbit = ~ids & (ids - 1);
217                                 /* clear least significant bit set */
218                                 ids ^= (ids ^ (ids - 1)) ^ vbit;
219                                 for (; vbit; vlan_id++)
220                                         vbit >>= 1;
221                                 ret = rte_eth_dev_vlan_filter(
222                                         PORT_ID(sdev), vlan_id, 1);
223                                 if (ret) {
224                                         ERROR("Failed to apply VLAN filter %hu",
225                                                 vlan_id);
226                                         return ret;
227                                 }
228                         }
229                 }
230         } else {
231                 DEBUG("VLAN filter already set");
232         }
233         /* rte_flow */
234         if (TAILQ_EMPTY(&PRIV(dev)->flow_list)) {
235                 DEBUG("rte_flow already set");
236         } else {
237                 DEBUG("Resetting rte_flow configuration");
238                 ret = rte_flow_flush(PORT_ID(sdev), &ferror);
239                 if (ret) {
240                         fs_flow_complain(&ferror);
241                         return ret;
242                 }
243                 i = 0;
244                 rte_errno = 0;
245                 DEBUG("Configuring rte_flow");
246                 TAILQ_FOREACH(flow, &PRIV(dev)->flow_list, next) {
247                         DEBUG("Creating flow #%" PRIu32, i++);
248                         flow->flows[SUB_ID(sdev)] =
249                                 rte_flow_create(PORT_ID(sdev),
250                                                 &flow->fd->attr,
251                                                 flow->fd->items,
252                                                 flow->fd->actions,
253                                                 &ferror);
254                         ret = rte_errno;
255                         if (ret)
256                                 break;
257                 }
258                 if (ret) {
259                         fs_flow_complain(&ferror);
260                         return ret;
261                 }
262         }
263         return 0;
264 }
265
266 static void
267 fs_dev_remove(struct sub_device *sdev)
268 {
269         int ret;
270
271         if (sdev == NULL)
272                 return;
273         switch (sdev->state) {
274         case DEV_STARTED:
275                 failsafe_rx_intr_uninstall_subdevice(sdev);
276                 rte_eth_dev_stop(PORT_ID(sdev));
277                 sdev->state = DEV_ACTIVE;
278                 /* fallthrough */
279         case DEV_ACTIVE:
280                 failsafe_eth_dev_unregister_callbacks(sdev);
281                 rte_eth_dev_close(PORT_ID(sdev));
282                 sdev->state = DEV_PROBED;
283                 /* fallthrough */
284         case DEV_PROBED:
285                 ret = rte_eal_hotplug_remove(sdev->bus->name,
286                                              sdev->dev->name);
287                 if (ret) {
288                         ERROR("Bus detach failed for sub_device %u",
289                               SUB_ID(sdev));
290                 } else {
291                         rte_eth_dev_release_port(ETH(sdev));
292                 }
293                 sdev->state = DEV_PARSED;
294                 /* fallthrough */
295         case DEV_PARSED:
296         case DEV_UNDEFINED:
297                 sdev->state = DEV_UNDEFINED;
298                 /* the end */
299                 break;
300         }
301         sdev->remove = 0;
302         failsafe_hotplug_alarm_install(sdev->fs_dev);
303 }
304
305 static void
306 fs_dev_stats_save(struct sub_device *sdev)
307 {
308         struct rte_eth_stats stats;
309         int err;
310
311         /* Attempt to read current stats. */
312         err = rte_eth_stats_get(PORT_ID(sdev), &stats);
313         if (err) {
314                 uint64_t timestamp = sdev->stats_snapshot.timestamp;
315
316                 WARN("Could not access latest statistics from sub-device %d.\n",
317                          SUB_ID(sdev));
318                 if (timestamp != 0)
319                         WARN("Using latest snapshot taken before %"PRIu64" seconds.\n",
320                                  (rte_rdtsc() - timestamp) / rte_get_tsc_hz());
321         }
322         failsafe_stats_increment(&PRIV(sdev->fs_dev)->stats_accumulator,
323                         err ? &sdev->stats_snapshot.stats : &stats);
324         memset(&sdev->stats_snapshot, 0, sizeof(sdev->stats_snapshot));
325 }
326
327 static inline int
328 fs_rxtx_clean(struct sub_device *sdev)
329 {
330         uint16_t i;
331
332         for (i = 0; i < ETH(sdev)->data->nb_rx_queues; i++)
333                 if (FS_ATOMIC_RX(sdev, i))
334                         return 0;
335         for (i = 0; i < ETH(sdev)->data->nb_tx_queues; i++)
336                 if (FS_ATOMIC_TX(sdev, i))
337                         return 0;
338         return 1;
339 }
340
341 void
342 failsafe_eth_dev_unregister_callbacks(struct sub_device *sdev)
343 {
344         int ret;
345
346         if (sdev == NULL)
347                 return;
348         if (sdev->rmv_callback) {
349                 ret = rte_eth_dev_callback_unregister(PORT_ID(sdev),
350                                                 RTE_ETH_EVENT_INTR_RMV,
351                                                 failsafe_eth_rmv_event_callback,
352                                                 sdev);
353                 if (ret)
354                         WARN("Failed to unregister RMV callback for sub_device"
355                              " %d", SUB_ID(sdev));
356                 sdev->rmv_callback = 0;
357         }
358         if (sdev->lsc_callback) {
359                 ret = rte_eth_dev_callback_unregister(PORT_ID(sdev),
360                                                 RTE_ETH_EVENT_INTR_LSC,
361                                                 failsafe_eth_lsc_event_callback,
362                                                 sdev);
363                 if (ret)
364                         WARN("Failed to unregister LSC callback for sub_device"
365                              " %d", SUB_ID(sdev));
366                 sdev->lsc_callback = 0;
367         }
368 }
369
370 void
371 failsafe_dev_remove(struct rte_eth_dev *dev)
372 {
373         struct sub_device *sdev;
374         uint8_t i;
375
376         FOREACH_SUBDEV_STATE(sdev, i, dev, DEV_ACTIVE)
377                 if (sdev->remove && fs_rxtx_clean(sdev)) {
378                         if (fs_lock(dev, 1) != 0)
379                                 return;
380                         fs_dev_stats_save(sdev);
381                         fs_dev_remove(sdev);
382                         fs_unlock(dev, 1);
383                 }
384 }
385
386 static int
387 failsafe_eth_dev_rx_queues_sync(struct rte_eth_dev *dev)
388 {
389         struct rxq *rxq;
390         int ret;
391         uint16_t i;
392
393         for (i = 0; i < dev->data->nb_rx_queues; i++) {
394                 rxq = dev->data->rx_queues[i];
395
396                 if (rxq->info.conf.rx_deferred_start &&
397                     dev->data->rx_queue_state[i] ==
398                                                 RTE_ETH_QUEUE_STATE_STARTED) {
399                         /*
400                          * The subdevice Rx queue does not launch on device
401                          * start if deferred start flag is set. It needs to be
402                          * started manually in case an appropriate failsafe Rx
403                          * queue has been started earlier.
404                          */
405                         ret = dev->dev_ops->rx_queue_start(dev, i);
406                         if (ret) {
407                                 ERROR("Could not synchronize Rx queue %d", i);
408                                 return ret;
409                         }
410                 } else if (dev->data->rx_queue_state[i] ==
411                                                 RTE_ETH_QUEUE_STATE_STOPPED) {
412                         /*
413                          * The subdevice Rx queue needs to be stopped manually
414                          * in case an appropriate failsafe Rx queue has been
415                          * stopped earlier.
416                          */
417                         ret = dev->dev_ops->rx_queue_stop(dev, i);
418                         if (ret) {
419                                 ERROR("Could not synchronize Rx queue %d", i);
420                                 return ret;
421                         }
422                 }
423         }
424         return 0;
425 }
426
427 static int
428 failsafe_eth_dev_tx_queues_sync(struct rte_eth_dev *dev)
429 {
430         struct txq *txq;
431         int ret;
432         uint16_t i;
433
434         for (i = 0; i < dev->data->nb_tx_queues; i++) {
435                 txq = dev->data->tx_queues[i];
436
437                 if (txq->info.conf.tx_deferred_start &&
438                     dev->data->tx_queue_state[i] ==
439                                                 RTE_ETH_QUEUE_STATE_STARTED) {
440                         /*
441                          * The subdevice Tx queue does not launch on device
442                          * start if deferred start flag is set. It needs to be
443                          * started manually in case an appropriate failsafe Tx
444                          * queue has been started earlier.
445                          */
446                         ret = dev->dev_ops->tx_queue_start(dev, i);
447                         if (ret) {
448                                 ERROR("Could not synchronize Tx queue %d", i);
449                                 return ret;
450                         }
451                 } else if (dev->data->tx_queue_state[i] ==
452                                                 RTE_ETH_QUEUE_STATE_STOPPED) {
453                         /*
454                          * The subdevice Tx queue needs to be stopped manually
455                          * in case an appropriate failsafe Tx queue has been
456                          * stopped earlier.
457                          */
458                         ret = dev->dev_ops->tx_queue_stop(dev, i);
459                         if (ret) {
460                                 ERROR("Could not synchronize Tx queue %d", i);
461                                 return ret;
462                         }
463                 }
464         }
465         return 0;
466 }
467
468 int
469 failsafe_eth_dev_state_sync(struct rte_eth_dev *dev)
470 {
471         struct sub_device *sdev;
472         uint32_t inactive;
473         int ret;
474         uint8_t i;
475
476         if (PRIV(dev)->state < DEV_PARSED)
477                 return 0;
478
479         ret = failsafe_args_parse_subs(dev);
480         if (ret)
481                 goto err_remove;
482
483         if (PRIV(dev)->state < DEV_PROBED)
484                 return 0;
485         ret = failsafe_eal_init(dev);
486         if (ret)
487                 goto err_remove;
488         if (PRIV(dev)->state < DEV_ACTIVE)
489                 return 0;
490         inactive = 0;
491         FOREACH_SUBDEV(sdev, i, dev) {
492                 if (sdev->state == DEV_PROBED) {
493                         inactive |= UINT32_C(1) << i;
494                         ret = eth_dev_flow_isolate_set(dev, sdev);
495                         if (ret) {
496                                 ERROR("Could not apply configuration to sub_device %d",
497                                       i);
498                                 goto err_remove;
499                         }
500                 }
501         }
502         ret = dev->dev_ops->dev_configure(dev);
503         if (ret)
504                 goto err_remove;
505         FOREACH_SUBDEV(sdev, i, dev) {
506                 if (inactive & (UINT32_C(1) << i)) {
507                         ret = fs_eth_dev_conf_apply(dev, sdev);
508                         if (ret) {
509                                 ERROR("Could not apply configuration to sub_device %d",
510                                       i);
511                                 goto err_remove;
512                         }
513                 }
514         }
515         /*
516          * If new devices have been configured, check if
517          * the link state has changed.
518          */
519         if (inactive)
520                 dev->dev_ops->link_update(dev, 1);
521         if (PRIV(dev)->state < DEV_STARTED)
522                 return 0;
523         ret = dev->dev_ops->dev_start(dev);
524         if (ret)
525                 goto err_remove;
526         ret = failsafe_eth_dev_rx_queues_sync(dev);
527         if (ret)
528                 goto err_remove;
529         ret = failsafe_eth_dev_tx_queues_sync(dev);
530         if (ret)
531                 goto err_remove;
532         return 0;
533 err_remove:
534         FOREACH_SUBDEV(sdev, i, dev)
535                 if (sdev->state != PRIV(dev)->state)
536                         sdev->remove = 1;
537         return ret;
538 }
539
540 void
541 failsafe_stats_increment(struct rte_eth_stats *to, struct rte_eth_stats *from)
542 {
543         uint32_t i;
544
545         RTE_ASSERT(to != NULL && from != NULL);
546         to->ipackets += from->ipackets;
547         to->opackets += from->opackets;
548         to->ibytes += from->ibytes;
549         to->obytes += from->obytes;
550         to->imissed += from->imissed;
551         to->ierrors += from->ierrors;
552         to->oerrors += from->oerrors;
553         to->rx_nombuf += from->rx_nombuf;
554         for (i = 0; i < RTE_ETHDEV_QUEUE_STAT_CNTRS; i++) {
555                 to->q_ipackets[i] += from->q_ipackets[i];
556                 to->q_opackets[i] += from->q_opackets[i];
557                 to->q_ibytes[i] += from->q_ibytes[i];
558                 to->q_obytes[i] += from->q_obytes[i];
559                 to->q_errors[i] += from->q_errors[i];
560         }
561 }
562
563 int
564 failsafe_eth_rmv_event_callback(uint16_t port_id __rte_unused,
565                                 enum rte_eth_event_type event __rte_unused,
566                                 void *cb_arg, void *out __rte_unused)
567 {
568         struct sub_device *sdev = cb_arg;
569
570         fs_lock(sdev->fs_dev, 0);
571         /* Switch as soon as possible tx_dev. */
572         fs_switch_dev(sdev->fs_dev, sdev);
573         /* Use safe bursts in any case. */
574         set_burst_fn(sdev->fs_dev, 1);
575         /*
576          * Async removal, the sub-PMD will try to unregister
577          * the callback at the source of the current thread context.
578          */
579         sdev->remove = 1;
580         fs_unlock(sdev->fs_dev, 0);
581         return 0;
582 }
583
584 int
585 failsafe_eth_lsc_event_callback(uint16_t port_id __rte_unused,
586                                 enum rte_eth_event_type event __rte_unused,
587                                 void *cb_arg, void *out __rte_unused)
588 {
589         struct rte_eth_dev *dev = cb_arg;
590         int ret;
591
592         ret = dev->dev_ops->link_update(dev, 0);
593         /* We must pass on the LSC event */
594         if (ret)
595                 return _rte_eth_dev_callback_process(dev,
596                                                      RTE_ETH_EVENT_INTR_LSC,
597                                                      NULL);
598         else
599                 return 0;
600 }
601
602 /* Take sub-device ownership before it becomes exposed to the application. */
603 int
604 failsafe_eth_new_event_callback(uint16_t port_id,
605                                 enum rte_eth_event_type event __rte_unused,
606                                 void *cb_arg, void *out __rte_unused)
607 {
608         struct rte_eth_dev *fs_dev = cb_arg;
609         struct sub_device *sdev;
610         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
611         uint8_t i;
612
613         FOREACH_SUBDEV_STATE(sdev, i, fs_dev, DEV_PARSED) {
614                 if (sdev->state >= DEV_PROBED)
615                         continue;
616                 if (strcmp(sdev->devargs.name, dev->device->name) != 0)
617                         continue;
618                 rte_eth_dev_owner_set(port_id, &PRIV(fs_dev)->my_owner);
619                 /* The actual owner will be checked after the port probing. */
620                 break;
621         }
622         return 0;
623 }