1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2017 6WIND S.A.
3 * Copyright 2017 Mellanox Technologies, Ltd
6 #ifndef _RTE_ETH_FAILSAFE_PRIVATE_H_
7 #define _RTE_ETH_FAILSAFE_PRIVATE_H_
10 #include <sys/queue.h>
13 #include <rte_atomic.h>
15 #include <rte_ethdev_driver.h>
16 #include <rte_devargs.h>
18 #include <rte_interrupts.h>
20 #define FAILSAFE_DRIVER_NAME "Fail-safe PMD"
21 #define FAILSAFE_OWNER_NAME "Fail-safe"
23 #define PMD_FAILSAFE_MAC_KVARG "mac"
24 #define PMD_FAILSAFE_HOTPLUG_POLL_KVARG "hotplug_poll"
25 #define PMD_FAILSAFE_PARAM_STRING \
27 "exec(<shell command>)," \
33 #define FAILSAFE_HOTPLUG_DEFAULT_TIMEOUT_MS 2000
35 #define FAILSAFE_MAX_ETHPORTS 2
36 #define FAILSAFE_MAX_ETHADDR 128
38 #define DEVARGS_MAXLEN 4096
40 enum rxp_service_state {
50 /* epoll file descriptor */
52 /* event vector to be used by epoll */
53 struct rte_epoll_event *evec;
58 enum rxp_service_state sstate;
64 /* next sub_device to poll */
65 struct sub_device *sdev;
66 unsigned int socket_id;
68 unsigned int enable_events:1;
69 struct rte_eth_rxq_info info;
70 rte_atomic64_t refcnt[];
76 unsigned int socket_id;
77 struct rte_eth_txq_info info;
78 rte_atomic64_t refcnt[];
82 TAILQ_ENTRY(rte_flow) next;
84 struct rte_flow *flows[FAILSAFE_MAX_ETHPORTS];
85 /* flow description for synchronization */
86 struct rte_flow_conv_rule rule;
99 struct rte_eth_stats stats;
104 * Allocated in shared memory.
107 /* Exhaustive DPDK device description */
108 struct sub_device *next;
109 struct rte_devargs devargs;
110 struct rte_bus *bus; /* for primary process only. */
111 struct rte_device *dev; /* for primary process only. */
113 /* Device state machine */
114 enum dev_state state;
115 /* Last stats snapshot passed to user */
116 struct fs_stats stats_snapshot;
117 /* Some device are defined as a command line */
119 /* Others are retrieved through a file descriptor */
121 /* fail-safe device backreference */
122 uint16_t fs_port_id; /* shared between processes */
123 /* sub device port id*/
124 uint16_t sdev_port_id; /* shared between processes */
125 /* flag calling for recollection */
126 volatile unsigned int remove:1;
127 /* flow isolation state */
129 /* RMV callback registration state */
130 unsigned int rmv_callback:1;
131 /* LSC callback registration state */
132 unsigned int lsc_callback:1;
136 * This is referenced by eth_dev->data->dev_private
137 * This is shared between processes.
140 struct rte_eth_dev_data *data; /* backreference to shared data. */
142 * Set of sub_devices.
143 * subs[0] is the preferred device
144 * any other is just another slave
146 struct sub_device *subs; /* shared between processes */
147 uint8_t subs_head; /* if head == tail, no subs */
148 uint8_t subs_tail; /* first invalid */
149 uint8_t subs_tx; /* current emitting device */
150 uint8_t current_probed;
152 TAILQ_HEAD(sub_flows, rte_flow) flow_list;
153 /* current number of mac_addr slots allocated. */
154 uint32_t nb_mac_addr;
155 struct rte_ether_addr mac_addrs[FAILSAFE_MAX_ETHADDR];
156 uint32_t mac_addr_pool[FAILSAFE_MAX_ETHADDR];
157 uint32_t nb_mcast_addr;
158 struct rte_ether_addr *mcast_addrs;
159 /* current capabilities */
160 struct rte_eth_dev_info infos;
161 struct rte_eth_dev_owner my_owner; /* Unique owner. */
162 struct rte_intr_handle intr_handle; /* Port interrupt handle. */
164 * Fail-safe state machine.
165 * This level will be tracking state of the EAL and eth
166 * layer at large as defined by the user application.
167 * It will then steer the sub_devices toward the same
168 * synchronized state.
170 enum dev_state state;
171 struct rte_eth_stats stats_accumulator;
173 * Rx interrupts/events proxy.
174 * The PMD issues Rx events to the EAL on behalf of its subdevices,
175 * it does that by registering an event-fd for each of its queues with
176 * the EAL. A PMD service thread listens to all the Rx events from the
177 * subdevices, when an Rx event is issued by a subdevice it will be
178 * caught by this service with will trigger an Rx event in the
179 * appropriate failsafe Rx queue.
182 pthread_mutex_t hotplug_mutex;
183 /* Hot-plug mutex is locked by the alarm mechanism. */
184 volatile unsigned int alarm_lock:1;
185 unsigned int pending_alarm:1; /* An alarm is pending */
186 /* flow isolation state */
192 int failsafe_rx_intr_install(struct rte_eth_dev *dev);
193 void failsafe_rx_intr_uninstall(struct rte_eth_dev *dev);
194 int failsafe_rx_intr_install_subdevice(struct sub_device *sdev);
195 void failsafe_rx_intr_uninstall_subdevice(struct sub_device *sdev);
199 int failsafe_hotplug_alarm_install(struct rte_eth_dev *dev);
200 int failsafe_hotplug_alarm_cancel(struct rte_eth_dev *dev);
204 void failsafe_set_burst_fn(struct rte_eth_dev *dev, int force_safe);
206 uint16_t failsafe_rx_burst(void *rxq,
207 struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
208 uint16_t failsafe_tx_burst(void *txq,
209 struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
211 uint16_t failsafe_rx_burst_fast(void *rxq,
212 struct rte_mbuf **rx_pkts, uint16_t nb_pkts);
213 uint16_t failsafe_tx_burst_fast(void *txq,
214 struct rte_mbuf **tx_pkts, uint16_t nb_pkts);
218 int failsafe_args_parse(struct rte_eth_dev *dev, const char *params);
219 void failsafe_args_free(struct rte_eth_dev *dev);
220 int failsafe_args_count_subdevice(struct rte_eth_dev *dev, const char *params);
221 int failsafe_args_parse_subs(struct rte_eth_dev *dev);
225 int failsafe_eal_init(struct rte_eth_dev *dev);
226 int failsafe_eal_uninit(struct rte_eth_dev *dev);
230 int failsafe_eth_dev_state_sync(struct rte_eth_dev *dev);
231 void failsafe_eth_dev_unregister_callbacks(struct sub_device *sdev);
232 void failsafe_dev_remove(struct rte_eth_dev *dev);
233 void failsafe_stats_increment(struct rte_eth_stats *to,
234 struct rte_eth_stats *from);
235 int failsafe_eth_rmv_event_callback(uint16_t port_id,
236 enum rte_eth_event_type type,
237 void *arg, void *out);
238 int failsafe_eth_lsc_event_callback(uint16_t port_id,
239 enum rte_eth_event_type event,
240 void *cb_arg, void *out);
241 int failsafe_eth_new_event_callback(uint16_t port_id,
242 enum rte_eth_event_type event,
243 void *cb_arg, void *out);
247 extern const char pmd_failsafe_driver_name[];
248 extern const struct eth_dev_ops failsafe_ops;
249 extern const struct rte_flow_ops fs_flow_ops;
250 extern uint64_t failsafe_hotplug_poll;
251 extern int failsafe_mac_from_arg;
255 /* dev: (struct rte_eth_dev *) fail-safe device */
257 ((struct fs_priv *)(dev)->data->dev_private)
259 /* sdev: (struct sub_device *) */
261 ((sdev)->sdev_port_id == RTE_MAX_ETHPORTS ? \
262 NULL : &rte_eth_devices[(sdev)->sdev_port_id])
264 /* sdev: (struct sub_device *) */
265 #define PORT_ID(sdev) \
266 ((sdev)->sdev_port_id)
268 /* sdev: (struct sub_device *) */
269 #define SUB_ID(sdev) \
273 * Stateful iterator construct over fail-safe sub-devices:
274 * s: (struct sub_device *), iterator
275 * i: (uint8_t), increment
276 * dev: (struct rte_eth_dev *), fail-safe ethdev
277 * state: (enum dev_state), minimum acceptable device state
279 #define FOREACH_SUBDEV_STATE(s, i, dev, state) \
280 for (s = fs_find_next((dev), 0, state, &i); \
282 s = fs_find_next((dev), i + 1, state, &i))
285 * Iterator construct over fail-safe sub-devices:
286 * s: (struct sub_device *), iterator
287 * i: (uint8_t), increment
288 * dev: (struct rte_eth_dev *), fail-safe ethdev
290 #define FOREACH_SUBDEV(s, i, dev) \
291 FOREACH_SUBDEV_STATE(s, i, dev, DEV_UNDEFINED)
293 /* dev: (struct rte_eth_dev *) fail-safe device */
294 #define PREFERRED_SUBDEV(dev) \
295 (&PRIV(dev)->subs[0])
297 /* dev: (struct rte_eth_dev *) fail-safe device */
298 #define TX_SUBDEV(dev) \
299 (PRIV(dev)->subs_tx >= PRIV(dev)->subs_tail ? NULL \
300 : (PRIV(dev)->subs[PRIV(dev)->subs_tx].state < DEV_PROBED ? NULL \
301 : &PRIV(dev)->subs[PRIV(dev)->subs_tx]))
304 * s: (struct sub_device *)
305 * ops: (struct eth_dev_ops) member
307 #define SUBOPS(s, ops) \
308 (ETH(s)->dev_ops->ops)
315 * a: (rte_atomic64_t)
317 #define FS_ATOMIC_P(a) \
318 rte_atomic64_set(&(a), 1)
321 * a: (rte_atomic64_t)
323 #define FS_ATOMIC_V(a) \
324 rte_atomic64_set(&(a), 0)
327 * s: (struct sub_device *)
330 #define FS_ATOMIC_RX(s, i) \
333 (fs_dev(s)->data->rx_queues[i]))->refcnt[(s)->sid])
335 * s: (struct sub_device *)
338 #define FS_ATOMIC_TX(s, i) \
341 (fs_dev(s)->data->tx_queues[i]))->refcnt[(s)->sid])
343 #ifdef RTE_EXEC_ENV_FREEBSD
344 #define FS_THREADID_TYPE void*
345 #define FS_THREADID_FMT "p"
347 #define FS_THREADID_TYPE unsigned long
348 #define FS_THREADID_FMT "lu"
351 extern int failsafe_logtype;
353 #define LOG__(l, m, ...) \
354 rte_log(RTE_LOG_ ## l, failsafe_logtype, \
355 "net_failsafe: " m "%c", __VA_ARGS__)
357 #define LOG_(level, ...) LOG__(level, __VA_ARGS__, '\n')
358 #define DEBUG(...) LOG_(DEBUG, __VA_ARGS__)
359 #define INFO(...) LOG_(INFO, __VA_ARGS__)
360 #define WARN(...) LOG_(WARNING, __VA_ARGS__)
361 #define ERROR(...) LOG_(ERR, __VA_ARGS__)
363 /* inlined functions */
365 static inline struct sub_device *
366 fs_find_next(struct rte_eth_dev *dev,
368 enum dev_state min_state,
371 struct sub_device *subs;
374 subs = PRIV(dev)->subs;
375 tail = PRIV(dev)->subs_tail;
377 if (subs[sid].state >= min_state)
387 static inline struct rte_eth_dev *
388 fs_dev(struct sub_device *sdev) {
389 return &rte_eth_devices[sdev->fs_port_id];
393 * Lock hot-plug mutex.
394 * is_alarm means that the caller is, for sure, the hot-plug alarm mechanism.
397 fs_lock(struct rte_eth_dev *dev, unsigned int is_alarm)
402 ret = pthread_mutex_trylock(&PRIV(dev)->hotplug_mutex);
404 DEBUG("Hot-plug mutex lock trying failed(%s), will try"
405 " again later...", strerror(ret));
408 PRIV(dev)->alarm_lock = 1;
410 ret = pthread_mutex_lock(&PRIV(dev)->hotplug_mutex);
412 ERROR("Cannot lock mutex(%s)", strerror(ret));
420 * Unlock hot-plug mutex.
421 * is_alarm means that the caller is, for sure, the hot-plug alarm mechanism.
424 fs_unlock(struct rte_eth_dev *dev, unsigned int is_alarm)
429 RTE_ASSERT(PRIV(dev)->alarm_lock == 1);
430 PRIV(dev)->alarm_lock = 0;
432 ret = pthread_mutex_unlock(&PRIV(dev)->hotplug_mutex);
434 ERROR("Cannot unlock hot-plug mutex(%s)", strerror(ret));
438 * Switch emitting device.
439 * If banned is set, banned must not be considered for
440 * the role of emitting device.
443 fs_switch_dev(struct rte_eth_dev *dev,
444 struct sub_device *banned)
446 struct sub_device *txd;
447 enum dev_state req_state;
449 req_state = PRIV(dev)->state;
450 txd = TX_SUBDEV(dev);
451 if (PREFERRED_SUBDEV(dev)->state >= req_state &&
452 PREFERRED_SUBDEV(dev) != banned) {
453 if (txd != PREFERRED_SUBDEV(dev) &&
455 (req_state == DEV_STARTED) ||
456 (txd && txd->state < DEV_STARTED))) {
457 DEBUG("Switching tx_dev to preferred sub_device");
458 PRIV(dev)->subs_tx = 0;
460 } else if ((txd && txd->state < req_state) ||
463 struct sub_device *sdev = NULL;
466 /* Using acceptable device */
467 FOREACH_SUBDEV_STATE(sdev, i, dev, req_state) {
470 DEBUG("Switching tx_dev to sub_device %d",
472 PRIV(dev)->subs_tx = i;
475 if (i >= PRIV(dev)->subs_tail || sdev == NULL) {
476 DEBUG("No device ready, deactivating tx_dev");
477 PRIV(dev)->subs_tx = PRIV(dev)->subs_tail;
482 failsafe_set_burst_fn(dev, 0);
487 * Adjust error value and rte_errno to the fail-safe actual error value.
490 fs_err(struct sub_device *sdev, int err)
492 /* A device removal shouldn't be reported as an error. */
493 if (sdev->remove == 1 || err == -EIO)
494 return rte_errno = 0;
497 #endif /* _RTE_ETH_FAILSAFE_PRIVATE_H_ */