1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2022 Intel Corporation
5 #include <rte_kvargs.h>
6 #include <rte_malloc.h>
8 #include "ethdev_driver.h"
9 #include "ethdev_private.h"
12 * A set of values to describe the possible states of a switch domain.
14 enum rte_eth_switch_domain_state {
15 RTE_ETH_SWITCH_DOMAIN_UNUSED = 0,
16 RTE_ETH_SWITCH_DOMAIN_ALLOCATED
20 * Array of switch domains available for allocation. Array is sized to
21 * RTE_MAX_ETHPORTS elements as there cannot be more active switch domains than
22 * ethdev ports in a single process.
24 static struct rte_eth_dev_switch {
25 enum rte_eth_switch_domain_state state;
26 } eth_dev_switch_domains[RTE_MAX_ETHPORTS];
28 static struct rte_eth_dev *
29 eth_dev_allocated(const char *name)
33 RTE_BUILD_BUG_ON(RTE_MAX_ETHPORTS >= UINT16_MAX);
35 for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
36 if (rte_eth_devices[i].data != NULL &&
37 strcmp(rte_eth_devices[i].data->name, name) == 0)
38 return &rte_eth_devices[i];
44 eth_dev_find_free_port(void)
48 for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
49 /* Using shared name field to find a free port. */
50 if (eth_dev_shared_data->data[i].name[0] == '\0') {
51 RTE_ASSERT(rte_eth_devices[i].state ==
56 return RTE_MAX_ETHPORTS;
59 static struct rte_eth_dev *
60 eth_dev_get(uint16_t port_id)
62 struct rte_eth_dev *eth_dev = &rte_eth_devices[port_id];
64 eth_dev->data = ð_dev_shared_data->data[port_id];
70 rte_eth_dev_allocate(const char *name)
73 struct rte_eth_dev *eth_dev = NULL;
76 name_len = strnlen(name, RTE_ETH_NAME_MAX_LEN);
78 RTE_ETHDEV_LOG(ERR, "Zero length Ethernet device name\n");
82 if (name_len >= RTE_ETH_NAME_MAX_LEN) {
83 RTE_ETHDEV_LOG(ERR, "Ethernet device name is too long\n");
87 eth_dev_shared_data_prepare();
89 /* Synchronize port creation between primary and secondary threads. */
90 rte_spinlock_lock(ð_dev_shared_data->ownership_lock);
92 if (eth_dev_allocated(name) != NULL) {
94 "Ethernet device with name %s already allocated\n",
99 port_id = eth_dev_find_free_port();
100 if (port_id == RTE_MAX_ETHPORTS) {
102 "Reached maximum number of Ethernet ports\n");
106 eth_dev = eth_dev_get(port_id);
107 strlcpy(eth_dev->data->name, name, sizeof(eth_dev->data->name));
108 eth_dev->data->port_id = port_id;
109 eth_dev->data->backer_port_id = RTE_MAX_ETHPORTS;
110 eth_dev->data->mtu = RTE_ETHER_MTU;
111 pthread_mutex_init(ð_dev->data->flow_ops_mutex, NULL);
114 rte_spinlock_unlock(ð_dev_shared_data->ownership_lock);
120 rte_eth_dev_allocated(const char *name)
122 struct rte_eth_dev *ethdev;
124 eth_dev_shared_data_prepare();
126 rte_spinlock_lock(ð_dev_shared_data->ownership_lock);
128 ethdev = eth_dev_allocated(name);
130 rte_spinlock_unlock(ð_dev_shared_data->ownership_lock);
136 * Attach to a port already registered by the primary process, which
137 * makes sure that the same device would have the same port ID both
138 * in the primary and secondary process.
141 rte_eth_dev_attach_secondary(const char *name)
144 struct rte_eth_dev *eth_dev = NULL;
146 eth_dev_shared_data_prepare();
148 /* Synchronize port attachment to primary port creation and release. */
149 rte_spinlock_lock(ð_dev_shared_data->ownership_lock);
151 for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
152 if (strcmp(eth_dev_shared_data->data[i].name, name) == 0)
155 if (i == RTE_MAX_ETHPORTS) {
157 "Device %s is not driven by the primary process\n",
160 eth_dev = eth_dev_get(i);
161 RTE_ASSERT(eth_dev->data->port_id == i);
164 rte_spinlock_unlock(ð_dev_shared_data->ownership_lock);
169 rte_eth_dev_callback_process(struct rte_eth_dev *dev,
170 enum rte_eth_event_type event, void *ret_param)
172 struct rte_eth_dev_callback *cb_lst;
173 struct rte_eth_dev_callback dev_cb;
176 rte_spinlock_lock(ð_dev_cb_lock);
177 TAILQ_FOREACH(cb_lst, &(dev->link_intr_cbs), next) {
178 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
182 if (ret_param != NULL)
183 dev_cb.ret_param = ret_param;
185 rte_spinlock_unlock(ð_dev_cb_lock);
186 rc = dev_cb.cb_fn(dev->data->port_id, dev_cb.event,
187 dev_cb.cb_arg, dev_cb.ret_param);
188 rte_spinlock_lock(ð_dev_cb_lock);
191 rte_spinlock_unlock(ð_dev_cb_lock);
196 rte_eth_dev_probing_finish(struct rte_eth_dev *dev)
202 * for secondary process, at that point we expect device
203 * to be already 'usable', so shared data and all function pointers
204 * for fast-path devops have to be setup properly inside rte_eth_dev.
206 if (rte_eal_process_type() == RTE_PROC_SECONDARY)
207 eth_dev_fp_ops_setup(rte_eth_fp_ops + dev->data->port_id, dev);
209 rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_NEW, NULL);
211 dev->state = RTE_ETH_DEV_ATTACHED;
215 rte_eth_dev_release_port(struct rte_eth_dev *eth_dev)
220 eth_dev_shared_data_prepare();
222 if (eth_dev->state != RTE_ETH_DEV_UNUSED)
223 rte_eth_dev_callback_process(eth_dev,
224 RTE_ETH_EVENT_DESTROY, NULL);
226 eth_dev_fp_ops_reset(rte_eth_fp_ops + eth_dev->data->port_id);
228 rte_spinlock_lock(ð_dev_shared_data->ownership_lock);
230 eth_dev->state = RTE_ETH_DEV_UNUSED;
231 eth_dev->device = NULL;
232 eth_dev->process_private = NULL;
233 eth_dev->intr_handle = NULL;
234 eth_dev->rx_pkt_burst = NULL;
235 eth_dev->tx_pkt_burst = NULL;
236 eth_dev->tx_pkt_prepare = NULL;
237 eth_dev->rx_queue_count = NULL;
238 eth_dev->rx_descriptor_status = NULL;
239 eth_dev->tx_descriptor_status = NULL;
240 eth_dev->dev_ops = NULL;
242 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
243 rte_free(eth_dev->data->rx_queues);
244 rte_free(eth_dev->data->tx_queues);
245 rte_free(eth_dev->data->mac_addrs);
246 rte_free(eth_dev->data->hash_mac_addrs);
247 rte_free(eth_dev->data->dev_private);
248 pthread_mutex_destroy(ð_dev->data->flow_ops_mutex);
249 memset(eth_dev->data, 0, sizeof(struct rte_eth_dev_data));
252 rte_spinlock_unlock(ð_dev_shared_data->ownership_lock);
258 rte_eth_dev_create(struct rte_device *device, const char *name,
259 size_t priv_data_size,
260 ethdev_bus_specific_init ethdev_bus_specific_init,
261 void *bus_init_params,
262 ethdev_init_t ethdev_init, void *init_params)
264 struct rte_eth_dev *ethdev;
267 RTE_FUNC_PTR_OR_ERR_RET(*ethdev_init, -EINVAL);
269 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
270 ethdev = rte_eth_dev_allocate(name);
274 if (priv_data_size) {
275 ethdev->data->dev_private = rte_zmalloc_socket(
276 name, priv_data_size, RTE_CACHE_LINE_SIZE,
279 if (!ethdev->data->dev_private) {
281 "failed to allocate private data\n");
287 ethdev = rte_eth_dev_attach_secondary(name);
290 "secondary process attach failed, ethdev doesn't exist\n");
295 ethdev->device = device;
297 if (ethdev_bus_specific_init) {
298 retval = ethdev_bus_specific_init(ethdev, bus_init_params);
301 "ethdev bus specific initialisation failed\n");
306 retval = ethdev_init(ethdev, init_params);
308 RTE_ETHDEV_LOG(ERR, "ethdev initialisation failed\n");
312 rte_eth_dev_probing_finish(ethdev);
317 rte_eth_dev_release_port(ethdev);
322 rte_eth_dev_destroy(struct rte_eth_dev *ethdev,
323 ethdev_uninit_t ethdev_uninit)
327 ethdev = rte_eth_dev_allocated(ethdev->data->name);
331 RTE_FUNC_PTR_OR_ERR_RET(*ethdev_uninit, -EINVAL);
333 ret = ethdev_uninit(ethdev);
337 return rte_eth_dev_release_port(ethdev);
341 rte_eth_dev_get_by_name(const char *name)
345 if (rte_eth_dev_get_port_by_name(name, &pid))
348 return &rte_eth_devices[pid];
352 rte_eth_dev_is_rx_hairpin_queue(struct rte_eth_dev *dev, uint16_t queue_id)
354 if (dev->data->rx_queue_state[queue_id] == RTE_ETH_QUEUE_STATE_HAIRPIN)
360 rte_eth_dev_is_tx_hairpin_queue(struct rte_eth_dev *dev, uint16_t queue_id)
362 if (dev->data->tx_queue_state[queue_id] == RTE_ETH_QUEUE_STATE_HAIRPIN)
368 rte_eth_dev_internal_reset(struct rte_eth_dev *dev)
370 if (dev->data->dev_started) {
371 RTE_ETHDEV_LOG(ERR, "Port %u must be stopped to allow reset\n",
376 eth_dev_rx_queue_config(dev, 0);
377 eth_dev_tx_queue_config(dev, 0);
379 memset(&dev->data->dev_conf, 0, sizeof(dev->data->dev_conf));
383 eth_dev_devargs_tokenise(struct rte_kvargs *arglist, const char *str_in)
386 struct rte_kvargs_pair *pair;
389 arglist->str = strdup(str_in);
390 if (arglist->str == NULL)
393 letter = arglist->str;
396 pair = &arglist->pairs[0];
399 case 0: /* Initial */
402 else if (*letter == '\0')
409 case 1: /* Parsing key */
410 if (*letter == '=') {
412 pair->value = letter + 1;
414 } else if (*letter == ',' || *letter == '\0')
419 case 2: /* Parsing value */
422 else if (*letter == ',') {
425 pair = &arglist->pairs[arglist->count];
427 } else if (*letter == '\0') {
430 pair = &arglist->pairs[arglist->count];
435 case 3: /* Parsing list */
438 else if (*letter == '\0')
447 rte_eth_devargs_parse(const char *dargs, struct rte_eth_devargs *eth_da)
449 struct rte_kvargs args;
450 struct rte_kvargs_pair *pair;
454 memset(eth_da, 0, sizeof(*eth_da));
456 result = eth_dev_devargs_tokenise(&args, dargs);
460 for (i = 0; i < args.count; i++) {
461 pair = &args.pairs[i];
462 if (strcmp("representor", pair->key) == 0) {
463 if (eth_da->type != RTE_ETH_REPRESENTOR_NONE) {
464 RTE_LOG(ERR, EAL, "duplicated representor key: %s\n",
469 result = rte_eth_devargs_parse_representor_ports(
470 pair->value, eth_da);
483 eth_dev_dma_mzone_name(char *name, size_t len, uint16_t port_id, uint16_t queue_id,
484 const char *ring_name)
486 return snprintf(name, len, "eth_p%d_q%d_%s",
487 port_id, queue_id, ring_name);
491 rte_eth_dma_zone_free(const struct rte_eth_dev *dev, const char *ring_name,
494 char z_name[RTE_MEMZONE_NAMESIZE];
495 const struct rte_memzone *mz;
498 rc = eth_dev_dma_mzone_name(z_name, sizeof(z_name), dev->data->port_id,
499 queue_id, ring_name);
500 if (rc >= RTE_MEMZONE_NAMESIZE) {
501 RTE_ETHDEV_LOG(ERR, "ring name too long\n");
502 return -ENAMETOOLONG;
505 mz = rte_memzone_lookup(z_name);
507 rc = rte_memzone_free(mz);
514 const struct rte_memzone *
515 rte_eth_dma_zone_reserve(const struct rte_eth_dev *dev, const char *ring_name,
516 uint16_t queue_id, size_t size, unsigned int align,
519 char z_name[RTE_MEMZONE_NAMESIZE];
520 const struct rte_memzone *mz;
523 rc = eth_dev_dma_mzone_name(z_name, sizeof(z_name), dev->data->port_id,
524 queue_id, ring_name);
525 if (rc >= RTE_MEMZONE_NAMESIZE) {
526 RTE_ETHDEV_LOG(ERR, "ring name too long\n");
527 rte_errno = ENAMETOOLONG;
531 mz = rte_memzone_lookup(z_name);
533 if ((socket_id != SOCKET_ID_ANY && socket_id != mz->socket_id) ||
535 ((uintptr_t)mz->addr & (align - 1)) != 0) {
537 "memzone %s does not justify the requested attributes\n",
545 return rte_memzone_reserve_aligned(z_name, size, socket_id,
546 RTE_MEMZONE_IOVA_CONTIG, align);
550 rte_eth_hairpin_queue_peer_bind(uint16_t cur_port, uint16_t cur_queue,
551 struct rte_hairpin_peer_info *peer_info,
554 struct rte_eth_dev *dev;
556 if (peer_info == NULL)
559 /* No need to check the validity again. */
560 dev = &rte_eth_devices[cur_port];
561 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->hairpin_queue_peer_bind,
564 return (*dev->dev_ops->hairpin_queue_peer_bind)(dev, cur_queue,
565 peer_info, direction);
569 rte_eth_hairpin_queue_peer_unbind(uint16_t cur_port, uint16_t cur_queue,
572 struct rte_eth_dev *dev;
574 /* No need to check the validity again. */
575 dev = &rte_eth_devices[cur_port];
576 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->hairpin_queue_peer_unbind,
579 return (*dev->dev_ops->hairpin_queue_peer_unbind)(dev, cur_queue,
584 rte_eth_hairpin_queue_peer_update(uint16_t peer_port, uint16_t peer_queue,
585 struct rte_hairpin_peer_info *cur_info,
586 struct rte_hairpin_peer_info *peer_info,
589 struct rte_eth_dev *dev;
591 /* Current queue information is not mandatory. */
592 if (peer_info == NULL)
595 /* No need to check the validity again. */
596 dev = &rte_eth_devices[peer_port];
597 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->hairpin_queue_peer_update,
600 return (*dev->dev_ops->hairpin_queue_peer_update)(dev, peer_queue,
601 cur_info, peer_info, direction);
605 rte_eth_ip_reassembly_dynfield_register(int *field_offset, int *flag_offset)
607 static const struct rte_mbuf_dynfield field_desc = {
608 .name = RTE_MBUF_DYNFIELD_IP_REASSEMBLY_NAME,
609 .size = sizeof(rte_eth_ip_reassembly_dynfield_t),
610 .align = __alignof__(rte_eth_ip_reassembly_dynfield_t),
612 static const struct rte_mbuf_dynflag ip_reassembly_dynflag = {
613 .name = RTE_MBUF_DYNFLAG_IP_REASSEMBLY_INCOMPLETE_NAME,
617 offset = rte_mbuf_dynfield_register(&field_desc);
620 if (field_offset != NULL)
621 *field_offset = offset;
623 offset = rte_mbuf_dynflag_register(&ip_reassembly_dynflag);
626 if (flag_offset != NULL)
627 *flag_offset = offset;
633 rte_eth_pkt_burst_dummy(void *queue __rte_unused,
634 struct rte_mbuf **pkts __rte_unused,
635 uint16_t nb_pkts __rte_unused)
641 rte_eth_representor_id_get(uint16_t port_id,
642 enum rte_eth_representor_type type,
643 int controller, int pf, int representor_port,
648 struct rte_eth_representor_info *info = NULL;
651 if (type == RTE_ETH_REPRESENTOR_NONE)
656 /* Get PMD representor range info. */
657 ret = rte_eth_representor_info_get(port_id, NULL);
658 if (ret == -ENOTSUP && type == RTE_ETH_REPRESENTOR_VF &&
659 controller == -1 && pf == -1) {
660 /* Direct mapping for legacy VF representor. */
661 *repr_id = representor_port;
663 } else if (ret < 0) {
667 size = sizeof(*info) + n * sizeof(info->ranges[0]);
668 info = calloc(1, size);
671 info->nb_ranges_alloc = n;
672 ret = rte_eth_representor_info_get(port_id, info);
676 /* Default controller and pf to caller. */
677 if (controller == -1)
678 controller = info->controller;
682 /* Locate representor ID. */
684 for (i = 0; i < info->nb_ranges; ++i) {
685 if (info->ranges[i].type != type)
687 if (info->ranges[i].controller != controller)
689 if (info->ranges[i].id_end < info->ranges[i].id_base) {
690 RTE_LOG(WARNING, EAL, "Port %hu invalid representor ID Range %u - %u, entry %d\n",
691 port_id, info->ranges[i].id_base,
692 info->ranges[i].id_end, i);
696 count = info->ranges[i].id_end - info->ranges[i].id_base + 1;
697 switch (info->ranges[i].type) {
698 case RTE_ETH_REPRESENTOR_PF:
699 if (pf < info->ranges[i].pf ||
700 pf >= info->ranges[i].pf + count)
702 *repr_id = info->ranges[i].id_base +
703 (pf - info->ranges[i].pf);
706 case RTE_ETH_REPRESENTOR_VF:
707 if (info->ranges[i].pf != pf)
709 if (representor_port < info->ranges[i].vf ||
710 representor_port >= info->ranges[i].vf + count)
712 *repr_id = info->ranges[i].id_base +
713 (representor_port - info->ranges[i].vf);
716 case RTE_ETH_REPRESENTOR_SF:
717 if (info->ranges[i].pf != pf)
719 if (representor_port < info->ranges[i].sf ||
720 representor_port >= info->ranges[i].sf + count)
722 *repr_id = info->ranges[i].id_base +
723 (representor_port - info->ranges[i].sf);
736 rte_eth_switch_domain_alloc(uint16_t *domain_id)
740 *domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
742 for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
743 if (eth_dev_switch_domains[i].state ==
744 RTE_ETH_SWITCH_DOMAIN_UNUSED) {
745 eth_dev_switch_domains[i].state =
746 RTE_ETH_SWITCH_DOMAIN_ALLOCATED;
756 rte_eth_switch_domain_free(uint16_t domain_id)
758 if (domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID ||
759 domain_id >= RTE_MAX_ETHPORTS)
762 if (eth_dev_switch_domains[domain_id].state !=
763 RTE_ETH_SWITCH_DOMAIN_ALLOCATED)
766 eth_dev_switch_domains[domain_id].state = RTE_ETH_SWITCH_DOMAIN_UNUSED;