36e5389b3a6cc7cd5a87a8685b193807e0f5bddd
[dpdk.git] / lib / librte_ethdev / rte_ethdev.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright(c) 2010-2017 Intel Corporation
3  */
4
5 #include <sys/types.h>
6 #include <sys/queue.h>
7 #include <ctype.h>
8 #include <stdio.h>
9 #include <stdlib.h>
10 #include <string.h>
11 #include <stdarg.h>
12 #include <errno.h>
13 #include <stdbool.h>
14 #include <stdint.h>
15 #include <inttypes.h>
16 #include <netinet/in.h>
17
18 #include <rte_byteorder.h>
19 #include <rte_log.h>
20 #include <rte_debug.h>
21 #include <rte_interrupts.h>
22 #include <rte_memory.h>
23 #include <rte_memcpy.h>
24 #include <rte_memzone.h>
25 #include <rte_launch.h>
26 #include <rte_eal.h>
27 #include <rte_per_lcore.h>
28 #include <rte_lcore.h>
29 #include <rte_atomic.h>
30 #include <rte_branch_prediction.h>
31 #include <rte_common.h>
32 #include <rte_mempool.h>
33 #include <rte_malloc.h>
34 #include <rte_mbuf.h>
35 #include <rte_errno.h>
36 #include <rte_spinlock.h>
37 #include <rte_string_fns.h>
38 #include <rte_kvargs.h>
39 #include <rte_class.h>
40
41 #include "rte_ether.h"
42 #include "rte_ethdev.h"
43 #include "rte_ethdev_driver.h"
44 #include "ethdev_profile.h"
45 #include "ethdev_private.h"
46
47 int rte_eth_dev_logtype;
48
49 static const char *MZ_RTE_ETH_DEV_DATA = "rte_eth_dev_data";
50 struct rte_eth_dev rte_eth_devices[RTE_MAX_ETHPORTS];
51 static uint16_t eth_dev_last_created_port;
52
53 /* spinlock for eth device callbacks */
54 static rte_spinlock_t rte_eth_dev_cb_lock = RTE_SPINLOCK_INITIALIZER;
55
56 /* spinlock for add/remove rx callbacks */
57 static rte_spinlock_t rte_eth_rx_cb_lock = RTE_SPINLOCK_INITIALIZER;
58
59 /* spinlock for add/remove tx callbacks */
60 static rte_spinlock_t rte_eth_tx_cb_lock = RTE_SPINLOCK_INITIALIZER;
61
62 /* spinlock for shared data allocation */
63 static rte_spinlock_t rte_eth_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
64
65 /* store statistics names and its offset in stats structure  */
66 struct rte_eth_xstats_name_off {
67         char name[RTE_ETH_XSTATS_NAME_SIZE];
68         unsigned offset;
69 };
70
71 /* Shared memory between primary and secondary processes. */
72 static struct {
73         uint64_t next_owner_id;
74         rte_spinlock_t ownership_lock;
75         struct rte_eth_dev_data data[RTE_MAX_ETHPORTS];
76 } *rte_eth_dev_shared_data;
77
78 static const struct rte_eth_xstats_name_off rte_stats_strings[] = {
79         {"rx_good_packets", offsetof(struct rte_eth_stats, ipackets)},
80         {"tx_good_packets", offsetof(struct rte_eth_stats, opackets)},
81         {"rx_good_bytes", offsetof(struct rte_eth_stats, ibytes)},
82         {"tx_good_bytes", offsetof(struct rte_eth_stats, obytes)},
83         {"rx_missed_errors", offsetof(struct rte_eth_stats, imissed)},
84         {"rx_errors", offsetof(struct rte_eth_stats, ierrors)},
85         {"tx_errors", offsetof(struct rte_eth_stats, oerrors)},
86         {"rx_mbuf_allocation_errors", offsetof(struct rte_eth_stats,
87                 rx_nombuf)},
88 };
89
90 #define RTE_NB_STATS (sizeof(rte_stats_strings) / sizeof(rte_stats_strings[0]))
91
92 static const struct rte_eth_xstats_name_off rte_rxq_stats_strings[] = {
93         {"packets", offsetof(struct rte_eth_stats, q_ipackets)},
94         {"bytes", offsetof(struct rte_eth_stats, q_ibytes)},
95         {"errors", offsetof(struct rte_eth_stats, q_errors)},
96 };
97
98 #define RTE_NB_RXQ_STATS (sizeof(rte_rxq_stats_strings) /       \
99                 sizeof(rte_rxq_stats_strings[0]))
100
101 static const struct rte_eth_xstats_name_off rte_txq_stats_strings[] = {
102         {"packets", offsetof(struct rte_eth_stats, q_opackets)},
103         {"bytes", offsetof(struct rte_eth_stats, q_obytes)},
104 };
105 #define RTE_NB_TXQ_STATS (sizeof(rte_txq_stats_strings) /       \
106                 sizeof(rte_txq_stats_strings[0]))
107
108 #define RTE_RX_OFFLOAD_BIT2STR(_name)   \
109         { DEV_RX_OFFLOAD_##_name, #_name }
110
111 static const struct {
112         uint64_t offload;
113         const char *name;
114 } rte_rx_offload_names[] = {
115         RTE_RX_OFFLOAD_BIT2STR(VLAN_STRIP),
116         RTE_RX_OFFLOAD_BIT2STR(IPV4_CKSUM),
117         RTE_RX_OFFLOAD_BIT2STR(UDP_CKSUM),
118         RTE_RX_OFFLOAD_BIT2STR(TCP_CKSUM),
119         RTE_RX_OFFLOAD_BIT2STR(TCP_LRO),
120         RTE_RX_OFFLOAD_BIT2STR(QINQ_STRIP),
121         RTE_RX_OFFLOAD_BIT2STR(OUTER_IPV4_CKSUM),
122         RTE_RX_OFFLOAD_BIT2STR(MACSEC_STRIP),
123         RTE_RX_OFFLOAD_BIT2STR(HEADER_SPLIT),
124         RTE_RX_OFFLOAD_BIT2STR(VLAN_FILTER),
125         RTE_RX_OFFLOAD_BIT2STR(VLAN_EXTEND),
126         RTE_RX_OFFLOAD_BIT2STR(JUMBO_FRAME),
127         RTE_RX_OFFLOAD_BIT2STR(SCATTER),
128         RTE_RX_OFFLOAD_BIT2STR(TIMESTAMP),
129         RTE_RX_OFFLOAD_BIT2STR(SECURITY),
130         RTE_RX_OFFLOAD_BIT2STR(KEEP_CRC),
131         RTE_RX_OFFLOAD_BIT2STR(SCTP_CKSUM),
132         RTE_RX_OFFLOAD_BIT2STR(OUTER_UDP_CKSUM),
133 };
134
135 #undef RTE_RX_OFFLOAD_BIT2STR
136
137 #define RTE_TX_OFFLOAD_BIT2STR(_name)   \
138         { DEV_TX_OFFLOAD_##_name, #_name }
139
140 static const struct {
141         uint64_t offload;
142         const char *name;
143 } rte_tx_offload_names[] = {
144         RTE_TX_OFFLOAD_BIT2STR(VLAN_INSERT),
145         RTE_TX_OFFLOAD_BIT2STR(IPV4_CKSUM),
146         RTE_TX_OFFLOAD_BIT2STR(UDP_CKSUM),
147         RTE_TX_OFFLOAD_BIT2STR(TCP_CKSUM),
148         RTE_TX_OFFLOAD_BIT2STR(SCTP_CKSUM),
149         RTE_TX_OFFLOAD_BIT2STR(TCP_TSO),
150         RTE_TX_OFFLOAD_BIT2STR(UDP_TSO),
151         RTE_TX_OFFLOAD_BIT2STR(OUTER_IPV4_CKSUM),
152         RTE_TX_OFFLOAD_BIT2STR(QINQ_INSERT),
153         RTE_TX_OFFLOAD_BIT2STR(VXLAN_TNL_TSO),
154         RTE_TX_OFFLOAD_BIT2STR(GRE_TNL_TSO),
155         RTE_TX_OFFLOAD_BIT2STR(IPIP_TNL_TSO),
156         RTE_TX_OFFLOAD_BIT2STR(GENEVE_TNL_TSO),
157         RTE_TX_OFFLOAD_BIT2STR(MACSEC_INSERT),
158         RTE_TX_OFFLOAD_BIT2STR(MT_LOCKFREE),
159         RTE_TX_OFFLOAD_BIT2STR(MULTI_SEGS),
160         RTE_TX_OFFLOAD_BIT2STR(MBUF_FAST_FREE),
161         RTE_TX_OFFLOAD_BIT2STR(SECURITY),
162         RTE_TX_OFFLOAD_BIT2STR(UDP_TNL_TSO),
163         RTE_TX_OFFLOAD_BIT2STR(IP_TNL_TSO),
164         RTE_TX_OFFLOAD_BIT2STR(OUTER_UDP_CKSUM),
165         RTE_TX_OFFLOAD_BIT2STR(MATCH_METADATA),
166 };
167
168 #undef RTE_TX_OFFLOAD_BIT2STR
169
170 /**
171  * The user application callback description.
172  *
173  * It contains callback address to be registered by user application,
174  * the pointer to the parameters for callback, and the event type.
175  */
176 struct rte_eth_dev_callback {
177         TAILQ_ENTRY(rte_eth_dev_callback) next; /**< Callbacks list */
178         rte_eth_dev_cb_fn cb_fn;                /**< Callback address */
179         void *cb_arg;                           /**< Parameter for callback */
180         void *ret_param;                        /**< Return parameter */
181         enum rte_eth_event_type event;          /**< Interrupt event type */
182         uint32_t active;                        /**< Callback is executing */
183 };
184
185 enum {
186         STAT_QMAP_TX = 0,
187         STAT_QMAP_RX
188 };
189
190 int __rte_experimental
191 rte_eth_iterator_init(struct rte_dev_iterator *iter, const char *devargs_str)
192 {
193         int ret;
194         struct rte_devargs devargs = {.args = NULL};
195         const char *bus_param_key;
196         char *bus_str = NULL;
197         char *cls_str = NULL;
198         int str_size;
199
200         memset(iter, 0, sizeof(*iter));
201
202         /*
203          * The devargs string may use various syntaxes:
204          *   - 0000:08:00.0,representor=[1-3]
205          *   - pci:0000:06:00.0,representor=[0,5]
206          *   - class=eth,mac=00:11:22:33:44:55
207          * A new syntax is in development (not yet supported):
208          *   - bus=X,paramX=x/class=Y,paramY=y/driver=Z,paramZ=z
209          */
210
211         /*
212          * Handle pure class filter (i.e. without any bus-level argument),
213          * from future new syntax.
214          * rte_devargs_parse() is not yet supporting the new syntax,
215          * that's why this simple case is temporarily parsed here.
216          */
217 #define iter_anybus_str "class=eth,"
218         if (strncmp(devargs_str, iter_anybus_str,
219                         strlen(iter_anybus_str)) == 0) {
220                 iter->cls_str = devargs_str + strlen(iter_anybus_str);
221                 goto end;
222         }
223
224         /* Split bus, device and parameters. */
225         ret = rte_devargs_parse(&devargs, devargs_str);
226         if (ret != 0)
227                 goto error;
228
229         /*
230          * Assume parameters of old syntax can match only at ethdev level.
231          * Extra parameters will be ignored, thanks to "+" prefix.
232          */
233         str_size = strlen(devargs.args) + 2;
234         cls_str = malloc(str_size);
235         if (cls_str == NULL) {
236                 ret = -ENOMEM;
237                 goto error;
238         }
239         ret = snprintf(cls_str, str_size, "+%s", devargs.args);
240         if (ret != str_size - 1) {
241                 ret = -EINVAL;
242                 goto error;
243         }
244         iter->cls_str = cls_str;
245         free(devargs.args); /* allocated by rte_devargs_parse() */
246         devargs.args = NULL;
247
248         iter->bus = devargs.bus;
249         if (iter->bus->dev_iterate == NULL) {
250                 ret = -ENOTSUP;
251                 goto error;
252         }
253
254         /* Convert bus args to new syntax for use with new API dev_iterate. */
255         if (strcmp(iter->bus->name, "vdev") == 0) {
256                 bus_param_key = "name";
257         } else if (strcmp(iter->bus->name, "pci") == 0) {
258                 bus_param_key = "addr";
259         } else {
260                 ret = -ENOTSUP;
261                 goto error;
262         }
263         str_size = strlen(bus_param_key) + strlen(devargs.name) + 2;
264         bus_str = malloc(str_size);
265         if (bus_str == NULL) {
266                 ret = -ENOMEM;
267                 goto error;
268         }
269         ret = snprintf(bus_str, str_size, "%s=%s",
270                         bus_param_key, devargs.name);
271         if (ret != str_size - 1) {
272                 ret = -EINVAL;
273                 goto error;
274         }
275         iter->bus_str = bus_str;
276
277 end:
278         iter->cls = rte_class_find_by_name("eth");
279         return 0;
280
281 error:
282         if (ret == -ENOTSUP)
283                 RTE_LOG(ERR, EAL, "Bus %s does not support iterating.\n",
284                                 iter->bus->name);
285         free(devargs.args);
286         free(bus_str);
287         free(cls_str);
288         return ret;
289 }
290
291 uint16_t __rte_experimental
292 rte_eth_iterator_next(struct rte_dev_iterator *iter)
293 {
294         if (iter->cls == NULL) /* invalid ethdev iterator */
295                 return RTE_MAX_ETHPORTS;
296
297         do { /* loop to try all matching rte_device */
298                 /* If not pure ethdev filter and */
299                 if (iter->bus != NULL &&
300                                 /* not in middle of rte_eth_dev iteration, */
301                                 iter->class_device == NULL) {
302                         /* get next rte_device to try. */
303                         iter->device = iter->bus->dev_iterate(
304                                         iter->device, iter->bus_str, iter);
305                         if (iter->device == NULL)
306                                 break; /* no more rte_device candidate */
307                 }
308                 /* A device is matching bus part, need to check ethdev part. */
309                 iter->class_device = iter->cls->dev_iterate(
310                                 iter->class_device, iter->cls_str, iter);
311                 if (iter->class_device != NULL)
312                         return eth_dev_to_id(iter->class_device); /* match */
313         } while (iter->bus != NULL); /* need to try next rte_device */
314
315         /* No more ethdev port to iterate. */
316         rte_eth_iterator_cleanup(iter);
317         return RTE_MAX_ETHPORTS;
318 }
319
320 void __rte_experimental
321 rte_eth_iterator_cleanup(struct rte_dev_iterator *iter)
322 {
323         if (iter->bus_str == NULL)
324                 return; /* nothing to free in pure class filter */
325         free(RTE_CAST_FIELD(iter, bus_str, char *)); /* workaround const */
326         free(RTE_CAST_FIELD(iter, cls_str, char *)); /* workaround const */
327         memset(iter, 0, sizeof(*iter));
328 }
329
330 uint16_t
331 rte_eth_find_next(uint16_t port_id)
332 {
333         while (port_id < RTE_MAX_ETHPORTS &&
334                rte_eth_devices[port_id].state != RTE_ETH_DEV_ATTACHED &&
335                rte_eth_devices[port_id].state != RTE_ETH_DEV_REMOVED)
336                 port_id++;
337
338         if (port_id >= RTE_MAX_ETHPORTS)
339                 return RTE_MAX_ETHPORTS;
340
341         return port_id;
342 }
343
344 static void
345 rte_eth_dev_shared_data_prepare(void)
346 {
347         const unsigned flags = 0;
348         const struct rte_memzone *mz;
349
350         rte_spinlock_lock(&rte_eth_shared_data_lock);
351
352         if (rte_eth_dev_shared_data == NULL) {
353                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
354                         /* Allocate port data and ownership shared memory. */
355                         mz = rte_memzone_reserve(MZ_RTE_ETH_DEV_DATA,
356                                         sizeof(*rte_eth_dev_shared_data),
357                                         rte_socket_id(), flags);
358                 } else
359                         mz = rte_memzone_lookup(MZ_RTE_ETH_DEV_DATA);
360                 if (mz == NULL)
361                         rte_panic("Cannot allocate ethdev shared data\n");
362
363                 rte_eth_dev_shared_data = mz->addr;
364                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
365                         rte_eth_dev_shared_data->next_owner_id =
366                                         RTE_ETH_DEV_NO_OWNER + 1;
367                         rte_spinlock_init(&rte_eth_dev_shared_data->ownership_lock);
368                         memset(rte_eth_dev_shared_data->data, 0,
369                                sizeof(rte_eth_dev_shared_data->data));
370                 }
371         }
372
373         rte_spinlock_unlock(&rte_eth_shared_data_lock);
374 }
375
376 static bool
377 is_allocated(const struct rte_eth_dev *ethdev)
378 {
379         return ethdev->data->name[0] != '\0';
380 }
381
382 static struct rte_eth_dev *
383 _rte_eth_dev_allocated(const char *name)
384 {
385         unsigned i;
386
387         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
388                 if (rte_eth_devices[i].data != NULL &&
389                     strcmp(rte_eth_devices[i].data->name, name) == 0)
390                         return &rte_eth_devices[i];
391         }
392         return NULL;
393 }
394
395 struct rte_eth_dev *
396 rte_eth_dev_allocated(const char *name)
397 {
398         struct rte_eth_dev *ethdev;
399
400         rte_eth_dev_shared_data_prepare();
401
402         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
403
404         ethdev = _rte_eth_dev_allocated(name);
405
406         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
407
408         return ethdev;
409 }
410
411 static uint16_t
412 rte_eth_dev_find_free_port(void)
413 {
414         unsigned i;
415
416         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
417                 /* Using shared name field to find a free port. */
418                 if (rte_eth_dev_shared_data->data[i].name[0] == '\0') {
419                         RTE_ASSERT(rte_eth_devices[i].state ==
420                                    RTE_ETH_DEV_UNUSED);
421                         return i;
422                 }
423         }
424         return RTE_MAX_ETHPORTS;
425 }
426
427 static struct rte_eth_dev *
428 eth_dev_get(uint16_t port_id)
429 {
430         struct rte_eth_dev *eth_dev = &rte_eth_devices[port_id];
431
432         eth_dev->data = &rte_eth_dev_shared_data->data[port_id];
433
434         eth_dev_last_created_port = port_id;
435
436         return eth_dev;
437 }
438
439 struct rte_eth_dev *
440 rte_eth_dev_allocate(const char *name)
441 {
442         uint16_t port_id;
443         struct rte_eth_dev *eth_dev = NULL;
444
445         rte_eth_dev_shared_data_prepare();
446
447         /* Synchronize port creation between primary and secondary threads. */
448         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
449
450         if (_rte_eth_dev_allocated(name) != NULL) {
451                 RTE_ETHDEV_LOG(ERR,
452                         "Ethernet device with name %s already allocated\n",
453                         name);
454                 goto unlock;
455         }
456
457         port_id = rte_eth_dev_find_free_port();
458         if (port_id == RTE_MAX_ETHPORTS) {
459                 RTE_ETHDEV_LOG(ERR,
460                         "Reached maximum number of Ethernet ports\n");
461                 goto unlock;
462         }
463
464         eth_dev = eth_dev_get(port_id);
465         snprintf(eth_dev->data->name, sizeof(eth_dev->data->name), "%s", name);
466         eth_dev->data->port_id = port_id;
467         eth_dev->data->mtu = ETHER_MTU;
468
469 unlock:
470         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
471
472         return eth_dev;
473 }
474
475 /*
476  * Attach to a port already registered by the primary process, which
477  * makes sure that the same device would have the same port id both
478  * in the primary and secondary process.
479  */
480 struct rte_eth_dev *
481 rte_eth_dev_attach_secondary(const char *name)
482 {
483         uint16_t i;
484         struct rte_eth_dev *eth_dev = NULL;
485
486         rte_eth_dev_shared_data_prepare();
487
488         /* Synchronize port attachment to primary port creation and release. */
489         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
490
491         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
492                 if (strcmp(rte_eth_dev_shared_data->data[i].name, name) == 0)
493                         break;
494         }
495         if (i == RTE_MAX_ETHPORTS) {
496                 RTE_ETHDEV_LOG(ERR,
497                         "Device %s is not driven by the primary process\n",
498                         name);
499         } else {
500                 eth_dev = eth_dev_get(i);
501                 RTE_ASSERT(eth_dev->data->port_id == i);
502         }
503
504         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
505         return eth_dev;
506 }
507
508 int
509 rte_eth_dev_release_port(struct rte_eth_dev *eth_dev)
510 {
511         if (eth_dev == NULL)
512                 return -EINVAL;
513
514         rte_eth_dev_shared_data_prepare();
515
516         if (eth_dev->state != RTE_ETH_DEV_UNUSED)
517                 _rte_eth_dev_callback_process(eth_dev,
518                                 RTE_ETH_EVENT_DESTROY, NULL);
519
520         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
521
522         eth_dev->state = RTE_ETH_DEV_UNUSED;
523
524         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
525                 rte_free(eth_dev->data->rx_queues);
526                 rte_free(eth_dev->data->tx_queues);
527                 rte_free(eth_dev->data->mac_addrs);
528                 rte_free(eth_dev->data->hash_mac_addrs);
529                 rte_free(eth_dev->data->dev_private);
530                 memset(eth_dev->data, 0, sizeof(struct rte_eth_dev_data));
531         }
532
533         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
534
535         return 0;
536 }
537
538 int
539 rte_eth_dev_is_valid_port(uint16_t port_id)
540 {
541         if (port_id >= RTE_MAX_ETHPORTS ||
542             (rte_eth_devices[port_id].state == RTE_ETH_DEV_UNUSED))
543                 return 0;
544         else
545                 return 1;
546 }
547
548 static int
549 rte_eth_is_valid_owner_id(uint64_t owner_id)
550 {
551         if (owner_id == RTE_ETH_DEV_NO_OWNER ||
552             rte_eth_dev_shared_data->next_owner_id <= owner_id)
553                 return 0;
554         return 1;
555 }
556
557 uint64_t
558 rte_eth_find_next_owned_by(uint16_t port_id, const uint64_t owner_id)
559 {
560         while (port_id < RTE_MAX_ETHPORTS &&
561                ((rte_eth_devices[port_id].state != RTE_ETH_DEV_ATTACHED &&
562                rte_eth_devices[port_id].state != RTE_ETH_DEV_REMOVED) ||
563                rte_eth_devices[port_id].data->owner.id != owner_id))
564                 port_id++;
565
566         if (port_id >= RTE_MAX_ETHPORTS)
567                 return RTE_MAX_ETHPORTS;
568
569         return port_id;
570 }
571
572 int __rte_experimental
573 rte_eth_dev_owner_new(uint64_t *owner_id)
574 {
575         rte_eth_dev_shared_data_prepare();
576
577         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
578
579         *owner_id = rte_eth_dev_shared_data->next_owner_id++;
580
581         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
582         return 0;
583 }
584
585 static int
586 _rte_eth_dev_owner_set(const uint16_t port_id, const uint64_t old_owner_id,
587                        const struct rte_eth_dev_owner *new_owner)
588 {
589         struct rte_eth_dev *ethdev = &rte_eth_devices[port_id];
590         struct rte_eth_dev_owner *port_owner;
591         int sret;
592
593         if (port_id >= RTE_MAX_ETHPORTS || !is_allocated(ethdev)) {
594                 RTE_ETHDEV_LOG(ERR, "Port id %"PRIu16" is not allocated\n",
595                         port_id);
596                 return -ENODEV;
597         }
598
599         if (!rte_eth_is_valid_owner_id(new_owner->id) &&
600             !rte_eth_is_valid_owner_id(old_owner_id)) {
601                 RTE_ETHDEV_LOG(ERR,
602                         "Invalid owner old_id=%016"PRIx64" new_id=%016"PRIx64"\n",
603                        old_owner_id, new_owner->id);
604                 return -EINVAL;
605         }
606
607         port_owner = &rte_eth_devices[port_id].data->owner;
608         if (port_owner->id != old_owner_id) {
609                 RTE_ETHDEV_LOG(ERR,
610                         "Cannot set owner to port %u already owned by %s_%016"PRIX64"\n",
611                         port_id, port_owner->name, port_owner->id);
612                 return -EPERM;
613         }
614
615         sret = snprintf(port_owner->name, RTE_ETH_MAX_OWNER_NAME_LEN, "%s",
616                         new_owner->name);
617         if (sret < 0 || sret >= RTE_ETH_MAX_OWNER_NAME_LEN)
618                 RTE_ETHDEV_LOG(ERR, "Port %u owner name was truncated\n",
619                         port_id);
620
621         port_owner->id = new_owner->id;
622
623         RTE_ETHDEV_LOG(DEBUG, "Port %u owner is %s_%016"PRIx64"\n",
624                 port_id, new_owner->name, new_owner->id);
625
626         return 0;
627 }
628
629 int __rte_experimental
630 rte_eth_dev_owner_set(const uint16_t port_id,
631                       const struct rte_eth_dev_owner *owner)
632 {
633         int ret;
634
635         rte_eth_dev_shared_data_prepare();
636
637         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
638
639         ret = _rte_eth_dev_owner_set(port_id, RTE_ETH_DEV_NO_OWNER, owner);
640
641         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
642         return ret;
643 }
644
645 int __rte_experimental
646 rte_eth_dev_owner_unset(const uint16_t port_id, const uint64_t owner_id)
647 {
648         const struct rte_eth_dev_owner new_owner = (struct rte_eth_dev_owner)
649                         {.id = RTE_ETH_DEV_NO_OWNER, .name = ""};
650         int ret;
651
652         rte_eth_dev_shared_data_prepare();
653
654         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
655
656         ret = _rte_eth_dev_owner_set(port_id, owner_id, &new_owner);
657
658         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
659         return ret;
660 }
661
662 void __rte_experimental
663 rte_eth_dev_owner_delete(const uint64_t owner_id)
664 {
665         uint16_t port_id;
666
667         rte_eth_dev_shared_data_prepare();
668
669         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
670
671         if (rte_eth_is_valid_owner_id(owner_id)) {
672                 for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++)
673                         if (rte_eth_devices[port_id].data->owner.id == owner_id)
674                                 memset(&rte_eth_devices[port_id].data->owner, 0,
675                                        sizeof(struct rte_eth_dev_owner));
676                 RTE_ETHDEV_LOG(NOTICE,
677                         "All port owners owned by %016"PRIx64" identifier have removed\n",
678                         owner_id);
679         } else {
680                 RTE_ETHDEV_LOG(ERR,
681                                "Invalid owner id=%016"PRIx64"\n",
682                                owner_id);
683         }
684
685         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
686 }
687
688 int __rte_experimental
689 rte_eth_dev_owner_get(const uint16_t port_id, struct rte_eth_dev_owner *owner)
690 {
691         int ret = 0;
692         struct rte_eth_dev *ethdev = &rte_eth_devices[port_id];
693
694         rte_eth_dev_shared_data_prepare();
695
696         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
697
698         if (port_id >= RTE_MAX_ETHPORTS || !is_allocated(ethdev)) {
699                 RTE_ETHDEV_LOG(ERR, "Port id %"PRIu16" is not allocated\n",
700                         port_id);
701                 ret = -ENODEV;
702         } else {
703                 rte_memcpy(owner, &ethdev->data->owner, sizeof(*owner));
704         }
705
706         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
707         return ret;
708 }
709
710 int
711 rte_eth_dev_socket_id(uint16_t port_id)
712 {
713         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -1);
714         return rte_eth_devices[port_id].data->numa_node;
715 }
716
717 void *
718 rte_eth_dev_get_sec_ctx(uint16_t port_id)
719 {
720         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, NULL);
721         return rte_eth_devices[port_id].security_ctx;
722 }
723
724 uint16_t
725 rte_eth_dev_count(void)
726 {
727         return rte_eth_dev_count_avail();
728 }
729
730 uint16_t
731 rte_eth_dev_count_avail(void)
732 {
733         uint16_t p;
734         uint16_t count;
735
736         count = 0;
737
738         RTE_ETH_FOREACH_DEV(p)
739                 count++;
740
741         return count;
742 }
743
744 uint16_t __rte_experimental
745 rte_eth_dev_count_total(void)
746 {
747         uint16_t port, count = 0;
748
749         for (port = 0; port < RTE_MAX_ETHPORTS; port++)
750                 if (rte_eth_devices[port].state != RTE_ETH_DEV_UNUSED)
751                         count++;
752
753         return count;
754 }
755
756 int
757 rte_eth_dev_get_name_by_port(uint16_t port_id, char *name)
758 {
759         char *tmp;
760
761         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
762
763         if (name == NULL) {
764                 RTE_ETHDEV_LOG(ERR, "Null pointer is specified\n");
765                 return -EINVAL;
766         }
767
768         /* shouldn't check 'rte_eth_devices[i].data',
769          * because it might be overwritten by VDEV PMD */
770         tmp = rte_eth_dev_shared_data->data[port_id].name;
771         strcpy(name, tmp);
772         return 0;
773 }
774
775 int
776 rte_eth_dev_get_port_by_name(const char *name, uint16_t *port_id)
777 {
778         uint32_t pid;
779
780         if (name == NULL) {
781                 RTE_ETHDEV_LOG(ERR, "Null pointer is specified\n");
782                 return -EINVAL;
783         }
784
785         for (pid = 0; pid < RTE_MAX_ETHPORTS; pid++) {
786                 if (rte_eth_devices[pid].state != RTE_ETH_DEV_UNUSED &&
787                     !strcmp(name, rte_eth_dev_shared_data->data[pid].name)) {
788                         *port_id = pid;
789                         return 0;
790                 }
791         }
792
793         return -ENODEV;
794 }
795
796 static int
797 eth_err(uint16_t port_id, int ret)
798 {
799         if (ret == 0)
800                 return 0;
801         if (rte_eth_dev_is_removed(port_id))
802                 return -EIO;
803         return ret;
804 }
805
806 static int
807 rte_eth_dev_rx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
808 {
809         uint16_t old_nb_queues = dev->data->nb_rx_queues;
810         void **rxq;
811         unsigned i;
812
813         if (dev->data->rx_queues == NULL && nb_queues != 0) { /* first time configuration */
814                 dev->data->rx_queues = rte_zmalloc("ethdev->rx_queues",
815                                 sizeof(dev->data->rx_queues[0]) * nb_queues,
816                                 RTE_CACHE_LINE_SIZE);
817                 if (dev->data->rx_queues == NULL) {
818                         dev->data->nb_rx_queues = 0;
819                         return -(ENOMEM);
820                 }
821         } else if (dev->data->rx_queues != NULL && nb_queues != 0) { /* re-configure */
822                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release, -ENOTSUP);
823
824                 rxq = dev->data->rx_queues;
825
826                 for (i = nb_queues; i < old_nb_queues; i++)
827                         (*dev->dev_ops->rx_queue_release)(rxq[i]);
828                 rxq = rte_realloc(rxq, sizeof(rxq[0]) * nb_queues,
829                                 RTE_CACHE_LINE_SIZE);
830                 if (rxq == NULL)
831                         return -(ENOMEM);
832                 if (nb_queues > old_nb_queues) {
833                         uint16_t new_qs = nb_queues - old_nb_queues;
834
835                         memset(rxq + old_nb_queues, 0,
836                                 sizeof(rxq[0]) * new_qs);
837                 }
838
839                 dev->data->rx_queues = rxq;
840
841         } else if (dev->data->rx_queues != NULL && nb_queues == 0) {
842                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release, -ENOTSUP);
843
844                 rxq = dev->data->rx_queues;
845
846                 for (i = nb_queues; i < old_nb_queues; i++)
847                         (*dev->dev_ops->rx_queue_release)(rxq[i]);
848
849                 rte_free(dev->data->rx_queues);
850                 dev->data->rx_queues = NULL;
851         }
852         dev->data->nb_rx_queues = nb_queues;
853         return 0;
854 }
855
856 int
857 rte_eth_dev_rx_queue_start(uint16_t port_id, uint16_t rx_queue_id)
858 {
859         struct rte_eth_dev *dev;
860
861         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
862
863         dev = &rte_eth_devices[port_id];
864         if (!dev->data->dev_started) {
865                 RTE_ETHDEV_LOG(ERR,
866                         "Port %u must be started before start any queue\n",
867                         port_id);
868                 return -EINVAL;
869         }
870
871         if (rx_queue_id >= dev->data->nb_rx_queues) {
872                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
873                 return -EINVAL;
874         }
875
876         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_start, -ENOTSUP);
877
878         if (dev->data->rx_queue_state[rx_queue_id] != RTE_ETH_QUEUE_STATE_STOPPED) {
879                 RTE_ETHDEV_LOG(INFO,
880                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already started\n",
881                         rx_queue_id, port_id);
882                 return 0;
883         }
884
885         return eth_err(port_id, dev->dev_ops->rx_queue_start(dev,
886                                                              rx_queue_id));
887
888 }
889
890 int
891 rte_eth_dev_rx_queue_stop(uint16_t port_id, uint16_t rx_queue_id)
892 {
893         struct rte_eth_dev *dev;
894
895         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
896
897         dev = &rte_eth_devices[port_id];
898         if (rx_queue_id >= dev->data->nb_rx_queues) {
899                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
900                 return -EINVAL;
901         }
902
903         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_stop, -ENOTSUP);
904
905         if (dev->data->rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STOPPED) {
906                 RTE_ETHDEV_LOG(INFO,
907                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already stopped\n",
908                         rx_queue_id, port_id);
909                 return 0;
910         }
911
912         return eth_err(port_id, dev->dev_ops->rx_queue_stop(dev, rx_queue_id));
913
914 }
915
916 int
917 rte_eth_dev_tx_queue_start(uint16_t port_id, uint16_t tx_queue_id)
918 {
919         struct rte_eth_dev *dev;
920
921         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
922
923         dev = &rte_eth_devices[port_id];
924         if (!dev->data->dev_started) {
925                 RTE_ETHDEV_LOG(ERR,
926                         "Port %u must be started before start any queue\n",
927                         port_id);
928                 return -EINVAL;
929         }
930
931         if (tx_queue_id >= dev->data->nb_tx_queues) {
932                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
933                 return -EINVAL;
934         }
935
936         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_start, -ENOTSUP);
937
938         if (dev->data->tx_queue_state[tx_queue_id] != RTE_ETH_QUEUE_STATE_STOPPED) {
939                 RTE_ETHDEV_LOG(INFO,
940                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already started\n",
941                         tx_queue_id, port_id);
942                 return 0;
943         }
944
945         return eth_err(port_id, dev->dev_ops->tx_queue_start(dev, tx_queue_id));
946 }
947
948 int
949 rte_eth_dev_tx_queue_stop(uint16_t port_id, uint16_t tx_queue_id)
950 {
951         struct rte_eth_dev *dev;
952
953         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
954
955         dev = &rte_eth_devices[port_id];
956         if (tx_queue_id >= dev->data->nb_tx_queues) {
957                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
958                 return -EINVAL;
959         }
960
961         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_stop, -ENOTSUP);
962
963         if (dev->data->tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STOPPED) {
964                 RTE_ETHDEV_LOG(INFO,
965                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already stopped\n",
966                         tx_queue_id, port_id);
967                 return 0;
968         }
969
970         return eth_err(port_id, dev->dev_ops->tx_queue_stop(dev, tx_queue_id));
971
972 }
973
974 static int
975 rte_eth_dev_tx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
976 {
977         uint16_t old_nb_queues = dev->data->nb_tx_queues;
978         void **txq;
979         unsigned i;
980
981         if (dev->data->tx_queues == NULL && nb_queues != 0) { /* first time configuration */
982                 dev->data->tx_queues = rte_zmalloc("ethdev->tx_queues",
983                                                    sizeof(dev->data->tx_queues[0]) * nb_queues,
984                                                    RTE_CACHE_LINE_SIZE);
985                 if (dev->data->tx_queues == NULL) {
986                         dev->data->nb_tx_queues = 0;
987                         return -(ENOMEM);
988                 }
989         } else if (dev->data->tx_queues != NULL && nb_queues != 0) { /* re-configure */
990                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release, -ENOTSUP);
991
992                 txq = dev->data->tx_queues;
993
994                 for (i = nb_queues; i < old_nb_queues; i++)
995                         (*dev->dev_ops->tx_queue_release)(txq[i]);
996                 txq = rte_realloc(txq, sizeof(txq[0]) * nb_queues,
997                                   RTE_CACHE_LINE_SIZE);
998                 if (txq == NULL)
999                         return -ENOMEM;
1000                 if (nb_queues > old_nb_queues) {
1001                         uint16_t new_qs = nb_queues - old_nb_queues;
1002
1003                         memset(txq + old_nb_queues, 0,
1004                                sizeof(txq[0]) * new_qs);
1005                 }
1006
1007                 dev->data->tx_queues = txq;
1008
1009         } else if (dev->data->tx_queues != NULL && nb_queues == 0) {
1010                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release, -ENOTSUP);
1011
1012                 txq = dev->data->tx_queues;
1013
1014                 for (i = nb_queues; i < old_nb_queues; i++)
1015                         (*dev->dev_ops->tx_queue_release)(txq[i]);
1016
1017                 rte_free(dev->data->tx_queues);
1018                 dev->data->tx_queues = NULL;
1019         }
1020         dev->data->nb_tx_queues = nb_queues;
1021         return 0;
1022 }
1023
1024 uint32_t
1025 rte_eth_speed_bitflag(uint32_t speed, int duplex)
1026 {
1027         switch (speed) {
1028         case ETH_SPEED_NUM_10M:
1029                 return duplex ? ETH_LINK_SPEED_10M : ETH_LINK_SPEED_10M_HD;
1030         case ETH_SPEED_NUM_100M:
1031                 return duplex ? ETH_LINK_SPEED_100M : ETH_LINK_SPEED_100M_HD;
1032         case ETH_SPEED_NUM_1G:
1033                 return ETH_LINK_SPEED_1G;
1034         case ETH_SPEED_NUM_2_5G:
1035                 return ETH_LINK_SPEED_2_5G;
1036         case ETH_SPEED_NUM_5G:
1037                 return ETH_LINK_SPEED_5G;
1038         case ETH_SPEED_NUM_10G:
1039                 return ETH_LINK_SPEED_10G;
1040         case ETH_SPEED_NUM_20G:
1041                 return ETH_LINK_SPEED_20G;
1042         case ETH_SPEED_NUM_25G:
1043                 return ETH_LINK_SPEED_25G;
1044         case ETH_SPEED_NUM_40G:
1045                 return ETH_LINK_SPEED_40G;
1046         case ETH_SPEED_NUM_50G:
1047                 return ETH_LINK_SPEED_50G;
1048         case ETH_SPEED_NUM_56G:
1049                 return ETH_LINK_SPEED_56G;
1050         case ETH_SPEED_NUM_100G:
1051                 return ETH_LINK_SPEED_100G;
1052         default:
1053                 return 0;
1054         }
1055 }
1056
1057 const char *
1058 rte_eth_dev_rx_offload_name(uint64_t offload)
1059 {
1060         const char *name = "UNKNOWN";
1061         unsigned int i;
1062
1063         for (i = 0; i < RTE_DIM(rte_rx_offload_names); ++i) {
1064                 if (offload == rte_rx_offload_names[i].offload) {
1065                         name = rte_rx_offload_names[i].name;
1066                         break;
1067                 }
1068         }
1069
1070         return name;
1071 }
1072
1073 const char *
1074 rte_eth_dev_tx_offload_name(uint64_t offload)
1075 {
1076         const char *name = "UNKNOWN";
1077         unsigned int i;
1078
1079         for (i = 0; i < RTE_DIM(rte_tx_offload_names); ++i) {
1080                 if (offload == rte_tx_offload_names[i].offload) {
1081                         name = rte_tx_offload_names[i].name;
1082                         break;
1083                 }
1084         }
1085
1086         return name;
1087 }
1088
1089 int
1090 rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
1091                       const struct rte_eth_conf *dev_conf)
1092 {
1093         struct rte_eth_dev *dev;
1094         struct rte_eth_dev_info dev_info;
1095         struct rte_eth_conf local_conf = *dev_conf;
1096         int diag;
1097
1098         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1099
1100         dev = &rte_eth_devices[port_id];
1101
1102         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
1103         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
1104
1105         rte_eth_dev_info_get(port_id, &dev_info);
1106
1107         /* If number of queues specified by application for both Rx and Tx is
1108          * zero, use driver preferred values. This cannot be done individually
1109          * as it is valid for either Tx or Rx (but not both) to be zero.
1110          * If driver does not provide any preferred valued, fall back on
1111          * EAL defaults.
1112          */
1113         if (nb_rx_q == 0 && nb_tx_q == 0) {
1114                 nb_rx_q = dev_info.default_rxportconf.nb_queues;
1115                 if (nb_rx_q == 0)
1116                         nb_rx_q = RTE_ETH_DEV_FALLBACK_RX_NBQUEUES;
1117                 nb_tx_q = dev_info.default_txportconf.nb_queues;
1118                 if (nb_tx_q == 0)
1119                         nb_tx_q = RTE_ETH_DEV_FALLBACK_TX_NBQUEUES;
1120         }
1121
1122         if (nb_rx_q > RTE_MAX_QUEUES_PER_PORT) {
1123                 RTE_ETHDEV_LOG(ERR,
1124                         "Number of RX queues requested (%u) is greater than max supported(%d)\n",
1125                         nb_rx_q, RTE_MAX_QUEUES_PER_PORT);
1126                 return -EINVAL;
1127         }
1128
1129         if (nb_tx_q > RTE_MAX_QUEUES_PER_PORT) {
1130                 RTE_ETHDEV_LOG(ERR,
1131                         "Number of TX queues requested (%u) is greater than max supported(%d)\n",
1132                         nb_tx_q, RTE_MAX_QUEUES_PER_PORT);
1133                 return -EINVAL;
1134         }
1135
1136         if (dev->data->dev_started) {
1137                 RTE_ETHDEV_LOG(ERR,
1138                         "Port %u must be stopped to allow configuration\n",
1139                         port_id);
1140                 return -EBUSY;
1141         }
1142
1143         /* Copy the dev_conf parameter into the dev structure */
1144         memcpy(&dev->data->dev_conf, &local_conf, sizeof(dev->data->dev_conf));
1145
1146         /*
1147          * Check that the numbers of RX and TX queues are not greater
1148          * than the maximum number of RX and TX queues supported by the
1149          * configured device.
1150          */
1151         if (nb_rx_q > dev_info.max_rx_queues) {
1152                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u nb_rx_queues=%u > %u\n",
1153                         port_id, nb_rx_q, dev_info.max_rx_queues);
1154                 return -EINVAL;
1155         }
1156
1157         if (nb_tx_q > dev_info.max_tx_queues) {
1158                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u nb_tx_queues=%u > %u\n",
1159                         port_id, nb_tx_q, dev_info.max_tx_queues);
1160                 return -EINVAL;
1161         }
1162
1163         /* Check that the device supports requested interrupts */
1164         if ((dev_conf->intr_conf.lsc == 1) &&
1165                         (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC))) {
1166                 RTE_ETHDEV_LOG(ERR, "Driver %s does not support lsc\n",
1167                         dev->device->driver->name);
1168                 return -EINVAL;
1169         }
1170         if ((dev_conf->intr_conf.rmv == 1) &&
1171                         (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_RMV))) {
1172                 RTE_ETHDEV_LOG(ERR, "Driver %s does not support rmv\n",
1173                         dev->device->driver->name);
1174                 return -EINVAL;
1175         }
1176
1177         /*
1178          * If jumbo frames are enabled, check that the maximum RX packet
1179          * length is supported by the configured device.
1180          */
1181         if (local_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
1182                 if (dev_conf->rxmode.max_rx_pkt_len > dev_info.max_rx_pktlen) {
1183                         RTE_ETHDEV_LOG(ERR,
1184                                 "Ethdev port_id=%u max_rx_pkt_len %u > max valid value %u\n",
1185                                 port_id, dev_conf->rxmode.max_rx_pkt_len,
1186                                 dev_info.max_rx_pktlen);
1187                         return -EINVAL;
1188                 } else if (dev_conf->rxmode.max_rx_pkt_len < ETHER_MIN_LEN) {
1189                         RTE_ETHDEV_LOG(ERR,
1190                                 "Ethdev port_id=%u max_rx_pkt_len %u < min valid value %u\n",
1191                                 port_id, dev_conf->rxmode.max_rx_pkt_len,
1192                                 (unsigned)ETHER_MIN_LEN);
1193                         return -EINVAL;
1194                 }
1195         } else {
1196                 if (dev_conf->rxmode.max_rx_pkt_len < ETHER_MIN_LEN ||
1197                         dev_conf->rxmode.max_rx_pkt_len > ETHER_MAX_LEN)
1198                         /* Use default value */
1199                         dev->data->dev_conf.rxmode.max_rx_pkt_len =
1200                                                         ETHER_MAX_LEN;
1201         }
1202
1203         /* Any requested offloading must be within its device capabilities */
1204         if ((local_conf.rxmode.offloads & dev_info.rx_offload_capa) !=
1205              local_conf.rxmode.offloads) {
1206                 RTE_ETHDEV_LOG(ERR,
1207                         "Ethdev port_id=%u requested Rx offloads 0x%"PRIx64" doesn't match Rx offloads "
1208                         "capabilities 0x%"PRIx64" in %s()\n",
1209                         port_id, local_conf.rxmode.offloads,
1210                         dev_info.rx_offload_capa,
1211                         __func__);
1212                 return -EINVAL;
1213         }
1214         if ((local_conf.txmode.offloads & dev_info.tx_offload_capa) !=
1215              local_conf.txmode.offloads) {
1216                 RTE_ETHDEV_LOG(ERR,
1217                         "Ethdev port_id=%u requested Tx offloads 0x%"PRIx64" doesn't match Tx offloads "
1218                         "capabilities 0x%"PRIx64" in %s()\n",
1219                         port_id, local_conf.txmode.offloads,
1220                         dev_info.tx_offload_capa,
1221                         __func__);
1222                 return -EINVAL;
1223         }
1224
1225         /* Check that device supports requested rss hash functions. */
1226         if ((dev_info.flow_type_rss_offloads |
1227              dev_conf->rx_adv_conf.rss_conf.rss_hf) !=
1228             dev_info.flow_type_rss_offloads) {
1229                 RTE_ETHDEV_LOG(ERR,
1230                         "Ethdev port_id=%u invalid rss_hf: 0x%"PRIx64", valid value: 0x%"PRIx64"\n",
1231                         port_id, dev_conf->rx_adv_conf.rss_conf.rss_hf,
1232                         dev_info.flow_type_rss_offloads);
1233                 return -EINVAL;
1234         }
1235
1236         /*
1237          * Setup new number of RX/TX queues and reconfigure device.
1238          */
1239         diag = rte_eth_dev_rx_queue_config(dev, nb_rx_q);
1240         if (diag != 0) {
1241                 RTE_ETHDEV_LOG(ERR,
1242                         "Port%u rte_eth_dev_rx_queue_config = %d\n",
1243                         port_id, diag);
1244                 return diag;
1245         }
1246
1247         diag = rte_eth_dev_tx_queue_config(dev, nb_tx_q);
1248         if (diag != 0) {
1249                 RTE_ETHDEV_LOG(ERR,
1250                         "Port%u rte_eth_dev_tx_queue_config = %d\n",
1251                         port_id, diag);
1252                 rte_eth_dev_rx_queue_config(dev, 0);
1253                 return diag;
1254         }
1255
1256         diag = (*dev->dev_ops->dev_configure)(dev);
1257         if (diag != 0) {
1258                 RTE_ETHDEV_LOG(ERR, "Port%u dev_configure = %d\n",
1259                         port_id, diag);
1260                 rte_eth_dev_rx_queue_config(dev, 0);
1261                 rte_eth_dev_tx_queue_config(dev, 0);
1262                 return eth_err(port_id, diag);
1263         }
1264
1265         /* Initialize Rx profiling if enabled at compilation time. */
1266         diag = __rte_eth_dev_profile_init(port_id, dev);
1267         if (diag != 0) {
1268                 RTE_ETHDEV_LOG(ERR, "Port%u __rte_eth_dev_profile_init = %d\n",
1269                         port_id, diag);
1270                 rte_eth_dev_rx_queue_config(dev, 0);
1271                 rte_eth_dev_tx_queue_config(dev, 0);
1272                 return eth_err(port_id, diag);
1273         }
1274
1275         return 0;
1276 }
1277
1278 void
1279 _rte_eth_dev_reset(struct rte_eth_dev *dev)
1280 {
1281         if (dev->data->dev_started) {
1282                 RTE_ETHDEV_LOG(ERR, "Port %u must be stopped to allow reset\n",
1283                         dev->data->port_id);
1284                 return;
1285         }
1286
1287         rte_eth_dev_rx_queue_config(dev, 0);
1288         rte_eth_dev_tx_queue_config(dev, 0);
1289
1290         memset(&dev->data->dev_conf, 0, sizeof(dev->data->dev_conf));
1291 }
1292
1293 static void
1294 rte_eth_dev_mac_restore(struct rte_eth_dev *dev,
1295                         struct rte_eth_dev_info *dev_info)
1296 {
1297         struct ether_addr *addr;
1298         uint16_t i;
1299         uint32_t pool = 0;
1300         uint64_t pool_mask;
1301
1302         /* replay MAC address configuration including default MAC */
1303         addr = &dev->data->mac_addrs[0];
1304         if (*dev->dev_ops->mac_addr_set != NULL)
1305                 (*dev->dev_ops->mac_addr_set)(dev, addr);
1306         else if (*dev->dev_ops->mac_addr_add != NULL)
1307                 (*dev->dev_ops->mac_addr_add)(dev, addr, 0, pool);
1308
1309         if (*dev->dev_ops->mac_addr_add != NULL) {
1310                 for (i = 1; i < dev_info->max_mac_addrs; i++) {
1311                         addr = &dev->data->mac_addrs[i];
1312
1313                         /* skip zero address */
1314                         if (is_zero_ether_addr(addr))
1315                                 continue;
1316
1317                         pool = 0;
1318                         pool_mask = dev->data->mac_pool_sel[i];
1319
1320                         do {
1321                                 if (pool_mask & 1ULL)
1322                                         (*dev->dev_ops->mac_addr_add)(dev,
1323                                                 addr, i, pool);
1324                                 pool_mask >>= 1;
1325                                 pool++;
1326                         } while (pool_mask);
1327                 }
1328         }
1329 }
1330
1331 static void
1332 rte_eth_dev_config_restore(struct rte_eth_dev *dev,
1333                            struct rte_eth_dev_info *dev_info, uint16_t port_id)
1334 {
1335         if (!(*dev_info->dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR))
1336                 rte_eth_dev_mac_restore(dev, dev_info);
1337
1338         /* replay promiscuous configuration */
1339         if (rte_eth_promiscuous_get(port_id) == 1)
1340                 rte_eth_promiscuous_enable(port_id);
1341         else if (rte_eth_promiscuous_get(port_id) == 0)
1342                 rte_eth_promiscuous_disable(port_id);
1343
1344         /* replay all multicast configuration */
1345         if (rte_eth_allmulticast_get(port_id) == 1)
1346                 rte_eth_allmulticast_enable(port_id);
1347         else if (rte_eth_allmulticast_get(port_id) == 0)
1348                 rte_eth_allmulticast_disable(port_id);
1349 }
1350
1351 int
1352 rte_eth_dev_start(uint16_t port_id)
1353 {
1354         struct rte_eth_dev *dev;
1355         struct rte_eth_dev_info dev_info;
1356         int diag;
1357
1358         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1359
1360         dev = &rte_eth_devices[port_id];
1361
1362         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_start, -ENOTSUP);
1363
1364         if (dev->data->dev_started != 0) {
1365                 RTE_ETHDEV_LOG(INFO,
1366                         "Device with port_id=%"PRIu16" already started\n",
1367                         port_id);
1368                 return 0;
1369         }
1370
1371         rte_eth_dev_info_get(port_id, &dev_info);
1372
1373         /* Lets restore MAC now if device does not support live change */
1374         if (*dev_info.dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR)
1375                 rte_eth_dev_mac_restore(dev, &dev_info);
1376
1377         diag = (*dev->dev_ops->dev_start)(dev);
1378         if (diag == 0)
1379                 dev->data->dev_started = 1;
1380         else
1381                 return eth_err(port_id, diag);
1382
1383         rte_eth_dev_config_restore(dev, &dev_info, port_id);
1384
1385         if (dev->data->dev_conf.intr_conf.lsc == 0) {
1386                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
1387                 (*dev->dev_ops->link_update)(dev, 0);
1388         }
1389         return 0;
1390 }
1391
1392 void
1393 rte_eth_dev_stop(uint16_t port_id)
1394 {
1395         struct rte_eth_dev *dev;
1396
1397         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1398         dev = &rte_eth_devices[port_id];
1399
1400         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_stop);
1401
1402         if (dev->data->dev_started == 0) {
1403                 RTE_ETHDEV_LOG(INFO,
1404                         "Device with port_id=%"PRIu16" already stopped\n",
1405                         port_id);
1406                 return;
1407         }
1408
1409         dev->data->dev_started = 0;
1410         (*dev->dev_ops->dev_stop)(dev);
1411 }
1412
1413 int
1414 rte_eth_dev_set_link_up(uint16_t port_id)
1415 {
1416         struct rte_eth_dev *dev;
1417
1418         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1419
1420         dev = &rte_eth_devices[port_id];
1421
1422         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_up, -ENOTSUP);
1423         return eth_err(port_id, (*dev->dev_ops->dev_set_link_up)(dev));
1424 }
1425
1426 int
1427 rte_eth_dev_set_link_down(uint16_t port_id)
1428 {
1429         struct rte_eth_dev *dev;
1430
1431         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1432
1433         dev = &rte_eth_devices[port_id];
1434
1435         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_down, -ENOTSUP);
1436         return eth_err(port_id, (*dev->dev_ops->dev_set_link_down)(dev));
1437 }
1438
1439 void
1440 rte_eth_dev_close(uint16_t port_id)
1441 {
1442         struct rte_eth_dev *dev;
1443
1444         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1445         dev = &rte_eth_devices[port_id];
1446
1447         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_close);
1448         dev->data->dev_started = 0;
1449         (*dev->dev_ops->dev_close)(dev);
1450
1451         /* check behaviour flag - temporary for PMD migration */
1452         if ((dev->data->dev_flags & RTE_ETH_DEV_CLOSE_REMOVE) != 0) {
1453                 /* new behaviour: send event + reset state + free all data */
1454                 rte_eth_dev_release_port(dev);
1455                 return;
1456         }
1457         RTE_ETHDEV_LOG(DEBUG, "Port closing is using an old behaviour.\n"
1458                         "The driver %s should migrate to the new behaviour.\n",
1459                         dev->device->driver->name);
1460         /* old behaviour: only free queue arrays */
1461         dev->data->nb_rx_queues = 0;
1462         rte_free(dev->data->rx_queues);
1463         dev->data->rx_queues = NULL;
1464         dev->data->nb_tx_queues = 0;
1465         rte_free(dev->data->tx_queues);
1466         dev->data->tx_queues = NULL;
1467 }
1468
1469 int
1470 rte_eth_dev_reset(uint16_t port_id)
1471 {
1472         struct rte_eth_dev *dev;
1473         int ret;
1474
1475         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1476         dev = &rte_eth_devices[port_id];
1477
1478         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_reset, -ENOTSUP);
1479
1480         rte_eth_dev_stop(port_id);
1481         ret = dev->dev_ops->dev_reset(dev);
1482
1483         return eth_err(port_id, ret);
1484 }
1485
1486 int __rte_experimental
1487 rte_eth_dev_is_removed(uint16_t port_id)
1488 {
1489         struct rte_eth_dev *dev;
1490         int ret;
1491
1492         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, 0);
1493
1494         dev = &rte_eth_devices[port_id];
1495
1496         if (dev->state == RTE_ETH_DEV_REMOVED)
1497                 return 1;
1498
1499         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->is_removed, 0);
1500
1501         ret = dev->dev_ops->is_removed(dev);
1502         if (ret != 0)
1503                 /* Device is physically removed. */
1504                 dev->state = RTE_ETH_DEV_REMOVED;
1505
1506         return ret;
1507 }
1508
1509 int
1510 rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
1511                        uint16_t nb_rx_desc, unsigned int socket_id,
1512                        const struct rte_eth_rxconf *rx_conf,
1513                        struct rte_mempool *mp)
1514 {
1515         int ret;
1516         uint32_t mbp_buf_size;
1517         struct rte_eth_dev *dev;
1518         struct rte_eth_dev_info dev_info;
1519         struct rte_eth_rxconf local_conf;
1520         void **rxq;
1521
1522         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1523
1524         dev = &rte_eth_devices[port_id];
1525         if (rx_queue_id >= dev->data->nb_rx_queues) {
1526                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
1527                 return -EINVAL;
1528         }
1529
1530         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
1531         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_setup, -ENOTSUP);
1532
1533         /*
1534          * Check the size of the mbuf data buffer.
1535          * This value must be provided in the private data of the memory pool.
1536          * First check that the memory pool has a valid private data.
1537          */
1538         rte_eth_dev_info_get(port_id, &dev_info);
1539         if (mp->private_data_size < sizeof(struct rte_pktmbuf_pool_private)) {
1540                 RTE_ETHDEV_LOG(ERR, "%s private_data_size %d < %d\n",
1541                         mp->name, (int)mp->private_data_size,
1542                         (int)sizeof(struct rte_pktmbuf_pool_private));
1543                 return -ENOSPC;
1544         }
1545         mbp_buf_size = rte_pktmbuf_data_room_size(mp);
1546
1547         if ((mbp_buf_size - RTE_PKTMBUF_HEADROOM) < dev_info.min_rx_bufsize) {
1548                 RTE_ETHDEV_LOG(ERR,
1549                         "%s mbuf_data_room_size %d < %d (RTE_PKTMBUF_HEADROOM=%d + min_rx_bufsize(dev)=%d)\n",
1550                         mp->name, (int)mbp_buf_size,
1551                         (int)(RTE_PKTMBUF_HEADROOM + dev_info.min_rx_bufsize),
1552                         (int)RTE_PKTMBUF_HEADROOM,
1553                         (int)dev_info.min_rx_bufsize);
1554                 return -EINVAL;
1555         }
1556
1557         /* Use default specified by driver, if nb_rx_desc is zero */
1558         if (nb_rx_desc == 0) {
1559                 nb_rx_desc = dev_info.default_rxportconf.ring_size;
1560                 /* If driver default is also zero, fall back on EAL default */
1561                 if (nb_rx_desc == 0)
1562                         nb_rx_desc = RTE_ETH_DEV_FALLBACK_RX_RINGSIZE;
1563         }
1564
1565         if (nb_rx_desc > dev_info.rx_desc_lim.nb_max ||
1566                         nb_rx_desc < dev_info.rx_desc_lim.nb_min ||
1567                         nb_rx_desc % dev_info.rx_desc_lim.nb_align != 0) {
1568
1569                 RTE_ETHDEV_LOG(ERR,
1570                         "Invalid value for nb_rx_desc(=%hu), should be: <= %hu, = %hu, and a product of %hu\n",
1571                         nb_rx_desc, dev_info.rx_desc_lim.nb_max,
1572                         dev_info.rx_desc_lim.nb_min,
1573                         dev_info.rx_desc_lim.nb_align);
1574                 return -EINVAL;
1575         }
1576
1577         if (dev->data->dev_started &&
1578                 !(dev_info.dev_capa &
1579                         RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP))
1580                 return -EBUSY;
1581
1582         if (dev->data->dev_started &&
1583                 (dev->data->rx_queue_state[rx_queue_id] !=
1584                         RTE_ETH_QUEUE_STATE_STOPPED))
1585                 return -EBUSY;
1586
1587         rxq = dev->data->rx_queues;
1588         if (rxq[rx_queue_id]) {
1589                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release,
1590                                         -ENOTSUP);
1591                 (*dev->dev_ops->rx_queue_release)(rxq[rx_queue_id]);
1592                 rxq[rx_queue_id] = NULL;
1593         }
1594
1595         if (rx_conf == NULL)
1596                 rx_conf = &dev_info.default_rxconf;
1597
1598         local_conf = *rx_conf;
1599
1600         /*
1601          * If an offloading has already been enabled in
1602          * rte_eth_dev_configure(), it has been enabled on all queues,
1603          * so there is no need to enable it in this queue again.
1604          * The local_conf.offloads input to underlying PMD only carries
1605          * those offloadings which are only enabled on this queue and
1606          * not enabled on all queues.
1607          */
1608         local_conf.offloads &= ~dev->data->dev_conf.rxmode.offloads;
1609
1610         /*
1611          * New added offloadings for this queue are those not enabled in
1612          * rte_eth_dev_configure() and they must be per-queue type.
1613          * A pure per-port offloading can't be enabled on a queue while
1614          * disabled on another queue. A pure per-port offloading can't
1615          * be enabled for any queue as new added one if it hasn't been
1616          * enabled in rte_eth_dev_configure().
1617          */
1618         if ((local_conf.offloads & dev_info.rx_queue_offload_capa) !=
1619              local_conf.offloads) {
1620                 RTE_ETHDEV_LOG(ERR,
1621                         "Ethdev port_id=%d rx_queue_id=%d, new added offloads 0x%"PRIx64" must be "
1622                         "within pre-queue offload capabilities 0x%"PRIx64" in %s()\n",
1623                         port_id, rx_queue_id, local_conf.offloads,
1624                         dev_info.rx_queue_offload_capa,
1625                         __func__);
1626                 return -EINVAL;
1627         }
1628
1629         ret = (*dev->dev_ops->rx_queue_setup)(dev, rx_queue_id, nb_rx_desc,
1630                                               socket_id, &local_conf, mp);
1631         if (!ret) {
1632                 if (!dev->data->min_rx_buf_size ||
1633                     dev->data->min_rx_buf_size > mbp_buf_size)
1634                         dev->data->min_rx_buf_size = mbp_buf_size;
1635         }
1636
1637         return eth_err(port_id, ret);
1638 }
1639
1640 int
1641 rte_eth_tx_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
1642                        uint16_t nb_tx_desc, unsigned int socket_id,
1643                        const struct rte_eth_txconf *tx_conf)
1644 {
1645         struct rte_eth_dev *dev;
1646         struct rte_eth_dev_info dev_info;
1647         struct rte_eth_txconf local_conf;
1648         void **txq;
1649
1650         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1651
1652         dev = &rte_eth_devices[port_id];
1653         if (tx_queue_id >= dev->data->nb_tx_queues) {
1654                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
1655                 return -EINVAL;
1656         }
1657
1658         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
1659         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_setup, -ENOTSUP);
1660
1661         rte_eth_dev_info_get(port_id, &dev_info);
1662
1663         /* Use default specified by driver, if nb_tx_desc is zero */
1664         if (nb_tx_desc == 0) {
1665                 nb_tx_desc = dev_info.default_txportconf.ring_size;
1666                 /* If driver default is zero, fall back on EAL default */
1667                 if (nb_tx_desc == 0)
1668                         nb_tx_desc = RTE_ETH_DEV_FALLBACK_TX_RINGSIZE;
1669         }
1670         if (nb_tx_desc > dev_info.tx_desc_lim.nb_max ||
1671             nb_tx_desc < dev_info.tx_desc_lim.nb_min ||
1672             nb_tx_desc % dev_info.tx_desc_lim.nb_align != 0) {
1673                 RTE_ETHDEV_LOG(ERR,
1674                         "Invalid value for nb_tx_desc(=%hu), should be: <= %hu, = %hu, and a product of %hu\n",
1675                         nb_tx_desc, dev_info.tx_desc_lim.nb_max,
1676                         dev_info.tx_desc_lim.nb_min,
1677                         dev_info.tx_desc_lim.nb_align);
1678                 return -EINVAL;
1679         }
1680
1681         if (dev->data->dev_started &&
1682                 !(dev_info.dev_capa &
1683                         RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP))
1684                 return -EBUSY;
1685
1686         if (dev->data->dev_started &&
1687                 (dev->data->tx_queue_state[tx_queue_id] !=
1688                         RTE_ETH_QUEUE_STATE_STOPPED))
1689                 return -EBUSY;
1690
1691         txq = dev->data->tx_queues;
1692         if (txq[tx_queue_id]) {
1693                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release,
1694                                         -ENOTSUP);
1695                 (*dev->dev_ops->tx_queue_release)(txq[tx_queue_id]);
1696                 txq[tx_queue_id] = NULL;
1697         }
1698
1699         if (tx_conf == NULL)
1700                 tx_conf = &dev_info.default_txconf;
1701
1702         local_conf = *tx_conf;
1703
1704         /*
1705          * If an offloading has already been enabled in
1706          * rte_eth_dev_configure(), it has been enabled on all queues,
1707          * so there is no need to enable it in this queue again.
1708          * The local_conf.offloads input to underlying PMD only carries
1709          * those offloadings which are only enabled on this queue and
1710          * not enabled on all queues.
1711          */
1712         local_conf.offloads &= ~dev->data->dev_conf.txmode.offloads;
1713
1714         /*
1715          * New added offloadings for this queue are those not enabled in
1716          * rte_eth_dev_configure() and they must be per-queue type.
1717          * A pure per-port offloading can't be enabled on a queue while
1718          * disabled on another queue. A pure per-port offloading can't
1719          * be enabled for any queue as new added one if it hasn't been
1720          * enabled in rte_eth_dev_configure().
1721          */
1722         if ((local_conf.offloads & dev_info.tx_queue_offload_capa) !=
1723              local_conf.offloads) {
1724                 RTE_ETHDEV_LOG(ERR,
1725                         "Ethdev port_id=%d tx_queue_id=%d, new added offloads 0x%"PRIx64" must be "
1726                         "within pre-queue offload capabilities 0x%"PRIx64" in %s()\n",
1727                         port_id, tx_queue_id, local_conf.offloads,
1728                         dev_info.tx_queue_offload_capa,
1729                         __func__);
1730                 return -EINVAL;
1731         }
1732
1733         return eth_err(port_id, (*dev->dev_ops->tx_queue_setup)(dev,
1734                        tx_queue_id, nb_tx_desc, socket_id, &local_conf));
1735 }
1736
1737 void
1738 rte_eth_tx_buffer_drop_callback(struct rte_mbuf **pkts, uint16_t unsent,
1739                 void *userdata __rte_unused)
1740 {
1741         unsigned i;
1742
1743         for (i = 0; i < unsent; i++)
1744                 rte_pktmbuf_free(pkts[i]);
1745 }
1746
1747 void
1748 rte_eth_tx_buffer_count_callback(struct rte_mbuf **pkts, uint16_t unsent,
1749                 void *userdata)
1750 {
1751         uint64_t *count = userdata;
1752         unsigned i;
1753
1754         for (i = 0; i < unsent; i++)
1755                 rte_pktmbuf_free(pkts[i]);
1756
1757         *count += unsent;
1758 }
1759
1760 int
1761 rte_eth_tx_buffer_set_err_callback(struct rte_eth_dev_tx_buffer *buffer,
1762                 buffer_tx_error_fn cbfn, void *userdata)
1763 {
1764         buffer->error_callback = cbfn;
1765         buffer->error_userdata = userdata;
1766         return 0;
1767 }
1768
1769 int
1770 rte_eth_tx_buffer_init(struct rte_eth_dev_tx_buffer *buffer, uint16_t size)
1771 {
1772         int ret = 0;
1773
1774         if (buffer == NULL)
1775                 return -EINVAL;
1776
1777         buffer->size = size;
1778         if (buffer->error_callback == NULL) {
1779                 ret = rte_eth_tx_buffer_set_err_callback(
1780                         buffer, rte_eth_tx_buffer_drop_callback, NULL);
1781         }
1782
1783         return ret;
1784 }
1785
1786 int
1787 rte_eth_tx_done_cleanup(uint16_t port_id, uint16_t queue_id, uint32_t free_cnt)
1788 {
1789         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
1790         int ret;
1791
1792         /* Validate Input Data. Bail if not valid or not supported. */
1793         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1794         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_done_cleanup, -ENOTSUP);
1795
1796         /* Call driver to free pending mbufs. */
1797         ret = (*dev->dev_ops->tx_done_cleanup)(dev->data->tx_queues[queue_id],
1798                                                free_cnt);
1799         return eth_err(port_id, ret);
1800 }
1801
1802 void
1803 rte_eth_promiscuous_enable(uint16_t port_id)
1804 {
1805         struct rte_eth_dev *dev;
1806
1807         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1808         dev = &rte_eth_devices[port_id];
1809
1810         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->promiscuous_enable);
1811         (*dev->dev_ops->promiscuous_enable)(dev);
1812         dev->data->promiscuous = 1;
1813 }
1814
1815 void
1816 rte_eth_promiscuous_disable(uint16_t port_id)
1817 {
1818         struct rte_eth_dev *dev;
1819
1820         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1821         dev = &rte_eth_devices[port_id];
1822
1823         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->promiscuous_disable);
1824         dev->data->promiscuous = 0;
1825         (*dev->dev_ops->promiscuous_disable)(dev);
1826 }
1827
1828 int
1829 rte_eth_promiscuous_get(uint16_t port_id)
1830 {
1831         struct rte_eth_dev *dev;
1832
1833         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1834
1835         dev = &rte_eth_devices[port_id];
1836         return dev->data->promiscuous;
1837 }
1838
1839 void
1840 rte_eth_allmulticast_enable(uint16_t port_id)
1841 {
1842         struct rte_eth_dev *dev;
1843
1844         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1845         dev = &rte_eth_devices[port_id];
1846
1847         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->allmulticast_enable);
1848         (*dev->dev_ops->allmulticast_enable)(dev);
1849         dev->data->all_multicast = 1;
1850 }
1851
1852 void
1853 rte_eth_allmulticast_disable(uint16_t port_id)
1854 {
1855         struct rte_eth_dev *dev;
1856
1857         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1858         dev = &rte_eth_devices[port_id];
1859
1860         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->allmulticast_disable);
1861         dev->data->all_multicast = 0;
1862         (*dev->dev_ops->allmulticast_disable)(dev);
1863 }
1864
1865 int
1866 rte_eth_allmulticast_get(uint16_t port_id)
1867 {
1868         struct rte_eth_dev *dev;
1869
1870         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1871
1872         dev = &rte_eth_devices[port_id];
1873         return dev->data->all_multicast;
1874 }
1875
1876 void
1877 rte_eth_link_get(uint16_t port_id, struct rte_eth_link *eth_link)
1878 {
1879         struct rte_eth_dev *dev;
1880
1881         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1882         dev = &rte_eth_devices[port_id];
1883
1884         if (dev->data->dev_conf.intr_conf.lsc &&
1885             dev->data->dev_started)
1886                 rte_eth_linkstatus_get(dev, eth_link);
1887         else {
1888                 RTE_FUNC_PTR_OR_RET(*dev->dev_ops->link_update);
1889                 (*dev->dev_ops->link_update)(dev, 1);
1890                 *eth_link = dev->data->dev_link;
1891         }
1892 }
1893
1894 void
1895 rte_eth_link_get_nowait(uint16_t port_id, struct rte_eth_link *eth_link)
1896 {
1897         struct rte_eth_dev *dev;
1898
1899         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1900         dev = &rte_eth_devices[port_id];
1901
1902         if (dev->data->dev_conf.intr_conf.lsc &&
1903             dev->data->dev_started)
1904                 rte_eth_linkstatus_get(dev, eth_link);
1905         else {
1906                 RTE_FUNC_PTR_OR_RET(*dev->dev_ops->link_update);
1907                 (*dev->dev_ops->link_update)(dev, 0);
1908                 *eth_link = dev->data->dev_link;
1909         }
1910 }
1911
1912 int
1913 rte_eth_stats_get(uint16_t port_id, struct rte_eth_stats *stats)
1914 {
1915         struct rte_eth_dev *dev;
1916
1917         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1918
1919         dev = &rte_eth_devices[port_id];
1920         memset(stats, 0, sizeof(*stats));
1921
1922         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
1923         stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
1924         return eth_err(port_id, (*dev->dev_ops->stats_get)(dev, stats));
1925 }
1926
1927 int
1928 rte_eth_stats_reset(uint16_t port_id)
1929 {
1930         struct rte_eth_dev *dev;
1931
1932         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1933         dev = &rte_eth_devices[port_id];
1934
1935         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_reset, -ENOTSUP);
1936         (*dev->dev_ops->stats_reset)(dev);
1937         dev->data->rx_mbuf_alloc_failed = 0;
1938
1939         return 0;
1940 }
1941
1942 static inline int
1943 get_xstats_basic_count(struct rte_eth_dev *dev)
1944 {
1945         uint16_t nb_rxqs, nb_txqs;
1946         int count;
1947
1948         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
1949         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
1950
1951         count = RTE_NB_STATS;
1952         count += nb_rxqs * RTE_NB_RXQ_STATS;
1953         count += nb_txqs * RTE_NB_TXQ_STATS;
1954
1955         return count;
1956 }
1957
1958 static int
1959 get_xstats_count(uint16_t port_id)
1960 {
1961         struct rte_eth_dev *dev;
1962         int count;
1963
1964         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1965         dev = &rte_eth_devices[port_id];
1966         if (dev->dev_ops->xstats_get_names_by_id != NULL) {
1967                 count = (*dev->dev_ops->xstats_get_names_by_id)(dev, NULL,
1968                                 NULL, 0);
1969                 if (count < 0)
1970                         return eth_err(port_id, count);
1971         }
1972         if (dev->dev_ops->xstats_get_names != NULL) {
1973                 count = (*dev->dev_ops->xstats_get_names)(dev, NULL, 0);
1974                 if (count < 0)
1975                         return eth_err(port_id, count);
1976         } else
1977                 count = 0;
1978
1979
1980         count += get_xstats_basic_count(dev);
1981
1982         return count;
1983 }
1984
1985 int
1986 rte_eth_xstats_get_id_by_name(uint16_t port_id, const char *xstat_name,
1987                 uint64_t *id)
1988 {
1989         int cnt_xstats, idx_xstat;
1990
1991         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1992
1993         if (!id) {
1994                 RTE_ETHDEV_LOG(ERR, "Id pointer is NULL\n");
1995                 return -ENOMEM;
1996         }
1997
1998         if (!xstat_name) {
1999                 RTE_ETHDEV_LOG(ERR, "xstat_name pointer is NULL\n");
2000                 return -ENOMEM;
2001         }
2002
2003         /* Get count */
2004         cnt_xstats = rte_eth_xstats_get_names_by_id(port_id, NULL, 0, NULL);
2005         if (cnt_xstats  < 0) {
2006                 RTE_ETHDEV_LOG(ERR, "Cannot get count of xstats\n");
2007                 return -ENODEV;
2008         }
2009
2010         /* Get id-name lookup table */
2011         struct rte_eth_xstat_name xstats_names[cnt_xstats];
2012
2013         if (cnt_xstats != rte_eth_xstats_get_names_by_id(
2014                         port_id, xstats_names, cnt_xstats, NULL)) {
2015                 RTE_ETHDEV_LOG(ERR, "Cannot get xstats lookup\n");
2016                 return -1;
2017         }
2018
2019         for (idx_xstat = 0; idx_xstat < cnt_xstats; idx_xstat++) {
2020                 if (!strcmp(xstats_names[idx_xstat].name, xstat_name)) {
2021                         *id = idx_xstat;
2022                         return 0;
2023                 };
2024         }
2025
2026         return -EINVAL;
2027 }
2028
2029 /* retrieve basic stats names */
2030 static int
2031 rte_eth_basic_stats_get_names(struct rte_eth_dev *dev,
2032         struct rte_eth_xstat_name *xstats_names)
2033 {
2034         int cnt_used_entries = 0;
2035         uint32_t idx, id_queue;
2036         uint16_t num_q;
2037
2038         for (idx = 0; idx < RTE_NB_STATS; idx++) {
2039                 snprintf(xstats_names[cnt_used_entries].name,
2040                         sizeof(xstats_names[0].name),
2041                         "%s", rte_stats_strings[idx].name);
2042                 cnt_used_entries++;
2043         }
2044         num_q = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2045         for (id_queue = 0; id_queue < num_q; id_queue++) {
2046                 for (idx = 0; idx < RTE_NB_RXQ_STATS; idx++) {
2047                         snprintf(xstats_names[cnt_used_entries].name,
2048                                 sizeof(xstats_names[0].name),
2049                                 "rx_q%u%s",
2050                                 id_queue, rte_rxq_stats_strings[idx].name);
2051                         cnt_used_entries++;
2052                 }
2053
2054         }
2055         num_q = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2056         for (id_queue = 0; id_queue < num_q; id_queue++) {
2057                 for (idx = 0; idx < RTE_NB_TXQ_STATS; idx++) {
2058                         snprintf(xstats_names[cnt_used_entries].name,
2059                                 sizeof(xstats_names[0].name),
2060                                 "tx_q%u%s",
2061                                 id_queue, rte_txq_stats_strings[idx].name);
2062                         cnt_used_entries++;
2063                 }
2064         }
2065         return cnt_used_entries;
2066 }
2067
2068 /* retrieve ethdev extended statistics names */
2069 int
2070 rte_eth_xstats_get_names_by_id(uint16_t port_id,
2071         struct rte_eth_xstat_name *xstats_names, unsigned int size,
2072         uint64_t *ids)
2073 {
2074         struct rte_eth_xstat_name *xstats_names_copy;
2075         unsigned int no_basic_stat_requested = 1;
2076         unsigned int no_ext_stat_requested = 1;
2077         unsigned int expected_entries;
2078         unsigned int basic_count;
2079         struct rte_eth_dev *dev;
2080         unsigned int i;
2081         int ret;
2082
2083         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2084         dev = &rte_eth_devices[port_id];
2085
2086         basic_count = get_xstats_basic_count(dev);
2087         ret = get_xstats_count(port_id);
2088         if (ret < 0)
2089                 return ret;
2090         expected_entries = (unsigned int)ret;
2091
2092         /* Return max number of stats if no ids given */
2093         if (!ids) {
2094                 if (!xstats_names)
2095                         return expected_entries;
2096                 else if (xstats_names && size < expected_entries)
2097                         return expected_entries;
2098         }
2099
2100         if (ids && !xstats_names)
2101                 return -EINVAL;
2102
2103         if (ids && dev->dev_ops->xstats_get_names_by_id != NULL && size > 0) {
2104                 uint64_t ids_copy[size];
2105
2106                 for (i = 0; i < size; i++) {
2107                         if (ids[i] < basic_count) {
2108                                 no_basic_stat_requested = 0;
2109                                 break;
2110                         }
2111
2112                         /*
2113                          * Convert ids to xstats ids that PMD knows.
2114                          * ids known by user are basic + extended stats.
2115                          */
2116                         ids_copy[i] = ids[i] - basic_count;
2117                 }
2118
2119                 if (no_basic_stat_requested)
2120                         return (*dev->dev_ops->xstats_get_names_by_id)(dev,
2121                                         xstats_names, ids_copy, size);
2122         }
2123
2124         /* Retrieve all stats */
2125         if (!ids) {
2126                 int num_stats = rte_eth_xstats_get_names(port_id, xstats_names,
2127                                 expected_entries);
2128                 if (num_stats < 0 || num_stats > (int)expected_entries)
2129                         return num_stats;
2130                 else
2131                         return expected_entries;
2132         }
2133
2134         xstats_names_copy = calloc(expected_entries,
2135                 sizeof(struct rte_eth_xstat_name));
2136
2137         if (!xstats_names_copy) {
2138                 RTE_ETHDEV_LOG(ERR, "Can't allocate memory\n");
2139                 return -ENOMEM;
2140         }
2141
2142         if (ids) {
2143                 for (i = 0; i < size; i++) {
2144                         if (ids[i] >= basic_count) {
2145                                 no_ext_stat_requested = 0;
2146                                 break;
2147                         }
2148                 }
2149         }
2150
2151         /* Fill xstats_names_copy structure */
2152         if (ids && no_ext_stat_requested) {
2153                 rte_eth_basic_stats_get_names(dev, xstats_names_copy);
2154         } else {
2155                 ret = rte_eth_xstats_get_names(port_id, xstats_names_copy,
2156                         expected_entries);
2157                 if (ret < 0) {
2158                         free(xstats_names_copy);
2159                         return ret;
2160                 }
2161         }
2162
2163         /* Filter stats */
2164         for (i = 0; i < size; i++) {
2165                 if (ids[i] >= expected_entries) {
2166                         RTE_ETHDEV_LOG(ERR, "Id value isn't valid\n");
2167                         free(xstats_names_copy);
2168                         return -1;
2169                 }
2170                 xstats_names[i] = xstats_names_copy[ids[i]];
2171         }
2172
2173         free(xstats_names_copy);
2174         return size;
2175 }
2176
2177 int
2178 rte_eth_xstats_get_names(uint16_t port_id,
2179         struct rte_eth_xstat_name *xstats_names,
2180         unsigned int size)
2181 {
2182         struct rte_eth_dev *dev;
2183         int cnt_used_entries;
2184         int cnt_expected_entries;
2185         int cnt_driver_entries;
2186
2187         cnt_expected_entries = get_xstats_count(port_id);
2188         if (xstats_names == NULL || cnt_expected_entries < 0 ||
2189                         (int)size < cnt_expected_entries)
2190                 return cnt_expected_entries;
2191
2192         /* port_id checked in get_xstats_count() */
2193         dev = &rte_eth_devices[port_id];
2194
2195         cnt_used_entries = rte_eth_basic_stats_get_names(
2196                 dev, xstats_names);
2197
2198         if (dev->dev_ops->xstats_get_names != NULL) {
2199                 /* If there are any driver-specific xstats, append them
2200                  * to end of list.
2201                  */
2202                 cnt_driver_entries = (*dev->dev_ops->xstats_get_names)(
2203                         dev,
2204                         xstats_names + cnt_used_entries,
2205                         size - cnt_used_entries);
2206                 if (cnt_driver_entries < 0)
2207                         return eth_err(port_id, cnt_driver_entries);
2208                 cnt_used_entries += cnt_driver_entries;
2209         }
2210
2211         return cnt_used_entries;
2212 }
2213
2214
2215 static int
2216 rte_eth_basic_stats_get(uint16_t port_id, struct rte_eth_xstat *xstats)
2217 {
2218         struct rte_eth_dev *dev;
2219         struct rte_eth_stats eth_stats;
2220         unsigned int count = 0, i, q;
2221         uint64_t val, *stats_ptr;
2222         uint16_t nb_rxqs, nb_txqs;
2223         int ret;
2224
2225         ret = rte_eth_stats_get(port_id, &eth_stats);
2226         if (ret < 0)
2227                 return ret;
2228
2229         dev = &rte_eth_devices[port_id];
2230
2231         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2232         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2233
2234         /* global stats */
2235         for (i = 0; i < RTE_NB_STATS; i++) {
2236                 stats_ptr = RTE_PTR_ADD(&eth_stats,
2237                                         rte_stats_strings[i].offset);
2238                 val = *stats_ptr;
2239                 xstats[count++].value = val;
2240         }
2241
2242         /* per-rxq stats */
2243         for (q = 0; q < nb_rxqs; q++) {
2244                 for (i = 0; i < RTE_NB_RXQ_STATS; i++) {
2245                         stats_ptr = RTE_PTR_ADD(&eth_stats,
2246                                         rte_rxq_stats_strings[i].offset +
2247                                         q * sizeof(uint64_t));
2248                         val = *stats_ptr;
2249                         xstats[count++].value = val;
2250                 }
2251         }
2252
2253         /* per-txq stats */
2254         for (q = 0; q < nb_txqs; q++) {
2255                 for (i = 0; i < RTE_NB_TXQ_STATS; i++) {
2256                         stats_ptr = RTE_PTR_ADD(&eth_stats,
2257                                         rte_txq_stats_strings[i].offset +
2258                                         q * sizeof(uint64_t));
2259                         val = *stats_ptr;
2260                         xstats[count++].value = val;
2261                 }
2262         }
2263         return count;
2264 }
2265
2266 /* retrieve ethdev extended statistics */
2267 int
2268 rte_eth_xstats_get_by_id(uint16_t port_id, const uint64_t *ids,
2269                          uint64_t *values, unsigned int size)
2270 {
2271         unsigned int no_basic_stat_requested = 1;
2272         unsigned int no_ext_stat_requested = 1;
2273         unsigned int num_xstats_filled;
2274         unsigned int basic_count;
2275         uint16_t expected_entries;
2276         struct rte_eth_dev *dev;
2277         unsigned int i;
2278         int ret;
2279
2280         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2281         ret = get_xstats_count(port_id);
2282         if (ret < 0)
2283                 return ret;
2284         expected_entries = (uint16_t)ret;
2285         struct rte_eth_xstat xstats[expected_entries];
2286         dev = &rte_eth_devices[port_id];
2287         basic_count = get_xstats_basic_count(dev);
2288
2289         /* Return max number of stats if no ids given */
2290         if (!ids) {
2291                 if (!values)
2292                         return expected_entries;
2293                 else if (values && size < expected_entries)
2294                         return expected_entries;
2295         }
2296
2297         if (ids && !values)
2298                 return -EINVAL;
2299
2300         if (ids && dev->dev_ops->xstats_get_by_id != NULL && size) {
2301                 unsigned int basic_count = get_xstats_basic_count(dev);
2302                 uint64_t ids_copy[size];
2303
2304                 for (i = 0; i < size; i++) {
2305                         if (ids[i] < basic_count) {
2306                                 no_basic_stat_requested = 0;
2307                                 break;
2308                         }
2309
2310                         /*
2311                          * Convert ids to xstats ids that PMD knows.
2312                          * ids known by user are basic + extended stats.
2313                          */
2314                         ids_copy[i] = ids[i] - basic_count;
2315                 }
2316
2317                 if (no_basic_stat_requested)
2318                         return (*dev->dev_ops->xstats_get_by_id)(dev, ids_copy,
2319                                         values, size);
2320         }
2321
2322         if (ids) {
2323                 for (i = 0; i < size; i++) {
2324                         if (ids[i] >= basic_count) {
2325                                 no_ext_stat_requested = 0;
2326                                 break;
2327                         }
2328                 }
2329         }
2330
2331         /* Fill the xstats structure */
2332         if (ids && no_ext_stat_requested)
2333                 ret = rte_eth_basic_stats_get(port_id, xstats);
2334         else
2335                 ret = rte_eth_xstats_get(port_id, xstats, expected_entries);
2336
2337         if (ret < 0)
2338                 return ret;
2339         num_xstats_filled = (unsigned int)ret;
2340
2341         /* Return all stats */
2342         if (!ids) {
2343                 for (i = 0; i < num_xstats_filled; i++)
2344                         values[i] = xstats[i].value;
2345                 return expected_entries;
2346         }
2347
2348         /* Filter stats */
2349         for (i = 0; i < size; i++) {
2350                 if (ids[i] >= expected_entries) {
2351                         RTE_ETHDEV_LOG(ERR, "Id value isn't valid\n");
2352                         return -1;
2353                 }
2354                 values[i] = xstats[ids[i]].value;
2355         }
2356         return size;
2357 }
2358
2359 int
2360 rte_eth_xstats_get(uint16_t port_id, struct rte_eth_xstat *xstats,
2361         unsigned int n)
2362 {
2363         struct rte_eth_dev *dev;
2364         unsigned int count = 0, i;
2365         signed int xcount = 0;
2366         uint16_t nb_rxqs, nb_txqs;
2367         int ret;
2368
2369         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2370
2371         dev = &rte_eth_devices[port_id];
2372
2373         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2374         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2375
2376         /* Return generic statistics */
2377         count = RTE_NB_STATS + (nb_rxqs * RTE_NB_RXQ_STATS) +
2378                 (nb_txqs * RTE_NB_TXQ_STATS);
2379
2380         /* implemented by the driver */
2381         if (dev->dev_ops->xstats_get != NULL) {
2382                 /* Retrieve the xstats from the driver at the end of the
2383                  * xstats struct.
2384                  */
2385                 xcount = (*dev->dev_ops->xstats_get)(dev,
2386                                      xstats ? xstats + count : NULL,
2387                                      (n > count) ? n - count : 0);
2388
2389                 if (xcount < 0)
2390                         return eth_err(port_id, xcount);
2391         }
2392
2393         if (n < count + xcount || xstats == NULL)
2394                 return count + xcount;
2395
2396         /* now fill the xstats structure */
2397         ret = rte_eth_basic_stats_get(port_id, xstats);
2398         if (ret < 0)
2399                 return ret;
2400         count = ret;
2401
2402         for (i = 0; i < count; i++)
2403                 xstats[i].id = i;
2404         /* add an offset to driver-specific stats */
2405         for ( ; i < count + xcount; i++)
2406                 xstats[i].id += count;
2407
2408         return count + xcount;
2409 }
2410
2411 /* reset ethdev extended statistics */
2412 void
2413 rte_eth_xstats_reset(uint16_t port_id)
2414 {
2415         struct rte_eth_dev *dev;
2416
2417         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2418         dev = &rte_eth_devices[port_id];
2419
2420         /* implemented by the driver */
2421         if (dev->dev_ops->xstats_reset != NULL) {
2422                 (*dev->dev_ops->xstats_reset)(dev);
2423                 return;
2424         }
2425
2426         /* fallback to default */
2427         rte_eth_stats_reset(port_id);
2428 }
2429
2430 static int
2431 set_queue_stats_mapping(uint16_t port_id, uint16_t queue_id, uint8_t stat_idx,
2432                 uint8_t is_rx)
2433 {
2434         struct rte_eth_dev *dev;
2435
2436         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2437
2438         dev = &rte_eth_devices[port_id];
2439
2440         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_stats_mapping_set, -ENOTSUP);
2441
2442         if (is_rx && (queue_id >= dev->data->nb_rx_queues))
2443                 return -EINVAL;
2444
2445         if (!is_rx && (queue_id >= dev->data->nb_tx_queues))
2446                 return -EINVAL;
2447
2448         if (stat_idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
2449                 return -EINVAL;
2450
2451         return (*dev->dev_ops->queue_stats_mapping_set)
2452                         (dev, queue_id, stat_idx, is_rx);
2453 }
2454
2455
2456 int
2457 rte_eth_dev_set_tx_queue_stats_mapping(uint16_t port_id, uint16_t tx_queue_id,
2458                 uint8_t stat_idx)
2459 {
2460         return eth_err(port_id, set_queue_stats_mapping(port_id, tx_queue_id,
2461                                                 stat_idx, STAT_QMAP_TX));
2462 }
2463
2464
2465 int
2466 rte_eth_dev_set_rx_queue_stats_mapping(uint16_t port_id, uint16_t rx_queue_id,
2467                 uint8_t stat_idx)
2468 {
2469         return eth_err(port_id, set_queue_stats_mapping(port_id, rx_queue_id,
2470                                                 stat_idx, STAT_QMAP_RX));
2471 }
2472
2473 int
2474 rte_eth_dev_fw_version_get(uint16_t port_id, char *fw_version, size_t fw_size)
2475 {
2476         struct rte_eth_dev *dev;
2477
2478         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2479         dev = &rte_eth_devices[port_id];
2480
2481         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fw_version_get, -ENOTSUP);
2482         return eth_err(port_id, (*dev->dev_ops->fw_version_get)(dev,
2483                                                         fw_version, fw_size));
2484 }
2485
2486 void
2487 rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info)
2488 {
2489         struct rte_eth_dev *dev;
2490         const struct rte_eth_desc_lim lim = {
2491                 .nb_max = UINT16_MAX,
2492                 .nb_min = 0,
2493                 .nb_align = 1,
2494         };
2495
2496         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2497         dev = &rte_eth_devices[port_id];
2498
2499         memset(dev_info, 0, sizeof(struct rte_eth_dev_info));
2500         dev_info->rx_desc_lim = lim;
2501         dev_info->tx_desc_lim = lim;
2502         dev_info->device = dev->device;
2503
2504         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_infos_get);
2505         (*dev->dev_ops->dev_infos_get)(dev, dev_info);
2506         dev_info->driver_name = dev->device->driver->name;
2507         dev_info->nb_rx_queues = dev->data->nb_rx_queues;
2508         dev_info->nb_tx_queues = dev->data->nb_tx_queues;
2509
2510         dev_info->dev_flags = &dev->data->dev_flags;
2511 }
2512
2513 int
2514 rte_eth_dev_get_supported_ptypes(uint16_t port_id, uint32_t ptype_mask,
2515                                  uint32_t *ptypes, int num)
2516 {
2517         int i, j;
2518         struct rte_eth_dev *dev;
2519         const uint32_t *all_ptypes;
2520
2521         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2522         dev = &rte_eth_devices[port_id];
2523         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_supported_ptypes_get, 0);
2524         all_ptypes = (*dev->dev_ops->dev_supported_ptypes_get)(dev);
2525
2526         if (!all_ptypes)
2527                 return 0;
2528
2529         for (i = 0, j = 0; all_ptypes[i] != RTE_PTYPE_UNKNOWN; ++i)
2530                 if (all_ptypes[i] & ptype_mask) {
2531                         if (j < num)
2532                                 ptypes[j] = all_ptypes[i];
2533                         j++;
2534                 }
2535
2536         return j;
2537 }
2538
2539 void
2540 rte_eth_macaddr_get(uint16_t port_id, struct ether_addr *mac_addr)
2541 {
2542         struct rte_eth_dev *dev;
2543
2544         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2545         dev = &rte_eth_devices[port_id];
2546         ether_addr_copy(&dev->data->mac_addrs[0], mac_addr);
2547 }
2548
2549
2550 int
2551 rte_eth_dev_get_mtu(uint16_t port_id, uint16_t *mtu)
2552 {
2553         struct rte_eth_dev *dev;
2554
2555         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2556
2557         dev = &rte_eth_devices[port_id];
2558         *mtu = dev->data->mtu;
2559         return 0;
2560 }
2561
2562 int
2563 rte_eth_dev_set_mtu(uint16_t port_id, uint16_t mtu)
2564 {
2565         int ret;
2566         struct rte_eth_dev *dev;
2567
2568         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2569         dev = &rte_eth_devices[port_id];
2570         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mtu_set, -ENOTSUP);
2571
2572         ret = (*dev->dev_ops->mtu_set)(dev, mtu);
2573         if (!ret)
2574                 dev->data->mtu = mtu;
2575
2576         return eth_err(port_id, ret);
2577 }
2578
2579 int
2580 rte_eth_dev_vlan_filter(uint16_t port_id, uint16_t vlan_id, int on)
2581 {
2582         struct rte_eth_dev *dev;
2583         int ret;
2584
2585         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2586         dev = &rte_eth_devices[port_id];
2587         if (!(dev->data->dev_conf.rxmode.offloads &
2588               DEV_RX_OFFLOAD_VLAN_FILTER)) {
2589                 RTE_ETHDEV_LOG(ERR, "Port %u: vlan-filtering disabled\n",
2590                         port_id);
2591                 return -ENOSYS;
2592         }
2593
2594         if (vlan_id > 4095) {
2595                 RTE_ETHDEV_LOG(ERR, "Port_id=%u invalid vlan_id=%u > 4095\n",
2596                         port_id, vlan_id);
2597                 return -EINVAL;
2598         }
2599         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_filter_set, -ENOTSUP);
2600
2601         ret = (*dev->dev_ops->vlan_filter_set)(dev, vlan_id, on);
2602         if (ret == 0) {
2603                 struct rte_vlan_filter_conf *vfc;
2604                 int vidx;
2605                 int vbit;
2606
2607                 vfc = &dev->data->vlan_filter_conf;
2608                 vidx = vlan_id / 64;
2609                 vbit = vlan_id % 64;
2610
2611                 if (on)
2612                         vfc->ids[vidx] |= UINT64_C(1) << vbit;
2613                 else
2614                         vfc->ids[vidx] &= ~(UINT64_C(1) << vbit);
2615         }
2616
2617         return eth_err(port_id, ret);
2618 }
2619
2620 int
2621 rte_eth_dev_set_vlan_strip_on_queue(uint16_t port_id, uint16_t rx_queue_id,
2622                                     int on)
2623 {
2624         struct rte_eth_dev *dev;
2625
2626         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2627         dev = &rte_eth_devices[port_id];
2628         if (rx_queue_id >= dev->data->nb_rx_queues) {
2629                 RTE_ETHDEV_LOG(ERR, "Invalid rx_queue_id=%u\n", rx_queue_id);
2630                 return -EINVAL;
2631         }
2632
2633         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_strip_queue_set, -ENOTSUP);
2634         (*dev->dev_ops->vlan_strip_queue_set)(dev, rx_queue_id, on);
2635
2636         return 0;
2637 }
2638
2639 int
2640 rte_eth_dev_set_vlan_ether_type(uint16_t port_id,
2641                                 enum rte_vlan_type vlan_type,
2642                                 uint16_t tpid)
2643 {
2644         struct rte_eth_dev *dev;
2645
2646         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2647         dev = &rte_eth_devices[port_id];
2648         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_tpid_set, -ENOTSUP);
2649
2650         return eth_err(port_id, (*dev->dev_ops->vlan_tpid_set)(dev, vlan_type,
2651                                                                tpid));
2652 }
2653
2654 int
2655 rte_eth_dev_set_vlan_offload(uint16_t port_id, int offload_mask)
2656 {
2657         struct rte_eth_dev *dev;
2658         int ret = 0;
2659         int mask = 0;
2660         int cur, org = 0;
2661         uint64_t orig_offloads;
2662
2663         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2664         dev = &rte_eth_devices[port_id];
2665
2666         /* save original values in case of failure */
2667         orig_offloads = dev->data->dev_conf.rxmode.offloads;
2668
2669         /*check which option changed by application*/
2670         cur = !!(offload_mask & ETH_VLAN_STRIP_OFFLOAD);
2671         org = !!(dev->data->dev_conf.rxmode.offloads &
2672                  DEV_RX_OFFLOAD_VLAN_STRIP);
2673         if (cur != org) {
2674                 if (cur)
2675                         dev->data->dev_conf.rxmode.offloads |=
2676                                 DEV_RX_OFFLOAD_VLAN_STRIP;
2677                 else
2678                         dev->data->dev_conf.rxmode.offloads &=
2679                                 ~DEV_RX_OFFLOAD_VLAN_STRIP;
2680                 mask |= ETH_VLAN_STRIP_MASK;
2681         }
2682
2683         cur = !!(offload_mask & ETH_VLAN_FILTER_OFFLOAD);
2684         org = !!(dev->data->dev_conf.rxmode.offloads &
2685                  DEV_RX_OFFLOAD_VLAN_FILTER);
2686         if (cur != org) {
2687                 if (cur)
2688                         dev->data->dev_conf.rxmode.offloads |=
2689                                 DEV_RX_OFFLOAD_VLAN_FILTER;
2690                 else
2691                         dev->data->dev_conf.rxmode.offloads &=
2692                                 ~DEV_RX_OFFLOAD_VLAN_FILTER;
2693                 mask |= ETH_VLAN_FILTER_MASK;
2694         }
2695
2696         cur = !!(offload_mask & ETH_VLAN_EXTEND_OFFLOAD);
2697         org = !!(dev->data->dev_conf.rxmode.offloads &
2698                  DEV_RX_OFFLOAD_VLAN_EXTEND);
2699         if (cur != org) {
2700                 if (cur)
2701                         dev->data->dev_conf.rxmode.offloads |=
2702                                 DEV_RX_OFFLOAD_VLAN_EXTEND;
2703                 else
2704                         dev->data->dev_conf.rxmode.offloads &=
2705                                 ~DEV_RX_OFFLOAD_VLAN_EXTEND;
2706                 mask |= ETH_VLAN_EXTEND_MASK;
2707         }
2708
2709         /*no change*/
2710         if (mask == 0)
2711                 return ret;
2712
2713         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_offload_set, -ENOTSUP);
2714         ret = (*dev->dev_ops->vlan_offload_set)(dev, mask);
2715         if (ret) {
2716                 /* hit an error restore  original values */
2717                 dev->data->dev_conf.rxmode.offloads = orig_offloads;
2718         }
2719
2720         return eth_err(port_id, ret);
2721 }
2722
2723 int
2724 rte_eth_dev_get_vlan_offload(uint16_t port_id)
2725 {
2726         struct rte_eth_dev *dev;
2727         int ret = 0;
2728
2729         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2730         dev = &rte_eth_devices[port_id];
2731
2732         if (dev->data->dev_conf.rxmode.offloads &
2733             DEV_RX_OFFLOAD_VLAN_STRIP)
2734                 ret |= ETH_VLAN_STRIP_OFFLOAD;
2735
2736         if (dev->data->dev_conf.rxmode.offloads &
2737             DEV_RX_OFFLOAD_VLAN_FILTER)
2738                 ret |= ETH_VLAN_FILTER_OFFLOAD;
2739
2740         if (dev->data->dev_conf.rxmode.offloads &
2741             DEV_RX_OFFLOAD_VLAN_EXTEND)
2742                 ret |= ETH_VLAN_EXTEND_OFFLOAD;
2743
2744         return ret;
2745 }
2746
2747 int
2748 rte_eth_dev_set_vlan_pvid(uint16_t port_id, uint16_t pvid, int on)
2749 {
2750         struct rte_eth_dev *dev;
2751
2752         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2753         dev = &rte_eth_devices[port_id];
2754         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_pvid_set, -ENOTSUP);
2755
2756         return eth_err(port_id, (*dev->dev_ops->vlan_pvid_set)(dev, pvid, on));
2757 }
2758
2759 int
2760 rte_eth_dev_flow_ctrl_get(uint16_t port_id, struct rte_eth_fc_conf *fc_conf)
2761 {
2762         struct rte_eth_dev *dev;
2763
2764         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2765         dev = &rte_eth_devices[port_id];
2766         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_get, -ENOTSUP);
2767         memset(fc_conf, 0, sizeof(*fc_conf));
2768         return eth_err(port_id, (*dev->dev_ops->flow_ctrl_get)(dev, fc_conf));
2769 }
2770
2771 int
2772 rte_eth_dev_flow_ctrl_set(uint16_t port_id, struct rte_eth_fc_conf *fc_conf)
2773 {
2774         struct rte_eth_dev *dev;
2775
2776         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2777         if ((fc_conf->send_xon != 0) && (fc_conf->send_xon != 1)) {
2778                 RTE_ETHDEV_LOG(ERR, "Invalid send_xon, only 0/1 allowed\n");
2779                 return -EINVAL;
2780         }
2781
2782         dev = &rte_eth_devices[port_id];
2783         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_set, -ENOTSUP);
2784         return eth_err(port_id, (*dev->dev_ops->flow_ctrl_set)(dev, fc_conf));
2785 }
2786
2787 int
2788 rte_eth_dev_priority_flow_ctrl_set(uint16_t port_id,
2789                                    struct rte_eth_pfc_conf *pfc_conf)
2790 {
2791         struct rte_eth_dev *dev;
2792
2793         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2794         if (pfc_conf->priority > (ETH_DCB_NUM_USER_PRIORITIES - 1)) {
2795                 RTE_ETHDEV_LOG(ERR, "Invalid priority, only 0-7 allowed\n");
2796                 return -EINVAL;
2797         }
2798
2799         dev = &rte_eth_devices[port_id];
2800         /* High water, low water validation are device specific */
2801         if  (*dev->dev_ops->priority_flow_ctrl_set)
2802                 return eth_err(port_id, (*dev->dev_ops->priority_flow_ctrl_set)
2803                                         (dev, pfc_conf));
2804         return -ENOTSUP;
2805 }
2806
2807 static int
2808 rte_eth_check_reta_mask(struct rte_eth_rss_reta_entry64 *reta_conf,
2809                         uint16_t reta_size)
2810 {
2811         uint16_t i, num;
2812
2813         if (!reta_conf)
2814                 return -EINVAL;
2815
2816         num = (reta_size + RTE_RETA_GROUP_SIZE - 1) / RTE_RETA_GROUP_SIZE;
2817         for (i = 0; i < num; i++) {
2818                 if (reta_conf[i].mask)
2819                         return 0;
2820         }
2821
2822         return -EINVAL;
2823 }
2824
2825 static int
2826 rte_eth_check_reta_entry(struct rte_eth_rss_reta_entry64 *reta_conf,
2827                          uint16_t reta_size,
2828                          uint16_t max_rxq)
2829 {
2830         uint16_t i, idx, shift;
2831
2832         if (!reta_conf)
2833                 return -EINVAL;
2834
2835         if (max_rxq == 0) {
2836                 RTE_ETHDEV_LOG(ERR, "No receive queue is available\n");
2837                 return -EINVAL;
2838         }
2839
2840         for (i = 0; i < reta_size; i++) {
2841                 idx = i / RTE_RETA_GROUP_SIZE;
2842                 shift = i % RTE_RETA_GROUP_SIZE;
2843                 if ((reta_conf[idx].mask & (1ULL << shift)) &&
2844                         (reta_conf[idx].reta[shift] >= max_rxq)) {
2845                         RTE_ETHDEV_LOG(ERR,
2846                                 "reta_conf[%u]->reta[%u]: %u exceeds the maximum rxq index: %u\n",
2847                                 idx, shift,
2848                                 reta_conf[idx].reta[shift], max_rxq);
2849                         return -EINVAL;
2850                 }
2851         }
2852
2853         return 0;
2854 }
2855
2856 int
2857 rte_eth_dev_rss_reta_update(uint16_t port_id,
2858                             struct rte_eth_rss_reta_entry64 *reta_conf,
2859                             uint16_t reta_size)
2860 {
2861         struct rte_eth_dev *dev;
2862         int ret;
2863
2864         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2865         /* Check mask bits */
2866         ret = rte_eth_check_reta_mask(reta_conf, reta_size);
2867         if (ret < 0)
2868                 return ret;
2869
2870         dev = &rte_eth_devices[port_id];
2871
2872         /* Check entry value */
2873         ret = rte_eth_check_reta_entry(reta_conf, reta_size,
2874                                 dev->data->nb_rx_queues);
2875         if (ret < 0)
2876                 return ret;
2877
2878         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_update, -ENOTSUP);
2879         return eth_err(port_id, (*dev->dev_ops->reta_update)(dev, reta_conf,
2880                                                              reta_size));
2881 }
2882
2883 int
2884 rte_eth_dev_rss_reta_query(uint16_t port_id,
2885                            struct rte_eth_rss_reta_entry64 *reta_conf,
2886                            uint16_t reta_size)
2887 {
2888         struct rte_eth_dev *dev;
2889         int ret;
2890
2891         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2892
2893         /* Check mask bits */
2894         ret = rte_eth_check_reta_mask(reta_conf, reta_size);
2895         if (ret < 0)
2896                 return ret;
2897
2898         dev = &rte_eth_devices[port_id];
2899         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_query, -ENOTSUP);
2900         return eth_err(port_id, (*dev->dev_ops->reta_query)(dev, reta_conf,
2901                                                             reta_size));
2902 }
2903
2904 int
2905 rte_eth_dev_rss_hash_update(uint16_t port_id,
2906                             struct rte_eth_rss_conf *rss_conf)
2907 {
2908         struct rte_eth_dev *dev;
2909         struct rte_eth_dev_info dev_info = { .flow_type_rss_offloads = 0, };
2910
2911         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2912         dev = &rte_eth_devices[port_id];
2913         rte_eth_dev_info_get(port_id, &dev_info);
2914         if ((dev_info.flow_type_rss_offloads | rss_conf->rss_hf) !=
2915             dev_info.flow_type_rss_offloads) {
2916                 RTE_ETHDEV_LOG(ERR,
2917                         "Ethdev port_id=%u invalid rss_hf: 0x%"PRIx64", valid value: 0x%"PRIx64"\n",
2918                         port_id, rss_conf->rss_hf,
2919                         dev_info.flow_type_rss_offloads);
2920                 return -EINVAL;
2921         }
2922         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_update, -ENOTSUP);
2923         return eth_err(port_id, (*dev->dev_ops->rss_hash_update)(dev,
2924                                                                  rss_conf));
2925 }
2926
2927 int
2928 rte_eth_dev_rss_hash_conf_get(uint16_t port_id,
2929                               struct rte_eth_rss_conf *rss_conf)
2930 {
2931         struct rte_eth_dev *dev;
2932
2933         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2934         dev = &rte_eth_devices[port_id];
2935         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_conf_get, -ENOTSUP);
2936         return eth_err(port_id, (*dev->dev_ops->rss_hash_conf_get)(dev,
2937                                                                    rss_conf));
2938 }
2939
2940 int
2941 rte_eth_dev_udp_tunnel_port_add(uint16_t port_id,
2942                                 struct rte_eth_udp_tunnel *udp_tunnel)
2943 {
2944         struct rte_eth_dev *dev;
2945
2946         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2947         if (udp_tunnel == NULL) {
2948                 RTE_ETHDEV_LOG(ERR, "Invalid udp_tunnel parameter\n");
2949                 return -EINVAL;
2950         }
2951
2952         if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
2953                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
2954                 return -EINVAL;
2955         }
2956
2957         dev = &rte_eth_devices[port_id];
2958         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_port_add, -ENOTSUP);
2959         return eth_err(port_id, (*dev->dev_ops->udp_tunnel_port_add)(dev,
2960                                                                 udp_tunnel));
2961 }
2962
2963 int
2964 rte_eth_dev_udp_tunnel_port_delete(uint16_t port_id,
2965                                    struct rte_eth_udp_tunnel *udp_tunnel)
2966 {
2967         struct rte_eth_dev *dev;
2968
2969         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2970         dev = &rte_eth_devices[port_id];
2971
2972         if (udp_tunnel == NULL) {
2973                 RTE_ETHDEV_LOG(ERR, "Invalid udp_tunnel parameter\n");
2974                 return -EINVAL;
2975         }
2976
2977         if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
2978                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
2979                 return -EINVAL;
2980         }
2981
2982         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_port_del, -ENOTSUP);
2983         return eth_err(port_id, (*dev->dev_ops->udp_tunnel_port_del)(dev,
2984                                                                 udp_tunnel));
2985 }
2986
2987 int
2988 rte_eth_led_on(uint16_t port_id)
2989 {
2990         struct rte_eth_dev *dev;
2991
2992         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2993         dev = &rte_eth_devices[port_id];
2994         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_on, -ENOTSUP);
2995         return eth_err(port_id, (*dev->dev_ops->dev_led_on)(dev));
2996 }
2997
2998 int
2999 rte_eth_led_off(uint16_t port_id)
3000 {
3001         struct rte_eth_dev *dev;
3002
3003         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3004         dev = &rte_eth_devices[port_id];
3005         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_off, -ENOTSUP);
3006         return eth_err(port_id, (*dev->dev_ops->dev_led_off)(dev));
3007 }
3008
3009 /*
3010  * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find
3011  * an empty spot.
3012  */
3013 static int
3014 get_mac_addr_index(uint16_t port_id, const struct ether_addr *addr)
3015 {
3016         struct rte_eth_dev_info dev_info;
3017         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3018         unsigned i;
3019
3020         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3021         rte_eth_dev_info_get(port_id, &dev_info);
3022
3023         for (i = 0; i < dev_info.max_mac_addrs; i++)
3024                 if (memcmp(addr, &dev->data->mac_addrs[i], ETHER_ADDR_LEN) == 0)
3025                         return i;
3026
3027         return -1;
3028 }
3029
3030 static const struct ether_addr null_mac_addr;
3031
3032 int
3033 rte_eth_dev_mac_addr_add(uint16_t port_id, struct ether_addr *addr,
3034                         uint32_t pool)
3035 {
3036         struct rte_eth_dev *dev;
3037         int index;
3038         uint64_t pool_mask;
3039         int ret;
3040
3041         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3042         dev = &rte_eth_devices[port_id];
3043         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_add, -ENOTSUP);
3044
3045         if (is_zero_ether_addr(addr)) {
3046                 RTE_ETHDEV_LOG(ERR, "Port %u: Cannot add NULL MAC address\n",
3047                         port_id);
3048                 return -EINVAL;
3049         }
3050         if (pool >= ETH_64_POOLS) {
3051                 RTE_ETHDEV_LOG(ERR, "Pool id must be 0-%d\n", ETH_64_POOLS - 1);
3052                 return -EINVAL;
3053         }
3054
3055         index = get_mac_addr_index(port_id, addr);
3056         if (index < 0) {
3057                 index = get_mac_addr_index(port_id, &null_mac_addr);
3058                 if (index < 0) {
3059                         RTE_ETHDEV_LOG(ERR, "Port %u: MAC address array full\n",
3060                                 port_id);
3061                         return -ENOSPC;
3062                 }
3063         } else {
3064                 pool_mask = dev->data->mac_pool_sel[index];
3065
3066                 /* Check if both MAC address and pool is already there, and do nothing */
3067                 if (pool_mask & (1ULL << pool))
3068                         return 0;
3069         }
3070
3071         /* Update NIC */
3072         ret = (*dev->dev_ops->mac_addr_add)(dev, addr, index, pool);
3073
3074         if (ret == 0) {
3075                 /* Update address in NIC data structure */
3076                 ether_addr_copy(addr, &dev->data->mac_addrs[index]);
3077
3078                 /* Update pool bitmap in NIC data structure */
3079                 dev->data->mac_pool_sel[index] |= (1ULL << pool);
3080         }
3081
3082         return eth_err(port_id, ret);
3083 }
3084
3085 int
3086 rte_eth_dev_mac_addr_remove(uint16_t port_id, struct ether_addr *addr)
3087 {
3088         struct rte_eth_dev *dev;
3089         int index;
3090
3091         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3092         dev = &rte_eth_devices[port_id];
3093         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_remove, -ENOTSUP);
3094
3095         index = get_mac_addr_index(port_id, addr);
3096         if (index == 0) {
3097                 RTE_ETHDEV_LOG(ERR,
3098                         "Port %u: Cannot remove default MAC address\n",
3099                         port_id);
3100                 return -EADDRINUSE;
3101         } else if (index < 0)
3102                 return 0;  /* Do nothing if address wasn't found */
3103
3104         /* Update NIC */
3105         (*dev->dev_ops->mac_addr_remove)(dev, index);
3106
3107         /* Update address in NIC data structure */
3108         ether_addr_copy(&null_mac_addr, &dev->data->mac_addrs[index]);
3109
3110         /* reset pool bitmap */
3111         dev->data->mac_pool_sel[index] = 0;
3112
3113         return 0;
3114 }
3115
3116 int
3117 rte_eth_dev_default_mac_addr_set(uint16_t port_id, struct ether_addr *addr)
3118 {
3119         struct rte_eth_dev *dev;
3120         int ret;
3121
3122         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3123
3124         if (!is_valid_assigned_ether_addr(addr))
3125                 return -EINVAL;
3126
3127         dev = &rte_eth_devices[port_id];
3128         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_set, -ENOTSUP);
3129
3130         ret = (*dev->dev_ops->mac_addr_set)(dev, addr);
3131         if (ret < 0)
3132                 return ret;
3133
3134         /* Update default address in NIC data structure */
3135         ether_addr_copy(addr, &dev->data->mac_addrs[0]);
3136
3137         return 0;
3138 }
3139
3140
3141 /*
3142  * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find
3143  * an empty spot.
3144  */
3145 static int
3146 get_hash_mac_addr_index(uint16_t port_id, const struct ether_addr *addr)
3147 {
3148         struct rte_eth_dev_info dev_info;
3149         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3150         unsigned i;
3151
3152         rte_eth_dev_info_get(port_id, &dev_info);
3153         if (!dev->data->hash_mac_addrs)
3154                 return -1;
3155
3156         for (i = 0; i < dev_info.max_hash_mac_addrs; i++)
3157                 if (memcmp(addr, &dev->data->hash_mac_addrs[i],
3158                         ETHER_ADDR_LEN) == 0)
3159                         return i;
3160
3161         return -1;
3162 }
3163
3164 int
3165 rte_eth_dev_uc_hash_table_set(uint16_t port_id, struct ether_addr *addr,
3166                                 uint8_t on)
3167 {
3168         int index;
3169         int ret;
3170         struct rte_eth_dev *dev;
3171
3172         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3173
3174         dev = &rte_eth_devices[port_id];
3175         if (is_zero_ether_addr(addr)) {
3176                 RTE_ETHDEV_LOG(ERR, "Port %u: Cannot add NULL MAC address\n",
3177                         port_id);
3178                 return -EINVAL;
3179         }
3180
3181         index = get_hash_mac_addr_index(port_id, addr);
3182         /* Check if it's already there, and do nothing */
3183         if ((index >= 0) && on)
3184                 return 0;
3185
3186         if (index < 0) {
3187                 if (!on) {
3188                         RTE_ETHDEV_LOG(ERR,
3189                                 "Port %u: the MAC address was not set in UTA\n",
3190                                 port_id);
3191                         return -EINVAL;
3192                 }
3193
3194                 index = get_hash_mac_addr_index(port_id, &null_mac_addr);
3195                 if (index < 0) {
3196                         RTE_ETHDEV_LOG(ERR, "Port %u: MAC address array full\n",
3197                                 port_id);
3198                         return -ENOSPC;
3199                 }
3200         }
3201
3202         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_hash_table_set, -ENOTSUP);
3203         ret = (*dev->dev_ops->uc_hash_table_set)(dev, addr, on);
3204         if (ret == 0) {
3205                 /* Update address in NIC data structure */
3206                 if (on)
3207                         ether_addr_copy(addr,
3208                                         &dev->data->hash_mac_addrs[index]);
3209                 else
3210                         ether_addr_copy(&null_mac_addr,
3211                                         &dev->data->hash_mac_addrs[index]);
3212         }
3213
3214         return eth_err(port_id, ret);
3215 }
3216
3217 int
3218 rte_eth_dev_uc_all_hash_table_set(uint16_t port_id, uint8_t on)
3219 {
3220         struct rte_eth_dev *dev;
3221
3222         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3223
3224         dev = &rte_eth_devices[port_id];
3225
3226         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_all_hash_table_set, -ENOTSUP);
3227         return eth_err(port_id, (*dev->dev_ops->uc_all_hash_table_set)(dev,
3228                                                                        on));
3229 }
3230
3231 int rte_eth_set_queue_rate_limit(uint16_t port_id, uint16_t queue_idx,
3232                                         uint16_t tx_rate)
3233 {
3234         struct rte_eth_dev *dev;
3235         struct rte_eth_dev_info dev_info;
3236         struct rte_eth_link link;
3237
3238         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3239
3240         dev = &rte_eth_devices[port_id];
3241         rte_eth_dev_info_get(port_id, &dev_info);
3242         link = dev->data->dev_link;
3243
3244         if (queue_idx > dev_info.max_tx_queues) {
3245                 RTE_ETHDEV_LOG(ERR,
3246                         "Set queue rate limit:port %u: invalid queue id=%u\n",
3247                         port_id, queue_idx);
3248                 return -EINVAL;
3249         }
3250
3251         if (tx_rate > link.link_speed) {
3252                 RTE_ETHDEV_LOG(ERR,
3253                         "Set queue rate limit:invalid tx_rate=%u, bigger than link speed= %d\n",
3254                         tx_rate, link.link_speed);
3255                 return -EINVAL;
3256         }
3257
3258         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_queue_rate_limit, -ENOTSUP);
3259         return eth_err(port_id, (*dev->dev_ops->set_queue_rate_limit)(dev,
3260                                                         queue_idx, tx_rate));
3261 }
3262
3263 int
3264 rte_eth_mirror_rule_set(uint16_t port_id,
3265                         struct rte_eth_mirror_conf *mirror_conf,
3266                         uint8_t rule_id, uint8_t on)
3267 {
3268         struct rte_eth_dev *dev;
3269
3270         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3271         if (mirror_conf->rule_type == 0) {
3272                 RTE_ETHDEV_LOG(ERR, "Mirror rule type can not be 0\n");
3273                 return -EINVAL;
3274         }
3275
3276         if (mirror_conf->dst_pool >= ETH_64_POOLS) {
3277                 RTE_ETHDEV_LOG(ERR, "Invalid dst pool, pool id must be 0-%d\n",
3278                         ETH_64_POOLS - 1);
3279                 return -EINVAL;
3280         }
3281
3282         if ((mirror_conf->rule_type & (ETH_MIRROR_VIRTUAL_POOL_UP |
3283              ETH_MIRROR_VIRTUAL_POOL_DOWN)) &&
3284             (mirror_conf->pool_mask == 0)) {
3285                 RTE_ETHDEV_LOG(ERR,
3286                         "Invalid mirror pool, pool mask can not be 0\n");
3287                 return -EINVAL;
3288         }
3289
3290         if ((mirror_conf->rule_type & ETH_MIRROR_VLAN) &&
3291             mirror_conf->vlan.vlan_mask == 0) {
3292                 RTE_ETHDEV_LOG(ERR,
3293                         "Invalid vlan mask, vlan mask can not be 0\n");
3294                 return -EINVAL;
3295         }
3296
3297         dev = &rte_eth_devices[port_id];
3298         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mirror_rule_set, -ENOTSUP);
3299
3300         return eth_err(port_id, (*dev->dev_ops->mirror_rule_set)(dev,
3301                                                 mirror_conf, rule_id, on));
3302 }
3303
3304 int
3305 rte_eth_mirror_rule_reset(uint16_t port_id, uint8_t rule_id)
3306 {
3307         struct rte_eth_dev *dev;
3308
3309         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3310
3311         dev = &rte_eth_devices[port_id];
3312         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mirror_rule_reset, -ENOTSUP);
3313
3314         return eth_err(port_id, (*dev->dev_ops->mirror_rule_reset)(dev,
3315                                                                    rule_id));
3316 }
3317
3318 RTE_INIT(eth_dev_init_cb_lists)
3319 {
3320         int i;
3321
3322         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3323                 TAILQ_INIT(&rte_eth_devices[i].link_intr_cbs);
3324 }
3325
3326 int
3327 rte_eth_dev_callback_register(uint16_t port_id,
3328                         enum rte_eth_event_type event,
3329                         rte_eth_dev_cb_fn cb_fn, void *cb_arg)
3330 {
3331         struct rte_eth_dev *dev;
3332         struct rte_eth_dev_callback *user_cb;
3333         uint32_t next_port; /* size is 32-bit to prevent loop wrap-around */
3334         uint16_t last_port;
3335
3336         if (!cb_fn)
3337                 return -EINVAL;
3338
3339         if (!rte_eth_dev_is_valid_port(port_id) && port_id != RTE_ETH_ALL) {
3340                 RTE_ETHDEV_LOG(ERR, "Invalid port_id=%d\n", port_id);
3341                 return -EINVAL;
3342         }
3343
3344         if (port_id == RTE_ETH_ALL) {
3345                 next_port = 0;
3346                 last_port = RTE_MAX_ETHPORTS - 1;
3347         } else {
3348                 next_port = last_port = port_id;
3349         }
3350
3351         rte_spinlock_lock(&rte_eth_dev_cb_lock);
3352
3353         do {
3354                 dev = &rte_eth_devices[next_port];
3355
3356                 TAILQ_FOREACH(user_cb, &(dev->link_intr_cbs), next) {
3357                         if (user_cb->cb_fn == cb_fn &&
3358                                 user_cb->cb_arg == cb_arg &&
3359                                 user_cb->event == event) {
3360                                 break;
3361                         }
3362                 }
3363
3364                 /* create a new callback. */
3365                 if (user_cb == NULL) {
3366                         user_cb = rte_zmalloc("INTR_USER_CALLBACK",
3367                                 sizeof(struct rte_eth_dev_callback), 0);
3368                         if (user_cb != NULL) {
3369                                 user_cb->cb_fn = cb_fn;
3370                                 user_cb->cb_arg = cb_arg;
3371                                 user_cb->event = event;
3372                                 TAILQ_INSERT_TAIL(&(dev->link_intr_cbs),
3373                                                   user_cb, next);
3374                         } else {
3375                                 rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3376                                 rte_eth_dev_callback_unregister(port_id, event,
3377                                                                 cb_fn, cb_arg);
3378                                 return -ENOMEM;
3379                         }
3380
3381                 }
3382         } while (++next_port <= last_port);
3383
3384         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3385         return 0;
3386 }
3387
3388 int
3389 rte_eth_dev_callback_unregister(uint16_t port_id,
3390                         enum rte_eth_event_type event,
3391                         rte_eth_dev_cb_fn cb_fn, void *cb_arg)
3392 {
3393         int ret;
3394         struct rte_eth_dev *dev;
3395         struct rte_eth_dev_callback *cb, *next;
3396         uint32_t next_port; /* size is 32-bit to prevent loop wrap-around */
3397         uint16_t last_port;
3398
3399         if (!cb_fn)
3400                 return -EINVAL;
3401
3402         if (!rte_eth_dev_is_valid_port(port_id) && port_id != RTE_ETH_ALL) {
3403                 RTE_ETHDEV_LOG(ERR, "Invalid port_id=%d\n", port_id);
3404                 return -EINVAL;
3405         }
3406
3407         if (port_id == RTE_ETH_ALL) {
3408                 next_port = 0;
3409                 last_port = RTE_MAX_ETHPORTS - 1;
3410         } else {
3411                 next_port = last_port = port_id;
3412         }
3413
3414         rte_spinlock_lock(&rte_eth_dev_cb_lock);
3415
3416         do {
3417                 dev = &rte_eth_devices[next_port];
3418                 ret = 0;
3419                 for (cb = TAILQ_FIRST(&dev->link_intr_cbs); cb != NULL;
3420                      cb = next) {
3421
3422                         next = TAILQ_NEXT(cb, next);
3423
3424                         if (cb->cb_fn != cb_fn || cb->event != event ||
3425                             (cb->cb_arg != (void *)-1 && cb->cb_arg != cb_arg))
3426                                 continue;
3427
3428                         /*
3429                          * if this callback is not executing right now,
3430                          * then remove it.
3431                          */
3432                         if (cb->active == 0) {
3433                                 TAILQ_REMOVE(&(dev->link_intr_cbs), cb, next);
3434                                 rte_free(cb);
3435                         } else {
3436                                 ret = -EAGAIN;
3437                         }
3438                 }
3439         } while (++next_port <= last_port);
3440
3441         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3442         return ret;
3443 }
3444
3445 int
3446 _rte_eth_dev_callback_process(struct rte_eth_dev *dev,
3447         enum rte_eth_event_type event, void *ret_param)
3448 {
3449         struct rte_eth_dev_callback *cb_lst;
3450         struct rte_eth_dev_callback dev_cb;
3451         int rc = 0;
3452
3453         rte_spinlock_lock(&rte_eth_dev_cb_lock);
3454         TAILQ_FOREACH(cb_lst, &(dev->link_intr_cbs), next) {
3455                 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
3456                         continue;
3457                 dev_cb = *cb_lst;
3458                 cb_lst->active = 1;
3459                 if (ret_param != NULL)
3460                         dev_cb.ret_param = ret_param;
3461
3462                 rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3463                 rc = dev_cb.cb_fn(dev->data->port_id, dev_cb.event,
3464                                 dev_cb.cb_arg, dev_cb.ret_param);
3465                 rte_spinlock_lock(&rte_eth_dev_cb_lock);
3466                 cb_lst->active = 0;
3467         }
3468         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3469         return rc;
3470 }
3471
3472 void
3473 rte_eth_dev_probing_finish(struct rte_eth_dev *dev)
3474 {
3475         if (dev == NULL)
3476                 return;
3477
3478         _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_NEW, NULL);
3479
3480         dev->state = RTE_ETH_DEV_ATTACHED;
3481 }
3482
3483 int
3484 rte_eth_dev_rx_intr_ctl(uint16_t port_id, int epfd, int op, void *data)
3485 {
3486         uint32_t vec;
3487         struct rte_eth_dev *dev;
3488         struct rte_intr_handle *intr_handle;
3489         uint16_t qid;
3490         int rc;
3491
3492         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3493
3494         dev = &rte_eth_devices[port_id];
3495
3496         if (!dev->intr_handle) {
3497                 RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
3498                 return -ENOTSUP;
3499         }
3500
3501         intr_handle = dev->intr_handle;
3502         if (!intr_handle->intr_vec) {
3503                 RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
3504                 return -EPERM;
3505         }
3506
3507         for (qid = 0; qid < dev->data->nb_rx_queues; qid++) {
3508                 vec = intr_handle->intr_vec[qid];
3509                 rc = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
3510                 if (rc && rc != -EEXIST) {
3511                         RTE_ETHDEV_LOG(ERR,
3512                                 "p %u q %u rx ctl error op %d epfd %d vec %u\n",
3513                                 port_id, qid, op, epfd, vec);
3514                 }
3515         }
3516
3517         return 0;
3518 }
3519
3520 int __rte_experimental
3521 rte_eth_dev_rx_intr_ctl_q_get_fd(uint16_t port_id, uint16_t queue_id)
3522 {
3523         struct rte_intr_handle *intr_handle;
3524         struct rte_eth_dev *dev;
3525         unsigned int efd_idx;
3526         uint32_t vec;
3527         int fd;
3528
3529         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -1);
3530
3531         dev = &rte_eth_devices[port_id];
3532
3533         if (queue_id >= dev->data->nb_rx_queues) {
3534                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
3535                 return -1;
3536         }
3537
3538         if (!dev->intr_handle) {
3539                 RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
3540                 return -1;
3541         }
3542
3543         intr_handle = dev->intr_handle;
3544         if (!intr_handle->intr_vec) {
3545                 RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
3546                 return -1;
3547         }
3548
3549         vec = intr_handle->intr_vec[queue_id];
3550         efd_idx = (vec >= RTE_INTR_VEC_RXTX_OFFSET) ?
3551                 (vec - RTE_INTR_VEC_RXTX_OFFSET) : vec;
3552         fd = intr_handle->efds[efd_idx];
3553
3554         return fd;
3555 }
3556
3557 const struct rte_memzone *
3558 rte_eth_dma_zone_reserve(const struct rte_eth_dev *dev, const char *ring_name,
3559                          uint16_t queue_id, size_t size, unsigned align,
3560                          int socket_id)
3561 {
3562         char z_name[RTE_MEMZONE_NAMESIZE];
3563         const struct rte_memzone *mz;
3564
3565         snprintf(z_name, sizeof(z_name), "eth_p%d_q%d_%s",
3566                  dev->data->port_id, queue_id, ring_name);
3567
3568         mz = rte_memzone_lookup(z_name);
3569         if (mz)
3570                 return mz;
3571
3572         return rte_memzone_reserve_aligned(z_name, size, socket_id,
3573                         RTE_MEMZONE_IOVA_CONTIG, align);
3574 }
3575
3576 int __rte_experimental
3577 rte_eth_dev_create(struct rte_device *device, const char *name,
3578         size_t priv_data_size,
3579         ethdev_bus_specific_init ethdev_bus_specific_init,
3580         void *bus_init_params,
3581         ethdev_init_t ethdev_init, void *init_params)
3582 {
3583         struct rte_eth_dev *ethdev;
3584         int retval;
3585
3586         RTE_FUNC_PTR_OR_ERR_RET(*ethdev_init, -EINVAL);
3587
3588         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
3589                 ethdev = rte_eth_dev_allocate(name);
3590                 if (!ethdev)
3591                         return -ENODEV;
3592
3593                 if (priv_data_size) {
3594                         ethdev->data->dev_private = rte_zmalloc_socket(
3595                                 name, priv_data_size, RTE_CACHE_LINE_SIZE,
3596                                 device->numa_node);
3597
3598                         if (!ethdev->data->dev_private) {
3599                                 RTE_LOG(ERR, EAL, "failed to allocate private data");
3600                                 retval = -ENOMEM;
3601                                 goto probe_failed;
3602                         }
3603                 }
3604         } else {
3605                 ethdev = rte_eth_dev_attach_secondary(name);
3606                 if (!ethdev) {
3607                         RTE_LOG(ERR, EAL, "secondary process attach failed, "
3608                                 "ethdev doesn't exist");
3609                         return  -ENODEV;
3610                 }
3611         }
3612
3613         ethdev->device = device;
3614
3615         if (ethdev_bus_specific_init) {
3616                 retval = ethdev_bus_specific_init(ethdev, bus_init_params);
3617                 if (retval) {
3618                         RTE_LOG(ERR, EAL,
3619                                 "ethdev bus specific initialisation failed");
3620                         goto probe_failed;
3621                 }
3622         }
3623
3624         retval = ethdev_init(ethdev, init_params);
3625         if (retval) {
3626                 RTE_LOG(ERR, EAL, "ethdev initialisation failed");
3627                 goto probe_failed;
3628         }
3629
3630         rte_eth_dev_probing_finish(ethdev);
3631
3632         return retval;
3633
3634 probe_failed:
3635         rte_eth_dev_release_port(ethdev);
3636         return retval;
3637 }
3638
3639 int  __rte_experimental
3640 rte_eth_dev_destroy(struct rte_eth_dev *ethdev,
3641         ethdev_uninit_t ethdev_uninit)
3642 {
3643         int ret;
3644
3645         ethdev = rte_eth_dev_allocated(ethdev->data->name);
3646         if (!ethdev)
3647                 return -ENODEV;
3648
3649         RTE_FUNC_PTR_OR_ERR_RET(*ethdev_uninit, -EINVAL);
3650
3651         ret = ethdev_uninit(ethdev);
3652         if (ret)
3653                 return ret;
3654
3655         return rte_eth_dev_release_port(ethdev);
3656 }
3657
3658 int
3659 rte_eth_dev_rx_intr_ctl_q(uint16_t port_id, uint16_t queue_id,
3660                           int epfd, int op, void *data)
3661 {
3662         uint32_t vec;
3663         struct rte_eth_dev *dev;
3664         struct rte_intr_handle *intr_handle;
3665         int rc;
3666
3667         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3668
3669         dev = &rte_eth_devices[port_id];
3670         if (queue_id >= dev->data->nb_rx_queues) {
3671                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
3672                 return -EINVAL;
3673         }
3674
3675         if (!dev->intr_handle) {
3676                 RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
3677                 return -ENOTSUP;
3678         }
3679
3680         intr_handle = dev->intr_handle;
3681         if (!intr_handle->intr_vec) {
3682                 RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
3683                 return -EPERM;
3684         }
3685
3686         vec = intr_handle->intr_vec[queue_id];
3687         rc = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
3688         if (rc && rc != -EEXIST) {
3689                 RTE_ETHDEV_LOG(ERR,
3690                         "p %u q %u rx ctl error op %d epfd %d vec %u\n",
3691                         port_id, queue_id, op, epfd, vec);
3692                 return rc;
3693         }
3694
3695         return 0;
3696 }
3697
3698 int
3699 rte_eth_dev_rx_intr_enable(uint16_t port_id,
3700                            uint16_t queue_id)
3701 {
3702         struct rte_eth_dev *dev;
3703
3704         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3705
3706         dev = &rte_eth_devices[port_id];
3707
3708         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_intr_enable, -ENOTSUP);
3709         return eth_err(port_id, (*dev->dev_ops->rx_queue_intr_enable)(dev,
3710                                                                 queue_id));
3711 }
3712
3713 int
3714 rte_eth_dev_rx_intr_disable(uint16_t port_id,
3715                             uint16_t queue_id)
3716 {
3717         struct rte_eth_dev *dev;
3718
3719         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3720
3721         dev = &rte_eth_devices[port_id];
3722
3723         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_intr_disable, -ENOTSUP);
3724         return eth_err(port_id, (*dev->dev_ops->rx_queue_intr_disable)(dev,
3725                                                                 queue_id));
3726 }
3727
3728
3729 int
3730 rte_eth_dev_filter_supported(uint16_t port_id,
3731                              enum rte_filter_type filter_type)
3732 {
3733         struct rte_eth_dev *dev;
3734
3735         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3736
3737         dev = &rte_eth_devices[port_id];
3738         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->filter_ctrl, -ENOTSUP);
3739         return (*dev->dev_ops->filter_ctrl)(dev, filter_type,
3740                                 RTE_ETH_FILTER_NOP, NULL);
3741 }
3742
3743 int
3744 rte_eth_dev_filter_ctrl(uint16_t port_id, enum rte_filter_type filter_type,
3745                         enum rte_filter_op filter_op, void *arg)
3746 {
3747         struct rte_eth_dev *dev;
3748
3749         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3750
3751         dev = &rte_eth_devices[port_id];
3752         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->filter_ctrl, -ENOTSUP);
3753         return eth_err(port_id, (*dev->dev_ops->filter_ctrl)(dev, filter_type,
3754                                                              filter_op, arg));
3755 }
3756
3757 const struct rte_eth_rxtx_callback *
3758 rte_eth_add_rx_callback(uint16_t port_id, uint16_t queue_id,
3759                 rte_rx_callback_fn fn, void *user_param)
3760 {
3761 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3762         rte_errno = ENOTSUP;
3763         return NULL;
3764 #endif
3765         /* check input parameters */
3766         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
3767                     queue_id >= rte_eth_devices[port_id].data->nb_rx_queues) {
3768                 rte_errno = EINVAL;
3769                 return NULL;
3770         }
3771         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
3772
3773         if (cb == NULL) {
3774                 rte_errno = ENOMEM;
3775                 return NULL;
3776         }
3777
3778         cb->fn.rx = fn;
3779         cb->param = user_param;
3780
3781         rte_spinlock_lock(&rte_eth_rx_cb_lock);
3782         /* Add the callbacks in fifo order. */
3783         struct rte_eth_rxtx_callback *tail =
3784                 rte_eth_devices[port_id].post_rx_burst_cbs[queue_id];
3785
3786         if (!tail) {
3787                 rte_eth_devices[port_id].post_rx_burst_cbs[queue_id] = cb;
3788
3789         } else {
3790                 while (tail->next)
3791                         tail = tail->next;
3792                 tail->next = cb;
3793         }
3794         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
3795
3796         return cb;
3797 }
3798
3799 const struct rte_eth_rxtx_callback *
3800 rte_eth_add_first_rx_callback(uint16_t port_id, uint16_t queue_id,
3801                 rte_rx_callback_fn fn, void *user_param)
3802 {
3803 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3804         rte_errno = ENOTSUP;
3805         return NULL;
3806 #endif
3807         /* check input parameters */
3808         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
3809                 queue_id >= rte_eth_devices[port_id].data->nb_rx_queues) {
3810                 rte_errno = EINVAL;
3811                 return NULL;
3812         }
3813
3814         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
3815
3816         if (cb == NULL) {
3817                 rte_errno = ENOMEM;
3818                 return NULL;
3819         }
3820
3821         cb->fn.rx = fn;
3822         cb->param = user_param;
3823
3824         rte_spinlock_lock(&rte_eth_rx_cb_lock);
3825         /* Add the callbacks at fisrt position*/
3826         cb->next = rte_eth_devices[port_id].post_rx_burst_cbs[queue_id];
3827         rte_smp_wmb();
3828         rte_eth_devices[port_id].post_rx_burst_cbs[queue_id] = cb;
3829         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
3830
3831         return cb;
3832 }
3833
3834 const struct rte_eth_rxtx_callback *
3835 rte_eth_add_tx_callback(uint16_t port_id, uint16_t queue_id,
3836                 rte_tx_callback_fn fn, void *user_param)
3837 {
3838 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3839         rte_errno = ENOTSUP;
3840         return NULL;
3841 #endif
3842         /* check input parameters */
3843         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
3844                     queue_id >= rte_eth_devices[port_id].data->nb_tx_queues) {
3845                 rte_errno = EINVAL;
3846                 return NULL;
3847         }
3848
3849         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
3850
3851         if (cb == NULL) {
3852                 rte_errno = ENOMEM;
3853                 return NULL;
3854         }
3855
3856         cb->fn.tx = fn;
3857         cb->param = user_param;
3858
3859         rte_spinlock_lock(&rte_eth_tx_cb_lock);
3860         /* Add the callbacks in fifo order. */
3861         struct rte_eth_rxtx_callback *tail =
3862                 rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id];
3863
3864         if (!tail) {
3865                 rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id] = cb;
3866
3867         } else {
3868                 while (tail->next)
3869                         tail = tail->next;
3870                 tail->next = cb;
3871         }
3872         rte_spinlock_unlock(&rte_eth_tx_cb_lock);
3873
3874         return cb;
3875 }
3876
3877 int
3878 rte_eth_remove_rx_callback(uint16_t port_id, uint16_t queue_id,
3879                 const struct rte_eth_rxtx_callback *user_cb)
3880 {
3881 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3882         return -ENOTSUP;
3883 #endif
3884         /* Check input parameters. */
3885         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
3886         if (user_cb == NULL ||
3887                         queue_id >= rte_eth_devices[port_id].data->nb_rx_queues)
3888                 return -EINVAL;
3889
3890         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3891         struct rte_eth_rxtx_callback *cb;
3892         struct rte_eth_rxtx_callback **prev_cb;
3893         int ret = -EINVAL;
3894
3895         rte_spinlock_lock(&rte_eth_rx_cb_lock);
3896         prev_cb = &dev->post_rx_burst_cbs[queue_id];
3897         for (; *prev_cb != NULL; prev_cb = &cb->next) {
3898                 cb = *prev_cb;
3899                 if (cb == user_cb) {
3900                         /* Remove the user cb from the callback list. */
3901                         *prev_cb = cb->next;
3902                         ret = 0;
3903                         break;
3904                 }
3905         }
3906         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
3907
3908         return ret;
3909 }
3910
3911 int
3912 rte_eth_remove_tx_callback(uint16_t port_id, uint16_t queue_id,
3913                 const struct rte_eth_rxtx_callback *user_cb)
3914 {
3915 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3916         return -ENOTSUP;
3917 #endif
3918         /* Check input parameters. */
3919         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
3920         if (user_cb == NULL ||
3921                         queue_id >= rte_eth_devices[port_id].data->nb_tx_queues)
3922                 return -EINVAL;
3923
3924         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3925         int ret = -EINVAL;
3926         struct rte_eth_rxtx_callback *cb;
3927         struct rte_eth_rxtx_callback **prev_cb;
3928
3929         rte_spinlock_lock(&rte_eth_tx_cb_lock);
3930         prev_cb = &dev->pre_tx_burst_cbs[queue_id];
3931         for (; *prev_cb != NULL; prev_cb = &cb->next) {
3932                 cb = *prev_cb;
3933                 if (cb == user_cb) {
3934                         /* Remove the user cb from the callback list. */
3935                         *prev_cb = cb->next;
3936                         ret = 0;
3937                         break;
3938                 }
3939         }
3940         rte_spinlock_unlock(&rte_eth_tx_cb_lock);
3941
3942         return ret;
3943 }
3944
3945 int
3946 rte_eth_rx_queue_info_get(uint16_t port_id, uint16_t queue_id,
3947         struct rte_eth_rxq_info *qinfo)
3948 {
3949         struct rte_eth_dev *dev;
3950
3951         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3952
3953         if (qinfo == NULL)
3954                 return -EINVAL;
3955
3956         dev = &rte_eth_devices[port_id];
3957         if (queue_id >= dev->data->nb_rx_queues) {
3958                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
3959                 return -EINVAL;
3960         }
3961
3962         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rxq_info_get, -ENOTSUP);
3963
3964         memset(qinfo, 0, sizeof(*qinfo));
3965         dev->dev_ops->rxq_info_get(dev, queue_id, qinfo);
3966         return 0;
3967 }
3968
3969 int
3970 rte_eth_tx_queue_info_get(uint16_t port_id, uint16_t queue_id,
3971         struct rte_eth_txq_info *qinfo)
3972 {
3973         struct rte_eth_dev *dev;
3974
3975         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3976
3977         if (qinfo == NULL)
3978                 return -EINVAL;
3979
3980         dev = &rte_eth_devices[port_id];
3981         if (queue_id >= dev->data->nb_tx_queues) {
3982                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", queue_id);
3983                 return -EINVAL;
3984         }
3985
3986         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->txq_info_get, -ENOTSUP);
3987
3988         memset(qinfo, 0, sizeof(*qinfo));
3989         dev->dev_ops->txq_info_get(dev, queue_id, qinfo);
3990
3991         return 0;
3992 }
3993
3994 int
3995 rte_eth_dev_set_mc_addr_list(uint16_t port_id,
3996                              struct ether_addr *mc_addr_set,
3997                              uint32_t nb_mc_addr)
3998 {
3999         struct rte_eth_dev *dev;
4000
4001         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4002
4003         dev = &rte_eth_devices[port_id];
4004         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_mc_addr_list, -ENOTSUP);
4005         return eth_err(port_id, dev->dev_ops->set_mc_addr_list(dev,
4006                                                 mc_addr_set, nb_mc_addr));
4007 }
4008
4009 int
4010 rte_eth_timesync_enable(uint16_t port_id)
4011 {
4012         struct rte_eth_dev *dev;
4013
4014         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4015         dev = &rte_eth_devices[port_id];
4016
4017         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_enable, -ENOTSUP);
4018         return eth_err(port_id, (*dev->dev_ops->timesync_enable)(dev));
4019 }
4020
4021 int
4022 rte_eth_timesync_disable(uint16_t port_id)
4023 {
4024         struct rte_eth_dev *dev;
4025
4026         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4027         dev = &rte_eth_devices[port_id];
4028
4029         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_disable, -ENOTSUP);
4030         return eth_err(port_id, (*dev->dev_ops->timesync_disable)(dev));
4031 }
4032
4033 int
4034 rte_eth_timesync_read_rx_timestamp(uint16_t port_id, struct timespec *timestamp,
4035                                    uint32_t flags)
4036 {
4037         struct rte_eth_dev *dev;
4038
4039         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4040         dev = &rte_eth_devices[port_id];
4041
4042         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_rx_timestamp, -ENOTSUP);
4043         return eth_err(port_id, (*dev->dev_ops->timesync_read_rx_timestamp)
4044                                 (dev, timestamp, flags));
4045 }
4046
4047 int
4048 rte_eth_timesync_read_tx_timestamp(uint16_t port_id,
4049                                    struct timespec *timestamp)
4050 {
4051         struct rte_eth_dev *dev;
4052
4053         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4054         dev = &rte_eth_devices[port_id];
4055
4056         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_tx_timestamp, -ENOTSUP);
4057         return eth_err(port_id, (*dev->dev_ops->timesync_read_tx_timestamp)
4058                                 (dev, timestamp));
4059 }
4060
4061 int
4062 rte_eth_timesync_adjust_time(uint16_t port_id, int64_t delta)
4063 {
4064         struct rte_eth_dev *dev;
4065
4066         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4067         dev = &rte_eth_devices[port_id];
4068
4069         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_adjust_time, -ENOTSUP);
4070         return eth_err(port_id, (*dev->dev_ops->timesync_adjust_time)(dev,
4071                                                                       delta));
4072 }
4073
4074 int
4075 rte_eth_timesync_read_time(uint16_t port_id, struct timespec *timestamp)
4076 {
4077         struct rte_eth_dev *dev;
4078
4079         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4080         dev = &rte_eth_devices[port_id];
4081
4082         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_time, -ENOTSUP);
4083         return eth_err(port_id, (*dev->dev_ops->timesync_read_time)(dev,
4084                                                                 timestamp));
4085 }
4086
4087 int
4088 rte_eth_timesync_write_time(uint16_t port_id, const struct timespec *timestamp)
4089 {
4090         struct rte_eth_dev *dev;
4091
4092         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4093         dev = &rte_eth_devices[port_id];
4094
4095         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_write_time, -ENOTSUP);
4096         return eth_err(port_id, (*dev->dev_ops->timesync_write_time)(dev,
4097                                                                 timestamp));
4098 }
4099
4100 int
4101 rte_eth_dev_get_reg_info(uint16_t port_id, struct rte_dev_reg_info *info)
4102 {
4103         struct rte_eth_dev *dev;
4104
4105         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4106
4107         dev = &rte_eth_devices[port_id];
4108         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_reg, -ENOTSUP);
4109         return eth_err(port_id, (*dev->dev_ops->get_reg)(dev, info));
4110 }
4111
4112 int
4113 rte_eth_dev_get_eeprom_length(uint16_t port_id)
4114 {
4115         struct rte_eth_dev *dev;
4116
4117         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4118
4119         dev = &rte_eth_devices[port_id];
4120         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom_length, -ENOTSUP);
4121         return eth_err(port_id, (*dev->dev_ops->get_eeprom_length)(dev));
4122 }
4123
4124 int
4125 rte_eth_dev_get_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info)
4126 {
4127         struct rte_eth_dev *dev;
4128
4129         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4130
4131         dev = &rte_eth_devices[port_id];
4132         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom, -ENOTSUP);
4133         return eth_err(port_id, (*dev->dev_ops->get_eeprom)(dev, info));
4134 }
4135
4136 int
4137 rte_eth_dev_set_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info)
4138 {
4139         struct rte_eth_dev *dev;
4140
4141         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4142
4143         dev = &rte_eth_devices[port_id];
4144         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_eeprom, -ENOTSUP);
4145         return eth_err(port_id, (*dev->dev_ops->set_eeprom)(dev, info));
4146 }
4147
4148 int __rte_experimental
4149 rte_eth_dev_get_module_info(uint16_t port_id,
4150                             struct rte_eth_dev_module_info *modinfo)
4151 {
4152         struct rte_eth_dev *dev;
4153
4154         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4155
4156         dev = &rte_eth_devices[port_id];
4157         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_module_info, -ENOTSUP);
4158         return (*dev->dev_ops->get_module_info)(dev, modinfo);
4159 }
4160
4161 int __rte_experimental
4162 rte_eth_dev_get_module_eeprom(uint16_t port_id,
4163                               struct rte_dev_eeprom_info *info)
4164 {
4165         struct rte_eth_dev *dev;
4166
4167         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4168
4169         dev = &rte_eth_devices[port_id];
4170         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_module_eeprom, -ENOTSUP);
4171         return (*dev->dev_ops->get_module_eeprom)(dev, info);
4172 }
4173
4174 int
4175 rte_eth_dev_get_dcb_info(uint16_t port_id,
4176                              struct rte_eth_dcb_info *dcb_info)
4177 {
4178         struct rte_eth_dev *dev;
4179
4180         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4181
4182         dev = &rte_eth_devices[port_id];
4183         memset(dcb_info, 0, sizeof(struct rte_eth_dcb_info));
4184
4185         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_dcb_info, -ENOTSUP);
4186         return eth_err(port_id, (*dev->dev_ops->get_dcb_info)(dev, dcb_info));
4187 }
4188
4189 int
4190 rte_eth_dev_l2_tunnel_eth_type_conf(uint16_t port_id,
4191                                     struct rte_eth_l2_tunnel_conf *l2_tunnel)
4192 {
4193         struct rte_eth_dev *dev;
4194
4195         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4196         if (l2_tunnel == NULL) {
4197                 RTE_ETHDEV_LOG(ERR, "Invalid l2_tunnel parameter\n");
4198                 return -EINVAL;
4199         }
4200
4201         if (l2_tunnel->l2_tunnel_type >= RTE_TUNNEL_TYPE_MAX) {
4202                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
4203                 return -EINVAL;
4204         }
4205
4206         dev = &rte_eth_devices[port_id];
4207         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->l2_tunnel_eth_type_conf,
4208                                 -ENOTSUP);
4209         return eth_err(port_id, (*dev->dev_ops->l2_tunnel_eth_type_conf)(dev,
4210                                                                 l2_tunnel));
4211 }
4212
4213 int
4214 rte_eth_dev_l2_tunnel_offload_set(uint16_t port_id,
4215                                   struct rte_eth_l2_tunnel_conf *l2_tunnel,
4216                                   uint32_t mask,
4217                                   uint8_t en)
4218 {
4219         struct rte_eth_dev *dev;
4220
4221         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4222
4223         if (l2_tunnel == NULL) {
4224                 RTE_ETHDEV_LOG(ERR, "Invalid l2_tunnel parameter\n");
4225                 return -EINVAL;
4226         }
4227
4228         if (l2_tunnel->l2_tunnel_type >= RTE_TUNNEL_TYPE_MAX) {
4229                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
4230                 return -EINVAL;
4231         }
4232
4233         if (mask == 0) {
4234                 RTE_ETHDEV_LOG(ERR, "Mask should have a value\n");
4235                 return -EINVAL;
4236         }
4237
4238         dev = &rte_eth_devices[port_id];
4239         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->l2_tunnel_offload_set,
4240                                 -ENOTSUP);
4241         return eth_err(port_id, (*dev->dev_ops->l2_tunnel_offload_set)(dev,
4242                                                         l2_tunnel, mask, en));
4243 }
4244
4245 static void
4246 rte_eth_dev_adjust_nb_desc(uint16_t *nb_desc,
4247                            const struct rte_eth_desc_lim *desc_lim)
4248 {
4249         if (desc_lim->nb_align != 0)
4250                 *nb_desc = RTE_ALIGN_CEIL(*nb_desc, desc_lim->nb_align);
4251
4252         if (desc_lim->nb_max != 0)
4253                 *nb_desc = RTE_MIN(*nb_desc, desc_lim->nb_max);
4254
4255         *nb_desc = RTE_MAX(*nb_desc, desc_lim->nb_min);
4256 }
4257
4258 int
4259 rte_eth_dev_adjust_nb_rx_tx_desc(uint16_t port_id,
4260                                  uint16_t *nb_rx_desc,
4261                                  uint16_t *nb_tx_desc)
4262 {
4263         struct rte_eth_dev *dev;
4264         struct rte_eth_dev_info dev_info;
4265
4266         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4267
4268         dev = &rte_eth_devices[port_id];
4269         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
4270
4271         rte_eth_dev_info_get(port_id, &dev_info);
4272
4273         if (nb_rx_desc != NULL)
4274                 rte_eth_dev_adjust_nb_desc(nb_rx_desc, &dev_info.rx_desc_lim);
4275
4276         if (nb_tx_desc != NULL)
4277                 rte_eth_dev_adjust_nb_desc(nb_tx_desc, &dev_info.tx_desc_lim);
4278
4279         return 0;
4280 }
4281
4282 int
4283 rte_eth_dev_pool_ops_supported(uint16_t port_id, const char *pool)
4284 {
4285         struct rte_eth_dev *dev;
4286
4287         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4288
4289         if (pool == NULL)
4290                 return -EINVAL;
4291
4292         dev = &rte_eth_devices[port_id];
4293
4294         if (*dev->dev_ops->pool_ops_supported == NULL)
4295                 return 1; /* all pools are supported */
4296
4297         return (*dev->dev_ops->pool_ops_supported)(dev, pool);
4298 }
4299
4300 /**
4301  * A set of values to describe the possible states of a switch domain.
4302  */
4303 enum rte_eth_switch_domain_state {
4304         RTE_ETH_SWITCH_DOMAIN_UNUSED = 0,
4305         RTE_ETH_SWITCH_DOMAIN_ALLOCATED
4306 };
4307
4308 /**
4309  * Array of switch domains available for allocation. Array is sized to
4310  * RTE_MAX_ETHPORTS elements as there cannot be more active switch domains than
4311  * ethdev ports in a single process.
4312  */
4313 static struct rte_eth_dev_switch {
4314         enum rte_eth_switch_domain_state state;
4315 } rte_eth_switch_domains[RTE_MAX_ETHPORTS];
4316
4317 int __rte_experimental
4318 rte_eth_switch_domain_alloc(uint16_t *domain_id)
4319 {
4320         unsigned int i;
4321
4322         *domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
4323
4324         for (i = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID + 1;
4325                 i < RTE_MAX_ETHPORTS; i++) {
4326                 if (rte_eth_switch_domains[i].state ==
4327                         RTE_ETH_SWITCH_DOMAIN_UNUSED) {
4328                         rte_eth_switch_domains[i].state =
4329                                 RTE_ETH_SWITCH_DOMAIN_ALLOCATED;
4330                         *domain_id = i;
4331                         return 0;
4332                 }
4333         }
4334
4335         return -ENOSPC;
4336 }
4337
4338 int __rte_experimental
4339 rte_eth_switch_domain_free(uint16_t domain_id)
4340 {
4341         if (domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID ||
4342                 domain_id >= RTE_MAX_ETHPORTS)
4343                 return -EINVAL;
4344
4345         if (rte_eth_switch_domains[domain_id].state !=
4346                 RTE_ETH_SWITCH_DOMAIN_ALLOCATED)
4347                 return -EINVAL;
4348
4349         rte_eth_switch_domains[domain_id].state = RTE_ETH_SWITCH_DOMAIN_UNUSED;
4350
4351         return 0;
4352 }
4353
4354 static int
4355 rte_eth_devargs_tokenise(struct rte_kvargs *arglist, const char *str_in)
4356 {
4357         int state;
4358         struct rte_kvargs_pair *pair;
4359         char *letter;
4360
4361         arglist->str = strdup(str_in);
4362         if (arglist->str == NULL)
4363                 return -ENOMEM;
4364
4365         letter = arglist->str;
4366         state = 0;
4367         arglist->count = 0;
4368         pair = &arglist->pairs[0];
4369         while (1) {
4370                 switch (state) {
4371                 case 0: /* Initial */
4372                         if (*letter == '=')
4373                                 return -EINVAL;
4374                         else if (*letter == '\0')
4375                                 return 0;
4376
4377                         state = 1;
4378                         pair->key = letter;
4379                         /* fall-thru */
4380
4381                 case 1: /* Parsing key */
4382                         if (*letter == '=') {
4383                                 *letter = '\0';
4384                                 pair->value = letter + 1;
4385                                 state = 2;
4386                         } else if (*letter == ',' || *letter == '\0')
4387                                 return -EINVAL;
4388                         break;
4389
4390
4391                 case 2: /* Parsing value */
4392                         if (*letter == '[')
4393                                 state = 3;
4394                         else if (*letter == ',') {
4395                                 *letter = '\0';
4396                                 arglist->count++;
4397                                 pair = &arglist->pairs[arglist->count];
4398                                 state = 0;
4399                         } else if (*letter == '\0') {
4400                                 letter--;
4401                                 arglist->count++;
4402                                 pair = &arglist->pairs[arglist->count];
4403                                 state = 0;
4404                         }
4405                         break;
4406
4407                 case 3: /* Parsing list */
4408                         if (*letter == ']')
4409                                 state = 2;
4410                         else if (*letter == '\0')
4411                                 return -EINVAL;
4412                         break;
4413                 }
4414                 letter++;
4415         }
4416 }
4417
4418 int __rte_experimental
4419 rte_eth_devargs_parse(const char *dargs, struct rte_eth_devargs *eth_da)
4420 {
4421         struct rte_kvargs args;
4422         struct rte_kvargs_pair *pair;
4423         unsigned int i;
4424         int result = 0;
4425
4426         memset(eth_da, 0, sizeof(*eth_da));
4427
4428         result = rte_eth_devargs_tokenise(&args, dargs);
4429         if (result < 0)
4430                 goto parse_cleanup;
4431
4432         for (i = 0; i < args.count; i++) {
4433                 pair = &args.pairs[i];
4434                 if (strcmp("representor", pair->key) == 0) {
4435                         result = rte_eth_devargs_parse_list(pair->value,
4436                                 rte_eth_devargs_parse_representor_ports,
4437                                 eth_da);
4438                         if (result < 0)
4439                                 goto parse_cleanup;
4440                 }
4441         }
4442
4443 parse_cleanup:
4444         if (args.str)
4445                 free(args.str);
4446
4447         return result;
4448 }
4449
4450 RTE_INIT(ethdev_init_log)
4451 {
4452         rte_eth_dev_logtype = rte_log_register("lib.ethdev");
4453         if (rte_eth_dev_logtype >= 0)
4454                 rte_log_set_level(rte_eth_dev_logtype, RTE_LOG_INFO);
4455 }