lib/librte_ethdev/rte_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <sys/types.h>
   6 #include <sys/queue.h>
   7 #include <ctype.h>
   8 #include <stdio.h>
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <stdarg.h>
  12 #include <errno.h>
  13 #include <stdbool.h>
  14 #include <stdint.h>
  15 #include <inttypes.h>
  16 #include <netinet/in.h>
  17
  18 #include <rte_byteorder.h>
  19 #include <rte_log.h>
  20 #include <rte_debug.h>
  21 #include <rte_interrupts.h>
  22 #include <rte_memory.h>
  23 #include <rte_memcpy.h>
  24 #include <rte_memzone.h>
  25 #include <rte_launch.h>
  26 #include <rte_eal.h>
  27 #include <rte_per_lcore.h>
  28 #include <rte_lcore.h>
  29 #include <rte_atomic.h>
  30 #include <rte_branch_prediction.h>
  31 #include <rte_common.h>
  32 #include <rte_mempool.h>
  33 #include <rte_malloc.h>
  34 #include <rte_mbuf.h>
  35 #include <rte_errno.h>
  36 #include <rte_spinlock.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_kvargs.h>
  39 #include <rte_class.h>
  40 #include <rte_ether.h>
  41 #include <rte_telemetry.h>
  42
  43 #include "rte_ethdev_trace.h"
  44 #include "rte_ethdev.h"
  45 #include "rte_ethdev_driver.h"
  46 #include "ethdev_profile.h"
  47 #include "ethdev_private.h"
  48
  49 static const char *MZ_RTE_ETH_DEV_DATA = "rte_eth_dev_data";
  50 struct rte_eth_dev rte_eth_devices[RTE_MAX_ETHPORTS];
  51
  52 /* spinlock for eth device callbacks */
  53 static rte_spinlock_t rte_eth_dev_cb_lock = RTE_SPINLOCK_INITIALIZER;
  54
  55 /* spinlock for add/remove rx callbacks */
  56 static rte_spinlock_t rte_eth_rx_cb_lock = RTE_SPINLOCK_INITIALIZER;
  57
  58 /* spinlock for add/remove tx callbacks */
  59 static rte_spinlock_t rte_eth_tx_cb_lock = RTE_SPINLOCK_INITIALIZER;
  60
  61 /* spinlock for shared data allocation */
  62 static rte_spinlock_t rte_eth_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
  63
  64 /* store statistics names and its offset in stats structure  */
  65 struct rte_eth_xstats_name_off {
  66         char name[RTE_ETH_XSTATS_NAME_SIZE];
  67         unsigned offset;
  68 };
  69
  70 /* Shared memory between primary and secondary processes. */
  71 static struct {
  72         uint64_t next_owner_id;
  73         rte_spinlock_t ownership_lock;
  74         struct rte_eth_dev_data data[RTE_MAX_ETHPORTS];
  75 } *rte_eth_dev_shared_data;
  76
  77 static const struct rte_eth_xstats_name_off rte_stats_strings[] = {
  78         {"rx_good_packets", offsetof(struct rte_eth_stats, ipackets)},
  79         {"tx_good_packets", offsetof(struct rte_eth_stats, opackets)},
  80         {"rx_good_bytes", offsetof(struct rte_eth_stats, ibytes)},
  81         {"tx_good_bytes", offsetof(struct rte_eth_stats, obytes)},
  82         {"rx_missed_errors", offsetof(struct rte_eth_stats, imissed)},
  83         {"rx_errors", offsetof(struct rte_eth_stats, ierrors)},
  84         {"tx_errors", offsetof(struct rte_eth_stats, oerrors)},
  85         {"rx_mbuf_allocation_errors", offsetof(struct rte_eth_stats,
  86                 rx_nombuf)},
  87 };
  88
  89 #define RTE_NB_STATS RTE_DIM(rte_stats_strings)
  90
  91 static const struct rte_eth_xstats_name_off rte_rxq_stats_strings[] = {
  92         {"packets", offsetof(struct rte_eth_stats, q_ipackets)},
  93         {"bytes", offsetof(struct rte_eth_stats, q_ibytes)},
  94         {"errors", offsetof(struct rte_eth_stats, q_errors)},
  95 };
  96
  97 #define RTE_NB_RXQ_STATS RTE_DIM(rte_rxq_stats_strings)
  98
  99 static const struct rte_eth_xstats_name_off rte_txq_stats_strings[] = {
 100         {"packets", offsetof(struct rte_eth_stats, q_opackets)},
 101         {"bytes", offsetof(struct rte_eth_stats, q_obytes)},
 102 };
 103 #define RTE_NB_TXQ_STATS RTE_DIM(rte_txq_stats_strings)
 104
 105 #define RTE_RX_OFFLOAD_BIT2STR(_name)   \
 106         { DEV_RX_OFFLOAD_##_name, #_name }
 107
 108 static const struct {
 109         uint64_t offload;
 110         const char *name;
 111 } rte_rx_offload_names[] = {
 112         RTE_RX_OFFLOAD_BIT2STR(VLAN_STRIP),
 113         RTE_RX_OFFLOAD_BIT2STR(IPV4_CKSUM),
 114         RTE_RX_OFFLOAD_BIT2STR(UDP_CKSUM),
 115         RTE_RX_OFFLOAD_BIT2STR(TCP_CKSUM),
 116         RTE_RX_OFFLOAD_BIT2STR(TCP_LRO),
 117         RTE_RX_OFFLOAD_BIT2STR(QINQ_STRIP),
 118         RTE_RX_OFFLOAD_BIT2STR(OUTER_IPV4_CKSUM),
 119         RTE_RX_OFFLOAD_BIT2STR(MACSEC_STRIP),
 120         RTE_RX_OFFLOAD_BIT2STR(HEADER_SPLIT),
 121         RTE_RX_OFFLOAD_BIT2STR(VLAN_FILTER),
 122         RTE_RX_OFFLOAD_BIT2STR(VLAN_EXTEND),
 123         RTE_RX_OFFLOAD_BIT2STR(JUMBO_FRAME),
 124         RTE_RX_OFFLOAD_BIT2STR(SCATTER),
 125         RTE_RX_OFFLOAD_BIT2STR(TIMESTAMP),
 126         RTE_RX_OFFLOAD_BIT2STR(SECURITY),
 127         RTE_RX_OFFLOAD_BIT2STR(KEEP_CRC),
 128         RTE_RX_OFFLOAD_BIT2STR(SCTP_CKSUM),
 129         RTE_RX_OFFLOAD_BIT2STR(OUTER_UDP_CKSUM),
 130         RTE_RX_OFFLOAD_BIT2STR(RSS_HASH),
 131 };
 132
 133 #undef RTE_RX_OFFLOAD_BIT2STR
 134
 135 #define RTE_TX_OFFLOAD_BIT2STR(_name)   \
 136         { DEV_TX_OFFLOAD_##_name, #_name }
 137
 138 static const struct {
 139         uint64_t offload;
 140         const char *name;
 141 } rte_tx_offload_names[] = {
 142         RTE_TX_OFFLOAD_BIT2STR(VLAN_INSERT),
 143         RTE_TX_OFFLOAD_BIT2STR(IPV4_CKSUM),
 144         RTE_TX_OFFLOAD_BIT2STR(UDP_CKSUM),
 145         RTE_TX_OFFLOAD_BIT2STR(TCP_CKSUM),
 146         RTE_TX_OFFLOAD_BIT2STR(SCTP_CKSUM),
 147         RTE_TX_OFFLOAD_BIT2STR(TCP_TSO),
 148         RTE_TX_OFFLOAD_BIT2STR(UDP_TSO),
 149         RTE_TX_OFFLOAD_BIT2STR(OUTER_IPV4_CKSUM),
 150         RTE_TX_OFFLOAD_BIT2STR(QINQ_INSERT),
 151         RTE_TX_OFFLOAD_BIT2STR(VXLAN_TNL_TSO),
 152         RTE_TX_OFFLOAD_BIT2STR(GRE_TNL_TSO),
 153         RTE_TX_OFFLOAD_BIT2STR(IPIP_TNL_TSO),
 154         RTE_TX_OFFLOAD_BIT2STR(GENEVE_TNL_TSO),
 155         RTE_TX_OFFLOAD_BIT2STR(MACSEC_INSERT),
 156         RTE_TX_OFFLOAD_BIT2STR(MT_LOCKFREE),
 157         RTE_TX_OFFLOAD_BIT2STR(MULTI_SEGS),
 158         RTE_TX_OFFLOAD_BIT2STR(MBUF_FAST_FREE),
 159         RTE_TX_OFFLOAD_BIT2STR(SECURITY),
 160         RTE_TX_OFFLOAD_BIT2STR(UDP_TNL_TSO),
 161         RTE_TX_OFFLOAD_BIT2STR(IP_TNL_TSO),
 162         RTE_TX_OFFLOAD_BIT2STR(OUTER_UDP_CKSUM),
 163         RTE_TX_OFFLOAD_BIT2STR(SEND_ON_TIMESTAMP),
 164 };
 165
 166 #undef RTE_TX_OFFLOAD_BIT2STR
 167
 168 /**
 169  * The user application callback description.
 170  *
 171  * It contains callback address to be registered by user application,
 172  * the pointer to the parameters for callback, and the event type.
 173  */
 174 struct rte_eth_dev_callback {
 175         TAILQ_ENTRY(rte_eth_dev_callback) next; /**< Callbacks list */
 176         rte_eth_dev_cb_fn cb_fn;                /**< Callback address */
 177         void *cb_arg;                           /**< Parameter for callback */
 178         void *ret_param;                        /**< Return parameter */
 179         enum rte_eth_event_type event;          /**< Interrupt event type */
 180         uint32_t active;                        /**< Callback is executing */
 181 };
 182
 183 enum {
 184         STAT_QMAP_TX = 0,
 185         STAT_QMAP_RX
 186 };
 187
 188 int
 189 rte_eth_iterator_init(struct rte_dev_iterator *iter, const char *devargs_str)
 190 {
 191         int ret;
 192         struct rte_devargs devargs = {.args = NULL};
 193         const char *bus_param_key;
 194         char *bus_str = NULL;
 195         char *cls_str = NULL;
 196         int str_size;
 197
 198         memset(iter, 0, sizeof(*iter));
 199
 200         /*
 201          * The devargs string may use various syntaxes:
 202          *   - 0000:08:00.0,representor=[1-3]
 203          *   - pci:0000:06:00.0,representor=[0,5]
 204          *   - class=eth,mac=00:11:22:33:44:55
 205          * A new syntax is in development (not yet supported):
 206          *   - bus=X,paramX=x/class=Y,paramY=y/driver=Z,paramZ=z
 207          */
 208
 209         /*
 210          * Handle pure class filter (i.e. without any bus-level argument),
 211          * from future new syntax.
 212          * rte_devargs_parse() is not yet supporting the new syntax,
 213          * that's why this simple case is temporarily parsed here.
 214          */
 215 #define iter_anybus_str "class=eth,"
 216         if (strncmp(devargs_str, iter_anybus_str,
 217                         strlen(iter_anybus_str)) == 0) {
 218                 iter->cls_str = devargs_str + strlen(iter_anybus_str);
 219                 goto end;
 220         }
 221
 222         /* Split bus, device and parameters. */
 223         ret = rte_devargs_parse(&devargs, devargs_str);
 224         if (ret != 0)
 225                 goto error;
 226
 227         /*
 228          * Assume parameters of old syntax can match only at ethdev level.
 229          * Extra parameters will be ignored, thanks to "+" prefix.
 230          */
 231         str_size = strlen(devargs.args) + 2;
 232         cls_str = malloc(str_size);
 233         if (cls_str == NULL) {
 234                 ret = -ENOMEM;
 235                 goto error;
 236         }
 237         ret = snprintf(cls_str, str_size, "+%s", devargs.args);
 238         if (ret != str_size - 1) {
 239                 ret = -EINVAL;
 240                 goto error;
 241         }
 242         iter->cls_str = cls_str;
 243         free(devargs.args); /* allocated by rte_devargs_parse() */
 244         devargs.args = NULL;
 245
 246         iter->bus = devargs.bus;
 247         if (iter->bus->dev_iterate == NULL) {
 248                 ret = -ENOTSUP;
 249                 goto error;
 250         }
 251
 252         /* Convert bus args to new syntax for use with new API dev_iterate. */
 253         if (strcmp(iter->bus->name, "vdev") == 0) {
 254                 bus_param_key = "name";
 255         } else if (strcmp(iter->bus->name, "pci") == 0) {
 256                 bus_param_key = "addr";
 257         } else {
 258                 ret = -ENOTSUP;
 259                 goto error;
 260         }
 261         str_size = strlen(bus_param_key) + strlen(devargs.name) + 2;
 262         bus_str = malloc(str_size);
 263         if (bus_str == NULL) {
 264                 ret = -ENOMEM;
 265                 goto error;
 266         }
 267         ret = snprintf(bus_str, str_size, "%s=%s",
 268                         bus_param_key, devargs.name);
 269         if (ret != str_size - 1) {
 270                 ret = -EINVAL;
 271                 goto error;
 272         }
 273         iter->bus_str = bus_str;
 274
 275 end:
 276         iter->cls = rte_class_find_by_name("eth");
 277         return 0;
 278
 279 error:
 280         if (ret == -ENOTSUP)
 281                 RTE_ETHDEV_LOG(ERR, "Bus %s does not support iterating.\n",
 282                                 iter->bus->name);
 283         free(devargs.args);
 284         free(bus_str);
 285         free(cls_str);
 286         return ret;
 287 }
 288
 289 uint16_t
 290 rte_eth_iterator_next(struct rte_dev_iterator *iter)
 291 {
 292         if (iter->cls == NULL) /* invalid ethdev iterator */
 293                 return RTE_MAX_ETHPORTS;
 294
 295         do { /* loop to try all matching rte_device */
 296                 /* If not pure ethdev filter and */
 297                 if (iter->bus != NULL &&
 298                                 /* not in middle of rte_eth_dev iteration, */
 299                                 iter->class_device == NULL) {
 300                         /* get next rte_device to try. */
 301                         iter->device = iter->bus->dev_iterate(
 302                                         iter->device, iter->bus_str, iter);
 303                         if (iter->device == NULL)
 304                                 break; /* no more rte_device candidate */
 305                 }
 306                 /* A device is matching bus part, need to check ethdev part. */
 307                 iter->class_device = iter->cls->dev_iterate(
 308                                 iter->class_device, iter->cls_str, iter);
 309                 if (iter->class_device != NULL)
 310                         return eth_dev_to_id(iter->class_device); /* match */
 311         } while (iter->bus != NULL); /* need to try next rte_device */
 312
 313         /* No more ethdev port to iterate. */
 314         rte_eth_iterator_cleanup(iter);
 315         return RTE_MAX_ETHPORTS;
 316 }
 317
 318 void
 319 rte_eth_iterator_cleanup(struct rte_dev_iterator *iter)
 320 {
 321         if (iter->bus_str == NULL)
 322                 return; /* nothing to free in pure class filter */
 323         free(RTE_CAST_FIELD(iter, bus_str, char *)); /* workaround const */
 324         free(RTE_CAST_FIELD(iter, cls_str, char *)); /* workaround const */
 325         memset(iter, 0, sizeof(*iter));
 326 }
 327
 328 uint16_t
 329 rte_eth_find_next(uint16_t port_id)
 330 {
 331         while (port_id < RTE_MAX_ETHPORTS &&
 332                         rte_eth_devices[port_id].state == RTE_ETH_DEV_UNUSED)
 333                 port_id++;
 334
 335         if (port_id >= RTE_MAX_ETHPORTS)
 336                 return RTE_MAX_ETHPORTS;
 337
 338         return port_id;
 339 }
 340
 341 /*
 342  * Macro to iterate over all valid ports for internal usage.
 343  * Note: RTE_ETH_FOREACH_DEV is different because filtering owned ports.
 344  */
 345 #define RTE_ETH_FOREACH_VALID_DEV(port_id) \
 346         for (port_id = rte_eth_find_next(0); \
 347              port_id < RTE_MAX_ETHPORTS; \
 348              port_id = rte_eth_find_next(port_id + 1))
 349
 350 uint16_t
 351 rte_eth_find_next_of(uint16_t port_id, const struct rte_device *parent)
 352 {
 353         port_id = rte_eth_find_next(port_id);
 354         while (port_id < RTE_MAX_ETHPORTS &&
 355                         rte_eth_devices[port_id].device != parent)
 356                 port_id = rte_eth_find_next(port_id + 1);
 357
 358         return port_id;
 359 }
 360
 361 uint16_t
 362 rte_eth_find_next_sibling(uint16_t port_id, uint16_t ref_port_id)
 363 {
 364         RTE_ETH_VALID_PORTID_OR_ERR_RET(ref_port_id, RTE_MAX_ETHPORTS);
 365         return rte_eth_find_next_of(port_id,
 366                         rte_eth_devices[ref_port_id].device);
 367 }
 368
 369 static void
 370 rte_eth_dev_shared_data_prepare(void)
 371 {
 372         const unsigned flags = 0;
 373         const struct rte_memzone *mz;
 374
 375         rte_spinlock_lock(&rte_eth_shared_data_lock);
 376
 377         if (rte_eth_dev_shared_data == NULL) {
 378                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 379                         /* Allocate port data and ownership shared memory. */
 380                         mz = rte_memzone_reserve(MZ_RTE_ETH_DEV_DATA,
 381                                         sizeof(*rte_eth_dev_shared_data),
 382                                         rte_socket_id(), flags);
 383                 } else
 384                         mz = rte_memzone_lookup(MZ_RTE_ETH_DEV_DATA);
 385                 if (mz == NULL)
 386                         rte_panic("Cannot allocate ethdev shared data\n");
 387
 388                 rte_eth_dev_shared_data = mz->addr;
 389                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 390                         rte_eth_dev_shared_data->next_owner_id =
 391                                         RTE_ETH_DEV_NO_OWNER + 1;
 392                         rte_spinlock_init(&rte_eth_dev_shared_data->ownership_lock);
 393                         memset(rte_eth_dev_shared_data->data, 0,
 394                                sizeof(rte_eth_dev_shared_data->data));
 395                 }
 396         }
 397
 398         rte_spinlock_unlock(&rte_eth_shared_data_lock);
 399 }
 400
 401 static bool
 402 is_allocated(const struct rte_eth_dev *ethdev)
 403 {
 404         return ethdev->data->name[0] != '\0';
 405 }
 406
 407 static struct rte_eth_dev *
 408 _rte_eth_dev_allocated(const char *name)
 409 {
 410         unsigned i;
 411
 412         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 413                 if (rte_eth_devices[i].data != NULL &&
 414                     strcmp(rte_eth_devices[i].data->name, name) == 0)
 415                         return &rte_eth_devices[i];
 416         }
 417         return NULL;
 418 }
 419
 420 struct rte_eth_dev *
 421 rte_eth_dev_allocated(const char *name)
 422 {
 423         struct rte_eth_dev *ethdev;
 424
 425         rte_eth_dev_shared_data_prepare();
 426
 427         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 428
 429         ethdev = _rte_eth_dev_allocated(name);
 430
 431         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 432
 433         return ethdev;
 434 }
 435
 436 static uint16_t
 437 rte_eth_dev_find_free_port(void)
 438 {
 439         unsigned i;
 440
 441         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 442                 /* Using shared name field to find a free port. */
 443                 if (rte_eth_dev_shared_data->data[i].name[0] == '\0') {
 444                         RTE_ASSERT(rte_eth_devices[i].state ==
 445                                    RTE_ETH_DEV_UNUSED);
 446                         return i;
 447                 }
 448         }
 449         return RTE_MAX_ETHPORTS;
 450 }
 451
 452 static struct rte_eth_dev *
 453 eth_dev_get(uint16_t port_id)
 454 {
 455         struct rte_eth_dev *eth_dev = &rte_eth_devices[port_id];
 456
 457         eth_dev->data = &rte_eth_dev_shared_data->data[port_id];
 458
 459         return eth_dev;
 460 }
 461
 462 struct rte_eth_dev *
 463 rte_eth_dev_allocate(const char *name)
 464 {
 465         uint16_t port_id;
 466         struct rte_eth_dev *eth_dev = NULL;
 467         size_t name_len;
 468
 469         name_len = strnlen(name, RTE_ETH_NAME_MAX_LEN);
 470         if (name_len == 0) {
 471                 RTE_ETHDEV_LOG(ERR, "Zero length Ethernet device name\n");
 472                 return NULL;
 473         }
 474
 475         if (name_len >= RTE_ETH_NAME_MAX_LEN) {
 476                 RTE_ETHDEV_LOG(ERR, "Ethernet device name is too long\n");
 477                 return NULL;
 478         }
 479
 480         rte_eth_dev_shared_data_prepare();
 481
 482         /* Synchronize port creation between primary and secondary threads. */
 483         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 484
 485         if (_rte_eth_dev_allocated(name) != NULL) {
 486                 RTE_ETHDEV_LOG(ERR,
 487                         "Ethernet device with name %s already allocated\n",
 488                         name);
 489                 goto unlock;
 490         }
 491
 492         port_id = rte_eth_dev_find_free_port();
 493         if (port_id == RTE_MAX_ETHPORTS) {
 494                 RTE_ETHDEV_LOG(ERR,
 495                         "Reached maximum number of Ethernet ports\n");
 496                 goto unlock;
 497         }
 498
 499         eth_dev = eth_dev_get(port_id);
 500         strlcpy(eth_dev->data->name, name, sizeof(eth_dev->data->name));
 501         eth_dev->data->port_id = port_id;
 502         eth_dev->data->mtu = RTE_ETHER_MTU;
 503
 504 unlock:
 505         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 506
 507         return eth_dev;
 508 }
 509
 510 /*
 511  * Attach to a port already registered by the primary process, which
 512  * makes sure that the same device would have the same port id both
 513  * in the primary and secondary process.
 514  */
 515 struct rte_eth_dev *
 516 rte_eth_dev_attach_secondary(const char *name)
 517 {
 518         uint16_t i;
 519         struct rte_eth_dev *eth_dev = NULL;
 520
 521         rte_eth_dev_shared_data_prepare();
 522
 523         /* Synchronize port attachment to primary port creation and release. */
 524         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 525
 526         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 527                 if (strcmp(rte_eth_dev_shared_data->data[i].name, name) == 0)
 528                         break;
 529         }
 530         if (i == RTE_MAX_ETHPORTS) {
 531                 RTE_ETHDEV_LOG(ERR,
 532                         "Device %s is not driven by the primary process\n",
 533                         name);
 534         } else {
 535                 eth_dev = eth_dev_get(i);
 536                 RTE_ASSERT(eth_dev->data->port_id == i);
 537         }
 538
 539         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 540         return eth_dev;
 541 }
 542
 543 int
 544 rte_eth_dev_release_port(struct rte_eth_dev *eth_dev)
 545 {
 546         if (eth_dev == NULL)
 547                 return -EINVAL;
 548
 549         rte_eth_dev_shared_data_prepare();
 550
 551         if (eth_dev->state != RTE_ETH_DEV_UNUSED)
 552                 rte_eth_dev_callback_process(eth_dev,
 553                                 RTE_ETH_EVENT_DESTROY, NULL);
 554
 555         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 556
 557         eth_dev->state = RTE_ETH_DEV_UNUSED;
 558         eth_dev->device = NULL;
 559         eth_dev->intr_handle = NULL;
 560
 561         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 562                 rte_free(eth_dev->data->rx_queues);
 563                 rte_free(eth_dev->data->tx_queues);
 564                 rte_free(eth_dev->data->mac_addrs);
 565                 rte_free(eth_dev->data->hash_mac_addrs);
 566                 rte_free(eth_dev->data->dev_private);
 567                 memset(eth_dev->data, 0, sizeof(struct rte_eth_dev_data));
 568         }
 569
 570         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 571
 572         return 0;
 573 }
 574
 575 int
 576 rte_eth_dev_is_valid_port(uint16_t port_id)
 577 {
 578         if (port_id >= RTE_MAX_ETHPORTS ||
 579             (rte_eth_devices[port_id].state == RTE_ETH_DEV_UNUSED))
 580                 return 0;
 581         else
 582                 return 1;
 583 }
 584
 585 static int
 586 rte_eth_is_valid_owner_id(uint64_t owner_id)
 587 {
 588         if (owner_id == RTE_ETH_DEV_NO_OWNER ||
 589             rte_eth_dev_shared_data->next_owner_id <= owner_id)
 590                 return 0;
 591         return 1;
 592 }
 593
 594 uint64_t
 595 rte_eth_find_next_owned_by(uint16_t port_id, const uint64_t owner_id)
 596 {
 597         port_id = rte_eth_find_next(port_id);
 598         while (port_id < RTE_MAX_ETHPORTS &&
 599                         rte_eth_devices[port_id].data->owner.id != owner_id)
 600                 port_id = rte_eth_find_next(port_id + 1);
 601
 602         return port_id;
 603 }
 604
 605 int
 606 rte_eth_dev_owner_new(uint64_t *owner_id)
 607 {
 608         rte_eth_dev_shared_data_prepare();
 609
 610         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 611
 612         *owner_id = rte_eth_dev_shared_data->next_owner_id++;
 613
 614         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 615         return 0;
 616 }
 617
 618 static int
 619 _rte_eth_dev_owner_set(const uint16_t port_id, const uint64_t old_owner_id,
 620                        const struct rte_eth_dev_owner *new_owner)
 621 {
 622         struct rte_eth_dev *ethdev = &rte_eth_devices[port_id];
 623         struct rte_eth_dev_owner *port_owner;
 624
 625         if (port_id >= RTE_MAX_ETHPORTS || !is_allocated(ethdev)) {
 626                 RTE_ETHDEV_LOG(ERR, "Port id %"PRIu16" is not allocated\n",
 627                         port_id);
 628                 return -ENODEV;
 629         }
 630
 631         if (!rte_eth_is_valid_owner_id(new_owner->id) &&
 632             !rte_eth_is_valid_owner_id(old_owner_id)) {
 633                 RTE_ETHDEV_LOG(ERR,
 634                         "Invalid owner old_id=%016"PRIx64" new_id=%016"PRIx64"\n",
 635                        old_owner_id, new_owner->id);
 636                 return -EINVAL;
 637         }
 638
 639         port_owner = &rte_eth_devices[port_id].data->owner;
 640         if (port_owner->id != old_owner_id) {
 641                 RTE_ETHDEV_LOG(ERR,
 642                         "Cannot set owner to port %u already owned by %s_%016"PRIX64"\n",
 643                         port_id, port_owner->name, port_owner->id);
 644                 return -EPERM;
 645         }
 646
 647         /* can not truncate (same structure) */
 648         strlcpy(port_owner->name, new_owner->name, RTE_ETH_MAX_OWNER_NAME_LEN);
 649
 650         port_owner->id = new_owner->id;
 651
 652         RTE_ETHDEV_LOG(DEBUG, "Port %u owner is %s_%016"PRIx64"\n",
 653                 port_id, new_owner->name, new_owner->id);
 654
 655         return 0;
 656 }
 657
 658 int
 659 rte_eth_dev_owner_set(const uint16_t port_id,
 660                       const struct rte_eth_dev_owner *owner)
 661 {
 662         int ret;
 663
 664         rte_eth_dev_shared_data_prepare();
 665
 666         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 667
 668         ret = _rte_eth_dev_owner_set(port_id, RTE_ETH_DEV_NO_OWNER, owner);
 669
 670         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 671         return ret;
 672 }
 673
 674 int
 675 rte_eth_dev_owner_unset(const uint16_t port_id, const uint64_t owner_id)
 676 {
 677         const struct rte_eth_dev_owner new_owner = (struct rte_eth_dev_owner)
 678                         {.id = RTE_ETH_DEV_NO_OWNER, .name = ""};
 679         int ret;
 680
 681         rte_eth_dev_shared_data_prepare();
 682
 683         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 684
 685         ret = _rte_eth_dev_owner_set(port_id, owner_id, &new_owner);
 686
 687         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 688         return ret;
 689 }
 690
 691 int
 692 rte_eth_dev_owner_delete(const uint64_t owner_id)
 693 {
 694         uint16_t port_id;
 695         int ret = 0;
 696
 697         rte_eth_dev_shared_data_prepare();
 698
 699         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 700
 701         if (rte_eth_is_valid_owner_id(owner_id)) {
 702                 for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++)
 703                         if (rte_eth_devices[port_id].data->owner.id == owner_id)
 704                                 memset(&rte_eth_devices[port_id].data->owner, 0,
 705                                        sizeof(struct rte_eth_dev_owner));
 706                 RTE_ETHDEV_LOG(NOTICE,
 707                         "All port owners owned by %016"PRIx64" identifier have removed\n",
 708                         owner_id);
 709         } else {
 710                 RTE_ETHDEV_LOG(ERR,
 711                                "Invalid owner id=%016"PRIx64"\n",
 712                                owner_id);
 713                 ret = -EINVAL;
 714         }
 715
 716         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 717
 718         return ret;
 719 }
 720
 721 int
 722 rte_eth_dev_owner_get(const uint16_t port_id, struct rte_eth_dev_owner *owner)
 723 {
 724         int ret = 0;
 725         struct rte_eth_dev *ethdev = &rte_eth_devices[port_id];
 726
 727         rte_eth_dev_shared_data_prepare();
 728
 729         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 730
 731         if (port_id >= RTE_MAX_ETHPORTS || !is_allocated(ethdev)) {
 732                 RTE_ETHDEV_LOG(ERR, "Port id %"PRIu16" is not allocated\n",
 733                         port_id);
 734                 ret = -ENODEV;
 735         } else {
 736                 rte_memcpy(owner, &ethdev->data->owner, sizeof(*owner));
 737         }
 738
 739         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 740         return ret;
 741 }
 742
 743 int
 744 rte_eth_dev_socket_id(uint16_t port_id)
 745 {
 746         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -1);
 747         return rte_eth_devices[port_id].data->numa_node;
 748 }
 749
 750 void *
 751 rte_eth_dev_get_sec_ctx(uint16_t port_id)
 752 {
 753         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, NULL);
 754         return rte_eth_devices[port_id].security_ctx;
 755 }
 756
 757 uint16_t
 758 rte_eth_dev_count_avail(void)
 759 {
 760         uint16_t p;
 761         uint16_t count;
 762
 763         count = 0;
 764
 765         RTE_ETH_FOREACH_DEV(p)
 766                 count++;
 767
 768         return count;
 769 }
 770
 771 uint16_t
 772 rte_eth_dev_count_total(void)
 773 {
 774         uint16_t port, count = 0;
 775
 776         RTE_ETH_FOREACH_VALID_DEV(port)
 777                 count++;
 778
 779         return count;
 780 }
 781
 782 int
 783 rte_eth_dev_get_name_by_port(uint16_t port_id, char *name)
 784 {
 785         char *tmp;
 786
 787         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 788
 789         if (name == NULL) {
 790                 RTE_ETHDEV_LOG(ERR, "Null pointer is specified\n");
 791                 return -EINVAL;
 792         }
 793
 794         /* shouldn't check 'rte_eth_devices[i].data',
 795          * because it might be overwritten by VDEV PMD */
 796         tmp = rte_eth_dev_shared_data->data[port_id].name;
 797         strcpy(name, tmp);
 798         return 0;
 799 }
 800
 801 int
 802 rte_eth_dev_get_port_by_name(const char *name, uint16_t *port_id)
 803 {
 804         uint32_t pid;
 805
 806         if (name == NULL) {
 807                 RTE_ETHDEV_LOG(ERR, "Null pointer is specified\n");
 808                 return -EINVAL;
 809         }
 810
 811         RTE_ETH_FOREACH_VALID_DEV(pid)
 812                 if (!strcmp(name, rte_eth_dev_shared_data->data[pid].name)) {
 813                         *port_id = pid;
 814                         return 0;
 815                 }
 816
 817         return -ENODEV;
 818 }
 819
 820 static int
 821 eth_err(uint16_t port_id, int ret)
 822 {
 823         if (ret == 0)
 824                 return 0;
 825         if (rte_eth_dev_is_removed(port_id))
 826                 return -EIO;
 827         return ret;
 828 }
 829
 830 static int
 831 rte_eth_dev_rx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
 832 {
 833         uint16_t old_nb_queues = dev->data->nb_rx_queues;
 834         void **rxq;
 835         unsigned i;
 836
 837         if (dev->data->rx_queues == NULL && nb_queues != 0) { /* first time configuration */
 838                 dev->data->rx_queues = rte_zmalloc("ethdev->rx_queues",
 839                                 sizeof(dev->data->rx_queues[0]) * nb_queues,
 840                                 RTE_CACHE_LINE_SIZE);
 841                 if (dev->data->rx_queues == NULL) {
 842                         dev->data->nb_rx_queues = 0;
 843                         return -(ENOMEM);
 844                 }
 845         } else if (dev->data->rx_queues != NULL && nb_queues != 0) { /* re-configure */
 846                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release, -ENOTSUP);
 847
 848                 rxq = dev->data->rx_queues;
 849
 850                 for (i = nb_queues; i < old_nb_queues; i++)
 851                         (*dev->dev_ops->rx_queue_release)(rxq[i]);
 852                 rxq = rte_realloc(rxq, sizeof(rxq[0]) * nb_queues,
 853                                 RTE_CACHE_LINE_SIZE);
 854                 if (rxq == NULL)
 855                         return -(ENOMEM);
 856                 if (nb_queues > old_nb_queues) {
 857                         uint16_t new_qs = nb_queues - old_nb_queues;
 858
 859                         memset(rxq + old_nb_queues, 0,
 860                                 sizeof(rxq[0]) * new_qs);
 861                 }
 862
 863                 dev->data->rx_queues = rxq;
 864
 865         } else if (dev->data->rx_queues != NULL && nb_queues == 0) {
 866                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release, -ENOTSUP);
 867
 868                 rxq = dev->data->rx_queues;
 869
 870                 for (i = nb_queues; i < old_nb_queues; i++)
 871                         (*dev->dev_ops->rx_queue_release)(rxq[i]);
 872
 873                 rte_free(dev->data->rx_queues);
 874                 dev->data->rx_queues = NULL;
 875         }
 876         dev->data->nb_rx_queues = nb_queues;
 877         return 0;
 878 }
 879
 880 int
 881 rte_eth_dev_rx_queue_start(uint16_t port_id, uint16_t rx_queue_id)
 882 {
 883         struct rte_eth_dev *dev;
 884
 885         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 886
 887         dev = &rte_eth_devices[port_id];
 888         if (!dev->data->dev_started) {
 889                 RTE_ETHDEV_LOG(ERR,
 890                         "Port %u must be started before start any queue\n",
 891                         port_id);
 892                 return -EINVAL;
 893         }
 894
 895         if (rx_queue_id >= dev->data->nb_rx_queues) {
 896                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
 897                 return -EINVAL;
 898         }
 899
 900         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_start, -ENOTSUP);
 901
 902         if (rte_eth_dev_is_rx_hairpin_queue(dev, rx_queue_id)) {
 903                 RTE_ETHDEV_LOG(INFO,
 904                         "Can't start Rx hairpin queue %"PRIu16" of device with port_id=%"PRIu16"\n",
 905                         rx_queue_id, port_id);
 906                 return -EINVAL;
 907         }
 908
 909         if (dev->data->rx_queue_state[rx_queue_id] != RTE_ETH_QUEUE_STATE_STOPPED) {
 910                 RTE_ETHDEV_LOG(INFO,
 911                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already started\n",
 912                         rx_queue_id, port_id);
 913                 return 0;
 914         }
 915
 916         return eth_err(port_id, dev->dev_ops->rx_queue_start(dev,
 917                                                              rx_queue_id));
 918
 919 }
 920
 921 int
 922 rte_eth_dev_rx_queue_stop(uint16_t port_id, uint16_t rx_queue_id)
 923 {
 924         struct rte_eth_dev *dev;
 925
 926         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 927
 928         dev = &rte_eth_devices[port_id];
 929         if (rx_queue_id >= dev->data->nb_rx_queues) {
 930                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
 931                 return -EINVAL;
 932         }
 933
 934         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_stop, -ENOTSUP);
 935
 936         if (rte_eth_dev_is_rx_hairpin_queue(dev, rx_queue_id)) {
 937                 RTE_ETHDEV_LOG(INFO,
 938                         "Can't stop Rx hairpin queue %"PRIu16" of device with port_id=%"PRIu16"\n",
 939                         rx_queue_id, port_id);
 940                 return -EINVAL;
 941         }
 942
 943         if (dev->data->rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STOPPED) {
 944                 RTE_ETHDEV_LOG(INFO,
 945                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already stopped\n",
 946                         rx_queue_id, port_id);
 947                 return 0;
 948         }
 949
 950         return eth_err(port_id, dev->dev_ops->rx_queue_stop(dev, rx_queue_id));
 951
 952 }
 953
 954 int
 955 rte_eth_dev_tx_queue_start(uint16_t port_id, uint16_t tx_queue_id)
 956 {
 957         struct rte_eth_dev *dev;
 958
 959         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 960
 961         dev = &rte_eth_devices[port_id];
 962         if (!dev->data->dev_started) {
 963                 RTE_ETHDEV_LOG(ERR,
 964                         "Port %u must be started before start any queue\n",
 965                         port_id);
 966                 return -EINVAL;
 967         }
 968
 969         if (tx_queue_id >= dev->data->nb_tx_queues) {
 970                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
 971                 return -EINVAL;
 972         }
 973
 974         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_start, -ENOTSUP);
 975
 976         if (rte_eth_dev_is_tx_hairpin_queue(dev, tx_queue_id)) {
 977                 RTE_ETHDEV_LOG(INFO,
 978                         "Can't start Tx hairpin queue %"PRIu16" of device with port_id=%"PRIu16"\n",
 979                         tx_queue_id, port_id);
 980                 return -EINVAL;
 981         }
 982
 983         if (dev->data->tx_queue_state[tx_queue_id] != RTE_ETH_QUEUE_STATE_STOPPED) {
 984                 RTE_ETHDEV_LOG(INFO,
 985                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already started\n",
 986                         tx_queue_id, port_id);
 987                 return 0;
 988         }
 989
 990         return eth_err(port_id, dev->dev_ops->tx_queue_start(dev, tx_queue_id));
 991 }
 992
 993 int
 994 rte_eth_dev_tx_queue_stop(uint16_t port_id, uint16_t tx_queue_id)
 995 {
 996         struct rte_eth_dev *dev;
 997
 998         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 999
1000         dev = &rte_eth_devices[port_id];
1001         if (tx_queue_id >= dev->data->nb_tx_queues) {
1002                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
1003                 return -EINVAL;
1004         }
1005
1006         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_stop, -ENOTSUP);
1007
1008         if (rte_eth_dev_is_tx_hairpin_queue(dev, tx_queue_id)) {
1009                 RTE_ETHDEV_LOG(INFO,
1010                         "Can't stop Tx hairpin queue %"PRIu16" of device with port_id=%"PRIu16"\n",
1011                         tx_queue_id, port_id);
1012                 return -EINVAL;
1013         }
1014
1015         if (dev->data->tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STOPPED) {
1016                 RTE_ETHDEV_LOG(INFO,
1017                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already stopped\n",
1018                         tx_queue_id, port_id);
1019                 return 0;
1020         }
1021
1022         return eth_err(port_id, dev->dev_ops->tx_queue_stop(dev, tx_queue_id));
1023
1024 }
1025
1026 static int
1027 rte_eth_dev_tx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
1028 {
1029         uint16_t old_nb_queues = dev->data->nb_tx_queues;
1030         void **txq;
1031         unsigned i;
1032
1033         if (dev->data->tx_queues == NULL && nb_queues != 0) { /* first time configuration */
1034                 dev->data->tx_queues = rte_zmalloc("ethdev->tx_queues",
1035                                                    sizeof(dev->data->tx_queues[0]) * nb_queues,
1036                                                    RTE_CACHE_LINE_SIZE);
1037                 if (dev->data->tx_queues == NULL) {
1038                         dev->data->nb_tx_queues = 0;
1039                         return -(ENOMEM);
1040                 }
1041         } else if (dev->data->tx_queues != NULL && nb_queues != 0) { /* re-configure */
1042                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release, -ENOTSUP);
1043
1044                 txq = dev->data->tx_queues;
1045
1046                 for (i = nb_queues; i < old_nb_queues; i++)
1047                         (*dev->dev_ops->tx_queue_release)(txq[i]);
1048                 txq = rte_realloc(txq, sizeof(txq[0]) * nb_queues,
1049                                   RTE_CACHE_LINE_SIZE);
1050                 if (txq == NULL)
1051                         return -ENOMEM;
1052                 if (nb_queues > old_nb_queues) {
1053                         uint16_t new_qs = nb_queues - old_nb_queues;
1054
1055                         memset(txq + old_nb_queues, 0,
1056                                sizeof(txq[0]) * new_qs);
1057                 }
1058
1059                 dev->data->tx_queues = txq;
1060
1061         } else if (dev->data->tx_queues != NULL && nb_queues == 0) {
1062                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release, -ENOTSUP);
1063
1064                 txq = dev->data->tx_queues;
1065
1066                 for (i = nb_queues; i < old_nb_queues; i++)
1067                         (*dev->dev_ops->tx_queue_release)(txq[i]);
1068
1069                 rte_free(dev->data->tx_queues);
1070                 dev->data->tx_queues = NULL;
1071         }
1072         dev->data->nb_tx_queues = nb_queues;
1073         return 0;
1074 }
1075
1076 uint32_t
1077 rte_eth_speed_bitflag(uint32_t speed, int duplex)
1078 {
1079         switch (speed) {
1080         case ETH_SPEED_NUM_10M:
1081                 return duplex ? ETH_LINK_SPEED_10M : ETH_LINK_SPEED_10M_HD;
1082         case ETH_SPEED_NUM_100M:
1083                 return duplex ? ETH_LINK_SPEED_100M : ETH_LINK_SPEED_100M_HD;
1084         case ETH_SPEED_NUM_1G:
1085                 return ETH_LINK_SPEED_1G;
1086         case ETH_SPEED_NUM_2_5G:
1087                 return ETH_LINK_SPEED_2_5G;
1088         case ETH_SPEED_NUM_5G:
1089                 return ETH_LINK_SPEED_5G;
1090         case ETH_SPEED_NUM_10G:
1091                 return ETH_LINK_SPEED_10G;
1092         case ETH_SPEED_NUM_20G:
1093                 return ETH_LINK_SPEED_20G;
1094         case ETH_SPEED_NUM_25G:
1095                 return ETH_LINK_SPEED_25G;
1096         case ETH_SPEED_NUM_40G:
1097                 return ETH_LINK_SPEED_40G;
1098         case ETH_SPEED_NUM_50G:
1099                 return ETH_LINK_SPEED_50G;
1100         case ETH_SPEED_NUM_56G:
1101                 return ETH_LINK_SPEED_56G;
1102         case ETH_SPEED_NUM_100G:
1103                 return ETH_LINK_SPEED_100G;
1104         case ETH_SPEED_NUM_200G:
1105                 return ETH_LINK_SPEED_200G;
1106         default:
1107                 return 0;
1108         }
1109 }
1110
1111 const char *
1112 rte_eth_dev_rx_offload_name(uint64_t offload)
1113 {
1114         const char *name = "UNKNOWN";
1115         unsigned int i;
1116
1117         for (i = 0; i < RTE_DIM(rte_rx_offload_names); ++i) {
1118                 if (offload == rte_rx_offload_names[i].offload) {
1119                         name = rte_rx_offload_names[i].name;
1120                         break;
1121                 }
1122         }
1123
1124         return name;
1125 }
1126
1127 const char *
1128 rte_eth_dev_tx_offload_name(uint64_t offload)
1129 {
1130         const char *name = "UNKNOWN";
1131         unsigned int i;
1132
1133         for (i = 0; i < RTE_DIM(rte_tx_offload_names); ++i) {
1134                 if (offload == rte_tx_offload_names[i].offload) {
1135                         name = rte_tx_offload_names[i].name;
1136                         break;
1137                 }
1138         }
1139
1140         return name;
1141 }
1142
1143 static inline int
1144 check_lro_pkt_size(uint16_t port_id, uint32_t config_size,
1145                    uint32_t max_rx_pkt_len, uint32_t dev_info_size)
1146 {
1147         int ret = 0;
1148
1149         if (dev_info_size == 0) {
1150                 if (config_size != max_rx_pkt_len) {
1151                         RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%d max_lro_pkt_size"
1152                                        " %u != %u is not allowed\n",
1153                                        port_id, config_size, max_rx_pkt_len);
1154                         ret = -EINVAL;
1155                 }
1156         } else if (config_size > dev_info_size) {
1157                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%d max_lro_pkt_size %u "
1158                                "> max allowed value %u\n", port_id, config_size,
1159                                dev_info_size);
1160                 ret = -EINVAL;
1161         } else if (config_size < RTE_ETHER_MIN_LEN) {
1162                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%d max_lro_pkt_size %u "
1163                                "< min allowed value %u\n", port_id, config_size,
1164                                (unsigned int)RTE_ETHER_MIN_LEN);
1165                 ret = -EINVAL;
1166         }
1167         return ret;
1168 }
1169
1170 /*
1171  * Validate offloads that are requested through rte_eth_dev_configure against
1172  * the offloads successfully set by the ethernet device.
1173  *
1174  * @param port_id
1175  *   The port identifier of the Ethernet device.
1176  * @param req_offloads
1177  *   The offloads that have been requested through `rte_eth_dev_configure`.
1178  * @param set_offloads
1179  *   The offloads successfully set by the ethernet device.
1180  * @param offload_type
1181  *   The offload type i.e. Rx/Tx string.
1182  * @param offload_name
1183  *   The function that prints the offload name.
1184  * @return
1185  *   - (0) if validation successful.
1186  *   - (-EINVAL) if requested offload has been silently disabled.
1187  *
1188  */
1189 static int
1190 validate_offloads(uint16_t port_id, uint64_t req_offloads,
1191                   uint64_t set_offloads, const char *offload_type,
1192                   const char *(*offload_name)(uint64_t))
1193 {
1194         uint64_t offloads_diff = req_offloads ^ set_offloads;
1195         uint64_t offload;
1196         int ret = 0;
1197
1198         while (offloads_diff != 0) {
1199                 /* Check if any offload is requested but not enabled. */
1200                 offload = 1ULL << __builtin_ctzll(offloads_diff);
1201                 if (offload & req_offloads) {
1202                         RTE_ETHDEV_LOG(ERR,
1203                                 "Port %u failed to enable %s offload %s\n",
1204                                 port_id, offload_type, offload_name(offload));
1205                         ret = -EINVAL;
1206                 }
1207
1208                 /* Check if offload couldn't be disabled. */
1209                 if (offload & set_offloads) {
1210                         RTE_ETHDEV_LOG(DEBUG,
1211                                 "Port %u %s offload %s is not requested but enabled\n",
1212                                 port_id, offload_type, offload_name(offload));
1213                 }
1214
1215                 offloads_diff &= ~offload;
1216         }
1217
1218         return ret;
1219 }
1220
1221 int
1222 rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
1223                       const struct rte_eth_conf *dev_conf)
1224 {
1225         struct rte_eth_dev *dev;
1226         struct rte_eth_dev_info dev_info;
1227         struct rte_eth_conf orig_conf;
1228         int diag;
1229         int ret;
1230
1231         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1232
1233         dev = &rte_eth_devices[port_id];
1234
1235         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
1236
1237         if (dev->data->dev_started) {
1238                 RTE_ETHDEV_LOG(ERR,
1239                         "Port %u must be stopped to allow configuration\n",
1240                         port_id);
1241                 return -EBUSY;
1242         }
1243
1244          /* Store original config, as rollback required on failure */
1245         memcpy(&orig_conf, &dev->data->dev_conf, sizeof(dev->data->dev_conf));
1246
1247         /*
1248          * Copy the dev_conf parameter into the dev structure.
1249          * rte_eth_dev_info_get() requires dev_conf, copy it before dev_info get
1250          */
1251         if (dev_conf != &dev->data->dev_conf)
1252                 memcpy(&dev->data->dev_conf, dev_conf,
1253                        sizeof(dev->data->dev_conf));
1254
1255         ret = rte_eth_dev_info_get(port_id, &dev_info);
1256         if (ret != 0)
1257                 goto rollback;
1258
1259         /* If number of queues specified by application for both Rx and Tx is
1260          * zero, use driver preferred values. This cannot be done individually
1261          * as it is valid for either Tx or Rx (but not both) to be zero.
1262          * If driver does not provide any preferred valued, fall back on
1263          * EAL defaults.
1264          */
1265         if (nb_rx_q == 0 && nb_tx_q == 0) {
1266                 nb_rx_q = dev_info.default_rxportconf.nb_queues;
1267                 if (nb_rx_q == 0)
1268                         nb_rx_q = RTE_ETH_DEV_FALLBACK_RX_NBQUEUES;
1269                 nb_tx_q = dev_info.default_txportconf.nb_queues;
1270                 if (nb_tx_q == 0)
1271                         nb_tx_q = RTE_ETH_DEV_FALLBACK_TX_NBQUEUES;
1272         }
1273
1274         if (nb_rx_q > RTE_MAX_QUEUES_PER_PORT) {
1275                 RTE_ETHDEV_LOG(ERR,
1276                         "Number of RX queues requested (%u) is greater than max supported(%d)\n",
1277                         nb_rx_q, RTE_MAX_QUEUES_PER_PORT);
1278                 ret = -EINVAL;
1279                 goto rollback;
1280         }
1281
1282         if (nb_tx_q > RTE_MAX_QUEUES_PER_PORT) {
1283                 RTE_ETHDEV_LOG(ERR,
1284                         "Number of TX queues requested (%u) is greater than max supported(%d)\n",
1285                         nb_tx_q, RTE_MAX_QUEUES_PER_PORT);
1286                 ret = -EINVAL;
1287                 goto rollback;
1288         }
1289
1290         /*
1291          * Check that the numbers of RX and TX queues are not greater
1292          * than the maximum number of RX and TX queues supported by the
1293          * configured device.
1294          */
1295         if (nb_rx_q > dev_info.max_rx_queues) {
1296                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u nb_rx_queues=%u > %u\n",
1297                         port_id, nb_rx_q, dev_info.max_rx_queues);
1298                 ret = -EINVAL;
1299                 goto rollback;
1300         }
1301
1302         if (nb_tx_q > dev_info.max_tx_queues) {
1303                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u nb_tx_queues=%u > %u\n",
1304                         port_id, nb_tx_q, dev_info.max_tx_queues);
1305                 ret = -EINVAL;
1306                 goto rollback;
1307         }
1308
1309         /* Check that the device supports requested interrupts */
1310         if ((dev_conf->intr_conf.lsc == 1) &&
1311                         (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC))) {
1312                 RTE_ETHDEV_LOG(ERR, "Driver %s does not support lsc\n",
1313                         dev->device->driver->name);
1314                 ret = -EINVAL;
1315                 goto rollback;
1316         }
1317         if ((dev_conf->intr_conf.rmv == 1) &&
1318                         (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_RMV))) {
1319                 RTE_ETHDEV_LOG(ERR, "Driver %s does not support rmv\n",
1320                         dev->device->driver->name);
1321                 ret = -EINVAL;
1322                 goto rollback;
1323         }
1324
1325         /*
1326          * If jumbo frames are enabled, check that the maximum RX packet
1327          * length is supported by the configured device.
1328          */
1329         if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
1330                 if (dev_conf->rxmode.max_rx_pkt_len > dev_info.max_rx_pktlen) {
1331                         RTE_ETHDEV_LOG(ERR,
1332                                 "Ethdev port_id=%u max_rx_pkt_len %u > max valid value %u\n",
1333                                 port_id, dev_conf->rxmode.max_rx_pkt_len,
1334                                 dev_info.max_rx_pktlen);
1335                         ret = -EINVAL;
1336                         goto rollback;
1337                 } else if (dev_conf->rxmode.max_rx_pkt_len < RTE_ETHER_MIN_LEN) {
1338                         RTE_ETHDEV_LOG(ERR,
1339                                 "Ethdev port_id=%u max_rx_pkt_len %u < min valid value %u\n",
1340                                 port_id, dev_conf->rxmode.max_rx_pkt_len,
1341                                 (unsigned int)RTE_ETHER_MIN_LEN);
1342                         ret = -EINVAL;
1343                         goto rollback;
1344                 }
1345         } else {
1346                 if (dev_conf->rxmode.max_rx_pkt_len < RTE_ETHER_MIN_LEN ||
1347                         dev_conf->rxmode.max_rx_pkt_len > RTE_ETHER_MAX_LEN)
1348                         /* Use default value */
1349                         dev->data->dev_conf.rxmode.max_rx_pkt_len =
1350                                                         RTE_ETHER_MAX_LEN;
1351         }
1352
1353         /*
1354          * If LRO is enabled, check that the maximum aggregated packet
1355          * size is supported by the configured device.
1356          */
1357         if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_TCP_LRO) {
1358                 if (dev_conf->rxmode.max_lro_pkt_size == 0)
1359                         dev->data->dev_conf.rxmode.max_lro_pkt_size =
1360                                 dev->data->dev_conf.rxmode.max_rx_pkt_len;
1361                 ret = check_lro_pkt_size(port_id,
1362                                 dev->data->dev_conf.rxmode.max_lro_pkt_size,
1363                                 dev->data->dev_conf.rxmode.max_rx_pkt_len,
1364                                 dev_info.max_lro_pkt_size);
1365                 if (ret != 0)
1366                         goto rollback;
1367         }
1368
1369         /* Any requested offloading must be within its device capabilities */
1370         if ((dev_conf->rxmode.offloads & dev_info.rx_offload_capa) !=
1371              dev_conf->rxmode.offloads) {
1372                 RTE_ETHDEV_LOG(ERR,
1373                         "Ethdev port_id=%u requested Rx offloads 0x%"PRIx64" doesn't match Rx offloads "
1374                         "capabilities 0x%"PRIx64" in %s()\n",
1375                         port_id, dev_conf->rxmode.offloads,
1376                         dev_info.rx_offload_capa,
1377                         __func__);
1378                 ret = -EINVAL;
1379                 goto rollback;
1380         }
1381         if ((dev_conf->txmode.offloads & dev_info.tx_offload_capa) !=
1382              dev_conf->txmode.offloads) {
1383                 RTE_ETHDEV_LOG(ERR,
1384                         "Ethdev port_id=%u requested Tx offloads 0x%"PRIx64" doesn't match Tx offloads "
1385                         "capabilities 0x%"PRIx64" in %s()\n",
1386                         port_id, dev_conf->txmode.offloads,
1387                         dev_info.tx_offload_capa,
1388                         __func__);
1389                 ret = -EINVAL;
1390                 goto rollback;
1391         }
1392
1393         dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf =
1394                 rte_eth_rss_hf_refine(dev_conf->rx_adv_conf.rss_conf.rss_hf);
1395
1396         /* Check that device supports requested rss hash functions. */
1397         if ((dev_info.flow_type_rss_offloads |
1398              dev_conf->rx_adv_conf.rss_conf.rss_hf) !=
1399             dev_info.flow_type_rss_offloads) {
1400                 RTE_ETHDEV_LOG(ERR,
1401                         "Ethdev port_id=%u invalid rss_hf: 0x%"PRIx64", valid value: 0x%"PRIx64"\n",
1402                         port_id, dev_conf->rx_adv_conf.rss_conf.rss_hf,
1403                         dev_info.flow_type_rss_offloads);
1404                 ret = -EINVAL;
1405                 goto rollback;
1406         }
1407
1408         /* Check if Rx RSS distribution is disabled but RSS hash is enabled. */
1409         if (((dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) == 0) &&
1410             (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_RSS_HASH)) {
1411                 RTE_ETHDEV_LOG(ERR,
1412                         "Ethdev port_id=%u config invalid Rx mq_mode without RSS but %s offload is requested\n",
1413                         port_id,
1414                         rte_eth_dev_rx_offload_name(DEV_RX_OFFLOAD_RSS_HASH));
1415                 ret = -EINVAL;
1416                 goto rollback;
1417         }
1418
1419         /*
1420          * Setup new number of RX/TX queues and reconfigure device.
1421          */
1422         diag = rte_eth_dev_rx_queue_config(dev, nb_rx_q);
1423         if (diag != 0) {
1424                 RTE_ETHDEV_LOG(ERR,
1425                         "Port%u rte_eth_dev_rx_queue_config = %d\n",
1426                         port_id, diag);
1427                 ret = diag;
1428                 goto rollback;
1429         }
1430
1431         diag = rte_eth_dev_tx_queue_config(dev, nb_tx_q);
1432         if (diag != 0) {
1433                 RTE_ETHDEV_LOG(ERR,
1434                         "Port%u rte_eth_dev_tx_queue_config = %d\n",
1435                         port_id, diag);
1436                 rte_eth_dev_rx_queue_config(dev, 0);
1437                 ret = diag;
1438                 goto rollback;
1439         }
1440
1441         diag = (*dev->dev_ops->dev_configure)(dev);
1442         if (diag != 0) {
1443                 RTE_ETHDEV_LOG(ERR, "Port%u dev_configure = %d\n",
1444                         port_id, diag);
1445                 ret = eth_err(port_id, diag);
1446                 goto reset_queues;
1447         }
1448
1449         /* Initialize Rx profiling if enabled at compilation time. */
1450         diag = __rte_eth_dev_profile_init(port_id, dev);
1451         if (diag != 0) {
1452                 RTE_ETHDEV_LOG(ERR, "Port%u __rte_eth_dev_profile_init = %d\n",
1453                         port_id, diag);
1454                 ret = eth_err(port_id, diag);
1455                 goto reset_queues;
1456         }
1457
1458         /* Validate Rx offloads. */
1459         diag = validate_offloads(port_id,
1460                         dev_conf->rxmode.offloads,
1461                         dev->data->dev_conf.rxmode.offloads, "Rx",
1462                         rte_eth_dev_rx_offload_name);
1463         if (diag != 0) {
1464                 ret = diag;
1465                 goto reset_queues;
1466         }
1467
1468         /* Validate Tx offloads. */
1469         diag = validate_offloads(port_id,
1470                         dev_conf->txmode.offloads,
1471                         dev->data->dev_conf.txmode.offloads, "Tx",
1472                         rte_eth_dev_tx_offload_name);
1473         if (diag != 0) {
1474                 ret = diag;
1475                 goto reset_queues;
1476         }
1477
1478         rte_ethdev_trace_configure(port_id, nb_rx_q, nb_tx_q, dev_conf, 0);
1479         return 0;
1480 reset_queues:
1481         rte_eth_dev_rx_queue_config(dev, 0);
1482         rte_eth_dev_tx_queue_config(dev, 0);
1483 rollback:
1484         memcpy(&dev->data->dev_conf, &orig_conf, sizeof(dev->data->dev_conf));
1485
1486         rte_ethdev_trace_configure(port_id, nb_rx_q, nb_tx_q, dev_conf, ret);
1487         return ret;
1488 }
1489
1490 void
1491 rte_eth_dev_internal_reset(struct rte_eth_dev *dev)
1492 {
1493         if (dev->data->dev_started) {
1494                 RTE_ETHDEV_LOG(ERR, "Port %u must be stopped to allow reset\n",
1495                         dev->data->port_id);
1496                 return;
1497         }
1498
1499         rte_eth_dev_rx_queue_config(dev, 0);
1500         rte_eth_dev_tx_queue_config(dev, 0);
1501
1502         memset(&dev->data->dev_conf, 0, sizeof(dev->data->dev_conf));
1503 }
1504
1505 static void
1506 rte_eth_dev_mac_restore(struct rte_eth_dev *dev,
1507                         struct rte_eth_dev_info *dev_info)
1508 {
1509         struct rte_ether_addr *addr;
1510         uint16_t i;
1511         uint32_t pool = 0;
1512         uint64_t pool_mask;
1513
1514         /* replay MAC address configuration including default MAC */
1515         addr = &dev->data->mac_addrs[0];
1516         if (*dev->dev_ops->mac_addr_set != NULL)
1517                 (*dev->dev_ops->mac_addr_set)(dev, addr);
1518         else if (*dev->dev_ops->mac_addr_add != NULL)
1519                 (*dev->dev_ops->mac_addr_add)(dev, addr, 0, pool);
1520
1521         if (*dev->dev_ops->mac_addr_add != NULL) {
1522                 for (i = 1; i < dev_info->max_mac_addrs; i++) {
1523                         addr = &dev->data->mac_addrs[i];
1524
1525                         /* skip zero address */
1526                         if (rte_is_zero_ether_addr(addr))
1527                                 continue;
1528
1529                         pool = 0;
1530                         pool_mask = dev->data->mac_pool_sel[i];
1531
1532                         do {
1533                                 if (pool_mask & 1ULL)
1534                                         (*dev->dev_ops->mac_addr_add)(dev,
1535                                                 addr, i, pool);
1536                                 pool_mask >>= 1;
1537                                 pool++;
1538                         } while (pool_mask);
1539                 }
1540         }
1541 }
1542
1543 static int
1544 rte_eth_dev_config_restore(struct rte_eth_dev *dev,
1545                            struct rte_eth_dev_info *dev_info, uint16_t port_id)
1546 {
1547         int ret;
1548
1549         if (!(*dev_info->dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR))
1550                 rte_eth_dev_mac_restore(dev, dev_info);
1551
1552         /* replay promiscuous configuration */
1553         /*
1554          * use callbacks directly since we don't need port_id check and
1555          * would like to bypass the same value set
1556          */
1557         if (rte_eth_promiscuous_get(port_id) == 1 &&
1558             *dev->dev_ops->promiscuous_enable != NULL) {
1559                 ret = eth_err(port_id,
1560                               (*dev->dev_ops->promiscuous_enable)(dev));
1561                 if (ret != 0 && ret != -ENOTSUP) {
1562                         RTE_ETHDEV_LOG(ERR,
1563                                 "Failed to enable promiscuous mode for device (port %u): %s\n",
1564                                 port_id, rte_strerror(-ret));
1565                         return ret;
1566                 }
1567         } else if (rte_eth_promiscuous_get(port_id) == 0 &&
1568                    *dev->dev_ops->promiscuous_disable != NULL) {
1569                 ret = eth_err(port_id,
1570                               (*dev->dev_ops->promiscuous_disable)(dev));
1571                 if (ret != 0 && ret != -ENOTSUP) {
1572                         RTE_ETHDEV_LOG(ERR,
1573                                 "Failed to disable promiscuous mode for device (port %u): %s\n",
1574                                 port_id, rte_strerror(-ret));
1575                         return ret;
1576                 }
1577         }
1578
1579         /* replay all multicast configuration */
1580         /*
1581          * use callbacks directly since we don't need port_id check and
1582          * would like to bypass the same value set
1583          */
1584         if (rte_eth_allmulticast_get(port_id) == 1 &&
1585             *dev->dev_ops->allmulticast_enable != NULL) {
1586                 ret = eth_err(port_id,
1587                               (*dev->dev_ops->allmulticast_enable)(dev));
1588                 if (ret != 0 && ret != -ENOTSUP) {
1589                         RTE_ETHDEV_LOG(ERR,
1590                                 "Failed to enable allmulticast mode for device (port %u): %s\n",
1591                                 port_id, rte_strerror(-ret));
1592                         return ret;
1593                 }
1594         } else if (rte_eth_allmulticast_get(port_id) == 0 &&
1595                    *dev->dev_ops->allmulticast_disable != NULL) {
1596                 ret = eth_err(port_id,
1597                               (*dev->dev_ops->allmulticast_disable)(dev));
1598                 if (ret != 0 && ret != -ENOTSUP) {
1599                         RTE_ETHDEV_LOG(ERR,
1600                                 "Failed to disable allmulticast mode for device (port %u): %s\n",
1601                                 port_id, rte_strerror(-ret));
1602                         return ret;
1603                 }
1604         }
1605
1606         return 0;
1607 }
1608
1609 int
1610 rte_eth_dev_start(uint16_t port_id)
1611 {
1612         struct rte_eth_dev *dev;
1613         struct rte_eth_dev_info dev_info;
1614         int diag;
1615         int ret;
1616
1617         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1618
1619         dev = &rte_eth_devices[port_id];
1620
1621         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_start, -ENOTSUP);
1622
1623         if (dev->data->dev_started != 0) {
1624                 RTE_ETHDEV_LOG(INFO,
1625                         "Device with port_id=%"PRIu16" already started\n",
1626                         port_id);
1627                 return 0;
1628         }
1629
1630         ret = rte_eth_dev_info_get(port_id, &dev_info);
1631         if (ret != 0)
1632                 return ret;
1633
1634         /* Lets restore MAC now if device does not support live change */
1635         if (*dev_info.dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR)
1636                 rte_eth_dev_mac_restore(dev, &dev_info);
1637
1638         diag = (*dev->dev_ops->dev_start)(dev);
1639         if (diag == 0)
1640                 dev->data->dev_started = 1;
1641         else
1642                 return eth_err(port_id, diag);
1643
1644         ret = rte_eth_dev_config_restore(dev, &dev_info, port_id);
1645         if (ret != 0) {
1646                 RTE_ETHDEV_LOG(ERR,
1647                         "Error during restoring configuration for device (port %u): %s\n",
1648                         port_id, rte_strerror(-ret));
1649                 rte_eth_dev_stop(port_id);
1650                 return ret;
1651         }
1652
1653         if (dev->data->dev_conf.intr_conf.lsc == 0) {
1654                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
1655                 (*dev->dev_ops->link_update)(dev, 0);
1656         }
1657
1658         rte_ethdev_trace_start(port_id);
1659         return 0;
1660 }
1661
1662 void
1663 rte_eth_dev_stop(uint16_t port_id)
1664 {
1665         struct rte_eth_dev *dev;
1666
1667         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1668         dev = &rte_eth_devices[port_id];
1669
1670         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_stop);
1671
1672         if (dev->data->dev_started == 0) {
1673                 RTE_ETHDEV_LOG(INFO,
1674                         "Device with port_id=%"PRIu16" already stopped\n",
1675                         port_id);
1676                 return;
1677         }
1678
1679         dev->data->dev_started = 0;
1680         (*dev->dev_ops->dev_stop)(dev);
1681         rte_ethdev_trace_stop(port_id);
1682 }
1683
1684 int
1685 rte_eth_dev_set_link_up(uint16_t port_id)
1686 {
1687         struct rte_eth_dev *dev;
1688
1689         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1690
1691         dev = &rte_eth_devices[port_id];
1692
1693         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_up, -ENOTSUP);
1694         return eth_err(port_id, (*dev->dev_ops->dev_set_link_up)(dev));
1695 }
1696
1697 int
1698 rte_eth_dev_set_link_down(uint16_t port_id)
1699 {
1700         struct rte_eth_dev *dev;
1701
1702         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1703
1704         dev = &rte_eth_devices[port_id];
1705
1706         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_down, -ENOTSUP);
1707         return eth_err(port_id, (*dev->dev_ops->dev_set_link_down)(dev));
1708 }
1709
1710 void
1711 rte_eth_dev_close(uint16_t port_id)
1712 {
1713         struct rte_eth_dev *dev;
1714
1715         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1716         dev = &rte_eth_devices[port_id];
1717
1718         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_close);
1719         dev->data->dev_started = 0;
1720         (*dev->dev_ops->dev_close)(dev);
1721
1722         rte_ethdev_trace_close(port_id);
1723         rte_eth_dev_release_port(dev);
1724 }
1725
1726 int
1727 rte_eth_dev_reset(uint16_t port_id)
1728 {
1729         struct rte_eth_dev *dev;
1730         int ret;
1731
1732         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1733         dev = &rte_eth_devices[port_id];
1734
1735         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_reset, -ENOTSUP);
1736
1737         rte_eth_dev_stop(port_id);
1738         ret = dev->dev_ops->dev_reset(dev);
1739
1740         return eth_err(port_id, ret);
1741 }
1742
1743 int
1744 rte_eth_dev_is_removed(uint16_t port_id)
1745 {
1746         struct rte_eth_dev *dev;
1747         int ret;
1748
1749         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, 0);
1750
1751         dev = &rte_eth_devices[port_id];
1752
1753         if (dev->state == RTE_ETH_DEV_REMOVED)
1754                 return 1;
1755
1756         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->is_removed, 0);
1757
1758         ret = dev->dev_ops->is_removed(dev);
1759         if (ret != 0)
1760                 /* Device is physically removed. */
1761                 dev->state = RTE_ETH_DEV_REMOVED;
1762
1763         return ret;
1764 }
1765
1766 int
1767 rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
1768                        uint16_t nb_rx_desc, unsigned int socket_id,
1769                        const struct rte_eth_rxconf *rx_conf,
1770                        struct rte_mempool *mp)
1771 {
1772         int ret;
1773         uint32_t mbp_buf_size;
1774         struct rte_eth_dev *dev;
1775         struct rte_eth_dev_info dev_info;
1776         struct rte_eth_rxconf local_conf;
1777         void **rxq;
1778
1779         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1780
1781         dev = &rte_eth_devices[port_id];
1782         if (rx_queue_id >= dev->data->nb_rx_queues) {
1783                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
1784                 return -EINVAL;
1785         }
1786
1787         if (mp == NULL) {
1788                 RTE_ETHDEV_LOG(ERR, "Invalid null mempool pointer\n");
1789                 return -EINVAL;
1790         }
1791
1792         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_setup, -ENOTSUP);
1793
1794         /*
1795          * Check the size of the mbuf data buffer.
1796          * This value must be provided in the private data of the memory pool.
1797          * First check that the memory pool has a valid private data.
1798          */
1799         ret = rte_eth_dev_info_get(port_id, &dev_info);
1800         if (ret != 0)
1801                 return ret;
1802
1803         if (mp->private_data_size < sizeof(struct rte_pktmbuf_pool_private)) {
1804                 RTE_ETHDEV_LOG(ERR, "%s private_data_size %d < %d\n",
1805                         mp->name, (int)mp->private_data_size,
1806                         (int)sizeof(struct rte_pktmbuf_pool_private));
1807                 return -ENOSPC;
1808         }
1809         mbp_buf_size = rte_pktmbuf_data_room_size(mp);
1810
1811         if (mbp_buf_size < dev_info.min_rx_bufsize + RTE_PKTMBUF_HEADROOM) {
1812                 RTE_ETHDEV_LOG(ERR,
1813                         "%s mbuf_data_room_size %d < %d (RTE_PKTMBUF_HEADROOM=%d + min_rx_bufsize(dev)=%d)\n",
1814                         mp->name, (int)mbp_buf_size,
1815                         (int)(RTE_PKTMBUF_HEADROOM + dev_info.min_rx_bufsize),
1816                         (int)RTE_PKTMBUF_HEADROOM,
1817                         (int)dev_info.min_rx_bufsize);
1818                 return -EINVAL;
1819         }
1820
1821         /* Use default specified by driver, if nb_rx_desc is zero */
1822         if (nb_rx_desc == 0) {
1823                 nb_rx_desc = dev_info.default_rxportconf.ring_size;
1824                 /* If driver default is also zero, fall back on EAL default */
1825                 if (nb_rx_desc == 0)
1826                         nb_rx_desc = RTE_ETH_DEV_FALLBACK_RX_RINGSIZE;
1827         }
1828
1829         if (nb_rx_desc > dev_info.rx_desc_lim.nb_max ||
1830                         nb_rx_desc < dev_info.rx_desc_lim.nb_min ||
1831                         nb_rx_desc % dev_info.rx_desc_lim.nb_align != 0) {
1832
1833                 RTE_ETHDEV_LOG(ERR,
1834                         "Invalid value for nb_rx_desc(=%hu), should be: <= %hu, >= %hu, and a product of %hu\n",
1835                         nb_rx_desc, dev_info.rx_desc_lim.nb_max,
1836                         dev_info.rx_desc_lim.nb_min,
1837                         dev_info.rx_desc_lim.nb_align);
1838                 return -EINVAL;
1839         }
1840
1841         if (dev->data->dev_started &&
1842                 !(dev_info.dev_capa &
1843                         RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP))
1844                 return -EBUSY;
1845
1846         if (dev->data->dev_started &&
1847                 (dev->data->rx_queue_state[rx_queue_id] !=
1848                         RTE_ETH_QUEUE_STATE_STOPPED))
1849                 return -EBUSY;
1850
1851         rxq = dev->data->rx_queues;
1852         if (rxq[rx_queue_id]) {
1853                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release,
1854                                         -ENOTSUP);
1855                 (*dev->dev_ops->rx_queue_release)(rxq[rx_queue_id]);
1856                 rxq[rx_queue_id] = NULL;
1857         }
1858
1859         if (rx_conf == NULL)
1860                 rx_conf = &dev_info.default_rxconf;
1861
1862         local_conf = *rx_conf;
1863
1864         /*
1865          * If an offloading has already been enabled in
1866          * rte_eth_dev_configure(), it has been enabled on all queues,
1867          * so there is no need to enable it in this queue again.
1868          * The local_conf.offloads input to underlying PMD only carries
1869          * those offloadings which are only enabled on this queue and
1870          * not enabled on all queues.
1871          */
1872         local_conf.offloads &= ~dev->data->dev_conf.rxmode.offloads;
1873
1874         /*
1875          * New added offloadings for this queue are those not enabled in
1876          * rte_eth_dev_configure() and they must be per-queue type.
1877          * A pure per-port offloading can't be enabled on a queue while
1878          * disabled on another queue. A pure per-port offloading can't
1879          * be enabled for any queue as new added one if it hasn't been
1880          * enabled in rte_eth_dev_configure().
1881          */
1882         if ((local_conf.offloads & dev_info.rx_queue_offload_capa) !=
1883              local_conf.offloads) {
1884                 RTE_ETHDEV_LOG(ERR,
1885                         "Ethdev port_id=%d rx_queue_id=%d, new added offloads 0x%"PRIx64" must be "
1886                         "within per-queue offload capabilities 0x%"PRIx64" in %s()\n",
1887                         port_id, rx_queue_id, local_conf.offloads,
1888                         dev_info.rx_queue_offload_capa,
1889                         __func__);
1890                 return -EINVAL;
1891         }
1892
1893         /*
1894          * If LRO is enabled, check that the maximum aggregated packet
1895          * size is supported by the configured device.
1896          */
1897         if (local_conf.offloads & DEV_RX_OFFLOAD_TCP_LRO) {
1898                 if (dev->data->dev_conf.rxmode.max_lro_pkt_size == 0)
1899                         dev->data->dev_conf.rxmode.max_lro_pkt_size =
1900                                 dev->data->dev_conf.rxmode.max_rx_pkt_len;
1901                 int ret = check_lro_pkt_size(port_id,
1902                                 dev->data->dev_conf.rxmode.max_lro_pkt_size,
1903                                 dev->data->dev_conf.rxmode.max_rx_pkt_len,
1904                                 dev_info.max_lro_pkt_size);
1905                 if (ret != 0)
1906                         return ret;
1907         }
1908
1909         ret = (*dev->dev_ops->rx_queue_setup)(dev, rx_queue_id, nb_rx_desc,
1910                                               socket_id, &local_conf, mp);
1911         if (!ret) {
1912                 if (!dev->data->min_rx_buf_size ||
1913                     dev->data->min_rx_buf_size > mbp_buf_size)
1914                         dev->data->min_rx_buf_size = mbp_buf_size;
1915         }
1916
1917         rte_ethdev_trace_rxq_setup(port_id, rx_queue_id, nb_rx_desc, mp,
1918                 rx_conf, ret);
1919         return eth_err(port_id, ret);
1920 }
1921
1922 int
1923 rte_eth_rx_hairpin_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
1924                                uint16_t nb_rx_desc,
1925                                const struct rte_eth_hairpin_conf *conf)
1926 {
1927         int ret;
1928         struct rte_eth_dev *dev;
1929         struct rte_eth_hairpin_cap cap;
1930         void **rxq;
1931         int i;
1932         int count;
1933
1934         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1935
1936         dev = &rte_eth_devices[port_id];
1937         if (rx_queue_id >= dev->data->nb_rx_queues) {
1938                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
1939                 return -EINVAL;
1940         }
1941         ret = rte_eth_dev_hairpin_capability_get(port_id, &cap);
1942         if (ret != 0)
1943                 return ret;
1944         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_hairpin_queue_setup,
1945                                 -ENOTSUP);
1946         /* if nb_rx_desc is zero use max number of desc from the driver. */
1947         if (nb_rx_desc == 0)
1948                 nb_rx_desc = cap.max_nb_desc;
1949         if (nb_rx_desc > cap.max_nb_desc) {
1950                 RTE_ETHDEV_LOG(ERR,
1951                         "Invalid value for nb_rx_desc(=%hu), should be: <= %hu",
1952                         nb_rx_desc, cap.max_nb_desc);
1953                 return -EINVAL;
1954         }
1955         if (conf->peer_count > cap.max_rx_2_tx) {
1956                 RTE_ETHDEV_LOG(ERR,
1957                         "Invalid value for number of peers for Rx queue(=%hu), should be: <= %hu",
1958                         conf->peer_count, cap.max_rx_2_tx);
1959                 return -EINVAL;
1960         }
1961         if (conf->peer_count == 0) {
1962                 RTE_ETHDEV_LOG(ERR,
1963                         "Invalid value for number of peers for Rx queue(=%hu), should be: > 0",
1964                         conf->peer_count);
1965                 return -EINVAL;
1966         }
1967         for (i = 0, count = 0; i < dev->data->nb_rx_queues &&
1968              cap.max_nb_queues != UINT16_MAX; i++) {
1969                 if (i == rx_queue_id || rte_eth_dev_is_rx_hairpin_queue(dev, i))
1970                         count++;
1971         }
1972         if (count > cap.max_nb_queues) {
1973                 RTE_ETHDEV_LOG(ERR, "To many Rx hairpin queues max is %d",
1974                 cap.max_nb_queues);
1975                 return -EINVAL;
1976         }
1977         if (dev->data->dev_started)
1978                 return -EBUSY;
1979         rxq = dev->data->rx_queues;
1980         if (rxq[rx_queue_id] != NULL) {
1981                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release,
1982                                         -ENOTSUP);
1983                 (*dev->dev_ops->rx_queue_release)(rxq[rx_queue_id]);
1984                 rxq[rx_queue_id] = NULL;
1985         }
1986         ret = (*dev->dev_ops->rx_hairpin_queue_setup)(dev, rx_queue_id,
1987                                                       nb_rx_desc, conf);
1988         if (ret == 0)
1989                 dev->data->rx_queue_state[rx_queue_id] =
1990                         RTE_ETH_QUEUE_STATE_HAIRPIN;
1991         return eth_err(port_id, ret);
1992 }
1993
1994 int
1995 rte_eth_tx_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
1996                        uint16_t nb_tx_desc, unsigned int socket_id,
1997                        const struct rte_eth_txconf *tx_conf)
1998 {
1999         struct rte_eth_dev *dev;
2000         struct rte_eth_dev_info dev_info;
2001         struct rte_eth_txconf local_conf;
2002         void **txq;
2003         int ret;
2004
2005         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2006
2007         dev = &rte_eth_devices[port_id];
2008         if (tx_queue_id >= dev->data->nb_tx_queues) {
2009                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
2010                 return -EINVAL;
2011         }
2012
2013         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_setup, -ENOTSUP);
2014
2015         ret = rte_eth_dev_info_get(port_id, &dev_info);
2016         if (ret != 0)
2017                 return ret;
2018
2019         /* Use default specified by driver, if nb_tx_desc is zero */
2020         if (nb_tx_desc == 0) {
2021                 nb_tx_desc = dev_info.default_txportconf.ring_size;
2022                 /* If driver default is zero, fall back on EAL default */
2023                 if (nb_tx_desc == 0)
2024                         nb_tx_desc = RTE_ETH_DEV_FALLBACK_TX_RINGSIZE;
2025         }
2026         if (nb_tx_desc > dev_info.tx_desc_lim.nb_max ||
2027             nb_tx_desc < dev_info.tx_desc_lim.nb_min ||
2028             nb_tx_desc % dev_info.tx_desc_lim.nb_align != 0) {
2029                 RTE_ETHDEV_LOG(ERR,
2030                         "Invalid value for nb_tx_desc(=%hu), should be: <= %hu, >= %hu, and a product of %hu\n",
2031                         nb_tx_desc, dev_info.tx_desc_lim.nb_max,
2032                         dev_info.tx_desc_lim.nb_min,
2033                         dev_info.tx_desc_lim.nb_align);
2034                 return -EINVAL;
2035         }
2036
2037         if (dev->data->dev_started &&
2038                 !(dev_info.dev_capa &
2039                         RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP))
2040                 return -EBUSY;
2041
2042         if (dev->data->dev_started &&
2043                 (dev->data->tx_queue_state[tx_queue_id] !=
2044                         RTE_ETH_QUEUE_STATE_STOPPED))
2045                 return -EBUSY;
2046
2047         txq = dev->data->tx_queues;
2048         if (txq[tx_queue_id]) {
2049                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release,
2050                                         -ENOTSUP);
2051                 (*dev->dev_ops->tx_queue_release)(txq[tx_queue_id]);
2052                 txq[tx_queue_id] = NULL;
2053         }
2054
2055         if (tx_conf == NULL)
2056                 tx_conf = &dev_info.default_txconf;
2057
2058         local_conf = *tx_conf;
2059
2060         /*
2061          * If an offloading has already been enabled in
2062          * rte_eth_dev_configure(), it has been enabled on all queues,
2063          * so there is no need to enable it in this queue again.
2064          * The local_conf.offloads input to underlying PMD only carries
2065          * those offloadings which are only enabled on this queue and
2066          * not enabled on all queues.
2067          */
2068         local_conf.offloads &= ~dev->data->dev_conf.txmode.offloads;
2069
2070         /*
2071          * New added offloadings for this queue are those not enabled in
2072          * rte_eth_dev_configure() and they must be per-queue type.
2073          * A pure per-port offloading can't be enabled on a queue while
2074          * disabled on another queue. A pure per-port offloading can't
2075          * be enabled for any queue as new added one if it hasn't been
2076          * enabled in rte_eth_dev_configure().
2077          */
2078         if ((local_conf.offloads & dev_info.tx_queue_offload_capa) !=
2079              local_conf.offloads) {
2080                 RTE_ETHDEV_LOG(ERR,
2081                         "Ethdev port_id=%d tx_queue_id=%d, new added offloads 0x%"PRIx64" must be "
2082                         "within per-queue offload capabilities 0x%"PRIx64" in %s()\n",
2083                         port_id, tx_queue_id, local_conf.offloads,
2084                         dev_info.tx_queue_offload_capa,
2085                         __func__);
2086                 return -EINVAL;
2087         }
2088
2089         rte_ethdev_trace_txq_setup(port_id, tx_queue_id, nb_tx_desc, tx_conf);
2090         return eth_err(port_id, (*dev->dev_ops->tx_queue_setup)(dev,
2091                        tx_queue_id, nb_tx_desc, socket_id, &local_conf));
2092 }
2093
2094 int
2095 rte_eth_tx_hairpin_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
2096                                uint16_t nb_tx_desc,
2097                                const struct rte_eth_hairpin_conf *conf)
2098 {
2099         struct rte_eth_dev *dev;
2100         struct rte_eth_hairpin_cap cap;
2101         void **txq;
2102         int i;
2103         int count;
2104         int ret;
2105
2106         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2107         dev = &rte_eth_devices[port_id];
2108         if (tx_queue_id >= dev->data->nb_tx_queues) {
2109                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
2110                 return -EINVAL;
2111         }
2112         ret = rte_eth_dev_hairpin_capability_get(port_id, &cap);
2113         if (ret != 0)
2114                 return ret;
2115         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_hairpin_queue_setup,
2116                                 -ENOTSUP);
2117         /* if nb_rx_desc is zero use max number of desc from the driver. */
2118         if (nb_tx_desc == 0)
2119                 nb_tx_desc = cap.max_nb_desc;
2120         if (nb_tx_desc > cap.max_nb_desc) {
2121                 RTE_ETHDEV_LOG(ERR,
2122                         "Invalid value for nb_tx_desc(=%hu), should be: <= %hu",
2123                         nb_tx_desc, cap.max_nb_desc);
2124                 return -EINVAL;
2125         }
2126         if (conf->peer_count > cap.max_tx_2_rx) {
2127                 RTE_ETHDEV_LOG(ERR,
2128                         "Invalid value for number of peers for Tx queue(=%hu), should be: <= %hu",
2129                         conf->peer_count, cap.max_tx_2_rx);
2130                 return -EINVAL;
2131         }
2132         if (conf->peer_count == 0) {
2133                 RTE_ETHDEV_LOG(ERR,
2134                         "Invalid value for number of peers for Tx queue(=%hu), should be: > 0",
2135                         conf->peer_count);
2136                 return -EINVAL;
2137         }
2138         for (i = 0, count = 0; i < dev->data->nb_tx_queues &&
2139              cap.max_nb_queues != UINT16_MAX; i++) {
2140                 if (i == tx_queue_id || rte_eth_dev_is_tx_hairpin_queue(dev, i))
2141                         count++;
2142         }
2143         if (count > cap.max_nb_queues) {
2144                 RTE_ETHDEV_LOG(ERR, "To many Tx hairpin queues max is %d",
2145                 cap.max_nb_queues);
2146                 return -EINVAL;
2147         }
2148         if (dev->data->dev_started)
2149                 return -EBUSY;
2150         txq = dev->data->tx_queues;
2151         if (txq[tx_queue_id] != NULL) {
2152                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release,
2153                                         -ENOTSUP);
2154                 (*dev->dev_ops->tx_queue_release)(txq[tx_queue_id]);
2155                 txq[tx_queue_id] = NULL;
2156         }
2157         ret = (*dev->dev_ops->tx_hairpin_queue_setup)
2158                 (dev, tx_queue_id, nb_tx_desc, conf);
2159         if (ret == 0)
2160                 dev->data->tx_queue_state[tx_queue_id] =
2161                         RTE_ETH_QUEUE_STATE_HAIRPIN;
2162         return eth_err(port_id, ret);
2163 }
2164
2165 void
2166 rte_eth_tx_buffer_drop_callback(struct rte_mbuf **pkts, uint16_t unsent,
2167                 void *userdata __rte_unused)
2168 {
2169         rte_pktmbuf_free_bulk(pkts, unsent);
2170 }
2171
2172 void
2173 rte_eth_tx_buffer_count_callback(struct rte_mbuf **pkts, uint16_t unsent,
2174                 void *userdata)
2175 {
2176         uint64_t *count = userdata;
2177
2178         rte_pktmbuf_free_bulk(pkts, unsent);
2179         *count += unsent;
2180 }
2181
2182 int
2183 rte_eth_tx_buffer_set_err_callback(struct rte_eth_dev_tx_buffer *buffer,
2184                 buffer_tx_error_fn cbfn, void *userdata)
2185 {
2186         buffer->error_callback = cbfn;
2187         buffer->error_userdata = userdata;
2188         return 0;
2189 }
2190
2191 int
2192 rte_eth_tx_buffer_init(struct rte_eth_dev_tx_buffer *buffer, uint16_t size)
2193 {
2194         int ret = 0;
2195
2196         if (buffer == NULL)
2197                 return -EINVAL;
2198
2199         buffer->size = size;
2200         if (buffer->error_callback == NULL) {
2201                 ret = rte_eth_tx_buffer_set_err_callback(
2202                         buffer, rte_eth_tx_buffer_drop_callback, NULL);
2203         }
2204
2205         return ret;
2206 }
2207
2208 int
2209 rte_eth_tx_done_cleanup(uint16_t port_id, uint16_t queue_id, uint32_t free_cnt)
2210 {
2211         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
2212         int ret;
2213
2214         /* Validate Input Data. Bail if not valid or not supported. */
2215         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2216         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_done_cleanup, -ENOTSUP);
2217
2218         /* Call driver to free pending mbufs. */
2219         ret = (*dev->dev_ops->tx_done_cleanup)(dev->data->tx_queues[queue_id],
2220                                                free_cnt);
2221         return eth_err(port_id, ret);
2222 }
2223
2224 int
2225 rte_eth_promiscuous_enable(uint16_t port_id)
2226 {
2227         struct rte_eth_dev *dev;
2228         int diag = 0;
2229
2230         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2231         dev = &rte_eth_devices[port_id];
2232
2233         if (dev->data->promiscuous == 1)
2234                 return 0;
2235
2236         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->promiscuous_enable, -ENOTSUP);
2237
2238         diag = (*dev->dev_ops->promiscuous_enable)(dev);
2239         dev->data->promiscuous = (diag == 0) ? 1 : 0;
2240
2241         return eth_err(port_id, diag);
2242 }
2243
2244 int
2245 rte_eth_promiscuous_disable(uint16_t port_id)
2246 {
2247         struct rte_eth_dev *dev;
2248         int diag = 0;
2249
2250         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2251         dev = &rte_eth_devices[port_id];
2252
2253         if (dev->data->promiscuous == 0)
2254                 return 0;
2255
2256         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->promiscuous_disable, -ENOTSUP);
2257
2258         dev->data->promiscuous = 0;
2259         diag = (*dev->dev_ops->promiscuous_disable)(dev);
2260         if (diag != 0)
2261                 dev->data->promiscuous = 1;
2262
2263         return eth_err(port_id, diag);
2264 }
2265
2266 int
2267 rte_eth_promiscuous_get(uint16_t port_id)
2268 {
2269         struct rte_eth_dev *dev;
2270
2271         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2272
2273         dev = &rte_eth_devices[port_id];
2274         return dev->data->promiscuous;
2275 }
2276
2277 int
2278 rte_eth_allmulticast_enable(uint16_t port_id)
2279 {
2280         struct rte_eth_dev *dev;
2281         int diag;
2282
2283         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2284         dev = &rte_eth_devices[port_id];
2285
2286         if (dev->data->all_multicast == 1)
2287                 return 0;
2288
2289         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->allmulticast_enable, -ENOTSUP);
2290         diag = (*dev->dev_ops->allmulticast_enable)(dev);
2291         dev->data->all_multicast = (diag == 0) ? 1 : 0;
2292
2293         return eth_err(port_id, diag);
2294 }
2295
2296 int
2297 rte_eth_allmulticast_disable(uint16_t port_id)
2298 {
2299         struct rte_eth_dev *dev;
2300         int diag;
2301
2302         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2303         dev = &rte_eth_devices[port_id];
2304
2305         if (dev->data->all_multicast == 0)
2306                 return 0;
2307
2308         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->allmulticast_disable, -ENOTSUP);
2309         dev->data->all_multicast = 0;
2310         diag = (*dev->dev_ops->allmulticast_disable)(dev);
2311         if (diag != 0)
2312                 dev->data->all_multicast = 1;
2313
2314         return eth_err(port_id, diag);
2315 }
2316
2317 int
2318 rte_eth_allmulticast_get(uint16_t port_id)
2319 {
2320         struct rte_eth_dev *dev;
2321
2322         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2323
2324         dev = &rte_eth_devices[port_id];
2325         return dev->data->all_multicast;
2326 }
2327
2328 int
2329 rte_eth_link_get(uint16_t port_id, struct rte_eth_link *eth_link)
2330 {
2331         struct rte_eth_dev *dev;
2332
2333         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2334         dev = &rte_eth_devices[port_id];
2335
2336         if (dev->data->dev_conf.intr_conf.lsc &&
2337             dev->data->dev_started)
2338                 rte_eth_linkstatus_get(dev, eth_link);
2339         else {
2340                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
2341                 (*dev->dev_ops->link_update)(dev, 1);
2342                 *eth_link = dev->data->dev_link;
2343         }
2344
2345         return 0;
2346 }
2347
2348 int
2349 rte_eth_link_get_nowait(uint16_t port_id, struct rte_eth_link *eth_link)
2350 {
2351         struct rte_eth_dev *dev;
2352
2353         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2354         dev = &rte_eth_devices[port_id];
2355
2356         if (dev->data->dev_conf.intr_conf.lsc &&
2357             dev->data->dev_started)
2358                 rte_eth_linkstatus_get(dev, eth_link);
2359         else {
2360                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
2361                 (*dev->dev_ops->link_update)(dev, 0);
2362                 *eth_link = dev->data->dev_link;
2363         }
2364
2365         return 0;
2366 }
2367
2368 const char *
2369 rte_eth_link_speed_to_str(uint32_t link_speed)
2370 {
2371         switch (link_speed) {
2372         case ETH_SPEED_NUM_NONE: return "None";
2373         case ETH_SPEED_NUM_10M:  return "10 Mbps";
2374         case ETH_SPEED_NUM_100M: return "100 Mbps";
2375         case ETH_SPEED_NUM_1G:   return "1 Gbps";
2376         case ETH_SPEED_NUM_2_5G: return "2.5 Gbps";
2377         case ETH_SPEED_NUM_5G:   return "5 Gbps";
2378         case ETH_SPEED_NUM_10G:  return "10 Gbps";
2379         case ETH_SPEED_NUM_20G:  return "20 Gbps";
2380         case ETH_SPEED_NUM_25G:  return "25 Gbps";
2381         case ETH_SPEED_NUM_40G:  return "40 Gbps";
2382         case ETH_SPEED_NUM_50G:  return "50 Gbps";
2383         case ETH_SPEED_NUM_56G:  return "56 Gbps";
2384         case ETH_SPEED_NUM_100G: return "100 Gbps";
2385         case ETH_SPEED_NUM_200G: return "200 Gbps";
2386         case ETH_SPEED_NUM_UNKNOWN: return "Unknown";
2387         default: return "Invalid";
2388         }
2389 }
2390
2391 int
2392 rte_eth_link_to_str(char *str, size_t len, const struct rte_eth_link *eth_link)
2393 {
2394         if (eth_link->link_status == ETH_LINK_DOWN)
2395                 return snprintf(str, len, "Link down");
2396         else
2397                 return snprintf(str, len, "Link up at %s %s %s",
2398                         rte_eth_link_speed_to_str(eth_link->link_speed),
2399                         (eth_link->link_duplex == ETH_LINK_FULL_DUPLEX) ?
2400                         "FDX" : "HDX",
2401                         (eth_link->link_autoneg == ETH_LINK_AUTONEG) ?
2402                         "Autoneg" : "Fixed");
2403 }
2404
2405 int
2406 rte_eth_stats_get(uint16_t port_id, struct rte_eth_stats *stats)
2407 {
2408         struct rte_eth_dev *dev;
2409
2410         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2411
2412         dev = &rte_eth_devices[port_id];
2413         memset(stats, 0, sizeof(*stats));
2414
2415         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
2416         stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
2417         return eth_err(port_id, (*dev->dev_ops->stats_get)(dev, stats));
2418 }
2419
2420 int
2421 rte_eth_stats_reset(uint16_t port_id)
2422 {
2423         struct rte_eth_dev *dev;
2424         int ret;
2425
2426         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2427         dev = &rte_eth_devices[port_id];
2428
2429         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_reset, -ENOTSUP);
2430         ret = (*dev->dev_ops->stats_reset)(dev);
2431         if (ret != 0)
2432                 return eth_err(port_id, ret);
2433
2434         dev->data->rx_mbuf_alloc_failed = 0;
2435
2436         return 0;
2437 }
2438
2439 static inline int
2440 get_xstats_basic_count(struct rte_eth_dev *dev)
2441 {
2442         uint16_t nb_rxqs, nb_txqs;
2443         int count;
2444
2445         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2446         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2447
2448         count = RTE_NB_STATS;
2449         count += nb_rxqs * RTE_NB_RXQ_STATS;
2450         count += nb_txqs * RTE_NB_TXQ_STATS;
2451
2452         return count;
2453 }
2454
2455 static int
2456 get_xstats_count(uint16_t port_id)
2457 {
2458         struct rte_eth_dev *dev;
2459         int count;
2460
2461         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2462         dev = &rte_eth_devices[port_id];
2463         if (dev->dev_ops->xstats_get_names_by_id != NULL) {
2464                 count = (*dev->dev_ops->xstats_get_names_by_id)(dev, NULL,
2465                                 NULL, 0);
2466                 if (count < 0)
2467                         return eth_err(port_id, count);
2468         }
2469         if (dev->dev_ops->xstats_get_names != NULL) {
2470                 count = (*dev->dev_ops->xstats_get_names)(dev, NULL, 0);
2471                 if (count < 0)
2472                         return eth_err(port_id, count);
2473         } else
2474                 count = 0;
2475
2476
2477         count += get_xstats_basic_count(dev);
2478
2479         return count;
2480 }
2481
2482 int
2483 rte_eth_xstats_get_id_by_name(uint16_t port_id, const char *xstat_name,
2484                 uint64_t *id)
2485 {
2486         int cnt_xstats, idx_xstat;
2487
2488         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2489
2490         if (!id) {
2491                 RTE_ETHDEV_LOG(ERR, "Id pointer is NULL\n");
2492                 return -ENOMEM;
2493         }
2494
2495         if (!xstat_name) {
2496                 RTE_ETHDEV_LOG(ERR, "xstat_name pointer is NULL\n");
2497                 return -ENOMEM;
2498         }
2499
2500         /* Get count */
2501         cnt_xstats = rte_eth_xstats_get_names_by_id(port_id, NULL, 0, NULL);
2502         if (cnt_xstats  < 0) {
2503                 RTE_ETHDEV_LOG(ERR, "Cannot get count of xstats\n");
2504                 return -ENODEV;
2505         }
2506
2507         /* Get id-name lookup table */
2508         struct rte_eth_xstat_name xstats_names[cnt_xstats];
2509
2510         if (cnt_xstats != rte_eth_xstats_get_names_by_id(
2511                         port_id, xstats_names, cnt_xstats, NULL)) {
2512                 RTE_ETHDEV_LOG(ERR, "Cannot get xstats lookup\n");
2513                 return -1;
2514         }
2515
2516         for (idx_xstat = 0; idx_xstat < cnt_xstats; idx_xstat++) {
2517                 if (!strcmp(xstats_names[idx_xstat].name, xstat_name)) {
2518                         *id = idx_xstat;
2519                         return 0;
2520                 };
2521         }
2522
2523         return -EINVAL;
2524 }
2525
2526 /* retrieve basic stats names */
2527 static int
2528 rte_eth_basic_stats_get_names(struct rte_eth_dev *dev,
2529         struct rte_eth_xstat_name *xstats_names)
2530 {
2531         int cnt_used_entries = 0;
2532         uint32_t idx, id_queue;
2533         uint16_t num_q;
2534
2535         for (idx = 0; idx < RTE_NB_STATS; idx++) {
2536                 strlcpy(xstats_names[cnt_used_entries].name,
2537                         rte_stats_strings[idx].name,
2538                         sizeof(xstats_names[0].name));
2539                 cnt_used_entries++;
2540         }
2541         num_q = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2542         for (id_queue = 0; id_queue < num_q; id_queue++) {
2543                 for (idx = 0; idx < RTE_NB_RXQ_STATS; idx++) {
2544                         snprintf(xstats_names[cnt_used_entries].name,
2545                                 sizeof(xstats_names[0].name),
2546                                 "rx_q%u%s",
2547                                 id_queue, rte_rxq_stats_strings[idx].name);
2548                         cnt_used_entries++;
2549                 }
2550
2551         }
2552         num_q = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2553         for (id_queue = 0; id_queue < num_q; id_queue++) {
2554                 for (idx = 0; idx < RTE_NB_TXQ_STATS; idx++) {
2555                         snprintf(xstats_names[cnt_used_entries].name,
2556                                 sizeof(xstats_names[0].name),
2557                                 "tx_q%u%s",
2558                                 id_queue, rte_txq_stats_strings[idx].name);
2559                         cnt_used_entries++;
2560                 }
2561         }
2562         return cnt_used_entries;
2563 }
2564
2565 /* retrieve ethdev extended statistics names */
2566 int
2567 rte_eth_xstats_get_names_by_id(uint16_t port_id,
2568         struct rte_eth_xstat_name *xstats_names, unsigned int size,
2569         uint64_t *ids)
2570 {
2571         struct rte_eth_xstat_name *xstats_names_copy;
2572         unsigned int no_basic_stat_requested = 1;
2573         unsigned int no_ext_stat_requested = 1;
2574         unsigned int expected_entries;
2575         unsigned int basic_count;
2576         struct rte_eth_dev *dev;
2577         unsigned int i;
2578         int ret;
2579
2580         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2581         dev = &rte_eth_devices[port_id];
2582
2583         basic_count = get_xstats_basic_count(dev);
2584         ret = get_xstats_count(port_id);
2585         if (ret < 0)
2586                 return ret;
2587         expected_entries = (unsigned int)ret;
2588
2589         /* Return max number of stats if no ids given */
2590         if (!ids) {
2591                 if (!xstats_names)
2592                         return expected_entries;
2593                 else if (xstats_names && size < expected_entries)
2594                         return expected_entries;
2595         }
2596
2597         if (ids && !xstats_names)
2598                 return -EINVAL;
2599
2600         if (ids && dev->dev_ops->xstats_get_names_by_id != NULL && size > 0) {
2601                 uint64_t ids_copy[size];
2602
2603                 for (i = 0; i < size; i++) {
2604                         if (ids[i] < basic_count) {
2605                                 no_basic_stat_requested = 0;
2606                                 break;
2607                         }
2608
2609                         /*
2610                          * Convert ids to xstats ids that PMD knows.
2611                          * ids known by user are basic + extended stats.
2612                          */
2613                         ids_copy[i] = ids[i] - basic_count;
2614                 }
2615
2616                 if (no_basic_stat_requested)
2617                         return (*dev->dev_ops->xstats_get_names_by_id)(dev,
2618                                         xstats_names, ids_copy, size);
2619         }
2620
2621         /* Retrieve all stats */
2622         if (!ids) {
2623                 int num_stats = rte_eth_xstats_get_names(port_id, xstats_names,
2624                                 expected_entries);
2625                 if (num_stats < 0 || num_stats > (int)expected_entries)
2626                         return num_stats;
2627                 else
2628                         return expected_entries;
2629         }
2630
2631         xstats_names_copy = calloc(expected_entries,
2632                 sizeof(struct rte_eth_xstat_name));
2633
2634         if (!xstats_names_copy) {
2635                 RTE_ETHDEV_LOG(ERR, "Can't allocate memory\n");
2636                 return -ENOMEM;
2637         }
2638
2639         if (ids) {
2640                 for (i = 0; i < size; i++) {
2641                         if (ids[i] >= basic_count) {
2642                                 no_ext_stat_requested = 0;
2643                                 break;
2644                         }
2645                 }
2646         }
2647
2648         /* Fill xstats_names_copy structure */
2649         if (ids && no_ext_stat_requested) {
2650                 rte_eth_basic_stats_get_names(dev, xstats_names_copy);
2651         } else {
2652                 ret = rte_eth_xstats_get_names(port_id, xstats_names_copy,
2653                         expected_entries);
2654                 if (ret < 0) {
2655                         free(xstats_names_copy);
2656                         return ret;
2657                 }
2658         }
2659
2660         /* Filter stats */
2661         for (i = 0; i < size; i++) {
2662                 if (ids[i] >= expected_entries) {
2663                         RTE_ETHDEV_LOG(ERR, "Id value isn't valid\n");
2664                         free(xstats_names_copy);
2665                         return -1;
2666                 }
2667                 xstats_names[i] = xstats_names_copy[ids[i]];
2668         }
2669
2670         free(xstats_names_copy);
2671         return size;
2672 }
2673
2674 int
2675 rte_eth_xstats_get_names(uint16_t port_id,
2676         struct rte_eth_xstat_name *xstats_names,
2677         unsigned int size)
2678 {
2679         struct rte_eth_dev *dev;
2680         int cnt_used_entries;
2681         int cnt_expected_entries;
2682         int cnt_driver_entries;
2683
2684         cnt_expected_entries = get_xstats_count(port_id);
2685         if (xstats_names == NULL || cnt_expected_entries < 0 ||
2686                         (int)size < cnt_expected_entries)
2687                 return cnt_expected_entries;
2688
2689         /* port_id checked in get_xstats_count() */
2690         dev = &rte_eth_devices[port_id];
2691
2692         cnt_used_entries = rte_eth_basic_stats_get_names(
2693                 dev, xstats_names);
2694
2695         if (dev->dev_ops->xstats_get_names != NULL) {
2696                 /* If there are any driver-specific xstats, append them
2697                  * to end of list.
2698                  */
2699                 cnt_driver_entries = (*dev->dev_ops->xstats_get_names)(
2700                         dev,
2701                         xstats_names + cnt_used_entries,
2702                         size - cnt_used_entries);
2703                 if (cnt_driver_entries < 0)
2704                         return eth_err(port_id, cnt_driver_entries);
2705                 cnt_used_entries += cnt_driver_entries;
2706         }
2707
2708         return cnt_used_entries;
2709 }
2710
2711
2712 static int
2713 rte_eth_basic_stats_get(uint16_t port_id, struct rte_eth_xstat *xstats)
2714 {
2715         struct rte_eth_dev *dev;
2716         struct rte_eth_stats eth_stats;
2717         unsigned int count = 0, i, q;
2718         uint64_t val, *stats_ptr;
2719         uint16_t nb_rxqs, nb_txqs;
2720         int ret;
2721
2722         ret = rte_eth_stats_get(port_id, &eth_stats);
2723         if (ret < 0)
2724                 return ret;
2725
2726         dev = &rte_eth_devices[port_id];
2727
2728         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2729         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2730
2731         /* global stats */
2732         for (i = 0; i < RTE_NB_STATS; i++) {
2733                 stats_ptr = RTE_PTR_ADD(&eth_stats,
2734                                         rte_stats_strings[i].offset);
2735                 val = *stats_ptr;
2736                 xstats[count++].value = val;
2737         }
2738
2739         /* per-rxq stats */
2740         for (q = 0; q < nb_rxqs; q++) {
2741                 for (i = 0; i < RTE_NB_RXQ_STATS; i++) {
2742                         stats_ptr = RTE_PTR_ADD(&eth_stats,
2743                                         rte_rxq_stats_strings[i].offset +
2744                                         q * sizeof(uint64_t));
2745                         val = *stats_ptr;
2746                         xstats[count++].value = val;
2747                 }
2748         }
2749
2750         /* per-txq stats */
2751         for (q = 0; q < nb_txqs; q++) {
2752                 for (i = 0; i < RTE_NB_TXQ_STATS; i++) {
2753                         stats_ptr = RTE_PTR_ADD(&eth_stats,
2754                                         rte_txq_stats_strings[i].offset +
2755                                         q * sizeof(uint64_t));
2756                         val = *stats_ptr;
2757                         xstats[count++].value = val;
2758                 }
2759         }
2760         return count;
2761 }
2762
2763 /* retrieve ethdev extended statistics */
2764 int
2765 rte_eth_xstats_get_by_id(uint16_t port_id, const uint64_t *ids,
2766                          uint64_t *values, unsigned int size)
2767 {
2768         unsigned int no_basic_stat_requested = 1;
2769         unsigned int no_ext_stat_requested = 1;
2770         unsigned int num_xstats_filled;
2771         unsigned int basic_count;
2772         uint16_t expected_entries;
2773         struct rte_eth_dev *dev;
2774         unsigned int i;
2775         int ret;
2776
2777         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2778         ret = get_xstats_count(port_id);
2779         if (ret < 0)
2780                 return ret;
2781         expected_entries = (uint16_t)ret;
2782         struct rte_eth_xstat xstats[expected_entries];
2783         dev = &rte_eth_devices[port_id];
2784         basic_count = get_xstats_basic_count(dev);
2785
2786         /* Return max number of stats if no ids given */
2787         if (!ids) {
2788                 if (!values)
2789                         return expected_entries;
2790                 else if (values && size < expected_entries)
2791                         return expected_entries;
2792         }
2793
2794         if (ids && !values)
2795                 return -EINVAL;
2796
2797         if (ids && dev->dev_ops->xstats_get_by_id != NULL && size) {
2798                 unsigned int basic_count = get_xstats_basic_count(dev);
2799                 uint64_t ids_copy[size];
2800
2801                 for (i = 0; i < size; i++) {
2802                         if (ids[i] < basic_count) {
2803                                 no_basic_stat_requested = 0;
2804                                 break;
2805                         }
2806
2807                         /*
2808                          * Convert ids to xstats ids that PMD knows.
2809                          * ids known by user are basic + extended stats.
2810                          */
2811                         ids_copy[i] = ids[i] - basic_count;
2812                 }
2813
2814                 if (no_basic_stat_requested)
2815                         return (*dev->dev_ops->xstats_get_by_id)(dev, ids_copy,
2816                                         values, size);
2817         }
2818
2819         if (ids) {
2820                 for (i = 0; i < size; i++) {
2821                         if (ids[i] >= basic_count) {
2822                                 no_ext_stat_requested = 0;
2823                                 break;
2824                         }
2825                 }
2826         }
2827
2828         /* Fill the xstats structure */
2829         if (ids && no_ext_stat_requested)
2830                 ret = rte_eth_basic_stats_get(port_id, xstats);
2831         else
2832                 ret = rte_eth_xstats_get(port_id, xstats, expected_entries);
2833
2834         if (ret < 0)
2835                 return ret;
2836         num_xstats_filled = (unsigned int)ret;
2837
2838         /* Return all stats */
2839         if (!ids) {
2840                 for (i = 0; i < num_xstats_filled; i++)
2841                         values[i] = xstats[i].value;
2842                 return expected_entries;
2843         }
2844
2845         /* Filter stats */
2846         for (i = 0; i < size; i++) {
2847                 if (ids[i] >= expected_entries) {
2848                         RTE_ETHDEV_LOG(ERR, "Id value isn't valid\n");
2849                         return -1;
2850                 }
2851                 values[i] = xstats[ids[i]].value;
2852         }
2853         return size;
2854 }
2855
2856 int
2857 rte_eth_xstats_get(uint16_t port_id, struct rte_eth_xstat *xstats,
2858         unsigned int n)
2859 {
2860         struct rte_eth_dev *dev;
2861         unsigned int count = 0, i;
2862         signed int xcount = 0;
2863         uint16_t nb_rxqs, nb_txqs;
2864         int ret;
2865
2866         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2867
2868         dev = &rte_eth_devices[port_id];
2869
2870         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2871         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2872
2873         /* Return generic statistics */
2874         count = RTE_NB_STATS + (nb_rxqs * RTE_NB_RXQ_STATS) +
2875                 (nb_txqs * RTE_NB_TXQ_STATS);
2876
2877         /* implemented by the driver */
2878         if (dev->dev_ops->xstats_get != NULL) {
2879                 /* Retrieve the xstats from the driver at the end of the
2880                  * xstats struct.
2881                  */
2882                 xcount = (*dev->dev_ops->xstats_get)(dev,
2883                                      xstats ? xstats + count : NULL,
2884                                      (n > count) ? n - count : 0);
2885
2886                 if (xcount < 0)
2887                         return eth_err(port_id, xcount);
2888         }
2889
2890         if (n < count + xcount || xstats == NULL)
2891                 return count + xcount;
2892
2893         /* now fill the xstats structure */
2894         ret = rte_eth_basic_stats_get(port_id, xstats);
2895         if (ret < 0)
2896                 return ret;
2897         count = ret;
2898
2899         for (i = 0; i < count; i++)
2900                 xstats[i].id = i;
2901         /* add an offset to driver-specific stats */
2902         for ( ; i < count + xcount; i++)
2903                 xstats[i].id += count;
2904
2905         return count + xcount;
2906 }
2907
2908 /* reset ethdev extended statistics */
2909 int
2910 rte_eth_xstats_reset(uint16_t port_id)
2911 {
2912         struct rte_eth_dev *dev;
2913
2914         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2915         dev = &rte_eth_devices[port_id];
2916
2917         /* implemented by the driver */
2918         if (dev->dev_ops->xstats_reset != NULL)
2919                 return eth_err(port_id, (*dev->dev_ops->xstats_reset)(dev));
2920
2921         /* fallback to default */
2922         return rte_eth_stats_reset(port_id);
2923 }
2924
2925 static int
2926 set_queue_stats_mapping(uint16_t port_id, uint16_t queue_id, uint8_t stat_idx,
2927                 uint8_t is_rx)
2928 {
2929         struct rte_eth_dev *dev;
2930
2931         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2932
2933         dev = &rte_eth_devices[port_id];
2934
2935         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_stats_mapping_set, -ENOTSUP);
2936
2937         if (is_rx && (queue_id >= dev->data->nb_rx_queues))
2938                 return -EINVAL;
2939
2940         if (!is_rx && (queue_id >= dev->data->nb_tx_queues))
2941                 return -EINVAL;
2942
2943         if (stat_idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
2944                 return -EINVAL;
2945
2946         return (*dev->dev_ops->queue_stats_mapping_set)
2947                         (dev, queue_id, stat_idx, is_rx);
2948 }
2949
2950
2951 int
2952 rte_eth_dev_set_tx_queue_stats_mapping(uint16_t port_id, uint16_t tx_queue_id,
2953                 uint8_t stat_idx)
2954 {
2955         return eth_err(port_id, set_queue_stats_mapping(port_id, tx_queue_id,
2956                                                 stat_idx, STAT_QMAP_TX));
2957 }
2958
2959
2960 int
2961 rte_eth_dev_set_rx_queue_stats_mapping(uint16_t port_id, uint16_t rx_queue_id,
2962                 uint8_t stat_idx)
2963 {
2964         return eth_err(port_id, set_queue_stats_mapping(port_id, rx_queue_id,
2965                                                 stat_idx, STAT_QMAP_RX));
2966 }
2967
2968 int
2969 rte_eth_dev_fw_version_get(uint16_t port_id, char *fw_version, size_t fw_size)
2970 {
2971         struct rte_eth_dev *dev;
2972
2973         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2974         dev = &rte_eth_devices[port_id];
2975
2976         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fw_version_get, -ENOTSUP);
2977         return eth_err(port_id, (*dev->dev_ops->fw_version_get)(dev,
2978                                                         fw_version, fw_size));
2979 }
2980
2981 int
2982 rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info)
2983 {
2984         struct rte_eth_dev *dev;
2985         const struct rte_eth_desc_lim lim = {
2986                 .nb_max = UINT16_MAX,
2987                 .nb_min = 0,
2988                 .nb_align = 1,
2989                 .nb_seg_max = UINT16_MAX,
2990                 .nb_mtu_seg_max = UINT16_MAX,
2991         };
2992         int diag;
2993
2994         /*
2995          * Init dev_info before port_id check since caller does not have
2996          * return status and does not know if get is successful or not.
2997          */
2998         memset(dev_info, 0, sizeof(struct rte_eth_dev_info));
2999         dev_info->switch_info.domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
3000
3001         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3002         dev = &rte_eth_devices[port_id];
3003
3004         dev_info->rx_desc_lim = lim;
3005         dev_info->tx_desc_lim = lim;
3006         dev_info->device = dev->device;
3007         dev_info->min_mtu = RTE_ETHER_MIN_MTU;
3008         dev_info->max_mtu = UINT16_MAX;
3009
3010         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
3011         diag = (*dev->dev_ops->dev_infos_get)(dev, dev_info);
3012         if (diag != 0) {
3013                 /* Cleanup already filled in device information */
3014                 memset(dev_info, 0, sizeof(struct rte_eth_dev_info));
3015                 return eth_err(port_id, diag);
3016         }
3017
3018         /* Maximum number of queues should be <= RTE_MAX_QUEUES_PER_PORT */
3019         dev_info->max_rx_queues = RTE_MIN(dev_info->max_rx_queues,
3020                         RTE_MAX_QUEUES_PER_PORT);
3021         dev_info->max_tx_queues = RTE_MIN(dev_info->max_tx_queues,
3022                         RTE_MAX_QUEUES_PER_PORT);
3023
3024         dev_info->driver_name = dev->device->driver->name;
3025         dev_info->nb_rx_queues = dev->data->nb_rx_queues;
3026         dev_info->nb_tx_queues = dev->data->nb_tx_queues;
3027
3028         dev_info->dev_flags = &dev->data->dev_flags;
3029
3030         return 0;
3031 }
3032
3033 int
3034 rte_eth_dev_get_supported_ptypes(uint16_t port_id, uint32_t ptype_mask,
3035                                  uint32_t *ptypes, int num)
3036 {
3037         int i, j;
3038         struct rte_eth_dev *dev;
3039         const uint32_t *all_ptypes;
3040
3041         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3042         dev = &rte_eth_devices[port_id];
3043         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_supported_ptypes_get, 0);
3044         all_ptypes = (*dev->dev_ops->dev_supported_ptypes_get)(dev);
3045
3046         if (!all_ptypes)
3047                 return 0;
3048
3049         for (i = 0, j = 0; all_ptypes[i] != RTE_PTYPE_UNKNOWN; ++i)
3050                 if (all_ptypes[i] & ptype_mask) {
3051                         if (j < num)
3052                                 ptypes[j] = all_ptypes[i];
3053                         j++;
3054                 }
3055
3056         return j;
3057 }
3058
3059 int
3060 rte_eth_dev_set_ptypes(uint16_t port_id, uint32_t ptype_mask,
3061                                  uint32_t *set_ptypes, unsigned int num)
3062 {
3063         const uint32_t valid_ptype_masks[] = {
3064                 RTE_PTYPE_L2_MASK,
3065                 RTE_PTYPE_L3_MASK,
3066                 RTE_PTYPE_L4_MASK,
3067                 RTE_PTYPE_TUNNEL_MASK,
3068                 RTE_PTYPE_INNER_L2_MASK,
3069                 RTE_PTYPE_INNER_L3_MASK,
3070                 RTE_PTYPE_INNER_L4_MASK,
3071         };
3072         const uint32_t *all_ptypes;
3073         struct rte_eth_dev *dev;
3074         uint32_t unused_mask;
3075         unsigned int i, j;
3076         int ret;
3077
3078         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3079         dev = &rte_eth_devices[port_id];
3080
3081         if (num > 0 && set_ptypes == NULL)
3082                 return -EINVAL;
3083
3084         if (*dev->dev_ops->dev_supported_ptypes_get == NULL ||
3085                         *dev->dev_ops->dev_ptypes_set == NULL) {
3086                 ret = 0;
3087                 goto ptype_unknown;
3088         }
3089
3090         if (ptype_mask == 0) {
3091                 ret = (*dev->dev_ops->dev_ptypes_set)(dev,
3092                                 ptype_mask);
3093                 goto ptype_unknown;
3094         }
3095
3096         unused_mask = ptype_mask;
3097         for (i = 0; i < RTE_DIM(valid_ptype_masks); i++) {
3098                 uint32_t mask = ptype_mask & valid_ptype_masks[i];
3099                 if (mask && mask != valid_ptype_masks[i]) {
3100                         ret = -EINVAL;
3101                         goto ptype_unknown;
3102                 }
3103                 unused_mask &= ~valid_ptype_masks[i];
3104         }
3105
3106         if (unused_mask) {
3107                 ret = -EINVAL;
3108                 goto ptype_unknown;
3109         }
3110
3111         all_ptypes = (*dev->dev_ops->dev_supported_ptypes_get)(dev);
3112         if (all_ptypes == NULL) {
3113                 ret = 0;
3114                 goto ptype_unknown;
3115         }
3116
3117         /*
3118          * Accommodate as many set_ptypes as possible. If the supplied
3119          * set_ptypes array is insufficient fill it partially.
3120          */
3121         for (i = 0, j = 0; set_ptypes != NULL &&
3122                                 (all_ptypes[i] != RTE_PTYPE_UNKNOWN); ++i) {
3123                 if (ptype_mask & all_ptypes[i]) {
3124                         if (j < num - 1) {
3125                                 set_ptypes[j] = all_ptypes[i];
3126                                 j++;
3127                                 continue;
3128                         }
3129                         break;
3130                 }
3131         }
3132
3133         if (set_ptypes != NULL && j < num)
3134                 set_ptypes[j] = RTE_PTYPE_UNKNOWN;
3135
3136         return (*dev->dev_ops->dev_ptypes_set)(dev, ptype_mask);
3137
3138 ptype_unknown:
3139         if (num > 0)
3140                 set_ptypes[0] = RTE_PTYPE_UNKNOWN;
3141
3142         return ret;
3143 }
3144
3145 int
3146 rte_eth_macaddr_get(uint16_t port_id, struct rte_ether_addr *mac_addr)
3147 {
3148         struct rte_eth_dev *dev;
3149
3150         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3151         dev = &rte_eth_devices[port_id];
3152         rte_ether_addr_copy(&dev->data->mac_addrs[0], mac_addr);
3153
3154         return 0;
3155 }
3156
3157 int
3158 rte_eth_dev_get_mtu(uint16_t port_id, uint16_t *mtu)
3159 {
3160         struct rte_eth_dev *dev;
3161
3162         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3163
3164         dev = &rte_eth_devices[port_id];
3165         *mtu = dev->data->mtu;
3166         return 0;
3167 }
3168
3169 int
3170 rte_eth_dev_set_mtu(uint16_t port_id, uint16_t mtu)
3171 {
3172         int ret;
3173         struct rte_eth_dev_info dev_info;
3174         struct rte_eth_dev *dev;
3175
3176         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3177         dev = &rte_eth_devices[port_id];
3178         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mtu_set, -ENOTSUP);
3179
3180         /*
3181          * Check if the device supports dev_infos_get, if it does not
3182          * skip min_mtu/max_mtu validation here as this requires values
3183          * that are populated within the call to rte_eth_dev_info_get()
3184          * which relies on dev->dev_ops->dev_infos_get.
3185          */
3186         if (*dev->dev_ops->dev_infos_get != NULL) {
3187                 ret = rte_eth_dev_info_get(port_id, &dev_info);
3188                 if (ret != 0)
3189                         return ret;
3190
3191                 if (mtu < dev_info.min_mtu || mtu > dev_info.max_mtu)
3192                         return -EINVAL;
3193         }
3194
3195         ret = (*dev->dev_ops->mtu_set)(dev, mtu);
3196         if (!ret)
3197                 dev->data->mtu = mtu;
3198
3199         return eth_err(port_id, ret);
3200 }
3201
3202 int
3203 rte_eth_dev_vlan_filter(uint16_t port_id, uint16_t vlan_id, int on)
3204 {
3205         struct rte_eth_dev *dev;
3206         int ret;
3207
3208         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3209         dev = &rte_eth_devices[port_id];
3210         if (!(dev->data->dev_conf.rxmode.offloads &
3211               DEV_RX_OFFLOAD_VLAN_FILTER)) {
3212                 RTE_ETHDEV_LOG(ERR, "Port %u: vlan-filtering disabled\n",
3213                         port_id);
3214                 return -ENOSYS;
3215         }
3216
3217         if (vlan_id > 4095) {
3218                 RTE_ETHDEV_LOG(ERR, "Port_id=%u invalid vlan_id=%u > 4095\n",
3219                         port_id, vlan_id);
3220                 return -EINVAL;
3221         }
3222         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_filter_set, -ENOTSUP);
3223
3224         ret = (*dev->dev_ops->vlan_filter_set)(dev, vlan_id, on);
3225         if (ret == 0) {
3226                 struct rte_vlan_filter_conf *vfc;
3227                 int vidx;
3228                 int vbit;
3229
3230                 vfc = &dev->data->vlan_filter_conf;
3231                 vidx = vlan_id / 64;
3232                 vbit = vlan_id % 64;
3233
3234                 if (on)
3235                         vfc->ids[vidx] |= UINT64_C(1) << vbit;
3236                 else
3237                         vfc->ids[vidx] &= ~(UINT64_C(1) << vbit);
3238         }
3239
3240         return eth_err(port_id, ret);
3241 }
3242
3243 int
3244 rte_eth_dev_set_vlan_strip_on_queue(uint16_t port_id, uint16_t rx_queue_id,
3245                                     int on)
3246 {
3247         struct rte_eth_dev *dev;
3248
3249         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3250         dev = &rte_eth_devices[port_id];
3251         if (rx_queue_id >= dev->data->nb_rx_queues) {
3252                 RTE_ETHDEV_LOG(ERR, "Invalid rx_queue_id=%u\n", rx_queue_id);
3253                 return -EINVAL;
3254         }
3255
3256         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_strip_queue_set, -ENOTSUP);
3257         (*dev->dev_ops->vlan_strip_queue_set)(dev, rx_queue_id, on);
3258
3259         return 0;
3260 }
3261
3262 int
3263 rte_eth_dev_set_vlan_ether_type(uint16_t port_id,
3264                                 enum rte_vlan_type vlan_type,
3265                                 uint16_t tpid)
3266 {
3267         struct rte_eth_dev *dev;
3268
3269         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3270         dev = &rte_eth_devices[port_id];
3271         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_tpid_set, -ENOTSUP);
3272
3273         return eth_err(port_id, (*dev->dev_ops->vlan_tpid_set)(dev, vlan_type,
3274                                                                tpid));
3275 }
3276
3277 int
3278 rte_eth_dev_set_vlan_offload(uint16_t port_id, int offload_mask)
3279 {
3280         struct rte_eth_dev_info dev_info;
3281         struct rte_eth_dev *dev;
3282         int ret = 0;
3283         int mask = 0;
3284         int cur, org = 0;
3285         uint64_t orig_offloads;
3286         uint64_t dev_offloads;
3287         uint64_t new_offloads;
3288
3289         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3290         dev = &rte_eth_devices[port_id];
3291
3292         /* save original values in case of failure */
3293         orig_offloads = dev->data->dev_conf.rxmode.offloads;
3294         dev_offloads = orig_offloads;
3295
3296         /* check which option changed by application */
3297         cur = !!(offload_mask & ETH_VLAN_STRIP_OFFLOAD);
3298         org = !!(dev_offloads & DEV_RX_OFFLOAD_VLAN_STRIP);
3299         if (cur != org) {
3300                 if (cur)
3301                         dev_offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
3302                 else
3303                         dev_offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP;
3304                 mask |= ETH_VLAN_STRIP_MASK;
3305         }
3306
3307         cur = !!(offload_mask & ETH_VLAN_FILTER_OFFLOAD);
3308         org = !!(dev_offloads & DEV_RX_OFFLOAD_VLAN_FILTER);
3309         if (cur != org) {
3310                 if (cur)
3311                         dev_offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3312                 else
3313                         dev_offloads &= ~DEV_RX_OFFLOAD_VLAN_FILTER;
3314                 mask |= ETH_VLAN_FILTER_MASK;
3315         }
3316
3317         cur = !!(offload_mask & ETH_VLAN_EXTEND_OFFLOAD);
3318         org = !!(dev_offloads & DEV_RX_OFFLOAD_VLAN_EXTEND);
3319         if (cur != org) {
3320                 if (cur)
3321                         dev_offloads |= DEV_RX_OFFLOAD_VLAN_EXTEND;
3322                 else
3323                         dev_offloads &= ~DEV_RX_OFFLOAD_VLAN_EXTEND;
3324                 mask |= ETH_VLAN_EXTEND_MASK;
3325         }
3326
3327         cur = !!(offload_mask & ETH_QINQ_STRIP_OFFLOAD);
3328         org = !!(dev_offloads & DEV_RX_OFFLOAD_QINQ_STRIP);
3329         if (cur != org) {
3330                 if (cur)
3331                         dev_offloads |= DEV_RX_OFFLOAD_QINQ_STRIP;
3332                 else
3333                         dev_offloads &= ~DEV_RX_OFFLOAD_QINQ_STRIP;
3334                 mask |= ETH_QINQ_STRIP_MASK;
3335         }
3336
3337         /*no change*/
3338         if (mask == 0)
3339                 return ret;
3340
3341         ret = rte_eth_dev_info_get(port_id, &dev_info);
3342         if (ret != 0)
3343                 return ret;
3344
3345         /* Rx VLAN offloading must be within its device capabilities */
3346         if ((dev_offloads & dev_info.rx_offload_capa) != dev_offloads) {
3347                 new_offloads = dev_offloads & ~orig_offloads;
3348                 RTE_ETHDEV_LOG(ERR,
3349                         "Ethdev port_id=%u requested new added VLAN offloads "
3350                         "0x%" PRIx64 " must be within Rx offloads capabilities "
3351                         "0x%" PRIx64 " in %s()\n",
3352                         port_id, new_offloads, dev_info.rx_offload_capa,
3353                         __func__);
3354                 return -EINVAL;
3355         }
3356
3357         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_offload_set, -ENOTSUP);
3358         dev->data->dev_conf.rxmode.offloads = dev_offloads;
3359         ret = (*dev->dev_ops->vlan_offload_set)(dev, mask);
3360         if (ret) {
3361                 /* hit an error restore  original values */
3362                 dev->data->dev_conf.rxmode.offloads = orig_offloads;
3363         }
3364
3365         return eth_err(port_id, ret);
3366 }
3367
3368 int
3369 rte_eth_dev_get_vlan_offload(uint16_t port_id)
3370 {
3371         struct rte_eth_dev *dev;
3372         uint64_t *dev_offloads;
3373         int ret = 0;
3374
3375         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3376         dev = &rte_eth_devices[port_id];
3377         dev_offloads = &dev->data->dev_conf.rxmode.offloads;
3378
3379         if (*dev_offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
3380                 ret |= ETH_VLAN_STRIP_OFFLOAD;
3381
3382         if (*dev_offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
3383                 ret |= ETH_VLAN_FILTER_OFFLOAD;
3384
3385         if (*dev_offloads & DEV_RX_OFFLOAD_VLAN_EXTEND)
3386                 ret |= ETH_VLAN_EXTEND_OFFLOAD;
3387
3388         if (*dev_offloads & DEV_RX_OFFLOAD_QINQ_STRIP)
3389                 ret |= ETH_QINQ_STRIP_OFFLOAD;
3390
3391         return ret;
3392 }
3393
3394 int
3395 rte_eth_dev_set_vlan_pvid(uint16_t port_id, uint16_t pvid, int on)
3396 {
3397         struct rte_eth_dev *dev;
3398
3399         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3400         dev = &rte_eth_devices[port_id];
3401         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_pvid_set, -ENOTSUP);
3402
3403         return eth_err(port_id, (*dev->dev_ops->vlan_pvid_set)(dev, pvid, on));
3404 }
3405
3406 int
3407 rte_eth_dev_flow_ctrl_get(uint16_t port_id, struct rte_eth_fc_conf *fc_conf)
3408 {
3409         struct rte_eth_dev *dev;
3410
3411         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3412         dev = &rte_eth_devices[port_id];
3413         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_get, -ENOTSUP);
3414         memset(fc_conf, 0, sizeof(*fc_conf));
3415         return eth_err(port_id, (*dev->dev_ops->flow_ctrl_get)(dev, fc_conf));
3416 }
3417
3418 int
3419 rte_eth_dev_flow_ctrl_set(uint16_t port_id, struct rte_eth_fc_conf *fc_conf)
3420 {
3421         struct rte_eth_dev *dev;
3422
3423         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3424         if ((fc_conf->send_xon != 0) && (fc_conf->send_xon != 1)) {
3425                 RTE_ETHDEV_LOG(ERR, "Invalid send_xon, only 0/1 allowed\n");
3426                 return -EINVAL;
3427         }
3428
3429         dev = &rte_eth_devices[port_id];
3430         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_set, -ENOTSUP);
3431         return eth_err(port_id, (*dev->dev_ops->flow_ctrl_set)(dev, fc_conf));
3432 }
3433
3434 int
3435 rte_eth_dev_priority_flow_ctrl_set(uint16_t port_id,
3436                                    struct rte_eth_pfc_conf *pfc_conf)
3437 {
3438         struct rte_eth_dev *dev;
3439
3440         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3441         if (pfc_conf->priority > (ETH_DCB_NUM_USER_PRIORITIES - 1)) {
3442                 RTE_ETHDEV_LOG(ERR, "Invalid priority, only 0-7 allowed\n");
3443                 return -EINVAL;
3444         }
3445
3446         dev = &rte_eth_devices[port_id];
3447         /* High water, low water validation are device specific */
3448         if  (*dev->dev_ops->priority_flow_ctrl_set)
3449                 return eth_err(port_id, (*dev->dev_ops->priority_flow_ctrl_set)
3450                                         (dev, pfc_conf));
3451         return -ENOTSUP;
3452 }
3453
3454 static int
3455 rte_eth_check_reta_mask(struct rte_eth_rss_reta_entry64 *reta_conf,
3456                         uint16_t reta_size)
3457 {
3458         uint16_t i, num;
3459
3460         if (!reta_conf)
3461                 return -EINVAL;
3462
3463         num = (reta_size + RTE_RETA_GROUP_SIZE - 1) / RTE_RETA_GROUP_SIZE;
3464         for (i = 0; i < num; i++) {
3465                 if (reta_conf[i].mask)
3466                         return 0;
3467         }
3468
3469         return -EINVAL;
3470 }
3471
3472 static int
3473 rte_eth_check_reta_entry(struct rte_eth_rss_reta_entry64 *reta_conf,
3474                          uint16_t reta_size,
3475                          uint16_t max_rxq)
3476 {
3477         uint16_t i, idx, shift;
3478
3479         if (!reta_conf)
3480                 return -EINVAL;
3481
3482         if (max_rxq == 0) {
3483                 RTE_ETHDEV_LOG(ERR, "No receive queue is available\n");
3484                 return -EINVAL;
3485         }
3486
3487         for (i = 0; i < reta_size; i++) {
3488                 idx = i / RTE_RETA_GROUP_SIZE;
3489                 shift = i % RTE_RETA_GROUP_SIZE;
3490                 if ((reta_conf[idx].mask & (1ULL << shift)) &&
3491                         (reta_conf[idx].reta[shift] >= max_rxq)) {
3492                         RTE_ETHDEV_LOG(ERR,
3493                                 "reta_conf[%u]->reta[%u]: %u exceeds the maximum rxq index: %u\n",
3494                                 idx, shift,
3495                                 reta_conf[idx].reta[shift], max_rxq);
3496                         return -EINVAL;
3497                 }
3498         }
3499
3500         return 0;
3501 }
3502
3503 int
3504 rte_eth_dev_rss_reta_update(uint16_t port_id,
3505                             struct rte_eth_rss_reta_entry64 *reta_conf,
3506                             uint16_t reta_size)
3507 {
3508         struct rte_eth_dev *dev;
3509         int ret;
3510
3511         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3512         /* Check mask bits */
3513         ret = rte_eth_check_reta_mask(reta_conf, reta_size);
3514         if (ret < 0)
3515                 return ret;
3516
3517         dev = &rte_eth_devices[port_id];
3518
3519         /* Check entry value */
3520         ret = rte_eth_check_reta_entry(reta_conf, reta_size,
3521                                 dev->data->nb_rx_queues);
3522         if (ret < 0)
3523                 return ret;
3524
3525         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_update, -ENOTSUP);
3526         return eth_err(port_id, (*dev->dev_ops->reta_update)(dev, reta_conf,
3527                                                              reta_size));
3528 }
3529
3530 int
3531 rte_eth_dev_rss_reta_query(uint16_t port_id,
3532                            struct rte_eth_rss_reta_entry64 *reta_conf,
3533                            uint16_t reta_size)
3534 {
3535         struct rte_eth_dev *dev;
3536         int ret;
3537
3538         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3539
3540         /* Check mask bits */
3541         ret = rte_eth_check_reta_mask(reta_conf, reta_size);
3542         if (ret < 0)
3543                 return ret;
3544
3545         dev = &rte_eth_devices[port_id];
3546         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_query, -ENOTSUP);
3547         return eth_err(port_id, (*dev->dev_ops->reta_query)(dev, reta_conf,
3548                                                             reta_size));
3549 }
3550
3551 int
3552 rte_eth_dev_rss_hash_update(uint16_t port_id,
3553                             struct rte_eth_rss_conf *rss_conf)
3554 {
3555         struct rte_eth_dev *dev;
3556         struct rte_eth_dev_info dev_info = { .flow_type_rss_offloads = 0, };
3557         int ret;
3558
3559         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3560
3561         ret = rte_eth_dev_info_get(port_id, &dev_info);
3562         if (ret != 0)
3563                 return ret;
3564
3565         rss_conf->rss_hf = rte_eth_rss_hf_refine(rss_conf->rss_hf);
3566
3567         dev = &rte_eth_devices[port_id];
3568         if ((dev_info.flow_type_rss_offloads | rss_conf->rss_hf) !=
3569             dev_info.flow_type_rss_offloads) {
3570                 RTE_ETHDEV_LOG(ERR,
3571                         "Ethdev port_id=%u invalid rss_hf: 0x%"PRIx64", valid value: 0x%"PRIx64"\n",
3572                         port_id, rss_conf->rss_hf,
3573                         dev_info.flow_type_rss_offloads);
3574                 return -EINVAL;
3575         }
3576         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_update, -ENOTSUP);
3577         return eth_err(port_id, (*dev->dev_ops->rss_hash_update)(dev,
3578                                                                  rss_conf));
3579 }
3580
3581 int
3582 rte_eth_dev_rss_hash_conf_get(uint16_t port_id,
3583                               struct rte_eth_rss_conf *rss_conf)
3584 {
3585         struct rte_eth_dev *dev;
3586
3587         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3588         dev = &rte_eth_devices[port_id];
3589         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_conf_get, -ENOTSUP);
3590         return eth_err(port_id, (*dev->dev_ops->rss_hash_conf_get)(dev,
3591                                                                    rss_conf));
3592 }
3593
3594 int
3595 rte_eth_dev_udp_tunnel_port_add(uint16_t port_id,
3596                                 struct rte_eth_udp_tunnel *udp_tunnel)
3597 {
3598         struct rte_eth_dev *dev;
3599
3600         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3601         if (udp_tunnel == NULL) {
3602                 RTE_ETHDEV_LOG(ERR, "Invalid udp_tunnel parameter\n");
3603                 return -EINVAL;
3604         }
3605
3606         if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
3607                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
3608                 return -EINVAL;
3609         }
3610
3611         dev = &rte_eth_devices[port_id];
3612         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_port_add, -ENOTSUP);
3613         return eth_err(port_id, (*dev->dev_ops->udp_tunnel_port_add)(dev,
3614                                                                 udp_tunnel));
3615 }
3616
3617 int
3618 rte_eth_dev_udp_tunnel_port_delete(uint16_t port_id,
3619                                    struct rte_eth_udp_tunnel *udp_tunnel)
3620 {
3621         struct rte_eth_dev *dev;
3622
3623         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3624         dev = &rte_eth_devices[port_id];
3625
3626         if (udp_tunnel == NULL) {
3627                 RTE_ETHDEV_LOG(ERR, "Invalid udp_tunnel parameter\n");
3628                 return -EINVAL;
3629         }
3630
3631         if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
3632                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
3633                 return -EINVAL;
3634         }
3635
3636         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_port_del, -ENOTSUP);
3637         return eth_err(port_id, (*dev->dev_ops->udp_tunnel_port_del)(dev,
3638                                                                 udp_tunnel));
3639 }
3640
3641 int
3642 rte_eth_led_on(uint16_t port_id)
3643 {
3644         struct rte_eth_dev *dev;
3645
3646         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3647         dev = &rte_eth_devices[port_id];
3648         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_on, -ENOTSUP);
3649         return eth_err(port_id, (*dev->dev_ops->dev_led_on)(dev));
3650 }
3651
3652 int
3653 rte_eth_led_off(uint16_t port_id)
3654 {
3655         struct rte_eth_dev *dev;
3656
3657         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3658         dev = &rte_eth_devices[port_id];
3659         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_off, -ENOTSUP);
3660         return eth_err(port_id, (*dev->dev_ops->dev_led_off)(dev));
3661 }
3662
3663 int
3664 rte_eth_fec_get_capability(uint16_t port_id,
3665                            struct rte_eth_fec_capa *speed_fec_capa,
3666                            unsigned int num)
3667 {
3668         struct rte_eth_dev *dev;
3669         int ret;
3670
3671         if (speed_fec_capa == NULL && num > 0)
3672                 return -EINVAL;
3673
3674         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3675         dev = &rte_eth_devices[port_id];
3676         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fec_get_capability, -ENOTSUP);
3677         ret = (*dev->dev_ops->fec_get_capability)(dev, speed_fec_capa, num);
3678
3679         return ret;
3680 }
3681
3682 int
3683 rte_eth_fec_get(uint16_t port_id, uint32_t *fec_capa)
3684 {
3685         struct rte_eth_dev *dev;
3686
3687         if (fec_capa == NULL)
3688                 return -EINVAL;
3689
3690         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3691         dev = &rte_eth_devices[port_id];
3692         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fec_get, -ENOTSUP);
3693         return eth_err(port_id, (*dev->dev_ops->fec_get)(dev, fec_capa));
3694 }
3695
3696 int
3697 rte_eth_fec_set(uint16_t port_id, uint32_t fec_capa)
3698 {
3699         struct rte_eth_dev *dev;
3700
3701         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3702         dev = &rte_eth_devices[port_id];
3703         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fec_set, -ENOTSUP);
3704         return eth_err(port_id, (*dev->dev_ops->fec_set)(dev, fec_capa));
3705 }
3706
3707 /*
3708  * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find
3709  * an empty spot.
3710  */
3711 static int
3712 get_mac_addr_index(uint16_t port_id, const struct rte_ether_addr *addr)
3713 {
3714         struct rte_eth_dev_info dev_info;
3715         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3716         unsigned i;
3717         int ret;
3718
3719         ret = rte_eth_dev_info_get(port_id, &dev_info);
3720         if (ret != 0)
3721                 return -1;
3722
3723         for (i = 0; i < dev_info.max_mac_addrs; i++)
3724                 if (memcmp(addr, &dev->data->mac_addrs[i],
3725                                 RTE_ETHER_ADDR_LEN) == 0)
3726                         return i;
3727
3728         return -1;
3729 }
3730
3731 static const struct rte_ether_addr null_mac_addr;
3732
3733 int
3734 rte_eth_dev_mac_addr_add(uint16_t port_id, struct rte_ether_addr *addr,
3735                         uint32_t pool)
3736 {
3737         struct rte_eth_dev *dev;
3738         int index;
3739         uint64_t pool_mask;
3740         int ret;
3741
3742         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3743         dev = &rte_eth_devices[port_id];
3744         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_add, -ENOTSUP);
3745
3746         if (rte_is_zero_ether_addr(addr)) {
3747                 RTE_ETHDEV_LOG(ERR, "Port %u: Cannot add NULL MAC address\n",
3748                         port_id);
3749                 return -EINVAL;
3750         }
3751         if (pool >= ETH_64_POOLS) {
3752                 RTE_ETHDEV_LOG(ERR, "Pool id must be 0-%d\n", ETH_64_POOLS - 1);
3753                 return -EINVAL;
3754         }
3755
3756         index = get_mac_addr_index(port_id, addr);
3757         if (index < 0) {
3758                 index = get_mac_addr_index(port_id, &null_mac_addr);
3759                 if (index < 0) {
3760                         RTE_ETHDEV_LOG(ERR, "Port %u: MAC address array full\n",
3761                                 port_id);
3762                         return -ENOSPC;
3763                 }
3764         } else {
3765                 pool_mask = dev->data->mac_pool_sel[index];
3766
3767                 /* Check if both MAC address and pool is already there, and do nothing */
3768                 if (pool_mask & (1ULL << pool))
3769                         return 0;
3770         }
3771
3772         /* Update NIC */
3773         ret = (*dev->dev_ops->mac_addr_add)(dev, addr, index, pool);
3774
3775         if (ret == 0) {
3776                 /* Update address in NIC data structure */
3777                 rte_ether_addr_copy(addr, &dev->data->mac_addrs[index]);
3778
3779                 /* Update pool bitmap in NIC data structure */
3780                 dev->data->mac_pool_sel[index] |= (1ULL << pool);
3781         }
3782
3783         return eth_err(port_id, ret);
3784 }
3785
3786 int
3787 rte_eth_dev_mac_addr_remove(uint16_t port_id, struct rte_ether_addr *addr)
3788 {
3789         struct rte_eth_dev *dev;
3790         int index;
3791
3792         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3793         dev = &rte_eth_devices[port_id];
3794         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_remove, -ENOTSUP);
3795
3796         index = get_mac_addr_index(port_id, addr);
3797         if (index == 0) {
3798                 RTE_ETHDEV_LOG(ERR,
3799                         "Port %u: Cannot remove default MAC address\n",
3800                         port_id);
3801                 return -EADDRINUSE;
3802         } else if (index < 0)
3803                 return 0;  /* Do nothing if address wasn't found */
3804
3805         /* Update NIC */
3806         (*dev->dev_ops->mac_addr_remove)(dev, index);
3807
3808         /* Update address in NIC data structure */
3809         rte_ether_addr_copy(&null_mac_addr, &dev->data->mac_addrs[index]);
3810
3811         /* reset pool bitmap */
3812         dev->data->mac_pool_sel[index] = 0;
3813
3814         return 0;
3815 }
3816
3817 int
3818 rte_eth_dev_default_mac_addr_set(uint16_t port_id, struct rte_ether_addr *addr)
3819 {
3820         struct rte_eth_dev *dev;
3821         int ret;
3822
3823         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3824
3825         if (!rte_is_valid_assigned_ether_addr(addr))
3826                 return -EINVAL;
3827
3828         dev = &rte_eth_devices[port_id];
3829         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_set, -ENOTSUP);
3830
3831         ret = (*dev->dev_ops->mac_addr_set)(dev, addr);
3832         if (ret < 0)
3833                 return ret;
3834
3835         /* Update default address in NIC data structure */
3836         rte_ether_addr_copy(addr, &dev->data->mac_addrs[0]);
3837
3838         return 0;
3839 }
3840
3841
3842 /*
3843  * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find
3844  * an empty spot.
3845  */
3846 static int
3847 get_hash_mac_addr_index(uint16_t port_id, const struct rte_ether_addr *addr)
3848 {
3849         struct rte_eth_dev_info dev_info;
3850         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3851         unsigned i;
3852         int ret;
3853
3854         ret = rte_eth_dev_info_get(port_id, &dev_info);
3855         if (ret != 0)
3856                 return -1;
3857
3858         if (!dev->data->hash_mac_addrs)
3859                 return -1;
3860
3861         for (i = 0; i < dev_info.max_hash_mac_addrs; i++)
3862                 if (memcmp(addr, &dev->data->hash_mac_addrs[i],
3863                         RTE_ETHER_ADDR_LEN) == 0)
3864                         return i;
3865
3866         return -1;
3867 }
3868
3869 int
3870 rte_eth_dev_uc_hash_table_set(uint16_t port_id, struct rte_ether_addr *addr,
3871                                 uint8_t on)
3872 {
3873         int index;
3874         int ret;
3875         struct rte_eth_dev *dev;
3876
3877         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3878
3879         dev = &rte_eth_devices[port_id];
3880         if (rte_is_zero_ether_addr(addr)) {
3881                 RTE_ETHDEV_LOG(ERR, "Port %u: Cannot add NULL MAC address\n",
3882                         port_id);
3883                 return -EINVAL;
3884         }
3885
3886         index = get_hash_mac_addr_index(port_id, addr);
3887         /* Check if it's already there, and do nothing */
3888         if ((index >= 0) && on)
3889                 return 0;
3890
3891         if (index < 0) {
3892                 if (!on) {
3893                         RTE_ETHDEV_LOG(ERR,
3894                                 "Port %u: the MAC address was not set in UTA\n",
3895                                 port_id);
3896                         return -EINVAL;
3897                 }
3898
3899                 index = get_hash_mac_addr_index(port_id, &null_mac_addr);
3900                 if (index < 0) {
3901                         RTE_ETHDEV_LOG(ERR, "Port %u: MAC address array full\n",
3902                                 port_id);
3903                         return -ENOSPC;
3904                 }
3905         }
3906
3907         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_hash_table_set, -ENOTSUP);
3908         ret = (*dev->dev_ops->uc_hash_table_set)(dev, addr, on);
3909         if (ret == 0) {
3910                 /* Update address in NIC data structure */
3911                 if (on)
3912                         rte_ether_addr_copy(addr,
3913                                         &dev->data->hash_mac_addrs[index]);
3914                 else
3915                         rte_ether_addr_copy(&null_mac_addr,
3916                                         &dev->data->hash_mac_addrs[index]);
3917         }
3918
3919         return eth_err(port_id, ret);
3920 }
3921
3922 int
3923 rte_eth_dev_uc_all_hash_table_set(uint16_t port_id, uint8_t on)
3924 {
3925         struct rte_eth_dev *dev;
3926
3927         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3928
3929         dev = &rte_eth_devices[port_id];
3930
3931         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_all_hash_table_set, -ENOTSUP);
3932         return eth_err(port_id, (*dev->dev_ops->uc_all_hash_table_set)(dev,
3933                                                                        on));
3934 }
3935
3936 int rte_eth_set_queue_rate_limit(uint16_t port_id, uint16_t queue_idx,
3937                                         uint16_t tx_rate)
3938 {
3939         struct rte_eth_dev *dev;
3940         struct rte_eth_dev_info dev_info;
3941         struct rte_eth_link link;
3942         int ret;
3943
3944         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3945
3946         ret = rte_eth_dev_info_get(port_id, &dev_info);
3947         if (ret != 0)
3948                 return ret;
3949
3950         dev = &rte_eth_devices[port_id];
3951         link = dev->data->dev_link;
3952
3953         if (queue_idx > dev_info.max_tx_queues) {
3954                 RTE_ETHDEV_LOG(ERR,
3955                         "Set queue rate limit:port %u: invalid queue id=%u\n",
3956                         port_id, queue_idx);
3957                 return -EINVAL;
3958         }
3959
3960         if (tx_rate > link.link_speed) {
3961                 RTE_ETHDEV_LOG(ERR,
3962                         "Set queue rate limit:invalid tx_rate=%u, bigger than link speed= %d\n",
3963                         tx_rate, link.link_speed);
3964                 return -EINVAL;
3965         }
3966
3967         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_queue_rate_limit, -ENOTSUP);
3968         return eth_err(port_id, (*dev->dev_ops->set_queue_rate_limit)(dev,
3969                                                         queue_idx, tx_rate));
3970 }
3971
3972 int
3973 rte_eth_mirror_rule_set(uint16_t port_id,
3974                         struct rte_eth_mirror_conf *mirror_conf,
3975                         uint8_t rule_id, uint8_t on)
3976 {
3977         struct rte_eth_dev *dev;
3978
3979         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3980         if (mirror_conf->rule_type == 0) {
3981                 RTE_ETHDEV_LOG(ERR, "Mirror rule type can not be 0\n");
3982                 return -EINVAL;
3983         }
3984
3985         if (mirror_conf->dst_pool >= ETH_64_POOLS) {
3986                 RTE_ETHDEV_LOG(ERR, "Invalid dst pool, pool id must be 0-%d\n",
3987                         ETH_64_POOLS - 1);
3988                 return -EINVAL;
3989         }
3990
3991         if ((mirror_conf->rule_type & (ETH_MIRROR_VIRTUAL_POOL_UP |
3992              ETH_MIRROR_VIRTUAL_POOL_DOWN)) &&
3993             (mirror_conf->pool_mask == 0)) {
3994                 RTE_ETHDEV_LOG(ERR,
3995                         "Invalid mirror pool, pool mask can not be 0\n");
3996                 return -EINVAL;
3997         }
3998
3999         if ((mirror_conf->rule_type & ETH_MIRROR_VLAN) &&
4000             mirror_conf->vlan.vlan_mask == 0) {
4001                 RTE_ETHDEV_LOG(ERR,
4002                         "Invalid vlan mask, vlan mask can not be 0\n");
4003                 return -EINVAL;
4004         }
4005
4006         dev = &rte_eth_devices[port_id];
4007         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mirror_rule_set, -ENOTSUP);
4008
4009         return eth_err(port_id, (*dev->dev_ops->mirror_rule_set)(dev,
4010                                                 mirror_conf, rule_id, on));
4011 }
4012
4013 int
4014 rte_eth_mirror_rule_reset(uint16_t port_id, uint8_t rule_id)
4015 {
4016         struct rte_eth_dev *dev;
4017
4018         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4019
4020         dev = &rte_eth_devices[port_id];
4021         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mirror_rule_reset, -ENOTSUP);
4022
4023         return eth_err(port_id, (*dev->dev_ops->mirror_rule_reset)(dev,
4024                                                                    rule_id));
4025 }
4026
4027 RTE_INIT(eth_dev_init_cb_lists)
4028 {
4029         int i;
4030
4031         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
4032                 TAILQ_INIT(&rte_eth_devices[i].link_intr_cbs);
4033 }
4034
4035 int
4036 rte_eth_dev_callback_register(uint16_t port_id,
4037                         enum rte_eth_event_type event,
4038                         rte_eth_dev_cb_fn cb_fn, void *cb_arg)
4039 {
4040         struct rte_eth_dev *dev;
4041         struct rte_eth_dev_callback *user_cb;
4042         uint32_t next_port; /* size is 32-bit to prevent loop wrap-around */
4043         uint16_t last_port;
4044
4045         if (!cb_fn)
4046                 return -EINVAL;
4047
4048         if (!rte_eth_dev_is_valid_port(port_id) && port_id != RTE_ETH_ALL) {
4049                 RTE_ETHDEV_LOG(ERR, "Invalid port_id=%d\n", port_id);
4050                 return -EINVAL;
4051         }
4052
4053         if (port_id == RTE_ETH_ALL) {
4054                 next_port = 0;
4055                 last_port = RTE_MAX_ETHPORTS - 1;
4056         } else {
4057                 next_port = last_port = port_id;
4058         }
4059
4060         rte_spinlock_lock(&rte_eth_dev_cb_lock);
4061
4062         do {
4063                 dev = &rte_eth_devices[next_port];
4064
4065                 TAILQ_FOREACH(user_cb, &(dev->link_intr_cbs), next) {
4066                         if (user_cb->cb_fn == cb_fn &&
4067                                 user_cb->cb_arg == cb_arg &&
4068                                 user_cb->event == event) {
4069                                 break;
4070                         }
4071                 }
4072
4073                 /* create a new callback. */
4074                 if (user_cb == NULL) {
4075                         user_cb = rte_zmalloc("INTR_USER_CALLBACK",
4076                                 sizeof(struct rte_eth_dev_callback), 0);
4077                         if (user_cb != NULL) {
4078                                 user_cb->cb_fn = cb_fn;
4079                                 user_cb->cb_arg = cb_arg;
4080                                 user_cb->event = event;
4081                                 TAILQ_INSERT_TAIL(&(dev->link_intr_cbs),
4082                                                   user_cb, next);
4083                         } else {
4084                                 rte_spinlock_unlock(&rte_eth_dev_cb_lock);
4085                                 rte_eth_dev_callback_unregister(port_id, event,
4086                                                                 cb_fn, cb_arg);
4087                                 return -ENOMEM;
4088                         }
4089
4090                 }
4091         } while (++next_port <= last_port);
4092
4093         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
4094         return 0;
4095 }
4096
4097 int
4098 rte_eth_dev_callback_unregister(uint16_t port_id,
4099                         enum rte_eth_event_type event,
4100                         rte_eth_dev_cb_fn cb_fn, void *cb_arg)
4101 {
4102         int ret;
4103         struct rte_eth_dev *dev;
4104         struct rte_eth_dev_callback *cb, *next;
4105         uint32_t next_port; /* size is 32-bit to prevent loop wrap-around */
4106         uint16_t last_port;
4107
4108         if (!cb_fn)
4109                 return -EINVAL;
4110
4111         if (!rte_eth_dev_is_valid_port(port_id) && port_id != RTE_ETH_ALL) {
4112                 RTE_ETHDEV_LOG(ERR, "Invalid port_id=%d\n", port_id);
4113                 return -EINVAL;
4114         }
4115
4116         if (port_id == RTE_ETH_ALL) {
4117                 next_port = 0;
4118                 last_port = RTE_MAX_ETHPORTS - 1;
4119         } else {
4120                 next_port = last_port = port_id;
4121         }
4122
4123         rte_spinlock_lock(&rte_eth_dev_cb_lock);
4124
4125         do {
4126                 dev = &rte_eth_devices[next_port];
4127                 ret = 0;
4128                 for (cb = TAILQ_FIRST(&dev->link_intr_cbs); cb != NULL;
4129                      cb = next) {
4130
4131                         next = TAILQ_NEXT(cb, next);
4132
4133                         if (cb->cb_fn != cb_fn || cb->event != event ||
4134                             (cb_arg != (void *)-1 && cb->cb_arg != cb_arg))
4135                                 continue;
4136
4137                         /*
4138                          * if this callback is not executing right now,
4139                          * then remove it.
4140                          */
4141                         if (cb->active == 0) {
4142                                 TAILQ_REMOVE(&(dev->link_intr_cbs), cb, next);
4143                                 rte_free(cb);
4144                         } else {
4145                                 ret = -EAGAIN;
4146                         }
4147                 }
4148         } while (++next_port <= last_port);
4149
4150         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
4151         return ret;
4152 }
4153
4154 int
4155 rte_eth_dev_callback_process(struct rte_eth_dev *dev,
4156         enum rte_eth_event_type event, void *ret_param)
4157 {
4158         struct rte_eth_dev_callback *cb_lst;
4159         struct rte_eth_dev_callback dev_cb;
4160         int rc = 0;
4161
4162         rte_spinlock_lock(&rte_eth_dev_cb_lock);
4163         TAILQ_FOREACH(cb_lst, &(dev->link_intr_cbs), next) {
4164                 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
4165                         continue;
4166                 dev_cb = *cb_lst;
4167                 cb_lst->active = 1;
4168                 if (ret_param != NULL)
4169                         dev_cb.ret_param = ret_param;
4170
4171                 rte_spinlock_unlock(&rte_eth_dev_cb_lock);
4172                 rc = dev_cb.cb_fn(dev->data->port_id, dev_cb.event,
4173                                 dev_cb.cb_arg, dev_cb.ret_param);
4174                 rte_spinlock_lock(&rte_eth_dev_cb_lock);
4175                 cb_lst->active = 0;
4176         }
4177         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
4178         return rc;
4179 }
4180
4181 void
4182 rte_eth_dev_probing_finish(struct rte_eth_dev *dev)
4183 {
4184         if (dev == NULL)
4185                 return;
4186
4187         rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_NEW, NULL);
4188
4189         dev->state = RTE_ETH_DEV_ATTACHED;
4190 }
4191
4192 int
4193 rte_eth_dev_rx_intr_ctl(uint16_t port_id, int epfd, int op, void *data)
4194 {
4195         uint32_t vec;
4196         struct rte_eth_dev *dev;
4197         struct rte_intr_handle *intr_handle;
4198         uint16_t qid;
4199         int rc;
4200
4201         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4202
4203         dev = &rte_eth_devices[port_id];
4204
4205         if (!dev->intr_handle) {
4206                 RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
4207                 return -ENOTSUP;
4208         }
4209
4210         intr_handle = dev->intr_handle;
4211         if (!intr_handle->intr_vec) {
4212                 RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
4213                 return -EPERM;
4214         }
4215
4216         for (qid = 0; qid < dev->data->nb_rx_queues; qid++) {
4217                 vec = intr_handle->intr_vec[qid];
4218                 rc = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
4219                 if (rc && rc != -EEXIST) {
4220                         RTE_ETHDEV_LOG(ERR,
4221                                 "p %u q %u rx ctl error op %d epfd %d vec %u\n",
4222                                 port_id, qid, op, epfd, vec);
4223                 }
4224         }
4225
4226         return 0;
4227 }
4228
4229 int
4230 rte_eth_dev_rx_intr_ctl_q_get_fd(uint16_t port_id, uint16_t queue_id)
4231 {
4232         struct rte_intr_handle *intr_handle;
4233         struct rte_eth_dev *dev;
4234         unsigned int efd_idx;
4235         uint32_t vec;
4236         int fd;
4237
4238         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -1);
4239
4240         dev = &rte_eth_devices[port_id];
4241
4242         if (queue_id >= dev->data->nb_rx_queues) {
4243                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
4244                 return -1;
4245         }
4246
4247         if (!dev->intr_handle) {
4248                 RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
4249                 return -1;
4250         }
4251
4252         intr_handle = dev->intr_handle;
4253         if (!intr_handle->intr_vec) {
4254                 RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
4255                 return -1;
4256         }
4257
4258         vec = intr_handle->intr_vec[queue_id];
4259         efd_idx = (vec >= RTE_INTR_VEC_RXTX_OFFSET) ?
4260                 (vec - RTE_INTR_VEC_RXTX_OFFSET) : vec;
4261         fd = intr_handle->efds[efd_idx];
4262
4263         return fd;
4264 }
4265
4266 static inline int
4267 eth_dma_mzone_name(char *name, size_t len, uint16_t port_id, uint16_t queue_id,
4268                 const char *ring_name)
4269 {
4270         return snprintf(name, len, "eth_p%d_q%d_%s",
4271                         port_id, queue_id, ring_name);
4272 }
4273
4274 const struct rte_memzone *
4275 rte_eth_dma_zone_reserve(const struct rte_eth_dev *dev, const char *ring_name,
4276                          uint16_t queue_id, size_t size, unsigned align,
4277                          int socket_id)
4278 {
4279         char z_name[RTE_MEMZONE_NAMESIZE];
4280         const struct rte_memzone *mz;
4281         int rc;
4282
4283         rc = eth_dma_mzone_name(z_name, sizeof(z_name), dev->data->port_id,
4284                         queue_id, ring_name);
4285         if (rc >= RTE_MEMZONE_NAMESIZE) {
4286                 RTE_ETHDEV_LOG(ERR, "ring name too long\n");
4287                 rte_errno = ENAMETOOLONG;
4288                 return NULL;
4289         }
4290
4291         mz = rte_memzone_lookup(z_name);
4292         if (mz) {
4293                 if ((socket_id != SOCKET_ID_ANY && socket_id != mz->socket_id) ||
4294                                 size > mz->len ||
4295                                 ((uintptr_t)mz->addr & (align - 1)) != 0) {
4296                         RTE_ETHDEV_LOG(ERR,
4297                                 "memzone %s does not justify the requested attributes\n",
4298                                 mz->name);
4299                         return NULL;
4300                 }
4301
4302                 return mz;
4303         }
4304
4305         return rte_memzone_reserve_aligned(z_name, size, socket_id,
4306                         RTE_MEMZONE_IOVA_CONTIG, align);
4307 }
4308
4309 int
4310 rte_eth_dma_zone_free(const struct rte_eth_dev *dev, const char *ring_name,
4311                 uint16_t queue_id)
4312 {
4313         char z_name[RTE_MEMZONE_NAMESIZE];
4314         const struct rte_memzone *mz;
4315         int rc = 0;
4316
4317         rc = eth_dma_mzone_name(z_name, sizeof(z_name), dev->data->port_id,
4318                         queue_id, ring_name);
4319         if (rc >= RTE_MEMZONE_NAMESIZE) {
4320                 RTE_ETHDEV_LOG(ERR, "ring name too long\n");
4321                 return -ENAMETOOLONG;
4322         }
4323
4324         mz = rte_memzone_lookup(z_name);
4325         if (mz)
4326                 rc = rte_memzone_free(mz);
4327         else
4328                 rc = -ENOENT;
4329
4330         return rc;
4331 }
4332
4333 int
4334 rte_eth_dev_create(struct rte_device *device, const char *name,
4335         size_t priv_data_size,
4336         ethdev_bus_specific_init ethdev_bus_specific_init,
4337         void *bus_init_params,
4338         ethdev_init_t ethdev_init, void *init_params)
4339 {
4340         struct rte_eth_dev *ethdev;
4341         int retval;
4342
4343         RTE_FUNC_PTR_OR_ERR_RET(*ethdev_init, -EINVAL);
4344
4345         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
4346                 ethdev = rte_eth_dev_allocate(name);
4347                 if (!ethdev)
4348                         return -ENODEV;
4349
4350                 if (priv_data_size) {
4351                         ethdev->data->dev_private = rte_zmalloc_socket(
4352                                 name, priv_data_size, RTE_CACHE_LINE_SIZE,
4353                                 device->numa_node);
4354
4355                         if (!ethdev->data->dev_private) {
4356                                 RTE_ETHDEV_LOG(ERR,
4357                                         "failed to allocate private data\n");
4358                                 retval = -ENOMEM;
4359                                 goto probe_failed;
4360                         }
4361                 }
4362         } else {
4363                 ethdev = rte_eth_dev_attach_secondary(name);
4364                 if (!ethdev) {
4365                         RTE_ETHDEV_LOG(ERR,
4366                                 "secondary process attach failed, ethdev doesn't exist\n");
4367                         return  -ENODEV;
4368                 }
4369         }
4370
4371         ethdev->device = device;
4372
4373         if (ethdev_bus_specific_init) {
4374                 retval = ethdev_bus_specific_init(ethdev, bus_init_params);
4375                 if (retval) {
4376                         RTE_ETHDEV_LOG(ERR,
4377                                 "ethdev bus specific initialisation failed\n");
4378                         goto probe_failed;
4379                 }
4380         }
4381
4382         retval = ethdev_init(ethdev, init_params);
4383         if (retval) {
4384                 RTE_ETHDEV_LOG(ERR, "ethdev initialisation failed\n");
4385                 goto probe_failed;
4386         }
4387
4388         rte_eth_dev_probing_finish(ethdev);
4389
4390         return retval;
4391
4392 probe_failed:
4393         rte_eth_dev_release_port(ethdev);
4394         return retval;
4395 }
4396
4397 int
4398 rte_eth_dev_destroy(struct rte_eth_dev *ethdev,
4399         ethdev_uninit_t ethdev_uninit)
4400 {
4401         int ret;
4402
4403         ethdev = rte_eth_dev_allocated(ethdev->data->name);
4404         if (!ethdev)
4405                 return -ENODEV;
4406
4407         RTE_FUNC_PTR_OR_ERR_RET(*ethdev_uninit, -EINVAL);
4408
4409         ret = ethdev_uninit(ethdev);
4410         if (ret)
4411                 return ret;
4412
4413         return rte_eth_dev_release_port(ethdev);
4414 }
4415
4416 int
4417 rte_eth_dev_rx_intr_ctl_q(uint16_t port_id, uint16_t queue_id,
4418                           int epfd, int op, void *data)
4419 {
4420         uint32_t vec;
4421         struct rte_eth_dev *dev;
4422         struct rte_intr_handle *intr_handle;
4423         int rc;
4424
4425         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4426
4427         dev = &rte_eth_devices[port_id];
4428         if (queue_id >= dev->data->nb_rx_queues) {
4429                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
4430                 return -EINVAL;
4431         }
4432
4433         if (!dev->intr_handle) {
4434                 RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
4435                 return -ENOTSUP;
4436         }
4437
4438         intr_handle = dev->intr_handle;
4439         if (!intr_handle->intr_vec) {
4440                 RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
4441                 return -EPERM;
4442         }
4443
4444         vec = intr_handle->intr_vec[queue_id];
4445         rc = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
4446         if (rc && rc != -EEXIST) {
4447                 RTE_ETHDEV_LOG(ERR,
4448                         "p %u q %u rx ctl error op %d epfd %d vec %u\n",
4449                         port_id, queue_id, op, epfd, vec);
4450                 return rc;
4451         }
4452
4453         return 0;
4454 }
4455
4456 int
4457 rte_eth_dev_rx_intr_enable(uint16_t port_id,
4458                            uint16_t queue_id)
4459 {
4460         struct rte_eth_dev *dev;
4461
4462         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4463
4464         dev = &rte_eth_devices[port_id];
4465
4466         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_intr_enable, -ENOTSUP);
4467         return eth_err(port_id, (*dev->dev_ops->rx_queue_intr_enable)(dev,
4468                                                                 queue_id));
4469 }
4470
4471 int
4472 rte_eth_dev_rx_intr_disable(uint16_t port_id,
4473                             uint16_t queue_id)
4474 {
4475         struct rte_eth_dev *dev;
4476
4477         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4478
4479         dev = &rte_eth_devices[port_id];
4480
4481         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_intr_disable, -ENOTSUP);
4482         return eth_err(port_id, (*dev->dev_ops->rx_queue_intr_disable)(dev,
4483                                                                 queue_id));
4484 }
4485
4486
4487 int
4488 rte_eth_dev_filter_supported(uint16_t port_id,
4489                              enum rte_filter_type filter_type)
4490 {
4491         struct rte_eth_dev *dev;
4492
4493         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4494
4495         dev = &rte_eth_devices[port_id];
4496         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->filter_ctrl, -ENOTSUP);
4497         return (*dev->dev_ops->filter_ctrl)(dev, filter_type,
4498                                 RTE_ETH_FILTER_NOP, NULL);
4499 }
4500
4501 int
4502 rte_eth_dev_filter_ctrl(uint16_t port_id, enum rte_filter_type filter_type,
4503                         enum rte_filter_op filter_op, void *arg)
4504 {
4505         struct rte_eth_dev *dev;
4506
4507         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4508
4509         dev = &rte_eth_devices[port_id];
4510         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->filter_ctrl, -ENOTSUP);
4511         return eth_err(port_id, (*dev->dev_ops->filter_ctrl)(dev, filter_type,
4512                                                              filter_op, arg));
4513 }
4514
4515 const struct rte_eth_rxtx_callback *
4516 rte_eth_add_rx_callback(uint16_t port_id, uint16_t queue_id,
4517                 rte_rx_callback_fn fn, void *user_param)
4518 {
4519 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
4520         rte_errno = ENOTSUP;
4521         return NULL;
4522 #endif
4523         struct rte_eth_dev *dev;
4524
4525         /* check input parameters */
4526         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
4527                     queue_id >= rte_eth_devices[port_id].data->nb_rx_queues) {
4528                 rte_errno = EINVAL;
4529                 return NULL;
4530         }
4531         dev = &rte_eth_devices[port_id];
4532         if (rte_eth_dev_is_rx_hairpin_queue(dev, queue_id)) {
4533                 rte_errno = EINVAL;
4534                 return NULL;
4535         }
4536         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
4537
4538         if (cb == NULL) {
4539                 rte_errno = ENOMEM;
4540                 return NULL;
4541         }
4542
4543         cb->fn.rx = fn;
4544         cb->param = user_param;
4545
4546         rte_spinlock_lock(&rte_eth_rx_cb_lock);
4547         /* Add the callbacks in fifo order. */
4548         struct rte_eth_rxtx_callback *tail =
4549                 rte_eth_devices[port_id].post_rx_burst_cbs[queue_id];
4550
4551         if (!tail) {
4552                 rte_eth_devices[port_id].post_rx_burst_cbs[queue_id] = cb;
4553
4554         } else {
4555                 while (tail->next)
4556                         tail = tail->next;
4557                 tail->next = cb;
4558         }
4559         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
4560
4561         return cb;
4562 }
4563
4564 const struct rte_eth_rxtx_callback *
4565 rte_eth_add_first_rx_callback(uint16_t port_id, uint16_t queue_id,
4566                 rte_rx_callback_fn fn, void *user_param)
4567 {
4568 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
4569         rte_errno = ENOTSUP;
4570         return NULL;
4571 #endif
4572         /* check input parameters */
4573         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
4574                 queue_id >= rte_eth_devices[port_id].data->nb_rx_queues) {
4575                 rte_errno = EINVAL;
4576                 return NULL;
4577         }
4578
4579         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
4580
4581         if (cb == NULL) {
4582                 rte_errno = ENOMEM;
4583                 return NULL;
4584         }
4585
4586         cb->fn.rx = fn;
4587         cb->param = user_param;
4588
4589         rte_spinlock_lock(&rte_eth_rx_cb_lock);
4590         /* Add the callbacks at first position */
4591         cb->next = rte_eth_devices[port_id].post_rx_burst_cbs[queue_id];
4592         rte_smp_wmb();
4593         rte_eth_devices[port_id].post_rx_burst_cbs[queue_id] = cb;
4594         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
4595
4596         return cb;
4597 }
4598
4599 const struct rte_eth_rxtx_callback *
4600 rte_eth_add_tx_callback(uint16_t port_id, uint16_t queue_id,
4601                 rte_tx_callback_fn fn, void *user_param)
4602 {
4603 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
4604         rte_errno = ENOTSUP;
4605         return NULL;
4606 #endif
4607         struct rte_eth_dev *dev;
4608
4609         /* check input parameters */
4610         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
4611                     queue_id >= rte_eth_devices[port_id].data->nb_tx_queues) {
4612                 rte_errno = EINVAL;
4613                 return NULL;
4614         }
4615
4616         dev = &rte_eth_devices[port_id];
4617         if (rte_eth_dev_is_tx_hairpin_queue(dev, queue_id)) {
4618                 rte_errno = EINVAL;
4619                 return NULL;
4620         }
4621
4622         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
4623
4624         if (cb == NULL) {
4625                 rte_errno = ENOMEM;
4626                 return NULL;
4627         }
4628
4629         cb->fn.tx = fn;
4630         cb->param = user_param;
4631
4632         rte_spinlock_lock(&rte_eth_tx_cb_lock);
4633         /* Add the callbacks in fifo order. */
4634         struct rte_eth_rxtx_callback *tail =
4635                 rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id];
4636
4637         if (!tail) {
4638                 rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id] = cb;
4639
4640         } else {
4641                 while (tail->next)
4642                         tail = tail->next;
4643                 tail->next = cb;
4644         }
4645         rte_spinlock_unlock(&rte_eth_tx_cb_lock);
4646
4647         return cb;
4648 }
4649
4650 int
4651 rte_eth_remove_rx_callback(uint16_t port_id, uint16_t queue_id,
4652                 const struct rte_eth_rxtx_callback *user_cb)
4653 {
4654 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
4655         return -ENOTSUP;
4656 #endif
4657         /* Check input parameters. */
4658         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
4659         if (user_cb == NULL ||
4660                         queue_id >= rte_eth_devices[port_id].data->nb_rx_queues)
4661                 return -EINVAL;
4662
4663         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
4664         struct rte_eth_rxtx_callback *cb;
4665         struct rte_eth_rxtx_callback **prev_cb;
4666         int ret = -EINVAL;
4667
4668         rte_spinlock_lock(&rte_eth_rx_cb_lock);
4669         prev_cb = &dev->post_rx_burst_cbs[queue_id];
4670         for (; *prev_cb != NULL; prev_cb = &cb->next) {
4671                 cb = *prev_cb;
4672                 if (cb == user_cb) {
4673                         /* Remove the user cb from the callback list. */
4674                         *prev_cb = cb->next;
4675                         ret = 0;
4676                         break;
4677                 }
4678         }
4679         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
4680
4681         return ret;
4682 }
4683
4684 int
4685 rte_eth_remove_tx_callback(uint16_t port_id, uint16_t queue_id,
4686                 const struct rte_eth_rxtx_callback *user_cb)
4687 {
4688 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
4689         return -ENOTSUP;
4690 #endif
4691         /* Check input parameters. */
4692         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
4693         if (user_cb == NULL ||
4694                         queue_id >= rte_eth_devices[port_id].data->nb_tx_queues)
4695                 return -EINVAL;
4696
4697         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
4698         int ret = -EINVAL;
4699         struct rte_eth_rxtx_callback *cb;
4700         struct rte_eth_rxtx_callback **prev_cb;
4701
4702         rte_spinlock_lock(&rte_eth_tx_cb_lock);
4703         prev_cb = &dev->pre_tx_burst_cbs[queue_id];
4704         for (; *prev_cb != NULL; prev_cb = &cb->next) {
4705                 cb = *prev_cb;
4706                 if (cb == user_cb) {
4707                         /* Remove the user cb from the callback list. */
4708                         *prev_cb = cb->next;
4709                         ret = 0;
4710                         break;
4711                 }
4712         }
4713         rte_spinlock_unlock(&rte_eth_tx_cb_lock);
4714
4715         return ret;
4716 }
4717
4718 int
4719 rte_eth_rx_queue_info_get(uint16_t port_id, uint16_t queue_id,
4720         struct rte_eth_rxq_info *qinfo)
4721 {
4722         struct rte_eth_dev *dev;
4723
4724         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4725
4726         if (qinfo == NULL)
4727                 return -EINVAL;
4728
4729         dev = &rte_eth_devices[port_id];
4730         if (queue_id >= dev->data->nb_rx_queues) {
4731                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
4732                 return -EINVAL;
4733         }
4734
4735         if (dev->data->rx_queues == NULL ||
4736                         dev->data->rx_queues[queue_id] == NULL) {
4737                 RTE_ETHDEV_LOG(ERR,
4738                                "Rx queue %"PRIu16" of device with port_id=%"
4739                                PRIu16" has not been setup\n",
4740                                queue_id, port_id);
4741                 return -EINVAL;
4742         }
4743
4744         if (rte_eth_dev_is_rx_hairpin_queue(dev, queue_id)) {
4745                 RTE_ETHDEV_LOG(INFO,
4746                         "Can't get hairpin Rx queue %"PRIu16" info of device with port_id=%"PRIu16"\n",
4747                         queue_id, port_id);
4748                 return -EINVAL;
4749         }
4750
4751         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rxq_info_get, -ENOTSUP);
4752
4753         memset(qinfo, 0, sizeof(*qinfo));
4754         dev->dev_ops->rxq_info_get(dev, queue_id, qinfo);
4755         return 0;
4756 }
4757
4758 int
4759 rte_eth_tx_queue_info_get(uint16_t port_id, uint16_t queue_id,
4760         struct rte_eth_txq_info *qinfo)
4761 {
4762         struct rte_eth_dev *dev;
4763
4764         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4765
4766         if (qinfo == NULL)
4767                 return -EINVAL;
4768
4769         dev = &rte_eth_devices[port_id];
4770         if (queue_id >= dev->data->nb_tx_queues) {
4771                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", queue_id);
4772                 return -EINVAL;
4773         }
4774
4775         if (dev->data->tx_queues == NULL ||
4776                         dev->data->tx_queues[queue_id] == NULL) {
4777                 RTE_ETHDEV_LOG(ERR,
4778                                "Tx queue %"PRIu16" of device with port_id=%"
4779                                PRIu16" has not been setup\n",
4780                                queue_id, port_id);
4781                 return -EINVAL;
4782         }
4783
4784         if (rte_eth_dev_is_tx_hairpin_queue(dev, queue_id)) {
4785                 RTE_ETHDEV_LOG(INFO,
4786                         "Can't get hairpin Tx queue %"PRIu16" info of device with port_id=%"PRIu16"\n",
4787                         queue_id, port_id);
4788                 return -EINVAL;
4789         }
4790
4791         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->txq_info_get, -ENOTSUP);
4792
4793         memset(qinfo, 0, sizeof(*qinfo));
4794         dev->dev_ops->txq_info_get(dev, queue_id, qinfo);
4795
4796         return 0;
4797 }
4798
4799 int
4800 rte_eth_rx_burst_mode_get(uint16_t port_id, uint16_t queue_id,
4801                           struct rte_eth_burst_mode *mode)
4802 {
4803         struct rte_eth_dev *dev;
4804
4805         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4806
4807         if (mode == NULL)
4808                 return -EINVAL;
4809
4810         dev = &rte_eth_devices[port_id];
4811
4812         if (queue_id >= dev->data->nb_rx_queues) {
4813                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
4814                 return -EINVAL;
4815         }
4816
4817         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_burst_mode_get, -ENOTSUP);
4818         memset(mode, 0, sizeof(*mode));
4819         return eth_err(port_id,
4820                        dev->dev_ops->rx_burst_mode_get(dev, queue_id, mode));
4821 }
4822
4823 int
4824 rte_eth_tx_burst_mode_get(uint16_t port_id, uint16_t queue_id,
4825                           struct rte_eth_burst_mode *mode)
4826 {
4827         struct rte_eth_dev *dev;
4828
4829         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4830
4831         if (mode == NULL)
4832                 return -EINVAL;
4833
4834         dev = &rte_eth_devices[port_id];
4835
4836         if (queue_id >= dev->data->nb_tx_queues) {
4837                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", queue_id);
4838                 return -EINVAL;
4839         }
4840
4841         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_burst_mode_get, -ENOTSUP);
4842         memset(mode, 0, sizeof(*mode));
4843         return eth_err(port_id,
4844                        dev->dev_ops->tx_burst_mode_get(dev, queue_id, mode));
4845 }
4846
4847 int
4848 rte_eth_dev_set_mc_addr_list(uint16_t port_id,
4849                              struct rte_ether_addr *mc_addr_set,
4850                              uint32_t nb_mc_addr)
4851 {
4852         struct rte_eth_dev *dev;
4853
4854         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4855
4856         dev = &rte_eth_devices[port_id];
4857         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_mc_addr_list, -ENOTSUP);
4858         return eth_err(port_id, dev->dev_ops->set_mc_addr_list(dev,
4859                                                 mc_addr_set, nb_mc_addr));
4860 }
4861
4862 int
4863 rte_eth_timesync_enable(uint16_t port_id)
4864 {
4865         struct rte_eth_dev *dev;
4866
4867         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4868         dev = &rte_eth_devices[port_id];
4869
4870         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_enable, -ENOTSUP);
4871         return eth_err(port_id, (*dev->dev_ops->timesync_enable)(dev));
4872 }
4873
4874 int
4875 rte_eth_timesync_disable(uint16_t port_id)
4876 {
4877         struct rte_eth_dev *dev;
4878
4879         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4880         dev = &rte_eth_devices[port_id];
4881
4882         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_disable, -ENOTSUP);
4883         return eth_err(port_id, (*dev->dev_ops->timesync_disable)(dev));
4884 }
4885
4886 int
4887 rte_eth_timesync_read_rx_timestamp(uint16_t port_id, struct timespec *timestamp,
4888                                    uint32_t flags)
4889 {
4890         struct rte_eth_dev *dev;
4891
4892         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4893         dev = &rte_eth_devices[port_id];
4894
4895         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_rx_timestamp, -ENOTSUP);
4896         return eth_err(port_id, (*dev->dev_ops->timesync_read_rx_timestamp)
4897                                 (dev, timestamp, flags));
4898 }
4899
4900 int
4901 rte_eth_timesync_read_tx_timestamp(uint16_t port_id,
4902                                    struct timespec *timestamp)
4903 {
4904         struct rte_eth_dev *dev;
4905
4906         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4907         dev = &rte_eth_devices[port_id];
4908
4909         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_tx_timestamp, -ENOTSUP);
4910         return eth_err(port_id, (*dev->dev_ops->timesync_read_tx_timestamp)
4911                                 (dev, timestamp));
4912 }
4913
4914 int
4915 rte_eth_timesync_adjust_time(uint16_t port_id, int64_t delta)
4916 {
4917         struct rte_eth_dev *dev;
4918
4919         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4920         dev = &rte_eth_devices[port_id];
4921
4922         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_adjust_time, -ENOTSUP);
4923         return eth_err(port_id, (*dev->dev_ops->timesync_adjust_time)(dev,
4924                                                                       delta));
4925 }
4926
4927 int
4928 rte_eth_timesync_read_time(uint16_t port_id, struct timespec *timestamp)
4929 {
4930         struct rte_eth_dev *dev;
4931
4932         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4933         dev = &rte_eth_devices[port_id];
4934
4935         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_time, -ENOTSUP);
4936         return eth_err(port_id, (*dev->dev_ops->timesync_read_time)(dev,
4937                                                                 timestamp));
4938 }
4939
4940 int
4941 rte_eth_timesync_write_time(uint16_t port_id, const struct timespec *timestamp)
4942 {
4943         struct rte_eth_dev *dev;
4944
4945         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4946         dev = &rte_eth_devices[port_id];
4947
4948         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_write_time, -ENOTSUP);
4949         return eth_err(port_id, (*dev->dev_ops->timesync_write_time)(dev,
4950                                                                 timestamp));
4951 }
4952
4953 int
4954 rte_eth_read_clock(uint16_t port_id, uint64_t *clock)
4955 {
4956         struct rte_eth_dev *dev;
4957
4958         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4959         dev = &rte_eth_devices[port_id];
4960
4961         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->read_clock, -ENOTSUP);
4962         return eth_err(port_id, (*dev->dev_ops->read_clock)(dev, clock));
4963 }
4964
4965 int
4966 rte_eth_dev_get_reg_info(uint16_t port_id, struct rte_dev_reg_info *info)
4967 {
4968         struct rte_eth_dev *dev;
4969
4970         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4971
4972         dev = &rte_eth_devices[port_id];
4973         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_reg, -ENOTSUP);
4974         return eth_err(port_id, (*dev->dev_ops->get_reg)(dev, info));
4975 }
4976
4977 int
4978 rte_eth_dev_get_eeprom_length(uint16_t port_id)
4979 {
4980         struct rte_eth_dev *dev;
4981
4982         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4983
4984         dev = &rte_eth_devices[port_id];
4985         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom_length, -ENOTSUP);
4986         return eth_err(port_id, (*dev->dev_ops->get_eeprom_length)(dev));
4987 }
4988
4989 int
4990 rte_eth_dev_get_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info)
4991 {
4992         struct rte_eth_dev *dev;
4993
4994         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4995
4996         dev = &rte_eth_devices[port_id];
4997         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom, -ENOTSUP);
4998         return eth_err(port_id, (*dev->dev_ops->get_eeprom)(dev, info));
4999 }
5000
5001 int
5002 rte_eth_dev_set_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info)
5003 {
5004         struct rte_eth_dev *dev;
5005
5006         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5007
5008         dev = &rte_eth_devices[port_id];
5009         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_eeprom, -ENOTSUP);
5010         return eth_err(port_id, (*dev->dev_ops->set_eeprom)(dev, info));
5011 }
5012
5013 int
5014 rte_eth_dev_get_module_info(uint16_t port_id,
5015                             struct rte_eth_dev_module_info *modinfo)
5016 {
5017         struct rte_eth_dev *dev;
5018
5019         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5020
5021         dev = &rte_eth_devices[port_id];
5022         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_module_info, -ENOTSUP);
5023         return (*dev->dev_ops->get_module_info)(dev, modinfo);
5024 }
5025
5026 int
5027 rte_eth_dev_get_module_eeprom(uint16_t port_id,
5028                               struct rte_dev_eeprom_info *info)
5029 {
5030         struct rte_eth_dev *dev;
5031
5032         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5033
5034         dev = &rte_eth_devices[port_id];
5035         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_module_eeprom, -ENOTSUP);
5036         return (*dev->dev_ops->get_module_eeprom)(dev, info);
5037 }
5038
5039 int
5040 rte_eth_dev_get_dcb_info(uint16_t port_id,
5041                              struct rte_eth_dcb_info *dcb_info)
5042 {
5043         struct rte_eth_dev *dev;
5044
5045         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5046
5047         dev = &rte_eth_devices[port_id];
5048         memset(dcb_info, 0, sizeof(struct rte_eth_dcb_info));
5049
5050         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_dcb_info, -ENOTSUP);
5051         return eth_err(port_id, (*dev->dev_ops->get_dcb_info)(dev, dcb_info));
5052 }
5053
5054 int
5055 rte_eth_dev_l2_tunnel_eth_type_conf(uint16_t port_id,
5056                                     struct rte_eth_l2_tunnel_conf *l2_tunnel)
5057 {
5058         struct rte_eth_dev *dev;
5059
5060         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5061         if (l2_tunnel == NULL) {
5062                 RTE_ETHDEV_LOG(ERR, "Invalid l2_tunnel parameter\n");
5063                 return -EINVAL;
5064         }
5065
5066         if (l2_tunnel->l2_tunnel_type >= RTE_TUNNEL_TYPE_MAX) {
5067                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
5068                 return -EINVAL;
5069         }
5070
5071         dev = &rte_eth_devices[port_id];
5072         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->l2_tunnel_eth_type_conf,
5073                                 -ENOTSUP);
5074         return eth_err(port_id, (*dev->dev_ops->l2_tunnel_eth_type_conf)(dev,
5075                                                                 l2_tunnel));
5076 }
5077
5078 int
5079 rte_eth_dev_l2_tunnel_offload_set(uint16_t port_id,
5080                                   struct rte_eth_l2_tunnel_conf *l2_tunnel,
5081                                   uint32_t mask,
5082                                   uint8_t en)
5083 {
5084         struct rte_eth_dev *dev;
5085
5086         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5087
5088         if (l2_tunnel == NULL) {
5089                 RTE_ETHDEV_LOG(ERR, "Invalid l2_tunnel parameter\n");
5090                 return -EINVAL;
5091         }
5092
5093         if (l2_tunnel->l2_tunnel_type >= RTE_TUNNEL_TYPE_MAX) {
5094                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
5095                 return -EINVAL;
5096         }
5097
5098         if (mask == 0) {
5099                 RTE_ETHDEV_LOG(ERR, "Mask should have a value\n");
5100                 return -EINVAL;
5101         }
5102
5103         dev = &rte_eth_devices[port_id];
5104         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->l2_tunnel_offload_set,
5105                                 -ENOTSUP);
5106         return eth_err(port_id, (*dev->dev_ops->l2_tunnel_offload_set)(dev,
5107                                                         l2_tunnel, mask, en));
5108 }
5109
5110 static void
5111 rte_eth_dev_adjust_nb_desc(uint16_t *nb_desc,
5112                            const struct rte_eth_desc_lim *desc_lim)
5113 {
5114         if (desc_lim->nb_align != 0)
5115                 *nb_desc = RTE_ALIGN_CEIL(*nb_desc, desc_lim->nb_align);
5116
5117         if (desc_lim->nb_max != 0)
5118                 *nb_desc = RTE_MIN(*nb_desc, desc_lim->nb_max);
5119
5120         *nb_desc = RTE_MAX(*nb_desc, desc_lim->nb_min);
5121 }
5122
5123 int
5124 rte_eth_dev_adjust_nb_rx_tx_desc(uint16_t port_id,
5125                                  uint16_t *nb_rx_desc,
5126                                  uint16_t *nb_tx_desc)
5127 {
5128         struct rte_eth_dev_info dev_info;
5129         int ret;
5130
5131         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5132
5133         ret = rte_eth_dev_info_get(port_id, &dev_info);
5134         if (ret != 0)
5135                 return ret;
5136
5137         if (nb_rx_desc != NULL)
5138                 rte_eth_dev_adjust_nb_desc(nb_rx_desc, &dev_info.rx_desc_lim);
5139
5140         if (nb_tx_desc != NULL)
5141                 rte_eth_dev_adjust_nb_desc(nb_tx_desc, &dev_info.tx_desc_lim);
5142
5143         return 0;
5144 }
5145
5146 int
5147 rte_eth_dev_hairpin_capability_get(uint16_t port_id,
5148                                    struct rte_eth_hairpin_cap *cap)
5149 {
5150         struct rte_eth_dev *dev;
5151
5152         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
5153
5154         dev = &rte_eth_devices[port_id];
5155         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->hairpin_cap_get, -ENOTSUP);
5156         memset(cap, 0, sizeof(*cap));
5157         return eth_err(port_id, (*dev->dev_ops->hairpin_cap_get)(dev, cap));
5158 }
5159
5160 int
5161 rte_eth_dev_is_rx_hairpin_queue(struct rte_eth_dev *dev, uint16_t queue_id)
5162 {
5163         if (dev->data->rx_queue_state[queue_id] ==
5164             RTE_ETH_QUEUE_STATE_HAIRPIN)
5165                 return 1;
5166         return 0;
5167 }
5168
5169 int
5170 rte_eth_dev_is_tx_hairpin_queue(struct rte_eth_dev *dev, uint16_t queue_id)
5171 {
5172         if (dev->data->tx_queue_state[queue_id] ==
5173             RTE_ETH_QUEUE_STATE_HAIRPIN)
5174                 return 1;
5175         return 0;
5176 }
5177
5178 int
5179 rte_eth_dev_pool_ops_supported(uint16_t port_id, const char *pool)
5180 {
5181         struct rte_eth_dev *dev;
5182
5183         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5184
5185         if (pool == NULL)
5186                 return -EINVAL;
5187
5188         dev = &rte_eth_devices[port_id];
5189
5190         if (*dev->dev_ops->pool_ops_supported == NULL)
5191                 return 1; /* all pools are supported */
5192
5193         return (*dev->dev_ops->pool_ops_supported)(dev, pool);
5194 }
5195
5196 /**
5197  * A set of values to describe the possible states of a switch domain.
5198  */
5199 enum rte_eth_switch_domain_state {
5200         RTE_ETH_SWITCH_DOMAIN_UNUSED = 0,
5201         RTE_ETH_SWITCH_DOMAIN_ALLOCATED
5202 };
5203
5204 /**
5205  * Array of switch domains available for allocation. Array is sized to
5206  * RTE_MAX_ETHPORTS elements as there cannot be more active switch domains than
5207  * ethdev ports in a single process.
5208  */
5209 static struct rte_eth_dev_switch {
5210         enum rte_eth_switch_domain_state state;
5211 } rte_eth_switch_domains[RTE_MAX_ETHPORTS];
5212
5213 int
5214 rte_eth_switch_domain_alloc(uint16_t *domain_id)
5215 {
5216         unsigned int i;
5217
5218         *domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
5219
5220         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
5221                 if (rte_eth_switch_domains[i].state ==
5222                         RTE_ETH_SWITCH_DOMAIN_UNUSED) {
5223                         rte_eth_switch_domains[i].state =
5224                                 RTE_ETH_SWITCH_DOMAIN_ALLOCATED;
5225                         *domain_id = i;
5226                         return 0;
5227                 }
5228         }
5229
5230         return -ENOSPC;
5231 }
5232
5233 int
5234 rte_eth_switch_domain_free(uint16_t domain_id)
5235 {
5236         if (domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID ||
5237                 domain_id >= RTE_MAX_ETHPORTS)
5238                 return -EINVAL;
5239
5240         if (rte_eth_switch_domains[domain_id].state !=
5241                 RTE_ETH_SWITCH_DOMAIN_ALLOCATED)
5242                 return -EINVAL;
5243
5244         rte_eth_switch_domains[domain_id].state = RTE_ETH_SWITCH_DOMAIN_UNUSED;
5245
5246         return 0;
5247 }
5248
5249 static int
5250 rte_eth_devargs_tokenise(struct rte_kvargs *arglist, const char *str_in)
5251 {
5252         int state;
5253         struct rte_kvargs_pair *pair;
5254         char *letter;
5255
5256         arglist->str = strdup(str_in);
5257         if (arglist->str == NULL)
5258                 return -ENOMEM;
5259
5260         letter = arglist->str;
5261         state = 0;
5262         arglist->count = 0;
5263         pair = &arglist->pairs[0];
5264         while (1) {
5265                 switch (state) {
5266                 case 0: /* Initial */
5267                         if (*letter == '=')
5268                                 return -EINVAL;
5269                         else if (*letter == '\0')
5270                                 return 0;
5271
5272                         state = 1;
5273                         pair->key = letter;
5274                         /* fall-thru */
5275
5276                 case 1: /* Parsing key */
5277                         if (*letter == '=') {
5278                                 *letter = '\0';
5279                                 pair->value = letter + 1;
5280                                 state = 2;
5281                         } else if (*letter == ',' || *letter == '\0')
5282                                 return -EINVAL;
5283                         break;
5284
5285
5286                 case 2: /* Parsing value */
5287                         if (*letter == '[')
5288                                 state = 3;
5289                         else if (*letter == ',') {
5290                                 *letter = '\0';
5291                                 arglist->count++;
5292                                 pair = &arglist->pairs[arglist->count];
5293                                 state = 0;
5294                         } else if (*letter == '\0') {
5295                                 letter--;
5296                                 arglist->count++;
5297                                 pair = &arglist->pairs[arglist->count];
5298                                 state = 0;
5299                         }
5300                         break;
5301
5302                 case 3: /* Parsing list */
5303                         if (*letter == ']')
5304                                 state = 2;
5305                         else if (*letter == '\0')
5306                                 return -EINVAL;
5307                         break;
5308                 }
5309                 letter++;
5310         }
5311 }
5312
5313 int
5314 rte_eth_devargs_parse(const char *dargs, struct rte_eth_devargs *eth_da)
5315 {
5316         struct rte_kvargs args;
5317         struct rte_kvargs_pair *pair;
5318         unsigned int i;
5319         int result = 0;
5320
5321         memset(eth_da, 0, sizeof(*eth_da));
5322
5323         result = rte_eth_devargs_tokenise(&args, dargs);
5324         if (result < 0)
5325                 goto parse_cleanup;
5326
5327         for (i = 0; i < args.count; i++) {
5328                 pair = &args.pairs[i];
5329                 if (strcmp("representor", pair->key) == 0) {
5330                         result = rte_eth_devargs_parse_list(pair->value,
5331                                 rte_eth_devargs_parse_representor_ports,
5332                                 eth_da);
5333                         if (result < 0)
5334                                 goto parse_cleanup;
5335                 }
5336         }
5337
5338 parse_cleanup:
5339         if (args.str)
5340                 free(args.str);
5341
5342         return result;
5343 }
5344
5345 static int
5346 handle_port_list(const char *cmd __rte_unused,
5347                 const char *params __rte_unused,
5348                 struct rte_tel_data *d)
5349 {
5350         int port_id;
5351
5352         rte_tel_data_start_array(d, RTE_TEL_INT_VAL);
5353         RTE_ETH_FOREACH_DEV(port_id)
5354                 rte_tel_data_add_array_int(d, port_id);
5355         return 0;
5356 }
5357
5358 static void
5359 add_port_queue_stats(struct rte_tel_data *d, uint64_t *q_stats,
5360                 const char *stat_name)
5361 {
5362         int q;
5363         struct rte_tel_data *q_data = rte_tel_data_alloc();
5364         rte_tel_data_start_array(q_data, RTE_TEL_U64_VAL);
5365         for (q = 0; q < RTE_ETHDEV_QUEUE_STAT_CNTRS; q++)
5366                 rte_tel_data_add_array_u64(q_data, q_stats[q]);
5367         rte_tel_data_add_dict_container(d, stat_name, q_data, 0);
5368 }
5369
5370 #define ADD_DICT_STAT(stats, s) rte_tel_data_add_dict_u64(d, #s, stats.s)
5371
5372 static int
5373 handle_port_stats(const char *cmd __rte_unused,
5374                 const char *params,
5375                 struct rte_tel_data *d)
5376 {
5377         struct rte_eth_stats stats;
5378         int port_id, ret;
5379
5380         if (params == NULL || strlen(params) == 0 || !isdigit(*params))
5381                 return -1;
5382
5383         port_id = atoi(params);
5384         if (!rte_eth_dev_is_valid_port(port_id))
5385                 return -1;
5386
5387         ret = rte_eth_stats_get(port_id, &stats);
5388         if (ret < 0)
5389                 return -1;
5390
5391         rte_tel_data_start_dict(d);
5392         ADD_DICT_STAT(stats, ipackets);
5393         ADD_DICT_STAT(stats, opackets);
5394         ADD_DICT_STAT(stats, ibytes);
5395         ADD_DICT_STAT(stats, obytes);
5396         ADD_DICT_STAT(stats, imissed);
5397         ADD_DICT_STAT(stats, ierrors);
5398         ADD_DICT_STAT(stats, oerrors);
5399         ADD_DICT_STAT(stats, rx_nombuf);
5400         add_port_queue_stats(d, stats.q_ipackets, "q_ipackets");
5401         add_port_queue_stats(d, stats.q_opackets, "q_opackets");
5402         add_port_queue_stats(d, stats.q_ibytes, "q_ibytes");
5403         add_port_queue_stats(d, stats.q_obytes, "q_obytes");
5404         add_port_queue_stats(d, stats.q_errors, "q_errors");
5405
5406         return 0;
5407 }
5408
5409 static int
5410 handle_port_xstats(const char *cmd __rte_unused,
5411                 const char *params,
5412                 struct rte_tel_data *d)
5413 {
5414         struct rte_eth_xstat *eth_xstats;
5415         struct rte_eth_xstat_name *xstat_names;
5416         int port_id, num_xstats;
5417         int i, ret;
5418         char *end_param;
5419
5420         if (params == NULL || strlen(params) == 0 || !isdigit(*params))
5421                 return -1;
5422
5423         port_id = strtoul(params, &end_param, 0);
5424         if (*end_param != '\0')
5425                 RTE_ETHDEV_LOG(NOTICE,
5426                         "Extra parameters passed to ethdev telemetry command, ignoring");
5427         if (!rte_eth_dev_is_valid_port(port_id))
5428                 return -1;
5429
5430         num_xstats = rte_eth_xstats_get(port_id, NULL, 0);
5431         if (num_xstats < 0)
5432                 return -1;
5433
5434         /* use one malloc for both names and stats */
5435         eth_xstats = malloc((sizeof(struct rte_eth_xstat) +
5436                         sizeof(struct rte_eth_xstat_name)) * num_xstats);
5437         if (eth_xstats == NULL)
5438                 return -1;
5439         xstat_names = (void *)&eth_xstats[num_xstats];
5440
5441         ret = rte_eth_xstats_get_names(port_id, xstat_names, num_xstats);
5442         if (ret < 0 || ret > num_xstats) {
5443                 free(eth_xstats);
5444                 return -1;
5445         }
5446
5447         ret = rte_eth_xstats_get(port_id, eth_xstats, num_xstats);
5448         if (ret < 0 || ret > num_xstats) {
5449                 free(eth_xstats);
5450                 return -1;
5451         }
5452
5453         rte_tel_data_start_dict(d);
5454         for (i = 0; i < num_xstats; i++)
5455                 rte_tel_data_add_dict_u64(d, xstat_names[i].name,
5456                                 eth_xstats[i].value);
5457         return 0;
5458 }
5459
5460 static int
5461 handle_port_link_status(const char *cmd __rte_unused,
5462                 const char *params,
5463                 struct rte_tel_data *d)
5464 {
5465         static const char *status_str = "status";
5466         int ret, port_id;
5467         struct rte_eth_link link;
5468         char *end_param;
5469
5470         if (params == NULL || strlen(params) == 0 || !isdigit(*params))
5471                 return -1;
5472
5473         port_id = strtoul(params, &end_param, 0);
5474         if (*end_param != '\0')
5475                 RTE_ETHDEV_LOG(NOTICE,
5476                         "Extra parameters passed to ethdev telemetry command, ignoring");
5477         if (!rte_eth_dev_is_valid_port(port_id))
5478                 return -1;
5479
5480         ret = rte_eth_link_get(port_id, &link);
5481         if (ret < 0)
5482                 return -1;
5483
5484         rte_tel_data_start_dict(d);
5485         if (!link.link_status) {
5486                 rte_tel_data_add_dict_string(d, status_str, "DOWN");
5487                 return 0;
5488         }
5489         rte_tel_data_add_dict_string(d, status_str, "UP");
5490         rte_tel_data_add_dict_u64(d, "speed", link.link_speed);
5491         rte_tel_data_add_dict_string(d, "duplex",
5492                         (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
5493                                 "full-duplex" : "half-duplex");
5494         return 0;
5495 }
5496
5497 RTE_LOG_REGISTER(rte_eth_dev_logtype, lib.ethdev, INFO);
5498
5499 RTE_INIT(ethdev_init_telemetry)
5500 {
5501         rte_telemetry_register_cmd("/ethdev/list", handle_port_list,
5502                         "Returns list of available ethdev ports. Takes no parameters");
5503         rte_telemetry_register_cmd("/ethdev/stats", handle_port_stats,
5504                         "Returns the common stats for a port. Parameters: int port_id");
5505         rte_telemetry_register_cmd("/ethdev/xstats", handle_port_xstats,
5506                         "Returns the extended stats for a port. Parameters: int port_id");
5507         rte_telemetry_register_cmd("/ethdev/link_status",
5508                         handle_port_link_status,
5509                         "Returns the link status for a port. Parameters: int port_id");
5510 }