lib/librte_ethdev/rte_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <sys/types.h>
   6 #include <sys/queue.h>
   7 #include <ctype.h>
   8 #include <stdio.h>
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <stdarg.h>
  12 #include <errno.h>
  13 #include <stdbool.h>
  14 #include <stdint.h>
  15 #include <inttypes.h>
  16 #include <netinet/in.h>
  17
  18 #include <rte_byteorder.h>
  19 #include <rte_log.h>
  20 #include <rte_debug.h>
  21 #include <rte_interrupts.h>
  22 #include <rte_memory.h>
  23 #include <rte_memcpy.h>
  24 #include <rte_memzone.h>
  25 #include <rte_launch.h>
  26 #include <rte_eal.h>
  27 #include <rte_per_lcore.h>
  28 #include <rte_lcore.h>
  29 #include <rte_atomic.h>
  30 #include <rte_branch_prediction.h>
  31 #include <rte_common.h>
  32 #include <rte_mempool.h>
  33 #include <rte_malloc.h>
  34 #include <rte_mbuf.h>
  35 #include <rte_errno.h>
  36 #include <rte_spinlock.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_kvargs.h>
  39 #include <rte_class.h>
  40
  41 #include "rte_ether.h"
  42 #include "rte_ethdev.h"
  43 #include "rte_ethdev_driver.h"
  44 #include "ethdev_profile.h"
  45 #include "ethdev_private.h"
  46
  47 int rte_eth_dev_logtype;
  48
  49 static const char *MZ_RTE_ETH_DEV_DATA = "rte_eth_dev_data";
  50 struct rte_eth_dev rte_eth_devices[RTE_MAX_ETHPORTS];
  51
  52 /* spinlock for eth device callbacks */
  53 static rte_spinlock_t rte_eth_dev_cb_lock = RTE_SPINLOCK_INITIALIZER;
  54
  55 /* spinlock for add/remove rx callbacks */
  56 static rte_spinlock_t rte_eth_rx_cb_lock = RTE_SPINLOCK_INITIALIZER;
  57
  58 /* spinlock for add/remove tx callbacks */
  59 static rte_spinlock_t rte_eth_tx_cb_lock = RTE_SPINLOCK_INITIALIZER;
  60
  61 /* spinlock for shared data allocation */
  62 static rte_spinlock_t rte_eth_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
  63
  64 /* store statistics names and its offset in stats structure  */
  65 struct rte_eth_xstats_name_off {
  66         char name[RTE_ETH_XSTATS_NAME_SIZE];
  67         unsigned offset;
  68 };
  69
  70 /* Shared memory between primary and secondary processes. */
  71 static struct {
  72         uint64_t next_owner_id;
  73         rte_spinlock_t ownership_lock;
  74         struct rte_eth_dev_data data[RTE_MAX_ETHPORTS];
  75 } *rte_eth_dev_shared_data;
  76
  77 static const struct rte_eth_xstats_name_off rte_stats_strings[] = {
  78         {"rx_good_packets", offsetof(struct rte_eth_stats, ipackets)},
  79         {"tx_good_packets", offsetof(struct rte_eth_stats, opackets)},
  80         {"rx_good_bytes", offsetof(struct rte_eth_stats, ibytes)},
  81         {"tx_good_bytes", offsetof(struct rte_eth_stats, obytes)},
  82         {"rx_missed_errors", offsetof(struct rte_eth_stats, imissed)},
  83         {"rx_errors", offsetof(struct rte_eth_stats, ierrors)},
  84         {"tx_errors", offsetof(struct rte_eth_stats, oerrors)},
  85         {"rx_mbuf_allocation_errors", offsetof(struct rte_eth_stats,
  86                 rx_nombuf)},
  87 };
  88
  89 #define RTE_NB_STATS (sizeof(rte_stats_strings) / sizeof(rte_stats_strings[0]))
  90
  91 static const struct rte_eth_xstats_name_off rte_rxq_stats_strings[] = {
  92         {"packets", offsetof(struct rte_eth_stats, q_ipackets)},
  93         {"bytes", offsetof(struct rte_eth_stats, q_ibytes)},
  94         {"errors", offsetof(struct rte_eth_stats, q_errors)},
  95 };
  96
  97 #define RTE_NB_RXQ_STATS (sizeof(rte_rxq_stats_strings) /       \
  98                 sizeof(rte_rxq_stats_strings[0]))
  99
 100 static const struct rte_eth_xstats_name_off rte_txq_stats_strings[] = {
 101         {"packets", offsetof(struct rte_eth_stats, q_opackets)},
 102         {"bytes", offsetof(struct rte_eth_stats, q_obytes)},
 103 };
 104 #define RTE_NB_TXQ_STATS (sizeof(rte_txq_stats_strings) /       \
 105                 sizeof(rte_txq_stats_strings[0]))
 106
 107 #define RTE_RX_OFFLOAD_BIT2STR(_name)   \
 108         { DEV_RX_OFFLOAD_##_name, #_name }
 109
 110 static const struct {
 111         uint64_t offload;
 112         const char *name;
 113 } rte_rx_offload_names[] = {
 114         RTE_RX_OFFLOAD_BIT2STR(VLAN_STRIP),
 115         RTE_RX_OFFLOAD_BIT2STR(IPV4_CKSUM),
 116         RTE_RX_OFFLOAD_BIT2STR(UDP_CKSUM),
 117         RTE_RX_OFFLOAD_BIT2STR(TCP_CKSUM),
 118         RTE_RX_OFFLOAD_BIT2STR(TCP_LRO),
 119         RTE_RX_OFFLOAD_BIT2STR(QINQ_STRIP),
 120         RTE_RX_OFFLOAD_BIT2STR(OUTER_IPV4_CKSUM),
 121         RTE_RX_OFFLOAD_BIT2STR(MACSEC_STRIP),
 122         RTE_RX_OFFLOAD_BIT2STR(HEADER_SPLIT),
 123         RTE_RX_OFFLOAD_BIT2STR(VLAN_FILTER),
 124         RTE_RX_OFFLOAD_BIT2STR(VLAN_EXTEND),
 125         RTE_RX_OFFLOAD_BIT2STR(JUMBO_FRAME),
 126         RTE_RX_OFFLOAD_BIT2STR(SCATTER),
 127         RTE_RX_OFFLOAD_BIT2STR(TIMESTAMP),
 128         RTE_RX_OFFLOAD_BIT2STR(SECURITY),
 129         RTE_RX_OFFLOAD_BIT2STR(KEEP_CRC),
 130         RTE_RX_OFFLOAD_BIT2STR(SCTP_CKSUM),
 131         RTE_RX_OFFLOAD_BIT2STR(OUTER_UDP_CKSUM),
 132 };
 133
 134 #undef RTE_RX_OFFLOAD_BIT2STR
 135
 136 #define RTE_TX_OFFLOAD_BIT2STR(_name)   \
 137         { DEV_TX_OFFLOAD_##_name, #_name }
 138
 139 static const struct {
 140         uint64_t offload;
 141         const char *name;
 142 } rte_tx_offload_names[] = {
 143         RTE_TX_OFFLOAD_BIT2STR(VLAN_INSERT),
 144         RTE_TX_OFFLOAD_BIT2STR(IPV4_CKSUM),
 145         RTE_TX_OFFLOAD_BIT2STR(UDP_CKSUM),
 146         RTE_TX_OFFLOAD_BIT2STR(TCP_CKSUM),
 147         RTE_TX_OFFLOAD_BIT2STR(SCTP_CKSUM),
 148         RTE_TX_OFFLOAD_BIT2STR(TCP_TSO),
 149         RTE_TX_OFFLOAD_BIT2STR(UDP_TSO),
 150         RTE_TX_OFFLOAD_BIT2STR(OUTER_IPV4_CKSUM),
 151         RTE_TX_OFFLOAD_BIT2STR(QINQ_INSERT),
 152         RTE_TX_OFFLOAD_BIT2STR(VXLAN_TNL_TSO),
 153         RTE_TX_OFFLOAD_BIT2STR(GRE_TNL_TSO),
 154         RTE_TX_OFFLOAD_BIT2STR(IPIP_TNL_TSO),
 155         RTE_TX_OFFLOAD_BIT2STR(GENEVE_TNL_TSO),
 156         RTE_TX_OFFLOAD_BIT2STR(MACSEC_INSERT),
 157         RTE_TX_OFFLOAD_BIT2STR(MT_LOCKFREE),
 158         RTE_TX_OFFLOAD_BIT2STR(MULTI_SEGS),
 159         RTE_TX_OFFLOAD_BIT2STR(MBUF_FAST_FREE),
 160         RTE_TX_OFFLOAD_BIT2STR(SECURITY),
 161         RTE_TX_OFFLOAD_BIT2STR(UDP_TNL_TSO),
 162         RTE_TX_OFFLOAD_BIT2STR(IP_TNL_TSO),
 163         RTE_TX_OFFLOAD_BIT2STR(OUTER_UDP_CKSUM),
 164         RTE_TX_OFFLOAD_BIT2STR(MATCH_METADATA),
 165 };
 166
 167 #undef RTE_TX_OFFLOAD_BIT2STR
 168
 169 /**
 170  * The user application callback description.
 171  *
 172  * It contains callback address to be registered by user application,
 173  * the pointer to the parameters for callback, and the event type.
 174  */
 175 struct rte_eth_dev_callback {
 176         TAILQ_ENTRY(rte_eth_dev_callback) next; /**< Callbacks list */
 177         rte_eth_dev_cb_fn cb_fn;                /**< Callback address */
 178         void *cb_arg;                           /**< Parameter for callback */
 179         void *ret_param;                        /**< Return parameter */
 180         enum rte_eth_event_type event;          /**< Interrupt event type */
 181         uint32_t active;                        /**< Callback is executing */
 182 };
 183
 184 enum {
 185         STAT_QMAP_TX = 0,
 186         STAT_QMAP_RX
 187 };
 188
 189 int
 190 rte_eth_iterator_init(struct rte_dev_iterator *iter, const char *devargs_str)
 191 {
 192         int ret;
 193         struct rte_devargs devargs = {.args = NULL};
 194         const char *bus_param_key;
 195         char *bus_str = NULL;
 196         char *cls_str = NULL;
 197         int str_size;
 198
 199         memset(iter, 0, sizeof(*iter));
 200
 201         /*
 202          * The devargs string may use various syntaxes:
 203          *   - 0000:08:00.0,representor=[1-3]
 204          *   - pci:0000:06:00.0,representor=[0,5]
 205          *   - class=eth,mac=00:11:22:33:44:55
 206          * A new syntax is in development (not yet supported):
 207          *   - bus=X,paramX=x/class=Y,paramY=y/driver=Z,paramZ=z
 208          */
 209
 210         /*
 211          * Handle pure class filter (i.e. without any bus-level argument),
 212          * from future new syntax.
 213          * rte_devargs_parse() is not yet supporting the new syntax,
 214          * that's why this simple case is temporarily parsed here.
 215          */
 216 #define iter_anybus_str "class=eth,"
 217         if (strncmp(devargs_str, iter_anybus_str,
 218                         strlen(iter_anybus_str)) == 0) {
 219                 iter->cls_str = devargs_str + strlen(iter_anybus_str);
 220                 goto end;
 221         }
 222
 223         /* Split bus, device and parameters. */
 224         ret = rte_devargs_parse(&devargs, devargs_str);
 225         if (ret != 0)
 226                 goto error;
 227
 228         /*
 229          * Assume parameters of old syntax can match only at ethdev level.
 230          * Extra parameters will be ignored, thanks to "+" prefix.
 231          */
 232         str_size = strlen(devargs.args) + 2;
 233         cls_str = malloc(str_size);
 234         if (cls_str == NULL) {
 235                 ret = -ENOMEM;
 236                 goto error;
 237         }
 238         ret = snprintf(cls_str, str_size, "+%s", devargs.args);
 239         if (ret != str_size - 1) {
 240                 ret = -EINVAL;
 241                 goto error;
 242         }
 243         iter->cls_str = cls_str;
 244         free(devargs.args); /* allocated by rte_devargs_parse() */
 245         devargs.args = NULL;
 246
 247         iter->bus = devargs.bus;
 248         if (iter->bus->dev_iterate == NULL) {
 249                 ret = -ENOTSUP;
 250                 goto error;
 251         }
 252
 253         /* Convert bus args to new syntax for use with new API dev_iterate. */
 254         if (strcmp(iter->bus->name, "vdev") == 0) {
 255                 bus_param_key = "name";
 256         } else if (strcmp(iter->bus->name, "pci") == 0) {
 257                 bus_param_key = "addr";
 258         } else {
 259                 ret = -ENOTSUP;
 260                 goto error;
 261         }
 262         str_size = strlen(bus_param_key) + strlen(devargs.name) + 2;
 263         bus_str = malloc(str_size);
 264         if (bus_str == NULL) {
 265                 ret = -ENOMEM;
 266                 goto error;
 267         }
 268         ret = snprintf(bus_str, str_size, "%s=%s",
 269                         bus_param_key, devargs.name);
 270         if (ret != str_size - 1) {
 271                 ret = -EINVAL;
 272                 goto error;
 273         }
 274         iter->bus_str = bus_str;
 275
 276 end:
 277         iter->cls = rte_class_find_by_name("eth");
 278         return 0;
 279
 280 error:
 281         if (ret == -ENOTSUP)
 282                 RTE_LOG(ERR, EAL, "Bus %s does not support iterating.\n",
 283                                 iter->bus->name);
 284         free(devargs.args);
 285         free(bus_str);
 286         free(cls_str);
 287         return ret;
 288 }
 289
 290 uint16_t
 291 rte_eth_iterator_next(struct rte_dev_iterator *iter)
 292 {
 293         if (iter->cls == NULL) /* invalid ethdev iterator */
 294                 return RTE_MAX_ETHPORTS;
 295
 296         do { /* loop to try all matching rte_device */
 297                 /* If not pure ethdev filter and */
 298                 if (iter->bus != NULL &&
 299                                 /* not in middle of rte_eth_dev iteration, */
 300                                 iter->class_device == NULL) {
 301                         /* get next rte_device to try. */
 302                         iter->device = iter->bus->dev_iterate(
 303                                         iter->device, iter->bus_str, iter);
 304                         if (iter->device == NULL)
 305                                 break; /* no more rte_device candidate */
 306                 }
 307                 /* A device is matching bus part, need to check ethdev part. */
 308                 iter->class_device = iter->cls->dev_iterate(
 309                                 iter->class_device, iter->cls_str, iter);
 310                 if (iter->class_device != NULL)
 311                         return eth_dev_to_id(iter->class_device); /* match */
 312         } while (iter->bus != NULL); /* need to try next rte_device */
 313
 314         /* No more ethdev port to iterate. */
 315         rte_eth_iterator_cleanup(iter);
 316         return RTE_MAX_ETHPORTS;
 317 }
 318
 319 void
 320 rte_eth_iterator_cleanup(struct rte_dev_iterator *iter)
 321 {
 322         if (iter->bus_str == NULL)
 323                 return; /* nothing to free in pure class filter */
 324         free(RTE_CAST_FIELD(iter, bus_str, char *)); /* workaround const */
 325         free(RTE_CAST_FIELD(iter, cls_str, char *)); /* workaround const */
 326         memset(iter, 0, sizeof(*iter));
 327 }
 328
 329 uint16_t
 330 rte_eth_find_next(uint16_t port_id)
 331 {
 332         while (port_id < RTE_MAX_ETHPORTS &&
 333                rte_eth_devices[port_id].state != RTE_ETH_DEV_ATTACHED &&
 334                rte_eth_devices[port_id].state != RTE_ETH_DEV_REMOVED)
 335                 port_id++;
 336
 337         if (port_id >= RTE_MAX_ETHPORTS)
 338                 return RTE_MAX_ETHPORTS;
 339
 340         return port_id;
 341 }
 342
 343 static void
 344 rte_eth_dev_shared_data_prepare(void)
 345 {
 346         const unsigned flags = 0;
 347         const struct rte_memzone *mz;
 348
 349         rte_spinlock_lock(&rte_eth_shared_data_lock);
 350
 351         if (rte_eth_dev_shared_data == NULL) {
 352                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 353                         /* Allocate port data and ownership shared memory. */
 354                         mz = rte_memzone_reserve(MZ_RTE_ETH_DEV_DATA,
 355                                         sizeof(*rte_eth_dev_shared_data),
 356                                         rte_socket_id(), flags);
 357                 } else
 358                         mz = rte_memzone_lookup(MZ_RTE_ETH_DEV_DATA);
 359                 if (mz == NULL)
 360                         rte_panic("Cannot allocate ethdev shared data\n");
 361
 362                 rte_eth_dev_shared_data = mz->addr;
 363                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 364                         rte_eth_dev_shared_data->next_owner_id =
 365                                         RTE_ETH_DEV_NO_OWNER + 1;
 366                         rte_spinlock_init(&rte_eth_dev_shared_data->ownership_lock);
 367                         memset(rte_eth_dev_shared_data->data, 0,
 368                                sizeof(rte_eth_dev_shared_data->data));
 369                 }
 370         }
 371
 372         rte_spinlock_unlock(&rte_eth_shared_data_lock);
 373 }
 374
 375 static bool
 376 is_allocated(const struct rte_eth_dev *ethdev)
 377 {
 378         return ethdev->data->name[0] != '\0';
 379 }
 380
 381 static struct rte_eth_dev *
 382 _rte_eth_dev_allocated(const char *name)
 383 {
 384         unsigned i;
 385
 386         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 387                 if (rte_eth_devices[i].data != NULL &&
 388                     strcmp(rte_eth_devices[i].data->name, name) == 0)
 389                         return &rte_eth_devices[i];
 390         }
 391         return NULL;
 392 }
 393
 394 struct rte_eth_dev *
 395 rte_eth_dev_allocated(const char *name)
 396 {
 397         struct rte_eth_dev *ethdev;
 398
 399         rte_eth_dev_shared_data_prepare();
 400
 401         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 402
 403         ethdev = _rte_eth_dev_allocated(name);
 404
 405         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 406
 407         return ethdev;
 408 }
 409
 410 static uint16_t
 411 rte_eth_dev_find_free_port(void)
 412 {
 413         unsigned i;
 414
 415         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 416                 /* Using shared name field to find a free port. */
 417                 if (rte_eth_dev_shared_data->data[i].name[0] == '\0') {
 418                         RTE_ASSERT(rte_eth_devices[i].state ==
 419                                    RTE_ETH_DEV_UNUSED);
 420                         return i;
 421                 }
 422         }
 423         return RTE_MAX_ETHPORTS;
 424 }
 425
 426 static struct rte_eth_dev *
 427 eth_dev_get(uint16_t port_id)
 428 {
 429         struct rte_eth_dev *eth_dev = &rte_eth_devices[port_id];
 430
 431         eth_dev->data = &rte_eth_dev_shared_data->data[port_id];
 432
 433         return eth_dev;
 434 }
 435
 436 struct rte_eth_dev *
 437 rte_eth_dev_allocate(const char *name)
 438 {
 439         uint16_t port_id;
 440         struct rte_eth_dev *eth_dev = NULL;
 441         size_t name_len;
 442
 443         name_len = strnlen(name, RTE_ETH_NAME_MAX_LEN);
 444         if (name_len == 0) {
 445                 RTE_ETHDEV_LOG(ERR, "Zero length Ethernet device name\n");
 446                 return NULL;
 447         }
 448
 449         if (name_len >= RTE_ETH_NAME_MAX_LEN) {
 450                 RTE_ETHDEV_LOG(ERR, "Ethernet device name is too long\n");
 451                 return NULL;
 452         }
 453
 454         rte_eth_dev_shared_data_prepare();
 455
 456         /* Synchronize port creation between primary and secondary threads. */
 457         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 458
 459         if (_rte_eth_dev_allocated(name) != NULL) {
 460                 RTE_ETHDEV_LOG(ERR,
 461                         "Ethernet device with name %s already allocated\n",
 462                         name);
 463                 goto unlock;
 464         }
 465
 466         port_id = rte_eth_dev_find_free_port();
 467         if (port_id == RTE_MAX_ETHPORTS) {
 468                 RTE_ETHDEV_LOG(ERR,
 469                         "Reached maximum number of Ethernet ports\n");
 470                 goto unlock;
 471         }
 472
 473         eth_dev = eth_dev_get(port_id);
 474         strlcpy(eth_dev->data->name, name, sizeof(eth_dev->data->name));
 475         eth_dev->data->port_id = port_id;
 476         eth_dev->data->mtu = ETHER_MTU;
 477
 478 unlock:
 479         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 480
 481         return eth_dev;
 482 }
 483
 484 /*
 485  * Attach to a port already registered by the primary process, which
 486  * makes sure that the same device would have the same port id both
 487  * in the primary and secondary process.
 488  */
 489 struct rte_eth_dev *
 490 rte_eth_dev_attach_secondary(const char *name)
 491 {
 492         uint16_t i;
 493         struct rte_eth_dev *eth_dev = NULL;
 494
 495         rte_eth_dev_shared_data_prepare();
 496
 497         /* Synchronize port attachment to primary port creation and release. */
 498         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 499
 500         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 501                 if (strcmp(rte_eth_dev_shared_data->data[i].name, name) == 0)
 502                         break;
 503         }
 504         if (i == RTE_MAX_ETHPORTS) {
 505                 RTE_ETHDEV_LOG(ERR,
 506                         "Device %s is not driven by the primary process\n",
 507                         name);
 508         } else {
 509                 eth_dev = eth_dev_get(i);
 510                 RTE_ASSERT(eth_dev->data->port_id == i);
 511         }
 512
 513         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 514         return eth_dev;
 515 }
 516
 517 int
 518 rte_eth_dev_release_port(struct rte_eth_dev *eth_dev)
 519 {
 520         if (eth_dev == NULL)
 521                 return -EINVAL;
 522
 523         rte_eth_dev_shared_data_prepare();
 524
 525         if (eth_dev->state != RTE_ETH_DEV_UNUSED)
 526                 _rte_eth_dev_callback_process(eth_dev,
 527                                 RTE_ETH_EVENT_DESTROY, NULL);
 528
 529         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 530
 531         eth_dev->state = RTE_ETH_DEV_UNUSED;
 532
 533         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 534                 rte_free(eth_dev->data->rx_queues);
 535                 rte_free(eth_dev->data->tx_queues);
 536                 rte_free(eth_dev->data->mac_addrs);
 537                 rte_free(eth_dev->data->hash_mac_addrs);
 538                 rte_free(eth_dev->data->dev_private);
 539                 memset(eth_dev->data, 0, sizeof(struct rte_eth_dev_data));
 540         }
 541
 542         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 543
 544         return 0;
 545 }
 546
 547 int
 548 rte_eth_dev_is_valid_port(uint16_t port_id)
 549 {
 550         if (port_id >= RTE_MAX_ETHPORTS ||
 551             (rte_eth_devices[port_id].state == RTE_ETH_DEV_UNUSED))
 552                 return 0;
 553         else
 554                 return 1;
 555 }
 556
 557 static int
 558 rte_eth_is_valid_owner_id(uint64_t owner_id)
 559 {
 560         if (owner_id == RTE_ETH_DEV_NO_OWNER ||
 561             rte_eth_dev_shared_data->next_owner_id <= owner_id)
 562                 return 0;
 563         return 1;
 564 }
 565
 566 uint64_t
 567 rte_eth_find_next_owned_by(uint16_t port_id, const uint64_t owner_id)
 568 {
 569         while (port_id < RTE_MAX_ETHPORTS &&
 570                ((rte_eth_devices[port_id].state != RTE_ETH_DEV_ATTACHED &&
 571                rte_eth_devices[port_id].state != RTE_ETH_DEV_REMOVED) ||
 572                rte_eth_devices[port_id].data->owner.id != owner_id))
 573                 port_id++;
 574
 575         if (port_id >= RTE_MAX_ETHPORTS)
 576                 return RTE_MAX_ETHPORTS;
 577
 578         return port_id;
 579 }
 580
 581 int __rte_experimental
 582 rte_eth_dev_owner_new(uint64_t *owner_id)
 583 {
 584         rte_eth_dev_shared_data_prepare();
 585
 586         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 587
 588         *owner_id = rte_eth_dev_shared_data->next_owner_id++;
 589
 590         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 591         return 0;
 592 }
 593
 594 static int
 595 _rte_eth_dev_owner_set(const uint16_t port_id, const uint64_t old_owner_id,
 596                        const struct rte_eth_dev_owner *new_owner)
 597 {
 598         struct rte_eth_dev *ethdev = &rte_eth_devices[port_id];
 599         struct rte_eth_dev_owner *port_owner;
 600
 601         if (port_id >= RTE_MAX_ETHPORTS || !is_allocated(ethdev)) {
 602                 RTE_ETHDEV_LOG(ERR, "Port id %"PRIu16" is not allocated\n",
 603                         port_id);
 604                 return -ENODEV;
 605         }
 606
 607         if (!rte_eth_is_valid_owner_id(new_owner->id) &&
 608             !rte_eth_is_valid_owner_id(old_owner_id)) {
 609                 RTE_ETHDEV_LOG(ERR,
 610                         "Invalid owner old_id=%016"PRIx64" new_id=%016"PRIx64"\n",
 611                        old_owner_id, new_owner->id);
 612                 return -EINVAL;
 613         }
 614
 615         port_owner = &rte_eth_devices[port_id].data->owner;
 616         if (port_owner->id != old_owner_id) {
 617                 RTE_ETHDEV_LOG(ERR,
 618                         "Cannot set owner to port %u already owned by %s_%016"PRIX64"\n",
 619                         port_id, port_owner->name, port_owner->id);
 620                 return -EPERM;
 621         }
 622
 623         /* can not truncate (same structure) */
 624         strlcpy(port_owner->name, new_owner->name, RTE_ETH_MAX_OWNER_NAME_LEN);
 625
 626         port_owner->id = new_owner->id;
 627
 628         RTE_ETHDEV_LOG(DEBUG, "Port %u owner is %s_%016"PRIx64"\n",
 629                 port_id, new_owner->name, new_owner->id);
 630
 631         return 0;
 632 }
 633
 634 int __rte_experimental
 635 rte_eth_dev_owner_set(const uint16_t port_id,
 636                       const struct rte_eth_dev_owner *owner)
 637 {
 638         int ret;
 639
 640         rte_eth_dev_shared_data_prepare();
 641
 642         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 643
 644         ret = _rte_eth_dev_owner_set(port_id, RTE_ETH_DEV_NO_OWNER, owner);
 645
 646         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 647         return ret;
 648 }
 649
 650 int __rte_experimental
 651 rte_eth_dev_owner_unset(const uint16_t port_id, const uint64_t owner_id)
 652 {
 653         const struct rte_eth_dev_owner new_owner = (struct rte_eth_dev_owner)
 654                         {.id = RTE_ETH_DEV_NO_OWNER, .name = ""};
 655         int ret;
 656
 657         rte_eth_dev_shared_data_prepare();
 658
 659         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 660
 661         ret = _rte_eth_dev_owner_set(port_id, owner_id, &new_owner);
 662
 663         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 664         return ret;
 665 }
 666
 667 void __rte_experimental
 668 rte_eth_dev_owner_delete(const uint64_t owner_id)
 669 {
 670         uint16_t port_id;
 671
 672         rte_eth_dev_shared_data_prepare();
 673
 674         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 675
 676         if (rte_eth_is_valid_owner_id(owner_id)) {
 677                 for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++)
 678                         if (rte_eth_devices[port_id].data->owner.id == owner_id)
 679                                 memset(&rte_eth_devices[port_id].data->owner, 0,
 680                                        sizeof(struct rte_eth_dev_owner));
 681                 RTE_ETHDEV_LOG(NOTICE,
 682                         "All port owners owned by %016"PRIx64" identifier have removed\n",
 683                         owner_id);
 684         } else {
 685                 RTE_ETHDEV_LOG(ERR,
 686                                "Invalid owner id=%016"PRIx64"\n",
 687                                owner_id);
 688         }
 689
 690         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 691 }
 692
 693 int __rte_experimental
 694 rte_eth_dev_owner_get(const uint16_t port_id, struct rte_eth_dev_owner *owner)
 695 {
 696         int ret = 0;
 697         struct rte_eth_dev *ethdev = &rte_eth_devices[port_id];
 698
 699         rte_eth_dev_shared_data_prepare();
 700
 701         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 702
 703         if (port_id >= RTE_MAX_ETHPORTS || !is_allocated(ethdev)) {
 704                 RTE_ETHDEV_LOG(ERR, "Port id %"PRIu16" is not allocated\n",
 705                         port_id);
 706                 ret = -ENODEV;
 707         } else {
 708                 rte_memcpy(owner, &ethdev->data->owner, sizeof(*owner));
 709         }
 710
 711         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 712         return ret;
 713 }
 714
 715 int
 716 rte_eth_dev_socket_id(uint16_t port_id)
 717 {
 718         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -1);
 719         return rte_eth_devices[port_id].data->numa_node;
 720 }
 721
 722 void *
 723 rte_eth_dev_get_sec_ctx(uint16_t port_id)
 724 {
 725         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, NULL);
 726         return rte_eth_devices[port_id].security_ctx;
 727 }
 728
 729 uint16_t
 730 rte_eth_dev_count(void)
 731 {
 732         return rte_eth_dev_count_avail();
 733 }
 734
 735 uint16_t
 736 rte_eth_dev_count_avail(void)
 737 {
 738         uint16_t p;
 739         uint16_t count;
 740
 741         count = 0;
 742
 743         RTE_ETH_FOREACH_DEV(p)
 744                 count++;
 745
 746         return count;
 747 }
 748
 749 uint16_t __rte_experimental
 750 rte_eth_dev_count_total(void)
 751 {
 752         uint16_t port, count = 0;
 753
 754         for (port = 0; port < RTE_MAX_ETHPORTS; port++)
 755                 if (rte_eth_devices[port].state != RTE_ETH_DEV_UNUSED)
 756                         count++;
 757
 758         return count;
 759 }
 760
 761 int
 762 rte_eth_dev_get_name_by_port(uint16_t port_id, char *name)
 763 {
 764         char *tmp;
 765
 766         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 767
 768         if (name == NULL) {
 769                 RTE_ETHDEV_LOG(ERR, "Null pointer is specified\n");
 770                 return -EINVAL;
 771         }
 772
 773         /* shouldn't check 'rte_eth_devices[i].data',
 774          * because it might be overwritten by VDEV PMD */
 775         tmp = rte_eth_dev_shared_data->data[port_id].name;
 776         strcpy(name, tmp);
 777         return 0;
 778 }
 779
 780 int
 781 rte_eth_dev_get_port_by_name(const char *name, uint16_t *port_id)
 782 {
 783         uint32_t pid;
 784
 785         if (name == NULL) {
 786                 RTE_ETHDEV_LOG(ERR, "Null pointer is specified\n");
 787                 return -EINVAL;
 788         }
 789
 790         for (pid = 0; pid < RTE_MAX_ETHPORTS; pid++) {
 791                 if (rte_eth_devices[pid].state != RTE_ETH_DEV_UNUSED &&
 792                     !strcmp(name, rte_eth_dev_shared_data->data[pid].name)) {
 793                         *port_id = pid;
 794                         return 0;
 795                 }
 796         }
 797
 798         return -ENODEV;
 799 }
 800
 801 static int
 802 eth_err(uint16_t port_id, int ret)
 803 {
 804         if (ret == 0)
 805                 return 0;
 806         if (rte_eth_dev_is_removed(port_id))
 807                 return -EIO;
 808         return ret;
 809 }
 810
 811 static int
 812 rte_eth_dev_rx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
 813 {
 814         uint16_t old_nb_queues = dev->data->nb_rx_queues;
 815         void **rxq;
 816         unsigned i;
 817
 818         if (dev->data->rx_queues == NULL && nb_queues != 0) { /* first time configuration */
 819                 dev->data->rx_queues = rte_zmalloc("ethdev->rx_queues",
 820                                 sizeof(dev->data->rx_queues[0]) * nb_queues,
 821                                 RTE_CACHE_LINE_SIZE);
 822                 if (dev->data->rx_queues == NULL) {
 823                         dev->data->nb_rx_queues = 0;
 824                         return -(ENOMEM);
 825                 }
 826         } else if (dev->data->rx_queues != NULL && nb_queues != 0) { /* re-configure */
 827                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release, -ENOTSUP);
 828
 829                 rxq = dev->data->rx_queues;
 830
 831                 for (i = nb_queues; i < old_nb_queues; i++)
 832                         (*dev->dev_ops->rx_queue_release)(rxq[i]);
 833                 rxq = rte_realloc(rxq, sizeof(rxq[0]) * nb_queues,
 834                                 RTE_CACHE_LINE_SIZE);
 835                 if (rxq == NULL)
 836                         return -(ENOMEM);
 837                 if (nb_queues > old_nb_queues) {
 838                         uint16_t new_qs = nb_queues - old_nb_queues;
 839
 840                         memset(rxq + old_nb_queues, 0,
 841                                 sizeof(rxq[0]) * new_qs);
 842                 }
 843
 844                 dev->data->rx_queues = rxq;
 845
 846         } else if (dev->data->rx_queues != NULL && nb_queues == 0) {
 847                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release, -ENOTSUP);
 848
 849                 rxq = dev->data->rx_queues;
 850
 851                 for (i = nb_queues; i < old_nb_queues; i++)
 852                         (*dev->dev_ops->rx_queue_release)(rxq[i]);
 853
 854                 rte_free(dev->data->rx_queues);
 855                 dev->data->rx_queues = NULL;
 856         }
 857         dev->data->nb_rx_queues = nb_queues;
 858         return 0;
 859 }
 860
 861 int
 862 rte_eth_dev_rx_queue_start(uint16_t port_id, uint16_t rx_queue_id)
 863 {
 864         struct rte_eth_dev *dev;
 865
 866         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 867
 868         dev = &rte_eth_devices[port_id];
 869         if (!dev->data->dev_started) {
 870                 RTE_ETHDEV_LOG(ERR,
 871                         "Port %u must be started before start any queue\n",
 872                         port_id);
 873                 return -EINVAL;
 874         }
 875
 876         if (rx_queue_id >= dev->data->nb_rx_queues) {
 877                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
 878                 return -EINVAL;
 879         }
 880
 881         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_start, -ENOTSUP);
 882
 883         if (dev->data->rx_queue_state[rx_queue_id] != RTE_ETH_QUEUE_STATE_STOPPED) {
 884                 RTE_ETHDEV_LOG(INFO,
 885                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already started\n",
 886                         rx_queue_id, port_id);
 887                 return 0;
 888         }
 889
 890         return eth_err(port_id, dev->dev_ops->rx_queue_start(dev,
 891                                                              rx_queue_id));
 892
 893 }
 894
 895 int
 896 rte_eth_dev_rx_queue_stop(uint16_t port_id, uint16_t rx_queue_id)
 897 {
 898         struct rte_eth_dev *dev;
 899
 900         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 901
 902         dev = &rte_eth_devices[port_id];
 903         if (rx_queue_id >= dev->data->nb_rx_queues) {
 904                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
 905                 return -EINVAL;
 906         }
 907
 908         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_stop, -ENOTSUP);
 909
 910         if (dev->data->rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STOPPED) {
 911                 RTE_ETHDEV_LOG(INFO,
 912                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already stopped\n",
 913                         rx_queue_id, port_id);
 914                 return 0;
 915         }
 916
 917         return eth_err(port_id, dev->dev_ops->rx_queue_stop(dev, rx_queue_id));
 918
 919 }
 920
 921 int
 922 rte_eth_dev_tx_queue_start(uint16_t port_id, uint16_t tx_queue_id)
 923 {
 924         struct rte_eth_dev *dev;
 925
 926         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 927
 928         dev = &rte_eth_devices[port_id];
 929         if (!dev->data->dev_started) {
 930                 RTE_ETHDEV_LOG(ERR,
 931                         "Port %u must be started before start any queue\n",
 932                         port_id);
 933                 return -EINVAL;
 934         }
 935
 936         if (tx_queue_id >= dev->data->nb_tx_queues) {
 937                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
 938                 return -EINVAL;
 939         }
 940
 941         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_start, -ENOTSUP);
 942
 943         if (dev->data->tx_queue_state[tx_queue_id] != RTE_ETH_QUEUE_STATE_STOPPED) {
 944                 RTE_ETHDEV_LOG(INFO,
 945                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already started\n",
 946                         tx_queue_id, port_id);
 947                 return 0;
 948         }
 949
 950         return eth_err(port_id, dev->dev_ops->tx_queue_start(dev, tx_queue_id));
 951 }
 952
 953 int
 954 rte_eth_dev_tx_queue_stop(uint16_t port_id, uint16_t tx_queue_id)
 955 {
 956         struct rte_eth_dev *dev;
 957
 958         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 959
 960         dev = &rte_eth_devices[port_id];
 961         if (tx_queue_id >= dev->data->nb_tx_queues) {
 962                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
 963                 return -EINVAL;
 964         }
 965
 966         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_stop, -ENOTSUP);
 967
 968         if (dev->data->tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STOPPED) {
 969                 RTE_ETHDEV_LOG(INFO,
 970                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already stopped\n",
 971                         tx_queue_id, port_id);
 972                 return 0;
 973         }
 974
 975         return eth_err(port_id, dev->dev_ops->tx_queue_stop(dev, tx_queue_id));
 976
 977 }
 978
 979 static int
 980 rte_eth_dev_tx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
 981 {
 982         uint16_t old_nb_queues = dev->data->nb_tx_queues;
 983         void **txq;
 984         unsigned i;
 985
 986         if (dev->data->tx_queues == NULL && nb_queues != 0) { /* first time configuration */
 987                 dev->data->tx_queues = rte_zmalloc("ethdev->tx_queues",
 988                                                    sizeof(dev->data->tx_queues[0]) * nb_queues,
 989                                                    RTE_CACHE_LINE_SIZE);
 990                 if (dev->data->tx_queues == NULL) {
 991                         dev->data->nb_tx_queues = 0;
 992                         return -(ENOMEM);
 993                 }
 994         } else if (dev->data->tx_queues != NULL && nb_queues != 0) { /* re-configure */
 995                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release, -ENOTSUP);
 996
 997                 txq = dev->data->tx_queues;
 998
 999                 for (i = nb_queues; i < old_nb_queues; i++)
1000                         (*dev->dev_ops->tx_queue_release)(txq[i]);
1001                 txq = rte_realloc(txq, sizeof(txq[0]) * nb_queues,
1002                                   RTE_CACHE_LINE_SIZE);
1003                 if (txq == NULL)
1004                         return -ENOMEM;
1005                 if (nb_queues > old_nb_queues) {
1006                         uint16_t new_qs = nb_queues - old_nb_queues;
1007
1008                         memset(txq + old_nb_queues, 0,
1009                                sizeof(txq[0]) * new_qs);
1010                 }
1011
1012                 dev->data->tx_queues = txq;
1013
1014         } else if (dev->data->tx_queues != NULL && nb_queues == 0) {
1015                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release, -ENOTSUP);
1016
1017                 txq = dev->data->tx_queues;
1018
1019                 for (i = nb_queues; i < old_nb_queues; i++)
1020                         (*dev->dev_ops->tx_queue_release)(txq[i]);
1021
1022                 rte_free(dev->data->tx_queues);
1023                 dev->data->tx_queues = NULL;
1024         }
1025         dev->data->nb_tx_queues = nb_queues;
1026         return 0;
1027 }
1028
1029 uint32_t
1030 rte_eth_speed_bitflag(uint32_t speed, int duplex)
1031 {
1032         switch (speed) {
1033         case ETH_SPEED_NUM_10M:
1034                 return duplex ? ETH_LINK_SPEED_10M : ETH_LINK_SPEED_10M_HD;
1035         case ETH_SPEED_NUM_100M:
1036                 return duplex ? ETH_LINK_SPEED_100M : ETH_LINK_SPEED_100M_HD;
1037         case ETH_SPEED_NUM_1G:
1038                 return ETH_LINK_SPEED_1G;
1039         case ETH_SPEED_NUM_2_5G:
1040                 return ETH_LINK_SPEED_2_5G;
1041         case ETH_SPEED_NUM_5G:
1042                 return ETH_LINK_SPEED_5G;
1043         case ETH_SPEED_NUM_10G:
1044                 return ETH_LINK_SPEED_10G;
1045         case ETH_SPEED_NUM_20G:
1046                 return ETH_LINK_SPEED_20G;
1047         case ETH_SPEED_NUM_25G:
1048                 return ETH_LINK_SPEED_25G;
1049         case ETH_SPEED_NUM_40G:
1050                 return ETH_LINK_SPEED_40G;
1051         case ETH_SPEED_NUM_50G:
1052                 return ETH_LINK_SPEED_50G;
1053         case ETH_SPEED_NUM_56G:
1054                 return ETH_LINK_SPEED_56G;
1055         case ETH_SPEED_NUM_100G:
1056                 return ETH_LINK_SPEED_100G;
1057         default:
1058                 return 0;
1059         }
1060 }
1061
1062 const char *
1063 rte_eth_dev_rx_offload_name(uint64_t offload)
1064 {
1065         const char *name = "UNKNOWN";
1066         unsigned int i;
1067
1068         for (i = 0; i < RTE_DIM(rte_rx_offload_names); ++i) {
1069                 if (offload == rte_rx_offload_names[i].offload) {
1070                         name = rte_rx_offload_names[i].name;
1071                         break;
1072                 }
1073         }
1074
1075         return name;
1076 }
1077
1078 const char *
1079 rte_eth_dev_tx_offload_name(uint64_t offload)
1080 {
1081         const char *name = "UNKNOWN";
1082         unsigned int i;
1083
1084         for (i = 0; i < RTE_DIM(rte_tx_offload_names); ++i) {
1085                 if (offload == rte_tx_offload_names[i].offload) {
1086                         name = rte_tx_offload_names[i].name;
1087                         break;
1088                 }
1089         }
1090
1091         return name;
1092 }
1093
1094 int
1095 rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
1096                       const struct rte_eth_conf *dev_conf)
1097 {
1098         struct rte_eth_dev *dev;
1099         struct rte_eth_dev_info dev_info;
1100         struct rte_eth_conf orig_conf;
1101         int diag;
1102         int ret;
1103
1104         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1105
1106         dev = &rte_eth_devices[port_id];
1107
1108         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
1109         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
1110
1111         if (dev->data->dev_started) {
1112                 RTE_ETHDEV_LOG(ERR,
1113                         "Port %u must be stopped to allow configuration\n",
1114                         port_id);
1115                 return -EBUSY;
1116         }
1117
1118          /* Store original config, as rollback required on failure */
1119         memcpy(&orig_conf, &dev->data->dev_conf, sizeof(dev->data->dev_conf));
1120
1121         /*
1122          * Copy the dev_conf parameter into the dev structure.
1123          * rte_eth_dev_info_get() requires dev_conf, copy it before dev_info get
1124          */
1125         memcpy(&dev->data->dev_conf, dev_conf, sizeof(dev->data->dev_conf));
1126
1127         rte_eth_dev_info_get(port_id, &dev_info);
1128
1129         /* If number of queues specified by application for both Rx and Tx is
1130          * zero, use driver preferred values. This cannot be done individually
1131          * as it is valid for either Tx or Rx (but not both) to be zero.
1132          * If driver does not provide any preferred valued, fall back on
1133          * EAL defaults.
1134          */
1135         if (nb_rx_q == 0 && nb_tx_q == 0) {
1136                 nb_rx_q = dev_info.default_rxportconf.nb_queues;
1137                 if (nb_rx_q == 0)
1138                         nb_rx_q = RTE_ETH_DEV_FALLBACK_RX_NBQUEUES;
1139                 nb_tx_q = dev_info.default_txportconf.nb_queues;
1140                 if (nb_tx_q == 0)
1141                         nb_tx_q = RTE_ETH_DEV_FALLBACK_TX_NBQUEUES;
1142         }
1143
1144         if (nb_rx_q > RTE_MAX_QUEUES_PER_PORT) {
1145                 RTE_ETHDEV_LOG(ERR,
1146                         "Number of RX queues requested (%u) is greater than max supported(%d)\n",
1147                         nb_rx_q, RTE_MAX_QUEUES_PER_PORT);
1148                 ret = -EINVAL;
1149                 goto rollback;
1150         }
1151
1152         if (nb_tx_q > RTE_MAX_QUEUES_PER_PORT) {
1153                 RTE_ETHDEV_LOG(ERR,
1154                         "Number of TX queues requested (%u) is greater than max supported(%d)\n",
1155                         nb_tx_q, RTE_MAX_QUEUES_PER_PORT);
1156                 ret = -EINVAL;
1157                 goto rollback;
1158         }
1159
1160         /*
1161          * Check that the numbers of RX and TX queues are not greater
1162          * than the maximum number of RX and TX queues supported by the
1163          * configured device.
1164          */
1165         if (nb_rx_q > dev_info.max_rx_queues) {
1166                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u nb_rx_queues=%u > %u\n",
1167                         port_id, nb_rx_q, dev_info.max_rx_queues);
1168                 ret = -EINVAL;
1169                 goto rollback;
1170         }
1171
1172         if (nb_tx_q > dev_info.max_tx_queues) {
1173                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u nb_tx_queues=%u > %u\n",
1174                         port_id, nb_tx_q, dev_info.max_tx_queues);
1175                 ret = -EINVAL;
1176                 goto rollback;
1177         }
1178
1179         /* Check that the device supports requested interrupts */
1180         if ((dev_conf->intr_conf.lsc == 1) &&
1181                         (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC))) {
1182                 RTE_ETHDEV_LOG(ERR, "Driver %s does not support lsc\n",
1183                         dev->device->driver->name);
1184                 ret = -EINVAL;
1185                 goto rollback;
1186         }
1187         if ((dev_conf->intr_conf.rmv == 1) &&
1188                         (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_RMV))) {
1189                 RTE_ETHDEV_LOG(ERR, "Driver %s does not support rmv\n",
1190                         dev->device->driver->name);
1191                 ret = -EINVAL;
1192                 goto rollback;
1193         }
1194
1195         /*
1196          * If jumbo frames are enabled, check that the maximum RX packet
1197          * length is supported by the configured device.
1198          */
1199         if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
1200                 if (dev_conf->rxmode.max_rx_pkt_len > dev_info.max_rx_pktlen) {
1201                         RTE_ETHDEV_LOG(ERR,
1202                                 "Ethdev port_id=%u max_rx_pkt_len %u > max valid value %u\n",
1203                                 port_id, dev_conf->rxmode.max_rx_pkt_len,
1204                                 dev_info.max_rx_pktlen);
1205                         ret = -EINVAL;
1206                         goto rollback;
1207                 } else if (dev_conf->rxmode.max_rx_pkt_len < ETHER_MIN_LEN) {
1208                         RTE_ETHDEV_LOG(ERR,
1209                                 "Ethdev port_id=%u max_rx_pkt_len %u < min valid value %u\n",
1210                                 port_id, dev_conf->rxmode.max_rx_pkt_len,
1211                                 (unsigned)ETHER_MIN_LEN);
1212                         ret = -EINVAL;
1213                         goto rollback;
1214                 }
1215         } else {
1216                 if (dev_conf->rxmode.max_rx_pkt_len < ETHER_MIN_LEN ||
1217                         dev_conf->rxmode.max_rx_pkt_len > ETHER_MAX_LEN)
1218                         /* Use default value */
1219                         dev->data->dev_conf.rxmode.max_rx_pkt_len =
1220                                                         ETHER_MAX_LEN;
1221         }
1222
1223         /* Any requested offloading must be within its device capabilities */
1224         if ((dev_conf->rxmode.offloads & dev_info.rx_offload_capa) !=
1225              dev_conf->rxmode.offloads) {
1226                 RTE_ETHDEV_LOG(ERR,
1227                         "Ethdev port_id=%u requested Rx offloads 0x%"PRIx64" doesn't match Rx offloads "
1228                         "capabilities 0x%"PRIx64" in %s()\n",
1229                         port_id, dev_conf->rxmode.offloads,
1230                         dev_info.rx_offload_capa,
1231                         __func__);
1232                 ret = -EINVAL;
1233                 goto rollback;
1234         }
1235         if ((dev_conf->txmode.offloads & dev_info.tx_offload_capa) !=
1236              dev_conf->txmode.offloads) {
1237                 RTE_ETHDEV_LOG(ERR,
1238                         "Ethdev port_id=%u requested Tx offloads 0x%"PRIx64" doesn't match Tx offloads "
1239                         "capabilities 0x%"PRIx64" in %s()\n",
1240                         port_id, dev_conf->txmode.offloads,
1241                         dev_info.tx_offload_capa,
1242                         __func__);
1243                 ret = -EINVAL;
1244                 goto rollback;
1245         }
1246
1247         /* Check that device supports requested rss hash functions. */
1248         if ((dev_info.flow_type_rss_offloads |
1249              dev_conf->rx_adv_conf.rss_conf.rss_hf) !=
1250             dev_info.flow_type_rss_offloads) {
1251                 RTE_ETHDEV_LOG(ERR,
1252                         "Ethdev port_id=%u invalid rss_hf: 0x%"PRIx64", valid value: 0x%"PRIx64"\n",
1253                         port_id, dev_conf->rx_adv_conf.rss_conf.rss_hf,
1254                         dev_info.flow_type_rss_offloads);
1255                 ret = -EINVAL;
1256                 goto rollback;
1257         }
1258
1259         /*
1260          * Setup new number of RX/TX queues and reconfigure device.
1261          */
1262         diag = rte_eth_dev_rx_queue_config(dev, nb_rx_q);
1263         if (diag != 0) {
1264                 RTE_ETHDEV_LOG(ERR,
1265                         "Port%u rte_eth_dev_rx_queue_config = %d\n",
1266                         port_id, diag);
1267                 ret = diag;
1268                 goto rollback;
1269         }
1270
1271         diag = rte_eth_dev_tx_queue_config(dev, nb_tx_q);
1272         if (diag != 0) {
1273                 RTE_ETHDEV_LOG(ERR,
1274                         "Port%u rte_eth_dev_tx_queue_config = %d\n",
1275                         port_id, diag);
1276                 rte_eth_dev_rx_queue_config(dev, 0);
1277                 ret = diag;
1278                 goto rollback;
1279         }
1280
1281         diag = (*dev->dev_ops->dev_configure)(dev);
1282         if (diag != 0) {
1283                 RTE_ETHDEV_LOG(ERR, "Port%u dev_configure = %d\n",
1284                         port_id, diag);
1285                 rte_eth_dev_rx_queue_config(dev, 0);
1286                 rte_eth_dev_tx_queue_config(dev, 0);
1287                 ret = eth_err(port_id, diag);
1288                 goto rollback;
1289         }
1290
1291         /* Initialize Rx profiling if enabled at compilation time. */
1292         diag = __rte_eth_dev_profile_init(port_id, dev);
1293         if (diag != 0) {
1294                 RTE_ETHDEV_LOG(ERR, "Port%u __rte_eth_dev_profile_init = %d\n",
1295                         port_id, diag);
1296                 rte_eth_dev_rx_queue_config(dev, 0);
1297                 rte_eth_dev_tx_queue_config(dev, 0);
1298                 ret = eth_err(port_id, diag);
1299                 goto rollback;
1300         }
1301
1302         return 0;
1303
1304 rollback:
1305         memcpy(&dev->data->dev_conf, &orig_conf, sizeof(dev->data->dev_conf));
1306
1307         return ret;
1308 }
1309
1310 void
1311 _rte_eth_dev_reset(struct rte_eth_dev *dev)
1312 {
1313         if (dev->data->dev_started) {
1314                 RTE_ETHDEV_LOG(ERR, "Port %u must be stopped to allow reset\n",
1315                         dev->data->port_id);
1316                 return;
1317         }
1318
1319         rte_eth_dev_rx_queue_config(dev, 0);
1320         rte_eth_dev_tx_queue_config(dev, 0);
1321
1322         memset(&dev->data->dev_conf, 0, sizeof(dev->data->dev_conf));
1323 }
1324
1325 static void
1326 rte_eth_dev_mac_restore(struct rte_eth_dev *dev,
1327                         struct rte_eth_dev_info *dev_info)
1328 {
1329         struct ether_addr *addr;
1330         uint16_t i;
1331         uint32_t pool = 0;
1332         uint64_t pool_mask;
1333
1334         /* replay MAC address configuration including default MAC */
1335         addr = &dev->data->mac_addrs[0];
1336         if (*dev->dev_ops->mac_addr_set != NULL)
1337                 (*dev->dev_ops->mac_addr_set)(dev, addr);
1338         else if (*dev->dev_ops->mac_addr_add != NULL)
1339                 (*dev->dev_ops->mac_addr_add)(dev, addr, 0, pool);
1340
1341         if (*dev->dev_ops->mac_addr_add != NULL) {
1342                 for (i = 1; i < dev_info->max_mac_addrs; i++) {
1343                         addr = &dev->data->mac_addrs[i];
1344
1345                         /* skip zero address */
1346                         if (is_zero_ether_addr(addr))
1347                                 continue;
1348
1349                         pool = 0;
1350                         pool_mask = dev->data->mac_pool_sel[i];
1351
1352                         do {
1353                                 if (pool_mask & 1ULL)
1354                                         (*dev->dev_ops->mac_addr_add)(dev,
1355                                                 addr, i, pool);
1356                                 pool_mask >>= 1;
1357                                 pool++;
1358                         } while (pool_mask);
1359                 }
1360         }
1361 }
1362
1363 static void
1364 rte_eth_dev_config_restore(struct rte_eth_dev *dev,
1365                            struct rte_eth_dev_info *dev_info, uint16_t port_id)
1366 {
1367         if (!(*dev_info->dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR))
1368                 rte_eth_dev_mac_restore(dev, dev_info);
1369
1370         /* replay promiscuous configuration */
1371         if (rte_eth_promiscuous_get(port_id) == 1)
1372                 rte_eth_promiscuous_enable(port_id);
1373         else if (rte_eth_promiscuous_get(port_id) == 0)
1374                 rte_eth_promiscuous_disable(port_id);
1375
1376         /* replay all multicast configuration */
1377         if (rte_eth_allmulticast_get(port_id) == 1)
1378                 rte_eth_allmulticast_enable(port_id);
1379         else if (rte_eth_allmulticast_get(port_id) == 0)
1380                 rte_eth_allmulticast_disable(port_id);
1381 }
1382
1383 int
1384 rte_eth_dev_start(uint16_t port_id)
1385 {
1386         struct rte_eth_dev *dev;
1387         struct rte_eth_dev_info dev_info;
1388         int diag;
1389
1390         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1391
1392         dev = &rte_eth_devices[port_id];
1393
1394         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_start, -ENOTSUP);
1395
1396         if (dev->data->dev_started != 0) {
1397                 RTE_ETHDEV_LOG(INFO,
1398                         "Device with port_id=%"PRIu16" already started\n",
1399                         port_id);
1400                 return 0;
1401         }
1402
1403         rte_eth_dev_info_get(port_id, &dev_info);
1404
1405         /* Lets restore MAC now if device does not support live change */
1406         if (*dev_info.dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR)
1407                 rte_eth_dev_mac_restore(dev, &dev_info);
1408
1409         diag = (*dev->dev_ops->dev_start)(dev);
1410         if (diag == 0)
1411                 dev->data->dev_started = 1;
1412         else
1413                 return eth_err(port_id, diag);
1414
1415         rte_eth_dev_config_restore(dev, &dev_info, port_id);
1416
1417         if (dev->data->dev_conf.intr_conf.lsc == 0) {
1418                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
1419                 (*dev->dev_ops->link_update)(dev, 0);
1420         }
1421         return 0;
1422 }
1423
1424 void
1425 rte_eth_dev_stop(uint16_t port_id)
1426 {
1427         struct rte_eth_dev *dev;
1428
1429         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1430         dev = &rte_eth_devices[port_id];
1431
1432         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_stop);
1433
1434         if (dev->data->dev_started == 0) {
1435                 RTE_ETHDEV_LOG(INFO,
1436                         "Device with port_id=%"PRIu16" already stopped\n",
1437                         port_id);
1438                 return;
1439         }
1440
1441         dev->data->dev_started = 0;
1442         (*dev->dev_ops->dev_stop)(dev);
1443 }
1444
1445 int
1446 rte_eth_dev_set_link_up(uint16_t port_id)
1447 {
1448         struct rte_eth_dev *dev;
1449
1450         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1451
1452         dev = &rte_eth_devices[port_id];
1453
1454         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_up, -ENOTSUP);
1455         return eth_err(port_id, (*dev->dev_ops->dev_set_link_up)(dev));
1456 }
1457
1458 int
1459 rte_eth_dev_set_link_down(uint16_t port_id)
1460 {
1461         struct rte_eth_dev *dev;
1462
1463         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1464
1465         dev = &rte_eth_devices[port_id];
1466
1467         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_down, -ENOTSUP);
1468         return eth_err(port_id, (*dev->dev_ops->dev_set_link_down)(dev));
1469 }
1470
1471 void
1472 rte_eth_dev_close(uint16_t port_id)
1473 {
1474         struct rte_eth_dev *dev;
1475
1476         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1477         dev = &rte_eth_devices[port_id];
1478
1479         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_close);
1480         dev->data->dev_started = 0;
1481         (*dev->dev_ops->dev_close)(dev);
1482
1483         /* check behaviour flag - temporary for PMD migration */
1484         if ((dev->data->dev_flags & RTE_ETH_DEV_CLOSE_REMOVE) != 0) {
1485                 /* new behaviour: send event + reset state + free all data */
1486                 rte_eth_dev_release_port(dev);
1487                 return;
1488         }
1489         RTE_ETHDEV_LOG(DEBUG, "Port closing is using an old behaviour.\n"
1490                         "The driver %s should migrate to the new behaviour.\n",
1491                         dev->device->driver->name);
1492         /* old behaviour: only free queue arrays */
1493         dev->data->nb_rx_queues = 0;
1494         rte_free(dev->data->rx_queues);
1495         dev->data->rx_queues = NULL;
1496         dev->data->nb_tx_queues = 0;
1497         rte_free(dev->data->tx_queues);
1498         dev->data->tx_queues = NULL;
1499 }
1500
1501 int
1502 rte_eth_dev_reset(uint16_t port_id)
1503 {
1504         struct rte_eth_dev *dev;
1505         int ret;
1506
1507         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1508         dev = &rte_eth_devices[port_id];
1509
1510         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_reset, -ENOTSUP);
1511
1512         rte_eth_dev_stop(port_id);
1513         ret = dev->dev_ops->dev_reset(dev);
1514
1515         return eth_err(port_id, ret);
1516 }
1517
1518 int __rte_experimental
1519 rte_eth_dev_is_removed(uint16_t port_id)
1520 {
1521         struct rte_eth_dev *dev;
1522         int ret;
1523
1524         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, 0);
1525
1526         dev = &rte_eth_devices[port_id];
1527
1528         if (dev->state == RTE_ETH_DEV_REMOVED)
1529                 return 1;
1530
1531         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->is_removed, 0);
1532
1533         ret = dev->dev_ops->is_removed(dev);
1534         if (ret != 0)
1535                 /* Device is physically removed. */
1536                 dev->state = RTE_ETH_DEV_REMOVED;
1537
1538         return ret;
1539 }
1540
1541 int
1542 rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
1543                        uint16_t nb_rx_desc, unsigned int socket_id,
1544                        const struct rte_eth_rxconf *rx_conf,
1545                        struct rte_mempool *mp)
1546 {
1547         int ret;
1548         uint32_t mbp_buf_size;
1549         struct rte_eth_dev *dev;
1550         struct rte_eth_dev_info dev_info;
1551         struct rte_eth_rxconf local_conf;
1552         void **rxq;
1553
1554         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1555
1556         dev = &rte_eth_devices[port_id];
1557         if (rx_queue_id >= dev->data->nb_rx_queues) {
1558                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
1559                 return -EINVAL;
1560         }
1561
1562         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
1563         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_setup, -ENOTSUP);
1564
1565         /*
1566          * Check the size of the mbuf data buffer.
1567          * This value must be provided in the private data of the memory pool.
1568          * First check that the memory pool has a valid private data.
1569          */
1570         rte_eth_dev_info_get(port_id, &dev_info);
1571         if (mp->private_data_size < sizeof(struct rte_pktmbuf_pool_private)) {
1572                 RTE_ETHDEV_LOG(ERR, "%s private_data_size %d < %d\n",
1573                         mp->name, (int)mp->private_data_size,
1574                         (int)sizeof(struct rte_pktmbuf_pool_private));
1575                 return -ENOSPC;
1576         }
1577         mbp_buf_size = rte_pktmbuf_data_room_size(mp);
1578
1579         if ((mbp_buf_size - RTE_PKTMBUF_HEADROOM) < dev_info.min_rx_bufsize) {
1580                 RTE_ETHDEV_LOG(ERR,
1581                         "%s mbuf_data_room_size %d < %d (RTE_PKTMBUF_HEADROOM=%d + min_rx_bufsize(dev)=%d)\n",
1582                         mp->name, (int)mbp_buf_size,
1583                         (int)(RTE_PKTMBUF_HEADROOM + dev_info.min_rx_bufsize),
1584                         (int)RTE_PKTMBUF_HEADROOM,
1585                         (int)dev_info.min_rx_bufsize);
1586                 return -EINVAL;
1587         }
1588
1589         /* Use default specified by driver, if nb_rx_desc is zero */
1590         if (nb_rx_desc == 0) {
1591                 nb_rx_desc = dev_info.default_rxportconf.ring_size;
1592                 /* If driver default is also zero, fall back on EAL default */
1593                 if (nb_rx_desc == 0)
1594                         nb_rx_desc = RTE_ETH_DEV_FALLBACK_RX_RINGSIZE;
1595         }
1596
1597         if (nb_rx_desc > dev_info.rx_desc_lim.nb_max ||
1598                         nb_rx_desc < dev_info.rx_desc_lim.nb_min ||
1599                         nb_rx_desc % dev_info.rx_desc_lim.nb_align != 0) {
1600
1601                 RTE_ETHDEV_LOG(ERR,
1602                         "Invalid value for nb_rx_desc(=%hu), should be: <= %hu, >= %hu, and a product of %hu\n",
1603                         nb_rx_desc, dev_info.rx_desc_lim.nb_max,
1604                         dev_info.rx_desc_lim.nb_min,
1605                         dev_info.rx_desc_lim.nb_align);
1606                 return -EINVAL;
1607         }
1608
1609         if (dev->data->dev_started &&
1610                 !(dev_info.dev_capa &
1611                         RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP))
1612                 return -EBUSY;
1613
1614         if (dev->data->dev_started &&
1615                 (dev->data->rx_queue_state[rx_queue_id] !=
1616                         RTE_ETH_QUEUE_STATE_STOPPED))
1617                 return -EBUSY;
1618
1619         rxq = dev->data->rx_queues;
1620         if (rxq[rx_queue_id]) {
1621                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release,
1622                                         -ENOTSUP);
1623                 (*dev->dev_ops->rx_queue_release)(rxq[rx_queue_id]);
1624                 rxq[rx_queue_id] = NULL;
1625         }
1626
1627         if (rx_conf == NULL)
1628                 rx_conf = &dev_info.default_rxconf;
1629
1630         local_conf = *rx_conf;
1631
1632         /*
1633          * If an offloading has already been enabled in
1634          * rte_eth_dev_configure(), it has been enabled on all queues,
1635          * so there is no need to enable it in this queue again.
1636          * The local_conf.offloads input to underlying PMD only carries
1637          * those offloadings which are only enabled on this queue and
1638          * not enabled on all queues.
1639          */
1640         local_conf.offloads &= ~dev->data->dev_conf.rxmode.offloads;
1641
1642         /*
1643          * New added offloadings for this queue are those not enabled in
1644          * rte_eth_dev_configure() and they must be per-queue type.
1645          * A pure per-port offloading can't be enabled on a queue while
1646          * disabled on another queue. A pure per-port offloading can't
1647          * be enabled for any queue as new added one if it hasn't been
1648          * enabled in rte_eth_dev_configure().
1649          */
1650         if ((local_conf.offloads & dev_info.rx_queue_offload_capa) !=
1651              local_conf.offloads) {
1652                 RTE_ETHDEV_LOG(ERR,
1653                         "Ethdev port_id=%d rx_queue_id=%d, new added offloads 0x%"PRIx64" must be "
1654                         "within pre-queue offload capabilities 0x%"PRIx64" in %s()\n",
1655                         port_id, rx_queue_id, local_conf.offloads,
1656                         dev_info.rx_queue_offload_capa,
1657                         __func__);
1658                 return -EINVAL;
1659         }
1660
1661         ret = (*dev->dev_ops->rx_queue_setup)(dev, rx_queue_id, nb_rx_desc,
1662                                               socket_id, &local_conf, mp);
1663         if (!ret) {
1664                 if (!dev->data->min_rx_buf_size ||
1665                     dev->data->min_rx_buf_size > mbp_buf_size)
1666                         dev->data->min_rx_buf_size = mbp_buf_size;
1667         }
1668
1669         return eth_err(port_id, ret);
1670 }
1671
1672 int
1673 rte_eth_tx_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
1674                        uint16_t nb_tx_desc, unsigned int socket_id,
1675                        const struct rte_eth_txconf *tx_conf)
1676 {
1677         struct rte_eth_dev *dev;
1678         struct rte_eth_dev_info dev_info;
1679         struct rte_eth_txconf local_conf;
1680         void **txq;
1681
1682         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1683
1684         dev = &rte_eth_devices[port_id];
1685         if (tx_queue_id >= dev->data->nb_tx_queues) {
1686                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
1687                 return -EINVAL;
1688         }
1689
1690         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
1691         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_setup, -ENOTSUP);
1692
1693         rte_eth_dev_info_get(port_id, &dev_info);
1694
1695         /* Use default specified by driver, if nb_tx_desc is zero */
1696         if (nb_tx_desc == 0) {
1697                 nb_tx_desc = dev_info.default_txportconf.ring_size;
1698                 /* If driver default is zero, fall back on EAL default */
1699                 if (nb_tx_desc == 0)
1700                         nb_tx_desc = RTE_ETH_DEV_FALLBACK_TX_RINGSIZE;
1701         }
1702         if (nb_tx_desc > dev_info.tx_desc_lim.nb_max ||
1703             nb_tx_desc < dev_info.tx_desc_lim.nb_min ||
1704             nb_tx_desc % dev_info.tx_desc_lim.nb_align != 0) {
1705                 RTE_ETHDEV_LOG(ERR,
1706                         "Invalid value for nb_tx_desc(=%hu), should be: <= %hu, >= %hu, and a product of %hu\n",
1707                         nb_tx_desc, dev_info.tx_desc_lim.nb_max,
1708                         dev_info.tx_desc_lim.nb_min,
1709                         dev_info.tx_desc_lim.nb_align);
1710                 return -EINVAL;
1711         }
1712
1713         if (dev->data->dev_started &&
1714                 !(dev_info.dev_capa &
1715                         RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP))
1716                 return -EBUSY;
1717
1718         if (dev->data->dev_started &&
1719                 (dev->data->tx_queue_state[tx_queue_id] !=
1720                         RTE_ETH_QUEUE_STATE_STOPPED))
1721                 return -EBUSY;
1722
1723         txq = dev->data->tx_queues;
1724         if (txq[tx_queue_id]) {
1725                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release,
1726                                         -ENOTSUP);
1727                 (*dev->dev_ops->tx_queue_release)(txq[tx_queue_id]);
1728                 txq[tx_queue_id] = NULL;
1729         }
1730
1731         if (tx_conf == NULL)
1732                 tx_conf = &dev_info.default_txconf;
1733
1734         local_conf = *tx_conf;
1735
1736         /*
1737          * If an offloading has already been enabled in
1738          * rte_eth_dev_configure(), it has been enabled on all queues,
1739          * so there is no need to enable it in this queue again.
1740          * The local_conf.offloads input to underlying PMD only carries
1741          * those offloadings which are only enabled on this queue and
1742          * not enabled on all queues.
1743          */
1744         local_conf.offloads &= ~dev->data->dev_conf.txmode.offloads;
1745
1746         /*
1747          * New added offloadings for this queue are those not enabled in
1748          * rte_eth_dev_configure() and they must be per-queue type.
1749          * A pure per-port offloading can't be enabled on a queue while
1750          * disabled on another queue. A pure per-port offloading can't
1751          * be enabled for any queue as new added one if it hasn't been
1752          * enabled in rte_eth_dev_configure().
1753          */
1754         if ((local_conf.offloads & dev_info.tx_queue_offload_capa) !=
1755              local_conf.offloads) {
1756                 RTE_ETHDEV_LOG(ERR,
1757                         "Ethdev port_id=%d tx_queue_id=%d, new added offloads 0x%"PRIx64" must be "
1758                         "within pre-queue offload capabilities 0x%"PRIx64" in %s()\n",
1759                         port_id, tx_queue_id, local_conf.offloads,
1760                         dev_info.tx_queue_offload_capa,
1761                         __func__);
1762                 return -EINVAL;
1763         }
1764
1765         return eth_err(port_id, (*dev->dev_ops->tx_queue_setup)(dev,
1766                        tx_queue_id, nb_tx_desc, socket_id, &local_conf));
1767 }
1768
1769 void
1770 rte_eth_tx_buffer_drop_callback(struct rte_mbuf **pkts, uint16_t unsent,
1771                 void *userdata __rte_unused)
1772 {
1773         unsigned i;
1774
1775         for (i = 0; i < unsent; i++)
1776                 rte_pktmbuf_free(pkts[i]);
1777 }
1778
1779 void
1780 rte_eth_tx_buffer_count_callback(struct rte_mbuf **pkts, uint16_t unsent,
1781                 void *userdata)
1782 {
1783         uint64_t *count = userdata;
1784         unsigned i;
1785
1786         for (i = 0; i < unsent; i++)
1787                 rte_pktmbuf_free(pkts[i]);
1788
1789         *count += unsent;
1790 }
1791
1792 int
1793 rte_eth_tx_buffer_set_err_callback(struct rte_eth_dev_tx_buffer *buffer,
1794                 buffer_tx_error_fn cbfn, void *userdata)
1795 {
1796         buffer->error_callback = cbfn;
1797         buffer->error_userdata = userdata;
1798         return 0;
1799 }
1800
1801 int
1802 rte_eth_tx_buffer_init(struct rte_eth_dev_tx_buffer *buffer, uint16_t size)
1803 {
1804         int ret = 0;
1805
1806         if (buffer == NULL)
1807                 return -EINVAL;
1808
1809         buffer->size = size;
1810         if (buffer->error_callback == NULL) {
1811                 ret = rte_eth_tx_buffer_set_err_callback(
1812                         buffer, rte_eth_tx_buffer_drop_callback, NULL);
1813         }
1814
1815         return ret;
1816 }
1817
1818 int
1819 rte_eth_tx_done_cleanup(uint16_t port_id, uint16_t queue_id, uint32_t free_cnt)
1820 {
1821         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
1822         int ret;
1823
1824         /* Validate Input Data. Bail if not valid or not supported. */
1825         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1826         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_done_cleanup, -ENOTSUP);
1827
1828         /* Call driver to free pending mbufs. */
1829         ret = (*dev->dev_ops->tx_done_cleanup)(dev->data->tx_queues[queue_id],
1830                                                free_cnt);
1831         return eth_err(port_id, ret);
1832 }
1833
1834 void
1835 rte_eth_promiscuous_enable(uint16_t port_id)
1836 {
1837         struct rte_eth_dev *dev;
1838
1839         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1840         dev = &rte_eth_devices[port_id];
1841
1842         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->promiscuous_enable);
1843         (*dev->dev_ops->promiscuous_enable)(dev);
1844         dev->data->promiscuous = 1;
1845 }
1846
1847 void
1848 rte_eth_promiscuous_disable(uint16_t port_id)
1849 {
1850         struct rte_eth_dev *dev;
1851
1852         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1853         dev = &rte_eth_devices[port_id];
1854
1855         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->promiscuous_disable);
1856         dev->data->promiscuous = 0;
1857         (*dev->dev_ops->promiscuous_disable)(dev);
1858 }
1859
1860 int
1861 rte_eth_promiscuous_get(uint16_t port_id)
1862 {
1863         struct rte_eth_dev *dev;
1864
1865         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1866
1867         dev = &rte_eth_devices[port_id];
1868         return dev->data->promiscuous;
1869 }
1870
1871 void
1872 rte_eth_allmulticast_enable(uint16_t port_id)
1873 {
1874         struct rte_eth_dev *dev;
1875
1876         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1877         dev = &rte_eth_devices[port_id];
1878
1879         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->allmulticast_enable);
1880         (*dev->dev_ops->allmulticast_enable)(dev);
1881         dev->data->all_multicast = 1;
1882 }
1883
1884 void
1885 rte_eth_allmulticast_disable(uint16_t port_id)
1886 {
1887         struct rte_eth_dev *dev;
1888
1889         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1890         dev = &rte_eth_devices[port_id];
1891
1892         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->allmulticast_disable);
1893         dev->data->all_multicast = 0;
1894         (*dev->dev_ops->allmulticast_disable)(dev);
1895 }
1896
1897 int
1898 rte_eth_allmulticast_get(uint16_t port_id)
1899 {
1900         struct rte_eth_dev *dev;
1901
1902         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1903
1904         dev = &rte_eth_devices[port_id];
1905         return dev->data->all_multicast;
1906 }
1907
1908 void
1909 rte_eth_link_get(uint16_t port_id, struct rte_eth_link *eth_link)
1910 {
1911         struct rte_eth_dev *dev;
1912
1913         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1914         dev = &rte_eth_devices[port_id];
1915
1916         if (dev->data->dev_conf.intr_conf.lsc &&
1917             dev->data->dev_started)
1918                 rte_eth_linkstatus_get(dev, eth_link);
1919         else {
1920                 RTE_FUNC_PTR_OR_RET(*dev->dev_ops->link_update);
1921                 (*dev->dev_ops->link_update)(dev, 1);
1922                 *eth_link = dev->data->dev_link;
1923         }
1924 }
1925
1926 void
1927 rte_eth_link_get_nowait(uint16_t port_id, struct rte_eth_link *eth_link)
1928 {
1929         struct rte_eth_dev *dev;
1930
1931         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1932         dev = &rte_eth_devices[port_id];
1933
1934         if (dev->data->dev_conf.intr_conf.lsc &&
1935             dev->data->dev_started)
1936                 rte_eth_linkstatus_get(dev, eth_link);
1937         else {
1938                 RTE_FUNC_PTR_OR_RET(*dev->dev_ops->link_update);
1939                 (*dev->dev_ops->link_update)(dev, 0);
1940                 *eth_link = dev->data->dev_link;
1941         }
1942 }
1943
1944 int
1945 rte_eth_stats_get(uint16_t port_id, struct rte_eth_stats *stats)
1946 {
1947         struct rte_eth_dev *dev;
1948
1949         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1950
1951         dev = &rte_eth_devices[port_id];
1952         memset(stats, 0, sizeof(*stats));
1953
1954         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
1955         stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
1956         return eth_err(port_id, (*dev->dev_ops->stats_get)(dev, stats));
1957 }
1958
1959 int
1960 rte_eth_stats_reset(uint16_t port_id)
1961 {
1962         struct rte_eth_dev *dev;
1963
1964         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1965         dev = &rte_eth_devices[port_id];
1966
1967         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_reset, -ENOTSUP);
1968         (*dev->dev_ops->stats_reset)(dev);
1969         dev->data->rx_mbuf_alloc_failed = 0;
1970
1971         return 0;
1972 }
1973
1974 static inline int
1975 get_xstats_basic_count(struct rte_eth_dev *dev)
1976 {
1977         uint16_t nb_rxqs, nb_txqs;
1978         int count;
1979
1980         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
1981         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
1982
1983         count = RTE_NB_STATS;
1984         count += nb_rxqs * RTE_NB_RXQ_STATS;
1985         count += nb_txqs * RTE_NB_TXQ_STATS;
1986
1987         return count;
1988 }
1989
1990 static int
1991 get_xstats_count(uint16_t port_id)
1992 {
1993         struct rte_eth_dev *dev;
1994         int count;
1995
1996         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1997         dev = &rte_eth_devices[port_id];
1998         if (dev->dev_ops->xstats_get_names_by_id != NULL) {
1999                 count = (*dev->dev_ops->xstats_get_names_by_id)(dev, NULL,
2000                                 NULL, 0);
2001                 if (count < 0)
2002                         return eth_err(port_id, count);
2003         }
2004         if (dev->dev_ops->xstats_get_names != NULL) {
2005                 count = (*dev->dev_ops->xstats_get_names)(dev, NULL, 0);
2006                 if (count < 0)
2007                         return eth_err(port_id, count);
2008         } else
2009                 count = 0;
2010
2011
2012         count += get_xstats_basic_count(dev);
2013
2014         return count;
2015 }
2016
2017 int
2018 rte_eth_xstats_get_id_by_name(uint16_t port_id, const char *xstat_name,
2019                 uint64_t *id)
2020 {
2021         int cnt_xstats, idx_xstat;
2022
2023         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2024
2025         if (!id) {
2026                 RTE_ETHDEV_LOG(ERR, "Id pointer is NULL\n");
2027                 return -ENOMEM;
2028         }
2029
2030         if (!xstat_name) {
2031                 RTE_ETHDEV_LOG(ERR, "xstat_name pointer is NULL\n");
2032                 return -ENOMEM;
2033         }
2034
2035         /* Get count */
2036         cnt_xstats = rte_eth_xstats_get_names_by_id(port_id, NULL, 0, NULL);
2037         if (cnt_xstats  < 0) {
2038                 RTE_ETHDEV_LOG(ERR, "Cannot get count of xstats\n");
2039                 return -ENODEV;
2040         }
2041
2042         /* Get id-name lookup table */
2043         struct rte_eth_xstat_name xstats_names[cnt_xstats];
2044
2045         if (cnt_xstats != rte_eth_xstats_get_names_by_id(
2046                         port_id, xstats_names, cnt_xstats, NULL)) {
2047                 RTE_ETHDEV_LOG(ERR, "Cannot get xstats lookup\n");
2048                 return -1;
2049         }
2050
2051         for (idx_xstat = 0; idx_xstat < cnt_xstats; idx_xstat++) {
2052                 if (!strcmp(xstats_names[idx_xstat].name, xstat_name)) {
2053                         *id = idx_xstat;
2054                         return 0;
2055                 };
2056         }
2057
2058         return -EINVAL;
2059 }
2060
2061 /* retrieve basic stats names */
2062 static int
2063 rte_eth_basic_stats_get_names(struct rte_eth_dev *dev,
2064         struct rte_eth_xstat_name *xstats_names)
2065 {
2066         int cnt_used_entries = 0;
2067         uint32_t idx, id_queue;
2068         uint16_t num_q;
2069
2070         for (idx = 0; idx < RTE_NB_STATS; idx++) {
2071                 snprintf(xstats_names[cnt_used_entries].name,
2072                         sizeof(xstats_names[0].name),
2073                         "%s", rte_stats_strings[idx].name);
2074                 cnt_used_entries++;
2075         }
2076         num_q = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2077         for (id_queue = 0; id_queue < num_q; id_queue++) {
2078                 for (idx = 0; idx < RTE_NB_RXQ_STATS; idx++) {
2079                         snprintf(xstats_names[cnt_used_entries].name,
2080                                 sizeof(xstats_names[0].name),
2081                                 "rx_q%u%s",
2082                                 id_queue, rte_rxq_stats_strings[idx].name);
2083                         cnt_used_entries++;
2084                 }
2085
2086         }
2087         num_q = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2088         for (id_queue = 0; id_queue < num_q; id_queue++) {
2089                 for (idx = 0; idx < RTE_NB_TXQ_STATS; idx++) {
2090                         snprintf(xstats_names[cnt_used_entries].name,
2091                                 sizeof(xstats_names[0].name),
2092                                 "tx_q%u%s",
2093                                 id_queue, rte_txq_stats_strings[idx].name);
2094                         cnt_used_entries++;
2095                 }
2096         }
2097         return cnt_used_entries;
2098 }
2099
2100 /* retrieve ethdev extended statistics names */
2101 int
2102 rte_eth_xstats_get_names_by_id(uint16_t port_id,
2103         struct rte_eth_xstat_name *xstats_names, unsigned int size,
2104         uint64_t *ids)
2105 {
2106         struct rte_eth_xstat_name *xstats_names_copy;
2107         unsigned int no_basic_stat_requested = 1;
2108         unsigned int no_ext_stat_requested = 1;
2109         unsigned int expected_entries;
2110         unsigned int basic_count;
2111         struct rte_eth_dev *dev;
2112         unsigned int i;
2113         int ret;
2114
2115         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2116         dev = &rte_eth_devices[port_id];
2117
2118         basic_count = get_xstats_basic_count(dev);
2119         ret = get_xstats_count(port_id);
2120         if (ret < 0)
2121                 return ret;
2122         expected_entries = (unsigned int)ret;
2123
2124         /* Return max number of stats if no ids given */
2125         if (!ids) {
2126                 if (!xstats_names)
2127                         return expected_entries;
2128                 else if (xstats_names && size < expected_entries)
2129                         return expected_entries;
2130         }
2131
2132         if (ids && !xstats_names)
2133                 return -EINVAL;
2134
2135         if (ids && dev->dev_ops->xstats_get_names_by_id != NULL && size > 0) {
2136                 uint64_t ids_copy[size];
2137
2138                 for (i = 0; i < size; i++) {
2139                         if (ids[i] < basic_count) {
2140                                 no_basic_stat_requested = 0;
2141                                 break;
2142                         }
2143
2144                         /*
2145                          * Convert ids to xstats ids that PMD knows.
2146                          * ids known by user are basic + extended stats.
2147                          */
2148                         ids_copy[i] = ids[i] - basic_count;
2149                 }
2150
2151                 if (no_basic_stat_requested)
2152                         return (*dev->dev_ops->xstats_get_names_by_id)(dev,
2153                                         xstats_names, ids_copy, size);
2154         }
2155
2156         /* Retrieve all stats */
2157         if (!ids) {
2158                 int num_stats = rte_eth_xstats_get_names(port_id, xstats_names,
2159                                 expected_entries);
2160                 if (num_stats < 0 || num_stats > (int)expected_entries)
2161                         return num_stats;
2162                 else
2163                         return expected_entries;
2164         }
2165
2166         xstats_names_copy = calloc(expected_entries,
2167                 sizeof(struct rte_eth_xstat_name));
2168
2169         if (!xstats_names_copy) {
2170                 RTE_ETHDEV_LOG(ERR, "Can't allocate memory\n");
2171                 return -ENOMEM;
2172         }
2173
2174         if (ids) {
2175                 for (i = 0; i < size; i++) {
2176                         if (ids[i] >= basic_count) {
2177                                 no_ext_stat_requested = 0;
2178                                 break;
2179                         }
2180                 }
2181         }
2182
2183         /* Fill xstats_names_copy structure */
2184         if (ids && no_ext_stat_requested) {
2185                 rte_eth_basic_stats_get_names(dev, xstats_names_copy);
2186         } else {
2187                 ret = rte_eth_xstats_get_names(port_id, xstats_names_copy,
2188                         expected_entries);
2189                 if (ret < 0) {
2190                         free(xstats_names_copy);
2191                         return ret;
2192                 }
2193         }
2194
2195         /* Filter stats */
2196         for (i = 0; i < size; i++) {
2197                 if (ids[i] >= expected_entries) {
2198                         RTE_ETHDEV_LOG(ERR, "Id value isn't valid\n");
2199                         free(xstats_names_copy);
2200                         return -1;
2201                 }
2202                 xstats_names[i] = xstats_names_copy[ids[i]];
2203         }
2204
2205         free(xstats_names_copy);
2206         return size;
2207 }
2208
2209 int
2210 rte_eth_xstats_get_names(uint16_t port_id,
2211         struct rte_eth_xstat_name *xstats_names,
2212         unsigned int size)
2213 {
2214         struct rte_eth_dev *dev;
2215         int cnt_used_entries;
2216         int cnt_expected_entries;
2217         int cnt_driver_entries;
2218
2219         cnt_expected_entries = get_xstats_count(port_id);
2220         if (xstats_names == NULL || cnt_expected_entries < 0 ||
2221                         (int)size < cnt_expected_entries)
2222                 return cnt_expected_entries;
2223
2224         /* port_id checked in get_xstats_count() */
2225         dev = &rte_eth_devices[port_id];
2226
2227         cnt_used_entries = rte_eth_basic_stats_get_names(
2228                 dev, xstats_names);
2229
2230         if (dev->dev_ops->xstats_get_names != NULL) {
2231                 /* If there are any driver-specific xstats, append them
2232                  * to end of list.
2233                  */
2234                 cnt_driver_entries = (*dev->dev_ops->xstats_get_names)(
2235                         dev,
2236                         xstats_names + cnt_used_entries,
2237                         size - cnt_used_entries);
2238                 if (cnt_driver_entries < 0)
2239                         return eth_err(port_id, cnt_driver_entries);
2240                 cnt_used_entries += cnt_driver_entries;
2241         }
2242
2243         return cnt_used_entries;
2244 }
2245
2246
2247 static int
2248 rte_eth_basic_stats_get(uint16_t port_id, struct rte_eth_xstat *xstats)
2249 {
2250         struct rte_eth_dev *dev;
2251         struct rte_eth_stats eth_stats;
2252         unsigned int count = 0, i, q;
2253         uint64_t val, *stats_ptr;
2254         uint16_t nb_rxqs, nb_txqs;
2255         int ret;
2256
2257         ret = rte_eth_stats_get(port_id, &eth_stats);
2258         if (ret < 0)
2259                 return ret;
2260
2261         dev = &rte_eth_devices[port_id];
2262
2263         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2264         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2265
2266         /* global stats */
2267         for (i = 0; i < RTE_NB_STATS; i++) {
2268                 stats_ptr = RTE_PTR_ADD(&eth_stats,
2269                                         rte_stats_strings[i].offset);
2270                 val = *stats_ptr;
2271                 xstats[count++].value = val;
2272         }
2273
2274         /* per-rxq stats */
2275         for (q = 0; q < nb_rxqs; q++) {
2276                 for (i = 0; i < RTE_NB_RXQ_STATS; i++) {
2277                         stats_ptr = RTE_PTR_ADD(&eth_stats,
2278                                         rte_rxq_stats_strings[i].offset +
2279                                         q * sizeof(uint64_t));
2280                         val = *stats_ptr;
2281                         xstats[count++].value = val;
2282                 }
2283         }
2284
2285         /* per-txq stats */
2286         for (q = 0; q < nb_txqs; q++) {
2287                 for (i = 0; i < RTE_NB_TXQ_STATS; i++) {
2288                         stats_ptr = RTE_PTR_ADD(&eth_stats,
2289                                         rte_txq_stats_strings[i].offset +
2290                                         q * sizeof(uint64_t));
2291                         val = *stats_ptr;
2292                         xstats[count++].value = val;
2293                 }
2294         }
2295         return count;
2296 }
2297
2298 /* retrieve ethdev extended statistics */
2299 int
2300 rte_eth_xstats_get_by_id(uint16_t port_id, const uint64_t *ids,
2301                          uint64_t *values, unsigned int size)
2302 {
2303         unsigned int no_basic_stat_requested = 1;
2304         unsigned int no_ext_stat_requested = 1;
2305         unsigned int num_xstats_filled;
2306         unsigned int basic_count;
2307         uint16_t expected_entries;
2308         struct rte_eth_dev *dev;
2309         unsigned int i;
2310         int ret;
2311
2312         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2313         ret = get_xstats_count(port_id);
2314         if (ret < 0)
2315                 return ret;
2316         expected_entries = (uint16_t)ret;
2317         struct rte_eth_xstat xstats[expected_entries];
2318         dev = &rte_eth_devices[port_id];
2319         basic_count = get_xstats_basic_count(dev);
2320
2321         /* Return max number of stats if no ids given */
2322         if (!ids) {
2323                 if (!values)
2324                         return expected_entries;
2325                 else if (values && size < expected_entries)
2326                         return expected_entries;
2327         }
2328
2329         if (ids && !values)
2330                 return -EINVAL;
2331
2332         if (ids && dev->dev_ops->xstats_get_by_id != NULL && size) {
2333                 unsigned int basic_count = get_xstats_basic_count(dev);
2334                 uint64_t ids_copy[size];
2335
2336                 for (i = 0; i < size; i++) {
2337                         if (ids[i] < basic_count) {
2338                                 no_basic_stat_requested = 0;
2339                                 break;
2340                         }
2341
2342                         /*
2343                          * Convert ids to xstats ids that PMD knows.
2344                          * ids known by user are basic + extended stats.
2345                          */
2346                         ids_copy[i] = ids[i] - basic_count;
2347                 }
2348
2349                 if (no_basic_stat_requested)
2350                         return (*dev->dev_ops->xstats_get_by_id)(dev, ids_copy,
2351                                         values, size);
2352         }
2353
2354         if (ids) {
2355                 for (i = 0; i < size; i++) {
2356                         if (ids[i] >= basic_count) {
2357                                 no_ext_stat_requested = 0;
2358                                 break;
2359                         }
2360                 }
2361         }
2362
2363         /* Fill the xstats structure */
2364         if (ids && no_ext_stat_requested)
2365                 ret = rte_eth_basic_stats_get(port_id, xstats);
2366         else
2367                 ret = rte_eth_xstats_get(port_id, xstats, expected_entries);
2368
2369         if (ret < 0)
2370                 return ret;
2371         num_xstats_filled = (unsigned int)ret;
2372
2373         /* Return all stats */
2374         if (!ids) {
2375                 for (i = 0; i < num_xstats_filled; i++)
2376                         values[i] = xstats[i].value;
2377                 return expected_entries;
2378         }
2379
2380         /* Filter stats */
2381         for (i = 0; i < size; i++) {
2382                 if (ids[i] >= expected_entries) {
2383                         RTE_ETHDEV_LOG(ERR, "Id value isn't valid\n");
2384                         return -1;
2385                 }
2386                 values[i] = xstats[ids[i]].value;
2387         }
2388         return size;
2389 }
2390
2391 int
2392 rte_eth_xstats_get(uint16_t port_id, struct rte_eth_xstat *xstats,
2393         unsigned int n)
2394 {
2395         struct rte_eth_dev *dev;
2396         unsigned int count = 0, i;
2397         signed int xcount = 0;
2398         uint16_t nb_rxqs, nb_txqs;
2399         int ret;
2400
2401         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2402
2403         dev = &rte_eth_devices[port_id];
2404
2405         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2406         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2407
2408         /* Return generic statistics */
2409         count = RTE_NB_STATS + (nb_rxqs * RTE_NB_RXQ_STATS) +
2410                 (nb_txqs * RTE_NB_TXQ_STATS);
2411
2412         /* implemented by the driver */
2413         if (dev->dev_ops->xstats_get != NULL) {
2414                 /* Retrieve the xstats from the driver at the end of the
2415                  * xstats struct.
2416                  */
2417                 xcount = (*dev->dev_ops->xstats_get)(dev,
2418                                      xstats ? xstats + count : NULL,
2419                                      (n > count) ? n - count : 0);
2420
2421                 if (xcount < 0)
2422                         return eth_err(port_id, xcount);
2423         }
2424
2425         if (n < count + xcount || xstats == NULL)
2426                 return count + xcount;
2427
2428         /* now fill the xstats structure */
2429         ret = rte_eth_basic_stats_get(port_id, xstats);
2430         if (ret < 0)
2431                 return ret;
2432         count = ret;
2433
2434         for (i = 0; i < count; i++)
2435                 xstats[i].id = i;
2436         /* add an offset to driver-specific stats */
2437         for ( ; i < count + xcount; i++)
2438                 xstats[i].id += count;
2439
2440         return count + xcount;
2441 }
2442
2443 /* reset ethdev extended statistics */
2444 void
2445 rte_eth_xstats_reset(uint16_t port_id)
2446 {
2447         struct rte_eth_dev *dev;
2448
2449         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2450         dev = &rte_eth_devices[port_id];
2451
2452         /* implemented by the driver */
2453         if (dev->dev_ops->xstats_reset != NULL) {
2454                 (*dev->dev_ops->xstats_reset)(dev);
2455                 return;
2456         }
2457
2458         /* fallback to default */
2459         rte_eth_stats_reset(port_id);
2460 }
2461
2462 static int
2463 set_queue_stats_mapping(uint16_t port_id, uint16_t queue_id, uint8_t stat_idx,
2464                 uint8_t is_rx)
2465 {
2466         struct rte_eth_dev *dev;
2467
2468         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2469
2470         dev = &rte_eth_devices[port_id];
2471
2472         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_stats_mapping_set, -ENOTSUP);
2473
2474         if (is_rx && (queue_id >= dev->data->nb_rx_queues))
2475                 return -EINVAL;
2476
2477         if (!is_rx && (queue_id >= dev->data->nb_tx_queues))
2478                 return -EINVAL;
2479
2480         if (stat_idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
2481                 return -EINVAL;
2482
2483         return (*dev->dev_ops->queue_stats_mapping_set)
2484                         (dev, queue_id, stat_idx, is_rx);
2485 }
2486
2487
2488 int
2489 rte_eth_dev_set_tx_queue_stats_mapping(uint16_t port_id, uint16_t tx_queue_id,
2490                 uint8_t stat_idx)
2491 {
2492         return eth_err(port_id, set_queue_stats_mapping(port_id, tx_queue_id,
2493                                                 stat_idx, STAT_QMAP_TX));
2494 }
2495
2496
2497 int
2498 rte_eth_dev_set_rx_queue_stats_mapping(uint16_t port_id, uint16_t rx_queue_id,
2499                 uint8_t stat_idx)
2500 {
2501         return eth_err(port_id, set_queue_stats_mapping(port_id, rx_queue_id,
2502                                                 stat_idx, STAT_QMAP_RX));
2503 }
2504
2505 int
2506 rte_eth_dev_fw_version_get(uint16_t port_id, char *fw_version, size_t fw_size)
2507 {
2508         struct rte_eth_dev *dev;
2509
2510         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2511         dev = &rte_eth_devices[port_id];
2512
2513         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fw_version_get, -ENOTSUP);
2514         return eth_err(port_id, (*dev->dev_ops->fw_version_get)(dev,
2515                                                         fw_version, fw_size));
2516 }
2517
2518 void
2519 rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info)
2520 {
2521         struct rte_eth_dev *dev;
2522         const struct rte_eth_desc_lim lim = {
2523                 .nb_max = UINT16_MAX,
2524                 .nb_min = 0,
2525                 .nb_align = 1,
2526         };
2527
2528         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2529         dev = &rte_eth_devices[port_id];
2530
2531         memset(dev_info, 0, sizeof(struct rte_eth_dev_info));
2532         dev_info->rx_desc_lim = lim;
2533         dev_info->tx_desc_lim = lim;
2534         dev_info->device = dev->device;
2535
2536         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_infos_get);
2537         (*dev->dev_ops->dev_infos_get)(dev, dev_info);
2538         dev_info->driver_name = dev->device->driver->name;
2539         dev_info->nb_rx_queues = dev->data->nb_rx_queues;
2540         dev_info->nb_tx_queues = dev->data->nb_tx_queues;
2541
2542         dev_info->dev_flags = &dev->data->dev_flags;
2543 }
2544
2545 int
2546 rte_eth_dev_get_supported_ptypes(uint16_t port_id, uint32_t ptype_mask,
2547                                  uint32_t *ptypes, int num)
2548 {
2549         int i, j;
2550         struct rte_eth_dev *dev;
2551         const uint32_t *all_ptypes;
2552
2553         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2554         dev = &rte_eth_devices[port_id];
2555         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_supported_ptypes_get, 0);
2556         all_ptypes = (*dev->dev_ops->dev_supported_ptypes_get)(dev);
2557
2558         if (!all_ptypes)
2559                 return 0;
2560
2561         for (i = 0, j = 0; all_ptypes[i] != RTE_PTYPE_UNKNOWN; ++i)
2562                 if (all_ptypes[i] & ptype_mask) {
2563                         if (j < num)
2564                                 ptypes[j] = all_ptypes[i];
2565                         j++;
2566                 }
2567
2568         return j;
2569 }
2570
2571 void
2572 rte_eth_macaddr_get(uint16_t port_id, struct ether_addr *mac_addr)
2573 {
2574         struct rte_eth_dev *dev;
2575
2576         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2577         dev = &rte_eth_devices[port_id];
2578         ether_addr_copy(&dev->data->mac_addrs[0], mac_addr);
2579 }
2580
2581
2582 int
2583 rte_eth_dev_get_mtu(uint16_t port_id, uint16_t *mtu)
2584 {
2585         struct rte_eth_dev *dev;
2586
2587         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2588
2589         dev = &rte_eth_devices[port_id];
2590         *mtu = dev->data->mtu;
2591         return 0;
2592 }
2593
2594 int
2595 rte_eth_dev_set_mtu(uint16_t port_id, uint16_t mtu)
2596 {
2597         int ret;
2598         struct rte_eth_dev *dev;
2599
2600         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2601         dev = &rte_eth_devices[port_id];
2602         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mtu_set, -ENOTSUP);
2603
2604         ret = (*dev->dev_ops->mtu_set)(dev, mtu);
2605         if (!ret)
2606                 dev->data->mtu = mtu;
2607
2608         return eth_err(port_id, ret);
2609 }
2610
2611 int
2612 rte_eth_dev_vlan_filter(uint16_t port_id, uint16_t vlan_id, int on)
2613 {
2614         struct rte_eth_dev *dev;
2615         int ret;
2616
2617         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2618         dev = &rte_eth_devices[port_id];
2619         if (!(dev->data->dev_conf.rxmode.offloads &
2620               DEV_RX_OFFLOAD_VLAN_FILTER)) {
2621                 RTE_ETHDEV_LOG(ERR, "Port %u: vlan-filtering disabled\n",
2622                         port_id);
2623                 return -ENOSYS;
2624         }
2625
2626         if (vlan_id > 4095) {
2627                 RTE_ETHDEV_LOG(ERR, "Port_id=%u invalid vlan_id=%u > 4095\n",
2628                         port_id, vlan_id);
2629                 return -EINVAL;
2630         }
2631         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_filter_set, -ENOTSUP);
2632
2633         ret = (*dev->dev_ops->vlan_filter_set)(dev, vlan_id, on);
2634         if (ret == 0) {
2635                 struct rte_vlan_filter_conf *vfc;
2636                 int vidx;
2637                 int vbit;
2638
2639                 vfc = &dev->data->vlan_filter_conf;
2640                 vidx = vlan_id / 64;
2641                 vbit = vlan_id % 64;
2642
2643                 if (on)
2644                         vfc->ids[vidx] |= UINT64_C(1) << vbit;
2645                 else
2646                         vfc->ids[vidx] &= ~(UINT64_C(1) << vbit);
2647         }
2648
2649         return eth_err(port_id, ret);
2650 }
2651
2652 int
2653 rte_eth_dev_set_vlan_strip_on_queue(uint16_t port_id, uint16_t rx_queue_id,
2654                                     int on)
2655 {
2656         struct rte_eth_dev *dev;
2657
2658         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2659         dev = &rte_eth_devices[port_id];
2660         if (rx_queue_id >= dev->data->nb_rx_queues) {
2661                 RTE_ETHDEV_LOG(ERR, "Invalid rx_queue_id=%u\n", rx_queue_id);
2662                 return -EINVAL;
2663         }
2664
2665         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_strip_queue_set, -ENOTSUP);
2666         (*dev->dev_ops->vlan_strip_queue_set)(dev, rx_queue_id, on);
2667
2668         return 0;
2669 }
2670
2671 int
2672 rte_eth_dev_set_vlan_ether_type(uint16_t port_id,
2673                                 enum rte_vlan_type vlan_type,
2674                                 uint16_t tpid)
2675 {
2676         struct rte_eth_dev *dev;
2677
2678         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2679         dev = &rte_eth_devices[port_id];
2680         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_tpid_set, -ENOTSUP);
2681
2682         return eth_err(port_id, (*dev->dev_ops->vlan_tpid_set)(dev, vlan_type,
2683                                                                tpid));
2684 }
2685
2686 int
2687 rte_eth_dev_set_vlan_offload(uint16_t port_id, int offload_mask)
2688 {
2689         struct rte_eth_dev *dev;
2690         int ret = 0;
2691         int mask = 0;
2692         int cur, org = 0;
2693         uint64_t orig_offloads;
2694
2695         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2696         dev = &rte_eth_devices[port_id];
2697
2698         /* save original values in case of failure */
2699         orig_offloads = dev->data->dev_conf.rxmode.offloads;
2700
2701         /*check which option changed by application*/
2702         cur = !!(offload_mask & ETH_VLAN_STRIP_OFFLOAD);
2703         org = !!(dev->data->dev_conf.rxmode.offloads &
2704                  DEV_RX_OFFLOAD_VLAN_STRIP);
2705         if (cur != org) {
2706                 if (cur)
2707                         dev->data->dev_conf.rxmode.offloads |=
2708                                 DEV_RX_OFFLOAD_VLAN_STRIP;
2709                 else
2710                         dev->data->dev_conf.rxmode.offloads &=
2711                                 ~DEV_RX_OFFLOAD_VLAN_STRIP;
2712                 mask |= ETH_VLAN_STRIP_MASK;
2713         }
2714
2715         cur = !!(offload_mask & ETH_VLAN_FILTER_OFFLOAD);
2716         org = !!(dev->data->dev_conf.rxmode.offloads &
2717                  DEV_RX_OFFLOAD_VLAN_FILTER);
2718         if (cur != org) {
2719                 if (cur)
2720                         dev->data->dev_conf.rxmode.offloads |=
2721                                 DEV_RX_OFFLOAD_VLAN_FILTER;
2722                 else
2723                         dev->data->dev_conf.rxmode.offloads &=
2724                                 ~DEV_RX_OFFLOAD_VLAN_FILTER;
2725                 mask |= ETH_VLAN_FILTER_MASK;
2726         }
2727
2728         cur = !!(offload_mask & ETH_VLAN_EXTEND_OFFLOAD);
2729         org = !!(dev->data->dev_conf.rxmode.offloads &
2730                  DEV_RX_OFFLOAD_VLAN_EXTEND);
2731         if (cur != org) {
2732                 if (cur)
2733                         dev->data->dev_conf.rxmode.offloads |=
2734                                 DEV_RX_OFFLOAD_VLAN_EXTEND;
2735                 else
2736                         dev->data->dev_conf.rxmode.offloads &=
2737                                 ~DEV_RX_OFFLOAD_VLAN_EXTEND;
2738                 mask |= ETH_VLAN_EXTEND_MASK;
2739         }
2740
2741         /*no change*/
2742         if (mask == 0)
2743                 return ret;
2744
2745         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_offload_set, -ENOTSUP);
2746         ret = (*dev->dev_ops->vlan_offload_set)(dev, mask);
2747         if (ret) {
2748                 /* hit an error restore  original values */
2749                 dev->data->dev_conf.rxmode.offloads = orig_offloads;
2750         }
2751
2752         return eth_err(port_id, ret);
2753 }
2754
2755 int
2756 rte_eth_dev_get_vlan_offload(uint16_t port_id)
2757 {
2758         struct rte_eth_dev *dev;
2759         int ret = 0;
2760
2761         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2762         dev = &rte_eth_devices[port_id];
2763
2764         if (dev->data->dev_conf.rxmode.offloads &
2765             DEV_RX_OFFLOAD_VLAN_STRIP)
2766                 ret |= ETH_VLAN_STRIP_OFFLOAD;
2767
2768         if (dev->data->dev_conf.rxmode.offloads &
2769             DEV_RX_OFFLOAD_VLAN_FILTER)
2770                 ret |= ETH_VLAN_FILTER_OFFLOAD;
2771
2772         if (dev->data->dev_conf.rxmode.offloads &
2773             DEV_RX_OFFLOAD_VLAN_EXTEND)
2774                 ret |= ETH_VLAN_EXTEND_OFFLOAD;
2775
2776         return ret;
2777 }
2778
2779 int
2780 rte_eth_dev_set_vlan_pvid(uint16_t port_id, uint16_t pvid, int on)
2781 {
2782         struct rte_eth_dev *dev;
2783
2784         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2785         dev = &rte_eth_devices[port_id];
2786         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_pvid_set, -ENOTSUP);
2787
2788         return eth_err(port_id, (*dev->dev_ops->vlan_pvid_set)(dev, pvid, on));
2789 }
2790
2791 int
2792 rte_eth_dev_flow_ctrl_get(uint16_t port_id, struct rte_eth_fc_conf *fc_conf)
2793 {
2794         struct rte_eth_dev *dev;
2795
2796         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2797         dev = &rte_eth_devices[port_id];
2798         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_get, -ENOTSUP);
2799         memset(fc_conf, 0, sizeof(*fc_conf));
2800         return eth_err(port_id, (*dev->dev_ops->flow_ctrl_get)(dev, fc_conf));
2801 }
2802
2803 int
2804 rte_eth_dev_flow_ctrl_set(uint16_t port_id, struct rte_eth_fc_conf *fc_conf)
2805 {
2806         struct rte_eth_dev *dev;
2807
2808         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2809         if ((fc_conf->send_xon != 0) && (fc_conf->send_xon != 1)) {
2810                 RTE_ETHDEV_LOG(ERR, "Invalid send_xon, only 0/1 allowed\n");
2811                 return -EINVAL;
2812         }
2813
2814         dev = &rte_eth_devices[port_id];
2815         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_set, -ENOTSUP);
2816         return eth_err(port_id, (*dev->dev_ops->flow_ctrl_set)(dev, fc_conf));
2817 }
2818
2819 int
2820 rte_eth_dev_priority_flow_ctrl_set(uint16_t port_id,
2821                                    struct rte_eth_pfc_conf *pfc_conf)
2822 {
2823         struct rte_eth_dev *dev;
2824
2825         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2826         if (pfc_conf->priority > (ETH_DCB_NUM_USER_PRIORITIES - 1)) {
2827                 RTE_ETHDEV_LOG(ERR, "Invalid priority, only 0-7 allowed\n");
2828                 return -EINVAL;
2829         }
2830
2831         dev = &rte_eth_devices[port_id];
2832         /* High water, low water validation are device specific */
2833         if  (*dev->dev_ops->priority_flow_ctrl_set)
2834                 return eth_err(port_id, (*dev->dev_ops->priority_flow_ctrl_set)
2835                                         (dev, pfc_conf));
2836         return -ENOTSUP;
2837 }
2838
2839 static int
2840 rte_eth_check_reta_mask(struct rte_eth_rss_reta_entry64 *reta_conf,
2841                         uint16_t reta_size)
2842 {
2843         uint16_t i, num;
2844
2845         if (!reta_conf)
2846                 return -EINVAL;
2847
2848         num = (reta_size + RTE_RETA_GROUP_SIZE - 1) / RTE_RETA_GROUP_SIZE;
2849         for (i = 0; i < num; i++) {
2850                 if (reta_conf[i].mask)
2851                         return 0;
2852         }
2853
2854         return -EINVAL;
2855 }
2856
2857 static int
2858 rte_eth_check_reta_entry(struct rte_eth_rss_reta_entry64 *reta_conf,
2859                          uint16_t reta_size,
2860                          uint16_t max_rxq)
2861 {
2862         uint16_t i, idx, shift;
2863
2864         if (!reta_conf)
2865                 return -EINVAL;
2866
2867         if (max_rxq == 0) {
2868                 RTE_ETHDEV_LOG(ERR, "No receive queue is available\n");
2869                 return -EINVAL;
2870         }
2871
2872         for (i = 0; i < reta_size; i++) {
2873                 idx = i / RTE_RETA_GROUP_SIZE;
2874                 shift = i % RTE_RETA_GROUP_SIZE;
2875                 if ((reta_conf[idx].mask & (1ULL << shift)) &&
2876                         (reta_conf[idx].reta[shift] >= max_rxq)) {
2877                         RTE_ETHDEV_LOG(ERR,
2878                                 "reta_conf[%u]->reta[%u]: %u exceeds the maximum rxq index: %u\n",
2879                                 idx, shift,
2880                                 reta_conf[idx].reta[shift], max_rxq);
2881                         return -EINVAL;
2882                 }
2883         }
2884
2885         return 0;
2886 }
2887
2888 int
2889 rte_eth_dev_rss_reta_update(uint16_t port_id,
2890                             struct rte_eth_rss_reta_entry64 *reta_conf,
2891                             uint16_t reta_size)
2892 {
2893         struct rte_eth_dev *dev;
2894         int ret;
2895
2896         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2897         /* Check mask bits */
2898         ret = rte_eth_check_reta_mask(reta_conf, reta_size);
2899         if (ret < 0)
2900                 return ret;
2901
2902         dev = &rte_eth_devices[port_id];
2903
2904         /* Check entry value */
2905         ret = rte_eth_check_reta_entry(reta_conf, reta_size,
2906                                 dev->data->nb_rx_queues);
2907         if (ret < 0)
2908                 return ret;
2909
2910         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_update, -ENOTSUP);
2911         return eth_err(port_id, (*dev->dev_ops->reta_update)(dev, reta_conf,
2912                                                              reta_size));
2913 }
2914
2915 int
2916 rte_eth_dev_rss_reta_query(uint16_t port_id,
2917                            struct rte_eth_rss_reta_entry64 *reta_conf,
2918                            uint16_t reta_size)
2919 {
2920         struct rte_eth_dev *dev;
2921         int ret;
2922
2923         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2924
2925         /* Check mask bits */
2926         ret = rte_eth_check_reta_mask(reta_conf, reta_size);
2927         if (ret < 0)
2928                 return ret;
2929
2930         dev = &rte_eth_devices[port_id];
2931         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_query, -ENOTSUP);
2932         return eth_err(port_id, (*dev->dev_ops->reta_query)(dev, reta_conf,
2933                                                             reta_size));
2934 }
2935
2936 int
2937 rte_eth_dev_rss_hash_update(uint16_t port_id,
2938                             struct rte_eth_rss_conf *rss_conf)
2939 {
2940         struct rte_eth_dev *dev;
2941         struct rte_eth_dev_info dev_info = { .flow_type_rss_offloads = 0, };
2942
2943         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2944         dev = &rte_eth_devices[port_id];
2945         rte_eth_dev_info_get(port_id, &dev_info);
2946         if ((dev_info.flow_type_rss_offloads | rss_conf->rss_hf) !=
2947             dev_info.flow_type_rss_offloads) {
2948                 RTE_ETHDEV_LOG(ERR,
2949                         "Ethdev port_id=%u invalid rss_hf: 0x%"PRIx64", valid value: 0x%"PRIx64"\n",
2950                         port_id, rss_conf->rss_hf,
2951                         dev_info.flow_type_rss_offloads);
2952                 return -EINVAL;
2953         }
2954         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_update, -ENOTSUP);
2955         return eth_err(port_id, (*dev->dev_ops->rss_hash_update)(dev,
2956                                                                  rss_conf));
2957 }
2958
2959 int
2960 rte_eth_dev_rss_hash_conf_get(uint16_t port_id,
2961                               struct rte_eth_rss_conf *rss_conf)
2962 {
2963         struct rte_eth_dev *dev;
2964
2965         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2966         dev = &rte_eth_devices[port_id];
2967         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_conf_get, -ENOTSUP);
2968         return eth_err(port_id, (*dev->dev_ops->rss_hash_conf_get)(dev,
2969                                                                    rss_conf));
2970 }
2971
2972 int
2973 rte_eth_dev_udp_tunnel_port_add(uint16_t port_id,
2974                                 struct rte_eth_udp_tunnel *udp_tunnel)
2975 {
2976         struct rte_eth_dev *dev;
2977
2978         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2979         if (udp_tunnel == NULL) {
2980                 RTE_ETHDEV_LOG(ERR, "Invalid udp_tunnel parameter\n");
2981                 return -EINVAL;
2982         }
2983
2984         if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
2985                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
2986                 return -EINVAL;
2987         }
2988
2989         dev = &rte_eth_devices[port_id];
2990         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_port_add, -ENOTSUP);
2991         return eth_err(port_id, (*dev->dev_ops->udp_tunnel_port_add)(dev,
2992                                                                 udp_tunnel));
2993 }
2994
2995 int
2996 rte_eth_dev_udp_tunnel_port_delete(uint16_t port_id,
2997                                    struct rte_eth_udp_tunnel *udp_tunnel)
2998 {
2999         struct rte_eth_dev *dev;
3000
3001         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3002         dev = &rte_eth_devices[port_id];
3003
3004         if (udp_tunnel == NULL) {
3005                 RTE_ETHDEV_LOG(ERR, "Invalid udp_tunnel parameter\n");
3006                 return -EINVAL;
3007         }
3008
3009         if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
3010                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
3011                 return -EINVAL;
3012         }
3013
3014         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_port_del, -ENOTSUP);
3015         return eth_err(port_id, (*dev->dev_ops->udp_tunnel_port_del)(dev,
3016                                                                 udp_tunnel));
3017 }
3018
3019 int
3020 rte_eth_led_on(uint16_t port_id)
3021 {
3022         struct rte_eth_dev *dev;
3023
3024         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3025         dev = &rte_eth_devices[port_id];
3026         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_on, -ENOTSUP);
3027         return eth_err(port_id, (*dev->dev_ops->dev_led_on)(dev));
3028 }
3029
3030 int
3031 rte_eth_led_off(uint16_t port_id)
3032 {
3033         struct rte_eth_dev *dev;
3034
3035         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3036         dev = &rte_eth_devices[port_id];
3037         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_off, -ENOTSUP);
3038         return eth_err(port_id, (*dev->dev_ops->dev_led_off)(dev));
3039 }
3040
3041 /*
3042  * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find
3043  * an empty spot.
3044  */
3045 static int
3046 get_mac_addr_index(uint16_t port_id, const struct ether_addr *addr)
3047 {
3048         struct rte_eth_dev_info dev_info;
3049         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3050         unsigned i;
3051
3052         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3053         rte_eth_dev_info_get(port_id, &dev_info);
3054
3055         for (i = 0; i < dev_info.max_mac_addrs; i++)
3056                 if (memcmp(addr, &dev->data->mac_addrs[i], ETHER_ADDR_LEN) == 0)
3057                         return i;
3058
3059         return -1;
3060 }
3061
3062 static const struct ether_addr null_mac_addr;
3063
3064 int
3065 rte_eth_dev_mac_addr_add(uint16_t port_id, struct ether_addr *addr,
3066                         uint32_t pool)
3067 {
3068         struct rte_eth_dev *dev;
3069         int index;
3070         uint64_t pool_mask;
3071         int ret;
3072
3073         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3074         dev = &rte_eth_devices[port_id];
3075         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_add, -ENOTSUP);
3076
3077         if (is_zero_ether_addr(addr)) {
3078                 RTE_ETHDEV_LOG(ERR, "Port %u: Cannot add NULL MAC address\n",
3079                         port_id);
3080                 return -EINVAL;
3081         }
3082         if (pool >= ETH_64_POOLS) {
3083                 RTE_ETHDEV_LOG(ERR, "Pool id must be 0-%d\n", ETH_64_POOLS - 1);
3084                 return -EINVAL;
3085         }
3086
3087         index = get_mac_addr_index(port_id, addr);
3088         if (index < 0) {
3089                 index = get_mac_addr_index(port_id, &null_mac_addr);
3090                 if (index < 0) {
3091                         RTE_ETHDEV_LOG(ERR, "Port %u: MAC address array full\n",
3092                                 port_id);
3093                         return -ENOSPC;
3094                 }
3095         } else {
3096                 pool_mask = dev->data->mac_pool_sel[index];
3097
3098                 /* Check if both MAC address and pool is already there, and do nothing */
3099                 if (pool_mask & (1ULL << pool))
3100                         return 0;
3101         }
3102
3103         /* Update NIC */
3104         ret = (*dev->dev_ops->mac_addr_add)(dev, addr, index, pool);
3105
3106         if (ret == 0) {
3107                 /* Update address in NIC data structure */
3108                 ether_addr_copy(addr, &dev->data->mac_addrs[index]);
3109
3110                 /* Update pool bitmap in NIC data structure */
3111                 dev->data->mac_pool_sel[index] |= (1ULL << pool);
3112         }
3113
3114         return eth_err(port_id, ret);
3115 }
3116
3117 int
3118 rte_eth_dev_mac_addr_remove(uint16_t port_id, struct ether_addr *addr)
3119 {
3120         struct rte_eth_dev *dev;
3121         int index;
3122
3123         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3124         dev = &rte_eth_devices[port_id];
3125         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_remove, -ENOTSUP);
3126
3127         index = get_mac_addr_index(port_id, addr);
3128         if (index == 0) {
3129                 RTE_ETHDEV_LOG(ERR,
3130                         "Port %u: Cannot remove default MAC address\n",
3131                         port_id);
3132                 return -EADDRINUSE;
3133         } else if (index < 0)
3134                 return 0;  /* Do nothing if address wasn't found */
3135
3136         /* Update NIC */
3137         (*dev->dev_ops->mac_addr_remove)(dev, index);
3138
3139         /* Update address in NIC data structure */
3140         ether_addr_copy(&null_mac_addr, &dev->data->mac_addrs[index]);
3141
3142         /* reset pool bitmap */
3143         dev->data->mac_pool_sel[index] = 0;
3144
3145         return 0;
3146 }
3147
3148 int
3149 rte_eth_dev_default_mac_addr_set(uint16_t port_id, struct ether_addr *addr)
3150 {
3151         struct rte_eth_dev *dev;
3152         int ret;
3153
3154         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3155
3156         if (!is_valid_assigned_ether_addr(addr))
3157                 return -EINVAL;
3158
3159         dev = &rte_eth_devices[port_id];
3160         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_set, -ENOTSUP);
3161
3162         ret = (*dev->dev_ops->mac_addr_set)(dev, addr);
3163         if (ret < 0)
3164                 return ret;
3165
3166         /* Update default address in NIC data structure */
3167         ether_addr_copy(addr, &dev->data->mac_addrs[0]);
3168
3169         return 0;
3170 }
3171
3172
3173 /*
3174  * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find
3175  * an empty spot.
3176  */
3177 static int
3178 get_hash_mac_addr_index(uint16_t port_id, const struct ether_addr *addr)
3179 {
3180         struct rte_eth_dev_info dev_info;
3181         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3182         unsigned i;
3183
3184         rte_eth_dev_info_get(port_id, &dev_info);
3185         if (!dev->data->hash_mac_addrs)
3186                 return -1;
3187
3188         for (i = 0; i < dev_info.max_hash_mac_addrs; i++)
3189                 if (memcmp(addr, &dev->data->hash_mac_addrs[i],
3190                         ETHER_ADDR_LEN) == 0)
3191                         return i;
3192
3193         return -1;
3194 }
3195
3196 int
3197 rte_eth_dev_uc_hash_table_set(uint16_t port_id, struct ether_addr *addr,
3198                                 uint8_t on)
3199 {
3200         int index;
3201         int ret;
3202         struct rte_eth_dev *dev;
3203
3204         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3205
3206         dev = &rte_eth_devices[port_id];
3207         if (is_zero_ether_addr(addr)) {
3208                 RTE_ETHDEV_LOG(ERR, "Port %u: Cannot add NULL MAC address\n",
3209                         port_id);
3210                 return -EINVAL;
3211         }
3212
3213         index = get_hash_mac_addr_index(port_id, addr);
3214         /* Check if it's already there, and do nothing */
3215         if ((index >= 0) && on)
3216                 return 0;
3217
3218         if (index < 0) {
3219                 if (!on) {
3220                         RTE_ETHDEV_LOG(ERR,
3221                                 "Port %u: the MAC address was not set in UTA\n",
3222                                 port_id);
3223                         return -EINVAL;
3224                 }
3225
3226                 index = get_hash_mac_addr_index(port_id, &null_mac_addr);
3227                 if (index < 0) {
3228                         RTE_ETHDEV_LOG(ERR, "Port %u: MAC address array full\n",
3229                                 port_id);
3230                         return -ENOSPC;
3231                 }
3232         }
3233
3234         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_hash_table_set, -ENOTSUP);
3235         ret = (*dev->dev_ops->uc_hash_table_set)(dev, addr, on);
3236         if (ret == 0) {
3237                 /* Update address in NIC data structure */
3238                 if (on)
3239                         ether_addr_copy(addr,
3240                                         &dev->data->hash_mac_addrs[index]);
3241                 else
3242                         ether_addr_copy(&null_mac_addr,
3243                                         &dev->data->hash_mac_addrs[index]);
3244         }
3245
3246         return eth_err(port_id, ret);
3247 }
3248
3249 int
3250 rte_eth_dev_uc_all_hash_table_set(uint16_t port_id, uint8_t on)
3251 {
3252         struct rte_eth_dev *dev;
3253
3254         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3255
3256         dev = &rte_eth_devices[port_id];
3257
3258         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_all_hash_table_set, -ENOTSUP);
3259         return eth_err(port_id, (*dev->dev_ops->uc_all_hash_table_set)(dev,
3260                                                                        on));
3261 }
3262
3263 int rte_eth_set_queue_rate_limit(uint16_t port_id, uint16_t queue_idx,
3264                                         uint16_t tx_rate)
3265 {
3266         struct rte_eth_dev *dev;
3267         struct rte_eth_dev_info dev_info;
3268         struct rte_eth_link link;
3269
3270         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3271
3272         dev = &rte_eth_devices[port_id];
3273         rte_eth_dev_info_get(port_id, &dev_info);
3274         link = dev->data->dev_link;
3275
3276         if (queue_idx > dev_info.max_tx_queues) {
3277                 RTE_ETHDEV_LOG(ERR,
3278                         "Set queue rate limit:port %u: invalid queue id=%u\n",
3279                         port_id, queue_idx);
3280                 return -EINVAL;
3281         }
3282
3283         if (tx_rate > link.link_speed) {
3284                 RTE_ETHDEV_LOG(ERR,
3285                         "Set queue rate limit:invalid tx_rate=%u, bigger than link speed= %d\n",
3286                         tx_rate, link.link_speed);
3287                 return -EINVAL;
3288         }
3289
3290         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_queue_rate_limit, -ENOTSUP);
3291         return eth_err(port_id, (*dev->dev_ops->set_queue_rate_limit)(dev,
3292                                                         queue_idx, tx_rate));
3293 }
3294
3295 int
3296 rte_eth_mirror_rule_set(uint16_t port_id,
3297                         struct rte_eth_mirror_conf *mirror_conf,
3298                         uint8_t rule_id, uint8_t on)
3299 {
3300         struct rte_eth_dev *dev;
3301
3302         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3303         if (mirror_conf->rule_type == 0) {
3304                 RTE_ETHDEV_LOG(ERR, "Mirror rule type can not be 0\n");
3305                 return -EINVAL;
3306         }
3307
3308         if (mirror_conf->dst_pool >= ETH_64_POOLS) {
3309                 RTE_ETHDEV_LOG(ERR, "Invalid dst pool, pool id must be 0-%d\n",
3310                         ETH_64_POOLS - 1);
3311                 return -EINVAL;
3312         }
3313
3314         if ((mirror_conf->rule_type & (ETH_MIRROR_VIRTUAL_POOL_UP |
3315              ETH_MIRROR_VIRTUAL_POOL_DOWN)) &&
3316             (mirror_conf->pool_mask == 0)) {
3317                 RTE_ETHDEV_LOG(ERR,
3318                         "Invalid mirror pool, pool mask can not be 0\n");
3319                 return -EINVAL;
3320         }
3321
3322         if ((mirror_conf->rule_type & ETH_MIRROR_VLAN) &&
3323             mirror_conf->vlan.vlan_mask == 0) {
3324                 RTE_ETHDEV_LOG(ERR,
3325                         "Invalid vlan mask, vlan mask can not be 0\n");
3326                 return -EINVAL;
3327         }
3328
3329         dev = &rte_eth_devices[port_id];
3330         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mirror_rule_set, -ENOTSUP);
3331
3332         return eth_err(port_id, (*dev->dev_ops->mirror_rule_set)(dev,
3333                                                 mirror_conf, rule_id, on));
3334 }
3335
3336 int
3337 rte_eth_mirror_rule_reset(uint16_t port_id, uint8_t rule_id)
3338 {
3339         struct rte_eth_dev *dev;
3340
3341         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3342
3343         dev = &rte_eth_devices[port_id];
3344         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mirror_rule_reset, -ENOTSUP);
3345
3346         return eth_err(port_id, (*dev->dev_ops->mirror_rule_reset)(dev,
3347                                                                    rule_id));
3348 }
3349
3350 RTE_INIT(eth_dev_init_cb_lists)
3351 {
3352         int i;
3353
3354         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3355                 TAILQ_INIT(&rte_eth_devices[i].link_intr_cbs);
3356 }
3357
3358 int
3359 rte_eth_dev_callback_register(uint16_t port_id,
3360                         enum rte_eth_event_type event,
3361                         rte_eth_dev_cb_fn cb_fn, void *cb_arg)
3362 {
3363         struct rte_eth_dev *dev;
3364         struct rte_eth_dev_callback *user_cb;
3365         uint32_t next_port; /* size is 32-bit to prevent loop wrap-around */
3366         uint16_t last_port;
3367
3368         if (!cb_fn)
3369                 return -EINVAL;
3370
3371         if (!rte_eth_dev_is_valid_port(port_id) && port_id != RTE_ETH_ALL) {
3372                 RTE_ETHDEV_LOG(ERR, "Invalid port_id=%d\n", port_id);
3373                 return -EINVAL;
3374         }
3375
3376         if (port_id == RTE_ETH_ALL) {
3377                 next_port = 0;
3378                 last_port = RTE_MAX_ETHPORTS - 1;
3379         } else {
3380                 next_port = last_port = port_id;
3381         }
3382
3383         rte_spinlock_lock(&rte_eth_dev_cb_lock);
3384
3385         do {
3386                 dev = &rte_eth_devices[next_port];
3387
3388                 TAILQ_FOREACH(user_cb, &(dev->link_intr_cbs), next) {
3389                         if (user_cb->cb_fn == cb_fn &&
3390                                 user_cb->cb_arg == cb_arg &&
3391                                 user_cb->event == event) {
3392                                 break;
3393                         }
3394                 }
3395
3396                 /* create a new callback. */
3397                 if (user_cb == NULL) {
3398                         user_cb = rte_zmalloc("INTR_USER_CALLBACK",
3399                                 sizeof(struct rte_eth_dev_callback), 0);
3400                         if (user_cb != NULL) {
3401                                 user_cb->cb_fn = cb_fn;
3402                                 user_cb->cb_arg = cb_arg;
3403                                 user_cb->event = event;
3404                                 TAILQ_INSERT_TAIL(&(dev->link_intr_cbs),
3405                                                   user_cb, next);
3406                         } else {
3407                                 rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3408                                 rte_eth_dev_callback_unregister(port_id, event,
3409                                                                 cb_fn, cb_arg);
3410                                 return -ENOMEM;
3411                         }
3412
3413                 }
3414         } while (++next_port <= last_port);
3415
3416         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3417         return 0;
3418 }
3419
3420 int
3421 rte_eth_dev_callback_unregister(uint16_t port_id,
3422                         enum rte_eth_event_type event,
3423                         rte_eth_dev_cb_fn cb_fn, void *cb_arg)
3424 {
3425         int ret;
3426         struct rte_eth_dev *dev;
3427         struct rte_eth_dev_callback *cb, *next;
3428         uint32_t next_port; /* size is 32-bit to prevent loop wrap-around */
3429         uint16_t last_port;
3430
3431         if (!cb_fn)
3432                 return -EINVAL;
3433
3434         if (!rte_eth_dev_is_valid_port(port_id) && port_id != RTE_ETH_ALL) {
3435                 RTE_ETHDEV_LOG(ERR, "Invalid port_id=%d\n", port_id);
3436                 return -EINVAL;
3437         }
3438
3439         if (port_id == RTE_ETH_ALL) {
3440                 next_port = 0;
3441                 last_port = RTE_MAX_ETHPORTS - 1;
3442         } else {
3443                 next_port = last_port = port_id;
3444         }
3445
3446         rte_spinlock_lock(&rte_eth_dev_cb_lock);
3447
3448         do {
3449                 dev = &rte_eth_devices[next_port];
3450                 ret = 0;
3451                 for (cb = TAILQ_FIRST(&dev->link_intr_cbs); cb != NULL;
3452                      cb = next) {
3453
3454                         next = TAILQ_NEXT(cb, next);
3455
3456                         if (cb->cb_fn != cb_fn || cb->event != event ||
3457                             (cb->cb_arg != (void *)-1 && cb->cb_arg != cb_arg))
3458                                 continue;
3459
3460                         /*
3461                          * if this callback is not executing right now,
3462                          * then remove it.
3463                          */
3464                         if (cb->active == 0) {
3465                                 TAILQ_REMOVE(&(dev->link_intr_cbs), cb, next);
3466                                 rte_free(cb);
3467                         } else {
3468                                 ret = -EAGAIN;
3469                         }
3470                 }
3471         } while (++next_port <= last_port);
3472
3473         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3474         return ret;
3475 }
3476
3477 int
3478 _rte_eth_dev_callback_process(struct rte_eth_dev *dev,
3479         enum rte_eth_event_type event, void *ret_param)
3480 {
3481         struct rte_eth_dev_callback *cb_lst;
3482         struct rte_eth_dev_callback dev_cb;
3483         int rc = 0;
3484
3485         rte_spinlock_lock(&rte_eth_dev_cb_lock);
3486         TAILQ_FOREACH(cb_lst, &(dev->link_intr_cbs), next) {
3487                 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
3488                         continue;
3489                 dev_cb = *cb_lst;
3490                 cb_lst->active = 1;
3491                 if (ret_param != NULL)
3492                         dev_cb.ret_param = ret_param;
3493
3494                 rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3495                 rc = dev_cb.cb_fn(dev->data->port_id, dev_cb.event,
3496                                 dev_cb.cb_arg, dev_cb.ret_param);
3497                 rte_spinlock_lock(&rte_eth_dev_cb_lock);
3498                 cb_lst->active = 0;
3499         }
3500         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3501         return rc;
3502 }
3503
3504 void
3505 rte_eth_dev_probing_finish(struct rte_eth_dev *dev)
3506 {
3507         if (dev == NULL)
3508                 return;
3509
3510         _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_NEW, NULL);
3511
3512         dev->state = RTE_ETH_DEV_ATTACHED;
3513 }
3514
3515 int
3516 rte_eth_dev_rx_intr_ctl(uint16_t port_id, int epfd, int op, void *data)
3517 {
3518         uint32_t vec;
3519         struct rte_eth_dev *dev;
3520         struct rte_intr_handle *intr_handle;
3521         uint16_t qid;
3522         int rc;
3523
3524         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3525
3526         dev = &rte_eth_devices[port_id];
3527
3528         if (!dev->intr_handle) {
3529                 RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
3530                 return -ENOTSUP;
3531         }
3532
3533         intr_handle = dev->intr_handle;
3534         if (!intr_handle->intr_vec) {
3535                 RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
3536                 return -EPERM;
3537         }
3538
3539         for (qid = 0; qid < dev->data->nb_rx_queues; qid++) {
3540                 vec = intr_handle->intr_vec[qid];
3541                 rc = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
3542                 if (rc && rc != -EEXIST) {
3543                         RTE_ETHDEV_LOG(ERR,
3544                                 "p %u q %u rx ctl error op %d epfd %d vec %u\n",
3545                                 port_id, qid, op, epfd, vec);
3546                 }
3547         }
3548
3549         return 0;
3550 }
3551
3552 int __rte_experimental
3553 rte_eth_dev_rx_intr_ctl_q_get_fd(uint16_t port_id, uint16_t queue_id)
3554 {
3555         struct rte_intr_handle *intr_handle;
3556         struct rte_eth_dev *dev;
3557         unsigned int efd_idx;
3558         uint32_t vec;
3559         int fd;
3560
3561         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -1);
3562
3563         dev = &rte_eth_devices[port_id];
3564
3565         if (queue_id >= dev->data->nb_rx_queues) {
3566                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
3567                 return -1;
3568         }
3569
3570         if (!dev->intr_handle) {
3571                 RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
3572                 return -1;
3573         }
3574
3575         intr_handle = dev->intr_handle;
3576         if (!intr_handle->intr_vec) {
3577                 RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
3578                 return -1;
3579         }
3580
3581         vec = intr_handle->intr_vec[queue_id];
3582         efd_idx = (vec >= RTE_INTR_VEC_RXTX_OFFSET) ?
3583                 (vec - RTE_INTR_VEC_RXTX_OFFSET) : vec;
3584         fd = intr_handle->efds[efd_idx];
3585
3586         return fd;
3587 }
3588
3589 const struct rte_memzone *
3590 rte_eth_dma_zone_reserve(const struct rte_eth_dev *dev, const char *ring_name,
3591                          uint16_t queue_id, size_t size, unsigned align,
3592                          int socket_id)
3593 {
3594         char z_name[RTE_MEMZONE_NAMESIZE];
3595         const struct rte_memzone *mz;
3596         int rc;
3597
3598         rc = snprintf(z_name, sizeof(z_name), "eth_p%d_q%d_%s",
3599                       dev->data->port_id, queue_id, ring_name);
3600         if (rc >= RTE_MEMZONE_NAMESIZE) {
3601                 RTE_ETHDEV_LOG(ERR, "ring name too long\n");
3602                 rte_errno = ENAMETOOLONG;
3603                 return NULL;
3604         }
3605
3606         mz = rte_memzone_lookup(z_name);
3607         if (mz)
3608                 return mz;
3609
3610         return rte_memzone_reserve_aligned(z_name, size, socket_id,
3611                         RTE_MEMZONE_IOVA_CONTIG, align);
3612 }
3613
3614 int __rte_experimental
3615 rte_eth_dev_create(struct rte_device *device, const char *name,
3616         size_t priv_data_size,
3617         ethdev_bus_specific_init ethdev_bus_specific_init,
3618         void *bus_init_params,
3619         ethdev_init_t ethdev_init, void *init_params)
3620 {
3621         struct rte_eth_dev *ethdev;
3622         int retval;
3623
3624         RTE_FUNC_PTR_OR_ERR_RET(*ethdev_init, -EINVAL);
3625
3626         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
3627                 ethdev = rte_eth_dev_allocate(name);
3628                 if (!ethdev)
3629                         return -ENODEV;
3630
3631                 if (priv_data_size) {
3632                         ethdev->data->dev_private = rte_zmalloc_socket(
3633                                 name, priv_data_size, RTE_CACHE_LINE_SIZE,
3634                                 device->numa_node);
3635
3636                         if (!ethdev->data->dev_private) {
3637                                 RTE_LOG(ERR, EAL, "failed to allocate private data");
3638                                 retval = -ENOMEM;
3639                                 goto probe_failed;
3640                         }
3641                 }
3642         } else {
3643                 ethdev = rte_eth_dev_attach_secondary(name);
3644                 if (!ethdev) {
3645                         RTE_LOG(ERR, EAL, "secondary process attach failed, "
3646                                 "ethdev doesn't exist");
3647                         return  -ENODEV;
3648                 }
3649         }
3650
3651         ethdev->device = device;
3652
3653         if (ethdev_bus_specific_init) {
3654                 retval = ethdev_bus_specific_init(ethdev, bus_init_params);
3655                 if (retval) {
3656                         RTE_LOG(ERR, EAL,
3657                                 "ethdev bus specific initialisation failed");
3658                         goto probe_failed;
3659                 }
3660         }
3661
3662         retval = ethdev_init(ethdev, init_params);
3663         if (retval) {
3664                 RTE_LOG(ERR, EAL, "ethdev initialisation failed");
3665                 goto probe_failed;
3666         }
3667
3668         rte_eth_dev_probing_finish(ethdev);
3669
3670         return retval;
3671
3672 probe_failed:
3673         rte_eth_dev_release_port(ethdev);
3674         return retval;
3675 }
3676
3677 int  __rte_experimental
3678 rte_eth_dev_destroy(struct rte_eth_dev *ethdev,
3679         ethdev_uninit_t ethdev_uninit)
3680 {
3681         int ret;
3682
3683         ethdev = rte_eth_dev_allocated(ethdev->data->name);
3684         if (!ethdev)
3685                 return -ENODEV;
3686
3687         RTE_FUNC_PTR_OR_ERR_RET(*ethdev_uninit, -EINVAL);
3688
3689         ret = ethdev_uninit(ethdev);
3690         if (ret)
3691                 return ret;
3692
3693         return rte_eth_dev_release_port(ethdev);
3694 }
3695
3696 int
3697 rte_eth_dev_rx_intr_ctl_q(uint16_t port_id, uint16_t queue_id,
3698                           int epfd, int op, void *data)
3699 {
3700         uint32_t vec;
3701         struct rte_eth_dev *dev;
3702         struct rte_intr_handle *intr_handle;
3703         int rc;
3704
3705         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3706
3707         dev = &rte_eth_devices[port_id];
3708         if (queue_id >= dev->data->nb_rx_queues) {
3709                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
3710                 return -EINVAL;
3711         }
3712
3713         if (!dev->intr_handle) {
3714                 RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
3715                 return -ENOTSUP;
3716         }
3717
3718         intr_handle = dev->intr_handle;
3719         if (!intr_handle->intr_vec) {
3720                 RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
3721                 return -EPERM;
3722         }
3723
3724         vec = intr_handle->intr_vec[queue_id];
3725         rc = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
3726         if (rc && rc != -EEXIST) {
3727                 RTE_ETHDEV_LOG(ERR,
3728                         "p %u q %u rx ctl error op %d epfd %d vec %u\n",
3729                         port_id, queue_id, op, epfd, vec);
3730                 return rc;
3731         }
3732
3733         return 0;
3734 }
3735
3736 int
3737 rte_eth_dev_rx_intr_enable(uint16_t port_id,
3738                            uint16_t queue_id)
3739 {
3740         struct rte_eth_dev *dev;
3741
3742         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3743
3744         dev = &rte_eth_devices[port_id];
3745
3746         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_intr_enable, -ENOTSUP);
3747         return eth_err(port_id, (*dev->dev_ops->rx_queue_intr_enable)(dev,
3748                                                                 queue_id));
3749 }
3750
3751 int
3752 rte_eth_dev_rx_intr_disable(uint16_t port_id,
3753                             uint16_t queue_id)
3754 {
3755         struct rte_eth_dev *dev;
3756
3757         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3758
3759         dev = &rte_eth_devices[port_id];
3760
3761         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_intr_disable, -ENOTSUP);
3762         return eth_err(port_id, (*dev->dev_ops->rx_queue_intr_disable)(dev,
3763                                                                 queue_id));
3764 }
3765
3766
3767 int
3768 rte_eth_dev_filter_supported(uint16_t port_id,
3769                              enum rte_filter_type filter_type)
3770 {
3771         struct rte_eth_dev *dev;
3772
3773         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3774
3775         dev = &rte_eth_devices[port_id];
3776         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->filter_ctrl, -ENOTSUP);
3777         return (*dev->dev_ops->filter_ctrl)(dev, filter_type,
3778                                 RTE_ETH_FILTER_NOP, NULL);
3779 }
3780
3781 int
3782 rte_eth_dev_filter_ctrl(uint16_t port_id, enum rte_filter_type filter_type,
3783                         enum rte_filter_op filter_op, void *arg)
3784 {
3785         struct rte_eth_dev *dev;
3786
3787         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3788
3789         dev = &rte_eth_devices[port_id];
3790         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->filter_ctrl, -ENOTSUP);
3791         return eth_err(port_id, (*dev->dev_ops->filter_ctrl)(dev, filter_type,
3792                                                              filter_op, arg));
3793 }
3794
3795 const struct rte_eth_rxtx_callback *
3796 rte_eth_add_rx_callback(uint16_t port_id, uint16_t queue_id,
3797                 rte_rx_callback_fn fn, void *user_param)
3798 {
3799 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3800         rte_errno = ENOTSUP;
3801         return NULL;
3802 #endif
3803         /* check input parameters */
3804         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
3805                     queue_id >= rte_eth_devices[port_id].data->nb_rx_queues) {
3806                 rte_errno = EINVAL;
3807                 return NULL;
3808         }
3809         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
3810
3811         if (cb == NULL) {
3812                 rte_errno = ENOMEM;
3813                 return NULL;
3814         }
3815
3816         cb->fn.rx = fn;
3817         cb->param = user_param;
3818
3819         rte_spinlock_lock(&rte_eth_rx_cb_lock);
3820         /* Add the callbacks in fifo order. */
3821         struct rte_eth_rxtx_callback *tail =
3822                 rte_eth_devices[port_id].post_rx_burst_cbs[queue_id];
3823
3824         if (!tail) {
3825                 rte_eth_devices[port_id].post_rx_burst_cbs[queue_id] = cb;
3826
3827         } else {
3828                 while (tail->next)
3829                         tail = tail->next;
3830                 tail->next = cb;
3831         }
3832         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
3833
3834         return cb;
3835 }
3836
3837 const struct rte_eth_rxtx_callback *
3838 rte_eth_add_first_rx_callback(uint16_t port_id, uint16_t queue_id,
3839                 rte_rx_callback_fn fn, void *user_param)
3840 {
3841 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3842         rte_errno = ENOTSUP;
3843         return NULL;
3844 #endif
3845         /* check input parameters */
3846         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
3847                 queue_id >= rte_eth_devices[port_id].data->nb_rx_queues) {
3848                 rte_errno = EINVAL;
3849                 return NULL;
3850         }
3851
3852         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
3853
3854         if (cb == NULL) {
3855                 rte_errno = ENOMEM;
3856                 return NULL;
3857         }
3858
3859         cb->fn.rx = fn;
3860         cb->param = user_param;
3861
3862         rte_spinlock_lock(&rte_eth_rx_cb_lock);
3863         /* Add the callbacks at fisrt position*/
3864         cb->next = rte_eth_devices[port_id].post_rx_burst_cbs[queue_id];
3865         rte_smp_wmb();
3866         rte_eth_devices[port_id].post_rx_burst_cbs[queue_id] = cb;
3867         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
3868
3869         return cb;
3870 }
3871
3872 const struct rte_eth_rxtx_callback *
3873 rte_eth_add_tx_callback(uint16_t port_id, uint16_t queue_id,
3874                 rte_tx_callback_fn fn, void *user_param)
3875 {
3876 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3877         rte_errno = ENOTSUP;
3878         return NULL;
3879 #endif
3880         /* check input parameters */
3881         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
3882                     queue_id >= rte_eth_devices[port_id].data->nb_tx_queues) {
3883                 rte_errno = EINVAL;
3884                 return NULL;
3885         }
3886
3887         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
3888
3889         if (cb == NULL) {
3890                 rte_errno = ENOMEM;
3891                 return NULL;
3892         }
3893
3894         cb->fn.tx = fn;
3895         cb->param = user_param;
3896
3897         rte_spinlock_lock(&rte_eth_tx_cb_lock);
3898         /* Add the callbacks in fifo order. */
3899         struct rte_eth_rxtx_callback *tail =
3900                 rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id];
3901
3902         if (!tail) {
3903                 rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id] = cb;
3904
3905         } else {
3906                 while (tail->next)
3907                         tail = tail->next;
3908                 tail->next = cb;
3909         }
3910         rte_spinlock_unlock(&rte_eth_tx_cb_lock);
3911
3912         return cb;
3913 }
3914
3915 int
3916 rte_eth_remove_rx_callback(uint16_t port_id, uint16_t queue_id,
3917                 const struct rte_eth_rxtx_callback *user_cb)
3918 {
3919 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3920         return -ENOTSUP;
3921 #endif
3922         /* Check input parameters. */
3923         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
3924         if (user_cb == NULL ||
3925                         queue_id >= rte_eth_devices[port_id].data->nb_rx_queues)
3926                 return -EINVAL;
3927
3928         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3929         struct rte_eth_rxtx_callback *cb;
3930         struct rte_eth_rxtx_callback **prev_cb;
3931         int ret = -EINVAL;
3932
3933         rte_spinlock_lock(&rte_eth_rx_cb_lock);
3934         prev_cb = &dev->post_rx_burst_cbs[queue_id];
3935         for (; *prev_cb != NULL; prev_cb = &cb->next) {
3936                 cb = *prev_cb;
3937                 if (cb == user_cb) {
3938                         /* Remove the user cb from the callback list. */
3939                         *prev_cb = cb->next;
3940                         ret = 0;
3941                         break;
3942                 }
3943         }
3944         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
3945
3946         return ret;
3947 }
3948
3949 int
3950 rte_eth_remove_tx_callback(uint16_t port_id, uint16_t queue_id,
3951                 const struct rte_eth_rxtx_callback *user_cb)
3952 {
3953 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3954         return -ENOTSUP;
3955 #endif
3956         /* Check input parameters. */
3957         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
3958         if (user_cb == NULL ||
3959                         queue_id >= rte_eth_devices[port_id].data->nb_tx_queues)
3960                 return -EINVAL;
3961
3962         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3963         int ret = -EINVAL;
3964         struct rte_eth_rxtx_callback *cb;
3965         struct rte_eth_rxtx_callback **prev_cb;
3966
3967         rte_spinlock_lock(&rte_eth_tx_cb_lock);
3968         prev_cb = &dev->pre_tx_burst_cbs[queue_id];
3969         for (; *prev_cb != NULL; prev_cb = &cb->next) {
3970                 cb = *prev_cb;
3971                 if (cb == user_cb) {
3972                         /* Remove the user cb from the callback list. */
3973                         *prev_cb = cb->next;
3974                         ret = 0;
3975                         break;
3976                 }
3977         }
3978         rte_spinlock_unlock(&rte_eth_tx_cb_lock);
3979
3980         return ret;
3981 }
3982
3983 int
3984 rte_eth_rx_queue_info_get(uint16_t port_id, uint16_t queue_id,
3985         struct rte_eth_rxq_info *qinfo)
3986 {
3987         struct rte_eth_dev *dev;
3988
3989         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3990
3991         if (qinfo == NULL)
3992                 return -EINVAL;
3993
3994         dev = &rte_eth_devices[port_id];
3995         if (queue_id >= dev->data->nb_rx_queues) {
3996                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
3997                 return -EINVAL;
3998         }
3999
4000         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rxq_info_get, -ENOTSUP);
4001
4002         memset(qinfo, 0, sizeof(*qinfo));
4003         dev->dev_ops->rxq_info_get(dev, queue_id, qinfo);
4004         return 0;
4005 }
4006
4007 int
4008 rte_eth_tx_queue_info_get(uint16_t port_id, uint16_t queue_id,
4009         struct rte_eth_txq_info *qinfo)
4010 {
4011         struct rte_eth_dev *dev;
4012
4013         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4014
4015         if (qinfo == NULL)
4016                 return -EINVAL;
4017
4018         dev = &rte_eth_devices[port_id];
4019         if (queue_id >= dev->data->nb_tx_queues) {
4020                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", queue_id);
4021                 return -EINVAL;
4022         }
4023
4024         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->txq_info_get, -ENOTSUP);
4025
4026         memset(qinfo, 0, sizeof(*qinfo));
4027         dev->dev_ops->txq_info_get(dev, queue_id, qinfo);
4028
4029         return 0;
4030 }
4031
4032 int
4033 rte_eth_dev_set_mc_addr_list(uint16_t port_id,
4034                              struct ether_addr *mc_addr_set,
4035                              uint32_t nb_mc_addr)
4036 {
4037         struct rte_eth_dev *dev;
4038
4039         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4040
4041         dev = &rte_eth_devices[port_id];
4042         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_mc_addr_list, -ENOTSUP);
4043         return eth_err(port_id, dev->dev_ops->set_mc_addr_list(dev,
4044                                                 mc_addr_set, nb_mc_addr));
4045 }
4046
4047 int
4048 rte_eth_timesync_enable(uint16_t port_id)
4049 {
4050         struct rte_eth_dev *dev;
4051
4052         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4053         dev = &rte_eth_devices[port_id];
4054
4055         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_enable, -ENOTSUP);
4056         return eth_err(port_id, (*dev->dev_ops->timesync_enable)(dev));
4057 }
4058
4059 int
4060 rte_eth_timesync_disable(uint16_t port_id)
4061 {
4062         struct rte_eth_dev *dev;
4063
4064         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4065         dev = &rte_eth_devices[port_id];
4066
4067         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_disable, -ENOTSUP);
4068         return eth_err(port_id, (*dev->dev_ops->timesync_disable)(dev));
4069 }
4070
4071 int
4072 rte_eth_timesync_read_rx_timestamp(uint16_t port_id, struct timespec *timestamp,
4073                                    uint32_t flags)
4074 {
4075         struct rte_eth_dev *dev;
4076
4077         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4078         dev = &rte_eth_devices[port_id];
4079
4080         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_rx_timestamp, -ENOTSUP);
4081         return eth_err(port_id, (*dev->dev_ops->timesync_read_rx_timestamp)
4082                                 (dev, timestamp, flags));
4083 }
4084
4085 int
4086 rte_eth_timesync_read_tx_timestamp(uint16_t port_id,
4087                                    struct timespec *timestamp)
4088 {
4089         struct rte_eth_dev *dev;
4090
4091         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4092         dev = &rte_eth_devices[port_id];
4093
4094         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_tx_timestamp, -ENOTSUP);
4095         return eth_err(port_id, (*dev->dev_ops->timesync_read_tx_timestamp)
4096                                 (dev, timestamp));
4097 }
4098
4099 int
4100 rte_eth_timesync_adjust_time(uint16_t port_id, int64_t delta)
4101 {
4102         struct rte_eth_dev *dev;
4103
4104         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4105         dev = &rte_eth_devices[port_id];
4106
4107         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_adjust_time, -ENOTSUP);
4108         return eth_err(port_id, (*dev->dev_ops->timesync_adjust_time)(dev,
4109                                                                       delta));
4110 }
4111
4112 int
4113 rte_eth_timesync_read_time(uint16_t port_id, struct timespec *timestamp)
4114 {
4115         struct rte_eth_dev *dev;
4116
4117         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4118         dev = &rte_eth_devices[port_id];
4119
4120         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_time, -ENOTSUP);
4121         return eth_err(port_id, (*dev->dev_ops->timesync_read_time)(dev,
4122                                                                 timestamp));
4123 }
4124
4125 int
4126 rte_eth_timesync_write_time(uint16_t port_id, const struct timespec *timestamp)
4127 {
4128         struct rte_eth_dev *dev;
4129
4130         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4131         dev = &rte_eth_devices[port_id];
4132
4133         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_write_time, -ENOTSUP);
4134         return eth_err(port_id, (*dev->dev_ops->timesync_write_time)(dev,
4135                                                                 timestamp));
4136 }
4137
4138 int
4139 rte_eth_dev_get_reg_info(uint16_t port_id, struct rte_dev_reg_info *info)
4140 {
4141         struct rte_eth_dev *dev;
4142
4143         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4144
4145         dev = &rte_eth_devices[port_id];
4146         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_reg, -ENOTSUP);
4147         return eth_err(port_id, (*dev->dev_ops->get_reg)(dev, info));
4148 }
4149
4150 int
4151 rte_eth_dev_get_eeprom_length(uint16_t port_id)
4152 {
4153         struct rte_eth_dev *dev;
4154
4155         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4156
4157         dev = &rte_eth_devices[port_id];
4158         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom_length, -ENOTSUP);
4159         return eth_err(port_id, (*dev->dev_ops->get_eeprom_length)(dev));
4160 }
4161
4162 int
4163 rte_eth_dev_get_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info)
4164 {
4165         struct rte_eth_dev *dev;
4166
4167         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4168
4169         dev = &rte_eth_devices[port_id];
4170         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom, -ENOTSUP);
4171         return eth_err(port_id, (*dev->dev_ops->get_eeprom)(dev, info));
4172 }
4173
4174 int
4175 rte_eth_dev_set_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info)
4176 {
4177         struct rte_eth_dev *dev;
4178
4179         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4180
4181         dev = &rte_eth_devices[port_id];
4182         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_eeprom, -ENOTSUP);
4183         return eth_err(port_id, (*dev->dev_ops->set_eeprom)(dev, info));
4184 }
4185
4186 int __rte_experimental
4187 rte_eth_dev_get_module_info(uint16_t port_id,
4188                             struct rte_eth_dev_module_info *modinfo)
4189 {
4190         struct rte_eth_dev *dev;
4191
4192         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4193
4194         dev = &rte_eth_devices[port_id];
4195         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_module_info, -ENOTSUP);
4196         return (*dev->dev_ops->get_module_info)(dev, modinfo);
4197 }
4198
4199 int __rte_experimental
4200 rte_eth_dev_get_module_eeprom(uint16_t port_id,
4201                               struct rte_dev_eeprom_info *info)
4202 {
4203         struct rte_eth_dev *dev;
4204
4205         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4206
4207         dev = &rte_eth_devices[port_id];
4208         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_module_eeprom, -ENOTSUP);
4209         return (*dev->dev_ops->get_module_eeprom)(dev, info);
4210 }
4211
4212 int
4213 rte_eth_dev_get_dcb_info(uint16_t port_id,
4214                              struct rte_eth_dcb_info *dcb_info)
4215 {
4216         struct rte_eth_dev *dev;
4217
4218         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4219
4220         dev = &rte_eth_devices[port_id];
4221         memset(dcb_info, 0, sizeof(struct rte_eth_dcb_info));
4222
4223         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_dcb_info, -ENOTSUP);
4224         return eth_err(port_id, (*dev->dev_ops->get_dcb_info)(dev, dcb_info));
4225 }
4226
4227 int
4228 rte_eth_dev_l2_tunnel_eth_type_conf(uint16_t port_id,
4229                                     struct rte_eth_l2_tunnel_conf *l2_tunnel)
4230 {
4231         struct rte_eth_dev *dev;
4232
4233         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4234         if (l2_tunnel == NULL) {
4235                 RTE_ETHDEV_LOG(ERR, "Invalid l2_tunnel parameter\n");
4236                 return -EINVAL;
4237         }
4238
4239         if (l2_tunnel->l2_tunnel_type >= RTE_TUNNEL_TYPE_MAX) {
4240                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
4241                 return -EINVAL;
4242         }
4243
4244         dev = &rte_eth_devices[port_id];
4245         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->l2_tunnel_eth_type_conf,
4246                                 -ENOTSUP);
4247         return eth_err(port_id, (*dev->dev_ops->l2_tunnel_eth_type_conf)(dev,
4248                                                                 l2_tunnel));
4249 }
4250
4251 int
4252 rte_eth_dev_l2_tunnel_offload_set(uint16_t port_id,
4253                                   struct rte_eth_l2_tunnel_conf *l2_tunnel,
4254                                   uint32_t mask,
4255                                   uint8_t en)
4256 {
4257         struct rte_eth_dev *dev;
4258
4259         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4260
4261         if (l2_tunnel == NULL) {
4262                 RTE_ETHDEV_LOG(ERR, "Invalid l2_tunnel parameter\n");
4263                 return -EINVAL;
4264         }
4265
4266         if (l2_tunnel->l2_tunnel_type >= RTE_TUNNEL_TYPE_MAX) {
4267                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
4268                 return -EINVAL;
4269         }
4270
4271         if (mask == 0) {
4272                 RTE_ETHDEV_LOG(ERR, "Mask should have a value\n");
4273                 return -EINVAL;
4274         }
4275
4276         dev = &rte_eth_devices[port_id];
4277         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->l2_tunnel_offload_set,
4278                                 -ENOTSUP);
4279         return eth_err(port_id, (*dev->dev_ops->l2_tunnel_offload_set)(dev,
4280                                                         l2_tunnel, mask, en));
4281 }
4282
4283 static void
4284 rte_eth_dev_adjust_nb_desc(uint16_t *nb_desc,
4285                            const struct rte_eth_desc_lim *desc_lim)
4286 {
4287         if (desc_lim->nb_align != 0)
4288                 *nb_desc = RTE_ALIGN_CEIL(*nb_desc, desc_lim->nb_align);
4289
4290         if (desc_lim->nb_max != 0)
4291                 *nb_desc = RTE_MIN(*nb_desc, desc_lim->nb_max);
4292
4293         *nb_desc = RTE_MAX(*nb_desc, desc_lim->nb_min);
4294 }
4295
4296 int
4297 rte_eth_dev_adjust_nb_rx_tx_desc(uint16_t port_id,
4298                                  uint16_t *nb_rx_desc,
4299                                  uint16_t *nb_tx_desc)
4300 {
4301         struct rte_eth_dev *dev;
4302         struct rte_eth_dev_info dev_info;
4303
4304         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4305
4306         dev = &rte_eth_devices[port_id];
4307         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
4308
4309         rte_eth_dev_info_get(port_id, &dev_info);
4310
4311         if (nb_rx_desc != NULL)
4312                 rte_eth_dev_adjust_nb_desc(nb_rx_desc, &dev_info.rx_desc_lim);
4313
4314         if (nb_tx_desc != NULL)
4315                 rte_eth_dev_adjust_nb_desc(nb_tx_desc, &dev_info.tx_desc_lim);
4316
4317         return 0;
4318 }
4319
4320 int
4321 rte_eth_dev_pool_ops_supported(uint16_t port_id, const char *pool)
4322 {
4323         struct rte_eth_dev *dev;
4324
4325         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4326
4327         if (pool == NULL)
4328                 return -EINVAL;
4329
4330         dev = &rte_eth_devices[port_id];
4331
4332         if (*dev->dev_ops->pool_ops_supported == NULL)
4333                 return 1; /* all pools are supported */
4334
4335         return (*dev->dev_ops->pool_ops_supported)(dev, pool);
4336 }
4337
4338 /**
4339  * A set of values to describe the possible states of a switch domain.
4340  */
4341 enum rte_eth_switch_domain_state {
4342         RTE_ETH_SWITCH_DOMAIN_UNUSED = 0,
4343         RTE_ETH_SWITCH_DOMAIN_ALLOCATED
4344 };
4345
4346 /**
4347  * Array of switch domains available for allocation. Array is sized to
4348  * RTE_MAX_ETHPORTS elements as there cannot be more active switch domains than
4349  * ethdev ports in a single process.
4350  */
4351 static struct rte_eth_dev_switch {
4352         enum rte_eth_switch_domain_state state;
4353 } rte_eth_switch_domains[RTE_MAX_ETHPORTS];
4354
4355 int __rte_experimental
4356 rte_eth_switch_domain_alloc(uint16_t *domain_id)
4357 {
4358         unsigned int i;
4359
4360         *domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
4361
4362         for (i = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID + 1;
4363                 i < RTE_MAX_ETHPORTS; i++) {
4364                 if (rte_eth_switch_domains[i].state ==
4365                         RTE_ETH_SWITCH_DOMAIN_UNUSED) {
4366                         rte_eth_switch_domains[i].state =
4367                                 RTE_ETH_SWITCH_DOMAIN_ALLOCATED;
4368                         *domain_id = i;
4369                         return 0;
4370                 }
4371         }
4372
4373         return -ENOSPC;
4374 }
4375
4376 int __rte_experimental
4377 rte_eth_switch_domain_free(uint16_t domain_id)
4378 {
4379         if (domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID ||
4380                 domain_id >= RTE_MAX_ETHPORTS)
4381                 return -EINVAL;
4382
4383         if (rte_eth_switch_domains[domain_id].state !=
4384                 RTE_ETH_SWITCH_DOMAIN_ALLOCATED)
4385                 return -EINVAL;
4386
4387         rte_eth_switch_domains[domain_id].state = RTE_ETH_SWITCH_DOMAIN_UNUSED;
4388
4389         return 0;
4390 }
4391
4392 static int
4393 rte_eth_devargs_tokenise(struct rte_kvargs *arglist, const char *str_in)
4394 {
4395         int state;
4396         struct rte_kvargs_pair *pair;
4397         char *letter;
4398
4399         arglist->str = strdup(str_in);
4400         if (arglist->str == NULL)
4401                 return -ENOMEM;
4402
4403         letter = arglist->str;
4404         state = 0;
4405         arglist->count = 0;
4406         pair = &arglist->pairs[0];
4407         while (1) {
4408                 switch (state) {
4409                 case 0: /* Initial */
4410                         if (*letter == '=')
4411                                 return -EINVAL;
4412                         else if (*letter == '\0')
4413                                 return 0;
4414
4415                         state = 1;
4416                         pair->key = letter;
4417                         /* fall-thru */
4418
4419                 case 1: /* Parsing key */
4420                         if (*letter == '=') {
4421                                 *letter = '\0';
4422                                 pair->value = letter + 1;
4423                                 state = 2;
4424                         } else if (*letter == ',' || *letter == '\0')
4425                                 return -EINVAL;
4426                         break;
4427
4428
4429                 case 2: /* Parsing value */
4430                         if (*letter == '[')
4431                                 state = 3;
4432                         else if (*letter == ',') {
4433                                 *letter = '\0';
4434                                 arglist->count++;
4435                                 pair = &arglist->pairs[arglist->count];
4436                                 state = 0;
4437                         } else if (*letter == '\0') {
4438                                 letter--;
4439                                 arglist->count++;
4440                                 pair = &arglist->pairs[arglist->count];
4441                                 state = 0;
4442                         }
4443                         break;
4444
4445                 case 3: /* Parsing list */
4446                         if (*letter == ']')
4447                                 state = 2;
4448                         else if (*letter == '\0')
4449                                 return -EINVAL;
4450                         break;
4451                 }
4452                 letter++;
4453         }
4454 }
4455
4456 int __rte_experimental
4457 rte_eth_devargs_parse(const char *dargs, struct rte_eth_devargs *eth_da)
4458 {
4459         struct rte_kvargs args;
4460         struct rte_kvargs_pair *pair;
4461         unsigned int i;
4462         int result = 0;
4463
4464         memset(eth_da, 0, sizeof(*eth_da));
4465
4466         result = rte_eth_devargs_tokenise(&args, dargs);
4467         if (result < 0)
4468                 goto parse_cleanup;
4469
4470         for (i = 0; i < args.count; i++) {
4471                 pair = &args.pairs[i];
4472                 if (strcmp("representor", pair->key) == 0) {
4473                         result = rte_eth_devargs_parse_list(pair->value,
4474                                 rte_eth_devargs_parse_representor_ports,
4475                                 eth_da);
4476                         if (result < 0)
4477                                 goto parse_cleanup;
4478                 }
4479         }
4480
4481 parse_cleanup:
4482         if (args.str)
4483                 free(args.str);
4484
4485         return result;
4486 }
4487
4488 RTE_INIT(ethdev_init_log)
4489 {
4490         rte_eth_dev_logtype = rte_log_register("lib.ethdev");
4491         if (rte_eth_dev_logtype >= 0)
4492                 rte_log_set_level(rte_eth_dev_logtype, RTE_LOG_INFO);
4493 }