lib/librte_ether/rte_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/types.h>
  35 #include <sys/queue.h>
  36 #include <ctype.h>
  37 #include <stdio.h>
  38 #include <stdlib.h>
  39 #include <string.h>
  40 #include <stdarg.h>
  41 #include <errno.h>
  42 #include <stdint.h>
  43 #include <inttypes.h>
  44 #include <netinet/in.h>
  45
  46 #include <rte_byteorder.h>
  47 #include <rte_log.h>
  48 #include <rte_debug.h>
  49 #include <rte_interrupts.h>
  50 #include <rte_pci.h>
  51 #include <rte_memory.h>
  52 #include <rte_memcpy.h>
  53 #include <rte_memzone.h>
  54 #include <rte_launch.h>
  55 #include <rte_eal.h>
  56 #include <rte_per_lcore.h>
  57 #include <rte_lcore.h>
  58 #include <rte_atomic.h>
  59 #include <rte_branch_prediction.h>
  60 #include <rte_common.h>
  61 #include <rte_ring.h>
  62 #include <rte_mempool.h>
  63 #include <rte_malloc.h>
  64 #include <rte_mbuf.h>
  65 #include <rte_errno.h>
  66 #include <rte_spinlock.h>
  67 #include <rte_string_fns.h>
  68
  69 #include "rte_ether.h"
  70 #include "rte_ethdev.h"
  71
  72 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
  73 #define PMD_DEBUG_TRACE(fmt, args...) do {                        \
  74                 RTE_LOG(ERR, PMD, "%s: " fmt, __func__, ## args); \
  75         } while (0)
  76 #else
  77 #define PMD_DEBUG_TRACE(fmt, args...)
  78 #endif
  79
  80 /* Macros for checking for restricting functions to primary instance only */
  81 #define PROC_PRIMARY_OR_ERR_RET(retval) do { \
  82         if (rte_eal_process_type() != RTE_PROC_PRIMARY) { \
  83                 PMD_DEBUG_TRACE("Cannot run in secondary processes\n"); \
  84                 return (retval); \
  85         } \
  86 } while (0)
  87
  88 #define PROC_PRIMARY_OR_RET() do { \
  89         if (rte_eal_process_type() != RTE_PROC_PRIMARY) { \
  90                 PMD_DEBUG_TRACE("Cannot run in secondary processes\n"); \
  91                 return; \
  92         } \
  93 } while (0)
  94
  95 /* Macros to check for invalid function pointers in dev_ops structure */
  96 #define FUNC_PTR_OR_ERR_RET(func, retval) do { \
  97         if ((func) == NULL) { \
  98                 PMD_DEBUG_TRACE("Function not supported\n"); \
  99                 return (retval); \
 100         } \
 101 } while (0)
 102
 103 #define FUNC_PTR_OR_RET(func) do { \
 104         if ((func) == NULL) { \
 105                 PMD_DEBUG_TRACE("Function not supported\n"); \
 106                 return; \
 107         } \
 108 } while (0)
 109
 110 /* Macros to check for valid port */
 111 #define VALID_PORTID_OR_ERR_RET(port_id, retval) do {           \
 112         if (!rte_eth_dev_is_valid_port(port_id)) {              \
 113                 PMD_DEBUG_TRACE("Invalid port_id=%d\n", port_id); \
 114                 return retval;                                  \
 115         }                                                       \
 116 } while (0)
 117
 118 #define VALID_PORTID_OR_RET(port_id) do {                       \
 119         if (!rte_eth_dev_is_valid_port(port_id)) {              \
 120                 PMD_DEBUG_TRACE("Invalid port_id=%d\n", port_id); \
 121                 return;                                         \
 122         }                                                       \
 123 } while (0)
 124
 125 static const char *MZ_RTE_ETH_DEV_DATA = "rte_eth_dev_data";
 126 struct rte_eth_dev rte_eth_devices[RTE_MAX_ETHPORTS];
 127 static struct rte_eth_dev_data *rte_eth_dev_data;
 128 static uint8_t nb_ports;
 129
 130 /* spinlock for eth device callbacks */
 131 static rte_spinlock_t rte_eth_dev_cb_lock = RTE_SPINLOCK_INITIALIZER;
 132
 133 /* store statistics names and its offset in stats structure  */
 134 struct rte_eth_xstats_name_off {
 135         char name[RTE_ETH_XSTATS_NAME_SIZE];
 136         unsigned offset;
 137 };
 138
 139 static const struct rte_eth_xstats_name_off rte_stats_strings[] = {
 140         {"rx_packets", offsetof(struct rte_eth_stats, ipackets)},
 141         {"tx_packets", offsetof(struct rte_eth_stats, opackets)},
 142         {"rx_bytes", offsetof(struct rte_eth_stats, ibytes)},
 143         {"tx_bytes", offsetof(struct rte_eth_stats, obytes)},
 144         {"tx_errors", offsetof(struct rte_eth_stats, oerrors)},
 145         {"rx_errors", offsetof(struct rte_eth_stats, ierrors)},
 146         {"alloc_rx_buff_failed", offsetof(struct rte_eth_stats, rx_nombuf)},
 147 };
 148 #define RTE_NB_STATS (sizeof(rte_stats_strings) / sizeof(rte_stats_strings[0]))
 149
 150 static const struct rte_eth_xstats_name_off rte_rxq_stats_strings[] = {
 151         {"rx_packets", offsetof(struct rte_eth_stats, q_ipackets)},
 152         {"rx_bytes", offsetof(struct rte_eth_stats, q_ibytes)},
 153 };
 154 #define RTE_NB_RXQ_STATS (sizeof(rte_rxq_stats_strings) /       \
 155                 sizeof(rte_rxq_stats_strings[0]))
 156
 157 static const struct rte_eth_xstats_name_off rte_txq_stats_strings[] = {
 158         {"tx_packets", offsetof(struct rte_eth_stats, q_opackets)},
 159         {"tx_bytes", offsetof(struct rte_eth_stats, q_obytes)},
 160         {"tx_errors", offsetof(struct rte_eth_stats, q_errors)},
 161 };
 162 #define RTE_NB_TXQ_STATS (sizeof(rte_txq_stats_strings) /       \
 163                 sizeof(rte_txq_stats_strings[0]))
 164
 165
 166 /**
 167  * The user application callback description.
 168  *
 169  * It contains callback address to be registered by user application,
 170  * the pointer to the parameters for callback, and the event type.
 171  */
 172 struct rte_eth_dev_callback {
 173         TAILQ_ENTRY(rte_eth_dev_callback) next; /**< Callbacks list */
 174         rte_eth_dev_cb_fn cb_fn;                /**< Callback address */
 175         void *cb_arg;                           /**< Parameter for callback */
 176         enum rte_eth_event_type event;          /**< Interrupt event type */
 177         uint32_t active;                        /**< Callback is executing */
 178 };
 179
 180 enum {
 181         STAT_QMAP_TX = 0,
 182         STAT_QMAP_RX
 183 };
 184
 185 enum {
 186         DEV_DETACHED = 0,
 187         DEV_ATTACHED
 188 };
 189
 190 static void
 191 rte_eth_dev_data_alloc(void)
 192 {
 193         const unsigned flags = 0;
 194         const struct rte_memzone *mz;
 195
 196         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 197                 mz = rte_memzone_reserve(MZ_RTE_ETH_DEV_DATA,
 198                                 RTE_MAX_ETHPORTS * sizeof(*rte_eth_dev_data),
 199                                 rte_socket_id(), flags);
 200         } else
 201                 mz = rte_memzone_lookup(MZ_RTE_ETH_DEV_DATA);
 202         if (mz == NULL)
 203                 rte_panic("Cannot allocate memzone for ethernet port data\n");
 204
 205         rte_eth_dev_data = mz->addr;
 206         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 207                 memset(rte_eth_dev_data, 0,
 208                                 RTE_MAX_ETHPORTS * sizeof(*rte_eth_dev_data));
 209 }
 210
 211 struct rte_eth_dev *
 212 rte_eth_dev_allocated(const char *name)
 213 {
 214         unsigned i;
 215
 216         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 217                 if ((rte_eth_devices[i].attached == DEV_ATTACHED) &&
 218                     strcmp(rte_eth_devices[i].data->name, name) == 0)
 219                         return &rte_eth_devices[i];
 220         }
 221         return NULL;
 222 }
 223
 224 static uint8_t
 225 rte_eth_dev_find_free_port(void)
 226 {
 227         unsigned i;
 228
 229         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 230                 if (rte_eth_devices[i].attached == DEV_DETACHED)
 231                         return i;
 232         }
 233         return RTE_MAX_ETHPORTS;
 234 }
 235
 236 struct rte_eth_dev *
 237 rte_eth_dev_allocate(const char *name, enum rte_eth_dev_type type)
 238 {
 239         uint8_t port_id;
 240         struct rte_eth_dev *eth_dev;
 241
 242         port_id = rte_eth_dev_find_free_port();
 243         if (port_id == RTE_MAX_ETHPORTS) {
 244                 PMD_DEBUG_TRACE("Reached maximum number of Ethernet ports\n");
 245                 return NULL;
 246         }
 247
 248         if (rte_eth_dev_data == NULL)
 249                 rte_eth_dev_data_alloc();
 250
 251         if (rte_eth_dev_allocated(name) != NULL) {
 252                 PMD_DEBUG_TRACE("Ethernet Device with name %s already allocated!\n",
 253                                 name);
 254                 return NULL;
 255         }
 256
 257         eth_dev = &rte_eth_devices[port_id];
 258         eth_dev->data = &rte_eth_dev_data[port_id];
 259         snprintf(eth_dev->data->name, sizeof(eth_dev->data->name), "%s", name);
 260         eth_dev->data->port_id = port_id;
 261         eth_dev->attached = DEV_ATTACHED;
 262         eth_dev->dev_type = type;
 263         nb_ports++;
 264         return eth_dev;
 265 }
 266
 267 static int
 268 rte_eth_dev_create_unique_device_name(char *name, size_t size,
 269                 struct rte_pci_device *pci_dev)
 270 {
 271         int ret;
 272
 273         if ((name == NULL) || (pci_dev == NULL))
 274                 return -EINVAL;
 275
 276         ret = snprintf(name, size, "%d:%d.%d",
 277                         pci_dev->addr.bus, pci_dev->addr.devid,
 278                         pci_dev->addr.function);
 279         if (ret < 0)
 280                 return ret;
 281         return 0;
 282 }
 283
 284 int
 285 rte_eth_dev_release_port(struct rte_eth_dev *eth_dev)
 286 {
 287         if (eth_dev == NULL)
 288                 return -EINVAL;
 289
 290         eth_dev->attached = DEV_DETACHED;
 291         nb_ports--;
 292         return 0;
 293 }
 294
 295 static int
 296 rte_eth_dev_init(struct rte_pci_driver *pci_drv,
 297                  struct rte_pci_device *pci_dev)
 298 {
 299         struct eth_driver    *eth_drv;
 300         struct rte_eth_dev *eth_dev;
 301         char ethdev_name[RTE_ETH_NAME_MAX_LEN];
 302
 303         int diag;
 304
 305         eth_drv = (struct eth_driver *)pci_drv;
 306
 307         /* Create unique Ethernet device name using PCI address */
 308         rte_eth_dev_create_unique_device_name(ethdev_name,
 309                         sizeof(ethdev_name), pci_dev);
 310
 311         eth_dev = rte_eth_dev_allocate(ethdev_name, RTE_ETH_DEV_PCI);
 312         if (eth_dev == NULL)
 313                 return -ENOMEM;
 314
 315         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 316                 eth_dev->data->dev_private = rte_zmalloc("ethdev private structure",
 317                                   eth_drv->dev_private_size,
 318                                   RTE_CACHE_LINE_SIZE);
 319                 if (eth_dev->data->dev_private == NULL)
 320                         rte_panic("Cannot allocate memzone for private port data\n");
 321         }
 322         eth_dev->pci_dev = pci_dev;
 323         eth_dev->driver = eth_drv;
 324         eth_dev->data->rx_mbuf_alloc_failed = 0;
 325
 326         /* init user callbacks */
 327         TAILQ_INIT(&(eth_dev->link_intr_cbs));
 328
 329         /*
 330          * Set the default MTU.
 331          */
 332         eth_dev->data->mtu = ETHER_MTU;
 333
 334         /* Invoke PMD device initialization function */
 335         diag = (*eth_drv->eth_dev_init)(eth_dev);
 336         if (diag == 0)
 337                 return 0;
 338
 339         PMD_DEBUG_TRACE("driver %s: eth_dev_init(vendor_id=0x%u device_id=0x%x) failed\n",
 340                         pci_drv->name,
 341                         (unsigned) pci_dev->id.vendor_id,
 342                         (unsigned) pci_dev->id.device_id);
 343         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 344                 rte_free(eth_dev->data->dev_private);
 345         rte_eth_dev_release_port(eth_dev);
 346         return diag;
 347 }
 348
 349 static int
 350 rte_eth_dev_uninit(struct rte_pci_device *pci_dev)
 351 {
 352         const struct eth_driver *eth_drv;
 353         struct rte_eth_dev *eth_dev;
 354         char ethdev_name[RTE_ETH_NAME_MAX_LEN];
 355         int ret;
 356
 357         if (pci_dev == NULL)
 358                 return -EINVAL;
 359
 360         /* Create unique Ethernet device name using PCI address */
 361         rte_eth_dev_create_unique_device_name(ethdev_name,
 362                         sizeof(ethdev_name), pci_dev);
 363
 364         eth_dev = rte_eth_dev_allocated(ethdev_name);
 365         if (eth_dev == NULL)
 366                 return -ENODEV;
 367
 368         eth_drv = (const struct eth_driver *)pci_dev->driver;
 369
 370         /* Invoke PMD device uninit function */
 371         if (*eth_drv->eth_dev_uninit) {
 372                 ret = (*eth_drv->eth_dev_uninit)(eth_dev);
 373                 if (ret)
 374                         return ret;
 375         }
 376
 377         /* free ether device */
 378         rte_eth_dev_release_port(eth_dev);
 379
 380         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 381                 rte_free(eth_dev->data->dev_private);
 382
 383         eth_dev->pci_dev = NULL;
 384         eth_dev->driver = NULL;
 385         eth_dev->data = NULL;
 386
 387         return 0;
 388 }
 389
 390 /**
 391  * Register an Ethernet [Poll Mode] driver.
 392  *
 393  * Function invoked by the initialization function of an Ethernet driver
 394  * to simultaneously register itself as a PCI driver and as an Ethernet
 395  * Poll Mode Driver.
 396  * Invokes the rte_eal_pci_register() function to register the *pci_drv*
 397  * structure embedded in the *eth_drv* structure, after having stored the
 398  * address of the rte_eth_dev_init() function in the *devinit* field of
 399  * the *pci_drv* structure.
 400  * During the PCI probing phase, the rte_eth_dev_init() function is
 401  * invoked for each PCI [Ethernet device] matching the embedded PCI
 402  * identifiers provided by the driver.
 403  */
 404 void
 405 rte_eth_driver_register(struct eth_driver *eth_drv)
 406 {
 407         eth_drv->pci_drv.devinit = rte_eth_dev_init;
 408         eth_drv->pci_drv.devuninit = rte_eth_dev_uninit;
 409         rte_eal_pci_register(&eth_drv->pci_drv);
 410 }
 411
 412 int
 413 rte_eth_dev_is_valid_port(uint8_t port_id)
 414 {
 415         if (port_id >= RTE_MAX_ETHPORTS ||
 416             rte_eth_devices[port_id].attached != DEV_ATTACHED)
 417                 return 0;
 418         else
 419                 return 1;
 420 }
 421
 422 int
 423 rte_eth_dev_socket_id(uint8_t port_id)
 424 {
 425         if (!rte_eth_dev_is_valid_port(port_id))
 426                 return -1;
 427         return rte_eth_devices[port_id].pci_dev->numa_node;
 428 }
 429
 430 uint8_t
 431 rte_eth_dev_count(void)
 432 {
 433         return nb_ports;
 434 }
 435
 436 static enum rte_eth_dev_type
 437 rte_eth_dev_get_device_type(uint8_t port_id)
 438 {
 439         if (!rte_eth_dev_is_valid_port(port_id))
 440                 return RTE_ETH_DEV_UNKNOWN;
 441         return rte_eth_devices[port_id].dev_type;
 442 }
 443
 444 static int
 445 rte_eth_dev_save(struct rte_eth_dev *devs, size_t size)
 446 {
 447         if ((devs == NULL) ||
 448             (size != sizeof(struct rte_eth_dev) * RTE_MAX_ETHPORTS))
 449                 return -EINVAL;
 450
 451         /* save current rte_eth_devices */
 452         memcpy(devs, rte_eth_devices, size);
 453         return 0;
 454 }
 455
 456 static int
 457 rte_eth_dev_get_changed_port(struct rte_eth_dev *devs, uint8_t *port_id)
 458 {
 459         if ((devs == NULL) || (port_id == NULL))
 460                 return -EINVAL;
 461
 462         /* check which port was attached or detached */
 463         for (*port_id = 0; *port_id < RTE_MAX_ETHPORTS; (*port_id)++, devs++) {
 464                 if (rte_eth_devices[*port_id].attached ^ devs->attached)
 465                         return 0;
 466         }
 467         return -ENODEV;
 468 }
 469
 470 static int
 471 rte_eth_dev_get_addr_by_port(uint8_t port_id, struct rte_pci_addr *addr)
 472 {
 473         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 474
 475         if (addr == NULL) {
 476                 PMD_DEBUG_TRACE("Null pointer is specified\n");
 477                 return -EINVAL;
 478         }
 479
 480         *addr = rte_eth_devices[port_id].pci_dev->addr;
 481         return 0;
 482 }
 483
 484 static int
 485 rte_eth_dev_get_name_by_port(uint8_t port_id, char *name)
 486 {
 487         char *tmp;
 488
 489         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 490
 491         if (name == NULL) {
 492                 PMD_DEBUG_TRACE("Null pointer is specified\n");
 493                 return -EINVAL;
 494         }
 495
 496         /* shouldn't check 'rte_eth_devices[i].data',
 497          * because it might be overwritten by VDEV PMD */
 498         tmp = rte_eth_dev_data[port_id].name;
 499         strcpy(name, tmp);
 500         return 0;
 501 }
 502
 503 static int
 504 rte_eth_dev_is_detachable(uint8_t port_id)
 505 {
 506         uint32_t drv_flags;
 507
 508         if (!rte_eth_dev_is_valid_port(port_id)) {
 509                 PMD_DEBUG_TRACE("Invalid port_id=%d\n", port_id);
 510                 return -EINVAL;
 511         }
 512
 513         if (rte_eth_devices[port_id].dev_type == RTE_ETH_DEV_PCI) {
 514                 switch (rte_eth_devices[port_id].pci_dev->kdrv) {
 515                 case RTE_KDRV_IGB_UIO:
 516                 case RTE_KDRV_UIO_GENERIC:
 517                 case RTE_KDRV_NIC_UIO:
 518                         break;
 519                 case RTE_KDRV_VFIO:
 520                 default:
 521                         return -ENOTSUP;
 522                 }
 523         }
 524
 525         drv_flags = rte_eth_devices[port_id].driver->pci_drv.drv_flags;
 526         return !(drv_flags & RTE_PCI_DRV_DETACHABLE);
 527 }
 528
 529 /* attach the new physical device, then store port_id of the device */
 530 static int
 531 rte_eth_dev_attach_pdev(struct rte_pci_addr *addr, uint8_t *port_id)
 532 {
 533         uint8_t new_port_id;
 534         struct rte_eth_dev devs[RTE_MAX_ETHPORTS];
 535
 536         if ((addr == NULL) || (port_id == NULL))
 537                 goto err;
 538
 539         /* save current port status */
 540         if (rte_eth_dev_save(devs, sizeof(devs)))
 541                 goto err;
 542         /* re-construct pci_device_list */
 543         if (rte_eal_pci_scan())
 544                 goto err;
 545         /* invoke probe func of the driver can handle the new device.
 546          * TODO:
 547          * rte_eal_pci_probe_one() should return port_id.
 548          * And rte_eth_dev_save() and rte_eth_dev_get_changed_port()
 549          * should be removed. */
 550         if (rte_eal_pci_probe_one(addr))
 551                 goto err;
 552         /* get port_id enabled by above procedures */
 553         if (rte_eth_dev_get_changed_port(devs, &new_port_id))
 554                 goto err;
 555
 556         *port_id = new_port_id;
 557         return 0;
 558 err:
 559         RTE_LOG(ERR, EAL, "Driver, cannot attach the device\n");
 560         return -1;
 561 }
 562
 563 /* detach the new physical device, then store pci_addr of the device */
 564 static int
 565 rte_eth_dev_detach_pdev(uint8_t port_id, struct rte_pci_addr *addr)
 566 {
 567         struct rte_pci_addr freed_addr;
 568         struct rte_pci_addr vp;
 569
 570         if (addr == NULL)
 571                 goto err;
 572
 573         /* check whether the driver supports detach feature, or not */
 574         if (rte_eth_dev_is_detachable(port_id))
 575                 goto err;
 576
 577         /* get pci address by port id */
 578         if (rte_eth_dev_get_addr_by_port(port_id, &freed_addr))
 579                 goto err;
 580
 581         /* Zeroed pci addr means the port comes from virtual device */
 582         vp.domain = vp.bus = vp.devid = vp.function = 0;
 583         if (rte_eal_compare_pci_addr(&vp, &freed_addr) == 0)
 584                 goto err;
 585
 586         /* invoke devuninit func of the pci driver,
 587          * also remove the device from pci_device_list */
 588         if (rte_eal_pci_detach(&freed_addr))
 589                 goto err;
 590
 591         *addr = freed_addr;
 592         return 0;
 593 err:
 594         RTE_LOG(ERR, EAL, "Driver, cannot detach the device\n");
 595         return -1;
 596 }
 597
 598 /* attach the new virtual device, then store port_id of the device */
 599 static int
 600 rte_eth_dev_attach_vdev(const char *vdevargs, uint8_t *port_id)
 601 {
 602         char *name = NULL, *args = NULL;
 603         uint8_t new_port_id;
 604         struct rte_eth_dev devs[RTE_MAX_ETHPORTS];
 605         int ret = -1;
 606
 607         if ((vdevargs == NULL) || (port_id == NULL))
 608                 goto end;
 609
 610         /* parse vdevargs, then retrieve device name and args */
 611         if (rte_eal_parse_devargs_str(vdevargs, &name, &args))
 612                 goto end;
 613
 614         /* save current port status */
 615         if (rte_eth_dev_save(devs, sizeof(devs)))
 616                 goto end;
 617         /* walk around dev_driver_list to find the driver of the device,
 618          * then invoke probe function o the driver.
 619          * TODO:
 620          * rte_eal_vdev_init() should return port_id,
 621          * And rte_eth_dev_save() and rte_eth_dev_get_changed_port()
 622          * should be removed. */
 623         if (rte_eal_vdev_init(name, args))
 624                 goto end;
 625         /* get port_id enabled by above procedures */
 626         if (rte_eth_dev_get_changed_port(devs, &new_port_id))
 627                 goto end;
 628         ret = 0;
 629         *port_id = new_port_id;
 630 end:
 631         if (name)
 632                 free(name);
 633         if (args)
 634                 free(args);
 635
 636         if (ret < 0)
 637                 RTE_LOG(ERR, EAL, "Driver, cannot attach the device\n");
 638         return ret;
 639 }
 640
 641 /* detach the new virtual device, then store the name of the device */
 642 static int
 643 rte_eth_dev_detach_vdev(uint8_t port_id, char *vdevname)
 644 {
 645         char name[RTE_ETH_NAME_MAX_LEN];
 646
 647         if (vdevname == NULL)
 648                 goto err;
 649
 650         /* check whether the driver supports detach feature, or not */
 651         if (rte_eth_dev_is_detachable(port_id))
 652                 goto err;
 653
 654         /* get device name by port id */
 655         if (rte_eth_dev_get_name_by_port(port_id, name))
 656                 goto err;
 657         /* walk around dev_driver_list to find the driver of the device,
 658          * then invoke uninit function of the driver */
 659         if (rte_eal_vdev_uninit(name))
 660                 goto err;
 661
 662         strncpy(vdevname, name, sizeof(name));
 663         return 0;
 664 err:
 665         RTE_LOG(ERR, EAL, "Driver, cannot detach the device\n");
 666         return -1;
 667 }
 668
 669 /* attach the new device, then store port_id of the device */
 670 int
 671 rte_eth_dev_attach(const char *devargs, uint8_t *port_id)
 672 {
 673         struct rte_pci_addr addr;
 674
 675         if ((devargs == NULL) || (port_id == NULL))
 676                 return -EINVAL;
 677
 678         if (eal_parse_pci_DomBDF(devargs, &addr) == 0)
 679                 return rte_eth_dev_attach_pdev(&addr, port_id);
 680         else
 681                 return rte_eth_dev_attach_vdev(devargs, port_id);
 682 }
 683
 684 /* detach the device, then store the name of the device */
 685 int
 686 rte_eth_dev_detach(uint8_t port_id, char *name)
 687 {
 688         struct rte_pci_addr addr;
 689         int ret;
 690
 691         if (name == NULL)
 692                 return -EINVAL;
 693
 694         if (rte_eth_dev_get_device_type(port_id) == RTE_ETH_DEV_PCI) {
 695                 ret = rte_eth_dev_get_addr_by_port(port_id, &addr);
 696                 if (ret < 0)
 697                         return ret;
 698
 699                 ret = rte_eth_dev_detach_pdev(port_id, &addr);
 700                 if (ret == 0)
 701                         snprintf(name, RTE_ETH_NAME_MAX_LEN,
 702                                 "%04x:%02x:%02x.%d",
 703                                 addr.domain, addr.bus,
 704                                 addr.devid, addr.function);
 705
 706                 return ret;
 707         } else
 708                 return rte_eth_dev_detach_vdev(port_id, name);
 709 }
 710
 711 static int
 712 rte_eth_dev_rx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
 713 {
 714         uint16_t old_nb_queues = dev->data->nb_rx_queues;
 715         void **rxq;
 716         unsigned i;
 717
 718         if (dev->data->rx_queues == NULL) { /* first time configuration */
 719                 dev->data->rx_queues = rte_zmalloc("ethdev->rx_queues",
 720                                 sizeof(dev->data->rx_queues[0]) * nb_queues,
 721                                 RTE_CACHE_LINE_SIZE);
 722                 if (dev->data->rx_queues == NULL) {
 723                         dev->data->nb_rx_queues = 0;
 724                         return -(ENOMEM);
 725                 }
 726         } else { /* re-configure */
 727                 FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release, -ENOTSUP);
 728
 729                 rxq = dev->data->rx_queues;
 730
 731                 for (i = nb_queues; i < old_nb_queues; i++)
 732                         (*dev->dev_ops->rx_queue_release)(rxq[i]);
 733                 rxq = rte_realloc(rxq, sizeof(rxq[0]) * nb_queues,
 734                                 RTE_CACHE_LINE_SIZE);
 735                 if (rxq == NULL)
 736                         return -(ENOMEM);
 737                 if (nb_queues > old_nb_queues) {
 738                         uint16_t new_qs = nb_queues - old_nb_queues;
 739
 740                         memset(rxq + old_nb_queues, 0,
 741                                 sizeof(rxq[0]) * new_qs);
 742                 }
 743
 744                 dev->data->rx_queues = rxq;
 745
 746         }
 747         dev->data->nb_rx_queues = nb_queues;
 748         return 0;
 749 }
 750
 751 int
 752 rte_eth_dev_rx_queue_start(uint8_t port_id, uint16_t rx_queue_id)
 753 {
 754         struct rte_eth_dev *dev;
 755
 756         /* This function is only safe when called from the primary process
 757          * in a multi-process setup*/
 758         PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
 759
 760         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 761
 762         dev = &rte_eth_devices[port_id];
 763         if (rx_queue_id >= dev->data->nb_rx_queues) {
 764                 PMD_DEBUG_TRACE("Invalid RX queue_id=%d\n", rx_queue_id);
 765                 return -EINVAL;
 766         }
 767
 768         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_start, -ENOTSUP);
 769
 770         return dev->dev_ops->rx_queue_start(dev, rx_queue_id);
 771
 772 }
 773
 774 int
 775 rte_eth_dev_rx_queue_stop(uint8_t port_id, uint16_t rx_queue_id)
 776 {
 777         struct rte_eth_dev *dev;
 778
 779         /* This function is only safe when called from the primary process
 780          * in a multi-process setup*/
 781         PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
 782
 783         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 784
 785         dev = &rte_eth_devices[port_id];
 786         if (rx_queue_id >= dev->data->nb_rx_queues) {
 787                 PMD_DEBUG_TRACE("Invalid RX queue_id=%d\n", rx_queue_id);
 788                 return -EINVAL;
 789         }
 790
 791         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_stop, -ENOTSUP);
 792
 793         return dev->dev_ops->rx_queue_stop(dev, rx_queue_id);
 794
 795 }
 796
 797 int
 798 rte_eth_dev_tx_queue_start(uint8_t port_id, uint16_t tx_queue_id)
 799 {
 800         struct rte_eth_dev *dev;
 801
 802         /* This function is only safe when called from the primary process
 803          * in a multi-process setup*/
 804         PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
 805
 806         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 807
 808         dev = &rte_eth_devices[port_id];
 809         if (tx_queue_id >= dev->data->nb_tx_queues) {
 810                 PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", tx_queue_id);
 811                 return -EINVAL;
 812         }
 813
 814         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_start, -ENOTSUP);
 815
 816         return dev->dev_ops->tx_queue_start(dev, tx_queue_id);
 817
 818 }
 819
 820 int
 821 rte_eth_dev_tx_queue_stop(uint8_t port_id, uint16_t tx_queue_id)
 822 {
 823         struct rte_eth_dev *dev;
 824
 825         /* This function is only safe when called from the primary process
 826          * in a multi-process setup*/
 827         PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
 828
 829         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 830
 831         dev = &rte_eth_devices[port_id];
 832         if (tx_queue_id >= dev->data->nb_tx_queues) {
 833                 PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", tx_queue_id);
 834                 return -EINVAL;
 835         }
 836
 837         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_stop, -ENOTSUP);
 838
 839         return dev->dev_ops->tx_queue_stop(dev, tx_queue_id);
 840
 841 }
 842
 843 static int
 844 rte_eth_dev_tx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
 845 {
 846         uint16_t old_nb_queues = dev->data->nb_tx_queues;
 847         void **txq;
 848         unsigned i;
 849
 850         if (dev->data->tx_queues == NULL) { /* first time configuration */
 851                 dev->data->tx_queues = rte_zmalloc("ethdev->tx_queues",
 852                                                    sizeof(dev->data->tx_queues[0]) * nb_queues,
 853                                                    RTE_CACHE_LINE_SIZE);
 854                 if (dev->data->tx_queues == NULL) {
 855                         dev->data->nb_tx_queues = 0;
 856                         return -(ENOMEM);
 857                 }
 858         } else { /* re-configure */
 859                 FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release, -ENOTSUP);
 860
 861                 txq = dev->data->tx_queues;
 862
 863                 for (i = nb_queues; i < old_nb_queues; i++)
 864                         (*dev->dev_ops->tx_queue_release)(txq[i]);
 865                 txq = rte_realloc(txq, sizeof(txq[0]) * nb_queues,
 866                                   RTE_CACHE_LINE_SIZE);
 867                 if (txq == NULL)
 868                         return -ENOMEM;
 869                 if (nb_queues > old_nb_queues) {
 870                         uint16_t new_qs = nb_queues - old_nb_queues;
 871
 872                         memset(txq + old_nb_queues, 0,
 873                                sizeof(txq[0]) * new_qs);
 874                 }
 875
 876                 dev->data->tx_queues = txq;
 877
 878         }
 879         dev->data->nb_tx_queues = nb_queues;
 880         return 0;
 881 }
 882
 883 int
 884 rte_eth_dev_configure(uint8_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
 885                       const struct rte_eth_conf *dev_conf)
 886 {
 887         struct rte_eth_dev *dev;
 888         struct rte_eth_dev_info dev_info;
 889         int diag;
 890
 891         /* This function is only safe when called from the primary process
 892          * in a multi-process setup*/
 893         PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
 894
 895         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 896
 897         if (nb_rx_q > RTE_MAX_QUEUES_PER_PORT) {
 898                 PMD_DEBUG_TRACE(
 899                         "Number of RX queues requested (%u) is greater than max supported(%d)\n",
 900                         nb_rx_q, RTE_MAX_QUEUES_PER_PORT);
 901                 return -EINVAL;
 902         }
 903
 904         if (nb_tx_q > RTE_MAX_QUEUES_PER_PORT) {
 905                 PMD_DEBUG_TRACE(
 906                         "Number of TX queues requested (%u) is greater than max supported(%d)\n",
 907                         nb_tx_q, RTE_MAX_QUEUES_PER_PORT);
 908                 return -EINVAL;
 909         }
 910
 911         dev = &rte_eth_devices[port_id];
 912
 913         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
 914         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
 915
 916         if (dev->data->dev_started) {
 917                 PMD_DEBUG_TRACE(
 918                     "port %d must be stopped to allow configuration\n", port_id);
 919                 return -EBUSY;
 920         }
 921
 922         /*
 923          * Check that the numbers of RX and TX queues are not greater
 924          * than the maximum number of RX and TX queues supported by the
 925          * configured device.
 926          */
 927         (*dev->dev_ops->dev_infos_get)(dev, &dev_info);
 928         if (nb_rx_q > dev_info.max_rx_queues) {
 929                 PMD_DEBUG_TRACE("ethdev port_id=%d nb_rx_queues=%d > %d\n",
 930                                 port_id, nb_rx_q, dev_info.max_rx_queues);
 931                 return -EINVAL;
 932         }
 933         if (nb_rx_q == 0) {
 934                 PMD_DEBUG_TRACE("ethdev port_id=%d nb_rx_q == 0\n", port_id);
 935                 return -EINVAL;
 936         }
 937
 938         if (nb_tx_q > dev_info.max_tx_queues) {
 939                 PMD_DEBUG_TRACE("ethdev port_id=%d nb_tx_queues=%d > %d\n",
 940                                 port_id, nb_tx_q, dev_info.max_tx_queues);
 941                 return -EINVAL;
 942         }
 943         if (nb_tx_q == 0) {
 944                 PMD_DEBUG_TRACE("ethdev port_id=%d nb_tx_q == 0\n", port_id);
 945                 return -EINVAL;
 946         }
 947
 948         /* Copy the dev_conf parameter into the dev structure */
 949         memcpy(&dev->data->dev_conf, dev_conf, sizeof(dev->data->dev_conf));
 950
 951         /*
 952          * If link state interrupt is enabled, check that the
 953          * device supports it.
 954          */
 955         if (dev_conf->intr_conf.lsc == 1) {
 956                 const struct rte_pci_driver *pci_drv = &dev->driver->pci_drv;
 957
 958                 if (!(pci_drv->drv_flags & RTE_PCI_DRV_INTR_LSC)) {
 959                         PMD_DEBUG_TRACE("driver %s does not support lsc\n",
 960                                         pci_drv->name);
 961                         return -EINVAL;
 962                 }
 963         }
 964
 965         /*
 966          * If jumbo frames are enabled, check that the maximum RX packet
 967          * length is supported by the configured device.
 968          */
 969         if (dev_conf->rxmode.jumbo_frame == 1) {
 970                 if (dev_conf->rxmode.max_rx_pkt_len >
 971                     dev_info.max_rx_pktlen) {
 972                         PMD_DEBUG_TRACE("ethdev port_id=%d max_rx_pkt_len %u"
 973                                 " > max valid value %u\n",
 974                                 port_id,
 975                                 (unsigned)dev_conf->rxmode.max_rx_pkt_len,
 976                                 (unsigned)dev_info.max_rx_pktlen);
 977                         return -EINVAL;
 978                 } else if (dev_conf->rxmode.max_rx_pkt_len < ETHER_MIN_LEN) {
 979                         PMD_DEBUG_TRACE("ethdev port_id=%d max_rx_pkt_len %u"
 980                                 " < min valid value %u\n",
 981                                 port_id,
 982                                 (unsigned)dev_conf->rxmode.max_rx_pkt_len,
 983                                 (unsigned)ETHER_MIN_LEN);
 984                         return -EINVAL;
 985                 }
 986         } else {
 987                 if (dev_conf->rxmode.max_rx_pkt_len < ETHER_MIN_LEN ||
 988                         dev_conf->rxmode.max_rx_pkt_len > ETHER_MAX_LEN)
 989                         /* Use default value */
 990                         dev->data->dev_conf.rxmode.max_rx_pkt_len =
 991                                                         ETHER_MAX_LEN;
 992         }
 993
 994         /*
 995          * Setup new number of RX/TX queues and reconfigure device.
 996          */
 997         diag = rte_eth_dev_rx_queue_config(dev, nb_rx_q);
 998         if (diag != 0) {
 999                 PMD_DEBUG_TRACE("port%d rte_eth_dev_rx_queue_config = %d\n",
1000                                 port_id, diag);
1001                 return diag;
1002         }
1003
1004         diag = rte_eth_dev_tx_queue_config(dev, nb_tx_q);
1005         if (diag != 0) {
1006                 PMD_DEBUG_TRACE("port%d rte_eth_dev_tx_queue_config = %d\n",
1007                                 port_id, diag);
1008                 rte_eth_dev_rx_queue_config(dev, 0);
1009                 return diag;
1010         }
1011
1012         diag = (*dev->dev_ops->dev_configure)(dev);
1013         if (diag != 0) {
1014                 PMD_DEBUG_TRACE("port%d dev_configure = %d\n",
1015                                 port_id, diag);
1016                 rte_eth_dev_rx_queue_config(dev, 0);
1017                 rte_eth_dev_tx_queue_config(dev, 0);
1018                 return diag;
1019         }
1020
1021         return 0;
1022 }
1023
1024 static void
1025 rte_eth_dev_config_restore(uint8_t port_id)
1026 {
1027         struct rte_eth_dev *dev;
1028         struct rte_eth_dev_info dev_info;
1029         struct ether_addr addr;
1030         uint16_t i;
1031         uint32_t pool = 0;
1032
1033         dev = &rte_eth_devices[port_id];
1034
1035         rte_eth_dev_info_get(port_id, &dev_info);
1036
1037         if (RTE_ETH_DEV_SRIOV(dev).active)
1038                 pool = RTE_ETH_DEV_SRIOV(dev).def_vmdq_idx;
1039
1040         /* replay MAC address configuration */
1041         for (i = 0; i < dev_info.max_mac_addrs; i++) {
1042                 addr = dev->data->mac_addrs[i];
1043
1044                 /* skip zero address */
1045                 if (is_zero_ether_addr(&addr))
1046                         continue;
1047
1048                 /* add address to the hardware */
1049                 if  (*dev->dev_ops->mac_addr_add &&
1050                         (dev->data->mac_pool_sel[i] & (1ULL << pool)))
1051                         (*dev->dev_ops->mac_addr_add)(dev, &addr, i, pool);
1052                 else {
1053                         PMD_DEBUG_TRACE("port %d: MAC address array not supported\n",
1054                                         port_id);
1055                         /* exit the loop but not return an error */
1056                         break;
1057                 }
1058         }
1059
1060         /* replay promiscuous configuration */
1061         if (rte_eth_promiscuous_get(port_id) == 1)
1062                 rte_eth_promiscuous_enable(port_id);
1063         else if (rte_eth_promiscuous_get(port_id) == 0)
1064                 rte_eth_promiscuous_disable(port_id);
1065
1066         /* replay all multicast configuration */
1067         if (rte_eth_allmulticast_get(port_id) == 1)
1068                 rte_eth_allmulticast_enable(port_id);
1069         else if (rte_eth_allmulticast_get(port_id) == 0)
1070                 rte_eth_allmulticast_disable(port_id);
1071 }
1072
1073 int
1074 rte_eth_dev_start(uint8_t port_id)
1075 {
1076         struct rte_eth_dev *dev;
1077         int diag;
1078
1079         /* This function is only safe when called from the primary process
1080          * in a multi-process setup*/
1081         PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
1082
1083         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1084
1085         dev = &rte_eth_devices[port_id];
1086
1087         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_start, -ENOTSUP);
1088
1089         if (dev->data->dev_started != 0) {
1090                 PMD_DEBUG_TRACE("Device with port_id=%" PRIu8
1091                         " already started\n",
1092                         port_id);
1093                 return 0;
1094         }
1095
1096         diag = (*dev->dev_ops->dev_start)(dev);
1097         if (diag == 0)
1098                 dev->data->dev_started = 1;
1099         else
1100                 return diag;
1101
1102         rte_eth_dev_config_restore(port_id);
1103
1104         if (dev->data->dev_conf.intr_conf.lsc != 0) {
1105                 FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
1106                 (*dev->dev_ops->link_update)(dev, 0);
1107         }
1108         return 0;
1109 }
1110
1111 void
1112 rte_eth_dev_stop(uint8_t port_id)
1113 {
1114         struct rte_eth_dev *dev;
1115
1116         /* This function is only safe when called from the primary process
1117          * in a multi-process setup*/
1118         PROC_PRIMARY_OR_RET();
1119
1120         VALID_PORTID_OR_RET(port_id);
1121         dev = &rte_eth_devices[port_id];
1122
1123         FUNC_PTR_OR_RET(*dev->dev_ops->dev_stop);
1124
1125         if (dev->data->dev_started == 0) {
1126                 PMD_DEBUG_TRACE("Device with port_id=%" PRIu8
1127                         " already stopped\n",
1128                         port_id);
1129                 return;
1130         }
1131
1132         dev->data->dev_started = 0;
1133         (*dev->dev_ops->dev_stop)(dev);
1134 }
1135
1136 int
1137 rte_eth_dev_set_link_up(uint8_t port_id)
1138 {
1139         struct rte_eth_dev *dev;
1140
1141         /* This function is only safe when called from the primary process
1142          * in a multi-process setup*/
1143         PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
1144
1145         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1146
1147         dev = &rte_eth_devices[port_id];
1148
1149         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_up, -ENOTSUP);
1150         return (*dev->dev_ops->dev_set_link_up)(dev);
1151 }
1152
1153 int
1154 rte_eth_dev_set_link_down(uint8_t port_id)
1155 {
1156         struct rte_eth_dev *dev;
1157
1158         /* This function is only safe when called from the primary process
1159          * in a multi-process setup*/
1160         PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
1161
1162         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1163
1164         dev = &rte_eth_devices[port_id];
1165
1166         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_down, -ENOTSUP);
1167         return (*dev->dev_ops->dev_set_link_down)(dev);
1168 }
1169
1170 void
1171 rte_eth_dev_close(uint8_t port_id)
1172 {
1173         struct rte_eth_dev *dev;
1174
1175         /* This function is only safe when called from the primary process
1176          * in a multi-process setup*/
1177         PROC_PRIMARY_OR_RET();
1178
1179         VALID_PORTID_OR_RET(port_id);
1180         dev = &rte_eth_devices[port_id];
1181
1182         FUNC_PTR_OR_RET(*dev->dev_ops->dev_close);
1183         dev->data->dev_started = 0;
1184         (*dev->dev_ops->dev_close)(dev);
1185
1186         rte_free(dev->data->rx_queues);
1187         dev->data->rx_queues = NULL;
1188         rte_free(dev->data->tx_queues);
1189         dev->data->tx_queues = NULL;
1190 }
1191
1192 int
1193 rte_eth_rx_queue_setup(uint8_t port_id, uint16_t rx_queue_id,
1194                        uint16_t nb_rx_desc, unsigned int socket_id,
1195                        const struct rte_eth_rxconf *rx_conf,
1196                        struct rte_mempool *mp)
1197 {
1198         int ret;
1199         uint32_t mbp_buf_size;
1200         struct rte_eth_dev *dev;
1201         struct rte_eth_dev_info dev_info;
1202
1203         /* This function is only safe when called from the primary process
1204          * in a multi-process setup*/
1205         PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
1206
1207         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1208
1209         dev = &rte_eth_devices[port_id];
1210         if (rx_queue_id >= dev->data->nb_rx_queues) {
1211                 PMD_DEBUG_TRACE("Invalid RX queue_id=%d\n", rx_queue_id);
1212                 return -EINVAL;
1213         }
1214
1215         if (dev->data->dev_started) {
1216                 PMD_DEBUG_TRACE(
1217                     "port %d must be stopped to allow configuration\n", port_id);
1218                 return -EBUSY;
1219         }
1220
1221         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
1222         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_setup, -ENOTSUP);
1223
1224         /*
1225          * Check the size of the mbuf data buffer.
1226          * This value must be provided in the private data of the memory pool.
1227          * First check that the memory pool has a valid private data.
1228          */
1229         rte_eth_dev_info_get(port_id, &dev_info);
1230         if (mp->private_data_size < sizeof(struct rte_pktmbuf_pool_private)) {
1231                 PMD_DEBUG_TRACE("%s private_data_size %d < %d\n",
1232                                 mp->name, (int) mp->private_data_size,
1233                                 (int) sizeof(struct rte_pktmbuf_pool_private));
1234                 return -ENOSPC;
1235         }
1236         mbp_buf_size = rte_pktmbuf_data_room_size(mp);
1237
1238         if ((mbp_buf_size - RTE_PKTMBUF_HEADROOM) < dev_info.min_rx_bufsize) {
1239                 PMD_DEBUG_TRACE("%s mbuf_data_room_size %d < %d "
1240                                 "(RTE_PKTMBUF_HEADROOM=%d + min_rx_bufsize(dev)"
1241                                 "=%d)\n",
1242                                 mp->name,
1243                                 (int)mbp_buf_size,
1244                                 (int)(RTE_PKTMBUF_HEADROOM +
1245                                       dev_info.min_rx_bufsize),
1246                                 (int)RTE_PKTMBUF_HEADROOM,
1247                                 (int)dev_info.min_rx_bufsize);
1248                 return -EINVAL;
1249         }
1250
1251         if (rx_conf == NULL)
1252                 rx_conf = &dev_info.default_rxconf;
1253
1254         ret = (*dev->dev_ops->rx_queue_setup)(dev, rx_queue_id, nb_rx_desc,
1255                                               socket_id, rx_conf, mp);
1256         if (!ret) {
1257                 if (!dev->data->min_rx_buf_size ||
1258                     dev->data->min_rx_buf_size > mbp_buf_size)
1259                         dev->data->min_rx_buf_size = mbp_buf_size;
1260         }
1261
1262         return ret;
1263 }
1264
1265 int
1266 rte_eth_tx_queue_setup(uint8_t port_id, uint16_t tx_queue_id,
1267                        uint16_t nb_tx_desc, unsigned int socket_id,
1268                        const struct rte_eth_txconf *tx_conf)
1269 {
1270         struct rte_eth_dev *dev;
1271         struct rte_eth_dev_info dev_info;
1272
1273         /* This function is only safe when called from the primary process
1274          * in a multi-process setup*/
1275         PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
1276
1277         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1278
1279         dev = &rte_eth_devices[port_id];
1280         if (tx_queue_id >= dev->data->nb_tx_queues) {
1281                 PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", tx_queue_id);
1282                 return -EINVAL;
1283         }
1284
1285         if (dev->data->dev_started) {
1286                 PMD_DEBUG_TRACE(
1287                     "port %d must be stopped to allow configuration\n", port_id);
1288                 return -EBUSY;
1289         }
1290
1291         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
1292         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_setup, -ENOTSUP);
1293
1294         rte_eth_dev_info_get(port_id, &dev_info);
1295
1296         if (tx_conf == NULL)
1297                 tx_conf = &dev_info.default_txconf;
1298
1299         return (*dev->dev_ops->tx_queue_setup)(dev, tx_queue_id, nb_tx_desc,
1300                                                socket_id, tx_conf);
1301 }
1302
1303 void
1304 rte_eth_promiscuous_enable(uint8_t port_id)
1305 {
1306         struct rte_eth_dev *dev;
1307
1308         VALID_PORTID_OR_RET(port_id);
1309         dev = &rte_eth_devices[port_id];
1310
1311         FUNC_PTR_OR_RET(*dev->dev_ops->promiscuous_enable);
1312         (*dev->dev_ops->promiscuous_enable)(dev);
1313         dev->data->promiscuous = 1;
1314 }
1315
1316 void
1317 rte_eth_promiscuous_disable(uint8_t port_id)
1318 {
1319         struct rte_eth_dev *dev;
1320
1321         VALID_PORTID_OR_RET(port_id);
1322         dev = &rte_eth_devices[port_id];
1323
1324         FUNC_PTR_OR_RET(*dev->dev_ops->promiscuous_disable);
1325         dev->data->promiscuous = 0;
1326         (*dev->dev_ops->promiscuous_disable)(dev);
1327 }
1328
1329 int
1330 rte_eth_promiscuous_get(uint8_t port_id)
1331 {
1332         struct rte_eth_dev *dev;
1333
1334         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1335
1336         dev = &rte_eth_devices[port_id];
1337         return dev->data->promiscuous;
1338 }
1339
1340 void
1341 rte_eth_allmulticast_enable(uint8_t port_id)
1342 {
1343         struct rte_eth_dev *dev;
1344
1345         VALID_PORTID_OR_RET(port_id);
1346         dev = &rte_eth_devices[port_id];
1347
1348         FUNC_PTR_OR_RET(*dev->dev_ops->allmulticast_enable);
1349         (*dev->dev_ops->allmulticast_enable)(dev);
1350         dev->data->all_multicast = 1;
1351 }
1352
1353 void
1354 rte_eth_allmulticast_disable(uint8_t port_id)
1355 {
1356         struct rte_eth_dev *dev;
1357
1358         VALID_PORTID_OR_RET(port_id);
1359         dev = &rte_eth_devices[port_id];
1360
1361         FUNC_PTR_OR_RET(*dev->dev_ops->allmulticast_disable);
1362         dev->data->all_multicast = 0;
1363         (*dev->dev_ops->allmulticast_disable)(dev);
1364 }
1365
1366 int
1367 rte_eth_allmulticast_get(uint8_t port_id)
1368 {
1369         struct rte_eth_dev *dev;
1370
1371         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1372
1373         dev = &rte_eth_devices[port_id];
1374         return dev->data->all_multicast;
1375 }
1376
1377 static inline int
1378 rte_eth_dev_atomic_read_link_status(struct rte_eth_dev *dev,
1379                                 struct rte_eth_link *link)
1380 {
1381         struct rte_eth_link *dst = link;
1382         struct rte_eth_link *src = &(dev->data->dev_link);
1383
1384         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
1385                                         *(uint64_t *)src) == 0)
1386                 return -1;
1387
1388         return 0;
1389 }
1390
1391 void
1392 rte_eth_link_get(uint8_t port_id, struct rte_eth_link *eth_link)
1393 {
1394         struct rte_eth_dev *dev;
1395
1396         VALID_PORTID_OR_RET(port_id);
1397         dev = &rte_eth_devices[port_id];
1398
1399         if (dev->data->dev_conf.intr_conf.lsc != 0)
1400                 rte_eth_dev_atomic_read_link_status(dev, eth_link);
1401         else {
1402                 FUNC_PTR_OR_RET(*dev->dev_ops->link_update);
1403                 (*dev->dev_ops->link_update)(dev, 1);
1404                 *eth_link = dev->data->dev_link;
1405         }
1406 }
1407
1408 void
1409 rte_eth_link_get_nowait(uint8_t port_id, struct rte_eth_link *eth_link)
1410 {
1411         struct rte_eth_dev *dev;
1412
1413         VALID_PORTID_OR_RET(port_id);
1414         dev = &rte_eth_devices[port_id];
1415
1416         if (dev->data->dev_conf.intr_conf.lsc != 0)
1417                 rte_eth_dev_atomic_read_link_status(dev, eth_link);
1418         else {
1419                 FUNC_PTR_OR_RET(*dev->dev_ops->link_update);
1420                 (*dev->dev_ops->link_update)(dev, 0);
1421                 *eth_link = dev->data->dev_link;
1422         }
1423 }
1424
1425 int
1426 rte_eth_stats_get(uint8_t port_id, struct rte_eth_stats *stats)
1427 {
1428         struct rte_eth_dev *dev;
1429
1430         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1431
1432         dev = &rte_eth_devices[port_id];
1433         memset(stats, 0, sizeof(*stats));
1434
1435         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
1436         (*dev->dev_ops->stats_get)(dev, stats);
1437         stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
1438         return 0;
1439 }
1440
1441 void
1442 rte_eth_stats_reset(uint8_t port_id)
1443 {
1444         struct rte_eth_dev *dev;
1445
1446         VALID_PORTID_OR_RET(port_id);
1447         dev = &rte_eth_devices[port_id];
1448
1449         FUNC_PTR_OR_RET(*dev->dev_ops->stats_reset);
1450         (*dev->dev_ops->stats_reset)(dev);
1451 }
1452
1453 /* retrieve ethdev extended statistics */
1454 int
1455 rte_eth_xstats_get(uint8_t port_id, struct rte_eth_xstats *xstats,
1456         unsigned n)
1457 {
1458         struct rte_eth_stats eth_stats;
1459         struct rte_eth_dev *dev;
1460         unsigned count = 0, i, q;
1461         signed xcount = 0;
1462         uint64_t val, *stats_ptr;
1463
1464         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1465
1466         dev = &rte_eth_devices[port_id];
1467
1468         /* Return generic statistics */
1469         count = RTE_NB_STATS;
1470         count += dev->data->nb_rx_queues * RTE_NB_RXQ_STATS;
1471         count += dev->data->nb_tx_queues * RTE_NB_TXQ_STATS;
1472
1473         /* implemented by the driver */
1474         if (dev->dev_ops->xstats_get != NULL) {
1475                 /* Retrieve the xstats from the driver at the end of the
1476                  * xstats struct.
1477                  */
1478                 xcount = (*dev->dev_ops->xstats_get)(dev, &xstats[count],
1479                          (n > count) ? n - count : 0);
1480
1481                 if (xcount < 0)
1482                         return xcount;
1483         }
1484
1485         if (n < count + xcount)
1486                 return count + xcount;
1487
1488         /* now fill the xstats structure */
1489         count = 0;
1490         rte_eth_stats_get(port_id, &eth_stats);
1491
1492         /* global stats */
1493         for (i = 0; i < RTE_NB_STATS; i++) {
1494                 stats_ptr = RTE_PTR_ADD(&eth_stats,
1495                                         rte_stats_strings[i].offset);
1496                 val = *stats_ptr;
1497                 snprintf(xstats[count].name, sizeof(xstats[count].name),
1498                         "%s", rte_stats_strings[i].name);
1499                 xstats[count++].value = val;
1500         }
1501
1502         /* per-rxq stats */
1503         for (q = 0; q < dev->data->nb_rx_queues; q++) {
1504                 for (i = 0; i < RTE_NB_RXQ_STATS; i++) {
1505                         stats_ptr = RTE_PTR_ADD(&eth_stats,
1506                                         rte_rxq_stats_strings[i].offset +
1507                                         q * sizeof(uint64_t));
1508                         val = *stats_ptr;
1509                         snprintf(xstats[count].name, sizeof(xstats[count].name),
1510                                 "rx_queue_%u_%s", q,
1511                                 rte_rxq_stats_strings[i].name);
1512                         xstats[count++].value = val;
1513                 }
1514         }
1515
1516         /* per-txq stats */
1517         for (q = 0; q < dev->data->nb_tx_queues; q++) {
1518                 for (i = 0; i < RTE_NB_TXQ_STATS; i++) {
1519                         stats_ptr = RTE_PTR_ADD(&eth_stats,
1520                                         rte_txq_stats_strings[i].offset +
1521                                         q * sizeof(uint64_t));
1522                         val = *stats_ptr;
1523                         snprintf(xstats[count].name, sizeof(xstats[count].name),
1524                                 "tx_queue_%u_%s", q,
1525                                 rte_txq_stats_strings[i].name);
1526                         xstats[count++].value = val;
1527                 }
1528         }
1529
1530         return count + xcount;
1531 }
1532
1533 /* reset ethdev extended statistics */
1534 void
1535 rte_eth_xstats_reset(uint8_t port_id)
1536 {
1537         struct rte_eth_dev *dev;
1538
1539         VALID_PORTID_OR_RET(port_id);
1540         dev = &rte_eth_devices[port_id];
1541
1542         /* implemented by the driver */
1543         if (dev->dev_ops->xstats_reset != NULL) {
1544                 (*dev->dev_ops->xstats_reset)(dev);
1545                 return;
1546         }
1547
1548         /* fallback to default */
1549         rte_eth_stats_reset(port_id);
1550 }
1551
1552 static int
1553 set_queue_stats_mapping(uint8_t port_id, uint16_t queue_id, uint8_t stat_idx,
1554                 uint8_t is_rx)
1555 {
1556         struct rte_eth_dev *dev;
1557
1558         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1559
1560         dev = &rte_eth_devices[port_id];
1561
1562         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_stats_mapping_set, -ENOTSUP);
1563         return (*dev->dev_ops->queue_stats_mapping_set)
1564                         (dev, queue_id, stat_idx, is_rx);
1565 }
1566
1567
1568 int
1569 rte_eth_dev_set_tx_queue_stats_mapping(uint8_t port_id, uint16_t tx_queue_id,
1570                 uint8_t stat_idx)
1571 {
1572         return set_queue_stats_mapping(port_id, tx_queue_id, stat_idx,
1573                         STAT_QMAP_TX);
1574 }
1575
1576
1577 int
1578 rte_eth_dev_set_rx_queue_stats_mapping(uint8_t port_id, uint16_t rx_queue_id,
1579                 uint8_t stat_idx)
1580 {
1581         return set_queue_stats_mapping(port_id, rx_queue_id, stat_idx,
1582                         STAT_QMAP_RX);
1583 }
1584
1585
1586 void
1587 rte_eth_dev_info_get(uint8_t port_id, struct rte_eth_dev_info *dev_info)
1588 {
1589         struct rte_eth_dev *dev;
1590
1591         VALID_PORTID_OR_RET(port_id);
1592         dev = &rte_eth_devices[port_id];
1593
1594         memset(dev_info, 0, sizeof(struct rte_eth_dev_info));
1595
1596         FUNC_PTR_OR_RET(*dev->dev_ops->dev_infos_get);
1597         (*dev->dev_ops->dev_infos_get)(dev, dev_info);
1598         dev_info->pci_dev = dev->pci_dev;
1599         if (dev->driver)
1600                 dev_info->driver_name = dev->driver->pci_drv.name;
1601 }
1602
1603 void
1604 rte_eth_macaddr_get(uint8_t port_id, struct ether_addr *mac_addr)
1605 {
1606         struct rte_eth_dev *dev;
1607
1608         VALID_PORTID_OR_RET(port_id);
1609         dev = &rte_eth_devices[port_id];
1610         ether_addr_copy(&dev->data->mac_addrs[0], mac_addr);
1611 }
1612
1613
1614 int
1615 rte_eth_dev_get_mtu(uint8_t port_id, uint16_t *mtu)
1616 {
1617         struct rte_eth_dev *dev;
1618
1619         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1620
1621         dev = &rte_eth_devices[port_id];
1622         *mtu = dev->data->mtu;
1623         return 0;
1624 }
1625
1626 int
1627 rte_eth_dev_set_mtu(uint8_t port_id, uint16_t mtu)
1628 {
1629         int ret;
1630         struct rte_eth_dev *dev;
1631
1632         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1633         dev = &rte_eth_devices[port_id];
1634         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mtu_set, -ENOTSUP);
1635
1636         ret = (*dev->dev_ops->mtu_set)(dev, mtu);
1637         if (!ret)
1638                 dev->data->mtu = mtu;
1639
1640         return ret;
1641 }
1642
1643 int
1644 rte_eth_dev_vlan_filter(uint8_t port_id, uint16_t vlan_id, int on)
1645 {
1646         struct rte_eth_dev *dev;
1647
1648         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1649         dev = &rte_eth_devices[port_id];
1650         if (!(dev->data->dev_conf.rxmode.hw_vlan_filter)) {
1651                 PMD_DEBUG_TRACE("port %d: vlan-filtering disabled\n", port_id);
1652                 return -ENOSYS;
1653         }
1654
1655         if (vlan_id > 4095) {
1656                 PMD_DEBUG_TRACE("(port_id=%d) invalid vlan_id=%u > 4095\n",
1657                                 port_id, (unsigned) vlan_id);
1658                 return -EINVAL;
1659         }
1660         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_filter_set, -ENOTSUP);
1661
1662         return (*dev->dev_ops->vlan_filter_set)(dev, vlan_id, on);
1663 }
1664
1665 int
1666 rte_eth_dev_set_vlan_strip_on_queue(uint8_t port_id, uint16_t rx_queue_id, int on)
1667 {
1668         struct rte_eth_dev *dev;
1669
1670         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1671         dev = &rte_eth_devices[port_id];
1672         if (rx_queue_id >= dev->data->nb_rx_queues) {
1673                 PMD_DEBUG_TRACE("Invalid rx_queue_id=%d\n", port_id);
1674                 return -EINVAL;
1675         }
1676
1677         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_strip_queue_set, -ENOTSUP);
1678         (*dev->dev_ops->vlan_strip_queue_set)(dev, rx_queue_id, on);
1679
1680         return 0;
1681 }
1682
1683 int
1684 rte_eth_dev_set_vlan_ether_type(uint8_t port_id, uint16_t tpid)
1685 {
1686         struct rte_eth_dev *dev;
1687
1688         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1689         dev = &rte_eth_devices[port_id];
1690         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_tpid_set, -ENOTSUP);
1691         (*dev->dev_ops->vlan_tpid_set)(dev, tpid);
1692
1693         return 0;
1694 }
1695
1696 int
1697 rte_eth_dev_set_vlan_offload(uint8_t port_id, int offload_mask)
1698 {
1699         struct rte_eth_dev *dev;
1700         int ret = 0;
1701         int mask = 0;
1702         int cur, org = 0;
1703
1704         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1705         dev = &rte_eth_devices[port_id];
1706
1707         /*check which option changed by application*/
1708         cur = !!(offload_mask & ETH_VLAN_STRIP_OFFLOAD);
1709         org = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
1710         if (cur != org) {
1711                 dev->data->dev_conf.rxmode.hw_vlan_strip = (uint8_t)cur;
1712                 mask |= ETH_VLAN_STRIP_MASK;
1713         }
1714
1715         cur = !!(offload_mask & ETH_VLAN_FILTER_OFFLOAD);
1716         org = !!(dev->data->dev_conf.rxmode.hw_vlan_filter);
1717         if (cur != org) {
1718                 dev->data->dev_conf.rxmode.hw_vlan_filter = (uint8_t)cur;
1719                 mask |= ETH_VLAN_FILTER_MASK;
1720         }
1721
1722         cur = !!(offload_mask & ETH_VLAN_EXTEND_OFFLOAD);
1723         org = !!(dev->data->dev_conf.rxmode.hw_vlan_extend);
1724         if (cur != org) {
1725                 dev->data->dev_conf.rxmode.hw_vlan_extend = (uint8_t)cur;
1726                 mask |= ETH_VLAN_EXTEND_MASK;
1727         }
1728
1729         /*no change*/
1730         if (mask == 0)
1731                 return ret;
1732
1733         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_offload_set, -ENOTSUP);
1734         (*dev->dev_ops->vlan_offload_set)(dev, mask);
1735
1736         return ret;
1737 }
1738
1739 int
1740 rte_eth_dev_get_vlan_offload(uint8_t port_id)
1741 {
1742         struct rte_eth_dev *dev;
1743         int ret = 0;
1744
1745         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1746         dev = &rte_eth_devices[port_id];
1747
1748         if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1749                 ret |= ETH_VLAN_STRIP_OFFLOAD;
1750
1751         if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1752                 ret |= ETH_VLAN_FILTER_OFFLOAD;
1753
1754         if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1755                 ret |= ETH_VLAN_EXTEND_OFFLOAD;
1756
1757         return ret;
1758 }
1759
1760 int
1761 rte_eth_dev_set_vlan_pvid(uint8_t port_id, uint16_t pvid, int on)
1762 {
1763         struct rte_eth_dev *dev;
1764
1765         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1766         dev = &rte_eth_devices[port_id];
1767         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_pvid_set, -ENOTSUP);
1768         (*dev->dev_ops->vlan_pvid_set)(dev, pvid, on);
1769
1770         return 0;
1771 }
1772
1773 int
1774 rte_eth_dev_flow_ctrl_get(uint8_t port_id, struct rte_eth_fc_conf *fc_conf)
1775 {
1776         struct rte_eth_dev *dev;
1777
1778         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1779         dev = &rte_eth_devices[port_id];
1780         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_get, -ENOTSUP);
1781         memset(fc_conf, 0, sizeof(*fc_conf));
1782         return (*dev->dev_ops->flow_ctrl_get)(dev, fc_conf);
1783 }
1784
1785 int
1786 rte_eth_dev_flow_ctrl_set(uint8_t port_id, struct rte_eth_fc_conf *fc_conf)
1787 {
1788         struct rte_eth_dev *dev;
1789
1790         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1791         if ((fc_conf->send_xon != 0) && (fc_conf->send_xon != 1)) {
1792                 PMD_DEBUG_TRACE("Invalid send_xon, only 0/1 allowed\n");
1793                 return -EINVAL;
1794         }
1795
1796         dev = &rte_eth_devices[port_id];
1797         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_set, -ENOTSUP);
1798         return (*dev->dev_ops->flow_ctrl_set)(dev, fc_conf);
1799 }
1800
1801 int
1802 rte_eth_dev_priority_flow_ctrl_set(uint8_t port_id, struct rte_eth_pfc_conf *pfc_conf)
1803 {
1804         struct rte_eth_dev *dev;
1805
1806         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1807         if (pfc_conf->priority > (ETH_DCB_NUM_USER_PRIORITIES - 1)) {
1808                 PMD_DEBUG_TRACE("Invalid priority, only 0-7 allowed\n");
1809                 return -EINVAL;
1810         }
1811
1812         dev = &rte_eth_devices[port_id];
1813         /* High water, low water validation are device specific */
1814         if  (*dev->dev_ops->priority_flow_ctrl_set)
1815                 return (*dev->dev_ops->priority_flow_ctrl_set)(dev, pfc_conf);
1816         return -ENOTSUP;
1817 }
1818
1819 static int
1820 rte_eth_check_reta_mask(struct rte_eth_rss_reta_entry64 *reta_conf,
1821                         uint16_t reta_size)
1822 {
1823         uint16_t i, num;
1824
1825         if (!reta_conf)
1826                 return -EINVAL;
1827
1828         if (reta_size != RTE_ALIGN(reta_size, RTE_RETA_GROUP_SIZE)) {
1829                 PMD_DEBUG_TRACE("Invalid reta size, should be %u aligned\n",
1830                                                         RTE_RETA_GROUP_SIZE);
1831                 return -EINVAL;
1832         }
1833
1834         num = reta_size / RTE_RETA_GROUP_SIZE;
1835         for (i = 0; i < num; i++) {
1836                 if (reta_conf[i].mask)
1837                         return 0;
1838         }
1839
1840         return -EINVAL;
1841 }
1842
1843 static int
1844 rte_eth_check_reta_entry(struct rte_eth_rss_reta_entry64 *reta_conf,
1845                          uint16_t reta_size,
1846                          uint8_t max_rxq)
1847 {
1848         uint16_t i, idx, shift;
1849
1850         if (!reta_conf)
1851                 return -EINVAL;
1852
1853         if (max_rxq == 0) {
1854                 PMD_DEBUG_TRACE("No receive queue is available\n");
1855                 return -EINVAL;
1856         }
1857
1858         for (i = 0; i < reta_size; i++) {
1859                 idx = i / RTE_RETA_GROUP_SIZE;
1860                 shift = i % RTE_RETA_GROUP_SIZE;
1861                 if ((reta_conf[idx].mask & (1ULL << shift)) &&
1862                         (reta_conf[idx].reta[shift] >= max_rxq)) {
1863                         PMD_DEBUG_TRACE("reta_conf[%u]->reta[%u]: %u exceeds "
1864                                 "the maximum rxq index: %u\n", idx, shift,
1865                                 reta_conf[idx].reta[shift], max_rxq);
1866                         return -EINVAL;
1867                 }
1868         }
1869
1870         return 0;
1871 }
1872
1873 int
1874 rte_eth_dev_rss_reta_update(uint8_t port_id,
1875                             struct rte_eth_rss_reta_entry64 *reta_conf,
1876                             uint16_t reta_size)
1877 {
1878         struct rte_eth_dev *dev;
1879         int ret;
1880
1881         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1882         /* Check mask bits */
1883         ret = rte_eth_check_reta_mask(reta_conf, reta_size);
1884         if (ret < 0)
1885                 return ret;
1886
1887         dev = &rte_eth_devices[port_id];
1888
1889         /* Check entry value */
1890         ret = rte_eth_check_reta_entry(reta_conf, reta_size,
1891                                 dev->data->nb_rx_queues);
1892         if (ret < 0)
1893                 return ret;
1894
1895         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_update, -ENOTSUP);
1896         return (*dev->dev_ops->reta_update)(dev, reta_conf, reta_size);
1897 }
1898
1899 int
1900 rte_eth_dev_rss_reta_query(uint8_t port_id,
1901                            struct rte_eth_rss_reta_entry64 *reta_conf,
1902                            uint16_t reta_size)
1903 {
1904         struct rte_eth_dev *dev;
1905         int ret;
1906
1907         if (port_id >= nb_ports) {
1908                 PMD_DEBUG_TRACE("Invalid port_id=%d\n", port_id);
1909                 return -ENODEV;
1910         }
1911
1912         /* Check mask bits */
1913         ret = rte_eth_check_reta_mask(reta_conf, reta_size);
1914         if (ret < 0)
1915                 return ret;
1916
1917         dev = &rte_eth_devices[port_id];
1918         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_query, -ENOTSUP);
1919         return (*dev->dev_ops->reta_query)(dev, reta_conf, reta_size);
1920 }
1921
1922 int
1923 rte_eth_dev_rss_hash_update(uint8_t port_id, struct rte_eth_rss_conf *rss_conf)
1924 {
1925         struct rte_eth_dev *dev;
1926         uint16_t rss_hash_protos;
1927
1928         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1929         rss_hash_protos = rss_conf->rss_hf;
1930         if ((rss_hash_protos != 0) &&
1931             ((rss_hash_protos & ETH_RSS_PROTO_MASK) == 0)) {
1932                 PMD_DEBUG_TRACE("Invalid rss_hash_protos=0x%x\n",
1933                                 rss_hash_protos);
1934                 return -EINVAL;
1935         }
1936         dev = &rte_eth_devices[port_id];
1937         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_update, -ENOTSUP);
1938         return (*dev->dev_ops->rss_hash_update)(dev, rss_conf);
1939 }
1940
1941 int
1942 rte_eth_dev_rss_hash_conf_get(uint8_t port_id,
1943                               struct rte_eth_rss_conf *rss_conf)
1944 {
1945         struct rte_eth_dev *dev;
1946
1947         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1948         dev = &rte_eth_devices[port_id];
1949         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_conf_get, -ENOTSUP);
1950         return (*dev->dev_ops->rss_hash_conf_get)(dev, rss_conf);
1951 }
1952
1953 int
1954 rte_eth_dev_udp_tunnel_add(uint8_t port_id,
1955                            struct rte_eth_udp_tunnel *udp_tunnel)
1956 {
1957         struct rte_eth_dev *dev;
1958
1959         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1960         if (udp_tunnel == NULL) {
1961                 PMD_DEBUG_TRACE("Invalid udp_tunnel parameter\n");
1962                 return -EINVAL;
1963         }
1964
1965         if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
1966                 PMD_DEBUG_TRACE("Invalid tunnel type\n");
1967                 return -EINVAL;
1968         }
1969
1970         dev = &rte_eth_devices[port_id];
1971         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_add, -ENOTSUP);
1972         return (*dev->dev_ops->udp_tunnel_add)(dev, udp_tunnel);
1973 }
1974
1975 int
1976 rte_eth_dev_udp_tunnel_delete(uint8_t port_id,
1977                               struct rte_eth_udp_tunnel *udp_tunnel)
1978 {
1979         struct rte_eth_dev *dev;
1980
1981         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1982         dev = &rte_eth_devices[port_id];
1983
1984         if (udp_tunnel == NULL) {
1985                 PMD_DEBUG_TRACE("Invalid udp_tunnel parameter\n");
1986                 return -EINVAL;
1987         }
1988
1989         if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
1990                 PMD_DEBUG_TRACE("Invalid tunnel type\n");
1991                 return -EINVAL;
1992         }
1993
1994         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_del, -ENOTSUP);
1995         return (*dev->dev_ops->udp_tunnel_del)(dev, udp_tunnel);
1996 }
1997
1998 int
1999 rte_eth_led_on(uint8_t port_id)
2000 {
2001         struct rte_eth_dev *dev;
2002
2003         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2004         dev = &rte_eth_devices[port_id];
2005         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_on, -ENOTSUP);
2006         return (*dev->dev_ops->dev_led_on)(dev);
2007 }
2008
2009 int
2010 rte_eth_led_off(uint8_t port_id)
2011 {
2012         struct rte_eth_dev *dev;
2013
2014         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2015         dev = &rte_eth_devices[port_id];
2016         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_off, -ENOTSUP);
2017         return (*dev->dev_ops->dev_led_off)(dev);
2018 }
2019
2020 /*
2021  * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find
2022  * an empty spot.
2023  */
2024 static int
2025 get_mac_addr_index(uint8_t port_id, const struct ether_addr *addr)
2026 {
2027         struct rte_eth_dev_info dev_info;
2028         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
2029         unsigned i;
2030
2031         rte_eth_dev_info_get(port_id, &dev_info);
2032
2033         for (i = 0; i < dev_info.max_mac_addrs; i++)
2034                 if (memcmp(addr, &dev->data->mac_addrs[i], ETHER_ADDR_LEN) == 0)
2035                         return i;
2036
2037         return -1;
2038 }
2039
2040 static const struct ether_addr null_mac_addr;
2041
2042 int
2043 rte_eth_dev_mac_addr_add(uint8_t port_id, struct ether_addr *addr,
2044                         uint32_t pool)
2045 {
2046         struct rte_eth_dev *dev;
2047         int index;
2048         uint64_t pool_mask;
2049
2050         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2051         dev = &rte_eth_devices[port_id];
2052         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_add, -ENOTSUP);
2053
2054         if (is_zero_ether_addr(addr)) {
2055                 PMD_DEBUG_TRACE("port %d: Cannot add NULL MAC address\n",
2056                         port_id);
2057                 return -EINVAL;
2058         }
2059         if (pool >= ETH_64_POOLS) {
2060                 PMD_DEBUG_TRACE("pool id must be 0-%d\n", ETH_64_POOLS - 1);
2061                 return -EINVAL;
2062         }
2063
2064         index = get_mac_addr_index(port_id, addr);
2065         if (index < 0) {
2066                 index = get_mac_addr_index(port_id, &null_mac_addr);
2067                 if (index < 0) {
2068                         PMD_DEBUG_TRACE("port %d: MAC address array full\n",
2069                                 port_id);
2070                         return -ENOSPC;
2071                 }
2072         } else {
2073                 pool_mask = dev->data->mac_pool_sel[index];
2074
2075                 /* Check if both MAC address and pool is already there, and do nothing */
2076                 if (pool_mask & (1ULL << pool))
2077                         return 0;
2078         }
2079
2080         /* Update NIC */
2081         (*dev->dev_ops->mac_addr_add)(dev, addr, index, pool);
2082
2083         /* Update address in NIC data structure */
2084         ether_addr_copy(addr, &dev->data->mac_addrs[index]);
2085
2086         /* Update pool bitmap in NIC data structure */
2087         dev->data->mac_pool_sel[index] |= (1ULL << pool);
2088
2089         return 0;
2090 }
2091
2092 int
2093 rte_eth_dev_mac_addr_remove(uint8_t port_id, struct ether_addr *addr)
2094 {
2095         struct rte_eth_dev *dev;
2096         int index;
2097
2098         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2099         dev = &rte_eth_devices[port_id];
2100         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_remove, -ENOTSUP);
2101
2102         index = get_mac_addr_index(port_id, addr);
2103         if (index == 0) {
2104                 PMD_DEBUG_TRACE("port %d: Cannot remove default MAC address\n", port_id);
2105                 return -EADDRINUSE;
2106         } else if (index < 0)
2107                 return 0;  /* Do nothing if address wasn't found */
2108
2109         /* Update NIC */
2110         (*dev->dev_ops->mac_addr_remove)(dev, index);
2111
2112         /* Update address in NIC data structure */
2113         ether_addr_copy(&null_mac_addr, &dev->data->mac_addrs[index]);
2114
2115         /* reset pool bitmap */
2116         dev->data->mac_pool_sel[index] = 0;
2117
2118         return 0;
2119 }
2120
2121 int
2122 rte_eth_dev_default_mac_addr_set(uint8_t port_id, struct ether_addr *addr)
2123 {
2124         struct rte_eth_dev *dev;
2125
2126         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2127
2128         if (!is_valid_assigned_ether_addr(addr))
2129                 return -EINVAL;
2130
2131         dev = &rte_eth_devices[port_id];
2132         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_set, -ENOTSUP);
2133
2134         /* Update default address in NIC data structure */
2135         ether_addr_copy(addr, &dev->data->mac_addrs[0]);
2136
2137         (*dev->dev_ops->mac_addr_set)(dev, addr);
2138
2139         return 0;
2140 }
2141
2142 int
2143 rte_eth_dev_set_vf_rxmode(uint8_t port_id,  uint16_t vf,
2144                                 uint16_t rx_mode, uint8_t on)
2145 {
2146         uint16_t num_vfs;
2147         struct rte_eth_dev *dev;
2148         struct rte_eth_dev_info dev_info;
2149
2150         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2151
2152         dev = &rte_eth_devices[port_id];
2153         rte_eth_dev_info_get(port_id, &dev_info);
2154
2155         num_vfs = dev_info.max_vfs;
2156         if (vf > num_vfs) {
2157                 PMD_DEBUG_TRACE("set VF RX mode:invalid VF id %d\n", vf);
2158                 return -EINVAL;
2159         }
2160
2161         if (rx_mode == 0) {
2162                 PMD_DEBUG_TRACE("set VF RX mode:mode mask ca not be zero\n");
2163                 return -EINVAL;
2164         }
2165         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_vf_rx_mode, -ENOTSUP);
2166         return (*dev->dev_ops->set_vf_rx_mode)(dev, vf, rx_mode, on);
2167 }
2168
2169 /*
2170  * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find
2171  * an empty spot.
2172  */
2173 static int
2174 get_hash_mac_addr_index(uint8_t port_id, const struct ether_addr *addr)
2175 {
2176         struct rte_eth_dev_info dev_info;
2177         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
2178         unsigned i;
2179
2180         rte_eth_dev_info_get(port_id, &dev_info);
2181         if (!dev->data->hash_mac_addrs)
2182                 return -1;
2183
2184         for (i = 0; i < dev_info.max_hash_mac_addrs; i++)
2185                 if (memcmp(addr, &dev->data->hash_mac_addrs[i],
2186                         ETHER_ADDR_LEN) == 0)
2187                         return i;
2188
2189         return -1;
2190 }
2191
2192 int
2193 rte_eth_dev_uc_hash_table_set(uint8_t port_id, struct ether_addr *addr,
2194                                 uint8_t on)
2195 {
2196         int index;
2197         int ret;
2198         struct rte_eth_dev *dev;
2199
2200         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2201
2202         dev = &rte_eth_devices[port_id];
2203         if (is_zero_ether_addr(addr)) {
2204                 PMD_DEBUG_TRACE("port %d: Cannot add NULL MAC address\n",
2205                         port_id);
2206                 return -EINVAL;
2207         }
2208
2209         index = get_hash_mac_addr_index(port_id, addr);
2210         /* Check if it's already there, and do nothing */
2211         if ((index >= 0) && (on))
2212                 return 0;
2213
2214         if (index < 0) {
2215                 if (!on) {
2216                         PMD_DEBUG_TRACE("port %d: the MAC address was not "
2217                                 "set in UTA\n", port_id);
2218                         return -EINVAL;
2219                 }
2220
2221                 index = get_hash_mac_addr_index(port_id, &null_mac_addr);
2222                 if (index < 0) {
2223                         PMD_DEBUG_TRACE("port %d: MAC address array full\n",
2224                                         port_id);
2225                         return -ENOSPC;
2226                 }
2227         }
2228
2229         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_hash_table_set, -ENOTSUP);
2230         ret = (*dev->dev_ops->uc_hash_table_set)(dev, addr, on);
2231         if (ret == 0) {
2232                 /* Update address in NIC data structure */
2233                 if (on)
2234                         ether_addr_copy(addr,
2235                                         &dev->data->hash_mac_addrs[index]);
2236                 else
2237                         ether_addr_copy(&null_mac_addr,
2238                                         &dev->data->hash_mac_addrs[index]);
2239         }
2240
2241         return ret;
2242 }
2243
2244 int
2245 rte_eth_dev_uc_all_hash_table_set(uint8_t port_id, uint8_t on)
2246 {
2247         struct rte_eth_dev *dev;
2248
2249         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2250
2251         dev = &rte_eth_devices[port_id];
2252
2253         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_all_hash_table_set, -ENOTSUP);
2254         return (*dev->dev_ops->uc_all_hash_table_set)(dev, on);
2255 }
2256
2257 int
2258 rte_eth_dev_set_vf_rx(uint8_t port_id, uint16_t vf, uint8_t on)
2259 {
2260         uint16_t num_vfs;
2261         struct rte_eth_dev *dev;
2262         struct rte_eth_dev_info dev_info;
2263
2264         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2265
2266         dev = &rte_eth_devices[port_id];
2267         rte_eth_dev_info_get(port_id, &dev_info);
2268
2269         num_vfs = dev_info.max_vfs;
2270         if (vf > num_vfs) {
2271                 PMD_DEBUG_TRACE("port %d: invalid vf id\n", port_id);
2272                 return -EINVAL;
2273         }
2274
2275         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_vf_rx, -ENOTSUP);
2276         return (*dev->dev_ops->set_vf_rx)(dev, vf, on);
2277 }
2278
2279 int
2280 rte_eth_dev_set_vf_tx(uint8_t port_id, uint16_t vf, uint8_t on)
2281 {
2282         uint16_t num_vfs;
2283         struct rte_eth_dev *dev;
2284         struct rte_eth_dev_info dev_info;
2285
2286         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2287
2288         dev = &rte_eth_devices[port_id];
2289         rte_eth_dev_info_get(port_id, &dev_info);
2290
2291         num_vfs = dev_info.max_vfs;
2292         if (vf > num_vfs) {
2293                 PMD_DEBUG_TRACE("set pool tx:invalid pool id=%d\n", vf);
2294                 return -EINVAL;
2295         }
2296
2297         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_vf_tx, -ENOTSUP);
2298         return (*dev->dev_ops->set_vf_tx)(dev, vf, on);
2299 }
2300
2301 int
2302 rte_eth_dev_set_vf_vlan_filter(uint8_t port_id, uint16_t vlan_id,
2303                                uint64_t vf_mask, uint8_t vlan_on)
2304 {
2305         struct rte_eth_dev *dev;
2306
2307         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2308
2309         dev = &rte_eth_devices[port_id];
2310
2311         if (vlan_id > ETHER_MAX_VLAN_ID) {
2312                 PMD_DEBUG_TRACE("VF VLAN filter:invalid VLAN id=%d\n",
2313                         vlan_id);
2314                 return -EINVAL;
2315         }
2316
2317         if (vf_mask == 0) {
2318                 PMD_DEBUG_TRACE("VF VLAN filter:pool_mask can not be 0\n");
2319                 return -EINVAL;
2320         }
2321
2322         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_vf_vlan_filter, -ENOTSUP);
2323         return (*dev->dev_ops->set_vf_vlan_filter)(dev, vlan_id,
2324                                                    vf_mask, vlan_on);
2325 }
2326
2327 int rte_eth_set_queue_rate_limit(uint8_t port_id, uint16_t queue_idx,
2328                                         uint16_t tx_rate)
2329 {
2330         struct rte_eth_dev *dev;
2331         struct rte_eth_dev_info dev_info;
2332         struct rte_eth_link link;
2333
2334         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2335
2336         dev = &rte_eth_devices[port_id];
2337         rte_eth_dev_info_get(port_id, &dev_info);
2338         link = dev->data->dev_link;
2339
2340         if (queue_idx > dev_info.max_tx_queues) {
2341                 PMD_DEBUG_TRACE("set queue rate limit:port %d: "
2342                                 "invalid queue id=%d\n", port_id, queue_idx);
2343                 return -EINVAL;
2344         }
2345
2346         if (tx_rate > link.link_speed) {
2347                 PMD_DEBUG_TRACE("set queue rate limit:invalid tx_rate=%d, "
2348                                 "bigger than link speed= %d\n",
2349                         tx_rate, link.link_speed);
2350                 return -EINVAL;
2351         }
2352
2353         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_queue_rate_limit, -ENOTSUP);
2354         return (*dev->dev_ops->set_queue_rate_limit)(dev, queue_idx, tx_rate);
2355 }
2356
2357 int rte_eth_set_vf_rate_limit(uint8_t port_id, uint16_t vf, uint16_t tx_rate,
2358                                 uint64_t q_msk)
2359 {
2360         struct rte_eth_dev *dev;
2361         struct rte_eth_dev_info dev_info;
2362         struct rte_eth_link link;
2363
2364         if (q_msk == 0)
2365                 return 0;
2366
2367         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2368
2369         dev = &rte_eth_devices[port_id];
2370         rte_eth_dev_info_get(port_id, &dev_info);
2371         link = dev->data->dev_link;
2372
2373         if (vf > dev_info.max_vfs) {
2374                 PMD_DEBUG_TRACE("set VF rate limit:port %d: "
2375                                 "invalid vf id=%d\n", port_id, vf);
2376                 return -EINVAL;
2377         }
2378
2379         if (tx_rate > link.link_speed) {
2380                 PMD_DEBUG_TRACE("set VF rate limit:invalid tx_rate=%d, "
2381                                 "bigger than link speed= %d\n",
2382                                 tx_rate, link.link_speed);
2383                 return -EINVAL;
2384         }
2385
2386         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_vf_rate_limit, -ENOTSUP);
2387         return (*dev->dev_ops->set_vf_rate_limit)(dev, vf, tx_rate, q_msk);
2388 }
2389
2390 int
2391 rte_eth_mirror_rule_set(uint8_t port_id,
2392                         struct rte_eth_mirror_conf *mirror_conf,
2393                         uint8_t rule_id, uint8_t on)
2394 {
2395         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
2396
2397         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2398         if (mirror_conf->rule_type == 0) {
2399                 PMD_DEBUG_TRACE("mirror rule type can not be 0.\n");
2400                 return -EINVAL;
2401         }
2402
2403         if (mirror_conf->dst_pool >= ETH_64_POOLS) {
2404                 PMD_DEBUG_TRACE("Invalid dst pool, pool id must be 0-%d\n",
2405                                 ETH_64_POOLS - 1);
2406                 return -EINVAL;
2407         }
2408
2409         if ((mirror_conf->rule_type & (ETH_MIRROR_VIRTUAL_POOL_UP |
2410              ETH_MIRROR_VIRTUAL_POOL_DOWN)) &&
2411             (mirror_conf->pool_mask == 0)) {
2412                 PMD_DEBUG_TRACE("Invalid mirror pool, pool mask can not be 0.\n");
2413                 return -EINVAL;
2414         }
2415
2416         if ((mirror_conf->rule_type & ETH_MIRROR_VLAN) &&
2417             mirror_conf->vlan.vlan_mask == 0) {
2418                 PMD_DEBUG_TRACE("Invalid vlan mask, vlan mask can not be 0.\n");
2419                 return -EINVAL;
2420         }
2421
2422         dev = &rte_eth_devices[port_id];
2423         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mirror_rule_set, -ENOTSUP);
2424
2425         return (*dev->dev_ops->mirror_rule_set)(dev, mirror_conf, rule_id, on);
2426 }
2427
2428 int
2429 rte_eth_mirror_rule_reset(uint8_t port_id, uint8_t rule_id)
2430 {
2431         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
2432
2433         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2434
2435         dev = &rte_eth_devices[port_id];
2436         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mirror_rule_reset, -ENOTSUP);
2437
2438         return (*dev->dev_ops->mirror_rule_reset)(dev, rule_id);
2439 }
2440
2441 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
2442 uint16_t
2443 rte_eth_rx_burst(uint8_t port_id, uint16_t queue_id,
2444                  struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
2445 {
2446         struct rte_eth_dev *dev;
2447
2448         VALID_PORTID_OR_ERR_RET(port_id, 0);
2449
2450         dev = &rte_eth_devices[port_id];
2451         FUNC_PTR_OR_ERR_RET(*dev->rx_pkt_burst, 0);
2452         if (queue_id >= dev->data->nb_rx_queues) {
2453                 PMD_DEBUG_TRACE("Invalid RX queue_id=%d\n", queue_id);
2454                 return 0;
2455         }
2456         return (*dev->rx_pkt_burst)(dev->data->rx_queues[queue_id],
2457                                                 rx_pkts, nb_pkts);
2458 }
2459
2460 uint16_t
2461 rte_eth_tx_burst(uint8_t port_id, uint16_t queue_id,
2462                  struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2463 {
2464         struct rte_eth_dev *dev;
2465
2466         VALID_PORTID_OR_ERR_RET(port_id, 0);
2467
2468         dev = &rte_eth_devices[port_id];
2469
2470         FUNC_PTR_OR_ERR_RET(*dev->tx_pkt_burst, 0);
2471         if (queue_id >= dev->data->nb_tx_queues) {
2472                 PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", queue_id);
2473                 return 0;
2474         }
2475         return (*dev->tx_pkt_burst)(dev->data->tx_queues[queue_id],
2476                                                 tx_pkts, nb_pkts);
2477 }
2478
2479 uint32_t
2480 rte_eth_rx_queue_count(uint8_t port_id, uint16_t queue_id)
2481 {
2482         struct rte_eth_dev *dev;
2483
2484         VALID_PORTID_OR_ERR_RET(port_id, 0);
2485
2486         dev = &rte_eth_devices[port_id];
2487         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_count, 0);
2488         return (*dev->dev_ops->rx_queue_count)(dev, queue_id);
2489 }
2490
2491 int
2492 rte_eth_rx_descriptor_done(uint8_t port_id, uint16_t queue_id, uint16_t offset)
2493 {
2494         struct rte_eth_dev *dev;
2495
2496         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2497
2498         dev = &rte_eth_devices[port_id];
2499         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_descriptor_done, -ENOTSUP);
2500         return (*dev->dev_ops->rx_descriptor_done)(dev->data->rx_queues[queue_id],
2501                                                    offset);
2502 }
2503 #endif
2504
2505 int
2506 rte_eth_dev_callback_register(uint8_t port_id,
2507                         enum rte_eth_event_type event,
2508                         rte_eth_dev_cb_fn cb_fn, void *cb_arg)
2509 {
2510         struct rte_eth_dev *dev;
2511         struct rte_eth_dev_callback *user_cb;
2512
2513         if (!cb_fn)
2514                 return -EINVAL;
2515
2516         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2517
2518         dev = &rte_eth_devices[port_id];
2519         rte_spinlock_lock(&rte_eth_dev_cb_lock);
2520
2521         TAILQ_FOREACH(user_cb, &(dev->link_intr_cbs), next) {
2522                 if (user_cb->cb_fn == cb_fn &&
2523                         user_cb->cb_arg == cb_arg &&
2524                         user_cb->event == event) {
2525                         break;
2526                 }
2527         }
2528
2529         /* create a new callback. */
2530         if (user_cb == NULL)
2531                 user_cb = rte_zmalloc("INTR_USER_CALLBACK",
2532                                       sizeof(struct rte_eth_dev_callback), 0);
2533         if (user_cb != NULL) {
2534                 user_cb->cb_fn = cb_fn;
2535                 user_cb->cb_arg = cb_arg;
2536                 user_cb->event = event;
2537                 TAILQ_INSERT_TAIL(&(dev->link_intr_cbs), user_cb, next);
2538         }
2539
2540         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
2541         return (user_cb == NULL) ? -ENOMEM : 0;
2542 }
2543
2544 int
2545 rte_eth_dev_callback_unregister(uint8_t port_id,
2546                         enum rte_eth_event_type event,
2547                         rte_eth_dev_cb_fn cb_fn, void *cb_arg)
2548 {
2549         int ret;
2550         struct rte_eth_dev *dev;
2551         struct rte_eth_dev_callback *cb, *next;
2552
2553         if (!cb_fn)
2554                 return -EINVAL;
2555
2556         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2557
2558         dev = &rte_eth_devices[port_id];
2559         rte_spinlock_lock(&rte_eth_dev_cb_lock);
2560
2561         ret = 0;
2562         for (cb = TAILQ_FIRST(&dev->link_intr_cbs); cb != NULL; cb = next) {
2563
2564                 next = TAILQ_NEXT(cb, next);
2565
2566                 if (cb->cb_fn != cb_fn || cb->event != event ||
2567                                 (cb->cb_arg != (void *)-1 &&
2568                                 cb->cb_arg != cb_arg))
2569                         continue;
2570
2571                 /*
2572                  * if this callback is not executing right now,
2573                  * then remove it.
2574                  */
2575                 if (cb->active == 0) {
2576                         TAILQ_REMOVE(&(dev->link_intr_cbs), cb, next);
2577                         rte_free(cb);
2578                 } else {
2579                         ret = -EAGAIN;
2580                 }
2581         }
2582
2583         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
2584         return ret;
2585 }
2586
2587 void
2588 _rte_eth_dev_callback_process(struct rte_eth_dev *dev,
2589         enum rte_eth_event_type event)
2590 {
2591         struct rte_eth_dev_callback *cb_lst;
2592         struct rte_eth_dev_callback dev_cb;
2593
2594         rte_spinlock_lock(&rte_eth_dev_cb_lock);
2595         TAILQ_FOREACH(cb_lst, &(dev->link_intr_cbs), next) {
2596                 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
2597                         continue;
2598                 dev_cb = *cb_lst;
2599                 cb_lst->active = 1;
2600                 rte_spinlock_unlock(&rte_eth_dev_cb_lock);
2601                 dev_cb.cb_fn(dev->data->port_id, dev_cb.event,
2602                                                 dev_cb.cb_arg);
2603                 rte_spinlock_lock(&rte_eth_dev_cb_lock);
2604                 cb_lst->active = 0;
2605         }
2606         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
2607 }
2608
2609 int
2610 rte_eth_dev_rx_intr_ctl(uint8_t port_id, int epfd, int op, void *data)
2611 {
2612         uint32_t vec;
2613         struct rte_eth_dev *dev;
2614         struct rte_intr_handle *intr_handle;
2615         uint16_t qid;
2616         int rc;
2617
2618         if (!rte_eth_dev_is_valid_port(port_id)) {
2619                 PMD_DEBUG_TRACE("Invalid port_id=%u\n", port_id);
2620                 return -ENODEV;
2621         }
2622
2623         dev = &rte_eth_devices[port_id];
2624         intr_handle = &dev->pci_dev->intr_handle;
2625         if (!intr_handle->intr_vec) {
2626                 PMD_DEBUG_TRACE("RX Intr vector unset\n");
2627                 return -EPERM;
2628         }
2629
2630         for (qid = 0; qid < dev->data->nb_rx_queues; qid++) {
2631                 vec = intr_handle->intr_vec[qid];
2632                 rc = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
2633                 if (rc && rc != -EEXIST) {
2634                         PMD_DEBUG_TRACE("p %u q %u rx ctl error"
2635                                         " op %d epfd %d vec %u\n",
2636                                         port_id, qid, op, epfd, vec);
2637                 }
2638         }
2639
2640         return 0;
2641 }
2642
2643 int
2644 rte_eth_dev_rx_intr_ctl_q(uint8_t port_id, uint16_t queue_id,
2645                           int epfd, int op, void *data)
2646 {
2647         uint32_t vec;
2648         struct rte_eth_dev *dev;
2649         struct rte_intr_handle *intr_handle;
2650         int rc;
2651
2652         if (!rte_eth_dev_is_valid_port(port_id)) {
2653                 PMD_DEBUG_TRACE("Invalid port_id=%u\n", port_id);
2654                 return -ENODEV;
2655         }
2656
2657         dev = &rte_eth_devices[port_id];
2658         if (queue_id >= dev->data->nb_rx_queues) {
2659                 PMD_DEBUG_TRACE("Invalid RX queue_id=%u\n", queue_id);
2660                 return -EINVAL;
2661         }
2662
2663         intr_handle = &dev->pci_dev->intr_handle;
2664         if (!intr_handle->intr_vec) {
2665                 PMD_DEBUG_TRACE("RX Intr vector unset\n");
2666                 return -EPERM;
2667         }
2668
2669         vec = intr_handle->intr_vec[queue_id];
2670         rc = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
2671         if (rc && rc != -EEXIST) {
2672                 PMD_DEBUG_TRACE("p %u q %u rx ctl error"
2673                                 " op %d epfd %d vec %u\n",
2674                                 port_id, queue_id, op, epfd, vec);
2675                 return rc;
2676         }
2677
2678         return 0;
2679 }
2680
2681 int
2682 rte_eth_dev_rx_intr_enable(uint8_t port_id,
2683                            uint16_t queue_id)
2684 {
2685         struct rte_eth_dev *dev;
2686
2687         if (!rte_eth_dev_is_valid_port(port_id)) {
2688                 PMD_DEBUG_TRACE("Invalid port_id=%d\n", port_id);
2689                 return -ENODEV;
2690         }
2691
2692         dev = &rte_eth_devices[port_id];
2693
2694         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_intr_enable, -ENOTSUP);
2695         return (*dev->dev_ops->rx_queue_intr_enable)(dev, queue_id);
2696 }
2697
2698 int
2699 rte_eth_dev_rx_intr_disable(uint8_t port_id,
2700                             uint16_t queue_id)
2701 {
2702         struct rte_eth_dev *dev;
2703
2704         if (!rte_eth_dev_is_valid_port(port_id)) {
2705                 PMD_DEBUG_TRACE("Invalid port_id=%d\n", port_id);
2706                 return -ENODEV;
2707         }
2708
2709         dev = &rte_eth_devices[port_id];
2710
2711         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_intr_disable, -ENOTSUP);
2712         return (*dev->dev_ops->rx_queue_intr_disable)(dev, queue_id);
2713 }
2714
2715 #ifdef RTE_NIC_BYPASS
2716 int rte_eth_dev_bypass_init(uint8_t port_id)
2717 {
2718         struct rte_eth_dev *dev;
2719
2720         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2721
2722         dev = &rte_eth_devices[port_id];
2723         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_init, -ENOTSUP);
2724         (*dev->dev_ops->bypass_init)(dev);
2725         return 0;
2726 }
2727
2728 int
2729 rte_eth_dev_bypass_state_show(uint8_t port_id, uint32_t *state)
2730 {
2731         struct rte_eth_dev *dev;
2732
2733         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2734
2735         dev = &rte_eth_devices[port_id];
2736         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_state_show, -ENOTSUP);
2737         (*dev->dev_ops->bypass_state_show)(dev, state);
2738         return 0;
2739 }
2740
2741 int
2742 rte_eth_dev_bypass_state_set(uint8_t port_id, uint32_t *new_state)
2743 {
2744         struct rte_eth_dev *dev;
2745
2746         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2747
2748         dev = &rte_eth_devices[port_id];
2749         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_state_set, -ENOTSUP);
2750         (*dev->dev_ops->bypass_state_set)(dev, new_state);
2751         return 0;
2752 }
2753
2754 int
2755 rte_eth_dev_bypass_event_show(uint8_t port_id, uint32_t event, uint32_t *state)
2756 {
2757         struct rte_eth_dev *dev;
2758
2759         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2760
2761         dev = &rte_eth_devices[port_id];
2762         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_state_show, -ENOTSUP);
2763         (*dev->dev_ops->bypass_event_show)(dev, event, state);
2764         return 0;
2765 }
2766
2767 int
2768 rte_eth_dev_bypass_event_store(uint8_t port_id, uint32_t event, uint32_t state)
2769 {
2770         struct rte_eth_dev *dev;
2771
2772         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2773
2774         dev = &rte_eth_devices[port_id];
2775
2776         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_event_set, -ENOTSUP);
2777         (*dev->dev_ops->bypass_event_set)(dev, event, state);
2778         return 0;
2779 }
2780
2781 int
2782 rte_eth_dev_wd_timeout_store(uint8_t port_id, uint32_t timeout)
2783 {
2784         struct rte_eth_dev *dev;
2785
2786         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2787
2788         dev = &rte_eth_devices[port_id];
2789
2790         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_wd_timeout_set, -ENOTSUP);
2791         (*dev->dev_ops->bypass_wd_timeout_set)(dev, timeout);
2792         return 0;
2793 }
2794
2795 int
2796 rte_eth_dev_bypass_ver_show(uint8_t port_id, uint32_t *ver)
2797 {
2798         struct rte_eth_dev *dev;
2799
2800         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2801
2802         dev = &rte_eth_devices[port_id];
2803
2804         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_ver_show, -ENOTSUP);
2805         (*dev->dev_ops->bypass_ver_show)(dev, ver);
2806         return 0;
2807 }
2808
2809 int
2810 rte_eth_dev_bypass_wd_timeout_show(uint8_t port_id, uint32_t *wd_timeout)
2811 {
2812         struct rte_eth_dev *dev;
2813
2814         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2815
2816         dev = &rte_eth_devices[port_id];
2817
2818         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_wd_timeout_show, -ENOTSUP);
2819         (*dev->dev_ops->bypass_wd_timeout_show)(dev, wd_timeout);
2820         return 0;
2821 }
2822
2823 int
2824 rte_eth_dev_bypass_wd_reset(uint8_t port_id)
2825 {
2826         struct rte_eth_dev *dev;
2827
2828         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2829
2830         dev = &rte_eth_devices[port_id];
2831
2832         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_wd_reset, -ENOTSUP);
2833         (*dev->dev_ops->bypass_wd_reset)(dev);
2834         return 0;
2835 }
2836 #endif
2837
2838 int
2839 rte_eth_dev_filter_supported(uint8_t port_id, enum rte_filter_type filter_type)
2840 {
2841         struct rte_eth_dev *dev;
2842
2843         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2844
2845         dev = &rte_eth_devices[port_id];
2846         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->filter_ctrl, -ENOTSUP);
2847         return (*dev->dev_ops->filter_ctrl)(dev, filter_type,
2848                                 RTE_ETH_FILTER_NOP, NULL);
2849 }
2850
2851 int
2852 rte_eth_dev_filter_ctrl(uint8_t port_id, enum rte_filter_type filter_type,
2853                        enum rte_filter_op filter_op, void *arg)
2854 {
2855         struct rte_eth_dev *dev;
2856
2857         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2858
2859         dev = &rte_eth_devices[port_id];
2860         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->filter_ctrl, -ENOTSUP);
2861         return (*dev->dev_ops->filter_ctrl)(dev, filter_type, filter_op, arg);
2862 }
2863
2864 void *
2865 rte_eth_add_rx_callback(uint8_t port_id, uint16_t queue_id,
2866                 rte_rx_callback_fn fn, void *user_param)
2867 {
2868 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
2869         rte_errno = ENOTSUP;
2870         return NULL;
2871 #endif
2872         /* check input parameters */
2873         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
2874                     queue_id >= rte_eth_devices[port_id].data->nb_rx_queues) {
2875                 rte_errno = EINVAL;
2876                 return NULL;
2877         }
2878
2879         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
2880
2881         if (cb == NULL) {
2882                 rte_errno = ENOMEM;
2883                 return NULL;
2884         }
2885
2886         cb->fn.rx = fn;
2887         cb->param = user_param;
2888
2889         /* Add the callbacks in fifo order. */
2890         struct rte_eth_rxtx_callback *tail =
2891                 rte_eth_devices[port_id].post_rx_burst_cbs[queue_id];
2892
2893         if (!tail) {
2894                 rte_eth_devices[port_id].post_rx_burst_cbs[queue_id] = cb;
2895
2896         } else {
2897                 while (tail->next)
2898                         tail = tail->next;
2899                 tail->next = cb;
2900         }
2901
2902         return cb;
2903 }
2904
2905 void *
2906 rte_eth_add_tx_callback(uint8_t port_id, uint16_t queue_id,
2907                 rte_tx_callback_fn fn, void *user_param)
2908 {
2909 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
2910         rte_errno = ENOTSUP;
2911         return NULL;
2912 #endif
2913         /* check input parameters */
2914         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
2915                     queue_id >= rte_eth_devices[port_id].data->nb_tx_queues) {
2916                 rte_errno = EINVAL;
2917                 return NULL;
2918         }
2919
2920         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
2921
2922         if (cb == NULL) {
2923                 rte_errno = ENOMEM;
2924                 return NULL;
2925         }
2926
2927         cb->fn.tx = fn;
2928         cb->param = user_param;
2929
2930         /* Add the callbacks in fifo order. */
2931         struct rte_eth_rxtx_callback *tail =
2932                 rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id];
2933
2934         if (!tail) {
2935                 rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id] = cb;
2936
2937         } else {
2938                 while (tail->next)
2939                         tail = tail->next;
2940                 tail->next = cb;
2941         }
2942
2943         return cb;
2944 }
2945
2946 int
2947 rte_eth_remove_rx_callback(uint8_t port_id, uint16_t queue_id,
2948                 struct rte_eth_rxtx_callback *user_cb)
2949 {
2950 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
2951         return -ENOTSUP;
2952 #endif
2953         /* Check input parameters. */
2954         if (!rte_eth_dev_is_valid_port(port_id) || user_cb == NULL ||
2955                     queue_id >= rte_eth_devices[port_id].data->nb_rx_queues) {
2956                 return -EINVAL;
2957         }
2958
2959         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
2960         struct rte_eth_rxtx_callback *cb = dev->post_rx_burst_cbs[queue_id];
2961         struct rte_eth_rxtx_callback *prev_cb;
2962
2963         /* Reset head pointer and remove user cb if first in the list. */
2964         if (cb == user_cb) {
2965                 dev->post_rx_burst_cbs[queue_id] = user_cb->next;
2966                 return 0;
2967         }
2968
2969         /* Remove the user cb from the callback list. */
2970         do {
2971                 prev_cb = cb;
2972                 cb = cb->next;
2973
2974                 if (cb == user_cb) {
2975                         prev_cb->next = user_cb->next;
2976                         return 0;
2977                 }
2978
2979         } while (cb != NULL);
2980
2981         /* Callback wasn't found. */
2982         return -EINVAL;
2983 }
2984
2985 int
2986 rte_eth_remove_tx_callback(uint8_t port_id, uint16_t queue_id,
2987                 struct rte_eth_rxtx_callback *user_cb)
2988 {
2989 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
2990         return -ENOTSUP;
2991 #endif
2992         /* Check input parameters. */
2993         if (!rte_eth_dev_is_valid_port(port_id) || user_cb == NULL ||
2994                     queue_id >= rte_eth_devices[port_id].data->nb_tx_queues) {
2995                 return -EINVAL;
2996         }
2997
2998         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
2999         struct rte_eth_rxtx_callback *cb = dev->pre_tx_burst_cbs[queue_id];
3000         struct rte_eth_rxtx_callback *prev_cb;
3001
3002         /* Reset head pointer and remove user cb if first in the list. */
3003         if (cb == user_cb) {
3004                 dev->pre_tx_burst_cbs[queue_id] = user_cb->next;
3005                 return 0;
3006         }
3007
3008         /* Remove the user cb from the callback list. */
3009         do {
3010                 prev_cb = cb;
3011                 cb = cb->next;
3012
3013                 if (cb == user_cb) {
3014                         prev_cb->next = user_cb->next;
3015                         return 0;
3016                 }
3017
3018         } while (cb != NULL);
3019
3020         /* Callback wasn't found. */
3021         return -EINVAL;
3022 }
3023
3024 int
3025 rte_eth_dev_set_mc_addr_list(uint8_t port_id,
3026                              struct ether_addr *mc_addr_set,
3027                              uint32_t nb_mc_addr)
3028 {
3029         struct rte_eth_dev *dev;
3030
3031         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3032
3033         dev = &rte_eth_devices[port_id];
3034         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_mc_addr_list, -ENOTSUP);
3035         return dev->dev_ops->set_mc_addr_list(dev, mc_addr_set, nb_mc_addr);
3036 }
3037
3038 int
3039 rte_eth_timesync_enable(uint8_t port_id)
3040 {
3041         struct rte_eth_dev *dev;
3042
3043         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3044         dev = &rte_eth_devices[port_id];
3045
3046         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_enable, -ENOTSUP);
3047         return (*dev->dev_ops->timesync_enable)(dev);
3048 }
3049
3050 int
3051 rte_eth_timesync_disable(uint8_t port_id)
3052 {
3053         struct rte_eth_dev *dev;
3054
3055         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3056         dev = &rte_eth_devices[port_id];
3057
3058         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_disable, -ENOTSUP);
3059         return (*dev->dev_ops->timesync_disable)(dev);
3060 }
3061
3062 int
3063 rte_eth_timesync_read_rx_timestamp(uint8_t port_id, struct timespec *timestamp,
3064                                    uint32_t flags)
3065 {
3066         struct rte_eth_dev *dev;
3067
3068         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3069         dev = &rte_eth_devices[port_id];
3070
3071         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_rx_timestamp, -ENOTSUP);
3072         return (*dev->dev_ops->timesync_read_rx_timestamp)(dev, timestamp, flags);
3073 }
3074
3075 int
3076 rte_eth_timesync_read_tx_timestamp(uint8_t port_id, struct timespec *timestamp)
3077 {
3078         struct rte_eth_dev *dev;
3079
3080         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3081         dev = &rte_eth_devices[port_id];
3082
3083         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_tx_timestamp, -ENOTSUP);
3084         return (*dev->dev_ops->timesync_read_tx_timestamp)(dev, timestamp);
3085 }
3086
3087 int
3088 rte_eth_dev_get_reg_length(uint8_t port_id)
3089 {
3090         struct rte_eth_dev *dev;
3091
3092         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3093
3094         dev = &rte_eth_devices[port_id];
3095         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_reg_length, -ENOTSUP);
3096         return (*dev->dev_ops->get_reg_length)(dev);
3097 }
3098
3099 int
3100 rte_eth_dev_get_reg_info(uint8_t port_id, struct rte_dev_reg_info *info)
3101 {
3102         struct rte_eth_dev *dev;
3103
3104         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3105
3106         dev = &rte_eth_devices[port_id];
3107         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_reg, -ENOTSUP);
3108         return (*dev->dev_ops->get_reg)(dev, info);
3109 }
3110
3111 int
3112 rte_eth_dev_get_eeprom_length(uint8_t port_id)
3113 {
3114         struct rte_eth_dev *dev;
3115
3116         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3117
3118         dev = &rte_eth_devices[port_id];
3119         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom_length, -ENOTSUP);
3120         return (*dev->dev_ops->get_eeprom_length)(dev);
3121 }
3122
3123 int
3124 rte_eth_dev_get_eeprom(uint8_t port_id, struct rte_dev_eeprom_info *info)
3125 {
3126         struct rte_eth_dev *dev;
3127
3128         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3129
3130         dev = &rte_eth_devices[port_id];
3131         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom, -ENOTSUP);
3132         return (*dev->dev_ops->get_eeprom)(dev, info);
3133 }
3134
3135 int
3136 rte_eth_dev_set_eeprom(uint8_t port_id, struct rte_dev_eeprom_info *info)
3137 {
3138         struct rte_eth_dev *dev;
3139
3140         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3141
3142         dev = &rte_eth_devices[port_id];
3143         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_eeprom, -ENOTSUP);
3144         return (*dev->dev_ops->set_eeprom)(dev, info);
3145 }
3146
3147 int
3148 rte_eth_dev_get_dcb_info(uint8_t port_id,
3149                              struct rte_eth_dcb_info *dcb_info)
3150 {
3151         struct rte_eth_dev *dev;
3152
3153         if (!rte_eth_dev_is_valid_port(port_id)) {
3154                 PMD_DEBUG_TRACE("Invalid port_id=%d\n", port_id);
3155                 return -ENODEV;
3156         }
3157
3158         dev = &rte_eth_devices[port_id];
3159         memset(dcb_info, 0, sizeof(struct rte_eth_dcb_info));
3160
3161         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_dcb_info, -ENOTSUP);
3162         return (*dev->dev_ops->get_dcb_info)(dev, dcb_info);
3163 }