lib/librte_ether/rte_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2015 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/types.h>
  35 #include <sys/queue.h>
  36 #include <ctype.h>
  37 #include <stdio.h>
  38 #include <stdlib.h>
  39 #include <string.h>
  40 #include <stdarg.h>
  41 #include <errno.h>
  42 #include <stdint.h>
  43 #include <inttypes.h>
  44 #include <netinet/in.h>
  45
  46 #include <rte_byteorder.h>
  47 #include <rte_log.h>
  48 #include <rte_debug.h>
  49 #include <rte_interrupts.h>
  50 #include <rte_pci.h>
  51 #include <rte_memory.h>
  52 #include <rte_memcpy.h>
  53 #include <rte_memzone.h>
  54 #include <rte_launch.h>
  55 #include <rte_eal.h>
  56 #include <rte_per_lcore.h>
  57 #include <rte_lcore.h>
  58 #include <rte_atomic.h>
  59 #include <rte_branch_prediction.h>
  60 #include <rte_common.h>
  61 #include <rte_ring.h>
  62 #include <rte_mempool.h>
  63 #include <rte_malloc.h>
  64 #include <rte_mbuf.h>
  65 #include <rte_errno.h>
  66 #include <rte_spinlock.h>
  67 #include <rte_string_fns.h>
  68
  69 #include "rte_ether.h"
  70 #include "rte_ethdev.h"
  71
  72 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
  73 #define PMD_DEBUG_TRACE(fmt, args...) do {                        \
  74                 RTE_LOG(ERR, PMD, "%s: " fmt, __func__, ## args); \
  75         } while (0)
  76 #else
  77 #define PMD_DEBUG_TRACE(fmt, args...)
  78 #endif
  79
  80 /* Macros for checking for restricting functions to primary instance only */
  81 #define PROC_PRIMARY_OR_ERR_RET(retval) do { \
  82         if (rte_eal_process_type() != RTE_PROC_PRIMARY) { \
  83                 PMD_DEBUG_TRACE("Cannot run in secondary processes\n"); \
  84                 return (retval); \
  85         } \
  86 } while (0)
  87
  88 #define PROC_PRIMARY_OR_RET() do { \
  89         if (rte_eal_process_type() != RTE_PROC_PRIMARY) { \
  90                 PMD_DEBUG_TRACE("Cannot run in secondary processes\n"); \
  91                 return; \
  92         } \
  93 } while (0)
  94
  95 /* Macros to check for invalid function pointers in dev_ops structure */
  96 #define FUNC_PTR_OR_ERR_RET(func, retval) do { \
  97         if ((func) == NULL) { \
  98                 PMD_DEBUG_TRACE("Function not supported\n"); \
  99                 return (retval); \
 100         } \
 101 } while (0)
 102
 103 #define FUNC_PTR_OR_RET(func) do { \
 104         if ((func) == NULL) { \
 105                 PMD_DEBUG_TRACE("Function not supported\n"); \
 106                 return; \
 107         } \
 108 } while (0)
 109
 110 /* Macros to check for valid port */
 111 #define VALID_PORTID_OR_ERR_RET(port_id, retval) do {           \
 112         if (!rte_eth_dev_is_valid_port(port_id)) {              \
 113                 PMD_DEBUG_TRACE("Invalid port_id=%d\n", port_id); \
 114                 return retval;                                  \
 115         }                                                       \
 116 } while (0)
 117
 118 #define VALID_PORTID_OR_RET(port_id) do {                       \
 119         if (!rte_eth_dev_is_valid_port(port_id)) {              \
 120                 PMD_DEBUG_TRACE("Invalid port_id=%d\n", port_id); \
 121                 return;                                         \
 122         }                                                       \
 123 } while (0)
 124
 125 static const char *MZ_RTE_ETH_DEV_DATA = "rte_eth_dev_data";
 126 struct rte_eth_dev rte_eth_devices[RTE_MAX_ETHPORTS];
 127 static struct rte_eth_dev_data *rte_eth_dev_data;
 128 static uint8_t nb_ports;
 129
 130 /* spinlock for eth device callbacks */
 131 static rte_spinlock_t rte_eth_dev_cb_lock = RTE_SPINLOCK_INITIALIZER;
 132
 133 /* store statistics names and its offset in stats structure  */
 134 struct rte_eth_xstats_name_off {
 135         char name[RTE_ETH_XSTATS_NAME_SIZE];
 136         unsigned offset;
 137 };
 138
 139 static const struct rte_eth_xstats_name_off rte_stats_strings[] = {
 140         {"rx_packets", offsetof(struct rte_eth_stats, ipackets)},
 141         {"tx_packets", offsetof(struct rte_eth_stats, opackets)},
 142         {"rx_bytes", offsetof(struct rte_eth_stats, ibytes)},
 143         {"tx_bytes", offsetof(struct rte_eth_stats, obytes)},
 144         {"tx_errors", offsetof(struct rte_eth_stats, oerrors)},
 145         {"rx_missed_errors", offsetof(struct rte_eth_stats, imissed)},
 146         {"rx_crc_errors", offsetof(struct rte_eth_stats, ibadcrc)},
 147         {"rx_bad_length_errors", offsetof(struct rte_eth_stats, ibadlen)},
 148         {"rx_errors", offsetof(struct rte_eth_stats, ierrors)},
 149         {"alloc_rx_buff_failed", offsetof(struct rte_eth_stats, rx_nombuf)},
 150         {"fdir_match", offsetof(struct rte_eth_stats, fdirmatch)},
 151         {"fdir_miss", offsetof(struct rte_eth_stats, fdirmiss)},
 152         {"tx_flow_control_xon", offsetof(struct rte_eth_stats, tx_pause_xon)},
 153         {"rx_flow_control_xon", offsetof(struct rte_eth_stats, rx_pause_xon)},
 154         {"tx_flow_control_xoff", offsetof(struct rte_eth_stats, tx_pause_xoff)},
 155         {"rx_flow_control_xoff", offsetof(struct rte_eth_stats, rx_pause_xoff)},
 156 };
 157 #define RTE_NB_STATS (sizeof(rte_stats_strings) / sizeof(rte_stats_strings[0]))
 158
 159 static const struct rte_eth_xstats_name_off rte_rxq_stats_strings[] = {
 160         {"rx_packets", offsetof(struct rte_eth_stats, q_ipackets)},
 161         {"rx_bytes", offsetof(struct rte_eth_stats, q_ibytes)},
 162 };
 163 #define RTE_NB_RXQ_STATS (sizeof(rte_rxq_stats_strings) /       \
 164                 sizeof(rte_rxq_stats_strings[0]))
 165
 166 static const struct rte_eth_xstats_name_off rte_txq_stats_strings[] = {
 167         {"tx_packets", offsetof(struct rte_eth_stats, q_opackets)},
 168         {"tx_bytes", offsetof(struct rte_eth_stats, q_obytes)},
 169         {"tx_errors", offsetof(struct rte_eth_stats, q_errors)},
 170 };
 171 #define RTE_NB_TXQ_STATS (sizeof(rte_txq_stats_strings) /       \
 172                 sizeof(rte_txq_stats_strings[0]))
 173
 174
 175 /**
 176  * The user application callback description.
 177  *
 178  * It contains callback address to be registered by user application,
 179  * the pointer to the parameters for callback, and the event type.
 180  */
 181 struct rte_eth_dev_callback {
 182         TAILQ_ENTRY(rte_eth_dev_callback) next; /**< Callbacks list */
 183         rte_eth_dev_cb_fn cb_fn;                /**< Callback address */
 184         void *cb_arg;                           /**< Parameter for callback */
 185         enum rte_eth_event_type event;          /**< Interrupt event type */
 186         uint32_t active;                        /**< Callback is executing */
 187 };
 188
 189 enum {
 190         STAT_QMAP_TX = 0,
 191         STAT_QMAP_RX
 192 };
 193
 194 enum {
 195         DEV_DETACHED = 0,
 196         DEV_ATTACHED
 197 };
 198
 199 static void
 200 rte_eth_dev_data_alloc(void)
 201 {
 202         const unsigned flags = 0;
 203         const struct rte_memzone *mz;
 204
 205         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 206                 mz = rte_memzone_reserve(MZ_RTE_ETH_DEV_DATA,
 207                                 RTE_MAX_ETHPORTS * sizeof(*rte_eth_dev_data),
 208                                 rte_socket_id(), flags);
 209         } else
 210                 mz = rte_memzone_lookup(MZ_RTE_ETH_DEV_DATA);
 211         if (mz == NULL)
 212                 rte_panic("Cannot allocate memzone for ethernet port data\n");
 213
 214         rte_eth_dev_data = mz->addr;
 215         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 216                 memset(rte_eth_dev_data, 0,
 217                                 RTE_MAX_ETHPORTS * sizeof(*rte_eth_dev_data));
 218 }
 219
 220 struct rte_eth_dev *
 221 rte_eth_dev_allocated(const char *name)
 222 {
 223         unsigned i;
 224
 225         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 226                 if ((rte_eth_devices[i].attached == DEV_ATTACHED) &&
 227                     strcmp(rte_eth_devices[i].data->name, name) == 0)
 228                         return &rte_eth_devices[i];
 229         }
 230         return NULL;
 231 }
 232
 233 static uint8_t
 234 rte_eth_dev_find_free_port(void)
 235 {
 236         unsigned i;
 237
 238         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 239                 if (rte_eth_devices[i].attached == DEV_DETACHED)
 240                         return i;
 241         }
 242         return RTE_MAX_ETHPORTS;
 243 }
 244
 245 struct rte_eth_dev *
 246 rte_eth_dev_allocate(const char *name, enum rte_eth_dev_type type)
 247 {
 248         uint8_t port_id;
 249         struct rte_eth_dev *eth_dev;
 250
 251         port_id = rte_eth_dev_find_free_port();
 252         if (port_id == RTE_MAX_ETHPORTS) {
 253                 PMD_DEBUG_TRACE("Reached maximum number of Ethernet ports\n");
 254                 return NULL;
 255         }
 256
 257         if (rte_eth_dev_data == NULL)
 258                 rte_eth_dev_data_alloc();
 259
 260         if (rte_eth_dev_allocated(name) != NULL) {
 261                 PMD_DEBUG_TRACE("Ethernet Device with name %s already allocated!\n",
 262                                 name);
 263                 return NULL;
 264         }
 265
 266         eth_dev = &rte_eth_devices[port_id];
 267         eth_dev->data = &rte_eth_dev_data[port_id];
 268         snprintf(eth_dev->data->name, sizeof(eth_dev->data->name), "%s", name);
 269         eth_dev->data->port_id = port_id;
 270         eth_dev->attached = DEV_ATTACHED;
 271         eth_dev->dev_type = type;
 272         nb_ports++;
 273         return eth_dev;
 274 }
 275
 276 static int
 277 rte_eth_dev_create_unique_device_name(char *name, size_t size,
 278                 struct rte_pci_device *pci_dev)
 279 {
 280         int ret;
 281
 282         if ((name == NULL) || (pci_dev == NULL))
 283                 return -EINVAL;
 284
 285         ret = snprintf(name, size, "%d:%d.%d",
 286                         pci_dev->addr.bus, pci_dev->addr.devid,
 287                         pci_dev->addr.function);
 288         if (ret < 0)
 289                 return ret;
 290         return 0;
 291 }
 292
 293 int
 294 rte_eth_dev_release_port(struct rte_eth_dev *eth_dev)
 295 {
 296         if (eth_dev == NULL)
 297                 return -EINVAL;
 298
 299         eth_dev->attached = 0;
 300         nb_ports--;
 301         return 0;
 302 }
 303
 304 static int
 305 rte_eth_dev_init(struct rte_pci_driver *pci_drv,
 306                  struct rte_pci_device *pci_dev)
 307 {
 308         struct eth_driver    *eth_drv;
 309         struct rte_eth_dev *eth_dev;
 310         char ethdev_name[RTE_ETH_NAME_MAX_LEN];
 311
 312         int diag;
 313
 314         eth_drv = (struct eth_driver *)pci_drv;
 315
 316         /* Create unique Ethernet device name using PCI address */
 317         rte_eth_dev_create_unique_device_name(ethdev_name,
 318                         sizeof(ethdev_name), pci_dev);
 319
 320         eth_dev = rte_eth_dev_allocate(ethdev_name, RTE_ETH_DEV_PCI);
 321         if (eth_dev == NULL)
 322                 return -ENOMEM;
 323
 324         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 325                 eth_dev->data->dev_private = rte_zmalloc("ethdev private structure",
 326                                   eth_drv->dev_private_size,
 327                                   RTE_CACHE_LINE_SIZE);
 328                 if (eth_dev->data->dev_private == NULL)
 329                         rte_panic("Cannot allocate memzone for private port data\n");
 330         }
 331         eth_dev->pci_dev = pci_dev;
 332         eth_dev->driver = eth_drv;
 333         eth_dev->data->rx_mbuf_alloc_failed = 0;
 334
 335         /* init user callbacks */
 336         TAILQ_INIT(&(eth_dev->link_intr_cbs));
 337
 338         /*
 339          * Set the default MTU.
 340          */
 341         eth_dev->data->mtu = ETHER_MTU;
 342
 343         /* Invoke PMD device initialization function */
 344         diag = (*eth_drv->eth_dev_init)(eth_dev);
 345         if (diag == 0)
 346                 return 0;
 347
 348         PMD_DEBUG_TRACE("driver %s: eth_dev_init(vendor_id=0x%u device_id=0x%x) failed\n",
 349                         pci_drv->name,
 350                         (unsigned) pci_dev->id.vendor_id,
 351                         (unsigned) pci_dev->id.device_id);
 352         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 353                 rte_free(eth_dev->data->dev_private);
 354         eth_dev->attached = DEV_DETACHED;
 355         nb_ports--;
 356         return diag;
 357 }
 358
 359 static int
 360 rte_eth_dev_uninit(struct rte_pci_device *pci_dev)
 361 {
 362         const struct eth_driver *eth_drv;
 363         struct rte_eth_dev *eth_dev;
 364         char ethdev_name[RTE_ETH_NAME_MAX_LEN];
 365         int ret;
 366
 367         if (pci_dev == NULL)
 368                 return -EINVAL;
 369
 370         /* Create unique Ethernet device name using PCI address */
 371         rte_eth_dev_create_unique_device_name(ethdev_name,
 372                         sizeof(ethdev_name), pci_dev);
 373
 374         eth_dev = rte_eth_dev_allocated(ethdev_name);
 375         if (eth_dev == NULL)
 376                 return -ENODEV;
 377
 378         eth_drv = (const struct eth_driver *)pci_dev->driver;
 379
 380         /* Invoke PMD device uninit function */
 381         if (*eth_drv->eth_dev_uninit) {
 382                 ret = (*eth_drv->eth_dev_uninit)(eth_dev);
 383                 if (ret)
 384                         return ret;
 385         }
 386
 387         /* free ether device */
 388         rte_eth_dev_release_port(eth_dev);
 389
 390         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 391                 rte_free(eth_dev->data->dev_private);
 392
 393         eth_dev->pci_dev = NULL;
 394         eth_dev->driver = NULL;
 395         eth_dev->data = NULL;
 396
 397         return 0;
 398 }
 399
 400 /**
 401  * Register an Ethernet [Poll Mode] driver.
 402  *
 403  * Function invoked by the initialization function of an Ethernet driver
 404  * to simultaneously register itself as a PCI driver and as an Ethernet
 405  * Poll Mode Driver.
 406  * Invokes the rte_eal_pci_register() function to register the *pci_drv*
 407  * structure embedded in the *eth_drv* structure, after having stored the
 408  * address of the rte_eth_dev_init() function in the *devinit* field of
 409  * the *pci_drv* structure.
 410  * During the PCI probing phase, the rte_eth_dev_init() function is
 411  * invoked for each PCI [Ethernet device] matching the embedded PCI
 412  * identifiers provided by the driver.
 413  */
 414 void
 415 rte_eth_driver_register(struct eth_driver *eth_drv)
 416 {
 417         eth_drv->pci_drv.devinit = rte_eth_dev_init;
 418         eth_drv->pci_drv.devuninit = rte_eth_dev_uninit;
 419         rte_eal_pci_register(&eth_drv->pci_drv);
 420 }
 421
 422 static int
 423 rte_eth_dev_is_valid_port(uint8_t port_id)
 424 {
 425         if (port_id >= RTE_MAX_ETHPORTS ||
 426             rte_eth_devices[port_id].attached != DEV_ATTACHED)
 427                 return 0;
 428         else
 429                 return 1;
 430 }
 431
 432 int
 433 rte_eth_dev_socket_id(uint8_t port_id)
 434 {
 435         if (!rte_eth_dev_is_valid_port(port_id))
 436                 return -1;
 437         return rte_eth_devices[port_id].pci_dev->numa_node;
 438 }
 439
 440 uint8_t
 441 rte_eth_dev_count(void)
 442 {
 443         return nb_ports;
 444 }
 445
 446 static enum rte_eth_dev_type
 447 rte_eth_dev_get_device_type(uint8_t port_id)
 448 {
 449         if (!rte_eth_dev_is_valid_port(port_id))
 450                 return RTE_ETH_DEV_UNKNOWN;
 451         return rte_eth_devices[port_id].dev_type;
 452 }
 453
 454 static int
 455 rte_eth_dev_save(struct rte_eth_dev *devs, size_t size)
 456 {
 457         if ((devs == NULL) ||
 458             (size != sizeof(struct rte_eth_dev) * RTE_MAX_ETHPORTS))
 459                 return -EINVAL;
 460
 461         /* save current rte_eth_devices */
 462         memcpy(devs, rte_eth_devices, size);
 463         return 0;
 464 }
 465
 466 static int
 467 rte_eth_dev_get_changed_port(struct rte_eth_dev *devs, uint8_t *port_id)
 468 {
 469         if ((devs == NULL) || (port_id == NULL))
 470                 return -EINVAL;
 471
 472         /* check which port was attached or detached */
 473         for (*port_id = 0; *port_id < RTE_MAX_ETHPORTS; (*port_id)++, devs++) {
 474                 if (rte_eth_devices[*port_id].attached ^ devs->attached)
 475                         return 0;
 476         }
 477         return -ENODEV;
 478 }
 479
 480 static int
 481 rte_eth_dev_get_addr_by_port(uint8_t port_id, struct rte_pci_addr *addr)
 482 {
 483         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 484
 485         if (addr == NULL) {
 486                 PMD_DEBUG_TRACE("Null pointer is specified\n");
 487                 return -EINVAL;
 488         }
 489
 490         *addr = rte_eth_devices[port_id].pci_dev->addr;
 491         return 0;
 492 }
 493
 494 static int
 495 rte_eth_dev_get_name_by_port(uint8_t port_id, char *name)
 496 {
 497         char *tmp;
 498
 499         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 500
 501         if (name == NULL) {
 502                 PMD_DEBUG_TRACE("Null pointer is specified\n");
 503                 return -EINVAL;
 504         }
 505
 506         /* shouldn't check 'rte_eth_devices[i].data',
 507          * because it might be overwritten by VDEV PMD */
 508         tmp = rte_eth_dev_data[port_id].name;
 509         strcpy(name, tmp);
 510         return 0;
 511 }
 512
 513 static int
 514 rte_eth_dev_is_detachable(uint8_t port_id)
 515 {
 516         uint32_t drv_flags;
 517
 518         if (port_id >= RTE_MAX_ETHPORTS) {
 519                 PMD_DEBUG_TRACE("Invalid port_id=%d\n", port_id);
 520                 return -EINVAL;
 521         }
 522
 523         if (rte_eth_devices[port_id].dev_type == RTE_ETH_DEV_PCI) {
 524                 switch (rte_eth_devices[port_id].pci_dev->kdrv) {
 525                 case RTE_KDRV_IGB_UIO:
 526                 case RTE_KDRV_UIO_GENERIC:
 527                 case RTE_KDRV_NIC_UIO:
 528                         break;
 529                 case RTE_KDRV_VFIO:
 530                 default:
 531                         return -ENOTSUP;
 532                 }
 533         }
 534
 535         drv_flags = rte_eth_devices[port_id].driver->pci_drv.drv_flags;
 536         return !(drv_flags & RTE_PCI_DRV_DETACHABLE);
 537 }
 538
 539 /* attach the new physical device, then store port_id of the device */
 540 static int
 541 rte_eth_dev_attach_pdev(struct rte_pci_addr *addr, uint8_t *port_id)
 542 {
 543         uint8_t new_port_id;
 544         struct rte_eth_dev devs[RTE_MAX_ETHPORTS];
 545
 546         if ((addr == NULL) || (port_id == NULL))
 547                 goto err;
 548
 549         /* save current port status */
 550         if (rte_eth_dev_save(devs, sizeof(devs)))
 551                 goto err;
 552         /* re-construct pci_device_list */
 553         if (rte_eal_pci_scan())
 554                 goto err;
 555         /* invoke probe func of the driver can handle the new device.
 556          * TODO:
 557          * rte_eal_pci_probe_one() should return port_id.
 558          * And rte_eth_dev_save() and rte_eth_dev_get_changed_port()
 559          * should be removed. */
 560         if (rte_eal_pci_probe_one(addr))
 561                 goto err;
 562         /* get port_id enabled by above procedures */
 563         if (rte_eth_dev_get_changed_port(devs, &new_port_id))
 564                 goto err;
 565
 566         *port_id = new_port_id;
 567         return 0;
 568 err:
 569         RTE_LOG(ERR, EAL, "Driver, cannot attach the device\n");
 570         return -1;
 571 }
 572
 573 /* detach the new physical device, then store pci_addr of the device */
 574 static int
 575 rte_eth_dev_detach_pdev(uint8_t port_id, struct rte_pci_addr *addr)
 576 {
 577         struct rte_pci_addr freed_addr;
 578         struct rte_pci_addr vp;
 579
 580         if (addr == NULL)
 581                 goto err;
 582
 583         /* check whether the driver supports detach feature, or not */
 584         if (rte_eth_dev_is_detachable(port_id))
 585                 goto err;
 586
 587         /* get pci address by port id */
 588         if (rte_eth_dev_get_addr_by_port(port_id, &freed_addr))
 589                 goto err;
 590
 591         /* Zeroed pci addr means the port comes from virtual device */
 592         vp.domain = vp.bus = vp.devid = vp.function = 0;
 593         if (rte_eal_compare_pci_addr(&vp, &freed_addr) == 0)
 594                 goto err;
 595
 596         /* invoke close func of the driver,
 597          * also remove the device from pci_device_list */
 598         if (rte_eal_pci_close_one(&freed_addr))
 599                 goto err;
 600
 601         *addr = freed_addr;
 602         return 0;
 603 err:
 604         RTE_LOG(ERR, EAL, "Driver, cannot detach the device\n");
 605         return -1;
 606 }
 607
 608 /* attach the new virtual device, then store port_id of the device */
 609 static int
 610 rte_eth_dev_attach_vdev(const char *vdevargs, uint8_t *port_id)
 611 {
 612         char *name = NULL, *args = NULL;
 613         uint8_t new_port_id;
 614         struct rte_eth_dev devs[RTE_MAX_ETHPORTS];
 615         int ret = -1;
 616
 617         if ((vdevargs == NULL) || (port_id == NULL))
 618                 goto end;
 619
 620         /* parse vdevargs, then retrieve device name and args */
 621         if (rte_eal_parse_devargs_str(vdevargs, &name, &args))
 622                 goto end;
 623
 624         /* save current port status */
 625         if (rte_eth_dev_save(devs, sizeof(devs)))
 626                 goto end;
 627         /* walk around dev_driver_list to find the driver of the device,
 628          * then invoke probe function o the driver.
 629          * TODO:
 630          * rte_eal_vdev_init() should return port_id,
 631          * And rte_eth_dev_save() and rte_eth_dev_get_changed_port()
 632          * should be removed. */
 633         if (rte_eal_vdev_init(name, args))
 634                 goto end;
 635         /* get port_id enabled by above procedures */
 636         if (rte_eth_dev_get_changed_port(devs, &new_port_id))
 637                 goto end;
 638         ret = 0;
 639         *port_id = new_port_id;
 640 end:
 641         if (name)
 642                 free(name);
 643         if (args)
 644                 free(args);
 645
 646         if (ret < 0)
 647                 RTE_LOG(ERR, EAL, "Driver, cannot attach the device\n");
 648         return ret;
 649 }
 650
 651 /* detach the new virtual device, then store the name of the device */
 652 static int
 653 rte_eth_dev_detach_vdev(uint8_t port_id, char *vdevname)
 654 {
 655         char name[RTE_ETH_NAME_MAX_LEN];
 656
 657         if (vdevname == NULL)
 658                 goto err;
 659
 660         /* check whether the driver supports detach feature, or not */
 661         if (rte_eth_dev_is_detachable(port_id))
 662                 goto err;
 663
 664         /* get device name by port id */
 665         if (rte_eth_dev_get_name_by_port(port_id, name))
 666                 goto err;
 667         /* walk around dev_driver_list to find the driver of the device,
 668          * then invoke close function o the driver */
 669         if (rte_eal_vdev_uninit(name))
 670                 goto err;
 671
 672         strncpy(vdevname, name, sizeof(name));
 673         return 0;
 674 err:
 675         RTE_LOG(ERR, EAL, "Driver, cannot detach the device\n");
 676         return -1;
 677 }
 678
 679 /* attach the new device, then store port_id of the device */
 680 int
 681 rte_eth_dev_attach(const char *devargs, uint8_t *port_id)
 682 {
 683         struct rte_pci_addr addr;
 684
 685         if ((devargs == NULL) || (port_id == NULL))
 686                 return -EINVAL;
 687
 688         if (eal_parse_pci_DomBDF(devargs, &addr) == 0)
 689                 return rte_eth_dev_attach_pdev(&addr, port_id);
 690         else
 691                 return rte_eth_dev_attach_vdev(devargs, port_id);
 692 }
 693
 694 /* detach the device, then store the name of the device */
 695 int
 696 rte_eth_dev_detach(uint8_t port_id, char *name)
 697 {
 698         struct rte_pci_addr addr;
 699         int ret;
 700
 701         if (name == NULL)
 702                 return -EINVAL;
 703
 704         if (rte_eth_dev_get_device_type(port_id) == RTE_ETH_DEV_PCI) {
 705                 ret = rte_eth_dev_get_addr_by_port(port_id, &addr);
 706                 if (ret < 0)
 707                         return ret;
 708
 709                 ret = rte_eth_dev_detach_pdev(port_id, &addr);
 710                 if (ret == 0)
 711                         snprintf(name, RTE_ETH_NAME_MAX_LEN,
 712                                 "%04x:%02x:%02x.%d",
 713                                 addr.domain, addr.bus,
 714                                 addr.devid, addr.function);
 715
 716                 return ret;
 717         } else
 718                 return rte_eth_dev_detach_vdev(port_id, name);
 719 }
 720
 721 static int
 722 rte_eth_dev_rx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
 723 {
 724         uint16_t old_nb_queues = dev->data->nb_rx_queues;
 725         void **rxq;
 726         unsigned i;
 727
 728         if (dev->data->rx_queues == NULL) { /* first time configuration */
 729                 dev->data->rx_queues = rte_zmalloc("ethdev->rx_queues",
 730                                 sizeof(dev->data->rx_queues[0]) * nb_queues,
 731                                 RTE_CACHE_LINE_SIZE);
 732                 if (dev->data->rx_queues == NULL) {
 733                         dev->data->nb_rx_queues = 0;
 734                         return -(ENOMEM);
 735                 }
 736         } else { /* re-configure */
 737                 FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release, -ENOTSUP);
 738
 739                 rxq = dev->data->rx_queues;
 740
 741                 for (i = nb_queues; i < old_nb_queues; i++)
 742                         (*dev->dev_ops->rx_queue_release)(rxq[i]);
 743                 rxq = rte_realloc(rxq, sizeof(rxq[0]) * nb_queues,
 744                                 RTE_CACHE_LINE_SIZE);
 745                 if (rxq == NULL)
 746                         return -(ENOMEM);
 747                 if (nb_queues > old_nb_queues) {
 748                         uint16_t new_qs = nb_queues - old_nb_queues;
 749
 750                         memset(rxq + old_nb_queues, 0,
 751                                 sizeof(rxq[0]) * new_qs);
 752                 }
 753
 754                 dev->data->rx_queues = rxq;
 755
 756         }
 757         dev->data->nb_rx_queues = nb_queues;
 758         return 0;
 759 }
 760
 761 int
 762 rte_eth_dev_rx_queue_start(uint8_t port_id, uint16_t rx_queue_id)
 763 {
 764         struct rte_eth_dev *dev;
 765
 766         /* This function is only safe when called from the primary process
 767          * in a multi-process setup*/
 768         PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
 769
 770         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 771
 772         dev = &rte_eth_devices[port_id];
 773         if (rx_queue_id >= dev->data->nb_rx_queues) {
 774                 PMD_DEBUG_TRACE("Invalid RX queue_id=%d\n", rx_queue_id);
 775                 return -EINVAL;
 776         }
 777
 778         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_start, -ENOTSUP);
 779
 780         return dev->dev_ops->rx_queue_start(dev, rx_queue_id);
 781
 782 }
 783
 784 int
 785 rte_eth_dev_rx_queue_stop(uint8_t port_id, uint16_t rx_queue_id)
 786 {
 787         struct rte_eth_dev *dev;
 788
 789         /* This function is only safe when called from the primary process
 790          * in a multi-process setup*/
 791         PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
 792
 793         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 794
 795         dev = &rte_eth_devices[port_id];
 796         if (rx_queue_id >= dev->data->nb_rx_queues) {
 797                 PMD_DEBUG_TRACE("Invalid RX queue_id=%d\n", rx_queue_id);
 798                 return -EINVAL;
 799         }
 800
 801         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_stop, -ENOTSUP);
 802
 803         return dev->dev_ops->rx_queue_stop(dev, rx_queue_id);
 804
 805 }
 806
 807 int
 808 rte_eth_dev_tx_queue_start(uint8_t port_id, uint16_t tx_queue_id)
 809 {
 810         struct rte_eth_dev *dev;
 811
 812         /* This function is only safe when called from the primary process
 813          * in a multi-process setup*/
 814         PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
 815
 816         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 817
 818         dev = &rte_eth_devices[port_id];
 819         if (tx_queue_id >= dev->data->nb_tx_queues) {
 820                 PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", tx_queue_id);
 821                 return -EINVAL;
 822         }
 823
 824         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_start, -ENOTSUP);
 825
 826         return dev->dev_ops->tx_queue_start(dev, tx_queue_id);
 827
 828 }
 829
 830 int
 831 rte_eth_dev_tx_queue_stop(uint8_t port_id, uint16_t tx_queue_id)
 832 {
 833         struct rte_eth_dev *dev;
 834
 835         /* This function is only safe when called from the primary process
 836          * in a multi-process setup*/
 837         PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
 838
 839         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 840
 841         dev = &rte_eth_devices[port_id];
 842         if (tx_queue_id >= dev->data->nb_tx_queues) {
 843                 PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", tx_queue_id);
 844                 return -EINVAL;
 845         }
 846
 847         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_stop, -ENOTSUP);
 848
 849         return dev->dev_ops->tx_queue_stop(dev, tx_queue_id);
 850
 851 }
 852
 853 static int
 854 rte_eth_dev_tx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
 855 {
 856         uint16_t old_nb_queues = dev->data->nb_tx_queues;
 857         void **txq;
 858         unsigned i;
 859
 860         if (dev->data->tx_queues == NULL) { /* first time configuration */
 861                 dev->data->tx_queues = rte_zmalloc("ethdev->tx_queues",
 862                                                    sizeof(dev->data->tx_queues[0]) * nb_queues,
 863                                                    RTE_CACHE_LINE_SIZE);
 864                 if (dev->data->tx_queues == NULL) {
 865                         dev->data->nb_tx_queues = 0;
 866                         return -(ENOMEM);
 867                 }
 868         } else { /* re-configure */
 869                 FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release, -ENOTSUP);
 870
 871                 txq = dev->data->tx_queues;
 872
 873                 for (i = nb_queues; i < old_nb_queues; i++)
 874                         (*dev->dev_ops->tx_queue_release)(txq[i]);
 875                 txq = rte_realloc(txq, sizeof(txq[0]) * nb_queues,
 876                                   RTE_CACHE_LINE_SIZE);
 877                 if (txq == NULL)
 878                         return -ENOMEM;
 879                 if (nb_queues > old_nb_queues) {
 880                         uint16_t new_qs = nb_queues - old_nb_queues;
 881
 882                         memset(txq + old_nb_queues, 0,
 883                                sizeof(txq[0]) * new_qs);
 884                 }
 885
 886                 dev->data->tx_queues = txq;
 887
 888         }
 889         dev->data->nb_tx_queues = nb_queues;
 890         return 0;
 891 }
 892
 893 static int
 894 rte_eth_dev_check_vf_rss_rxq_num(uint8_t port_id, uint16_t nb_rx_q)
 895 {
 896         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 897
 898         switch (nb_rx_q) {
 899         case 1:
 900         case 2:
 901                 RTE_ETH_DEV_SRIOV(dev).active =
 902                         ETH_64_POOLS;
 903                 break;
 904         case 4:
 905                 RTE_ETH_DEV_SRIOV(dev).active =
 906                         ETH_32_POOLS;
 907                 break;
 908         default:
 909                 return -EINVAL;
 910         }
 911
 912         RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = nb_rx_q;
 913         RTE_ETH_DEV_SRIOV(dev).def_pool_q_idx =
 914                 dev->pci_dev->max_vfs * nb_rx_q;
 915
 916         return 0;
 917 }
 918
 919 static int
 920 rte_eth_dev_check_mq_mode(uint8_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
 921                           const struct rte_eth_conf *dev_conf)
 922 {
 923         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
 924
 925         if (RTE_ETH_DEV_SRIOV(dev).active != 0) {
 926                 /* check multi-queue mode */
 927                 if ((dev_conf->rxmode.mq_mode == ETH_MQ_RX_DCB) ||
 928                     (dev_conf->rxmode.mq_mode == ETH_MQ_RX_DCB_RSS) ||
 929                     (dev_conf->txmode.mq_mode == ETH_MQ_TX_DCB)) {
 930                         /* SRIOV only works in VMDq enable mode */
 931                         PMD_DEBUG_TRACE("ethdev port_id=%" PRIu8
 932                                         " SRIOV active, "
 933                                         "wrong VMDQ mq_mode rx %u tx %u\n",
 934                                         port_id,
 935                                         dev_conf->rxmode.mq_mode,
 936                                         dev_conf->txmode.mq_mode);
 937                         return -EINVAL;
 938                 }
 939
 940                 switch (dev_conf->rxmode.mq_mode) {
 941                 case ETH_MQ_RX_VMDQ_DCB:
 942                 case ETH_MQ_RX_VMDQ_DCB_RSS:
 943                         /* DCB/RSS VMDQ in SRIOV mode, not implement yet */
 944                         PMD_DEBUG_TRACE("ethdev port_id=%" PRIu8
 945                                         " SRIOV active, "
 946                                         "unsupported VMDQ mq_mode rx %u\n",
 947                                         port_id, dev_conf->rxmode.mq_mode);
 948                         return -EINVAL;
 949                 case ETH_MQ_RX_RSS:
 950                         PMD_DEBUG_TRACE("ethdev port_id=%" PRIu8
 951                                         " SRIOV active, "
 952                                         "Rx mq mode is changed from:"
 953                                         "mq_mode %u into VMDQ mq_mode %u\n",
 954                                         port_id,
 955                                         dev_conf->rxmode.mq_mode,
 956                                         dev->data->dev_conf.rxmode.mq_mode);
 957                 case ETH_MQ_RX_VMDQ_RSS:
 958                         dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_RSS;
 959                         if (nb_rx_q <= RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool)
 960                                 if (rte_eth_dev_check_vf_rss_rxq_num(port_id, nb_rx_q) != 0) {
 961                                         PMD_DEBUG_TRACE("ethdev port_id=%d"
 962                                                         " SRIOV active, invalid queue"
 963                                                         " number for VMDQ RSS, allowed"
 964                                                         " value are 1, 2 or 4\n",
 965                                                         port_id);
 966                                         return -EINVAL;
 967                                 }
 968                         break;
 969                 default: /* ETH_MQ_RX_VMDQ_ONLY or ETH_MQ_RX_NONE */
 970                         /* if nothing mq mode configure, use default scheme */
 971                         dev->data->dev_conf.rxmode.mq_mode = ETH_MQ_RX_VMDQ_ONLY;
 972                         if (RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool > 1)
 973                                 RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool = 1;
 974                         break;
 975                 }
 976
 977                 switch (dev_conf->txmode.mq_mode) {
 978                 case ETH_MQ_TX_VMDQ_DCB:
 979                         /* DCB VMDQ in SRIOV mode, not implement yet */
 980                         PMD_DEBUG_TRACE("ethdev port_id=%" PRIu8
 981                                         " SRIOV active, "
 982                                         "unsupported VMDQ mq_mode tx %u\n",
 983                                         port_id, dev_conf->txmode.mq_mode);
 984                         return -EINVAL;
 985                 default: /* ETH_MQ_TX_VMDQ_ONLY or ETH_MQ_TX_NONE */
 986                         /* if nothing mq mode configure, use default scheme */
 987                         dev->data->dev_conf.txmode.mq_mode = ETH_MQ_TX_VMDQ_ONLY;
 988                         break;
 989                 }
 990
 991                 /* check valid queue number */
 992                 if ((nb_rx_q > RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool) ||
 993                     (nb_tx_q > RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool)) {
 994                         PMD_DEBUG_TRACE("ethdev port_id=%d SRIOV active, "
 995                                         "queue number must less equal to %d\n",
 996                                         port_id,
 997                                         RTE_ETH_DEV_SRIOV(dev).nb_q_per_pool);
 998                         return -EINVAL;
 999                 }
1000         } else {
1001                 /* For vmdb+dcb mode check our configuration before we go further */
1002                 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_VMDQ_DCB) {
1003                         const struct rte_eth_vmdq_dcb_conf *conf;
1004
1005                         if (nb_rx_q != ETH_VMDQ_DCB_NUM_QUEUES) {
1006                                 PMD_DEBUG_TRACE("ethdev port_id=%d VMDQ+DCB, nb_rx_q "
1007                                                 "!= %d\n",
1008                                                 port_id, ETH_VMDQ_DCB_NUM_QUEUES);
1009                                 return -EINVAL;
1010                         }
1011                         conf = &(dev_conf->rx_adv_conf.vmdq_dcb_conf);
1012                         if (!(conf->nb_queue_pools == ETH_16_POOLS ||
1013                               conf->nb_queue_pools == ETH_32_POOLS)) {
1014                                 PMD_DEBUG_TRACE("ethdev port_id=%d VMDQ+DCB selected, "
1015                                                 "nb_queue_pools must be %d or %d\n",
1016                                                 port_id, ETH_16_POOLS, ETH_32_POOLS);
1017                                 return -EINVAL;
1018                         }
1019                 }
1020                 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_VMDQ_DCB) {
1021                         const struct rte_eth_vmdq_dcb_tx_conf *conf;
1022
1023                         if (nb_tx_q != ETH_VMDQ_DCB_NUM_QUEUES) {
1024                                 PMD_DEBUG_TRACE("ethdev port_id=%d VMDQ+DCB, nb_tx_q "
1025                                                 "!= %d\n",
1026                                                 port_id, ETH_VMDQ_DCB_NUM_QUEUES);
1027                                 return -EINVAL;
1028                         }
1029                         conf = &(dev_conf->tx_adv_conf.vmdq_dcb_tx_conf);
1030                         if (!(conf->nb_queue_pools == ETH_16_POOLS ||
1031                               conf->nb_queue_pools == ETH_32_POOLS)) {
1032                                 PMD_DEBUG_TRACE("ethdev port_id=%d VMDQ+DCB selected, "
1033                                                 "nb_queue_pools != %d or nb_queue_pools "
1034                                                 "!= %d\n",
1035                                                 port_id, ETH_16_POOLS, ETH_32_POOLS);
1036                                 return -EINVAL;
1037                         }
1038                 }
1039
1040                 /* For DCB mode check our configuration before we go further */
1041                 if (dev_conf->rxmode.mq_mode == ETH_MQ_RX_DCB) {
1042                         const struct rte_eth_dcb_rx_conf *conf;
1043
1044                         if (nb_rx_q != ETH_DCB_NUM_QUEUES) {
1045                                 PMD_DEBUG_TRACE("ethdev port_id=%d DCB, nb_rx_q "
1046                                                 "!= %d\n",
1047                                                 port_id, ETH_DCB_NUM_QUEUES);
1048                                 return -EINVAL;
1049                         }
1050                         conf = &(dev_conf->rx_adv_conf.dcb_rx_conf);
1051                         if (!(conf->nb_tcs == ETH_4_TCS ||
1052                               conf->nb_tcs == ETH_8_TCS)) {
1053                                 PMD_DEBUG_TRACE("ethdev port_id=%d DCB selected, "
1054                                                 "nb_tcs != %d or nb_tcs "
1055                                                 "!= %d\n",
1056                                                 port_id, ETH_4_TCS, ETH_8_TCS);
1057                                 return -EINVAL;
1058                         }
1059                 }
1060
1061                 if (dev_conf->txmode.mq_mode == ETH_MQ_TX_DCB) {
1062                         const struct rte_eth_dcb_tx_conf *conf;
1063
1064                         if (nb_tx_q != ETH_DCB_NUM_QUEUES) {
1065                                 PMD_DEBUG_TRACE("ethdev port_id=%d DCB, nb_tx_q "
1066                                                 "!= %d\n",
1067                                                 port_id, ETH_DCB_NUM_QUEUES);
1068                                 return -EINVAL;
1069                         }
1070                         conf = &(dev_conf->tx_adv_conf.dcb_tx_conf);
1071                         if (!(conf->nb_tcs == ETH_4_TCS ||
1072                               conf->nb_tcs == ETH_8_TCS)) {
1073                                 PMD_DEBUG_TRACE("ethdev port_id=%d DCB selected, "
1074                                                 "nb_tcs != %d or nb_tcs "
1075                                                 "!= %d\n",
1076                                                 port_id, ETH_4_TCS, ETH_8_TCS);
1077                                 return -EINVAL;
1078                         }
1079                 }
1080         }
1081         return 0;
1082 }
1083
1084 int
1085 rte_eth_dev_configure(uint8_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
1086                       const struct rte_eth_conf *dev_conf)
1087 {
1088         struct rte_eth_dev *dev;
1089         struct rte_eth_dev_info dev_info;
1090         int diag;
1091
1092         /* This function is only safe when called from the primary process
1093          * in a multi-process setup*/
1094         PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
1095
1096         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1097
1098         if (nb_rx_q > RTE_MAX_QUEUES_PER_PORT) {
1099                 PMD_DEBUG_TRACE(
1100                         "Number of RX queues requested (%u) is greater than max supported(%d)\n",
1101                         nb_rx_q, RTE_MAX_QUEUES_PER_PORT);
1102                 return -EINVAL;
1103         }
1104
1105         if (nb_tx_q > RTE_MAX_QUEUES_PER_PORT) {
1106                 PMD_DEBUG_TRACE(
1107                         "Number of TX queues requested (%u) is greater than max supported(%d)\n",
1108                         nb_tx_q, RTE_MAX_QUEUES_PER_PORT);
1109                 return -EINVAL;
1110         }
1111
1112         dev = &rte_eth_devices[port_id];
1113
1114         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
1115         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
1116
1117         if (dev->data->dev_started) {
1118                 PMD_DEBUG_TRACE(
1119                     "port %d must be stopped to allow configuration\n", port_id);
1120                 return -EBUSY;
1121         }
1122
1123         /*
1124          * Check that the numbers of RX and TX queues are not greater
1125          * than the maximum number of RX and TX queues supported by the
1126          * configured device.
1127          */
1128         (*dev->dev_ops->dev_infos_get)(dev, &dev_info);
1129         if (nb_rx_q > dev_info.max_rx_queues) {
1130                 PMD_DEBUG_TRACE("ethdev port_id=%d nb_rx_queues=%d > %d\n",
1131                                 port_id, nb_rx_q, dev_info.max_rx_queues);
1132                 return -EINVAL;
1133         }
1134         if (nb_rx_q == 0) {
1135                 PMD_DEBUG_TRACE("ethdev port_id=%d nb_rx_q == 0\n", port_id);
1136                 return -EINVAL;
1137         }
1138
1139         if (nb_tx_q > dev_info.max_tx_queues) {
1140                 PMD_DEBUG_TRACE("ethdev port_id=%d nb_tx_queues=%d > %d\n",
1141                                 port_id, nb_tx_q, dev_info.max_tx_queues);
1142                 return -EINVAL;
1143         }
1144         if (nb_tx_q == 0) {
1145                 PMD_DEBUG_TRACE("ethdev port_id=%d nb_tx_q == 0\n", port_id);
1146                 return -EINVAL;
1147         }
1148
1149         /* Copy the dev_conf parameter into the dev structure */
1150         memcpy(&dev->data->dev_conf, dev_conf, sizeof(dev->data->dev_conf));
1151
1152         /*
1153          * If link state interrupt is enabled, check that the
1154          * device supports it.
1155          */
1156         if (dev_conf->intr_conf.lsc == 1) {
1157                 const struct rte_pci_driver *pci_drv = &dev->driver->pci_drv;
1158
1159                 if (!(pci_drv->drv_flags & RTE_PCI_DRV_INTR_LSC)) {
1160                         PMD_DEBUG_TRACE("driver %s does not support lsc\n",
1161                                         pci_drv->name);
1162                         return -EINVAL;
1163                 }
1164         }
1165
1166         /*
1167          * If jumbo frames are enabled, check that the maximum RX packet
1168          * length is supported by the configured device.
1169          */
1170         if (dev_conf->rxmode.jumbo_frame == 1) {
1171                 if (dev_conf->rxmode.max_rx_pkt_len >
1172                     dev_info.max_rx_pktlen) {
1173                         PMD_DEBUG_TRACE("ethdev port_id=%d max_rx_pkt_len %u"
1174                                 " > max valid value %u\n",
1175                                 port_id,
1176                                 (unsigned)dev_conf->rxmode.max_rx_pkt_len,
1177                                 (unsigned)dev_info.max_rx_pktlen);
1178                         return -EINVAL;
1179                 } else if (dev_conf->rxmode.max_rx_pkt_len < ETHER_MIN_LEN) {
1180                         PMD_DEBUG_TRACE("ethdev port_id=%d max_rx_pkt_len %u"
1181                                 " < min valid value %u\n",
1182                                 port_id,
1183                                 (unsigned)dev_conf->rxmode.max_rx_pkt_len,
1184                                 (unsigned)ETHER_MIN_LEN);
1185                         return -EINVAL;
1186                 }
1187         } else {
1188                 if (dev_conf->rxmode.max_rx_pkt_len < ETHER_MIN_LEN ||
1189                         dev_conf->rxmode.max_rx_pkt_len > ETHER_MAX_LEN)
1190                         /* Use default value */
1191                         dev->data->dev_conf.rxmode.max_rx_pkt_len =
1192                                                         ETHER_MAX_LEN;
1193         }
1194
1195         /* multiple queue mode checking */
1196         diag = rte_eth_dev_check_mq_mode(port_id, nb_rx_q, nb_tx_q, dev_conf);
1197         if (diag != 0) {
1198                 PMD_DEBUG_TRACE("port%d rte_eth_dev_check_mq_mode = %d\n",
1199                                 port_id, diag);
1200                 return diag;
1201         }
1202
1203         /*
1204          * Setup new number of RX/TX queues and reconfigure device.
1205          */
1206         diag = rte_eth_dev_rx_queue_config(dev, nb_rx_q);
1207         if (diag != 0) {
1208                 PMD_DEBUG_TRACE("port%d rte_eth_dev_rx_queue_config = %d\n",
1209                                 port_id, diag);
1210                 return diag;
1211         }
1212
1213         diag = rte_eth_dev_tx_queue_config(dev, nb_tx_q);
1214         if (diag != 0) {
1215                 PMD_DEBUG_TRACE("port%d rte_eth_dev_tx_queue_config = %d\n",
1216                                 port_id, diag);
1217                 rte_eth_dev_rx_queue_config(dev, 0);
1218                 return diag;
1219         }
1220
1221         diag = (*dev->dev_ops->dev_configure)(dev);
1222         if (diag != 0) {
1223                 PMD_DEBUG_TRACE("port%d dev_configure = %d\n",
1224                                 port_id, diag);
1225                 rte_eth_dev_rx_queue_config(dev, 0);
1226                 rte_eth_dev_tx_queue_config(dev, 0);
1227                 return diag;
1228         }
1229
1230         return 0;
1231 }
1232
1233 static void
1234 rte_eth_dev_config_restore(uint8_t port_id)
1235 {
1236         struct rte_eth_dev *dev;
1237         struct rte_eth_dev_info dev_info;
1238         struct ether_addr addr;
1239         uint16_t i;
1240         uint32_t pool = 0;
1241
1242         dev = &rte_eth_devices[port_id];
1243
1244         rte_eth_dev_info_get(port_id, &dev_info);
1245
1246         if (RTE_ETH_DEV_SRIOV(dev).active)
1247                 pool = RTE_ETH_DEV_SRIOV(dev).def_vmdq_idx;
1248
1249         /* replay MAC address configuration */
1250         for (i = 0; i < dev_info.max_mac_addrs; i++) {
1251                 addr = dev->data->mac_addrs[i];
1252
1253                 /* skip zero address */
1254                 if (is_zero_ether_addr(&addr))
1255                         continue;
1256
1257                 /* add address to the hardware */
1258                 if  (*dev->dev_ops->mac_addr_add &&
1259                         (dev->data->mac_pool_sel[i] & (1ULL << pool)))
1260                         (*dev->dev_ops->mac_addr_add)(dev, &addr, i, pool);
1261                 else {
1262                         PMD_DEBUG_TRACE("port %d: MAC address array not supported\n",
1263                                         port_id);
1264                         /* exit the loop but not return an error */
1265                         break;
1266                 }
1267         }
1268
1269         /* replay promiscuous configuration */
1270         if (rte_eth_promiscuous_get(port_id) == 1)
1271                 rte_eth_promiscuous_enable(port_id);
1272         else if (rte_eth_promiscuous_get(port_id) == 0)
1273                 rte_eth_promiscuous_disable(port_id);
1274
1275         /* replay all multicast configuration */
1276         if (rte_eth_allmulticast_get(port_id) == 1)
1277                 rte_eth_allmulticast_enable(port_id);
1278         else if (rte_eth_allmulticast_get(port_id) == 0)
1279                 rte_eth_allmulticast_disable(port_id);
1280 }
1281
1282 int
1283 rte_eth_dev_start(uint8_t port_id)
1284 {
1285         struct rte_eth_dev *dev;
1286         int diag;
1287
1288         /* This function is only safe when called from the primary process
1289          * in a multi-process setup*/
1290         PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
1291
1292         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1293
1294         dev = &rte_eth_devices[port_id];
1295
1296         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_start, -ENOTSUP);
1297
1298         if (dev->data->dev_started != 0) {
1299                 PMD_DEBUG_TRACE("Device with port_id=%" PRIu8
1300                         " already started\n",
1301                         port_id);
1302                 return 0;
1303         }
1304
1305         diag = (*dev->dev_ops->dev_start)(dev);
1306         if (diag == 0)
1307                 dev->data->dev_started = 1;
1308         else
1309                 return diag;
1310
1311         rte_eth_dev_config_restore(port_id);
1312
1313         if (dev->data->dev_conf.intr_conf.lsc != 0) {
1314                 FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
1315                 (*dev->dev_ops->link_update)(dev, 0);
1316         }
1317         return 0;
1318 }
1319
1320 void
1321 rte_eth_dev_stop(uint8_t port_id)
1322 {
1323         struct rte_eth_dev *dev;
1324
1325         /* This function is only safe when called from the primary process
1326          * in a multi-process setup*/
1327         PROC_PRIMARY_OR_RET();
1328
1329         VALID_PORTID_OR_RET(port_id);
1330         dev = &rte_eth_devices[port_id];
1331
1332         FUNC_PTR_OR_RET(*dev->dev_ops->dev_stop);
1333
1334         if (dev->data->dev_started == 0) {
1335                 PMD_DEBUG_TRACE("Device with port_id=%" PRIu8
1336                         " already stopped\n",
1337                         port_id);
1338                 return;
1339         }
1340
1341         dev->data->dev_started = 0;
1342         (*dev->dev_ops->dev_stop)(dev);
1343 }
1344
1345 int
1346 rte_eth_dev_set_link_up(uint8_t port_id)
1347 {
1348         struct rte_eth_dev *dev;
1349
1350         /* This function is only safe when called from the primary process
1351          * in a multi-process setup*/
1352         PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
1353
1354         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1355
1356         dev = &rte_eth_devices[port_id];
1357
1358         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_up, -ENOTSUP);
1359         return (*dev->dev_ops->dev_set_link_up)(dev);
1360 }
1361
1362 int
1363 rte_eth_dev_set_link_down(uint8_t port_id)
1364 {
1365         struct rte_eth_dev *dev;
1366
1367         /* This function is only safe when called from the primary process
1368          * in a multi-process setup*/
1369         PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
1370
1371         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1372
1373         dev = &rte_eth_devices[port_id];
1374
1375         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_down, -ENOTSUP);
1376         return (*dev->dev_ops->dev_set_link_down)(dev);
1377 }
1378
1379 void
1380 rte_eth_dev_close(uint8_t port_id)
1381 {
1382         struct rte_eth_dev *dev;
1383
1384         /* This function is only safe when called from the primary process
1385          * in a multi-process setup*/
1386         PROC_PRIMARY_OR_RET();
1387
1388         VALID_PORTID_OR_RET(port_id);
1389         dev = &rte_eth_devices[port_id];
1390
1391         FUNC_PTR_OR_RET(*dev->dev_ops->dev_close);
1392         dev->data->dev_started = 0;
1393         (*dev->dev_ops->dev_close)(dev);
1394 }
1395
1396 int
1397 rte_eth_rx_queue_setup(uint8_t port_id, uint16_t rx_queue_id,
1398                        uint16_t nb_rx_desc, unsigned int socket_id,
1399                        const struct rte_eth_rxconf *rx_conf,
1400                        struct rte_mempool *mp)
1401 {
1402         int ret;
1403         uint32_t mbp_buf_size;
1404         struct rte_eth_dev *dev;
1405         struct rte_eth_dev_info dev_info;
1406
1407         /* This function is only safe when called from the primary process
1408          * in a multi-process setup*/
1409         PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
1410
1411         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1412
1413         dev = &rte_eth_devices[port_id];
1414         if (rx_queue_id >= dev->data->nb_rx_queues) {
1415                 PMD_DEBUG_TRACE("Invalid RX queue_id=%d\n", rx_queue_id);
1416                 return -EINVAL;
1417         }
1418
1419         if (dev->data->dev_started) {
1420                 PMD_DEBUG_TRACE(
1421                     "port %d must be stopped to allow configuration\n", port_id);
1422                 return -EBUSY;
1423         }
1424
1425         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
1426         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_setup, -ENOTSUP);
1427
1428         /*
1429          * Check the size of the mbuf data buffer.
1430          * This value must be provided in the private data of the memory pool.
1431          * First check that the memory pool has a valid private data.
1432          */
1433         rte_eth_dev_info_get(port_id, &dev_info);
1434         if (mp->private_data_size < sizeof(struct rte_pktmbuf_pool_private)) {
1435                 PMD_DEBUG_TRACE("%s private_data_size %d < %d\n",
1436                                 mp->name, (int) mp->private_data_size,
1437                                 (int) sizeof(struct rte_pktmbuf_pool_private));
1438                 return -ENOSPC;
1439         }
1440         mbp_buf_size = rte_pktmbuf_data_room_size(mp);
1441
1442         if ((mbp_buf_size - RTE_PKTMBUF_HEADROOM) < dev_info.min_rx_bufsize) {
1443                 PMD_DEBUG_TRACE("%s mbuf_data_room_size %d < %d "
1444                                 "(RTE_PKTMBUF_HEADROOM=%d + min_rx_bufsize(dev)"
1445                                 "=%d)\n",
1446                                 mp->name,
1447                                 (int)mbp_buf_size,
1448                                 (int)(RTE_PKTMBUF_HEADROOM +
1449                                       dev_info.min_rx_bufsize),
1450                                 (int)RTE_PKTMBUF_HEADROOM,
1451                                 (int)dev_info.min_rx_bufsize);
1452                 return -EINVAL;
1453         }
1454
1455         if (rx_conf == NULL)
1456                 rx_conf = &dev_info.default_rxconf;
1457
1458         ret = (*dev->dev_ops->rx_queue_setup)(dev, rx_queue_id, nb_rx_desc,
1459                                               socket_id, rx_conf, mp);
1460         if (!ret) {
1461                 if (!dev->data->min_rx_buf_size ||
1462                     dev->data->min_rx_buf_size > mbp_buf_size)
1463                         dev->data->min_rx_buf_size = mbp_buf_size;
1464         }
1465
1466         return ret;
1467 }
1468
1469 int
1470 rte_eth_tx_queue_setup(uint8_t port_id, uint16_t tx_queue_id,
1471                        uint16_t nb_tx_desc, unsigned int socket_id,
1472                        const struct rte_eth_txconf *tx_conf)
1473 {
1474         struct rte_eth_dev *dev;
1475         struct rte_eth_dev_info dev_info;
1476
1477         /* This function is only safe when called from the primary process
1478          * in a multi-process setup*/
1479         PROC_PRIMARY_OR_ERR_RET(-E_RTE_SECONDARY);
1480
1481         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1482
1483         dev = &rte_eth_devices[port_id];
1484         if (tx_queue_id >= dev->data->nb_tx_queues) {
1485                 PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", tx_queue_id);
1486                 return -EINVAL;
1487         }
1488
1489         if (dev->data->dev_started) {
1490                 PMD_DEBUG_TRACE(
1491                     "port %d must be stopped to allow configuration\n", port_id);
1492                 return -EBUSY;
1493         }
1494
1495         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
1496         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_setup, -ENOTSUP);
1497
1498         rte_eth_dev_info_get(port_id, &dev_info);
1499
1500         if (tx_conf == NULL)
1501                 tx_conf = &dev_info.default_txconf;
1502
1503         return (*dev->dev_ops->tx_queue_setup)(dev, tx_queue_id, nb_tx_desc,
1504                                                socket_id, tx_conf);
1505 }
1506
1507 void
1508 rte_eth_promiscuous_enable(uint8_t port_id)
1509 {
1510         struct rte_eth_dev *dev;
1511
1512         VALID_PORTID_OR_RET(port_id);
1513         dev = &rte_eth_devices[port_id];
1514
1515         FUNC_PTR_OR_RET(*dev->dev_ops->promiscuous_enable);
1516         (*dev->dev_ops->promiscuous_enable)(dev);
1517         dev->data->promiscuous = 1;
1518 }
1519
1520 void
1521 rte_eth_promiscuous_disable(uint8_t port_id)
1522 {
1523         struct rte_eth_dev *dev;
1524
1525         VALID_PORTID_OR_RET(port_id);
1526         dev = &rte_eth_devices[port_id];
1527
1528         FUNC_PTR_OR_RET(*dev->dev_ops->promiscuous_disable);
1529         dev->data->promiscuous = 0;
1530         (*dev->dev_ops->promiscuous_disable)(dev);
1531 }
1532
1533 int
1534 rte_eth_promiscuous_get(uint8_t port_id)
1535 {
1536         struct rte_eth_dev *dev;
1537
1538         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1539
1540         dev = &rte_eth_devices[port_id];
1541         return dev->data->promiscuous;
1542 }
1543
1544 void
1545 rte_eth_allmulticast_enable(uint8_t port_id)
1546 {
1547         struct rte_eth_dev *dev;
1548
1549         VALID_PORTID_OR_RET(port_id);
1550         dev = &rte_eth_devices[port_id];
1551
1552         FUNC_PTR_OR_RET(*dev->dev_ops->allmulticast_enable);
1553         (*dev->dev_ops->allmulticast_enable)(dev);
1554         dev->data->all_multicast = 1;
1555 }
1556
1557 void
1558 rte_eth_allmulticast_disable(uint8_t port_id)
1559 {
1560         struct rte_eth_dev *dev;
1561
1562         VALID_PORTID_OR_RET(port_id);
1563         dev = &rte_eth_devices[port_id];
1564
1565         FUNC_PTR_OR_RET(*dev->dev_ops->allmulticast_disable);
1566         dev->data->all_multicast = 0;
1567         (*dev->dev_ops->allmulticast_disable)(dev);
1568 }
1569
1570 int
1571 rte_eth_allmulticast_get(uint8_t port_id)
1572 {
1573         struct rte_eth_dev *dev;
1574
1575         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1576
1577         dev = &rte_eth_devices[port_id];
1578         return dev->data->all_multicast;
1579 }
1580
1581 static inline int
1582 rte_eth_dev_atomic_read_link_status(struct rte_eth_dev *dev,
1583                                 struct rte_eth_link *link)
1584 {
1585         struct rte_eth_link *dst = link;
1586         struct rte_eth_link *src = &(dev->data->dev_link);
1587
1588         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
1589                                         *(uint64_t *)src) == 0)
1590                 return -1;
1591
1592         return 0;
1593 }
1594
1595 void
1596 rte_eth_link_get(uint8_t port_id, struct rte_eth_link *eth_link)
1597 {
1598         struct rte_eth_dev *dev;
1599
1600         VALID_PORTID_OR_RET(port_id);
1601         dev = &rte_eth_devices[port_id];
1602
1603         if (dev->data->dev_conf.intr_conf.lsc != 0)
1604                 rte_eth_dev_atomic_read_link_status(dev, eth_link);
1605         else {
1606                 FUNC_PTR_OR_RET(*dev->dev_ops->link_update);
1607                 (*dev->dev_ops->link_update)(dev, 1);
1608                 *eth_link = dev->data->dev_link;
1609         }
1610 }
1611
1612 void
1613 rte_eth_link_get_nowait(uint8_t port_id, struct rte_eth_link *eth_link)
1614 {
1615         struct rte_eth_dev *dev;
1616
1617         VALID_PORTID_OR_RET(port_id);
1618         dev = &rte_eth_devices[port_id];
1619
1620         if (dev->data->dev_conf.intr_conf.lsc != 0)
1621                 rte_eth_dev_atomic_read_link_status(dev, eth_link);
1622         else {
1623                 FUNC_PTR_OR_RET(*dev->dev_ops->link_update);
1624                 (*dev->dev_ops->link_update)(dev, 0);
1625                 *eth_link = dev->data->dev_link;
1626         }
1627 }
1628
1629 int
1630 rte_eth_stats_get(uint8_t port_id, struct rte_eth_stats *stats)
1631 {
1632         struct rte_eth_dev *dev;
1633
1634         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1635
1636         dev = &rte_eth_devices[port_id];
1637         memset(stats, 0, sizeof(*stats));
1638
1639         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
1640         (*dev->dev_ops->stats_get)(dev, stats);
1641         stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
1642         return 0;
1643 }
1644
1645 void
1646 rte_eth_stats_reset(uint8_t port_id)
1647 {
1648         struct rte_eth_dev *dev;
1649
1650         VALID_PORTID_OR_RET(port_id);
1651         dev = &rte_eth_devices[port_id];
1652
1653         FUNC_PTR_OR_RET(*dev->dev_ops->stats_reset);
1654         (*dev->dev_ops->stats_reset)(dev);
1655 }
1656
1657 /* retrieve ethdev extended statistics */
1658 int
1659 rte_eth_xstats_get(uint8_t port_id, struct rte_eth_xstats *xstats,
1660         unsigned n)
1661 {
1662         struct rte_eth_stats eth_stats;
1663         struct rte_eth_dev *dev;
1664         unsigned count = 0, i, q;
1665         signed xcount = 0;
1666         uint64_t val, *stats_ptr;
1667
1668         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1669
1670         dev = &rte_eth_devices[port_id];
1671
1672         /* Return generic statistics */
1673         count = RTE_NB_STATS;
1674         count += dev->data->nb_rx_queues * RTE_NB_RXQ_STATS;
1675         count += dev->data->nb_tx_queues * RTE_NB_TXQ_STATS;
1676
1677         /* implemented by the driver */
1678         if (dev->dev_ops->xstats_get != NULL) {
1679                 /* Retrieve the xstats from the driver at the end of the
1680                  * xstats struct.
1681                  */
1682                 xcount = (*dev->dev_ops->xstats_get)(dev, &xstats[count],
1683                          (n > count) ? n - count : 0);
1684
1685                 if (xcount < 0)
1686                         return xcount;
1687         }
1688
1689         if (n < count + xcount)
1690                 return count + xcount;
1691
1692         /* now fill the xstats structure */
1693         count = 0;
1694         rte_eth_stats_get(port_id, &eth_stats);
1695
1696         /* global stats */
1697         for (i = 0; i < RTE_NB_STATS; i++) {
1698                 stats_ptr = RTE_PTR_ADD(&eth_stats,
1699                                         rte_stats_strings[i].offset);
1700                 val = *stats_ptr;
1701                 snprintf(xstats[count].name, sizeof(xstats[count].name),
1702                         "%s", rte_stats_strings[i].name);
1703                 xstats[count++].value = val;
1704         }
1705
1706         /* per-rxq stats */
1707         for (q = 0; q < dev->data->nb_rx_queues; q++) {
1708                 for (i = 0; i < RTE_NB_RXQ_STATS; i++) {
1709                         stats_ptr = RTE_PTR_ADD(&eth_stats,
1710                                         rte_rxq_stats_strings[i].offset +
1711                                         q * sizeof(uint64_t));
1712                         val = *stats_ptr;
1713                         snprintf(xstats[count].name, sizeof(xstats[count].name),
1714                                 "rx_queue_%u_%s", q,
1715                                 rte_rxq_stats_strings[i].name);
1716                         xstats[count++].value = val;
1717                 }
1718         }
1719
1720         /* per-txq stats */
1721         for (q = 0; q < dev->data->nb_tx_queues; q++) {
1722                 for (i = 0; i < RTE_NB_TXQ_STATS; i++) {
1723                         stats_ptr = RTE_PTR_ADD(&eth_stats,
1724                                         rte_txq_stats_strings[i].offset +
1725                                         q * sizeof(uint64_t));
1726                         val = *stats_ptr;
1727                         snprintf(xstats[count].name, sizeof(xstats[count].name),
1728                                 "tx_queue_%u_%s", q,
1729                                 rte_txq_stats_strings[i].name);
1730                         xstats[count++].value = val;
1731                 }
1732         }
1733
1734         return count + xcount;
1735 }
1736
1737 /* reset ethdev extended statistics */
1738 void
1739 rte_eth_xstats_reset(uint8_t port_id)
1740 {
1741         struct rte_eth_dev *dev;
1742
1743         VALID_PORTID_OR_RET(port_id);
1744         dev = &rte_eth_devices[port_id];
1745
1746         /* implemented by the driver */
1747         if (dev->dev_ops->xstats_reset != NULL) {
1748                 (*dev->dev_ops->xstats_reset)(dev);
1749                 return;
1750         }
1751
1752         /* fallback to default */
1753         rte_eth_stats_reset(port_id);
1754 }
1755
1756 static int
1757 set_queue_stats_mapping(uint8_t port_id, uint16_t queue_id, uint8_t stat_idx,
1758                 uint8_t is_rx)
1759 {
1760         struct rte_eth_dev *dev;
1761
1762         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1763
1764         dev = &rte_eth_devices[port_id];
1765
1766         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_stats_mapping_set, -ENOTSUP);
1767         return (*dev->dev_ops->queue_stats_mapping_set)
1768                         (dev, queue_id, stat_idx, is_rx);
1769 }
1770
1771
1772 int
1773 rte_eth_dev_set_tx_queue_stats_mapping(uint8_t port_id, uint16_t tx_queue_id,
1774                 uint8_t stat_idx)
1775 {
1776         return set_queue_stats_mapping(port_id, tx_queue_id, stat_idx,
1777                         STAT_QMAP_TX);
1778 }
1779
1780
1781 int
1782 rte_eth_dev_set_rx_queue_stats_mapping(uint8_t port_id, uint16_t rx_queue_id,
1783                 uint8_t stat_idx)
1784 {
1785         return set_queue_stats_mapping(port_id, rx_queue_id, stat_idx,
1786                         STAT_QMAP_RX);
1787 }
1788
1789
1790 void
1791 rte_eth_dev_info_get(uint8_t port_id, struct rte_eth_dev_info *dev_info)
1792 {
1793         struct rte_eth_dev *dev;
1794
1795         VALID_PORTID_OR_RET(port_id);
1796         dev = &rte_eth_devices[port_id];
1797
1798         memset(dev_info, 0, sizeof(struct rte_eth_dev_info));
1799
1800         FUNC_PTR_OR_RET(*dev->dev_ops->dev_infos_get);
1801         (*dev->dev_ops->dev_infos_get)(dev, dev_info);
1802         dev_info->pci_dev = dev->pci_dev;
1803         if (dev->driver)
1804                 dev_info->driver_name = dev->driver->pci_drv.name;
1805 }
1806
1807 void
1808 rte_eth_macaddr_get(uint8_t port_id, struct ether_addr *mac_addr)
1809 {
1810         struct rte_eth_dev *dev;
1811
1812         VALID_PORTID_OR_RET(port_id);
1813         dev = &rte_eth_devices[port_id];
1814         ether_addr_copy(&dev->data->mac_addrs[0], mac_addr);
1815 }
1816
1817
1818 int
1819 rte_eth_dev_get_mtu(uint8_t port_id, uint16_t *mtu)
1820 {
1821         struct rte_eth_dev *dev;
1822
1823         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1824
1825         dev = &rte_eth_devices[port_id];
1826         *mtu = dev->data->mtu;
1827         return 0;
1828 }
1829
1830 int
1831 rte_eth_dev_set_mtu(uint8_t port_id, uint16_t mtu)
1832 {
1833         int ret;
1834         struct rte_eth_dev *dev;
1835
1836         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1837         dev = &rte_eth_devices[port_id];
1838         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mtu_set, -ENOTSUP);
1839
1840         ret = (*dev->dev_ops->mtu_set)(dev, mtu);
1841         if (!ret)
1842                 dev->data->mtu = mtu;
1843
1844         return ret;
1845 }
1846
1847 int
1848 rte_eth_dev_vlan_filter(uint8_t port_id, uint16_t vlan_id, int on)
1849 {
1850         struct rte_eth_dev *dev;
1851
1852         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1853         dev = &rte_eth_devices[port_id];
1854         if (!(dev->data->dev_conf.rxmode.hw_vlan_filter)) {
1855                 PMD_DEBUG_TRACE("port %d: vlan-filtering disabled\n", port_id);
1856                 return -ENOSYS;
1857         }
1858
1859         if (vlan_id > 4095) {
1860                 PMD_DEBUG_TRACE("(port_id=%d) invalid vlan_id=%u > 4095\n",
1861                                 port_id, (unsigned) vlan_id);
1862                 return -EINVAL;
1863         }
1864         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_filter_set, -ENOTSUP);
1865
1866         return (*dev->dev_ops->vlan_filter_set)(dev, vlan_id, on);
1867 }
1868
1869 int
1870 rte_eth_dev_set_vlan_strip_on_queue(uint8_t port_id, uint16_t rx_queue_id, int on)
1871 {
1872         struct rte_eth_dev *dev;
1873
1874         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1875         dev = &rte_eth_devices[port_id];
1876         if (rx_queue_id >= dev->data->nb_rx_queues) {
1877                 PMD_DEBUG_TRACE("Invalid rx_queue_id=%d\n", port_id);
1878                 return -EINVAL;
1879         }
1880
1881         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_strip_queue_set, -ENOTSUP);
1882         (*dev->dev_ops->vlan_strip_queue_set)(dev, rx_queue_id, on);
1883
1884         return 0;
1885 }
1886
1887 int
1888 rte_eth_dev_set_vlan_ether_type(uint8_t port_id, uint16_t tpid)
1889 {
1890         struct rte_eth_dev *dev;
1891
1892         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1893         dev = &rte_eth_devices[port_id];
1894         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_tpid_set, -ENOTSUP);
1895         (*dev->dev_ops->vlan_tpid_set)(dev, tpid);
1896
1897         return 0;
1898 }
1899
1900 int
1901 rte_eth_dev_set_vlan_offload(uint8_t port_id, int offload_mask)
1902 {
1903         struct rte_eth_dev *dev;
1904         int ret = 0;
1905         int mask = 0;
1906         int cur, org = 0;
1907
1908         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1909         dev = &rte_eth_devices[port_id];
1910
1911         /*check which option changed by application*/
1912         cur = !!(offload_mask & ETH_VLAN_STRIP_OFFLOAD);
1913         org = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
1914         if (cur != org) {
1915                 dev->data->dev_conf.rxmode.hw_vlan_strip = (uint8_t)cur;
1916                 mask |= ETH_VLAN_STRIP_MASK;
1917         }
1918
1919         cur = !!(offload_mask & ETH_VLAN_FILTER_OFFLOAD);
1920         org = !!(dev->data->dev_conf.rxmode.hw_vlan_filter);
1921         if (cur != org) {
1922                 dev->data->dev_conf.rxmode.hw_vlan_filter = (uint8_t)cur;
1923                 mask |= ETH_VLAN_FILTER_MASK;
1924         }
1925
1926         cur = !!(offload_mask & ETH_VLAN_EXTEND_OFFLOAD);
1927         org = !!(dev->data->dev_conf.rxmode.hw_vlan_extend);
1928         if (cur != org) {
1929                 dev->data->dev_conf.rxmode.hw_vlan_extend = (uint8_t)cur;
1930                 mask |= ETH_VLAN_EXTEND_MASK;
1931         }
1932
1933         /*no change*/
1934         if (mask == 0)
1935                 return ret;
1936
1937         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_offload_set, -ENOTSUP);
1938         (*dev->dev_ops->vlan_offload_set)(dev, mask);
1939
1940         return ret;
1941 }
1942
1943 int
1944 rte_eth_dev_get_vlan_offload(uint8_t port_id)
1945 {
1946         struct rte_eth_dev *dev;
1947         int ret = 0;
1948
1949         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1950         dev = &rte_eth_devices[port_id];
1951
1952         if (dev->data->dev_conf.rxmode.hw_vlan_strip)
1953                 ret |= ETH_VLAN_STRIP_OFFLOAD;
1954
1955         if (dev->data->dev_conf.rxmode.hw_vlan_filter)
1956                 ret |= ETH_VLAN_FILTER_OFFLOAD;
1957
1958         if (dev->data->dev_conf.rxmode.hw_vlan_extend)
1959                 ret |= ETH_VLAN_EXTEND_OFFLOAD;
1960
1961         return ret;
1962 }
1963
1964 int
1965 rte_eth_dev_set_vlan_pvid(uint8_t port_id, uint16_t pvid, int on)
1966 {
1967         struct rte_eth_dev *dev;
1968
1969         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1970         dev = &rte_eth_devices[port_id];
1971         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_pvid_set, -ENOTSUP);
1972         (*dev->dev_ops->vlan_pvid_set)(dev, pvid, on);
1973
1974         return 0;
1975 }
1976
1977 int
1978 rte_eth_dev_fdir_add_signature_filter(uint8_t port_id,
1979                                       struct rte_fdir_filter *fdir_filter,
1980                                       uint8_t queue)
1981 {
1982         struct rte_eth_dev *dev;
1983
1984         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1985         dev = &rte_eth_devices[port_id];
1986
1987         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_SIGNATURE) {
1988                 PMD_DEBUG_TRACE("port %d: invalid FDIR mode=%u\n",
1989                                 port_id, dev->data->dev_conf.fdir_conf.mode);
1990                 return -ENOSYS;
1991         }
1992
1993         if ((fdir_filter->l4type == RTE_FDIR_L4TYPE_SCTP
1994              || fdir_filter->l4type == RTE_FDIR_L4TYPE_NONE)
1995             && (fdir_filter->port_src || fdir_filter->port_dst)) {
1996                 PMD_DEBUG_TRACE(" Port are meaningless for SCTP and "
1997                                 "None l4type, source & destinations ports "
1998                                 "should be null!\n");
1999                 return -EINVAL;
2000         }
2001
2002         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fdir_add_signature_filter, -ENOTSUP);
2003         return (*dev->dev_ops->fdir_add_signature_filter)(dev, fdir_filter,
2004                                                                 queue);
2005 }
2006
2007 int
2008 rte_eth_dev_fdir_update_signature_filter(uint8_t port_id,
2009                                          struct rte_fdir_filter *fdir_filter,
2010                                          uint8_t queue)
2011 {
2012         struct rte_eth_dev *dev;
2013
2014         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2015         dev = &rte_eth_devices[port_id];
2016
2017         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_SIGNATURE) {
2018                 PMD_DEBUG_TRACE("port %d: invalid FDIR mode=%u\n",
2019                                 port_id, dev->data->dev_conf.fdir_conf.mode);
2020                 return -ENOSYS;
2021         }
2022
2023         if ((fdir_filter->l4type == RTE_FDIR_L4TYPE_SCTP
2024              || fdir_filter->l4type == RTE_FDIR_L4TYPE_NONE)
2025             && (fdir_filter->port_src || fdir_filter->port_dst)) {
2026                 PMD_DEBUG_TRACE(" Port are meaningless for SCTP and "
2027                                 "None l4type, source & destinations ports "
2028                                 "should be null!\n");
2029                 return -EINVAL;
2030         }
2031
2032         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fdir_update_signature_filter, -ENOTSUP);
2033         return (*dev->dev_ops->fdir_update_signature_filter)(dev, fdir_filter,
2034                                                                 queue);
2035
2036 }
2037
2038 int
2039 rte_eth_dev_fdir_remove_signature_filter(uint8_t port_id,
2040                                          struct rte_fdir_filter *fdir_filter)
2041 {
2042         struct rte_eth_dev *dev;
2043
2044         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2045         dev = &rte_eth_devices[port_id];
2046
2047         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_SIGNATURE) {
2048                 PMD_DEBUG_TRACE("port %d: invalid FDIR mode=%u\n",
2049                                 port_id, dev->data->dev_conf.fdir_conf.mode);
2050                 return -ENOSYS;
2051         }
2052
2053         if ((fdir_filter->l4type == RTE_FDIR_L4TYPE_SCTP
2054              || fdir_filter->l4type == RTE_FDIR_L4TYPE_NONE)
2055             && (fdir_filter->port_src || fdir_filter->port_dst)) {
2056                 PMD_DEBUG_TRACE(" Port are meaningless for SCTP and "
2057                                 "None l4type source & destinations ports "
2058                                 "should be null!\n");
2059                 return -EINVAL;
2060         }
2061
2062         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fdir_remove_signature_filter, -ENOTSUP);
2063         return (*dev->dev_ops->fdir_remove_signature_filter)(dev, fdir_filter);
2064 }
2065
2066 int
2067 rte_eth_dev_fdir_get_infos(uint8_t port_id, struct rte_eth_fdir *fdir)
2068 {
2069         struct rte_eth_dev *dev;
2070
2071         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2072         dev = &rte_eth_devices[port_id];
2073         if (!(dev->data->dev_conf.fdir_conf.mode)) {
2074                 PMD_DEBUG_TRACE("port %d: pkt-filter disabled\n", port_id);
2075                 return -ENOSYS;
2076         }
2077
2078         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fdir_infos_get, -ENOTSUP);
2079
2080         (*dev->dev_ops->fdir_infos_get)(dev, fdir);
2081         return 0;
2082 }
2083
2084 int
2085 rte_eth_dev_fdir_add_perfect_filter(uint8_t port_id,
2086                                     struct rte_fdir_filter *fdir_filter,
2087                                     uint16_t soft_id, uint8_t queue,
2088                                     uint8_t drop)
2089 {
2090         struct rte_eth_dev *dev;
2091
2092         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2093         dev = &rte_eth_devices[port_id];
2094
2095         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_PERFECT) {
2096                 PMD_DEBUG_TRACE("port %d: invalid FDIR mode=%u\n",
2097                                 port_id, dev->data->dev_conf.fdir_conf.mode);
2098                 return -ENOSYS;
2099         }
2100
2101         if ((fdir_filter->l4type == RTE_FDIR_L4TYPE_SCTP
2102              || fdir_filter->l4type == RTE_FDIR_L4TYPE_NONE)
2103             && (fdir_filter->port_src || fdir_filter->port_dst)) {
2104                 PMD_DEBUG_TRACE(" Port are meaningless for SCTP and "
2105                                 "None l4type, source & destinations ports "
2106                                 "should be null!\n");
2107                 return -EINVAL;
2108         }
2109
2110         /* For now IPv6 is not supported with perfect filter */
2111         if (fdir_filter->iptype == RTE_FDIR_IPTYPE_IPV6)
2112                 return -ENOTSUP;
2113
2114         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fdir_add_perfect_filter, -ENOTSUP);
2115         return (*dev->dev_ops->fdir_add_perfect_filter)(dev, fdir_filter,
2116                                                                 soft_id, queue,
2117                                                                 drop);
2118 }
2119
2120 int
2121 rte_eth_dev_fdir_update_perfect_filter(uint8_t port_id,
2122                                        struct rte_fdir_filter *fdir_filter,
2123                                        uint16_t soft_id, uint8_t queue,
2124                                        uint8_t drop)
2125 {
2126         struct rte_eth_dev *dev;
2127
2128         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2129         dev = &rte_eth_devices[port_id];
2130
2131         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_PERFECT) {
2132                 PMD_DEBUG_TRACE("port %d: invalid FDIR mode=%u\n",
2133                                 port_id, dev->data->dev_conf.fdir_conf.mode);
2134                 return -ENOSYS;
2135         }
2136
2137         if ((fdir_filter->l4type == RTE_FDIR_L4TYPE_SCTP
2138              || fdir_filter->l4type == RTE_FDIR_L4TYPE_NONE)
2139             && (fdir_filter->port_src || fdir_filter->port_dst)) {
2140                 PMD_DEBUG_TRACE(" Port are meaningless for SCTP and "
2141                                 "None l4type, source & destinations ports "
2142                                 "should be null!\n");
2143                 return -EINVAL;
2144         }
2145
2146         /* For now IPv6 is not supported with perfect filter */
2147         if (fdir_filter->iptype == RTE_FDIR_IPTYPE_IPV6)
2148                 return -ENOTSUP;
2149
2150         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fdir_update_perfect_filter, -ENOTSUP);
2151         return (*dev->dev_ops->fdir_update_perfect_filter)(dev, fdir_filter,
2152                                                         soft_id, queue, drop);
2153 }
2154
2155 int
2156 rte_eth_dev_fdir_remove_perfect_filter(uint8_t port_id,
2157                                        struct rte_fdir_filter *fdir_filter,
2158                                        uint16_t soft_id)
2159 {
2160         struct rte_eth_dev *dev;
2161
2162         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2163         dev = &rte_eth_devices[port_id];
2164
2165         if (dev->data->dev_conf.fdir_conf.mode != RTE_FDIR_MODE_PERFECT) {
2166                 PMD_DEBUG_TRACE("port %d: invalid FDIR mode=%u\n",
2167                                 port_id, dev->data->dev_conf.fdir_conf.mode);
2168                 return -ENOSYS;
2169         }
2170
2171         if ((fdir_filter->l4type == RTE_FDIR_L4TYPE_SCTP
2172              || fdir_filter->l4type == RTE_FDIR_L4TYPE_NONE)
2173             && (fdir_filter->port_src || fdir_filter->port_dst)) {
2174                 PMD_DEBUG_TRACE(" Port are meaningless for SCTP and "
2175                                 "None l4type, source & destinations ports "
2176                                 "should be null!\n");
2177                 return -EINVAL;
2178         }
2179
2180         /* For now IPv6 is not supported with perfect filter */
2181         if (fdir_filter->iptype == RTE_FDIR_IPTYPE_IPV6)
2182                 return -ENOTSUP;
2183
2184         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fdir_remove_perfect_filter, -ENOTSUP);
2185         return (*dev->dev_ops->fdir_remove_perfect_filter)(dev, fdir_filter,
2186                                                                 soft_id);
2187 }
2188
2189 int
2190 rte_eth_dev_fdir_set_masks(uint8_t port_id, struct rte_fdir_masks *fdir_mask)
2191 {
2192         struct rte_eth_dev *dev;
2193
2194         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2195         dev = &rte_eth_devices[port_id];
2196         if (!(dev->data->dev_conf.fdir_conf.mode)) {
2197                 PMD_DEBUG_TRACE("port %d: pkt-filter disabled\n", port_id);
2198                 return -ENOSYS;
2199         }
2200
2201         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fdir_set_masks, -ENOTSUP);
2202         return (*dev->dev_ops->fdir_set_masks)(dev, fdir_mask);
2203 }
2204
2205 int
2206 rte_eth_dev_flow_ctrl_get(uint8_t port_id, struct rte_eth_fc_conf *fc_conf)
2207 {
2208         struct rte_eth_dev *dev;
2209
2210         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2211         dev = &rte_eth_devices[port_id];
2212         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_get, -ENOTSUP);
2213         memset(fc_conf, 0, sizeof(*fc_conf));
2214         return (*dev->dev_ops->flow_ctrl_get)(dev, fc_conf);
2215 }
2216
2217 int
2218 rte_eth_dev_flow_ctrl_set(uint8_t port_id, struct rte_eth_fc_conf *fc_conf)
2219 {
2220         struct rte_eth_dev *dev;
2221
2222         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2223         if ((fc_conf->send_xon != 0) && (fc_conf->send_xon != 1)) {
2224                 PMD_DEBUG_TRACE("Invalid send_xon, only 0/1 allowed\n");
2225                 return -EINVAL;
2226         }
2227
2228         dev = &rte_eth_devices[port_id];
2229         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_set, -ENOTSUP);
2230         return (*dev->dev_ops->flow_ctrl_set)(dev, fc_conf);
2231 }
2232
2233 int
2234 rte_eth_dev_priority_flow_ctrl_set(uint8_t port_id, struct rte_eth_pfc_conf *pfc_conf)
2235 {
2236         struct rte_eth_dev *dev;
2237
2238         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2239         if (pfc_conf->priority > (ETH_DCB_NUM_USER_PRIORITIES - 1)) {
2240                 PMD_DEBUG_TRACE("Invalid priority, only 0-7 allowed\n");
2241                 return -EINVAL;
2242         }
2243
2244         dev = &rte_eth_devices[port_id];
2245         /* High water, low water validation are device specific */
2246         if  (*dev->dev_ops->priority_flow_ctrl_set)
2247                 return (*dev->dev_ops->priority_flow_ctrl_set)(dev, pfc_conf);
2248         return -ENOTSUP;
2249 }
2250
2251 static int
2252 rte_eth_check_reta_mask(struct rte_eth_rss_reta_entry64 *reta_conf,
2253                         uint16_t reta_size)
2254 {
2255         uint16_t i, num;
2256
2257         if (!reta_conf)
2258                 return -EINVAL;
2259
2260         if (reta_size != RTE_ALIGN(reta_size, RTE_RETA_GROUP_SIZE)) {
2261                 PMD_DEBUG_TRACE("Invalid reta size, should be %u aligned\n",
2262                                                         RTE_RETA_GROUP_SIZE);
2263                 return -EINVAL;
2264         }
2265
2266         num = reta_size / RTE_RETA_GROUP_SIZE;
2267         for (i = 0; i < num; i++) {
2268                 if (reta_conf[i].mask)
2269                         return 0;
2270         }
2271
2272         return -EINVAL;
2273 }
2274
2275 static int
2276 rte_eth_check_reta_entry(struct rte_eth_rss_reta_entry64 *reta_conf,
2277                          uint16_t reta_size,
2278                          uint8_t max_rxq)
2279 {
2280         uint16_t i, idx, shift;
2281
2282         if (!reta_conf)
2283                 return -EINVAL;
2284
2285         if (max_rxq == 0) {
2286                 PMD_DEBUG_TRACE("No receive queue is available\n");
2287                 return -EINVAL;
2288         }
2289
2290         for (i = 0; i < reta_size; i++) {
2291                 idx = i / RTE_RETA_GROUP_SIZE;
2292                 shift = i % RTE_RETA_GROUP_SIZE;
2293                 if ((reta_conf[idx].mask & (1ULL << shift)) &&
2294                         (reta_conf[idx].reta[shift] >= max_rxq)) {
2295                         PMD_DEBUG_TRACE("reta_conf[%u]->reta[%u]: %u exceeds "
2296                                 "the maximum rxq index: %u\n", idx, shift,
2297                                 reta_conf[idx].reta[shift], max_rxq);
2298                         return -EINVAL;
2299                 }
2300         }
2301
2302         return 0;
2303 }
2304
2305 int
2306 rte_eth_dev_rss_reta_update(uint8_t port_id,
2307                             struct rte_eth_rss_reta_entry64 *reta_conf,
2308                             uint16_t reta_size)
2309 {
2310         struct rte_eth_dev *dev;
2311         int ret;
2312
2313         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2314         /* Check mask bits */
2315         ret = rte_eth_check_reta_mask(reta_conf, reta_size);
2316         if (ret < 0)
2317                 return ret;
2318
2319         dev = &rte_eth_devices[port_id];
2320
2321         /* Check entry value */
2322         ret = rte_eth_check_reta_entry(reta_conf, reta_size,
2323                                 dev->data->nb_rx_queues);
2324         if (ret < 0)
2325                 return ret;
2326
2327         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_update, -ENOTSUP);
2328         return (*dev->dev_ops->reta_update)(dev, reta_conf, reta_size);
2329 }
2330
2331 int
2332 rte_eth_dev_rss_reta_query(uint8_t port_id,
2333                            struct rte_eth_rss_reta_entry64 *reta_conf,
2334                            uint16_t reta_size)
2335 {
2336         struct rte_eth_dev *dev;
2337         int ret;
2338
2339         if (port_id >= nb_ports) {
2340                 PMD_DEBUG_TRACE("Invalid port_id=%d\n", port_id);
2341                 return -ENODEV;
2342         }
2343
2344         /* Check mask bits */
2345         ret = rte_eth_check_reta_mask(reta_conf, reta_size);
2346         if (ret < 0)
2347                 return ret;
2348
2349         dev = &rte_eth_devices[port_id];
2350         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_query, -ENOTSUP);
2351         return (*dev->dev_ops->reta_query)(dev, reta_conf, reta_size);
2352 }
2353
2354 int
2355 rte_eth_dev_rss_hash_update(uint8_t port_id, struct rte_eth_rss_conf *rss_conf)
2356 {
2357         struct rte_eth_dev *dev;
2358         uint16_t rss_hash_protos;
2359
2360         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2361         rss_hash_protos = rss_conf->rss_hf;
2362         if ((rss_hash_protos != 0) &&
2363             ((rss_hash_protos & ETH_RSS_PROTO_MASK) == 0)) {
2364                 PMD_DEBUG_TRACE("Invalid rss_hash_protos=0x%x\n",
2365                                 rss_hash_protos);
2366                 return -EINVAL;
2367         }
2368         dev = &rte_eth_devices[port_id];
2369         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_update, -ENOTSUP);
2370         return (*dev->dev_ops->rss_hash_update)(dev, rss_conf);
2371 }
2372
2373 int
2374 rte_eth_dev_rss_hash_conf_get(uint8_t port_id,
2375                               struct rte_eth_rss_conf *rss_conf)
2376 {
2377         struct rte_eth_dev *dev;
2378
2379         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2380         dev = &rte_eth_devices[port_id];
2381         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_conf_get, -ENOTSUP);
2382         return (*dev->dev_ops->rss_hash_conf_get)(dev, rss_conf);
2383 }
2384
2385 int
2386 rte_eth_dev_udp_tunnel_add(uint8_t port_id,
2387                            struct rte_eth_udp_tunnel *udp_tunnel)
2388 {
2389         struct rte_eth_dev *dev;
2390
2391         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2392         if (udp_tunnel == NULL) {
2393                 PMD_DEBUG_TRACE("Invalid udp_tunnel parameter\n");
2394                 return -EINVAL;
2395         }
2396
2397         if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
2398                 PMD_DEBUG_TRACE("Invalid tunnel type\n");
2399                 return -EINVAL;
2400         }
2401
2402         dev = &rte_eth_devices[port_id];
2403         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_add, -ENOTSUP);
2404         return (*dev->dev_ops->udp_tunnel_add)(dev, udp_tunnel);
2405 }
2406
2407 int
2408 rte_eth_dev_udp_tunnel_delete(uint8_t port_id,
2409                               struct rte_eth_udp_tunnel *udp_tunnel)
2410 {
2411         struct rte_eth_dev *dev;
2412
2413         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2414         dev = &rte_eth_devices[port_id];
2415
2416         if (udp_tunnel == NULL) {
2417                 PMD_DEBUG_TRACE("Invalid udp_tunnel parameter\n");
2418                 return -EINVAL;
2419         }
2420
2421         if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
2422                 PMD_DEBUG_TRACE("Invalid tunnel type\n");
2423                 return -EINVAL;
2424         }
2425
2426         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_del, -ENOTSUP);
2427         return (*dev->dev_ops->udp_tunnel_del)(dev, udp_tunnel);
2428 }
2429
2430 int
2431 rte_eth_led_on(uint8_t port_id)
2432 {
2433         struct rte_eth_dev *dev;
2434
2435         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2436         dev = &rte_eth_devices[port_id];
2437         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_on, -ENOTSUP);
2438         return (*dev->dev_ops->dev_led_on)(dev);
2439 }
2440
2441 int
2442 rte_eth_led_off(uint8_t port_id)
2443 {
2444         struct rte_eth_dev *dev;
2445
2446         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2447         dev = &rte_eth_devices[port_id];
2448         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_off, -ENOTSUP);
2449         return (*dev->dev_ops->dev_led_off)(dev);
2450 }
2451
2452 /*
2453  * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find
2454  * an empty spot.
2455  */
2456 static int
2457 get_mac_addr_index(uint8_t port_id, const struct ether_addr *addr)
2458 {
2459         struct rte_eth_dev_info dev_info;
2460         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
2461         unsigned i;
2462
2463         rte_eth_dev_info_get(port_id, &dev_info);
2464
2465         for (i = 0; i < dev_info.max_mac_addrs; i++)
2466                 if (memcmp(addr, &dev->data->mac_addrs[i], ETHER_ADDR_LEN) == 0)
2467                         return i;
2468
2469         return -1;
2470 }
2471
2472 static const struct ether_addr null_mac_addr;
2473
2474 int
2475 rte_eth_dev_mac_addr_add(uint8_t port_id, struct ether_addr *addr,
2476                         uint32_t pool)
2477 {
2478         struct rte_eth_dev *dev;
2479         int index;
2480         uint64_t pool_mask;
2481
2482         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2483         dev = &rte_eth_devices[port_id];
2484         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_add, -ENOTSUP);
2485
2486         if (is_zero_ether_addr(addr)) {
2487                 PMD_DEBUG_TRACE("port %d: Cannot add NULL MAC address\n",
2488                         port_id);
2489                 return -EINVAL;
2490         }
2491         if (pool >= ETH_64_POOLS) {
2492                 PMD_DEBUG_TRACE("pool id must be 0-%d\n", ETH_64_POOLS - 1);
2493                 return -EINVAL;
2494         }
2495
2496         index = get_mac_addr_index(port_id, addr);
2497         if (index < 0) {
2498                 index = get_mac_addr_index(port_id, &null_mac_addr);
2499                 if (index < 0) {
2500                         PMD_DEBUG_TRACE("port %d: MAC address array full\n",
2501                                 port_id);
2502                         return -ENOSPC;
2503                 }
2504         } else {
2505                 pool_mask = dev->data->mac_pool_sel[index];
2506
2507                 /* Check if both MAC address and pool is already there, and do nothing */
2508                 if (pool_mask & (1ULL << pool))
2509                         return 0;
2510         }
2511
2512         /* Update NIC */
2513         (*dev->dev_ops->mac_addr_add)(dev, addr, index, pool);
2514
2515         /* Update address in NIC data structure */
2516         ether_addr_copy(addr, &dev->data->mac_addrs[index]);
2517
2518         /* Update pool bitmap in NIC data structure */
2519         dev->data->mac_pool_sel[index] |= (1ULL << pool);
2520
2521         return 0;
2522 }
2523
2524 int
2525 rte_eth_dev_mac_addr_remove(uint8_t port_id, struct ether_addr *addr)
2526 {
2527         struct rte_eth_dev *dev;
2528         int index;
2529
2530         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2531         dev = &rte_eth_devices[port_id];
2532         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_remove, -ENOTSUP);
2533
2534         index = get_mac_addr_index(port_id, addr);
2535         if (index == 0) {
2536                 PMD_DEBUG_TRACE("port %d: Cannot remove default MAC address\n", port_id);
2537                 return -EADDRINUSE;
2538         } else if (index < 0)
2539                 return 0;  /* Do nothing if address wasn't found */
2540
2541         /* Update NIC */
2542         (*dev->dev_ops->mac_addr_remove)(dev, index);
2543
2544         /* Update address in NIC data structure */
2545         ether_addr_copy(&null_mac_addr, &dev->data->mac_addrs[index]);
2546
2547         /* reset pool bitmap */
2548         dev->data->mac_pool_sel[index] = 0;
2549
2550         return 0;
2551 }
2552
2553 int
2554 rte_eth_dev_default_mac_addr_set(uint8_t port_id, struct ether_addr *addr)
2555 {
2556         struct rte_eth_dev *dev;
2557
2558         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2559
2560         if (!is_valid_assigned_ether_addr(addr))
2561                 return -EINVAL;
2562
2563         dev = &rte_eth_devices[port_id];
2564         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_set, -ENOTSUP);
2565
2566         /* Update default address in NIC data structure */
2567         ether_addr_copy(addr, &dev->data->mac_addrs[0]);
2568
2569         (*dev->dev_ops->mac_addr_set)(dev, addr);
2570
2571         return 0;
2572 }
2573
2574 int
2575 rte_eth_dev_set_vf_rxmode(uint8_t port_id,  uint16_t vf,
2576                                 uint16_t rx_mode, uint8_t on)
2577 {
2578         uint16_t num_vfs;
2579         struct rte_eth_dev *dev;
2580         struct rte_eth_dev_info dev_info;
2581
2582         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2583
2584         dev = &rte_eth_devices[port_id];
2585         rte_eth_dev_info_get(port_id, &dev_info);
2586
2587         num_vfs = dev_info.max_vfs;
2588         if (vf > num_vfs) {
2589                 PMD_DEBUG_TRACE("set VF RX mode:invalid VF id %d\n", vf);
2590                 return -EINVAL;
2591         }
2592
2593         if (rx_mode == 0) {
2594                 PMD_DEBUG_TRACE("set VF RX mode:mode mask ca not be zero\n");
2595                 return -EINVAL;
2596         }
2597         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_vf_rx_mode, -ENOTSUP);
2598         return (*dev->dev_ops->set_vf_rx_mode)(dev, vf, rx_mode, on);
2599 }
2600
2601 /*
2602  * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find
2603  * an empty spot.
2604  */
2605 static int
2606 get_hash_mac_addr_index(uint8_t port_id, const struct ether_addr *addr)
2607 {
2608         struct rte_eth_dev_info dev_info;
2609         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
2610         unsigned i;
2611
2612         rte_eth_dev_info_get(port_id, &dev_info);
2613         if (!dev->data->hash_mac_addrs)
2614                 return -1;
2615
2616         for (i = 0; i < dev_info.max_hash_mac_addrs; i++)
2617                 if (memcmp(addr, &dev->data->hash_mac_addrs[i],
2618                         ETHER_ADDR_LEN) == 0)
2619                         return i;
2620
2621         return -1;
2622 }
2623
2624 int
2625 rte_eth_dev_uc_hash_table_set(uint8_t port_id, struct ether_addr *addr,
2626                                 uint8_t on)
2627 {
2628         int index;
2629         int ret;
2630         struct rte_eth_dev *dev;
2631
2632         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2633
2634         dev = &rte_eth_devices[port_id];
2635         if (is_zero_ether_addr(addr)) {
2636                 PMD_DEBUG_TRACE("port %d: Cannot add NULL MAC address\n",
2637                         port_id);
2638                 return -EINVAL;
2639         }
2640
2641         index = get_hash_mac_addr_index(port_id, addr);
2642         /* Check if it's already there, and do nothing */
2643         if ((index >= 0) && (on))
2644                 return 0;
2645
2646         if (index < 0) {
2647                 if (!on) {
2648                         PMD_DEBUG_TRACE("port %d: the MAC address was not "
2649                                 "set in UTA\n", port_id);
2650                         return -EINVAL;
2651                 }
2652
2653                 index = get_hash_mac_addr_index(port_id, &null_mac_addr);
2654                 if (index < 0) {
2655                         PMD_DEBUG_TRACE("port %d: MAC address array full\n",
2656                                         port_id);
2657                         return -ENOSPC;
2658                 }
2659         }
2660
2661         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_hash_table_set, -ENOTSUP);
2662         ret = (*dev->dev_ops->uc_hash_table_set)(dev, addr, on);
2663         if (ret == 0) {
2664                 /* Update address in NIC data structure */
2665                 if (on)
2666                         ether_addr_copy(addr,
2667                                         &dev->data->hash_mac_addrs[index]);
2668                 else
2669                         ether_addr_copy(&null_mac_addr,
2670                                         &dev->data->hash_mac_addrs[index]);
2671         }
2672
2673         return ret;
2674 }
2675
2676 int
2677 rte_eth_dev_uc_all_hash_table_set(uint8_t port_id, uint8_t on)
2678 {
2679         struct rte_eth_dev *dev;
2680
2681         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2682
2683         dev = &rte_eth_devices[port_id];
2684
2685         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_all_hash_table_set, -ENOTSUP);
2686         return (*dev->dev_ops->uc_all_hash_table_set)(dev, on);
2687 }
2688
2689 int
2690 rte_eth_dev_set_vf_rx(uint8_t port_id, uint16_t vf, uint8_t on)
2691 {
2692         uint16_t num_vfs;
2693         struct rte_eth_dev *dev;
2694         struct rte_eth_dev_info dev_info;
2695
2696         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2697
2698         dev = &rte_eth_devices[port_id];
2699         rte_eth_dev_info_get(port_id, &dev_info);
2700
2701         num_vfs = dev_info.max_vfs;
2702         if (vf > num_vfs) {
2703                 PMD_DEBUG_TRACE("port %d: invalid vf id\n", port_id);
2704                 return -EINVAL;
2705         }
2706
2707         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_vf_rx, -ENOTSUP);
2708         return (*dev->dev_ops->set_vf_rx)(dev, vf, on);
2709 }
2710
2711 int
2712 rte_eth_dev_set_vf_tx(uint8_t port_id, uint16_t vf, uint8_t on)
2713 {
2714         uint16_t num_vfs;
2715         struct rte_eth_dev *dev;
2716         struct rte_eth_dev_info dev_info;
2717
2718         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2719
2720         dev = &rte_eth_devices[port_id];
2721         rte_eth_dev_info_get(port_id, &dev_info);
2722
2723         num_vfs = dev_info.max_vfs;
2724         if (vf > num_vfs) {
2725                 PMD_DEBUG_TRACE("set pool tx:invalid pool id=%d\n", vf);
2726                 return -EINVAL;
2727         }
2728
2729         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_vf_tx, -ENOTSUP);
2730         return (*dev->dev_ops->set_vf_tx)(dev, vf, on);
2731 }
2732
2733 int
2734 rte_eth_dev_set_vf_vlan_filter(uint8_t port_id, uint16_t vlan_id,
2735                                uint64_t vf_mask, uint8_t vlan_on)
2736 {
2737         struct rte_eth_dev *dev;
2738
2739         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2740
2741         dev = &rte_eth_devices[port_id];
2742
2743         if (vlan_id > ETHER_MAX_VLAN_ID) {
2744                 PMD_DEBUG_TRACE("VF VLAN filter:invalid VLAN id=%d\n",
2745                         vlan_id);
2746                 return -EINVAL;
2747         }
2748
2749         if (vf_mask == 0) {
2750                 PMD_DEBUG_TRACE("VF VLAN filter:pool_mask can not be 0\n");
2751                 return -EINVAL;
2752         }
2753
2754         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_vf_vlan_filter, -ENOTSUP);
2755         return (*dev->dev_ops->set_vf_vlan_filter)(dev, vlan_id,
2756                                                    vf_mask, vlan_on);
2757 }
2758
2759 int rte_eth_set_queue_rate_limit(uint8_t port_id, uint16_t queue_idx,
2760                                         uint16_t tx_rate)
2761 {
2762         struct rte_eth_dev *dev;
2763         struct rte_eth_dev_info dev_info;
2764         struct rte_eth_link link;
2765
2766         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2767
2768         dev = &rte_eth_devices[port_id];
2769         rte_eth_dev_info_get(port_id, &dev_info);
2770         link = dev->data->dev_link;
2771
2772         if (queue_idx > dev_info.max_tx_queues) {
2773                 PMD_DEBUG_TRACE("set queue rate limit:port %d: "
2774                                 "invalid queue id=%d\n", port_id, queue_idx);
2775                 return -EINVAL;
2776         }
2777
2778         if (tx_rate > link.link_speed) {
2779                 PMD_DEBUG_TRACE("set queue rate limit:invalid tx_rate=%d, "
2780                                 "bigger than link speed= %d\n",
2781                         tx_rate, link.link_speed);
2782                 return -EINVAL;
2783         }
2784
2785         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_queue_rate_limit, -ENOTSUP);
2786         return (*dev->dev_ops->set_queue_rate_limit)(dev, queue_idx, tx_rate);
2787 }
2788
2789 int rte_eth_set_vf_rate_limit(uint8_t port_id, uint16_t vf, uint16_t tx_rate,
2790                                 uint64_t q_msk)
2791 {
2792         struct rte_eth_dev *dev;
2793         struct rte_eth_dev_info dev_info;
2794         struct rte_eth_link link;
2795
2796         if (q_msk == 0)
2797                 return 0;
2798
2799         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2800
2801         dev = &rte_eth_devices[port_id];
2802         rte_eth_dev_info_get(port_id, &dev_info);
2803         link = dev->data->dev_link;
2804
2805         if (vf > dev_info.max_vfs) {
2806                 PMD_DEBUG_TRACE("set VF rate limit:port %d: "
2807                                 "invalid vf id=%d\n", port_id, vf);
2808                 return -EINVAL;
2809         }
2810
2811         if (tx_rate > link.link_speed) {
2812                 PMD_DEBUG_TRACE("set VF rate limit:invalid tx_rate=%d, "
2813                                 "bigger than link speed= %d\n",
2814                                 tx_rate, link.link_speed);
2815                 return -EINVAL;
2816         }
2817
2818         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_vf_rate_limit, -ENOTSUP);
2819         return (*dev->dev_ops->set_vf_rate_limit)(dev, vf, tx_rate, q_msk);
2820 }
2821
2822 int
2823 rte_eth_mirror_rule_set(uint8_t port_id,
2824                         struct rte_eth_mirror_conf *mirror_conf,
2825                         uint8_t rule_id, uint8_t on)
2826 {
2827         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
2828
2829         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2830         if (mirror_conf->rule_type == 0) {
2831                 PMD_DEBUG_TRACE("mirror rule type can not be 0.\n");
2832                 return -EINVAL;
2833         }
2834
2835         if (mirror_conf->dst_pool >= ETH_64_POOLS) {
2836                 PMD_DEBUG_TRACE("Invalid dst pool, pool id must be 0-%d\n",
2837                                 ETH_64_POOLS - 1);
2838                 return -EINVAL;
2839         }
2840
2841         if ((mirror_conf->rule_type & (ETH_MIRROR_VIRTUAL_POOL_UP |
2842              ETH_MIRROR_VIRTUAL_POOL_DOWN)) &&
2843             (mirror_conf->pool_mask == 0)) {
2844                 PMD_DEBUG_TRACE("Invalid mirror pool, pool mask can not be 0.\n");
2845                 return -EINVAL;
2846         }
2847
2848         if ((mirror_conf->rule_type & ETH_MIRROR_VLAN) &&
2849             mirror_conf->vlan.vlan_mask == 0) {
2850                 PMD_DEBUG_TRACE("Invalid vlan mask, vlan mask can not be 0.\n");
2851                 return -EINVAL;
2852         }
2853
2854         dev = &rte_eth_devices[port_id];
2855         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mirror_rule_set, -ENOTSUP);
2856
2857         return (*dev->dev_ops->mirror_rule_set)(dev, mirror_conf, rule_id, on);
2858 }
2859
2860 int
2861 rte_eth_mirror_rule_reset(uint8_t port_id, uint8_t rule_id)
2862 {
2863         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
2864
2865         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2866
2867         dev = &rte_eth_devices[port_id];
2868         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mirror_rule_reset, -ENOTSUP);
2869
2870         return (*dev->dev_ops->mirror_rule_reset)(dev, rule_id);
2871 }
2872
2873 #ifdef RTE_LIBRTE_ETHDEV_DEBUG
2874 uint16_t
2875 rte_eth_rx_burst(uint8_t port_id, uint16_t queue_id,
2876                  struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
2877 {
2878         struct rte_eth_dev *dev;
2879
2880         VALID_PORTID_OR_ERR_RET(port_id, 0);
2881
2882         dev = &rte_eth_devices[port_id];
2883         FUNC_PTR_OR_ERR_RET(*dev->rx_pkt_burst, 0);
2884         if (queue_id >= dev->data->nb_rx_queues) {
2885                 PMD_DEBUG_TRACE("Invalid RX queue_id=%d\n", queue_id);
2886                 return 0;
2887         }
2888         return (*dev->rx_pkt_burst)(dev->data->rx_queues[queue_id],
2889                                                 rx_pkts, nb_pkts);
2890 }
2891
2892 uint16_t
2893 rte_eth_tx_burst(uint8_t port_id, uint16_t queue_id,
2894                  struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2895 {
2896         struct rte_eth_dev *dev;
2897
2898         VALID_PORTID_OR_ERR_RET(port_id, 0);
2899
2900         dev = &rte_eth_devices[port_id];
2901
2902         FUNC_PTR_OR_ERR_RET(*dev->tx_pkt_burst, 0);
2903         if (queue_id >= dev->data->nb_tx_queues) {
2904                 PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", queue_id);
2905                 return 0;
2906         }
2907         return (*dev->tx_pkt_burst)(dev->data->tx_queues[queue_id],
2908                                                 tx_pkts, nb_pkts);
2909 }
2910
2911 uint32_t
2912 rte_eth_rx_queue_count(uint8_t port_id, uint16_t queue_id)
2913 {
2914         struct rte_eth_dev *dev;
2915
2916         VALID_PORTID_OR_ERR_RET(port_id, 0);
2917
2918         dev = &rte_eth_devices[port_id];
2919         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_count, 0);
2920         return (*dev->dev_ops->rx_queue_count)(dev, queue_id);
2921 }
2922
2923 int
2924 rte_eth_rx_descriptor_done(uint8_t port_id, uint16_t queue_id, uint16_t offset)
2925 {
2926         struct rte_eth_dev *dev;
2927
2928         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2929
2930         dev = &rte_eth_devices[port_id];
2931         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_descriptor_done, -ENOTSUP);
2932         return (*dev->dev_ops->rx_descriptor_done)(dev->data->rx_queues[queue_id],
2933                                                    offset);
2934 }
2935 #endif
2936
2937 int
2938 rte_eth_dev_callback_register(uint8_t port_id,
2939                         enum rte_eth_event_type event,
2940                         rte_eth_dev_cb_fn cb_fn, void *cb_arg)
2941 {
2942         struct rte_eth_dev *dev;
2943         struct rte_eth_dev_callback *user_cb;
2944
2945         if (!cb_fn)
2946                 return -EINVAL;
2947
2948         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2949
2950         dev = &rte_eth_devices[port_id];
2951         rte_spinlock_lock(&rte_eth_dev_cb_lock);
2952
2953         TAILQ_FOREACH(user_cb, &(dev->link_intr_cbs), next) {
2954                 if (user_cb->cb_fn == cb_fn &&
2955                         user_cb->cb_arg == cb_arg &&
2956                         user_cb->event == event) {
2957                         break;
2958                 }
2959         }
2960
2961         /* create a new callback. */
2962         if (user_cb == NULL &&
2963             (user_cb = rte_zmalloc("INTR_USER_CALLBACK",
2964                                    sizeof(struct rte_eth_dev_callback), 0))) {
2965                 user_cb->cb_fn = cb_fn;
2966                 user_cb->cb_arg = cb_arg;
2967                 user_cb->event = event;
2968                 TAILQ_INSERT_TAIL(&(dev->link_intr_cbs), user_cb, next);
2969         }
2970
2971         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
2972         return (user_cb == NULL) ? -ENOMEM : 0;
2973 }
2974
2975 int
2976 rte_eth_dev_callback_unregister(uint8_t port_id,
2977                         enum rte_eth_event_type event,
2978                         rte_eth_dev_cb_fn cb_fn, void *cb_arg)
2979 {
2980         int ret;
2981         struct rte_eth_dev *dev;
2982         struct rte_eth_dev_callback *cb, *next;
2983
2984         if (!cb_fn)
2985                 return -EINVAL;
2986
2987         VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2988
2989         dev = &rte_eth_devices[port_id];
2990         rte_spinlock_lock(&rte_eth_dev_cb_lock);
2991
2992         ret = 0;
2993         for (cb = TAILQ_FIRST(&dev->link_intr_cbs); cb != NULL; cb = next) {
2994
2995                 next = TAILQ_NEXT(cb, next);
2996
2997                 if (cb->cb_fn != cb_fn || cb->event != event ||
2998                                 (cb->cb_arg != (void *)-1 &&
2999                                 cb->cb_arg != cb_arg))
3000                         continue;
3001
3002                 /*
3003                  * if this callback is not executing right now,
3004                  * then remove it.
3005                  */
3006                 if (cb->active == 0) {
3007                         TAILQ_REMOVE(&(dev->link_intr_cbs), cb, next);
3008                         rte_free(cb);
3009                 } else {
3010                         ret = -EAGAIN;
3011                 }
3012         }
3013
3014         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3015         return ret;
3016 }
3017
3018 void
3019 _rte_eth_dev_callback_process(struct rte_eth_dev *dev,
3020         enum rte_eth_event_type event)
3021 {
3022         struct rte_eth_dev_callback *cb_lst;
3023         struct rte_eth_dev_callback dev_cb;
3024
3025         rte_spinlock_lock(&rte_eth_dev_cb_lock);
3026         TAILQ_FOREACH(cb_lst, &(dev->link_intr_cbs), next) {
3027                 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
3028                         continue;
3029                 dev_cb = *cb_lst;
3030                 cb_lst->active = 1;
3031                 rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3032                 dev_cb.cb_fn(dev->data->port_id, dev_cb.event,
3033                                                 dev_cb.cb_arg);
3034                 rte_spinlock_lock(&rte_eth_dev_cb_lock);
3035                 cb_lst->active = 0;
3036         }
3037         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3038 }
3039 #ifdef RTE_NIC_BYPASS
3040 int rte_eth_dev_bypass_init(uint8_t port_id)
3041 {
3042         struct rte_eth_dev *dev;
3043
3044         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3045
3046         dev = &rte_eth_devices[port_id];
3047         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_init, -ENOTSUP);
3048         (*dev->dev_ops->bypass_init)(dev);
3049         return 0;
3050 }
3051
3052 int
3053 rte_eth_dev_bypass_state_show(uint8_t port_id, uint32_t *state)
3054 {
3055         struct rte_eth_dev *dev;
3056
3057         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3058
3059         dev = &rte_eth_devices[port_id];
3060         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_state_show, -ENOTSUP);
3061         (*dev->dev_ops->bypass_state_show)(dev, state);
3062         return 0;
3063 }
3064
3065 int
3066 rte_eth_dev_bypass_state_set(uint8_t port_id, uint32_t *new_state)
3067 {
3068         struct rte_eth_dev *dev;
3069
3070         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3071
3072         dev = &rte_eth_devices[port_id];
3073         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_state_set, -ENOTSUP);
3074         (*dev->dev_ops->bypass_state_set)(dev, new_state);
3075         return 0;
3076 }
3077
3078 int
3079 rte_eth_dev_bypass_event_show(uint8_t port_id, uint32_t event, uint32_t *state)
3080 {
3081         struct rte_eth_dev *dev;
3082
3083         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3084
3085         dev = &rte_eth_devices[port_id];
3086         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_state_show, -ENOTSUP);
3087         (*dev->dev_ops->bypass_event_show)(dev, event, state);
3088         return 0;
3089 }
3090
3091 int
3092 rte_eth_dev_bypass_event_store(uint8_t port_id, uint32_t event, uint32_t state)
3093 {
3094         struct rte_eth_dev *dev;
3095
3096         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3097
3098         dev = &rte_eth_devices[port_id];
3099
3100         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_event_set, -ENOTSUP);
3101         (*dev->dev_ops->bypass_event_set)(dev, event, state);
3102         return 0;
3103 }
3104
3105 int
3106 rte_eth_dev_wd_timeout_store(uint8_t port_id, uint32_t timeout)
3107 {
3108         struct rte_eth_dev *dev;
3109
3110         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3111
3112         dev = &rte_eth_devices[port_id];
3113
3114         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_wd_timeout_set, -ENOTSUP);
3115         (*dev->dev_ops->bypass_wd_timeout_set)(dev, timeout);
3116         return 0;
3117 }
3118
3119 int
3120 rte_eth_dev_bypass_ver_show(uint8_t port_id, uint32_t *ver)
3121 {
3122         struct rte_eth_dev *dev;
3123
3124         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3125
3126         dev = &rte_eth_devices[port_id];
3127
3128         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_ver_show, -ENOTSUP);
3129         (*dev->dev_ops->bypass_ver_show)(dev, ver);
3130         return 0;
3131 }
3132
3133 int
3134 rte_eth_dev_bypass_wd_timeout_show(uint8_t port_id, uint32_t *wd_timeout)
3135 {
3136         struct rte_eth_dev *dev;
3137
3138         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3139
3140         dev = &rte_eth_devices[port_id];
3141
3142         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_wd_timeout_show, -ENOTSUP);
3143         (*dev->dev_ops->bypass_wd_timeout_show)(dev, wd_timeout);
3144         return 0;
3145 }
3146
3147 int
3148 rte_eth_dev_bypass_wd_reset(uint8_t port_id)
3149 {
3150         struct rte_eth_dev *dev;
3151
3152         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3153
3154         dev = &rte_eth_devices[port_id];
3155
3156         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->bypass_wd_reset, -ENOTSUP);
3157         (*dev->dev_ops->bypass_wd_reset)(dev);
3158         return 0;
3159 }
3160 #endif
3161
3162 int
3163 rte_eth_dev_filter_supported(uint8_t port_id, enum rte_filter_type filter_type)
3164 {
3165         struct rte_eth_dev *dev;
3166
3167         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3168
3169         dev = &rte_eth_devices[port_id];
3170         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->filter_ctrl, -ENOTSUP);
3171         return (*dev->dev_ops->filter_ctrl)(dev, filter_type,
3172                                 RTE_ETH_FILTER_NOP, NULL);
3173 }
3174
3175 int
3176 rte_eth_dev_filter_ctrl(uint8_t port_id, enum rte_filter_type filter_type,
3177                        enum rte_filter_op filter_op, void *arg)
3178 {
3179         struct rte_eth_dev *dev;
3180
3181         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3182
3183         dev = &rte_eth_devices[port_id];
3184         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->filter_ctrl, -ENOTSUP);
3185         return (*dev->dev_ops->filter_ctrl)(dev, filter_type, filter_op, arg);
3186 }
3187
3188 void *
3189 rte_eth_add_rx_callback(uint8_t port_id, uint16_t queue_id,
3190                 rte_rx_callback_fn fn, void *user_param)
3191 {
3192 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3193         rte_errno = ENOTSUP;
3194         return NULL;
3195 #endif
3196         /* check input parameters */
3197         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
3198                     queue_id >= rte_eth_devices[port_id].data->nb_rx_queues) {
3199                 rte_errno = EINVAL;
3200                 return NULL;
3201         }
3202
3203         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
3204
3205         if (cb == NULL) {
3206                 rte_errno = ENOMEM;
3207                 return NULL;
3208         }
3209
3210         cb->fn.rx = fn;
3211         cb->param = user_param;
3212
3213         /* Add the callbacks in fifo order. */
3214         struct rte_eth_rxtx_callback *tail =
3215                 rte_eth_devices[port_id].post_rx_burst_cbs[queue_id];
3216
3217         if (!tail) {
3218                 rte_eth_devices[port_id].post_rx_burst_cbs[queue_id] = cb;
3219
3220         } else {
3221                 while (tail->next)
3222                         tail = tail->next;
3223                 tail->next = cb;
3224         }
3225
3226         return cb;
3227 }
3228
3229 void *
3230 rte_eth_add_tx_callback(uint8_t port_id, uint16_t queue_id,
3231                 rte_tx_callback_fn fn, void *user_param)
3232 {
3233 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3234         rte_errno = ENOTSUP;
3235         return NULL;
3236 #endif
3237         /* check input parameters */
3238         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
3239                     queue_id >= rte_eth_devices[port_id].data->nb_tx_queues) {
3240                 rte_errno = EINVAL;
3241                 return NULL;
3242         }
3243
3244         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
3245
3246         if (cb == NULL) {
3247                 rte_errno = ENOMEM;
3248                 return NULL;
3249         }
3250
3251         cb->fn.tx = fn;
3252         cb->param = user_param;
3253
3254         /* Add the callbacks in fifo order. */
3255         struct rte_eth_rxtx_callback *tail =
3256                 rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id];
3257
3258         if (!tail) {
3259                 rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id] = cb;
3260
3261         } else {
3262                 while (tail->next)
3263                         tail = tail->next;
3264                 tail->next = cb;
3265         }
3266
3267         return cb;
3268 }
3269
3270 int
3271 rte_eth_remove_rx_callback(uint8_t port_id, uint16_t queue_id,
3272                 struct rte_eth_rxtx_callback *user_cb)
3273 {
3274 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3275         return -ENOTSUP;
3276 #endif
3277         /* Check input parameters. */
3278         if (!rte_eth_dev_is_valid_port(port_id) || user_cb == NULL ||
3279                     queue_id >= rte_eth_devices[port_id].data->nb_rx_queues) {
3280                 return -EINVAL;
3281         }
3282
3283         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3284         struct rte_eth_rxtx_callback *cb = dev->post_rx_burst_cbs[queue_id];
3285         struct rte_eth_rxtx_callback *prev_cb;
3286
3287         /* Reset head pointer and remove user cb if first in the list. */
3288         if (cb == user_cb) {
3289                 dev->post_rx_burst_cbs[queue_id] = user_cb->next;
3290                 return 0;
3291         }
3292
3293         /* Remove the user cb from the callback list. */
3294         do {
3295                 prev_cb = cb;
3296                 cb = cb->next;
3297
3298                 if (cb == user_cb) {
3299                         prev_cb->next = user_cb->next;
3300                         return 0;
3301                 }
3302
3303         } while (cb != NULL);
3304
3305         /* Callback wasn't found. */
3306         return -EINVAL;
3307 }
3308
3309 int
3310 rte_eth_remove_tx_callback(uint8_t port_id, uint16_t queue_id,
3311                 struct rte_eth_rxtx_callback *user_cb)
3312 {
3313 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3314         return -ENOTSUP;
3315 #endif
3316         /* Check input parameters. */
3317         if (!rte_eth_dev_is_valid_port(port_id) || user_cb == NULL ||
3318                     queue_id >= rte_eth_devices[port_id].data->nb_tx_queues) {
3319                 return -EINVAL;
3320         }
3321
3322         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3323         struct rte_eth_rxtx_callback *cb = dev->pre_tx_burst_cbs[queue_id];
3324         struct rte_eth_rxtx_callback *prev_cb;
3325
3326         /* Reset head pointer and remove user cb if first in the list. */
3327         if (cb == user_cb) {
3328                 dev->pre_tx_burst_cbs[queue_id] = user_cb->next;
3329                 return 0;
3330         }
3331
3332         /* Remove the user cb from the callback list. */
3333         do {
3334                 prev_cb = cb;
3335                 cb = cb->next;
3336
3337                 if (cb == user_cb) {
3338                         prev_cb->next = user_cb->next;
3339                         return 0;
3340                 }
3341
3342         } while (cb != NULL);
3343
3344         /* Callback wasn't found. */
3345         return -EINVAL;
3346 }
3347
3348 int
3349 rte_eth_dev_set_mc_addr_list(uint8_t port_id,
3350                              struct ether_addr *mc_addr_set,
3351                              uint32_t nb_mc_addr)
3352 {
3353         struct rte_eth_dev *dev;
3354
3355         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3356
3357         dev = &rte_eth_devices[port_id];
3358         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_mc_addr_list, -ENOTSUP);
3359         return dev->dev_ops->set_mc_addr_list(dev, mc_addr_set, nb_mc_addr);
3360 }
3361
3362 int
3363 rte_eth_timesync_enable(uint8_t port_id)
3364 {
3365         struct rte_eth_dev *dev;
3366
3367         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3368         dev = &rte_eth_devices[port_id];
3369
3370         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_enable, -ENOTSUP);
3371         return (*dev->dev_ops->timesync_enable)(dev);
3372 }
3373
3374 int
3375 rte_eth_timesync_disable(uint8_t port_id)
3376 {
3377         struct rte_eth_dev *dev;
3378
3379         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3380         dev = &rte_eth_devices[port_id];
3381
3382         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_disable, -ENOTSUP);
3383         return (*dev->dev_ops->timesync_disable)(dev);
3384 }
3385
3386 int
3387 rte_eth_timesync_read_rx_timestamp(uint8_t port_id, struct timespec *timestamp,
3388                                    uint32_t flags)
3389 {
3390         struct rte_eth_dev *dev;
3391
3392         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3393         dev = &rte_eth_devices[port_id];
3394
3395         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_rx_timestamp, -ENOTSUP);
3396         return (*dev->dev_ops->timesync_read_rx_timestamp)(dev, timestamp, flags);
3397 }
3398
3399 int
3400 rte_eth_timesync_read_tx_timestamp(uint8_t port_id, struct timespec *timestamp)
3401 {
3402         struct rte_eth_dev *dev;
3403
3404         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3405         dev = &rte_eth_devices[port_id];
3406
3407         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_tx_timestamp, -ENOTSUP);
3408         return (*dev->dev_ops->timesync_read_tx_timestamp)(dev, timestamp);
3409 }
3410
3411 int
3412 rte_eth_dev_get_reg_length(uint8_t port_id)
3413 {
3414         struct rte_eth_dev *dev;
3415
3416         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3417
3418         dev = &rte_eth_devices[port_id];
3419         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_reg_length, -ENOTSUP);
3420         return (*dev->dev_ops->get_reg_length)(dev);
3421 }
3422
3423 int
3424 rte_eth_dev_get_reg_info(uint8_t port_id, struct rte_dev_reg_info *info)
3425 {
3426         struct rte_eth_dev *dev;
3427
3428         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3429
3430         dev = &rte_eth_devices[port_id];
3431         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_reg, -ENOTSUP);
3432         return (*dev->dev_ops->get_reg)(dev, info);
3433 }
3434
3435 int
3436 rte_eth_dev_get_eeprom_length(uint8_t port_id)
3437 {
3438         struct rte_eth_dev *dev;
3439
3440         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3441
3442         dev = &rte_eth_devices[port_id];
3443         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom_length, -ENOTSUP);
3444         return (*dev->dev_ops->get_eeprom_length)(dev);
3445 }
3446
3447 int
3448 rte_eth_dev_get_eeprom(uint8_t port_id, struct rte_dev_eeprom_info *info)
3449 {
3450         struct rte_eth_dev *dev;
3451
3452         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3453
3454         dev = &rte_eth_devices[port_id];
3455         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom, -ENOTSUP);
3456         return (*dev->dev_ops->get_eeprom)(dev, info);
3457 }
3458
3459 int
3460 rte_eth_dev_set_eeprom(uint8_t port_id, struct rte_dev_eeprom_info *info)
3461 {
3462         struct rte_eth_dev *dev;
3463
3464         VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3465
3466         dev = &rte_eth_devices[port_id];
3467         FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_eeprom, -ENOTSUP);
3468         return (*dev->dev_ops->set_eeprom)(dev, info);
3469 }