lib/librte_ether/rte_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <sys/types.h>
  35 #include <sys/queue.h>
  36 #include <ctype.h>
  37 #include <stdio.h>
  38 #include <stdlib.h>
  39 #include <string.h>
  40 #include <stdarg.h>
  41 #include <errno.h>
  42 #include <stdint.h>
  43 #include <inttypes.h>
  44 #include <netinet/in.h>
  45
  46 #include <rte_byteorder.h>
  47 #include <rte_log.h>
  48 #include <rte_debug.h>
  49 #include <rte_interrupts.h>
  50 #include <rte_pci.h>
  51 #include <rte_memory.h>
  52 #include <rte_memcpy.h>
  53 #include <rte_memzone.h>
  54 #include <rte_launch.h>
  55 #include <rte_eal.h>
  56 #include <rte_per_lcore.h>
  57 #include <rte_lcore.h>
  58 #include <rte_atomic.h>
  59 #include <rte_branch_prediction.h>
  60 #include <rte_common.h>
  61 #include <rte_mempool.h>
  62 #include <rte_malloc.h>
  63 #include <rte_mbuf.h>
  64 #include <rte_errno.h>
  65 #include <rte_spinlock.h>
  66 #include <rte_string_fns.h>
  67
  68 #include "rte_ether.h"
  69 #include "rte_ethdev.h"
  70
  71 static const char *MZ_RTE_ETH_DEV_DATA = "rte_eth_dev_data";
  72 struct rte_eth_dev rte_eth_devices[RTE_MAX_ETHPORTS];
  73 static struct rte_eth_dev_data *rte_eth_dev_data;
  74 static uint8_t eth_dev_last_created_port;
  75 static uint8_t nb_ports;
  76
  77 /* spinlock for eth device callbacks */
  78 static rte_spinlock_t rte_eth_dev_cb_lock = RTE_SPINLOCK_INITIALIZER;
  79
  80 /* spinlock for add/remove rx callbacks */
  81 static rte_spinlock_t rte_eth_rx_cb_lock = RTE_SPINLOCK_INITIALIZER;
  82
  83 /* spinlock for add/remove tx callbacks */
  84 static rte_spinlock_t rte_eth_tx_cb_lock = RTE_SPINLOCK_INITIALIZER;
  85
  86 /* store statistics names and its offset in stats structure  */
  87 struct rte_eth_xstats_name_off {
  88         char name[RTE_ETH_XSTATS_NAME_SIZE];
  89         unsigned offset;
  90 };
  91
  92 static const struct rte_eth_xstats_name_off rte_stats_strings[] = {
  93         {"rx_good_packets", offsetof(struct rte_eth_stats, ipackets)},
  94         {"tx_good_packets", offsetof(struct rte_eth_stats, opackets)},
  95         {"rx_good_bytes", offsetof(struct rte_eth_stats, ibytes)},
  96         {"tx_good_bytes", offsetof(struct rte_eth_stats, obytes)},
  97         {"rx_errors", offsetof(struct rte_eth_stats, ierrors)},
  98         {"tx_errors", offsetof(struct rte_eth_stats, oerrors)},
  99         {"rx_mbuf_allocation_errors", offsetof(struct rte_eth_stats,
 100                 rx_nombuf)},
 101 };
 102
 103 #define RTE_NB_STATS (sizeof(rte_stats_strings) / sizeof(rte_stats_strings[0]))
 104
 105 static const struct rte_eth_xstats_name_off rte_rxq_stats_strings[] = {
 106         {"packets", offsetof(struct rte_eth_stats, q_ipackets)},
 107         {"bytes", offsetof(struct rte_eth_stats, q_ibytes)},
 108         {"errors", offsetof(struct rte_eth_stats, q_errors)},
 109 };
 110
 111 #define RTE_NB_RXQ_STATS (sizeof(rte_rxq_stats_strings) /       \
 112                 sizeof(rte_rxq_stats_strings[0]))
 113
 114 static const struct rte_eth_xstats_name_off rte_txq_stats_strings[] = {
 115         {"packets", offsetof(struct rte_eth_stats, q_opackets)},
 116         {"bytes", offsetof(struct rte_eth_stats, q_obytes)},
 117 };
 118 #define RTE_NB_TXQ_STATS (sizeof(rte_txq_stats_strings) /       \
 119                 sizeof(rte_txq_stats_strings[0]))
 120
 121
 122 /**
 123  * The user application callback description.
 124  *
 125  * It contains callback address to be registered by user application,
 126  * the pointer to the parameters for callback, and the event type.
 127  */
 128 struct rte_eth_dev_callback {
 129         TAILQ_ENTRY(rte_eth_dev_callback) next; /**< Callbacks list */
 130         rte_eth_dev_cb_fn cb_fn;                /**< Callback address */
 131         void *cb_arg;                           /**< Parameter for callback */
 132         enum rte_eth_event_type event;          /**< Interrupt event type */
 133         uint32_t active;                        /**< Callback is executing */
 134 };
 135
 136 enum {
 137         STAT_QMAP_TX = 0,
 138         STAT_QMAP_RX
 139 };
 140
 141 uint8_t
 142 rte_eth_find_next(uint8_t port_id)
 143 {
 144         while (port_id < RTE_MAX_ETHPORTS &&
 145                rte_eth_devices[port_id].state != RTE_ETH_DEV_ATTACHED)
 146                 port_id++;
 147
 148         if (port_id >= RTE_MAX_ETHPORTS)
 149                 return RTE_MAX_ETHPORTS;
 150
 151         return port_id;
 152 }
 153
 154 static void
 155 rte_eth_dev_data_alloc(void)
 156 {
 157         const unsigned flags = 0;
 158         const struct rte_memzone *mz;
 159
 160         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 161                 mz = rte_memzone_reserve(MZ_RTE_ETH_DEV_DATA,
 162                                 RTE_MAX_ETHPORTS * sizeof(*rte_eth_dev_data),
 163                                 rte_socket_id(), flags);
 164         } else
 165                 mz = rte_memzone_lookup(MZ_RTE_ETH_DEV_DATA);
 166         if (mz == NULL)
 167                 rte_panic("Cannot allocate memzone for ethernet port data\n");
 168
 169         rte_eth_dev_data = mz->addr;
 170         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
 171                 memset(rte_eth_dev_data, 0,
 172                                 RTE_MAX_ETHPORTS * sizeof(*rte_eth_dev_data));
 173 }
 174
 175 struct rte_eth_dev *
 176 rte_eth_dev_allocated(const char *name)
 177 {
 178         unsigned i;
 179
 180         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 181                 if ((rte_eth_devices[i].state == RTE_ETH_DEV_ATTACHED) &&
 182                     strcmp(rte_eth_devices[i].data->name, name) == 0)
 183                         return &rte_eth_devices[i];
 184         }
 185         return NULL;
 186 }
 187
 188 static uint8_t
 189 rte_eth_dev_find_free_port(void)
 190 {
 191         unsigned i;
 192
 193         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 194                 if (rte_eth_devices[i].state == RTE_ETH_DEV_UNUSED)
 195                         return i;
 196         }
 197         return RTE_MAX_ETHPORTS;
 198 }
 199
 200 static struct rte_eth_dev *
 201 eth_dev_get(uint8_t port_id)
 202 {
 203         struct rte_eth_dev *eth_dev = &rte_eth_devices[port_id];
 204
 205         eth_dev->data = &rte_eth_dev_data[port_id];
 206         eth_dev->state = RTE_ETH_DEV_ATTACHED;
 207         TAILQ_INIT(&(eth_dev->link_intr_cbs));
 208
 209         eth_dev_last_created_port = port_id;
 210         nb_ports++;
 211
 212         return eth_dev;
 213 }
 214
 215 struct rte_eth_dev *
 216 rte_eth_dev_allocate(const char *name)
 217 {
 218         uint8_t port_id;
 219         struct rte_eth_dev *eth_dev;
 220
 221         port_id = rte_eth_dev_find_free_port();
 222         if (port_id == RTE_MAX_ETHPORTS) {
 223                 RTE_PMD_DEBUG_TRACE("Reached maximum number of Ethernet ports\n");
 224                 return NULL;
 225         }
 226
 227         if (rte_eth_dev_data == NULL)
 228                 rte_eth_dev_data_alloc();
 229
 230         if (rte_eth_dev_allocated(name) != NULL) {
 231                 RTE_PMD_DEBUG_TRACE("Ethernet Device with name %s already allocated!\n",
 232                                 name);
 233                 return NULL;
 234         }
 235
 236         memset(&rte_eth_dev_data[port_id], 0, sizeof(struct rte_eth_dev_data));
 237         eth_dev = eth_dev_get(port_id);
 238         snprintf(eth_dev->data->name, sizeof(eth_dev->data->name), "%s", name);
 239         eth_dev->data->port_id = port_id;
 240         eth_dev->data->mtu = ETHER_MTU;
 241
 242         return eth_dev;
 243 }
 244
 245 /*
 246  * Attach to a port already registered by the primary process, which
 247  * makes sure that the same device would have the same port id both
 248  * in the primary and secondary process.
 249  */
 250 struct rte_eth_dev *
 251 rte_eth_dev_attach_secondary(const char *name)
 252 {
 253         uint8_t i;
 254         struct rte_eth_dev *eth_dev;
 255
 256         if (rte_eth_dev_data == NULL)
 257                 rte_eth_dev_data_alloc();
 258
 259         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 260                 if (strcmp(rte_eth_dev_data[i].name, name) == 0)
 261                         break;
 262         }
 263         if (i == RTE_MAX_ETHPORTS) {
 264                 RTE_PMD_DEBUG_TRACE(
 265                         "device %s is not driven by the primary process\n",
 266                         name);
 267                 return NULL;
 268         }
 269
 270         eth_dev = eth_dev_get(i);
 271         RTE_ASSERT(eth_dev->data->port_id == i);
 272
 273         return eth_dev;
 274 }
 275
 276 int
 277 rte_eth_dev_release_port(struct rte_eth_dev *eth_dev)
 278 {
 279         if (eth_dev == NULL)
 280                 return -EINVAL;
 281
 282         eth_dev->state = RTE_ETH_DEV_UNUSED;
 283         nb_ports--;
 284         return 0;
 285 }
 286
 287 int
 288 rte_eth_dev_is_valid_port(uint8_t port_id)
 289 {
 290         if (port_id >= RTE_MAX_ETHPORTS ||
 291             rte_eth_devices[port_id].state != RTE_ETH_DEV_ATTACHED)
 292                 return 0;
 293         else
 294                 return 1;
 295 }
 296
 297 int
 298 rte_eth_dev_socket_id(uint8_t port_id)
 299 {
 300         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -1);
 301         return rte_eth_devices[port_id].data->numa_node;
 302 }
 303
 304 uint8_t
 305 rte_eth_dev_count(void)
 306 {
 307         return nb_ports;
 308 }
 309
 310 int
 311 rte_eth_dev_get_name_by_port(uint8_t port_id, char *name)
 312 {
 313         char *tmp;
 314
 315         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 316
 317         if (name == NULL) {
 318                 RTE_PMD_DEBUG_TRACE("Null pointer is specified\n");
 319                 return -EINVAL;
 320         }
 321
 322         /* shouldn't check 'rte_eth_devices[i].data',
 323          * because it might be overwritten by VDEV PMD */
 324         tmp = rte_eth_dev_data[port_id].name;
 325         strcpy(name, tmp);
 326         return 0;
 327 }
 328
 329 int
 330 rte_eth_dev_get_port_by_name(const char *name, uint8_t *port_id)
 331 {
 332         int i;
 333
 334         if (name == NULL) {
 335                 RTE_PMD_DEBUG_TRACE("Null pointer is specified\n");
 336                 return -EINVAL;
 337         }
 338
 339         if (!nb_ports)
 340                 return -ENODEV;
 341
 342         RTE_ETH_FOREACH_DEV(i) {
 343                 if (!strncmp(name,
 344                         rte_eth_dev_data[i].name, strlen(name))) {
 345
 346                         *port_id = i;
 347
 348                         return 0;
 349                 }
 350         }
 351         return -ENODEV;
 352 }
 353
 354 static int
 355 rte_eth_dev_is_detachable(uint8_t port_id)
 356 {
 357         uint32_t dev_flags;
 358
 359         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 360
 361         switch (rte_eth_devices[port_id].data->kdrv) {
 362         case RTE_KDRV_IGB_UIO:
 363         case RTE_KDRV_UIO_GENERIC:
 364         case RTE_KDRV_NIC_UIO:
 365         case RTE_KDRV_NONE:
 366         case RTE_KDRV_VFIO:
 367                 break;
 368         default:
 369                 return -ENOTSUP;
 370         }
 371         dev_flags = rte_eth_devices[port_id].data->dev_flags;
 372         if ((dev_flags & RTE_ETH_DEV_DETACHABLE) &&
 373                 (!(dev_flags & RTE_ETH_DEV_BONDED_SLAVE)))
 374                 return 0;
 375         else
 376                 return 1;
 377 }
 378
 379 /* attach the new device, then store port_id of the device */
 380 int
 381 rte_eth_dev_attach(const char *devargs, uint8_t *port_id)
 382 {
 383         int ret = -1;
 384         int current = rte_eth_dev_count();
 385         char *name = NULL;
 386         char *args = NULL;
 387
 388         if ((devargs == NULL) || (port_id == NULL)) {
 389                 ret = -EINVAL;
 390                 goto err;
 391         }
 392
 393         /* parse devargs, then retrieve device name and args */
 394         if (rte_eal_parse_devargs_str(devargs, &name, &args))
 395                 goto err;
 396
 397         ret = rte_eal_dev_attach(name, args);
 398         if (ret < 0)
 399                 goto err;
 400
 401         /* no point looking at the port count if no port exists */
 402         if (!rte_eth_dev_count()) {
 403                 RTE_LOG(ERR, EAL, "No port found for device (%s)\n", name);
 404                 ret = -1;
 405                 goto err;
 406         }
 407
 408         /* if nothing happened, there is a bug here, since some driver told us
 409          * it did attach a device, but did not create a port.
 410          */
 411         if (current == rte_eth_dev_count()) {
 412                 ret = -1;
 413                 goto err;
 414         }
 415
 416         *port_id = eth_dev_last_created_port;
 417         ret = 0;
 418
 419 err:
 420         free(name);
 421         free(args);
 422         return ret;
 423 }
 424
 425 /* detach the device, then store the name of the device */
 426 int
 427 rte_eth_dev_detach(uint8_t port_id, char *name)
 428 {
 429         int ret = -1;
 430
 431         if (name == NULL) {
 432                 ret = -EINVAL;
 433                 goto err;
 434         }
 435
 436         /* FIXME: move this to eal, once device flags are relocated there */
 437         if (rte_eth_dev_is_detachable(port_id))
 438                 goto err;
 439
 440         snprintf(name, sizeof(rte_eth_devices[port_id].data->name),
 441                  "%s", rte_eth_devices[port_id].data->name);
 442         ret = rte_eal_dev_detach(name);
 443         if (ret < 0)
 444                 goto err;
 445
 446         return 0;
 447
 448 err:
 449         return ret;
 450 }
 451
 452 static int
 453 rte_eth_dev_rx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
 454 {
 455         uint16_t old_nb_queues = dev->data->nb_rx_queues;
 456         void **rxq;
 457         unsigned i;
 458
 459         if (dev->data->rx_queues == NULL && nb_queues != 0) { /* first time configuration */
 460                 dev->data->rx_queues = rte_zmalloc("ethdev->rx_queues",
 461                                 sizeof(dev->data->rx_queues[0]) * nb_queues,
 462                                 RTE_CACHE_LINE_SIZE);
 463                 if (dev->data->rx_queues == NULL) {
 464                         dev->data->nb_rx_queues = 0;
 465                         return -(ENOMEM);
 466                 }
 467         } else if (dev->data->rx_queues != NULL && nb_queues != 0) { /* re-configure */
 468                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release, -ENOTSUP);
 469
 470                 rxq = dev->data->rx_queues;
 471
 472                 for (i = nb_queues; i < old_nb_queues; i++)
 473                         (*dev->dev_ops->rx_queue_release)(rxq[i]);
 474                 rxq = rte_realloc(rxq, sizeof(rxq[0]) * nb_queues,
 475                                 RTE_CACHE_LINE_SIZE);
 476                 if (rxq == NULL)
 477                         return -(ENOMEM);
 478                 if (nb_queues > old_nb_queues) {
 479                         uint16_t new_qs = nb_queues - old_nb_queues;
 480
 481                         memset(rxq + old_nb_queues, 0,
 482                                 sizeof(rxq[0]) * new_qs);
 483                 }
 484
 485                 dev->data->rx_queues = rxq;
 486
 487         } else if (dev->data->rx_queues != NULL && nb_queues == 0) {
 488                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release, -ENOTSUP);
 489
 490                 rxq = dev->data->rx_queues;
 491
 492                 for (i = nb_queues; i < old_nb_queues; i++)
 493                         (*dev->dev_ops->rx_queue_release)(rxq[i]);
 494
 495                 rte_free(dev->data->rx_queues);
 496                 dev->data->rx_queues = NULL;
 497         }
 498         dev->data->nb_rx_queues = nb_queues;
 499         return 0;
 500 }
 501
 502 int
 503 rte_eth_dev_rx_queue_start(uint8_t port_id, uint16_t rx_queue_id)
 504 {
 505         struct rte_eth_dev *dev;
 506
 507         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 508
 509         dev = &rte_eth_devices[port_id];
 510         if (rx_queue_id >= dev->data->nb_rx_queues) {
 511                 RTE_PMD_DEBUG_TRACE("Invalid RX queue_id=%d\n", rx_queue_id);
 512                 return -EINVAL;
 513         }
 514
 515         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_start, -ENOTSUP);
 516
 517         if (dev->data->rx_queue_state[rx_queue_id] != RTE_ETH_QUEUE_STATE_STOPPED) {
 518                 RTE_PMD_DEBUG_TRACE("Queue %" PRIu16" of device with port_id=%" PRIu8
 519                         " already started\n",
 520                         rx_queue_id, port_id);
 521                 return 0;
 522         }
 523
 524         return dev->dev_ops->rx_queue_start(dev, rx_queue_id);
 525
 526 }
 527
 528 int
 529 rte_eth_dev_rx_queue_stop(uint8_t port_id, uint16_t rx_queue_id)
 530 {
 531         struct rte_eth_dev *dev;
 532
 533         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 534
 535         dev = &rte_eth_devices[port_id];
 536         if (rx_queue_id >= dev->data->nb_rx_queues) {
 537                 RTE_PMD_DEBUG_TRACE("Invalid RX queue_id=%d\n", rx_queue_id);
 538                 return -EINVAL;
 539         }
 540
 541         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_stop, -ENOTSUP);
 542
 543         if (dev->data->rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STOPPED) {
 544                 RTE_PMD_DEBUG_TRACE("Queue %" PRIu16" of device with port_id=%" PRIu8
 545                         " already stopped\n",
 546                         rx_queue_id, port_id);
 547                 return 0;
 548         }
 549
 550         return dev->dev_ops->rx_queue_stop(dev, rx_queue_id);
 551
 552 }
 553
 554 int
 555 rte_eth_dev_tx_queue_start(uint8_t port_id, uint16_t tx_queue_id)
 556 {
 557         struct rte_eth_dev *dev;
 558
 559         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 560
 561         dev = &rte_eth_devices[port_id];
 562         if (tx_queue_id >= dev->data->nb_tx_queues) {
 563                 RTE_PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", tx_queue_id);
 564                 return -EINVAL;
 565         }
 566
 567         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_start, -ENOTSUP);
 568
 569         if (dev->data->tx_queue_state[tx_queue_id] != RTE_ETH_QUEUE_STATE_STOPPED) {
 570                 RTE_PMD_DEBUG_TRACE("Queue %" PRIu16" of device with port_id=%" PRIu8
 571                         " already started\n",
 572                         tx_queue_id, port_id);
 573                 return 0;
 574         }
 575
 576         return dev->dev_ops->tx_queue_start(dev, tx_queue_id);
 577
 578 }
 579
 580 int
 581 rte_eth_dev_tx_queue_stop(uint8_t port_id, uint16_t tx_queue_id)
 582 {
 583         struct rte_eth_dev *dev;
 584
 585         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 586
 587         dev = &rte_eth_devices[port_id];
 588         if (tx_queue_id >= dev->data->nb_tx_queues) {
 589                 RTE_PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", tx_queue_id);
 590                 return -EINVAL;
 591         }
 592
 593         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_stop, -ENOTSUP);
 594
 595         if (dev->data->tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STOPPED) {
 596                 RTE_PMD_DEBUG_TRACE("Queue %" PRIu16" of device with port_id=%" PRIu8
 597                         " already stopped\n",
 598                         tx_queue_id, port_id);
 599                 return 0;
 600         }
 601
 602         return dev->dev_ops->tx_queue_stop(dev, tx_queue_id);
 603
 604 }
 605
 606 static int
 607 rte_eth_dev_tx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
 608 {
 609         uint16_t old_nb_queues = dev->data->nb_tx_queues;
 610         void **txq;
 611         unsigned i;
 612
 613         if (dev->data->tx_queues == NULL && nb_queues != 0) { /* first time configuration */
 614                 dev->data->tx_queues = rte_zmalloc("ethdev->tx_queues",
 615                                                    sizeof(dev->data->tx_queues[0]) * nb_queues,
 616                                                    RTE_CACHE_LINE_SIZE);
 617                 if (dev->data->tx_queues == NULL) {
 618                         dev->data->nb_tx_queues = 0;
 619                         return -(ENOMEM);
 620                 }
 621         } else if (dev->data->tx_queues != NULL && nb_queues != 0) { /* re-configure */
 622                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release, -ENOTSUP);
 623
 624                 txq = dev->data->tx_queues;
 625
 626                 for (i = nb_queues; i < old_nb_queues; i++)
 627                         (*dev->dev_ops->tx_queue_release)(txq[i]);
 628                 txq = rte_realloc(txq, sizeof(txq[0]) * nb_queues,
 629                                   RTE_CACHE_LINE_SIZE);
 630                 if (txq == NULL)
 631                         return -ENOMEM;
 632                 if (nb_queues > old_nb_queues) {
 633                         uint16_t new_qs = nb_queues - old_nb_queues;
 634
 635                         memset(txq + old_nb_queues, 0,
 636                                sizeof(txq[0]) * new_qs);
 637                 }
 638
 639                 dev->data->tx_queues = txq;
 640
 641         } else if (dev->data->tx_queues != NULL && nb_queues == 0) {
 642                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release, -ENOTSUP);
 643
 644                 txq = dev->data->tx_queues;
 645
 646                 for (i = nb_queues; i < old_nb_queues; i++)
 647                         (*dev->dev_ops->tx_queue_release)(txq[i]);
 648
 649                 rte_free(dev->data->tx_queues);
 650                 dev->data->tx_queues = NULL;
 651         }
 652         dev->data->nb_tx_queues = nb_queues;
 653         return 0;
 654 }
 655
 656 uint32_t
 657 rte_eth_speed_bitflag(uint32_t speed, int duplex)
 658 {
 659         switch (speed) {
 660         case ETH_SPEED_NUM_10M:
 661                 return duplex ? ETH_LINK_SPEED_10M : ETH_LINK_SPEED_10M_HD;
 662         case ETH_SPEED_NUM_100M:
 663                 return duplex ? ETH_LINK_SPEED_100M : ETH_LINK_SPEED_100M_HD;
 664         case ETH_SPEED_NUM_1G:
 665                 return ETH_LINK_SPEED_1G;
 666         case ETH_SPEED_NUM_2_5G:
 667                 return ETH_LINK_SPEED_2_5G;
 668         case ETH_SPEED_NUM_5G:
 669                 return ETH_LINK_SPEED_5G;
 670         case ETH_SPEED_NUM_10G:
 671                 return ETH_LINK_SPEED_10G;
 672         case ETH_SPEED_NUM_20G:
 673                 return ETH_LINK_SPEED_20G;
 674         case ETH_SPEED_NUM_25G:
 675                 return ETH_LINK_SPEED_25G;
 676         case ETH_SPEED_NUM_40G:
 677                 return ETH_LINK_SPEED_40G;
 678         case ETH_SPEED_NUM_50G:
 679                 return ETH_LINK_SPEED_50G;
 680         case ETH_SPEED_NUM_56G:
 681                 return ETH_LINK_SPEED_56G;
 682         case ETH_SPEED_NUM_100G:
 683                 return ETH_LINK_SPEED_100G;
 684         default:
 685                 return 0;
 686         }
 687 }
 688
 689 int
 690 rte_eth_dev_configure(uint8_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
 691                       const struct rte_eth_conf *dev_conf)
 692 {
 693         struct rte_eth_dev *dev;
 694         struct rte_eth_dev_info dev_info;
 695         int diag;
 696
 697         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 698
 699         if (nb_rx_q > RTE_MAX_QUEUES_PER_PORT) {
 700                 RTE_PMD_DEBUG_TRACE(
 701                         "Number of RX queues requested (%u) is greater than max supported(%d)\n",
 702                         nb_rx_q, RTE_MAX_QUEUES_PER_PORT);
 703                 return -EINVAL;
 704         }
 705
 706         if (nb_tx_q > RTE_MAX_QUEUES_PER_PORT) {
 707                 RTE_PMD_DEBUG_TRACE(
 708                         "Number of TX queues requested (%u) is greater than max supported(%d)\n",
 709                         nb_tx_q, RTE_MAX_QUEUES_PER_PORT);
 710                 return -EINVAL;
 711         }
 712
 713         dev = &rte_eth_devices[port_id];
 714
 715         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
 716         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
 717
 718         if (dev->data->dev_started) {
 719                 RTE_PMD_DEBUG_TRACE(
 720                     "port %d must be stopped to allow configuration\n", port_id);
 721                 return -EBUSY;
 722         }
 723
 724         /* Copy the dev_conf parameter into the dev structure */
 725         memcpy(&dev->data->dev_conf, dev_conf, sizeof(dev->data->dev_conf));
 726
 727         /*
 728          * Check that the numbers of RX and TX queues are not greater
 729          * than the maximum number of RX and TX queues supported by the
 730          * configured device.
 731          */
 732         (*dev->dev_ops->dev_infos_get)(dev, &dev_info);
 733
 734         if (nb_rx_q == 0 && nb_tx_q == 0) {
 735                 RTE_PMD_DEBUG_TRACE("ethdev port_id=%d both rx and tx queue cannot be 0\n", port_id);
 736                 return -EINVAL;
 737         }
 738
 739         if (nb_rx_q > dev_info.max_rx_queues) {
 740                 RTE_PMD_DEBUG_TRACE("ethdev port_id=%d nb_rx_queues=%d > %d\n",
 741                                 port_id, nb_rx_q, dev_info.max_rx_queues);
 742                 return -EINVAL;
 743         }
 744
 745         if (nb_tx_q > dev_info.max_tx_queues) {
 746                 RTE_PMD_DEBUG_TRACE("ethdev port_id=%d nb_tx_queues=%d > %d\n",
 747                                 port_id, nb_tx_q, dev_info.max_tx_queues);
 748                 return -EINVAL;
 749         }
 750
 751         /* Check that the device supports requested interrupts */
 752         if ((dev_conf->intr_conf.lsc == 1) &&
 753                 (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC))) {
 754                         RTE_PMD_DEBUG_TRACE("driver %s does not support lsc\n",
 755                                         dev->data->drv_name);
 756                         return -EINVAL;
 757         }
 758         if ((dev_conf->intr_conf.rmv == 1) &&
 759             (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_RMV))) {
 760                 RTE_PMD_DEBUG_TRACE("driver %s does not support rmv\n",
 761                                     dev->data->drv_name);
 762                 return -EINVAL;
 763         }
 764
 765         /*
 766          * If jumbo frames are enabled, check that the maximum RX packet
 767          * length is supported by the configured device.
 768          */
 769         if (dev_conf->rxmode.jumbo_frame == 1) {
 770                 if (dev_conf->rxmode.max_rx_pkt_len >
 771                     dev_info.max_rx_pktlen) {
 772                         RTE_PMD_DEBUG_TRACE("ethdev port_id=%d max_rx_pkt_len %u"
 773                                 " > max valid value %u\n",
 774                                 port_id,
 775                                 (unsigned)dev_conf->rxmode.max_rx_pkt_len,
 776                                 (unsigned)dev_info.max_rx_pktlen);
 777                         return -EINVAL;
 778                 } else if (dev_conf->rxmode.max_rx_pkt_len < ETHER_MIN_LEN) {
 779                         RTE_PMD_DEBUG_TRACE("ethdev port_id=%d max_rx_pkt_len %u"
 780                                 " < min valid value %u\n",
 781                                 port_id,
 782                                 (unsigned)dev_conf->rxmode.max_rx_pkt_len,
 783                                 (unsigned)ETHER_MIN_LEN);
 784                         return -EINVAL;
 785                 }
 786         } else {
 787                 if (dev_conf->rxmode.max_rx_pkt_len < ETHER_MIN_LEN ||
 788                         dev_conf->rxmode.max_rx_pkt_len > ETHER_MAX_LEN)
 789                         /* Use default value */
 790                         dev->data->dev_conf.rxmode.max_rx_pkt_len =
 791                                                         ETHER_MAX_LEN;
 792         }
 793
 794         /*
 795          * Setup new number of RX/TX queues and reconfigure device.
 796          */
 797         diag = rte_eth_dev_rx_queue_config(dev, nb_rx_q);
 798         if (diag != 0) {
 799                 RTE_PMD_DEBUG_TRACE("port%d rte_eth_dev_rx_queue_config = %d\n",
 800                                 port_id, diag);
 801                 return diag;
 802         }
 803
 804         diag = rte_eth_dev_tx_queue_config(dev, nb_tx_q);
 805         if (diag != 0) {
 806                 RTE_PMD_DEBUG_TRACE("port%d rte_eth_dev_tx_queue_config = %d\n",
 807                                 port_id, diag);
 808                 rte_eth_dev_rx_queue_config(dev, 0);
 809                 return diag;
 810         }
 811
 812         diag = (*dev->dev_ops->dev_configure)(dev);
 813         if (diag != 0) {
 814                 RTE_PMD_DEBUG_TRACE("port%d dev_configure = %d\n",
 815                                 port_id, diag);
 816                 rte_eth_dev_rx_queue_config(dev, 0);
 817                 rte_eth_dev_tx_queue_config(dev, 0);
 818                 return diag;
 819         }
 820
 821         return 0;
 822 }
 823
 824 void
 825 _rte_eth_dev_reset(struct rte_eth_dev *dev)
 826 {
 827         if (dev->data->dev_started) {
 828                 RTE_PMD_DEBUG_TRACE(
 829                         "port %d must be stopped to allow reset\n",
 830                         dev->data->port_id);
 831                 return;
 832         }
 833
 834         rte_eth_dev_rx_queue_config(dev, 0);
 835         rte_eth_dev_tx_queue_config(dev, 0);
 836
 837         memset(&dev->data->dev_conf, 0, sizeof(dev->data->dev_conf));
 838 }
 839
 840 static void
 841 rte_eth_dev_config_restore(uint8_t port_id)
 842 {
 843         struct rte_eth_dev *dev;
 844         struct rte_eth_dev_info dev_info;
 845         struct ether_addr *addr;
 846         uint16_t i;
 847         uint32_t pool = 0;
 848         uint64_t pool_mask;
 849
 850         dev = &rte_eth_devices[port_id];
 851
 852         rte_eth_dev_info_get(port_id, &dev_info);
 853
 854         /* replay MAC address configuration including default MAC */
 855         addr = &dev->data->mac_addrs[0];
 856         if (*dev->dev_ops->mac_addr_set != NULL)
 857                 (*dev->dev_ops->mac_addr_set)(dev, addr);
 858         else if (*dev->dev_ops->mac_addr_add != NULL)
 859                 (*dev->dev_ops->mac_addr_add)(dev, addr, 0, pool);
 860
 861         if (*dev->dev_ops->mac_addr_add != NULL) {
 862                 for (i = 1; i < dev_info.max_mac_addrs; i++) {
 863                         addr = &dev->data->mac_addrs[i];
 864
 865                         /* skip zero address */
 866                         if (is_zero_ether_addr(addr))
 867                                 continue;
 868
 869                         pool = 0;
 870                         pool_mask = dev->data->mac_pool_sel[i];
 871
 872                         do {
 873                                 if (pool_mask & 1ULL)
 874                                         (*dev->dev_ops->mac_addr_add)(dev,
 875                                                 addr, i, pool);
 876                                 pool_mask >>= 1;
 877                                 pool++;
 878                         } while (pool_mask);
 879                 }
 880         }
 881
 882         /* replay promiscuous configuration */
 883         if (rte_eth_promiscuous_get(port_id) == 1)
 884                 rte_eth_promiscuous_enable(port_id);
 885         else if (rte_eth_promiscuous_get(port_id) == 0)
 886                 rte_eth_promiscuous_disable(port_id);
 887
 888         /* replay all multicast configuration */
 889         if (rte_eth_allmulticast_get(port_id) == 1)
 890                 rte_eth_allmulticast_enable(port_id);
 891         else if (rte_eth_allmulticast_get(port_id) == 0)
 892                 rte_eth_allmulticast_disable(port_id);
 893 }
 894
 895 int
 896 rte_eth_dev_start(uint8_t port_id)
 897 {
 898         struct rte_eth_dev *dev;
 899         int diag;
 900
 901         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 902
 903         dev = &rte_eth_devices[port_id];
 904
 905         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_start, -ENOTSUP);
 906
 907         if (dev->data->dev_started != 0) {
 908                 RTE_PMD_DEBUG_TRACE("Device with port_id=%" PRIu8
 909                         " already started\n",
 910                         port_id);
 911                 return 0;
 912         }
 913
 914         diag = (*dev->dev_ops->dev_start)(dev);
 915         if (diag == 0)
 916                 dev->data->dev_started = 1;
 917         else
 918                 return diag;
 919
 920         rte_eth_dev_config_restore(port_id);
 921
 922         if (dev->data->dev_conf.intr_conf.lsc == 0) {
 923                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
 924                 (*dev->dev_ops->link_update)(dev, 0);
 925         }
 926         return 0;
 927 }
 928
 929 void
 930 rte_eth_dev_stop(uint8_t port_id)
 931 {
 932         struct rte_eth_dev *dev;
 933
 934         RTE_ETH_VALID_PORTID_OR_RET(port_id);
 935         dev = &rte_eth_devices[port_id];
 936
 937         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_stop);
 938
 939         if (dev->data->dev_started == 0) {
 940                 RTE_PMD_DEBUG_TRACE("Device with port_id=%" PRIu8
 941                         " already stopped\n",
 942                         port_id);
 943                 return;
 944         }
 945
 946         dev->data->dev_started = 0;
 947         (*dev->dev_ops->dev_stop)(dev);
 948 }
 949
 950 int
 951 rte_eth_dev_set_link_up(uint8_t port_id)
 952 {
 953         struct rte_eth_dev *dev;
 954
 955         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 956
 957         dev = &rte_eth_devices[port_id];
 958
 959         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_up, -ENOTSUP);
 960         return (*dev->dev_ops->dev_set_link_up)(dev);
 961 }
 962
 963 int
 964 rte_eth_dev_set_link_down(uint8_t port_id)
 965 {
 966         struct rte_eth_dev *dev;
 967
 968         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 969
 970         dev = &rte_eth_devices[port_id];
 971
 972         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_down, -ENOTSUP);
 973         return (*dev->dev_ops->dev_set_link_down)(dev);
 974 }
 975
 976 void
 977 rte_eth_dev_close(uint8_t port_id)
 978 {
 979         struct rte_eth_dev *dev;
 980
 981         RTE_ETH_VALID_PORTID_OR_RET(port_id);
 982         dev = &rte_eth_devices[port_id];
 983
 984         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_close);
 985         dev->data->dev_started = 0;
 986         (*dev->dev_ops->dev_close)(dev);
 987
 988         dev->data->nb_rx_queues = 0;
 989         rte_free(dev->data->rx_queues);
 990         dev->data->rx_queues = NULL;
 991         dev->data->nb_tx_queues = 0;
 992         rte_free(dev->data->tx_queues);
 993         dev->data->tx_queues = NULL;
 994 }
 995
 996 int
 997 rte_eth_rx_queue_setup(uint8_t port_id, uint16_t rx_queue_id,
 998                        uint16_t nb_rx_desc, unsigned int socket_id,
 999                        const struct rte_eth_rxconf *rx_conf,
1000                        struct rte_mempool *mp)
1001 {
1002         int ret;
1003         uint32_t mbp_buf_size;
1004         struct rte_eth_dev *dev;
1005         struct rte_eth_dev_info dev_info;
1006         void **rxq;
1007
1008         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1009
1010         dev = &rte_eth_devices[port_id];
1011         if (rx_queue_id >= dev->data->nb_rx_queues) {
1012                 RTE_PMD_DEBUG_TRACE("Invalid RX queue_id=%d\n", rx_queue_id);
1013                 return -EINVAL;
1014         }
1015
1016         if (dev->data->dev_started) {
1017                 RTE_PMD_DEBUG_TRACE(
1018                     "port %d must be stopped to allow configuration\n", port_id);
1019                 return -EBUSY;
1020         }
1021
1022         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
1023         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_setup, -ENOTSUP);
1024
1025         /*
1026          * Check the size of the mbuf data buffer.
1027          * This value must be provided in the private data of the memory pool.
1028          * First check that the memory pool has a valid private data.
1029          */
1030         rte_eth_dev_info_get(port_id, &dev_info);
1031         if (mp->private_data_size < sizeof(struct rte_pktmbuf_pool_private)) {
1032                 RTE_PMD_DEBUG_TRACE("%s private_data_size %d < %d\n",
1033                                 mp->name, (int) mp->private_data_size,
1034                                 (int) sizeof(struct rte_pktmbuf_pool_private));
1035                 return -ENOSPC;
1036         }
1037         mbp_buf_size = rte_pktmbuf_data_room_size(mp);
1038
1039         if ((mbp_buf_size - RTE_PKTMBUF_HEADROOM) < dev_info.min_rx_bufsize) {
1040                 RTE_PMD_DEBUG_TRACE("%s mbuf_data_room_size %d < %d "
1041                                 "(RTE_PKTMBUF_HEADROOM=%d + min_rx_bufsize(dev)"
1042                                 "=%d)\n",
1043                                 mp->name,
1044                                 (int)mbp_buf_size,
1045                                 (int)(RTE_PKTMBUF_HEADROOM +
1046                                       dev_info.min_rx_bufsize),
1047                                 (int)RTE_PKTMBUF_HEADROOM,
1048                                 (int)dev_info.min_rx_bufsize);
1049                 return -EINVAL;
1050         }
1051
1052         if (nb_rx_desc > dev_info.rx_desc_lim.nb_max ||
1053                         nb_rx_desc < dev_info.rx_desc_lim.nb_min ||
1054                         nb_rx_desc % dev_info.rx_desc_lim.nb_align != 0) {
1055
1056                 RTE_PMD_DEBUG_TRACE("Invalid value for nb_rx_desc(=%hu), "
1057                         "should be: <= %hu, = %hu, and a product of %hu\n",
1058                         nb_rx_desc,
1059                         dev_info.rx_desc_lim.nb_max,
1060                         dev_info.rx_desc_lim.nb_min,
1061                         dev_info.rx_desc_lim.nb_align);
1062                 return -EINVAL;
1063         }
1064
1065         rxq = dev->data->rx_queues;
1066         if (rxq[rx_queue_id]) {
1067                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release,
1068                                         -ENOTSUP);
1069                 (*dev->dev_ops->rx_queue_release)(rxq[rx_queue_id]);
1070                 rxq[rx_queue_id] = NULL;
1071         }
1072
1073         if (rx_conf == NULL)
1074                 rx_conf = &dev_info.default_rxconf;
1075
1076         ret = (*dev->dev_ops->rx_queue_setup)(dev, rx_queue_id, nb_rx_desc,
1077                                               socket_id, rx_conf, mp);
1078         if (!ret) {
1079                 if (!dev->data->min_rx_buf_size ||
1080                     dev->data->min_rx_buf_size > mbp_buf_size)
1081                         dev->data->min_rx_buf_size = mbp_buf_size;
1082         }
1083
1084         return ret;
1085 }
1086
1087 int
1088 rte_eth_tx_queue_setup(uint8_t port_id, uint16_t tx_queue_id,
1089                        uint16_t nb_tx_desc, unsigned int socket_id,
1090                        const struct rte_eth_txconf *tx_conf)
1091 {
1092         struct rte_eth_dev *dev;
1093         struct rte_eth_dev_info dev_info;
1094         void **txq;
1095
1096         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1097
1098         dev = &rte_eth_devices[port_id];
1099         if (tx_queue_id >= dev->data->nb_tx_queues) {
1100                 RTE_PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", tx_queue_id);
1101                 return -EINVAL;
1102         }
1103
1104         if (dev->data->dev_started) {
1105                 RTE_PMD_DEBUG_TRACE(
1106                     "port %d must be stopped to allow configuration\n", port_id);
1107                 return -EBUSY;
1108         }
1109
1110         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
1111         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_setup, -ENOTSUP);
1112
1113         rte_eth_dev_info_get(port_id, &dev_info);
1114
1115         if (nb_tx_desc > dev_info.tx_desc_lim.nb_max ||
1116             nb_tx_desc < dev_info.tx_desc_lim.nb_min ||
1117             nb_tx_desc % dev_info.tx_desc_lim.nb_align != 0) {
1118                 RTE_PMD_DEBUG_TRACE("Invalid value for nb_tx_desc(=%hu), "
1119                                 "should be: <= %hu, = %hu, and a product of %hu\n",
1120                                 nb_tx_desc,
1121                                 dev_info.tx_desc_lim.nb_max,
1122                                 dev_info.tx_desc_lim.nb_min,
1123                                 dev_info.tx_desc_lim.nb_align);
1124                 return -EINVAL;
1125         }
1126
1127         txq = dev->data->tx_queues;
1128         if (txq[tx_queue_id]) {
1129                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release,
1130                                         -ENOTSUP);
1131                 (*dev->dev_ops->tx_queue_release)(txq[tx_queue_id]);
1132                 txq[tx_queue_id] = NULL;
1133         }
1134
1135         if (tx_conf == NULL)
1136                 tx_conf = &dev_info.default_txconf;
1137
1138         return (*dev->dev_ops->tx_queue_setup)(dev, tx_queue_id, nb_tx_desc,
1139                                                socket_id, tx_conf);
1140 }
1141
1142 void
1143 rte_eth_tx_buffer_drop_callback(struct rte_mbuf **pkts, uint16_t unsent,
1144                 void *userdata __rte_unused)
1145 {
1146         unsigned i;
1147
1148         for (i = 0; i < unsent; i++)
1149                 rte_pktmbuf_free(pkts[i]);
1150 }
1151
1152 void
1153 rte_eth_tx_buffer_count_callback(struct rte_mbuf **pkts, uint16_t unsent,
1154                 void *userdata)
1155 {
1156         uint64_t *count = userdata;
1157         unsigned i;
1158
1159         for (i = 0; i < unsent; i++)
1160                 rte_pktmbuf_free(pkts[i]);
1161
1162         *count += unsent;
1163 }
1164
1165 int
1166 rte_eth_tx_buffer_set_err_callback(struct rte_eth_dev_tx_buffer *buffer,
1167                 buffer_tx_error_fn cbfn, void *userdata)
1168 {
1169         buffer->error_callback = cbfn;
1170         buffer->error_userdata = userdata;
1171         return 0;
1172 }
1173
1174 int
1175 rte_eth_tx_buffer_init(struct rte_eth_dev_tx_buffer *buffer, uint16_t size)
1176 {
1177         int ret = 0;
1178
1179         if (buffer == NULL)
1180                 return -EINVAL;
1181
1182         buffer->size = size;
1183         if (buffer->error_callback == NULL) {
1184                 ret = rte_eth_tx_buffer_set_err_callback(
1185                         buffer, rte_eth_tx_buffer_drop_callback, NULL);
1186         }
1187
1188         return ret;
1189 }
1190
1191 int
1192 rte_eth_tx_done_cleanup(uint8_t port_id, uint16_t queue_id, uint32_t free_cnt)
1193 {
1194         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
1195
1196         /* Validate Input Data. Bail if not valid or not supported. */
1197         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1198         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_done_cleanup, -ENOTSUP);
1199
1200         /* Call driver to free pending mbufs. */
1201         return (*dev->dev_ops->tx_done_cleanup)(dev->data->tx_queues[queue_id],
1202                         free_cnt);
1203 }
1204
1205 void
1206 rte_eth_promiscuous_enable(uint8_t port_id)
1207 {
1208         struct rte_eth_dev *dev;
1209
1210         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1211         dev = &rte_eth_devices[port_id];
1212
1213         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->promiscuous_enable);
1214         (*dev->dev_ops->promiscuous_enable)(dev);
1215         dev->data->promiscuous = 1;
1216 }
1217
1218 void
1219 rte_eth_promiscuous_disable(uint8_t port_id)
1220 {
1221         struct rte_eth_dev *dev;
1222
1223         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1224         dev = &rte_eth_devices[port_id];
1225
1226         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->promiscuous_disable);
1227         dev->data->promiscuous = 0;
1228         (*dev->dev_ops->promiscuous_disable)(dev);
1229 }
1230
1231 int
1232 rte_eth_promiscuous_get(uint8_t port_id)
1233 {
1234         struct rte_eth_dev *dev;
1235
1236         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1237
1238         dev = &rte_eth_devices[port_id];
1239         return dev->data->promiscuous;
1240 }
1241
1242 void
1243 rte_eth_allmulticast_enable(uint8_t port_id)
1244 {
1245         struct rte_eth_dev *dev;
1246
1247         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1248         dev = &rte_eth_devices[port_id];
1249
1250         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->allmulticast_enable);
1251         (*dev->dev_ops->allmulticast_enable)(dev);
1252         dev->data->all_multicast = 1;
1253 }
1254
1255 void
1256 rte_eth_allmulticast_disable(uint8_t port_id)
1257 {
1258         struct rte_eth_dev *dev;
1259
1260         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1261         dev = &rte_eth_devices[port_id];
1262
1263         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->allmulticast_disable);
1264         dev->data->all_multicast = 0;
1265         (*dev->dev_ops->allmulticast_disable)(dev);
1266 }
1267
1268 int
1269 rte_eth_allmulticast_get(uint8_t port_id)
1270 {
1271         struct rte_eth_dev *dev;
1272
1273         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1274
1275         dev = &rte_eth_devices[port_id];
1276         return dev->data->all_multicast;
1277 }
1278
1279 static inline int
1280 rte_eth_dev_atomic_read_link_status(struct rte_eth_dev *dev,
1281                                 struct rte_eth_link *link)
1282 {
1283         struct rte_eth_link *dst = link;
1284         struct rte_eth_link *src = &(dev->data->dev_link);
1285
1286         if (rte_atomic64_cmpset((uint64_t *)dst, *(uint64_t *)dst,
1287                                         *(uint64_t *)src) == 0)
1288                 return -1;
1289
1290         return 0;
1291 }
1292
1293 void
1294 rte_eth_link_get(uint8_t port_id, struct rte_eth_link *eth_link)
1295 {
1296         struct rte_eth_dev *dev;
1297
1298         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1299         dev = &rte_eth_devices[port_id];
1300
1301         if (dev->data->dev_conf.intr_conf.lsc != 0)
1302                 rte_eth_dev_atomic_read_link_status(dev, eth_link);
1303         else {
1304                 RTE_FUNC_PTR_OR_RET(*dev->dev_ops->link_update);
1305                 (*dev->dev_ops->link_update)(dev, 1);
1306                 *eth_link = dev->data->dev_link;
1307         }
1308 }
1309
1310 void
1311 rte_eth_link_get_nowait(uint8_t port_id, struct rte_eth_link *eth_link)
1312 {
1313         struct rte_eth_dev *dev;
1314
1315         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1316         dev = &rte_eth_devices[port_id];
1317
1318         if (dev->data->dev_conf.intr_conf.lsc != 0)
1319                 rte_eth_dev_atomic_read_link_status(dev, eth_link);
1320         else {
1321                 RTE_FUNC_PTR_OR_RET(*dev->dev_ops->link_update);
1322                 (*dev->dev_ops->link_update)(dev, 0);
1323                 *eth_link = dev->data->dev_link;
1324         }
1325 }
1326
1327 int
1328 rte_eth_stats_get(uint8_t port_id, struct rte_eth_stats *stats)
1329 {
1330         struct rte_eth_dev *dev;
1331
1332         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1333
1334         dev = &rte_eth_devices[port_id];
1335         memset(stats, 0, sizeof(*stats));
1336
1337         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
1338         stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
1339         (*dev->dev_ops->stats_get)(dev, stats);
1340         return 0;
1341 }
1342
1343 void
1344 rte_eth_stats_reset(uint8_t port_id)
1345 {
1346         struct rte_eth_dev *dev;
1347
1348         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1349         dev = &rte_eth_devices[port_id];
1350
1351         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->stats_reset);
1352         (*dev->dev_ops->stats_reset)(dev);
1353         dev->data->rx_mbuf_alloc_failed = 0;
1354 }
1355
1356 static int
1357 get_xstats_count(uint8_t port_id)
1358 {
1359         struct rte_eth_dev *dev;
1360         int count;
1361
1362         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1363         dev = &rte_eth_devices[port_id];
1364         if (dev->dev_ops->xstats_get_names_by_id != NULL) {
1365                 count = (*dev->dev_ops->xstats_get_names_by_id)(dev, NULL,
1366                                 NULL, 0);
1367                 if (count < 0)
1368                         return count;
1369         }
1370         if (dev->dev_ops->xstats_get_names != NULL) {
1371                 count = (*dev->dev_ops->xstats_get_names)(dev, NULL, 0);
1372                 if (count < 0)
1373                         return count;
1374         } else
1375                 count = 0;
1376
1377         count += RTE_NB_STATS;
1378         count += RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS) *
1379                  RTE_NB_RXQ_STATS;
1380         count += RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS) *
1381                  RTE_NB_TXQ_STATS;
1382         return count;
1383 }
1384
1385 int
1386 rte_eth_xstats_get_id_by_name(uint8_t port_id, const char *xstat_name,
1387                 uint64_t *id)
1388 {
1389         int cnt_xstats, idx_xstat;
1390
1391         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1392
1393         if (!id) {
1394                 RTE_PMD_DEBUG_TRACE("Error: id pointer is NULL\n");
1395                 return -ENOMEM;
1396         }
1397
1398         if (!xstat_name) {
1399                 RTE_PMD_DEBUG_TRACE("Error: xstat_name pointer is NULL\n");
1400                 return -ENOMEM;
1401         }
1402
1403         /* Get count */
1404         cnt_xstats = rte_eth_xstats_get_names_by_id(port_id, NULL, 0, NULL);
1405         if (cnt_xstats  < 0) {
1406                 RTE_PMD_DEBUG_TRACE("Error: Cannot get count of xstats\n");
1407                 return -ENODEV;
1408         }
1409
1410         /* Get id-name lookup table */
1411         struct rte_eth_xstat_name xstats_names[cnt_xstats];
1412
1413         if (cnt_xstats != rte_eth_xstats_get_names_by_id(
1414                         port_id, xstats_names, cnt_xstats, NULL)) {
1415                 RTE_PMD_DEBUG_TRACE("Error: Cannot get xstats lookup\n");
1416                 return -1;
1417         }
1418
1419         for (idx_xstat = 0; idx_xstat < cnt_xstats; idx_xstat++) {
1420                 if (!strcmp(xstats_names[idx_xstat].name, xstat_name)) {
1421                         *id = idx_xstat;
1422                         return 0;
1423                 };
1424         }
1425
1426         return -EINVAL;
1427 }
1428
1429 int
1430 rte_eth_xstats_get_names_by_id(uint8_t port_id,
1431         struct rte_eth_xstat_name *xstats_names, unsigned int size,
1432         uint64_t *ids)
1433 {
1434         /* Get all xstats */
1435         if (!ids) {
1436                 struct rte_eth_dev *dev;
1437                 int cnt_used_entries;
1438                 int cnt_expected_entries;
1439                 int cnt_driver_entries;
1440                 uint32_t idx, id_queue;
1441                 uint16_t num_q;
1442
1443                 cnt_expected_entries = get_xstats_count(port_id);
1444                 if (xstats_names == NULL || cnt_expected_entries < 0 ||
1445                                 (int)size < cnt_expected_entries)
1446                         return cnt_expected_entries;
1447
1448                 /* port_id checked in get_xstats_count() */
1449                 dev = &rte_eth_devices[port_id];
1450                 cnt_used_entries = 0;
1451
1452                 for (idx = 0; idx < RTE_NB_STATS; idx++) {
1453                         snprintf(xstats_names[cnt_used_entries].name,
1454                                 sizeof(xstats_names[0].name),
1455                                 "%s", rte_stats_strings[idx].name);
1456                         cnt_used_entries++;
1457                 }
1458                 num_q = RTE_MIN(dev->data->nb_rx_queues,
1459                                 RTE_ETHDEV_QUEUE_STAT_CNTRS);
1460                 for (id_queue = 0; id_queue < num_q; id_queue++) {
1461                         for (idx = 0; idx < RTE_NB_RXQ_STATS; idx++) {
1462                                 snprintf(xstats_names[cnt_used_entries].name,
1463                                         sizeof(xstats_names[0].name),
1464                                         "rx_q%u%s",
1465                                         id_queue,
1466                                         rte_rxq_stats_strings[idx].name);
1467                                 cnt_used_entries++;
1468                         }
1469
1470                 }
1471                 num_q = RTE_MIN(dev->data->nb_tx_queues,
1472                                 RTE_ETHDEV_QUEUE_STAT_CNTRS);
1473                 for (id_queue = 0; id_queue < num_q; id_queue++) {
1474                         for (idx = 0; idx < RTE_NB_TXQ_STATS; idx++) {
1475                                 snprintf(xstats_names[cnt_used_entries].name,
1476                                         sizeof(xstats_names[0].name),
1477                                         "tx_q%u%s",
1478                                         id_queue,
1479                                         rte_txq_stats_strings[idx].name);
1480                                 cnt_used_entries++;
1481                         }
1482                 }
1483
1484                 if (dev->dev_ops->xstats_get_names_by_id != NULL) {
1485                         /* If there are any driver-specific xstats, append them
1486                          * to end of list.
1487                          */
1488                         cnt_driver_entries =
1489                                 (*dev->dev_ops->xstats_get_names_by_id)(
1490                                 dev,
1491                                 xstats_names + cnt_used_entries,
1492                                 NULL,
1493                                 size - cnt_used_entries);
1494                         if (cnt_driver_entries < 0)
1495                                 return cnt_driver_entries;
1496                         cnt_used_entries += cnt_driver_entries;
1497
1498                 } else if (dev->dev_ops->xstats_get_names != NULL) {
1499                         /* If there are any driver-specific xstats, append them
1500                          * to end of list.
1501                          */
1502                         cnt_driver_entries = (*dev->dev_ops->xstats_get_names)(
1503                                 dev,
1504                                 xstats_names + cnt_used_entries,
1505                                 size - cnt_used_entries);
1506                         if (cnt_driver_entries < 0)
1507                                 return cnt_driver_entries;
1508                         cnt_used_entries += cnt_driver_entries;
1509                 }
1510
1511                 return cnt_used_entries;
1512         }
1513         /* Get only xstats given by IDS */
1514         else {
1515                 uint16_t len, i;
1516                 struct rte_eth_xstat_name *xstats_names_copy;
1517
1518                 len = rte_eth_xstats_get_names_by_id(port_id, NULL, 0, NULL);
1519
1520                 xstats_names_copy =
1521                                 malloc(sizeof(struct rte_eth_xstat_name) * len);
1522                 if (!xstats_names_copy) {
1523                         RTE_PMD_DEBUG_TRACE(
1524                              "ERROR: can't allocate memory for values_copy\n");
1525                         free(xstats_names_copy);
1526                         return -1;
1527                 }
1528
1529                 rte_eth_xstats_get_names_by_id(port_id, xstats_names_copy,
1530                                 len, NULL);
1531
1532                 for (i = 0; i < size; i++) {
1533                         if (ids[i] >= len) {
1534                                 RTE_PMD_DEBUG_TRACE(
1535                                         "ERROR: id value isn't valid\n");
1536                                 return -1;
1537                         }
1538                         strcpy(xstats_names[i].name,
1539                                         xstats_names_copy[ids[i]].name);
1540                 }
1541                 free(xstats_names_copy);
1542                 return size;
1543         }
1544 }
1545
1546 int
1547 rte_eth_xstats_get_names(uint8_t port_id,
1548         struct rte_eth_xstat_name *xstats_names,
1549         unsigned int size)
1550 {
1551         struct rte_eth_dev *dev;
1552         int cnt_used_entries;
1553         int cnt_expected_entries;
1554         int cnt_driver_entries;
1555         uint32_t idx, id_queue;
1556         uint16_t num_q;
1557
1558         cnt_expected_entries = get_xstats_count(port_id);
1559         if (xstats_names == NULL || cnt_expected_entries < 0 ||
1560                         (int)size < cnt_expected_entries)
1561                 return cnt_expected_entries;
1562
1563         /* port_id checked in get_xstats_count() */
1564         dev = &rte_eth_devices[port_id];
1565         cnt_used_entries = 0;
1566
1567         for (idx = 0; idx < RTE_NB_STATS; idx++) {
1568                 snprintf(xstats_names[cnt_used_entries].name,
1569                         sizeof(xstats_names[0].name),
1570                         "%s", rte_stats_strings[idx].name);
1571                 cnt_used_entries++;
1572         }
1573         num_q = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
1574         for (id_queue = 0; id_queue < num_q; id_queue++) {
1575                 for (idx = 0; idx < RTE_NB_RXQ_STATS; idx++) {
1576                         snprintf(xstats_names[cnt_used_entries].name,
1577                                 sizeof(xstats_names[0].name),
1578                                 "rx_q%u%s",
1579                                 id_queue, rte_rxq_stats_strings[idx].name);
1580                         cnt_used_entries++;
1581                 }
1582
1583         }
1584         num_q = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
1585         for (id_queue = 0; id_queue < num_q; id_queue++) {
1586                 for (idx = 0; idx < RTE_NB_TXQ_STATS; idx++) {
1587                         snprintf(xstats_names[cnt_used_entries].name,
1588                                 sizeof(xstats_names[0].name),
1589                                 "tx_q%u%s",
1590                                 id_queue, rte_txq_stats_strings[idx].name);
1591                         cnt_used_entries++;
1592                 }
1593         }
1594
1595         if (dev->dev_ops->xstats_get_names != NULL) {
1596                 /* If there are any driver-specific xstats, append them
1597                  * to end of list.
1598                  */
1599                 cnt_driver_entries = (*dev->dev_ops->xstats_get_names)(
1600                         dev,
1601                         xstats_names + cnt_used_entries,
1602                         size - cnt_used_entries);
1603                 if (cnt_driver_entries < 0)
1604                         return cnt_driver_entries;
1605                 cnt_used_entries += cnt_driver_entries;
1606         }
1607
1608         return cnt_used_entries;
1609 }
1610
1611 /* retrieve ethdev extended statistics */
1612 int
1613 rte_eth_xstats_get_by_id(uint8_t port_id, const uint64_t *ids, uint64_t *values,
1614         unsigned int n)
1615 {
1616         /* If need all xstats */
1617         if (!ids) {
1618                 struct rte_eth_stats eth_stats;
1619                 struct rte_eth_dev *dev;
1620                 unsigned int count = 0, i, q;
1621                 signed int xcount = 0;
1622                 uint64_t val, *stats_ptr;
1623                 uint16_t nb_rxqs, nb_txqs;
1624
1625                 RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1626                 dev = &rte_eth_devices[port_id];
1627
1628                 nb_rxqs = RTE_MIN(dev->data->nb_rx_queues,
1629                                 RTE_ETHDEV_QUEUE_STAT_CNTRS);
1630                 nb_txqs = RTE_MIN(dev->data->nb_tx_queues,
1631                                 RTE_ETHDEV_QUEUE_STAT_CNTRS);
1632
1633                 /* Return generic statistics */
1634                 count = RTE_NB_STATS + (nb_rxqs * RTE_NB_RXQ_STATS) +
1635                         (nb_txqs * RTE_NB_TXQ_STATS);
1636
1637
1638                 /* implemented by the driver */
1639                 if (dev->dev_ops->xstats_get_by_id != NULL) {
1640                         /* Retrieve the xstats from the driver at the end of the
1641                          * xstats struct. Retrieve all xstats.
1642                          */
1643                         xcount = (*dev->dev_ops->xstats_get_by_id)(dev,
1644                                         NULL,
1645                                         values ? values + count : NULL,
1646                                         (n > count) ? n - count : 0);
1647
1648                         if (xcount < 0)
1649                                 return xcount;
1650                 /* implemented by the driver */
1651                 } else if (dev->dev_ops->xstats_get != NULL) {
1652                         /* Retrieve the xstats from the driver at the end of the
1653                          * xstats struct. Retrieve all xstats.
1654                          * Compatibility for PMD without xstats_get_by_ids
1655                          */
1656                         unsigned int size = (n > count) ? n - count : 1;
1657                         struct rte_eth_xstat xstats[size];
1658
1659                         xcount = (*dev->dev_ops->xstats_get)(dev,
1660                                         values ? xstats : NULL, size);
1661
1662                         if (xcount < 0)
1663                                 return xcount;
1664
1665                         if (values != NULL)
1666                                 for (i = 0 ; i < (unsigned int)xcount; i++)
1667                                         values[i + count] = xstats[i].value;
1668                 }
1669
1670                 if (n < count + xcount || values == NULL)
1671                         return count + xcount;
1672
1673                 /* now fill the xstats structure */
1674                 count = 0;
1675                 rte_eth_stats_get(port_id, &eth_stats);
1676
1677                 /* global stats */
1678                 for (i = 0; i < RTE_NB_STATS; i++) {
1679                         stats_ptr = RTE_PTR_ADD(&eth_stats,
1680                                                 rte_stats_strings[i].offset);
1681                         val = *stats_ptr;
1682                         values[count++] = val;
1683                 }
1684
1685                 /* per-rxq stats */
1686                 for (q = 0; q < nb_rxqs; q++) {
1687                         for (i = 0; i < RTE_NB_RXQ_STATS; i++) {
1688                                 stats_ptr = RTE_PTR_ADD(&eth_stats,
1689                                             rte_rxq_stats_strings[i].offset +
1690                                             q * sizeof(uint64_t));
1691                                 val = *stats_ptr;
1692                                 values[count++] = val;
1693                         }
1694                 }
1695
1696                 /* per-txq stats */
1697                 for (q = 0; q < nb_txqs; q++) {
1698                         for (i = 0; i < RTE_NB_TXQ_STATS; i++) {
1699                                 stats_ptr = RTE_PTR_ADD(&eth_stats,
1700                                             rte_txq_stats_strings[i].offset +
1701                                             q * sizeof(uint64_t));
1702                                 val = *stats_ptr;
1703                                 values[count++] = val;
1704                         }
1705                 }
1706
1707                 return count + xcount;
1708         }
1709         /* Need only xstats given by IDS array */
1710         else {
1711                 uint16_t i, size;
1712                 uint64_t *values_copy;
1713
1714                 size = rte_eth_xstats_get_by_id(port_id, NULL, NULL, 0);
1715
1716                 values_copy = malloc(sizeof(*values_copy) * size);
1717                 if (!values_copy) {
1718                         RTE_PMD_DEBUG_TRACE(
1719                             "ERROR: can't allocate memory for values_copy\n");
1720                         return -1;
1721                 }
1722
1723                 rte_eth_xstats_get_by_id(port_id, NULL, values_copy, size);
1724
1725                 for (i = 0; i < n; i++) {
1726                         if (ids[i] >= size) {
1727                                 RTE_PMD_DEBUG_TRACE(
1728                                         "ERROR: id value isn't valid\n");
1729                                 return -1;
1730                         }
1731                         values[i] = values_copy[ids[i]];
1732                 }
1733                 free(values_copy);
1734                 return n;
1735         }
1736 }
1737
1738 int
1739 rte_eth_xstats_get(uint8_t port_id, struct rte_eth_xstat *xstats,
1740         unsigned int n)
1741 {
1742         struct rte_eth_stats eth_stats;
1743         struct rte_eth_dev *dev;
1744         unsigned int count = 0, i, q;
1745         signed int xcount = 0;
1746         uint64_t val, *stats_ptr;
1747         uint16_t nb_rxqs, nb_txqs;
1748
1749         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1750
1751         dev = &rte_eth_devices[port_id];
1752
1753         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
1754         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
1755
1756         /* Return generic statistics */
1757         count = RTE_NB_STATS + (nb_rxqs * RTE_NB_RXQ_STATS) +
1758                 (nb_txqs * RTE_NB_TXQ_STATS);
1759
1760         /* implemented by the driver */
1761         if (dev->dev_ops->xstats_get != NULL) {
1762                 /* Retrieve the xstats from the driver at the end of the
1763                  * xstats struct.
1764                  */
1765                 xcount = (*dev->dev_ops->xstats_get)(dev,
1766                                      xstats ? xstats + count : NULL,
1767                                      (n > count) ? n - count : 0);
1768
1769                 if (xcount < 0)
1770                         return xcount;
1771         }
1772
1773         if (n < count + xcount || xstats == NULL)
1774                 return count + xcount;
1775
1776         /* now fill the xstats structure */
1777         count = 0;
1778         rte_eth_stats_get(port_id, &eth_stats);
1779
1780         /* global stats */
1781         for (i = 0; i < RTE_NB_STATS; i++) {
1782                 stats_ptr = RTE_PTR_ADD(&eth_stats,
1783                                         rte_stats_strings[i].offset);
1784                 val = *stats_ptr;
1785                 xstats[count++].value = val;
1786         }
1787
1788         /* per-rxq stats */
1789         for (q = 0; q < nb_rxqs; q++) {
1790                 for (i = 0; i < RTE_NB_RXQ_STATS; i++) {
1791                         stats_ptr = RTE_PTR_ADD(&eth_stats,
1792                                         rte_rxq_stats_strings[i].offset +
1793                                         q * sizeof(uint64_t));
1794                         val = *stats_ptr;
1795                         xstats[count++].value = val;
1796                 }
1797         }
1798
1799         /* per-txq stats */
1800         for (q = 0; q < nb_txqs; q++) {
1801                 for (i = 0; i < RTE_NB_TXQ_STATS; i++) {
1802                         stats_ptr = RTE_PTR_ADD(&eth_stats,
1803                                         rte_txq_stats_strings[i].offset +
1804                                         q * sizeof(uint64_t));
1805                         val = *stats_ptr;
1806                         xstats[count++].value = val;
1807                 }
1808         }
1809
1810         for (i = 0; i < count; i++)
1811                 xstats[i].id = i;
1812         /* add an offset to driver-specific stats */
1813         for ( ; i < count + xcount; i++)
1814                 xstats[i].id += count;
1815
1816         return count + xcount;
1817 }
1818
1819 /* reset ethdev extended statistics */
1820 void
1821 rte_eth_xstats_reset(uint8_t port_id)
1822 {
1823         struct rte_eth_dev *dev;
1824
1825         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1826         dev = &rte_eth_devices[port_id];
1827
1828         /* implemented by the driver */
1829         if (dev->dev_ops->xstats_reset != NULL) {
1830                 (*dev->dev_ops->xstats_reset)(dev);
1831                 return;
1832         }
1833
1834         /* fallback to default */
1835         rte_eth_stats_reset(port_id);
1836 }
1837
1838 static int
1839 set_queue_stats_mapping(uint8_t port_id, uint16_t queue_id, uint8_t stat_idx,
1840                 uint8_t is_rx)
1841 {
1842         struct rte_eth_dev *dev;
1843
1844         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1845
1846         dev = &rte_eth_devices[port_id];
1847
1848         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_stats_mapping_set, -ENOTSUP);
1849         return (*dev->dev_ops->queue_stats_mapping_set)
1850                         (dev, queue_id, stat_idx, is_rx);
1851 }
1852
1853
1854 int
1855 rte_eth_dev_set_tx_queue_stats_mapping(uint8_t port_id, uint16_t tx_queue_id,
1856                 uint8_t stat_idx)
1857 {
1858         return set_queue_stats_mapping(port_id, tx_queue_id, stat_idx,
1859                         STAT_QMAP_TX);
1860 }
1861
1862
1863 int
1864 rte_eth_dev_set_rx_queue_stats_mapping(uint8_t port_id, uint16_t rx_queue_id,
1865                 uint8_t stat_idx)
1866 {
1867         return set_queue_stats_mapping(port_id, rx_queue_id, stat_idx,
1868                         STAT_QMAP_RX);
1869 }
1870
1871 int
1872 rte_eth_dev_fw_version_get(uint8_t port_id, char *fw_version, size_t fw_size)
1873 {
1874         struct rte_eth_dev *dev;
1875
1876         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1877         dev = &rte_eth_devices[port_id];
1878
1879         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fw_version_get, -ENOTSUP);
1880         return (*dev->dev_ops->fw_version_get)(dev, fw_version, fw_size);
1881 }
1882
1883 void
1884 rte_eth_dev_info_get(uint8_t port_id, struct rte_eth_dev_info *dev_info)
1885 {
1886         struct rte_eth_dev *dev;
1887         const struct rte_eth_desc_lim lim = {
1888                 .nb_max = UINT16_MAX,
1889                 .nb_min = 0,
1890                 .nb_align = 1,
1891         };
1892
1893         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1894         dev = &rte_eth_devices[port_id];
1895
1896         memset(dev_info, 0, sizeof(struct rte_eth_dev_info));
1897         dev_info->rx_desc_lim = lim;
1898         dev_info->tx_desc_lim = lim;
1899
1900         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_infos_get);
1901         (*dev->dev_ops->dev_infos_get)(dev, dev_info);
1902         dev_info->driver_name = dev->data->drv_name;
1903         dev_info->nb_rx_queues = dev->data->nb_rx_queues;
1904         dev_info->nb_tx_queues = dev->data->nb_tx_queues;
1905 }
1906
1907 int
1908 rte_eth_dev_get_supported_ptypes(uint8_t port_id, uint32_t ptype_mask,
1909                                  uint32_t *ptypes, int num)
1910 {
1911         int i, j;
1912         struct rte_eth_dev *dev;
1913         const uint32_t *all_ptypes;
1914
1915         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1916         dev = &rte_eth_devices[port_id];
1917         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_supported_ptypes_get, 0);
1918         all_ptypes = (*dev->dev_ops->dev_supported_ptypes_get)(dev);
1919
1920         if (!all_ptypes)
1921                 return 0;
1922
1923         for (i = 0, j = 0; all_ptypes[i] != RTE_PTYPE_UNKNOWN; ++i)
1924                 if (all_ptypes[i] & ptype_mask) {
1925                         if (j < num)
1926                                 ptypes[j] = all_ptypes[i];
1927                         j++;
1928                 }
1929
1930         return j;
1931 }
1932
1933 void
1934 rte_eth_macaddr_get(uint8_t port_id, struct ether_addr *mac_addr)
1935 {
1936         struct rte_eth_dev *dev;
1937
1938         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1939         dev = &rte_eth_devices[port_id];
1940         ether_addr_copy(&dev->data->mac_addrs[0], mac_addr);
1941 }
1942
1943
1944 int
1945 rte_eth_dev_get_mtu(uint8_t port_id, uint16_t *mtu)
1946 {
1947         struct rte_eth_dev *dev;
1948
1949         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1950
1951         dev = &rte_eth_devices[port_id];
1952         *mtu = dev->data->mtu;
1953         return 0;
1954 }
1955
1956 int
1957 rte_eth_dev_set_mtu(uint8_t port_id, uint16_t mtu)
1958 {
1959         int ret;
1960         struct rte_eth_dev *dev;
1961
1962         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1963         dev = &rte_eth_devices[port_id];
1964         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mtu_set, -ENOTSUP);
1965
1966         ret = (*dev->dev_ops->mtu_set)(dev, mtu);
1967         if (!ret)
1968                 dev->data->mtu = mtu;
1969
1970         return ret;
1971 }
1972
1973 int
1974 rte_eth_dev_vlan_filter(uint8_t port_id, uint16_t vlan_id, int on)
1975 {
1976         struct rte_eth_dev *dev;
1977
1978         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1979         dev = &rte_eth_devices[port_id];
1980         if (!(dev->data->dev_conf.rxmode.hw_vlan_filter)) {
1981                 RTE_PMD_DEBUG_TRACE("port %d: vlan-filtering disabled\n", port_id);
1982                 return -ENOSYS;
1983         }
1984
1985         if (vlan_id > 4095) {
1986                 RTE_PMD_DEBUG_TRACE("(port_id=%d) invalid vlan_id=%u > 4095\n",
1987                                 port_id, (unsigned) vlan_id);
1988                 return -EINVAL;
1989         }
1990         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_filter_set, -ENOTSUP);
1991
1992         return (*dev->dev_ops->vlan_filter_set)(dev, vlan_id, on);
1993 }
1994
1995 int
1996 rte_eth_dev_set_vlan_strip_on_queue(uint8_t port_id, uint16_t rx_queue_id, int on)
1997 {
1998         struct rte_eth_dev *dev;
1999
2000         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2001         dev = &rte_eth_devices[port_id];
2002         if (rx_queue_id >= dev->data->nb_rx_queues) {
2003                 RTE_PMD_DEBUG_TRACE("Invalid rx_queue_id=%d\n", port_id);
2004                 return -EINVAL;
2005         }
2006
2007         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_strip_queue_set, -ENOTSUP);
2008         (*dev->dev_ops->vlan_strip_queue_set)(dev, rx_queue_id, on);
2009
2010         return 0;
2011 }
2012
2013 int
2014 rte_eth_dev_set_vlan_ether_type(uint8_t port_id,
2015                                 enum rte_vlan_type vlan_type,
2016                                 uint16_t tpid)
2017 {
2018         struct rte_eth_dev *dev;
2019
2020         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2021         dev = &rte_eth_devices[port_id];
2022         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_tpid_set, -ENOTSUP);
2023
2024         return (*dev->dev_ops->vlan_tpid_set)(dev, vlan_type, tpid);
2025 }
2026
2027 int
2028 rte_eth_dev_set_vlan_offload(uint8_t port_id, int offload_mask)
2029 {
2030         struct rte_eth_dev *dev;
2031         int ret = 0;
2032         int mask = 0;
2033         int cur, org = 0;
2034
2035         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2036         dev = &rte_eth_devices[port_id];
2037
2038         /*check which option changed by application*/
2039         cur = !!(offload_mask & ETH_VLAN_STRIP_OFFLOAD);
2040         org = !!(dev->data->dev_conf.rxmode.hw_vlan_strip);
2041         if (cur != org) {
2042                 dev->data->dev_conf.rxmode.hw_vlan_strip = (uint8_t)cur;
2043                 mask |= ETH_VLAN_STRIP_MASK;
2044         }
2045
2046         cur = !!(offload_mask & ETH_VLAN_FILTER_OFFLOAD);
2047         org = !!(dev->data->dev_conf.rxmode.hw_vlan_filter);
2048         if (cur != org) {
2049                 dev->data->dev_conf.rxmode.hw_vlan_filter = (uint8_t)cur;
2050                 mask |= ETH_VLAN_FILTER_MASK;
2051         }
2052
2053         cur = !!(offload_mask & ETH_VLAN_EXTEND_OFFLOAD);
2054         org = !!(dev->data->dev_conf.rxmode.hw_vlan_extend);
2055         if (cur != org) {
2056                 dev->data->dev_conf.rxmode.hw_vlan_extend = (uint8_t)cur;
2057                 mask |= ETH_VLAN_EXTEND_MASK;
2058         }
2059
2060         /*no change*/
2061         if (mask == 0)
2062                 return ret;
2063
2064         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_offload_set, -ENOTSUP);
2065         (*dev->dev_ops->vlan_offload_set)(dev, mask);
2066
2067         return ret;
2068 }
2069
2070 int
2071 rte_eth_dev_get_vlan_offload(uint8_t port_id)
2072 {
2073         struct rte_eth_dev *dev;
2074         int ret = 0;
2075
2076         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2077         dev = &rte_eth_devices[port_id];
2078
2079         if (dev->data->dev_conf.rxmode.hw_vlan_strip)
2080                 ret |= ETH_VLAN_STRIP_OFFLOAD;
2081
2082         if (dev->data->dev_conf.rxmode.hw_vlan_filter)
2083                 ret |= ETH_VLAN_FILTER_OFFLOAD;
2084
2085         if (dev->data->dev_conf.rxmode.hw_vlan_extend)
2086                 ret |= ETH_VLAN_EXTEND_OFFLOAD;
2087
2088         return ret;
2089 }
2090
2091 int
2092 rte_eth_dev_set_vlan_pvid(uint8_t port_id, uint16_t pvid, int on)
2093 {
2094         struct rte_eth_dev *dev;
2095
2096         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2097         dev = &rte_eth_devices[port_id];
2098         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_pvid_set, -ENOTSUP);
2099         (*dev->dev_ops->vlan_pvid_set)(dev, pvid, on);
2100
2101         return 0;
2102 }
2103
2104 int
2105 rte_eth_dev_flow_ctrl_get(uint8_t port_id, struct rte_eth_fc_conf *fc_conf)
2106 {
2107         struct rte_eth_dev *dev;
2108
2109         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2110         dev = &rte_eth_devices[port_id];
2111         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_get, -ENOTSUP);
2112         memset(fc_conf, 0, sizeof(*fc_conf));
2113         return (*dev->dev_ops->flow_ctrl_get)(dev, fc_conf);
2114 }
2115
2116 int
2117 rte_eth_dev_flow_ctrl_set(uint8_t port_id, struct rte_eth_fc_conf *fc_conf)
2118 {
2119         struct rte_eth_dev *dev;
2120
2121         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2122         if ((fc_conf->send_xon != 0) && (fc_conf->send_xon != 1)) {
2123                 RTE_PMD_DEBUG_TRACE("Invalid send_xon, only 0/1 allowed\n");
2124                 return -EINVAL;
2125         }
2126
2127         dev = &rte_eth_devices[port_id];
2128         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_set, -ENOTSUP);
2129         return (*dev->dev_ops->flow_ctrl_set)(dev, fc_conf);
2130 }
2131
2132 int
2133 rte_eth_dev_priority_flow_ctrl_set(uint8_t port_id, struct rte_eth_pfc_conf *pfc_conf)
2134 {
2135         struct rte_eth_dev *dev;
2136
2137         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2138         if (pfc_conf->priority > (ETH_DCB_NUM_USER_PRIORITIES - 1)) {
2139                 RTE_PMD_DEBUG_TRACE("Invalid priority, only 0-7 allowed\n");
2140                 return -EINVAL;
2141         }
2142
2143         dev = &rte_eth_devices[port_id];
2144         /* High water, low water validation are device specific */
2145         if  (*dev->dev_ops->priority_flow_ctrl_set)
2146                 return (*dev->dev_ops->priority_flow_ctrl_set)(dev, pfc_conf);
2147         return -ENOTSUP;
2148 }
2149
2150 static int
2151 rte_eth_check_reta_mask(struct rte_eth_rss_reta_entry64 *reta_conf,
2152                         uint16_t reta_size)
2153 {
2154         uint16_t i, num;
2155
2156         if (!reta_conf)
2157                 return -EINVAL;
2158
2159         num = (reta_size + RTE_RETA_GROUP_SIZE - 1) / RTE_RETA_GROUP_SIZE;
2160         for (i = 0; i < num; i++) {
2161                 if (reta_conf[i].mask)
2162                         return 0;
2163         }
2164
2165         return -EINVAL;
2166 }
2167
2168 static int
2169 rte_eth_check_reta_entry(struct rte_eth_rss_reta_entry64 *reta_conf,
2170                          uint16_t reta_size,
2171                          uint16_t max_rxq)
2172 {
2173         uint16_t i, idx, shift;
2174
2175         if (!reta_conf)
2176                 return -EINVAL;
2177
2178         if (max_rxq == 0) {
2179                 RTE_PMD_DEBUG_TRACE("No receive queue is available\n");
2180                 return -EINVAL;
2181         }
2182
2183         for (i = 0; i < reta_size; i++) {
2184                 idx = i / RTE_RETA_GROUP_SIZE;
2185                 shift = i % RTE_RETA_GROUP_SIZE;
2186                 if ((reta_conf[idx].mask & (1ULL << shift)) &&
2187                         (reta_conf[idx].reta[shift] >= max_rxq)) {
2188                         RTE_PMD_DEBUG_TRACE("reta_conf[%u]->reta[%u]: %u exceeds "
2189                                 "the maximum rxq index: %u\n", idx, shift,
2190                                 reta_conf[idx].reta[shift], max_rxq);
2191                         return -EINVAL;
2192                 }
2193         }
2194
2195         return 0;
2196 }
2197
2198 int
2199 rte_eth_dev_rss_reta_update(uint8_t port_id,
2200                             struct rte_eth_rss_reta_entry64 *reta_conf,
2201                             uint16_t reta_size)
2202 {
2203         struct rte_eth_dev *dev;
2204         int ret;
2205
2206         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2207         /* Check mask bits */
2208         ret = rte_eth_check_reta_mask(reta_conf, reta_size);
2209         if (ret < 0)
2210                 return ret;
2211
2212         dev = &rte_eth_devices[port_id];
2213
2214         /* Check entry value */
2215         ret = rte_eth_check_reta_entry(reta_conf, reta_size,
2216                                 dev->data->nb_rx_queues);
2217         if (ret < 0)
2218                 return ret;
2219
2220         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_update, -ENOTSUP);
2221         return (*dev->dev_ops->reta_update)(dev, reta_conf, reta_size);
2222 }
2223
2224 int
2225 rte_eth_dev_rss_reta_query(uint8_t port_id,
2226                            struct rte_eth_rss_reta_entry64 *reta_conf,
2227                            uint16_t reta_size)
2228 {
2229         struct rte_eth_dev *dev;
2230         int ret;
2231
2232         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2233
2234         /* Check mask bits */
2235         ret = rte_eth_check_reta_mask(reta_conf, reta_size);
2236         if (ret < 0)
2237                 return ret;
2238
2239         dev = &rte_eth_devices[port_id];
2240         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_query, -ENOTSUP);
2241         return (*dev->dev_ops->reta_query)(dev, reta_conf, reta_size);
2242 }
2243
2244 int
2245 rte_eth_dev_rss_hash_update(uint8_t port_id, struct rte_eth_rss_conf *rss_conf)
2246 {
2247         struct rte_eth_dev *dev;
2248         uint16_t rss_hash_protos;
2249
2250         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2251         rss_hash_protos = rss_conf->rss_hf;
2252         if ((rss_hash_protos != 0) &&
2253             ((rss_hash_protos & ETH_RSS_PROTO_MASK) == 0)) {
2254                 RTE_PMD_DEBUG_TRACE("Invalid rss_hash_protos=0x%x\n",
2255                                 rss_hash_protos);
2256                 return -EINVAL;
2257         }
2258         dev = &rte_eth_devices[port_id];
2259         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_update, -ENOTSUP);
2260         return (*dev->dev_ops->rss_hash_update)(dev, rss_conf);
2261 }
2262
2263 int
2264 rte_eth_dev_rss_hash_conf_get(uint8_t port_id,
2265                               struct rte_eth_rss_conf *rss_conf)
2266 {
2267         struct rte_eth_dev *dev;
2268
2269         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2270         dev = &rte_eth_devices[port_id];
2271         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_conf_get, -ENOTSUP);
2272         return (*dev->dev_ops->rss_hash_conf_get)(dev, rss_conf);
2273 }
2274
2275 int
2276 rte_eth_dev_udp_tunnel_port_add(uint8_t port_id,
2277                                 struct rte_eth_udp_tunnel *udp_tunnel)
2278 {
2279         struct rte_eth_dev *dev;
2280
2281         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2282         if (udp_tunnel == NULL) {
2283                 RTE_PMD_DEBUG_TRACE("Invalid udp_tunnel parameter\n");
2284                 return -EINVAL;
2285         }
2286
2287         if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
2288                 RTE_PMD_DEBUG_TRACE("Invalid tunnel type\n");
2289                 return -EINVAL;
2290         }
2291
2292         dev = &rte_eth_devices[port_id];
2293         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_port_add, -ENOTSUP);
2294         return (*dev->dev_ops->udp_tunnel_port_add)(dev, udp_tunnel);
2295 }
2296
2297 int
2298 rte_eth_dev_udp_tunnel_port_delete(uint8_t port_id,
2299                                    struct rte_eth_udp_tunnel *udp_tunnel)
2300 {
2301         struct rte_eth_dev *dev;
2302
2303         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2304         dev = &rte_eth_devices[port_id];
2305
2306         if (udp_tunnel == NULL) {
2307                 RTE_PMD_DEBUG_TRACE("Invalid udp_tunnel parameter\n");
2308                 return -EINVAL;
2309         }
2310
2311         if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
2312                 RTE_PMD_DEBUG_TRACE("Invalid tunnel type\n");
2313                 return -EINVAL;
2314         }
2315
2316         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_port_del, -ENOTSUP);
2317         return (*dev->dev_ops->udp_tunnel_port_del)(dev, udp_tunnel);
2318 }
2319
2320 int
2321 rte_eth_led_on(uint8_t port_id)
2322 {
2323         struct rte_eth_dev *dev;
2324
2325         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2326         dev = &rte_eth_devices[port_id];
2327         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_on, -ENOTSUP);
2328         return (*dev->dev_ops->dev_led_on)(dev);
2329 }
2330
2331 int
2332 rte_eth_led_off(uint8_t port_id)
2333 {
2334         struct rte_eth_dev *dev;
2335
2336         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2337         dev = &rte_eth_devices[port_id];
2338         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_off, -ENOTSUP);
2339         return (*dev->dev_ops->dev_led_off)(dev);
2340 }
2341
2342 /*
2343  * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find
2344  * an empty spot.
2345  */
2346 static int
2347 get_mac_addr_index(uint8_t port_id, const struct ether_addr *addr)
2348 {
2349         struct rte_eth_dev_info dev_info;
2350         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
2351         unsigned i;
2352
2353         rte_eth_dev_info_get(port_id, &dev_info);
2354
2355         for (i = 0; i < dev_info.max_mac_addrs; i++)
2356                 if (memcmp(addr, &dev->data->mac_addrs[i], ETHER_ADDR_LEN) == 0)
2357                         return i;
2358
2359         return -1;
2360 }
2361
2362 static const struct ether_addr null_mac_addr;
2363
2364 int
2365 rte_eth_dev_mac_addr_add(uint8_t port_id, struct ether_addr *addr,
2366                         uint32_t pool)
2367 {
2368         struct rte_eth_dev *dev;
2369         int index;
2370         uint64_t pool_mask;
2371         int ret;
2372
2373         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2374         dev = &rte_eth_devices[port_id];
2375         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_add, -ENOTSUP);
2376
2377         if (is_zero_ether_addr(addr)) {
2378                 RTE_PMD_DEBUG_TRACE("port %d: Cannot add NULL MAC address\n",
2379                         port_id);
2380                 return -EINVAL;
2381         }
2382         if (pool >= ETH_64_POOLS) {
2383                 RTE_PMD_DEBUG_TRACE("pool id must be 0-%d\n", ETH_64_POOLS - 1);
2384                 return -EINVAL;
2385         }
2386
2387         index = get_mac_addr_index(port_id, addr);
2388         if (index < 0) {
2389                 index = get_mac_addr_index(port_id, &null_mac_addr);
2390                 if (index < 0) {
2391                         RTE_PMD_DEBUG_TRACE("port %d: MAC address array full\n",
2392                                 port_id);
2393                         return -ENOSPC;
2394                 }
2395         } else {
2396                 pool_mask = dev->data->mac_pool_sel[index];
2397
2398                 /* Check if both MAC address and pool is already there, and do nothing */
2399                 if (pool_mask & (1ULL << pool))
2400                         return 0;
2401         }
2402
2403         /* Update NIC */
2404         ret = (*dev->dev_ops->mac_addr_add)(dev, addr, index, pool);
2405
2406         if (ret == 0) {
2407                 /* Update address in NIC data structure */
2408                 ether_addr_copy(addr, &dev->data->mac_addrs[index]);
2409
2410                 /* Update pool bitmap in NIC data structure */
2411                 dev->data->mac_pool_sel[index] |= (1ULL << pool);
2412         }
2413
2414         return ret;
2415 }
2416
2417 int
2418 rte_eth_dev_mac_addr_remove(uint8_t port_id, struct ether_addr *addr)
2419 {
2420         struct rte_eth_dev *dev;
2421         int index;
2422
2423         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2424         dev = &rte_eth_devices[port_id];
2425         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_remove, -ENOTSUP);
2426
2427         index = get_mac_addr_index(port_id, addr);
2428         if (index == 0) {
2429                 RTE_PMD_DEBUG_TRACE("port %d: Cannot remove default MAC address\n", port_id);
2430                 return -EADDRINUSE;
2431         } else if (index < 0)
2432                 return 0;  /* Do nothing if address wasn't found */
2433
2434         /* Update NIC */
2435         (*dev->dev_ops->mac_addr_remove)(dev, index);
2436
2437         /* Update address in NIC data structure */
2438         ether_addr_copy(&null_mac_addr, &dev->data->mac_addrs[index]);
2439
2440         /* reset pool bitmap */
2441         dev->data->mac_pool_sel[index] = 0;
2442
2443         return 0;
2444 }
2445
2446 int
2447 rte_eth_dev_default_mac_addr_set(uint8_t port_id, struct ether_addr *addr)
2448 {
2449         struct rte_eth_dev *dev;
2450
2451         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2452
2453         if (!is_valid_assigned_ether_addr(addr))
2454                 return -EINVAL;
2455
2456         dev = &rte_eth_devices[port_id];
2457         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_set, -ENOTSUP);
2458
2459         /* Update default address in NIC data structure */
2460         ether_addr_copy(addr, &dev->data->mac_addrs[0]);
2461
2462         (*dev->dev_ops->mac_addr_set)(dev, addr);
2463
2464         return 0;
2465 }
2466
2467
2468 /*
2469  * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find
2470  * an empty spot.
2471  */
2472 static int
2473 get_hash_mac_addr_index(uint8_t port_id, const struct ether_addr *addr)
2474 {
2475         struct rte_eth_dev_info dev_info;
2476         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
2477         unsigned i;
2478
2479         rte_eth_dev_info_get(port_id, &dev_info);
2480         if (!dev->data->hash_mac_addrs)
2481                 return -1;
2482
2483         for (i = 0; i < dev_info.max_hash_mac_addrs; i++)
2484                 if (memcmp(addr, &dev->data->hash_mac_addrs[i],
2485                         ETHER_ADDR_LEN) == 0)
2486                         return i;
2487
2488         return -1;
2489 }
2490
2491 int
2492 rte_eth_dev_uc_hash_table_set(uint8_t port_id, struct ether_addr *addr,
2493                                 uint8_t on)
2494 {
2495         int index;
2496         int ret;
2497         struct rte_eth_dev *dev;
2498
2499         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2500
2501         dev = &rte_eth_devices[port_id];
2502         if (is_zero_ether_addr(addr)) {
2503                 RTE_PMD_DEBUG_TRACE("port %d: Cannot add NULL MAC address\n",
2504                         port_id);
2505                 return -EINVAL;
2506         }
2507
2508         index = get_hash_mac_addr_index(port_id, addr);
2509         /* Check if it's already there, and do nothing */
2510         if ((index >= 0) && (on))
2511                 return 0;
2512
2513         if (index < 0) {
2514                 if (!on) {
2515                         RTE_PMD_DEBUG_TRACE("port %d: the MAC address was not "
2516                                 "set in UTA\n", port_id);
2517                         return -EINVAL;
2518                 }
2519
2520                 index = get_hash_mac_addr_index(port_id, &null_mac_addr);
2521                 if (index < 0) {
2522                         RTE_PMD_DEBUG_TRACE("port %d: MAC address array full\n",
2523                                         port_id);
2524                         return -ENOSPC;
2525                 }
2526         }
2527
2528         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_hash_table_set, -ENOTSUP);
2529         ret = (*dev->dev_ops->uc_hash_table_set)(dev, addr, on);
2530         if (ret == 0) {
2531                 /* Update address in NIC data structure */
2532                 if (on)
2533                         ether_addr_copy(addr,
2534                                         &dev->data->hash_mac_addrs[index]);
2535                 else
2536                         ether_addr_copy(&null_mac_addr,
2537                                         &dev->data->hash_mac_addrs[index]);
2538         }
2539
2540         return ret;
2541 }
2542
2543 int
2544 rte_eth_dev_uc_all_hash_table_set(uint8_t port_id, uint8_t on)
2545 {
2546         struct rte_eth_dev *dev;
2547
2548         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2549
2550         dev = &rte_eth_devices[port_id];
2551
2552         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_all_hash_table_set, -ENOTSUP);
2553         return (*dev->dev_ops->uc_all_hash_table_set)(dev, on);
2554 }
2555
2556 int rte_eth_set_queue_rate_limit(uint8_t port_id, uint16_t queue_idx,
2557                                         uint16_t tx_rate)
2558 {
2559         struct rte_eth_dev *dev;
2560         struct rte_eth_dev_info dev_info;
2561         struct rte_eth_link link;
2562
2563         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2564
2565         dev = &rte_eth_devices[port_id];
2566         rte_eth_dev_info_get(port_id, &dev_info);
2567         link = dev->data->dev_link;
2568
2569         if (queue_idx > dev_info.max_tx_queues) {
2570                 RTE_PMD_DEBUG_TRACE("set queue rate limit:port %d: "
2571                                 "invalid queue id=%d\n", port_id, queue_idx);
2572                 return -EINVAL;
2573         }
2574
2575         if (tx_rate > link.link_speed) {
2576                 RTE_PMD_DEBUG_TRACE("set queue rate limit:invalid tx_rate=%d, "
2577                                 "bigger than link speed= %d\n",
2578                         tx_rate, link.link_speed);
2579                 return -EINVAL;
2580         }
2581
2582         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_queue_rate_limit, -ENOTSUP);
2583         return (*dev->dev_ops->set_queue_rate_limit)(dev, queue_idx, tx_rate);
2584 }
2585
2586 int
2587 rte_eth_mirror_rule_set(uint8_t port_id,
2588                         struct rte_eth_mirror_conf *mirror_conf,
2589                         uint8_t rule_id, uint8_t on)
2590 {
2591         struct rte_eth_dev *dev;
2592
2593         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2594         if (mirror_conf->rule_type == 0) {
2595                 RTE_PMD_DEBUG_TRACE("mirror rule type can not be 0.\n");
2596                 return -EINVAL;
2597         }
2598
2599         if (mirror_conf->dst_pool >= ETH_64_POOLS) {
2600                 RTE_PMD_DEBUG_TRACE("Invalid dst pool, pool id must be 0-%d\n",
2601                                 ETH_64_POOLS - 1);
2602                 return -EINVAL;
2603         }
2604
2605         if ((mirror_conf->rule_type & (ETH_MIRROR_VIRTUAL_POOL_UP |
2606              ETH_MIRROR_VIRTUAL_POOL_DOWN)) &&
2607             (mirror_conf->pool_mask == 0)) {
2608                 RTE_PMD_DEBUG_TRACE("Invalid mirror pool, pool mask can not be 0.\n");
2609                 return -EINVAL;
2610         }
2611
2612         if ((mirror_conf->rule_type & ETH_MIRROR_VLAN) &&
2613             mirror_conf->vlan.vlan_mask == 0) {
2614                 RTE_PMD_DEBUG_TRACE("Invalid vlan mask, vlan mask can not be 0.\n");
2615                 return -EINVAL;
2616         }
2617
2618         dev = &rte_eth_devices[port_id];
2619         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mirror_rule_set, -ENOTSUP);
2620
2621         return (*dev->dev_ops->mirror_rule_set)(dev, mirror_conf, rule_id, on);
2622 }
2623
2624 int
2625 rte_eth_mirror_rule_reset(uint8_t port_id, uint8_t rule_id)
2626 {
2627         struct rte_eth_dev *dev;
2628
2629         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2630
2631         dev = &rte_eth_devices[port_id];
2632         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mirror_rule_reset, -ENOTSUP);
2633
2634         return (*dev->dev_ops->mirror_rule_reset)(dev, rule_id);
2635 }
2636
2637 int
2638 rte_eth_dev_callback_register(uint8_t port_id,
2639                         enum rte_eth_event_type event,
2640                         rte_eth_dev_cb_fn cb_fn, void *cb_arg)
2641 {
2642         struct rte_eth_dev *dev;
2643         struct rte_eth_dev_callback *user_cb;
2644
2645         if (!cb_fn)
2646                 return -EINVAL;
2647
2648         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2649
2650         dev = &rte_eth_devices[port_id];
2651         rte_spinlock_lock(&rte_eth_dev_cb_lock);
2652
2653         TAILQ_FOREACH(user_cb, &(dev->link_intr_cbs), next) {
2654                 if (user_cb->cb_fn == cb_fn &&
2655                         user_cb->cb_arg == cb_arg &&
2656                         user_cb->event == event) {
2657                         break;
2658                 }
2659         }
2660
2661         /* create a new callback. */
2662         if (user_cb == NULL) {
2663                 user_cb = rte_zmalloc("INTR_USER_CALLBACK",
2664                                         sizeof(struct rte_eth_dev_callback), 0);
2665                 if (user_cb != NULL) {
2666                         user_cb->cb_fn = cb_fn;
2667                         user_cb->cb_arg = cb_arg;
2668                         user_cb->event = event;
2669                         TAILQ_INSERT_TAIL(&(dev->link_intr_cbs), user_cb, next);
2670                 }
2671         }
2672
2673         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
2674         return (user_cb == NULL) ? -ENOMEM : 0;
2675 }
2676
2677 int
2678 rte_eth_dev_callback_unregister(uint8_t port_id,
2679                         enum rte_eth_event_type event,
2680                         rte_eth_dev_cb_fn cb_fn, void *cb_arg)
2681 {
2682         int ret;
2683         struct rte_eth_dev *dev;
2684         struct rte_eth_dev_callback *cb, *next;
2685
2686         if (!cb_fn)
2687                 return -EINVAL;
2688
2689         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2690
2691         dev = &rte_eth_devices[port_id];
2692         rte_spinlock_lock(&rte_eth_dev_cb_lock);
2693
2694         ret = 0;
2695         for (cb = TAILQ_FIRST(&dev->link_intr_cbs); cb != NULL; cb = next) {
2696
2697                 next = TAILQ_NEXT(cb, next);
2698
2699                 if (cb->cb_fn != cb_fn || cb->event != event ||
2700                                 (cb->cb_arg != (void *)-1 &&
2701                                 cb->cb_arg != cb_arg))
2702                         continue;
2703
2704                 /*
2705                  * if this callback is not executing right now,
2706                  * then remove it.
2707                  */
2708                 if (cb->active == 0) {
2709                         TAILQ_REMOVE(&(dev->link_intr_cbs), cb, next);
2710                         rte_free(cb);
2711                 } else {
2712                         ret = -EAGAIN;
2713                 }
2714         }
2715
2716         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
2717         return ret;
2718 }
2719
2720 void
2721 _rte_eth_dev_callback_process(struct rte_eth_dev *dev,
2722         enum rte_eth_event_type event, void *cb_arg)
2723 {
2724         struct rte_eth_dev_callback *cb_lst;
2725         struct rte_eth_dev_callback dev_cb;
2726
2727         rte_spinlock_lock(&rte_eth_dev_cb_lock);
2728         TAILQ_FOREACH(cb_lst, &(dev->link_intr_cbs), next) {
2729                 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
2730                         continue;
2731                 dev_cb = *cb_lst;
2732                 cb_lst->active = 1;
2733                 if (cb_arg != NULL)
2734                         dev_cb.cb_arg = cb_arg;
2735
2736                 rte_spinlock_unlock(&rte_eth_dev_cb_lock);
2737                 dev_cb.cb_fn(dev->data->port_id, dev_cb.event,
2738                                                 dev_cb.cb_arg);
2739                 rte_spinlock_lock(&rte_eth_dev_cb_lock);
2740                 cb_lst->active = 0;
2741         }
2742         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
2743 }
2744
2745 int
2746 rte_eth_dev_rx_intr_ctl(uint8_t port_id, int epfd, int op, void *data)
2747 {
2748         uint32_t vec;
2749         struct rte_eth_dev *dev;
2750         struct rte_intr_handle *intr_handle;
2751         uint16_t qid;
2752         int rc;
2753
2754         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2755
2756         dev = &rte_eth_devices[port_id];
2757
2758         if (!dev->intr_handle) {
2759                 RTE_PMD_DEBUG_TRACE("RX Intr handle unset\n");
2760                 return -ENOTSUP;
2761         }
2762
2763         intr_handle = dev->intr_handle;
2764         if (!intr_handle->intr_vec) {
2765                 RTE_PMD_DEBUG_TRACE("RX Intr vector unset\n");
2766                 return -EPERM;
2767         }
2768
2769         for (qid = 0; qid < dev->data->nb_rx_queues; qid++) {
2770                 vec = intr_handle->intr_vec[qid];
2771                 rc = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
2772                 if (rc && rc != -EEXIST) {
2773                         RTE_PMD_DEBUG_TRACE("p %u q %u rx ctl error"
2774                                         " op %d epfd %d vec %u\n",
2775                                         port_id, qid, op, epfd, vec);
2776                 }
2777         }
2778
2779         return 0;
2780 }
2781
2782 const struct rte_memzone *
2783 rte_eth_dma_zone_reserve(const struct rte_eth_dev *dev, const char *ring_name,
2784                          uint16_t queue_id, size_t size, unsigned align,
2785                          int socket_id)
2786 {
2787         char z_name[RTE_MEMZONE_NAMESIZE];
2788         const struct rte_memzone *mz;
2789
2790         snprintf(z_name, sizeof(z_name), "%s_%s_%d_%d",
2791                  dev->data->drv_name, ring_name,
2792                  dev->data->port_id, queue_id);
2793
2794         mz = rte_memzone_lookup(z_name);
2795         if (mz)
2796                 return mz;
2797
2798         if (rte_xen_dom0_supported())
2799                 return rte_memzone_reserve_bounded(z_name, size, socket_id,
2800                                                    0, align, RTE_PGSIZE_2M);
2801         else
2802                 return rte_memzone_reserve_aligned(z_name, size, socket_id,
2803                                                    0, align);
2804 }
2805
2806 int
2807 rte_eth_dev_rx_intr_ctl_q(uint8_t port_id, uint16_t queue_id,
2808                           int epfd, int op, void *data)
2809 {
2810         uint32_t vec;
2811         struct rte_eth_dev *dev;
2812         struct rte_intr_handle *intr_handle;
2813         int rc;
2814
2815         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2816
2817         dev = &rte_eth_devices[port_id];
2818         if (queue_id >= dev->data->nb_rx_queues) {
2819                 RTE_PMD_DEBUG_TRACE("Invalid RX queue_id=%u\n", queue_id);
2820                 return -EINVAL;
2821         }
2822
2823         if (!dev->intr_handle) {
2824                 RTE_PMD_DEBUG_TRACE("RX Intr handle unset\n");
2825                 return -ENOTSUP;
2826         }
2827
2828         intr_handle = dev->intr_handle;
2829         if (!intr_handle->intr_vec) {
2830                 RTE_PMD_DEBUG_TRACE("RX Intr vector unset\n");
2831                 return -EPERM;
2832         }
2833
2834         vec = intr_handle->intr_vec[queue_id];
2835         rc = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
2836         if (rc && rc != -EEXIST) {
2837                 RTE_PMD_DEBUG_TRACE("p %u q %u rx ctl error"
2838                                 " op %d epfd %d vec %u\n",
2839                                 port_id, queue_id, op, epfd, vec);
2840                 return rc;
2841         }
2842
2843         return 0;
2844 }
2845
2846 int
2847 rte_eth_dev_rx_intr_enable(uint8_t port_id,
2848                            uint16_t queue_id)
2849 {
2850         struct rte_eth_dev *dev;
2851
2852         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2853
2854         dev = &rte_eth_devices[port_id];
2855
2856         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_intr_enable, -ENOTSUP);
2857         return (*dev->dev_ops->rx_queue_intr_enable)(dev, queue_id);
2858 }
2859
2860 int
2861 rte_eth_dev_rx_intr_disable(uint8_t port_id,
2862                             uint16_t queue_id)
2863 {
2864         struct rte_eth_dev *dev;
2865
2866         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2867
2868         dev = &rte_eth_devices[port_id];
2869
2870         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_intr_disable, -ENOTSUP);
2871         return (*dev->dev_ops->rx_queue_intr_disable)(dev, queue_id);
2872 }
2873
2874
2875 int
2876 rte_eth_dev_filter_supported(uint8_t port_id, enum rte_filter_type filter_type)
2877 {
2878         struct rte_eth_dev *dev;
2879
2880         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2881
2882         dev = &rte_eth_devices[port_id];
2883         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->filter_ctrl, -ENOTSUP);
2884         return (*dev->dev_ops->filter_ctrl)(dev, filter_type,
2885                                 RTE_ETH_FILTER_NOP, NULL);
2886 }
2887
2888 int
2889 rte_eth_dev_filter_ctrl(uint8_t port_id, enum rte_filter_type filter_type,
2890                        enum rte_filter_op filter_op, void *arg)
2891 {
2892         struct rte_eth_dev *dev;
2893
2894         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2895
2896         dev = &rte_eth_devices[port_id];
2897         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->filter_ctrl, -ENOTSUP);
2898         return (*dev->dev_ops->filter_ctrl)(dev, filter_type, filter_op, arg);
2899 }
2900
2901 void *
2902 rte_eth_add_rx_callback(uint8_t port_id, uint16_t queue_id,
2903                 rte_rx_callback_fn fn, void *user_param)
2904 {
2905 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
2906         rte_errno = ENOTSUP;
2907         return NULL;
2908 #endif
2909         /* check input parameters */
2910         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
2911                     queue_id >= rte_eth_devices[port_id].data->nb_rx_queues) {
2912                 rte_errno = EINVAL;
2913                 return NULL;
2914         }
2915         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
2916
2917         if (cb == NULL) {
2918                 rte_errno = ENOMEM;
2919                 return NULL;
2920         }
2921
2922         cb->fn.rx = fn;
2923         cb->param = user_param;
2924
2925         rte_spinlock_lock(&rte_eth_rx_cb_lock);
2926         /* Add the callbacks in fifo order. */
2927         struct rte_eth_rxtx_callback *tail =
2928                 rte_eth_devices[port_id].post_rx_burst_cbs[queue_id];
2929
2930         if (!tail) {
2931                 rte_eth_devices[port_id].post_rx_burst_cbs[queue_id] = cb;
2932
2933         } else {
2934                 while (tail->next)
2935                         tail = tail->next;
2936                 tail->next = cb;
2937         }
2938         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
2939
2940         return cb;
2941 }
2942
2943 void *
2944 rte_eth_add_first_rx_callback(uint8_t port_id, uint16_t queue_id,
2945                 rte_rx_callback_fn fn, void *user_param)
2946 {
2947 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
2948         rte_errno = ENOTSUP;
2949         return NULL;
2950 #endif
2951         /* check input parameters */
2952         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
2953                 queue_id >= rte_eth_devices[port_id].data->nb_rx_queues) {
2954                 rte_errno = EINVAL;
2955                 return NULL;
2956         }
2957
2958         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
2959
2960         if (cb == NULL) {
2961                 rte_errno = ENOMEM;
2962                 return NULL;
2963         }
2964
2965         cb->fn.rx = fn;
2966         cb->param = user_param;
2967
2968         rte_spinlock_lock(&rte_eth_rx_cb_lock);
2969         /* Add the callbacks at fisrt position*/
2970         cb->next = rte_eth_devices[port_id].post_rx_burst_cbs[queue_id];
2971         rte_smp_wmb();
2972         rte_eth_devices[port_id].post_rx_burst_cbs[queue_id] = cb;
2973         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
2974
2975         return cb;
2976 }
2977
2978 void *
2979 rte_eth_add_tx_callback(uint8_t port_id, uint16_t queue_id,
2980                 rte_tx_callback_fn fn, void *user_param)
2981 {
2982 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
2983         rte_errno = ENOTSUP;
2984         return NULL;
2985 #endif
2986         /* check input parameters */
2987         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
2988                     queue_id >= rte_eth_devices[port_id].data->nb_tx_queues) {
2989                 rte_errno = EINVAL;
2990                 return NULL;
2991         }
2992
2993         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
2994
2995         if (cb == NULL) {
2996                 rte_errno = ENOMEM;
2997                 return NULL;
2998         }
2999
3000         cb->fn.tx = fn;
3001         cb->param = user_param;
3002
3003         rte_spinlock_lock(&rte_eth_tx_cb_lock);
3004         /* Add the callbacks in fifo order. */
3005         struct rte_eth_rxtx_callback *tail =
3006                 rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id];
3007
3008         if (!tail) {
3009                 rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id] = cb;
3010
3011         } else {
3012                 while (tail->next)
3013                         tail = tail->next;
3014                 tail->next = cb;
3015         }
3016         rte_spinlock_unlock(&rte_eth_tx_cb_lock);
3017
3018         return cb;
3019 }
3020
3021 int
3022 rte_eth_remove_rx_callback(uint8_t port_id, uint16_t queue_id,
3023                 struct rte_eth_rxtx_callback *user_cb)
3024 {
3025 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3026         return -ENOTSUP;
3027 #endif
3028         /* Check input parameters. */
3029         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
3030         if (user_cb == NULL ||
3031                         queue_id >= rte_eth_devices[port_id].data->nb_rx_queues)
3032                 return -EINVAL;
3033
3034         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3035         struct rte_eth_rxtx_callback *cb;
3036         struct rte_eth_rxtx_callback **prev_cb;
3037         int ret = -EINVAL;
3038
3039         rte_spinlock_lock(&rte_eth_rx_cb_lock);
3040         prev_cb = &dev->post_rx_burst_cbs[queue_id];
3041         for (; *prev_cb != NULL; prev_cb = &cb->next) {
3042                 cb = *prev_cb;
3043                 if (cb == user_cb) {
3044                         /* Remove the user cb from the callback list. */
3045                         *prev_cb = cb->next;
3046                         ret = 0;
3047                         break;
3048                 }
3049         }
3050         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
3051
3052         return ret;
3053 }
3054
3055 int
3056 rte_eth_remove_tx_callback(uint8_t port_id, uint16_t queue_id,
3057                 struct rte_eth_rxtx_callback *user_cb)
3058 {
3059 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3060         return -ENOTSUP;
3061 #endif
3062         /* Check input parameters. */
3063         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
3064         if (user_cb == NULL ||
3065                         queue_id >= rte_eth_devices[port_id].data->nb_tx_queues)
3066                 return -EINVAL;
3067
3068         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3069         int ret = -EINVAL;
3070         struct rte_eth_rxtx_callback *cb;
3071         struct rte_eth_rxtx_callback **prev_cb;
3072
3073         rte_spinlock_lock(&rte_eth_tx_cb_lock);
3074         prev_cb = &dev->pre_tx_burst_cbs[queue_id];
3075         for (; *prev_cb != NULL; prev_cb = &cb->next) {
3076                 cb = *prev_cb;
3077                 if (cb == user_cb) {
3078                         /* Remove the user cb from the callback list. */
3079                         *prev_cb = cb->next;
3080                         ret = 0;
3081                         break;
3082                 }
3083         }
3084         rte_spinlock_unlock(&rte_eth_tx_cb_lock);
3085
3086         return ret;
3087 }
3088
3089 int
3090 rte_eth_rx_queue_info_get(uint8_t port_id, uint16_t queue_id,
3091         struct rte_eth_rxq_info *qinfo)
3092 {
3093         struct rte_eth_dev *dev;
3094
3095         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3096
3097         if (qinfo == NULL)
3098                 return -EINVAL;
3099
3100         dev = &rte_eth_devices[port_id];
3101         if (queue_id >= dev->data->nb_rx_queues) {
3102                 RTE_PMD_DEBUG_TRACE("Invalid RX queue_id=%d\n", queue_id);
3103                 return -EINVAL;
3104         }
3105
3106         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rxq_info_get, -ENOTSUP);
3107
3108         memset(qinfo, 0, sizeof(*qinfo));
3109         dev->dev_ops->rxq_info_get(dev, queue_id, qinfo);
3110         return 0;
3111 }
3112
3113 int
3114 rte_eth_tx_queue_info_get(uint8_t port_id, uint16_t queue_id,
3115         struct rte_eth_txq_info *qinfo)
3116 {
3117         struct rte_eth_dev *dev;
3118
3119         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3120
3121         if (qinfo == NULL)
3122                 return -EINVAL;
3123
3124         dev = &rte_eth_devices[port_id];
3125         if (queue_id >= dev->data->nb_tx_queues) {
3126                 RTE_PMD_DEBUG_TRACE("Invalid TX queue_id=%d\n", queue_id);
3127                 return -EINVAL;
3128         }
3129
3130         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->txq_info_get, -ENOTSUP);
3131
3132         memset(qinfo, 0, sizeof(*qinfo));
3133         dev->dev_ops->txq_info_get(dev, queue_id, qinfo);
3134         return 0;
3135 }
3136
3137 int
3138 rte_eth_dev_set_mc_addr_list(uint8_t port_id,
3139                              struct ether_addr *mc_addr_set,
3140                              uint32_t nb_mc_addr)
3141 {
3142         struct rte_eth_dev *dev;
3143
3144         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3145
3146         dev = &rte_eth_devices[port_id];
3147         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_mc_addr_list, -ENOTSUP);
3148         return dev->dev_ops->set_mc_addr_list(dev, mc_addr_set, nb_mc_addr);
3149 }
3150
3151 int
3152 rte_eth_timesync_enable(uint8_t port_id)
3153 {
3154         struct rte_eth_dev *dev;
3155
3156         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3157         dev = &rte_eth_devices[port_id];
3158
3159         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_enable, -ENOTSUP);
3160         return (*dev->dev_ops->timesync_enable)(dev);
3161 }
3162
3163 int
3164 rte_eth_timesync_disable(uint8_t port_id)
3165 {
3166         struct rte_eth_dev *dev;
3167
3168         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3169         dev = &rte_eth_devices[port_id];
3170
3171         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_disable, -ENOTSUP);
3172         return (*dev->dev_ops->timesync_disable)(dev);
3173 }
3174
3175 int
3176 rte_eth_timesync_read_rx_timestamp(uint8_t port_id, struct timespec *timestamp,
3177                                    uint32_t flags)
3178 {
3179         struct rte_eth_dev *dev;
3180
3181         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3182         dev = &rte_eth_devices[port_id];
3183
3184         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_rx_timestamp, -ENOTSUP);
3185         return (*dev->dev_ops->timesync_read_rx_timestamp)(dev, timestamp, flags);
3186 }
3187
3188 int
3189 rte_eth_timesync_read_tx_timestamp(uint8_t port_id, struct timespec *timestamp)
3190 {
3191         struct rte_eth_dev *dev;
3192
3193         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3194         dev = &rte_eth_devices[port_id];
3195
3196         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_tx_timestamp, -ENOTSUP);
3197         return (*dev->dev_ops->timesync_read_tx_timestamp)(dev, timestamp);
3198 }
3199
3200 int
3201 rte_eth_timesync_adjust_time(uint8_t port_id, int64_t delta)
3202 {
3203         struct rte_eth_dev *dev;
3204
3205         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3206         dev = &rte_eth_devices[port_id];
3207
3208         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_adjust_time, -ENOTSUP);
3209         return (*dev->dev_ops->timesync_adjust_time)(dev, delta);
3210 }
3211
3212 int
3213 rte_eth_timesync_read_time(uint8_t port_id, struct timespec *timestamp)
3214 {
3215         struct rte_eth_dev *dev;
3216
3217         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3218         dev = &rte_eth_devices[port_id];
3219
3220         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_time, -ENOTSUP);
3221         return (*dev->dev_ops->timesync_read_time)(dev, timestamp);
3222 }
3223
3224 int
3225 rte_eth_timesync_write_time(uint8_t port_id, const struct timespec *timestamp)
3226 {
3227         struct rte_eth_dev *dev;
3228
3229         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3230         dev = &rte_eth_devices[port_id];
3231
3232         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_write_time, -ENOTSUP);
3233         return (*dev->dev_ops->timesync_write_time)(dev, timestamp);
3234 }
3235
3236 int
3237 rte_eth_dev_get_reg_info(uint8_t port_id, struct rte_dev_reg_info *info)
3238 {
3239         struct rte_eth_dev *dev;
3240
3241         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3242
3243         dev = &rte_eth_devices[port_id];
3244         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_reg, -ENOTSUP);
3245         return (*dev->dev_ops->get_reg)(dev, info);
3246 }
3247
3248 int
3249 rte_eth_dev_get_eeprom_length(uint8_t port_id)
3250 {
3251         struct rte_eth_dev *dev;
3252
3253         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3254
3255         dev = &rte_eth_devices[port_id];
3256         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom_length, -ENOTSUP);
3257         return (*dev->dev_ops->get_eeprom_length)(dev);
3258 }
3259
3260 int
3261 rte_eth_dev_get_eeprom(uint8_t port_id, struct rte_dev_eeprom_info *info)
3262 {
3263         struct rte_eth_dev *dev;
3264
3265         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3266
3267         dev = &rte_eth_devices[port_id];
3268         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom, -ENOTSUP);
3269         return (*dev->dev_ops->get_eeprom)(dev, info);
3270 }
3271
3272 int
3273 rte_eth_dev_set_eeprom(uint8_t port_id, struct rte_dev_eeprom_info *info)
3274 {
3275         struct rte_eth_dev *dev;
3276
3277         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3278
3279         dev = &rte_eth_devices[port_id];
3280         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_eeprom, -ENOTSUP);
3281         return (*dev->dev_ops->set_eeprom)(dev, info);
3282 }
3283
3284 int
3285 rte_eth_dev_get_dcb_info(uint8_t port_id,
3286                              struct rte_eth_dcb_info *dcb_info)
3287 {
3288         struct rte_eth_dev *dev;
3289
3290         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3291
3292         dev = &rte_eth_devices[port_id];
3293         memset(dcb_info, 0, sizeof(struct rte_eth_dcb_info));
3294
3295         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_dcb_info, -ENOTSUP);
3296         return (*dev->dev_ops->get_dcb_info)(dev, dcb_info);
3297 }
3298
3299 int
3300 rte_eth_dev_l2_tunnel_eth_type_conf(uint8_t port_id,
3301                                     struct rte_eth_l2_tunnel_conf *l2_tunnel)
3302 {
3303         struct rte_eth_dev *dev;
3304
3305         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3306         if (l2_tunnel == NULL) {
3307                 RTE_PMD_DEBUG_TRACE("Invalid l2_tunnel parameter\n");
3308                 return -EINVAL;
3309         }
3310
3311         if (l2_tunnel->l2_tunnel_type >= RTE_TUNNEL_TYPE_MAX) {
3312                 RTE_PMD_DEBUG_TRACE("Invalid tunnel type\n");
3313                 return -EINVAL;
3314         }
3315
3316         dev = &rte_eth_devices[port_id];
3317         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->l2_tunnel_eth_type_conf,
3318                                 -ENOTSUP);
3319         return (*dev->dev_ops->l2_tunnel_eth_type_conf)(dev, l2_tunnel);
3320 }
3321
3322 int
3323 rte_eth_dev_l2_tunnel_offload_set(uint8_t port_id,
3324                                   struct rte_eth_l2_tunnel_conf *l2_tunnel,
3325                                   uint32_t mask,
3326                                   uint8_t en)
3327 {
3328         struct rte_eth_dev *dev;
3329
3330         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3331
3332         if (l2_tunnel == NULL) {
3333                 RTE_PMD_DEBUG_TRACE("Invalid l2_tunnel parameter\n");
3334                 return -EINVAL;
3335         }
3336
3337         if (l2_tunnel->l2_tunnel_type >= RTE_TUNNEL_TYPE_MAX) {
3338                 RTE_PMD_DEBUG_TRACE("Invalid tunnel type.\n");
3339                 return -EINVAL;
3340         }
3341
3342         if (mask == 0) {
3343                 RTE_PMD_DEBUG_TRACE("Mask should have a value.\n");
3344                 return -EINVAL;
3345         }
3346
3347         dev = &rte_eth_devices[port_id];
3348         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->l2_tunnel_offload_set,
3349                                 -ENOTSUP);
3350         return (*dev->dev_ops->l2_tunnel_offload_set)(dev, l2_tunnel, mask, en);
3351 }