lib/librte_ethdev/rte_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <sys/types.h>
   6 #include <sys/queue.h>
   7 #include <ctype.h>
   8 #include <stdio.h>
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <stdarg.h>
  12 #include <errno.h>
  13 #include <stdbool.h>
  14 #include <stdint.h>
  15 #include <inttypes.h>
  16 #include <netinet/in.h>
  17
  18 #include <rte_byteorder.h>
  19 #include <rte_log.h>
  20 #include <rte_debug.h>
  21 #include <rte_interrupts.h>
  22 #include <rte_memory.h>
  23 #include <rte_memcpy.h>
  24 #include <rte_memzone.h>
  25 #include <rte_launch.h>
  26 #include <rte_eal.h>
  27 #include <rte_per_lcore.h>
  28 #include <rte_lcore.h>
  29 #include <rte_atomic.h>
  30 #include <rte_branch_prediction.h>
  31 #include <rte_common.h>
  32 #include <rte_mempool.h>
  33 #include <rte_malloc.h>
  34 #include <rte_mbuf.h>
  35 #include <rte_errno.h>
  36 #include <rte_spinlock.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_kvargs.h>
  39
  40 #include "rte_ether.h"
  41 #include "rte_ethdev.h"
  42 #include "rte_ethdev_driver.h"
  43 #include "ethdev_profile.h"
  44
  45 int rte_eth_dev_logtype;
  46
  47 static const char *MZ_RTE_ETH_DEV_DATA = "rte_eth_dev_data";
  48 struct rte_eth_dev rte_eth_devices[RTE_MAX_ETHPORTS];
  49 static uint16_t eth_dev_last_created_port;
  50
  51 /* spinlock for eth device callbacks */
  52 static rte_spinlock_t rte_eth_dev_cb_lock = RTE_SPINLOCK_INITIALIZER;
  53
  54 /* spinlock for add/remove rx callbacks */
  55 static rte_spinlock_t rte_eth_rx_cb_lock = RTE_SPINLOCK_INITIALIZER;
  56
  57 /* spinlock for add/remove tx callbacks */
  58 static rte_spinlock_t rte_eth_tx_cb_lock = RTE_SPINLOCK_INITIALIZER;
  59
  60 /* spinlock for shared data allocation */
  61 static rte_spinlock_t rte_eth_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
  62
  63 /* store statistics names and its offset in stats structure  */
  64 struct rte_eth_xstats_name_off {
  65         char name[RTE_ETH_XSTATS_NAME_SIZE];
  66         unsigned offset;
  67 };
  68
  69 /* Shared memory between primary and secondary processes. */
  70 static struct {
  71         uint64_t next_owner_id;
  72         rte_spinlock_t ownership_lock;
  73         struct rte_eth_dev_data data[RTE_MAX_ETHPORTS];
  74 } *rte_eth_dev_shared_data;
  75
  76 static const struct rte_eth_xstats_name_off rte_stats_strings[] = {
  77         {"rx_good_packets", offsetof(struct rte_eth_stats, ipackets)},
  78         {"tx_good_packets", offsetof(struct rte_eth_stats, opackets)},
  79         {"rx_good_bytes", offsetof(struct rte_eth_stats, ibytes)},
  80         {"tx_good_bytes", offsetof(struct rte_eth_stats, obytes)},
  81         {"rx_missed_errors", offsetof(struct rte_eth_stats, imissed)},
  82         {"rx_errors", offsetof(struct rte_eth_stats, ierrors)},
  83         {"tx_errors", offsetof(struct rte_eth_stats, oerrors)},
  84         {"rx_mbuf_allocation_errors", offsetof(struct rte_eth_stats,
  85                 rx_nombuf)},
  86 };
  87
  88 #define RTE_NB_STATS (sizeof(rte_stats_strings) / sizeof(rte_stats_strings[0]))
  89
  90 static const struct rte_eth_xstats_name_off rte_rxq_stats_strings[] = {
  91         {"packets", offsetof(struct rte_eth_stats, q_ipackets)},
  92         {"bytes", offsetof(struct rte_eth_stats, q_ibytes)},
  93         {"errors", offsetof(struct rte_eth_stats, q_errors)},
  94 };
  95
  96 #define RTE_NB_RXQ_STATS (sizeof(rte_rxq_stats_strings) /       \
  97                 sizeof(rte_rxq_stats_strings[0]))
  98
  99 static const struct rte_eth_xstats_name_off rte_txq_stats_strings[] = {
 100         {"packets", offsetof(struct rte_eth_stats, q_opackets)},
 101         {"bytes", offsetof(struct rte_eth_stats, q_obytes)},
 102 };
 103 #define RTE_NB_TXQ_STATS (sizeof(rte_txq_stats_strings) /       \
 104                 sizeof(rte_txq_stats_strings[0]))
 105
 106 #define RTE_RX_OFFLOAD_BIT2STR(_name)   \
 107         { DEV_RX_OFFLOAD_##_name, #_name }
 108
 109 static const struct {
 110         uint64_t offload;
 111         const char *name;
 112 } rte_rx_offload_names[] = {
 113         RTE_RX_OFFLOAD_BIT2STR(VLAN_STRIP),
 114         RTE_RX_OFFLOAD_BIT2STR(IPV4_CKSUM),
 115         RTE_RX_OFFLOAD_BIT2STR(UDP_CKSUM),
 116         RTE_RX_OFFLOAD_BIT2STR(TCP_CKSUM),
 117         RTE_RX_OFFLOAD_BIT2STR(TCP_LRO),
 118         RTE_RX_OFFLOAD_BIT2STR(QINQ_STRIP),
 119         RTE_RX_OFFLOAD_BIT2STR(OUTER_IPV4_CKSUM),
 120         RTE_RX_OFFLOAD_BIT2STR(MACSEC_STRIP),
 121         RTE_RX_OFFLOAD_BIT2STR(HEADER_SPLIT),
 122         RTE_RX_OFFLOAD_BIT2STR(VLAN_FILTER),
 123         RTE_RX_OFFLOAD_BIT2STR(VLAN_EXTEND),
 124         RTE_RX_OFFLOAD_BIT2STR(JUMBO_FRAME),
 125         RTE_RX_OFFLOAD_BIT2STR(SCATTER),
 126         RTE_RX_OFFLOAD_BIT2STR(TIMESTAMP),
 127         RTE_RX_OFFLOAD_BIT2STR(SECURITY),
 128         RTE_RX_OFFLOAD_BIT2STR(KEEP_CRC),
 129         RTE_RX_OFFLOAD_BIT2STR(SCTP_CKSUM),
 130         RTE_RX_OFFLOAD_BIT2STR(OUTER_UDP_CKSUM),
 131 };
 132
 133 #undef RTE_RX_OFFLOAD_BIT2STR
 134
 135 #define RTE_TX_OFFLOAD_BIT2STR(_name)   \
 136         { DEV_TX_OFFLOAD_##_name, #_name }
 137
 138 static const struct {
 139         uint64_t offload;
 140         const char *name;
 141 } rte_tx_offload_names[] = {
 142         RTE_TX_OFFLOAD_BIT2STR(VLAN_INSERT),
 143         RTE_TX_OFFLOAD_BIT2STR(IPV4_CKSUM),
 144         RTE_TX_OFFLOAD_BIT2STR(UDP_CKSUM),
 145         RTE_TX_OFFLOAD_BIT2STR(TCP_CKSUM),
 146         RTE_TX_OFFLOAD_BIT2STR(SCTP_CKSUM),
 147         RTE_TX_OFFLOAD_BIT2STR(TCP_TSO),
 148         RTE_TX_OFFLOAD_BIT2STR(UDP_TSO),
 149         RTE_TX_OFFLOAD_BIT2STR(OUTER_IPV4_CKSUM),
 150         RTE_TX_OFFLOAD_BIT2STR(QINQ_INSERT),
 151         RTE_TX_OFFLOAD_BIT2STR(VXLAN_TNL_TSO),
 152         RTE_TX_OFFLOAD_BIT2STR(GRE_TNL_TSO),
 153         RTE_TX_OFFLOAD_BIT2STR(IPIP_TNL_TSO),
 154         RTE_TX_OFFLOAD_BIT2STR(GENEVE_TNL_TSO),
 155         RTE_TX_OFFLOAD_BIT2STR(MACSEC_INSERT),
 156         RTE_TX_OFFLOAD_BIT2STR(MT_LOCKFREE),
 157         RTE_TX_OFFLOAD_BIT2STR(MULTI_SEGS),
 158         RTE_TX_OFFLOAD_BIT2STR(MBUF_FAST_FREE),
 159         RTE_TX_OFFLOAD_BIT2STR(SECURITY),
 160         RTE_TX_OFFLOAD_BIT2STR(UDP_TNL_TSO),
 161         RTE_TX_OFFLOAD_BIT2STR(IP_TNL_TSO),
 162         RTE_TX_OFFLOAD_BIT2STR(OUTER_UDP_CKSUM),
 163 };
 164
 165 #undef RTE_TX_OFFLOAD_BIT2STR
 166
 167 /**
 168  * The user application callback description.
 169  *
 170  * It contains callback address to be registered by user application,
 171  * the pointer to the parameters for callback, and the event type.
 172  */
 173 struct rte_eth_dev_callback {
 174         TAILQ_ENTRY(rte_eth_dev_callback) next; /**< Callbacks list */
 175         rte_eth_dev_cb_fn cb_fn;                /**< Callback address */
 176         void *cb_arg;                           /**< Parameter for callback */
 177         void *ret_param;                        /**< Return parameter */
 178         enum rte_eth_event_type event;          /**< Interrupt event type */
 179         uint32_t active;                        /**< Callback is executing */
 180 };
 181
 182 enum {
 183         STAT_QMAP_TX = 0,
 184         STAT_QMAP_RX
 185 };
 186
 187 uint16_t
 188 rte_eth_find_next(uint16_t port_id)
 189 {
 190         while (port_id < RTE_MAX_ETHPORTS &&
 191                rte_eth_devices[port_id].state != RTE_ETH_DEV_ATTACHED &&
 192                rte_eth_devices[port_id].state != RTE_ETH_DEV_REMOVED)
 193                 port_id++;
 194
 195         if (port_id >= RTE_MAX_ETHPORTS)
 196                 return RTE_MAX_ETHPORTS;
 197
 198         return port_id;
 199 }
 200
 201 static void
 202 rte_eth_dev_shared_data_prepare(void)
 203 {
 204         const unsigned flags = 0;
 205         const struct rte_memzone *mz;
 206
 207         rte_spinlock_lock(&rte_eth_shared_data_lock);
 208
 209         if (rte_eth_dev_shared_data == NULL) {
 210                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 211                         /* Allocate port data and ownership shared memory. */
 212                         mz = rte_memzone_reserve(MZ_RTE_ETH_DEV_DATA,
 213                                         sizeof(*rte_eth_dev_shared_data),
 214                                         rte_socket_id(), flags);
 215                 } else
 216                         mz = rte_memzone_lookup(MZ_RTE_ETH_DEV_DATA);
 217                 if (mz == NULL)
 218                         rte_panic("Cannot allocate ethdev shared data\n");
 219
 220                 rte_eth_dev_shared_data = mz->addr;
 221                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 222                         rte_eth_dev_shared_data->next_owner_id =
 223                                         RTE_ETH_DEV_NO_OWNER + 1;
 224                         rte_spinlock_init(&rte_eth_dev_shared_data->ownership_lock);
 225                         memset(rte_eth_dev_shared_data->data, 0,
 226                                sizeof(rte_eth_dev_shared_data->data));
 227                 }
 228         }
 229
 230         rte_spinlock_unlock(&rte_eth_shared_data_lock);
 231 }
 232
 233 static bool
 234 is_allocated(const struct rte_eth_dev *ethdev)
 235 {
 236         return ethdev->data->name[0] != '\0';
 237 }
 238
 239 static struct rte_eth_dev *
 240 _rte_eth_dev_allocated(const char *name)
 241 {
 242         unsigned i;
 243
 244         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 245                 if (rte_eth_devices[i].data != NULL &&
 246                     strcmp(rte_eth_devices[i].data->name, name) == 0)
 247                         return &rte_eth_devices[i];
 248         }
 249         return NULL;
 250 }
 251
 252 struct rte_eth_dev *
 253 rte_eth_dev_allocated(const char *name)
 254 {
 255         struct rte_eth_dev *ethdev;
 256
 257         rte_eth_dev_shared_data_prepare();
 258
 259         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 260
 261         ethdev = _rte_eth_dev_allocated(name);
 262
 263         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 264
 265         return ethdev;
 266 }
 267
 268 static uint16_t
 269 rte_eth_dev_find_free_port(void)
 270 {
 271         unsigned i;
 272
 273         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 274                 /* Using shared name field to find a free port. */
 275                 if (rte_eth_dev_shared_data->data[i].name[0] == '\0') {
 276                         RTE_ASSERT(rte_eth_devices[i].state ==
 277                                    RTE_ETH_DEV_UNUSED);
 278                         return i;
 279                 }
 280         }
 281         return RTE_MAX_ETHPORTS;
 282 }
 283
 284 static struct rte_eth_dev *
 285 eth_dev_get(uint16_t port_id)
 286 {
 287         struct rte_eth_dev *eth_dev = &rte_eth_devices[port_id];
 288
 289         eth_dev->data = &rte_eth_dev_shared_data->data[port_id];
 290
 291         eth_dev_last_created_port = port_id;
 292
 293         return eth_dev;
 294 }
 295
 296 struct rte_eth_dev *
 297 rte_eth_dev_allocate(const char *name)
 298 {
 299         uint16_t port_id;
 300         struct rte_eth_dev *eth_dev = NULL;
 301
 302         rte_eth_dev_shared_data_prepare();
 303
 304         /* Synchronize port creation between primary and secondary threads. */
 305         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 306
 307         if (_rte_eth_dev_allocated(name) != NULL) {
 308                 RTE_ETHDEV_LOG(ERR,
 309                         "Ethernet device with name %s already allocated\n",
 310                         name);
 311                 goto unlock;
 312         }
 313
 314         port_id = rte_eth_dev_find_free_port();
 315         if (port_id == RTE_MAX_ETHPORTS) {
 316                 RTE_ETHDEV_LOG(ERR,
 317                         "Reached maximum number of Ethernet ports\n");
 318                 goto unlock;
 319         }
 320
 321         eth_dev = eth_dev_get(port_id);
 322         snprintf(eth_dev->data->name, sizeof(eth_dev->data->name), "%s", name);
 323         eth_dev->data->port_id = port_id;
 324         eth_dev->data->mtu = ETHER_MTU;
 325
 326 unlock:
 327         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 328
 329         return eth_dev;
 330 }
 331
 332 /*
 333  * Attach to a port already registered by the primary process, which
 334  * makes sure that the same device would have the same port id both
 335  * in the primary and secondary process.
 336  */
 337 struct rte_eth_dev *
 338 rte_eth_dev_attach_secondary(const char *name)
 339 {
 340         uint16_t i;
 341         struct rte_eth_dev *eth_dev = NULL;
 342
 343         rte_eth_dev_shared_data_prepare();
 344
 345         /* Synchronize port attachment to primary port creation and release. */
 346         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 347
 348         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 349                 if (strcmp(rte_eth_dev_shared_data->data[i].name, name) == 0)
 350                         break;
 351         }
 352         if (i == RTE_MAX_ETHPORTS) {
 353                 RTE_ETHDEV_LOG(ERR,
 354                         "Device %s is not driven by the primary process\n",
 355                         name);
 356         } else {
 357                 eth_dev = eth_dev_get(i);
 358                 RTE_ASSERT(eth_dev->data->port_id == i);
 359         }
 360
 361         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 362         return eth_dev;
 363 }
 364
 365 int
 366 rte_eth_dev_release_port(struct rte_eth_dev *eth_dev)
 367 {
 368         if (eth_dev == NULL)
 369                 return -EINVAL;
 370
 371         rte_eth_dev_shared_data_prepare();
 372
 373         _rte_eth_dev_callback_process(eth_dev, RTE_ETH_EVENT_DESTROY, NULL);
 374
 375         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 376
 377         eth_dev->state = RTE_ETH_DEV_UNUSED;
 378
 379         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 380                 rte_free(eth_dev->data->rx_queues);
 381                 rte_free(eth_dev->data->tx_queues);
 382                 rte_free(eth_dev->data->mac_addrs);
 383                 rte_free(eth_dev->data->hash_mac_addrs);
 384                 rte_free(eth_dev->data->dev_private);
 385                 memset(eth_dev->data, 0, sizeof(struct rte_eth_dev_data));
 386         }
 387
 388         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 389
 390         return 0;
 391 }
 392
 393 int
 394 rte_eth_dev_is_valid_port(uint16_t port_id)
 395 {
 396         if (port_id >= RTE_MAX_ETHPORTS ||
 397             (rte_eth_devices[port_id].state == RTE_ETH_DEV_UNUSED))
 398                 return 0;
 399         else
 400                 return 1;
 401 }
 402
 403 static int
 404 rte_eth_is_valid_owner_id(uint64_t owner_id)
 405 {
 406         if (owner_id == RTE_ETH_DEV_NO_OWNER ||
 407             rte_eth_dev_shared_data->next_owner_id <= owner_id)
 408                 return 0;
 409         return 1;
 410 }
 411
 412 uint64_t
 413 rte_eth_find_next_owned_by(uint16_t port_id, const uint64_t owner_id)
 414 {
 415         while (port_id < RTE_MAX_ETHPORTS &&
 416                ((rte_eth_devices[port_id].state != RTE_ETH_DEV_ATTACHED &&
 417                rte_eth_devices[port_id].state != RTE_ETH_DEV_REMOVED) ||
 418                rte_eth_devices[port_id].data->owner.id != owner_id))
 419                 port_id++;
 420
 421         if (port_id >= RTE_MAX_ETHPORTS)
 422                 return RTE_MAX_ETHPORTS;
 423
 424         return port_id;
 425 }
 426
 427 int __rte_experimental
 428 rte_eth_dev_owner_new(uint64_t *owner_id)
 429 {
 430         rte_eth_dev_shared_data_prepare();
 431
 432         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 433
 434         *owner_id = rte_eth_dev_shared_data->next_owner_id++;
 435
 436         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 437         return 0;
 438 }
 439
 440 static int
 441 _rte_eth_dev_owner_set(const uint16_t port_id, const uint64_t old_owner_id,
 442                        const struct rte_eth_dev_owner *new_owner)
 443 {
 444         struct rte_eth_dev *ethdev = &rte_eth_devices[port_id];
 445         struct rte_eth_dev_owner *port_owner;
 446         int sret;
 447
 448         if (port_id >= RTE_MAX_ETHPORTS || !is_allocated(ethdev)) {
 449                 RTE_ETHDEV_LOG(ERR, "Port id %"PRIu16" is not allocated\n",
 450                         port_id);
 451                 return -ENODEV;
 452         }
 453
 454         if (!rte_eth_is_valid_owner_id(new_owner->id) &&
 455             !rte_eth_is_valid_owner_id(old_owner_id)) {
 456                 RTE_ETHDEV_LOG(ERR,
 457                         "Invalid owner old_id=%016"PRIx64" new_id=%016"PRIx64"\n",
 458                        old_owner_id, new_owner->id);
 459                 return -EINVAL;
 460         }
 461
 462         port_owner = &rte_eth_devices[port_id].data->owner;
 463         if (port_owner->id != old_owner_id) {
 464                 RTE_ETHDEV_LOG(ERR,
 465                         "Cannot set owner to port %u already owned by %s_%016"PRIX64"\n",
 466                         port_id, port_owner->name, port_owner->id);
 467                 return -EPERM;
 468         }
 469
 470         sret = snprintf(port_owner->name, RTE_ETH_MAX_OWNER_NAME_LEN, "%s",
 471                         new_owner->name);
 472         if (sret < 0 || sret >= RTE_ETH_MAX_OWNER_NAME_LEN)
 473                 RTE_ETHDEV_LOG(ERR, "Port %u owner name was truncated\n",
 474                         port_id);
 475
 476         port_owner->id = new_owner->id;
 477
 478         RTE_ETHDEV_LOG(DEBUG, "Port %u owner is %s_%016"PRIx64"\n",
 479                 port_id, new_owner->name, new_owner->id);
 480
 481         return 0;
 482 }
 483
 484 int __rte_experimental
 485 rte_eth_dev_owner_set(const uint16_t port_id,
 486                       const struct rte_eth_dev_owner *owner)
 487 {
 488         int ret;
 489
 490         rte_eth_dev_shared_data_prepare();
 491
 492         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 493
 494         ret = _rte_eth_dev_owner_set(port_id, RTE_ETH_DEV_NO_OWNER, owner);
 495
 496         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 497         return ret;
 498 }
 499
 500 int __rte_experimental
 501 rte_eth_dev_owner_unset(const uint16_t port_id, const uint64_t owner_id)
 502 {
 503         const struct rte_eth_dev_owner new_owner = (struct rte_eth_dev_owner)
 504                         {.id = RTE_ETH_DEV_NO_OWNER, .name = ""};
 505         int ret;
 506
 507         rte_eth_dev_shared_data_prepare();
 508
 509         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 510
 511         ret = _rte_eth_dev_owner_set(port_id, owner_id, &new_owner);
 512
 513         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 514         return ret;
 515 }
 516
 517 void __rte_experimental
 518 rte_eth_dev_owner_delete(const uint64_t owner_id)
 519 {
 520         uint16_t port_id;
 521
 522         rte_eth_dev_shared_data_prepare();
 523
 524         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 525
 526         if (rte_eth_is_valid_owner_id(owner_id)) {
 527                 for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++)
 528                         if (rte_eth_devices[port_id].data->owner.id == owner_id)
 529                                 memset(&rte_eth_devices[port_id].data->owner, 0,
 530                                        sizeof(struct rte_eth_dev_owner));
 531                 RTE_ETHDEV_LOG(NOTICE,
 532                         "All port owners owned by %016"PRIx64" identifier have removed\n",
 533                         owner_id);
 534         } else {
 535                 RTE_ETHDEV_LOG(ERR,
 536                                "Invalid owner id=%016"PRIx64"\n",
 537                                owner_id);
 538         }
 539
 540         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 541 }
 542
 543 int __rte_experimental
 544 rte_eth_dev_owner_get(const uint16_t port_id, struct rte_eth_dev_owner *owner)
 545 {
 546         int ret = 0;
 547         struct rte_eth_dev *ethdev = &rte_eth_devices[port_id];
 548
 549         rte_eth_dev_shared_data_prepare();
 550
 551         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 552
 553         if (port_id >= RTE_MAX_ETHPORTS || !is_allocated(ethdev)) {
 554                 RTE_ETHDEV_LOG(ERR, "Port id %"PRIu16" is not allocated\n",
 555                         port_id);
 556                 ret = -ENODEV;
 557         } else {
 558                 rte_memcpy(owner, &ethdev->data->owner, sizeof(*owner));
 559         }
 560
 561         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 562         return ret;
 563 }
 564
 565 int
 566 rte_eth_dev_socket_id(uint16_t port_id)
 567 {
 568         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -1);
 569         return rte_eth_devices[port_id].data->numa_node;
 570 }
 571
 572 void *
 573 rte_eth_dev_get_sec_ctx(uint16_t port_id)
 574 {
 575         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, NULL);
 576         return rte_eth_devices[port_id].security_ctx;
 577 }
 578
 579 uint16_t
 580 rte_eth_dev_count(void)
 581 {
 582         return rte_eth_dev_count_avail();
 583 }
 584
 585 uint16_t
 586 rte_eth_dev_count_avail(void)
 587 {
 588         uint16_t p;
 589         uint16_t count;
 590
 591         count = 0;
 592
 593         RTE_ETH_FOREACH_DEV(p)
 594                 count++;
 595
 596         return count;
 597 }
 598
 599 uint16_t __rte_experimental
 600 rte_eth_dev_count_total(void)
 601 {
 602         uint16_t port, count = 0;
 603
 604         for (port = 0; port < RTE_MAX_ETHPORTS; port++)
 605                 if (rte_eth_devices[port].state != RTE_ETH_DEV_UNUSED)
 606                         count++;
 607
 608         return count;
 609 }
 610
 611 int
 612 rte_eth_dev_get_name_by_port(uint16_t port_id, char *name)
 613 {
 614         char *tmp;
 615
 616         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 617
 618         if (name == NULL) {
 619                 RTE_ETHDEV_LOG(ERR, "Null pointer is specified\n");
 620                 return -EINVAL;
 621         }
 622
 623         /* shouldn't check 'rte_eth_devices[i].data',
 624          * because it might be overwritten by VDEV PMD */
 625         tmp = rte_eth_dev_shared_data->data[port_id].name;
 626         strcpy(name, tmp);
 627         return 0;
 628 }
 629
 630 int
 631 rte_eth_dev_get_port_by_name(const char *name, uint16_t *port_id)
 632 {
 633         uint32_t pid;
 634
 635         if (name == NULL) {
 636                 RTE_ETHDEV_LOG(ERR, "Null pointer is specified\n");
 637                 return -EINVAL;
 638         }
 639
 640         for (pid = 0; pid < RTE_MAX_ETHPORTS; pid++) {
 641                 if (rte_eth_devices[pid].state != RTE_ETH_DEV_UNUSED &&
 642                     !strcmp(name, rte_eth_dev_shared_data->data[pid].name)) {
 643                         *port_id = pid;
 644                         return 0;
 645                 }
 646         }
 647
 648         return -ENODEV;
 649 }
 650
 651 static int
 652 eth_err(uint16_t port_id, int ret)
 653 {
 654         if (ret == 0)
 655                 return 0;
 656         if (rte_eth_dev_is_removed(port_id))
 657                 return -EIO;
 658         return ret;
 659 }
 660
 661 /* attach the new device, then store port_id of the device */
 662 int
 663 rte_eth_dev_attach(const char *devargs, uint16_t *port_id)
 664 {
 665         int current = rte_eth_dev_count_total();
 666         struct rte_devargs da;
 667         int ret = -1;
 668
 669         memset(&da, 0, sizeof(da));
 670
 671         if ((devargs == NULL) || (port_id == NULL)) {
 672                 ret = -EINVAL;
 673                 goto err;
 674         }
 675
 676         /* parse devargs */
 677         if (rte_devargs_parse(&da, devargs))
 678                 goto err;
 679
 680         ret = rte_eal_hotplug_add(da.bus->name, da.name, da.args);
 681         if (ret < 0)
 682                 goto err;
 683
 684         /* no point looking at the port count if no port exists */
 685         if (!rte_eth_dev_count_total()) {
 686                 RTE_ETHDEV_LOG(ERR, "No port found for device (%s)\n", da.name);
 687                 ret = -1;
 688                 goto err;
 689         }
 690
 691         /* if nothing happened, there is a bug here, since some driver told us
 692          * it did attach a device, but did not create a port.
 693          * FIXME: race condition in case of plug-out of another device
 694          */
 695         if (current == rte_eth_dev_count_total()) {
 696                 ret = -1;
 697                 goto err;
 698         }
 699
 700         *port_id = eth_dev_last_created_port;
 701         ret = 0;
 702
 703 err:
 704         free(da.args);
 705         return ret;
 706 }
 707
 708 /* detach the device, then store the name of the device */
 709 int
 710 rte_eth_dev_detach(uint16_t port_id, char *name __rte_unused)
 711 {
 712         struct rte_device *dev;
 713         struct rte_bus *bus;
 714         uint32_t dev_flags;
 715         int ret = -1;
 716
 717         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 718
 719         dev_flags = rte_eth_devices[port_id].data->dev_flags;
 720         if (dev_flags & RTE_ETH_DEV_BONDED_SLAVE) {
 721                 RTE_ETHDEV_LOG(ERR,
 722                         "Port %"PRIu16" is bonded, cannot detach\n", port_id);
 723                 return -ENOTSUP;
 724         }
 725
 726         dev = rte_eth_devices[port_id].device;
 727         if (dev == NULL)
 728                 return -EINVAL;
 729
 730         bus = rte_bus_find_by_device(dev);
 731         if (bus == NULL)
 732                 return -ENOENT;
 733
 734         ret = rte_eal_hotplug_remove(bus->name, dev->name);
 735         if (ret < 0)
 736                 return ret;
 737
 738         rte_eth_dev_release_port(&rte_eth_devices[port_id]);
 739         return 0;
 740 }
 741
 742 static int
 743 rte_eth_dev_rx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
 744 {
 745         uint16_t old_nb_queues = dev->data->nb_rx_queues;
 746         void **rxq;
 747         unsigned i;
 748
 749         if (dev->data->rx_queues == NULL && nb_queues != 0) { /* first time configuration */
 750                 dev->data->rx_queues = rte_zmalloc("ethdev->rx_queues",
 751                                 sizeof(dev->data->rx_queues[0]) * nb_queues,
 752                                 RTE_CACHE_LINE_SIZE);
 753                 if (dev->data->rx_queues == NULL) {
 754                         dev->data->nb_rx_queues = 0;
 755                         return -(ENOMEM);
 756                 }
 757         } else if (dev->data->rx_queues != NULL && nb_queues != 0) { /* re-configure */
 758                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release, -ENOTSUP);
 759
 760                 rxq = dev->data->rx_queues;
 761
 762                 for (i = nb_queues; i < old_nb_queues; i++)
 763                         (*dev->dev_ops->rx_queue_release)(rxq[i]);
 764                 rxq = rte_realloc(rxq, sizeof(rxq[0]) * nb_queues,
 765                                 RTE_CACHE_LINE_SIZE);
 766                 if (rxq == NULL)
 767                         return -(ENOMEM);
 768                 if (nb_queues > old_nb_queues) {
 769                         uint16_t new_qs = nb_queues - old_nb_queues;
 770
 771                         memset(rxq + old_nb_queues, 0,
 772                                 sizeof(rxq[0]) * new_qs);
 773                 }
 774
 775                 dev->data->rx_queues = rxq;
 776
 777         } else if (dev->data->rx_queues != NULL && nb_queues == 0) {
 778                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release, -ENOTSUP);
 779
 780                 rxq = dev->data->rx_queues;
 781
 782                 for (i = nb_queues; i < old_nb_queues; i++)
 783                         (*dev->dev_ops->rx_queue_release)(rxq[i]);
 784
 785                 rte_free(dev->data->rx_queues);
 786                 dev->data->rx_queues = NULL;
 787         }
 788         dev->data->nb_rx_queues = nb_queues;
 789         return 0;
 790 }
 791
 792 int
 793 rte_eth_dev_rx_queue_start(uint16_t port_id, uint16_t rx_queue_id)
 794 {
 795         struct rte_eth_dev *dev;
 796
 797         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 798
 799         dev = &rte_eth_devices[port_id];
 800         if (!dev->data->dev_started) {
 801                 RTE_ETHDEV_LOG(ERR,
 802                         "Port %u must be started before start any queue\n",
 803                         port_id);
 804                 return -EINVAL;
 805         }
 806
 807         if (rx_queue_id >= dev->data->nb_rx_queues) {
 808                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
 809                 return -EINVAL;
 810         }
 811
 812         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_start, -ENOTSUP);
 813
 814         if (dev->data->rx_queue_state[rx_queue_id] != RTE_ETH_QUEUE_STATE_STOPPED) {
 815                 RTE_ETHDEV_LOG(INFO,
 816                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already started\n",
 817                         rx_queue_id, port_id);
 818                 return 0;
 819         }
 820
 821         return eth_err(port_id, dev->dev_ops->rx_queue_start(dev,
 822                                                              rx_queue_id));
 823
 824 }
 825
 826 int
 827 rte_eth_dev_rx_queue_stop(uint16_t port_id, uint16_t rx_queue_id)
 828 {
 829         struct rte_eth_dev *dev;
 830
 831         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 832
 833         dev = &rte_eth_devices[port_id];
 834         if (rx_queue_id >= dev->data->nb_rx_queues) {
 835                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
 836                 return -EINVAL;
 837         }
 838
 839         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_stop, -ENOTSUP);
 840
 841         if (dev->data->rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STOPPED) {
 842                 RTE_ETHDEV_LOG(INFO,
 843                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already stopped\n",
 844                         rx_queue_id, port_id);
 845                 return 0;
 846         }
 847
 848         return eth_err(port_id, dev->dev_ops->rx_queue_stop(dev, rx_queue_id));
 849
 850 }
 851
 852 int
 853 rte_eth_dev_tx_queue_start(uint16_t port_id, uint16_t tx_queue_id)
 854 {
 855         struct rte_eth_dev *dev;
 856
 857         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 858
 859         dev = &rte_eth_devices[port_id];
 860         if (!dev->data->dev_started) {
 861                 RTE_ETHDEV_LOG(ERR,
 862                         "Port %u must be started before start any queue\n",
 863                         port_id);
 864                 return -EINVAL;
 865         }
 866
 867         if (tx_queue_id >= dev->data->nb_tx_queues) {
 868                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
 869                 return -EINVAL;
 870         }
 871
 872         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_start, -ENOTSUP);
 873
 874         if (dev->data->tx_queue_state[tx_queue_id] != RTE_ETH_QUEUE_STATE_STOPPED) {
 875                 RTE_ETHDEV_LOG(INFO,
 876                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already started\n",
 877                         tx_queue_id, port_id);
 878                 return 0;
 879         }
 880
 881         return eth_err(port_id, dev->dev_ops->tx_queue_start(dev, tx_queue_id));
 882 }
 883
 884 int
 885 rte_eth_dev_tx_queue_stop(uint16_t port_id, uint16_t tx_queue_id)
 886 {
 887         struct rte_eth_dev *dev;
 888
 889         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 890
 891         dev = &rte_eth_devices[port_id];
 892         if (tx_queue_id >= dev->data->nb_tx_queues) {
 893                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
 894                 return -EINVAL;
 895         }
 896
 897         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_stop, -ENOTSUP);
 898
 899         if (dev->data->tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STOPPED) {
 900                 RTE_ETHDEV_LOG(INFO,
 901                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already stopped\n",
 902                         tx_queue_id, port_id);
 903                 return 0;
 904         }
 905
 906         return eth_err(port_id, dev->dev_ops->tx_queue_stop(dev, tx_queue_id));
 907
 908 }
 909
 910 static int
 911 rte_eth_dev_tx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
 912 {
 913         uint16_t old_nb_queues = dev->data->nb_tx_queues;
 914         void **txq;
 915         unsigned i;
 916
 917         if (dev->data->tx_queues == NULL && nb_queues != 0) { /* first time configuration */
 918                 dev->data->tx_queues = rte_zmalloc("ethdev->tx_queues",
 919                                                    sizeof(dev->data->tx_queues[0]) * nb_queues,
 920                                                    RTE_CACHE_LINE_SIZE);
 921                 if (dev->data->tx_queues == NULL) {
 922                         dev->data->nb_tx_queues = 0;
 923                         return -(ENOMEM);
 924                 }
 925         } else if (dev->data->tx_queues != NULL && nb_queues != 0) { /* re-configure */
 926                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release, -ENOTSUP);
 927
 928                 txq = dev->data->tx_queues;
 929
 930                 for (i = nb_queues; i < old_nb_queues; i++)
 931                         (*dev->dev_ops->tx_queue_release)(txq[i]);
 932                 txq = rte_realloc(txq, sizeof(txq[0]) * nb_queues,
 933                                   RTE_CACHE_LINE_SIZE);
 934                 if (txq == NULL)
 935                         return -ENOMEM;
 936                 if (nb_queues > old_nb_queues) {
 937                         uint16_t new_qs = nb_queues - old_nb_queues;
 938
 939                         memset(txq + old_nb_queues, 0,
 940                                sizeof(txq[0]) * new_qs);
 941                 }
 942
 943                 dev->data->tx_queues = txq;
 944
 945         } else if (dev->data->tx_queues != NULL && nb_queues == 0) {
 946                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release, -ENOTSUP);
 947
 948                 txq = dev->data->tx_queues;
 949
 950                 for (i = nb_queues; i < old_nb_queues; i++)
 951                         (*dev->dev_ops->tx_queue_release)(txq[i]);
 952
 953                 rte_free(dev->data->tx_queues);
 954                 dev->data->tx_queues = NULL;
 955         }
 956         dev->data->nb_tx_queues = nb_queues;
 957         return 0;
 958 }
 959
 960 uint32_t
 961 rte_eth_speed_bitflag(uint32_t speed, int duplex)
 962 {
 963         switch (speed) {
 964         case ETH_SPEED_NUM_10M:
 965                 return duplex ? ETH_LINK_SPEED_10M : ETH_LINK_SPEED_10M_HD;
 966         case ETH_SPEED_NUM_100M:
 967                 return duplex ? ETH_LINK_SPEED_100M : ETH_LINK_SPEED_100M_HD;
 968         case ETH_SPEED_NUM_1G:
 969                 return ETH_LINK_SPEED_1G;
 970         case ETH_SPEED_NUM_2_5G:
 971                 return ETH_LINK_SPEED_2_5G;
 972         case ETH_SPEED_NUM_5G:
 973                 return ETH_LINK_SPEED_5G;
 974         case ETH_SPEED_NUM_10G:
 975                 return ETH_LINK_SPEED_10G;
 976         case ETH_SPEED_NUM_20G:
 977                 return ETH_LINK_SPEED_20G;
 978         case ETH_SPEED_NUM_25G:
 979                 return ETH_LINK_SPEED_25G;
 980         case ETH_SPEED_NUM_40G:
 981                 return ETH_LINK_SPEED_40G;
 982         case ETH_SPEED_NUM_50G:
 983                 return ETH_LINK_SPEED_50G;
 984         case ETH_SPEED_NUM_56G:
 985                 return ETH_LINK_SPEED_56G;
 986         case ETH_SPEED_NUM_100G:
 987                 return ETH_LINK_SPEED_100G;
 988         default:
 989                 return 0;
 990         }
 991 }
 992
 993 const char * __rte_experimental
 994 rte_eth_dev_rx_offload_name(uint64_t offload)
 995 {
 996         const char *name = "UNKNOWN";
 997         unsigned int i;
 998
 999         for (i = 0; i < RTE_DIM(rte_rx_offload_names); ++i) {
1000                 if (offload == rte_rx_offload_names[i].offload) {
1001                         name = rte_rx_offload_names[i].name;
1002                         break;
1003                 }
1004         }
1005
1006         return name;
1007 }
1008
1009 const char * __rte_experimental
1010 rte_eth_dev_tx_offload_name(uint64_t offload)
1011 {
1012         const char *name = "UNKNOWN";
1013         unsigned int i;
1014
1015         for (i = 0; i < RTE_DIM(rte_tx_offload_names); ++i) {
1016                 if (offload == rte_tx_offload_names[i].offload) {
1017                         name = rte_tx_offload_names[i].name;
1018                         break;
1019                 }
1020         }
1021
1022         return name;
1023 }
1024
1025 int
1026 rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
1027                       const struct rte_eth_conf *dev_conf)
1028 {
1029         struct rte_eth_dev *dev;
1030         struct rte_eth_dev_info dev_info;
1031         struct rte_eth_conf local_conf = *dev_conf;
1032         int diag;
1033
1034         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1035
1036         dev = &rte_eth_devices[port_id];
1037
1038         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
1039         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
1040
1041         rte_eth_dev_info_get(port_id, &dev_info);
1042
1043         /* If number of queues specified by application for both Rx and Tx is
1044          * zero, use driver preferred values. This cannot be done individually
1045          * as it is valid for either Tx or Rx (but not both) to be zero.
1046          * If driver does not provide any preferred valued, fall back on
1047          * EAL defaults.
1048          */
1049         if (nb_rx_q == 0 && nb_tx_q == 0) {
1050                 nb_rx_q = dev_info.default_rxportconf.nb_queues;
1051                 if (nb_rx_q == 0)
1052                         nb_rx_q = RTE_ETH_DEV_FALLBACK_RX_NBQUEUES;
1053                 nb_tx_q = dev_info.default_txportconf.nb_queues;
1054                 if (nb_tx_q == 0)
1055                         nb_tx_q = RTE_ETH_DEV_FALLBACK_TX_NBQUEUES;
1056         }
1057
1058         if (nb_rx_q > RTE_MAX_QUEUES_PER_PORT) {
1059                 RTE_ETHDEV_LOG(ERR,
1060                         "Number of RX queues requested (%u) is greater than max supported(%d)\n",
1061                         nb_rx_q, RTE_MAX_QUEUES_PER_PORT);
1062                 return -EINVAL;
1063         }
1064
1065         if (nb_tx_q > RTE_MAX_QUEUES_PER_PORT) {
1066                 RTE_ETHDEV_LOG(ERR,
1067                         "Number of TX queues requested (%u) is greater than max supported(%d)\n",
1068                         nb_tx_q, RTE_MAX_QUEUES_PER_PORT);
1069                 return -EINVAL;
1070         }
1071
1072         if (dev->data->dev_started) {
1073                 RTE_ETHDEV_LOG(ERR,
1074                         "Port %u must be stopped to allow configuration\n",
1075                         port_id);
1076                 return -EBUSY;
1077         }
1078
1079         /* Copy the dev_conf parameter into the dev structure */
1080         memcpy(&dev->data->dev_conf, &local_conf, sizeof(dev->data->dev_conf));
1081
1082         /*
1083          * Check that the numbers of RX and TX queues are not greater
1084          * than the maximum number of RX and TX queues supported by the
1085          * configured device.
1086          */
1087         if (nb_rx_q > dev_info.max_rx_queues) {
1088                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u nb_rx_queues=%u > %u\n",
1089                         port_id, nb_rx_q, dev_info.max_rx_queues);
1090                 return -EINVAL;
1091         }
1092
1093         if (nb_tx_q > dev_info.max_tx_queues) {
1094                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u nb_tx_queues=%u > %u\n",
1095                         port_id, nb_tx_q, dev_info.max_tx_queues);
1096                 return -EINVAL;
1097         }
1098
1099         /* Check that the device supports requested interrupts */
1100         if ((dev_conf->intr_conf.lsc == 1) &&
1101                         (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC))) {
1102                 RTE_ETHDEV_LOG(ERR, "Driver %s does not support lsc\n",
1103                         dev->device->driver->name);
1104                 return -EINVAL;
1105         }
1106         if ((dev_conf->intr_conf.rmv == 1) &&
1107                         (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_RMV))) {
1108                 RTE_ETHDEV_LOG(ERR, "Driver %s does not support rmv\n",
1109                         dev->device->driver->name);
1110                 return -EINVAL;
1111         }
1112
1113         /*
1114          * If jumbo frames are enabled, check that the maximum RX packet
1115          * length is supported by the configured device.
1116          */
1117         if (local_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
1118                 if (dev_conf->rxmode.max_rx_pkt_len > dev_info.max_rx_pktlen) {
1119                         RTE_ETHDEV_LOG(ERR,
1120                                 "Ethdev port_id=%u max_rx_pkt_len %u > max valid value %u\n",
1121                                 port_id, dev_conf->rxmode.max_rx_pkt_len,
1122                                 dev_info.max_rx_pktlen);
1123                         return -EINVAL;
1124                 } else if (dev_conf->rxmode.max_rx_pkt_len < ETHER_MIN_LEN) {
1125                         RTE_ETHDEV_LOG(ERR,
1126                                 "Ethdev port_id=%u max_rx_pkt_len %u < min valid value %u\n",
1127                                 port_id, dev_conf->rxmode.max_rx_pkt_len,
1128                                 (unsigned)ETHER_MIN_LEN);
1129                         return -EINVAL;
1130                 }
1131         } else {
1132                 if (dev_conf->rxmode.max_rx_pkt_len < ETHER_MIN_LEN ||
1133                         dev_conf->rxmode.max_rx_pkt_len > ETHER_MAX_LEN)
1134                         /* Use default value */
1135                         dev->data->dev_conf.rxmode.max_rx_pkt_len =
1136                                                         ETHER_MAX_LEN;
1137         }
1138
1139         /* Any requested offloading must be within its device capabilities */
1140         if ((local_conf.rxmode.offloads & dev_info.rx_offload_capa) !=
1141              local_conf.rxmode.offloads) {
1142                 RTE_ETHDEV_LOG(ERR,
1143                         "Ethdev port_id=%u requested Rx offloads 0x%"PRIx64" doesn't match Rx offloads "
1144                         "capabilities 0x%"PRIx64" in %s()\n",
1145                         port_id, local_conf.rxmode.offloads,
1146                         dev_info.rx_offload_capa,
1147                         __func__);
1148                 return -EINVAL;
1149         }
1150         if ((local_conf.txmode.offloads & dev_info.tx_offload_capa) !=
1151              local_conf.txmode.offloads) {
1152                 RTE_ETHDEV_LOG(ERR,
1153                         "Ethdev port_id=%u requested Tx offloads 0x%"PRIx64" doesn't match Tx offloads "
1154                         "capabilities 0x%"PRIx64" in %s()\n",
1155                         port_id, local_conf.txmode.offloads,
1156                         dev_info.tx_offload_capa,
1157                         __func__);
1158                 return -EINVAL;
1159         }
1160
1161         /* Check that device supports requested rss hash functions. */
1162         if ((dev_info.flow_type_rss_offloads |
1163              dev_conf->rx_adv_conf.rss_conf.rss_hf) !=
1164             dev_info.flow_type_rss_offloads) {
1165                 RTE_ETHDEV_LOG(ERR,
1166                         "Ethdev port_id=%u invalid rss_hf: 0x%"PRIx64", valid value: 0x%"PRIx64"\n",
1167                         port_id, dev_conf->rx_adv_conf.rss_conf.rss_hf,
1168                         dev_info.flow_type_rss_offloads);
1169                 return -EINVAL;
1170         }
1171
1172         /*
1173          * Setup new number of RX/TX queues and reconfigure device.
1174          */
1175         diag = rte_eth_dev_rx_queue_config(dev, nb_rx_q);
1176         if (diag != 0) {
1177                 RTE_ETHDEV_LOG(ERR,
1178                         "Port%u rte_eth_dev_rx_queue_config = %d\n",
1179                         port_id, diag);
1180                 return diag;
1181         }
1182
1183         diag = rte_eth_dev_tx_queue_config(dev, nb_tx_q);
1184         if (diag != 0) {
1185                 RTE_ETHDEV_LOG(ERR,
1186                         "Port%u rte_eth_dev_tx_queue_config = %d\n",
1187                         port_id, diag);
1188                 rte_eth_dev_rx_queue_config(dev, 0);
1189                 return diag;
1190         }
1191
1192         diag = (*dev->dev_ops->dev_configure)(dev);
1193         if (diag != 0) {
1194                 RTE_ETHDEV_LOG(ERR, "Port%u dev_configure = %d\n",
1195                         port_id, diag);
1196                 rte_eth_dev_rx_queue_config(dev, 0);
1197                 rte_eth_dev_tx_queue_config(dev, 0);
1198                 return eth_err(port_id, diag);
1199         }
1200
1201         /* Initialize Rx profiling if enabled at compilation time. */
1202         diag = __rte_eth_dev_profile_init(port_id, dev);
1203         if (diag != 0) {
1204                 RTE_ETHDEV_LOG(ERR, "Port%u __rte_eth_dev_profile_init = %d\n",
1205                         port_id, diag);
1206                 rte_eth_dev_rx_queue_config(dev, 0);
1207                 rte_eth_dev_tx_queue_config(dev, 0);
1208                 return eth_err(port_id, diag);
1209         }
1210
1211         return 0;
1212 }
1213
1214 void
1215 _rte_eth_dev_reset(struct rte_eth_dev *dev)
1216 {
1217         if (dev->data->dev_started) {
1218                 RTE_ETHDEV_LOG(ERR, "Port %u must be stopped to allow reset\n",
1219                         dev->data->port_id);
1220                 return;
1221         }
1222
1223         rte_eth_dev_rx_queue_config(dev, 0);
1224         rte_eth_dev_tx_queue_config(dev, 0);
1225
1226         memset(&dev->data->dev_conf, 0, sizeof(dev->data->dev_conf));
1227 }
1228
1229 static void
1230 rte_eth_dev_mac_restore(struct rte_eth_dev *dev,
1231                         struct rte_eth_dev_info *dev_info)
1232 {
1233         struct ether_addr *addr;
1234         uint16_t i;
1235         uint32_t pool = 0;
1236         uint64_t pool_mask;
1237
1238         /* replay MAC address configuration including default MAC */
1239         addr = &dev->data->mac_addrs[0];
1240         if (*dev->dev_ops->mac_addr_set != NULL)
1241                 (*dev->dev_ops->mac_addr_set)(dev, addr);
1242         else if (*dev->dev_ops->mac_addr_add != NULL)
1243                 (*dev->dev_ops->mac_addr_add)(dev, addr, 0, pool);
1244
1245         if (*dev->dev_ops->mac_addr_add != NULL) {
1246                 for (i = 1; i < dev_info->max_mac_addrs; i++) {
1247                         addr = &dev->data->mac_addrs[i];
1248
1249                         /* skip zero address */
1250                         if (is_zero_ether_addr(addr))
1251                                 continue;
1252
1253                         pool = 0;
1254                         pool_mask = dev->data->mac_pool_sel[i];
1255
1256                         do {
1257                                 if (pool_mask & 1ULL)
1258                                         (*dev->dev_ops->mac_addr_add)(dev,
1259                                                 addr, i, pool);
1260                                 pool_mask >>= 1;
1261                                 pool++;
1262                         } while (pool_mask);
1263                 }
1264         }
1265 }
1266
1267 static void
1268 rte_eth_dev_config_restore(struct rte_eth_dev *dev,
1269                            struct rte_eth_dev_info *dev_info, uint16_t port_id)
1270 {
1271         if (!(*dev_info->dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR))
1272                 rte_eth_dev_mac_restore(dev, dev_info);
1273
1274         /* replay promiscuous configuration */
1275         if (rte_eth_promiscuous_get(port_id) == 1)
1276                 rte_eth_promiscuous_enable(port_id);
1277         else if (rte_eth_promiscuous_get(port_id) == 0)
1278                 rte_eth_promiscuous_disable(port_id);
1279
1280         /* replay all multicast configuration */
1281         if (rte_eth_allmulticast_get(port_id) == 1)
1282                 rte_eth_allmulticast_enable(port_id);
1283         else if (rte_eth_allmulticast_get(port_id) == 0)
1284                 rte_eth_allmulticast_disable(port_id);
1285 }
1286
1287 int
1288 rte_eth_dev_start(uint16_t port_id)
1289 {
1290         struct rte_eth_dev *dev;
1291         struct rte_eth_dev_info dev_info;
1292         int diag;
1293
1294         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1295
1296         dev = &rte_eth_devices[port_id];
1297
1298         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_start, -ENOTSUP);
1299
1300         if (dev->data->dev_started != 0) {
1301                 RTE_ETHDEV_LOG(INFO,
1302                         "Device with port_id=%"PRIu16" already started\n",
1303                         port_id);
1304                 return 0;
1305         }
1306
1307         rte_eth_dev_info_get(port_id, &dev_info);
1308
1309         /* Lets restore MAC now if device does not support live change */
1310         if (*dev_info.dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR)
1311                 rte_eth_dev_mac_restore(dev, &dev_info);
1312
1313         diag = (*dev->dev_ops->dev_start)(dev);
1314         if (diag == 0)
1315                 dev->data->dev_started = 1;
1316         else
1317                 return eth_err(port_id, diag);
1318
1319         rte_eth_dev_config_restore(dev, &dev_info, port_id);
1320
1321         if (dev->data->dev_conf.intr_conf.lsc == 0) {
1322                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
1323                 (*dev->dev_ops->link_update)(dev, 0);
1324         }
1325         return 0;
1326 }
1327
1328 void
1329 rte_eth_dev_stop(uint16_t port_id)
1330 {
1331         struct rte_eth_dev *dev;
1332
1333         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1334         dev = &rte_eth_devices[port_id];
1335
1336         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_stop);
1337
1338         if (dev->data->dev_started == 0) {
1339                 RTE_ETHDEV_LOG(INFO,
1340                         "Device with port_id=%"PRIu16" already stopped\n",
1341                         port_id);
1342                 return;
1343         }
1344
1345         dev->data->dev_started = 0;
1346         (*dev->dev_ops->dev_stop)(dev);
1347 }
1348
1349 int
1350 rte_eth_dev_set_link_up(uint16_t port_id)
1351 {
1352         struct rte_eth_dev *dev;
1353
1354         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1355
1356         dev = &rte_eth_devices[port_id];
1357
1358         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_up, -ENOTSUP);
1359         return eth_err(port_id, (*dev->dev_ops->dev_set_link_up)(dev));
1360 }
1361
1362 int
1363 rte_eth_dev_set_link_down(uint16_t port_id)
1364 {
1365         struct rte_eth_dev *dev;
1366
1367         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1368
1369         dev = &rte_eth_devices[port_id];
1370
1371         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_down, -ENOTSUP);
1372         return eth_err(port_id, (*dev->dev_ops->dev_set_link_down)(dev));
1373 }
1374
1375 void
1376 rte_eth_dev_close(uint16_t port_id)
1377 {
1378         struct rte_eth_dev *dev;
1379
1380         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1381         dev = &rte_eth_devices[port_id];
1382
1383         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_close);
1384         dev->data->dev_started = 0;
1385         (*dev->dev_ops->dev_close)(dev);
1386
1387         /* check behaviour flag - temporary for PMD migration */
1388         if ((dev->data->dev_flags & RTE_ETH_DEV_CLOSE_REMOVE) != 0) {
1389                 /* new behaviour: send event + reset state + free all data */
1390                 rte_eth_dev_release_port(dev);
1391                 return;
1392         }
1393         RTE_ETHDEV_LOG(DEBUG, "Port closing is using an old behaviour.\n"
1394                         "The driver %s should migrate to the new behaviour.\n",
1395                         dev->device->driver->name);
1396         /* old behaviour: only free queue arrays */
1397         dev->data->nb_rx_queues = 0;
1398         rte_free(dev->data->rx_queues);
1399         dev->data->rx_queues = NULL;
1400         dev->data->nb_tx_queues = 0;
1401         rte_free(dev->data->tx_queues);
1402         dev->data->tx_queues = NULL;
1403 }
1404
1405 int
1406 rte_eth_dev_reset(uint16_t port_id)
1407 {
1408         struct rte_eth_dev *dev;
1409         int ret;
1410
1411         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1412         dev = &rte_eth_devices[port_id];
1413
1414         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_reset, -ENOTSUP);
1415
1416         rte_eth_dev_stop(port_id);
1417         ret = dev->dev_ops->dev_reset(dev);
1418
1419         return eth_err(port_id, ret);
1420 }
1421
1422 int __rte_experimental
1423 rte_eth_dev_is_removed(uint16_t port_id)
1424 {
1425         struct rte_eth_dev *dev;
1426         int ret;
1427
1428         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, 0);
1429
1430         dev = &rte_eth_devices[port_id];
1431
1432         if (dev->state == RTE_ETH_DEV_REMOVED)
1433                 return 1;
1434
1435         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->is_removed, 0);
1436
1437         ret = dev->dev_ops->is_removed(dev);
1438         if (ret != 0)
1439                 /* Device is physically removed. */
1440                 dev->state = RTE_ETH_DEV_REMOVED;
1441
1442         return ret;
1443 }
1444
1445 int
1446 rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
1447                        uint16_t nb_rx_desc, unsigned int socket_id,
1448                        const struct rte_eth_rxconf *rx_conf,
1449                        struct rte_mempool *mp)
1450 {
1451         int ret;
1452         uint32_t mbp_buf_size;
1453         struct rte_eth_dev *dev;
1454         struct rte_eth_dev_info dev_info;
1455         struct rte_eth_rxconf local_conf;
1456         void **rxq;
1457
1458         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1459
1460         dev = &rte_eth_devices[port_id];
1461         if (rx_queue_id >= dev->data->nb_rx_queues) {
1462                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
1463                 return -EINVAL;
1464         }
1465
1466         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
1467         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_setup, -ENOTSUP);
1468
1469         /*
1470          * Check the size of the mbuf data buffer.
1471          * This value must be provided in the private data of the memory pool.
1472          * First check that the memory pool has a valid private data.
1473          */
1474         rte_eth_dev_info_get(port_id, &dev_info);
1475         if (mp->private_data_size < sizeof(struct rte_pktmbuf_pool_private)) {
1476                 RTE_ETHDEV_LOG(ERR, "%s private_data_size %d < %d\n",
1477                         mp->name, (int)mp->private_data_size,
1478                         (int)sizeof(struct rte_pktmbuf_pool_private));
1479                 return -ENOSPC;
1480         }
1481         mbp_buf_size = rte_pktmbuf_data_room_size(mp);
1482
1483         if ((mbp_buf_size - RTE_PKTMBUF_HEADROOM) < dev_info.min_rx_bufsize) {
1484                 RTE_ETHDEV_LOG(ERR,
1485                         "%s mbuf_data_room_size %d < %d (RTE_PKTMBUF_HEADROOM=%d + min_rx_bufsize(dev)=%d)\n",
1486                         mp->name, (int)mbp_buf_size,
1487                         (int)(RTE_PKTMBUF_HEADROOM + dev_info.min_rx_bufsize),
1488                         (int)RTE_PKTMBUF_HEADROOM,
1489                         (int)dev_info.min_rx_bufsize);
1490                 return -EINVAL;
1491         }
1492
1493         /* Use default specified by driver, if nb_rx_desc is zero */
1494         if (nb_rx_desc == 0) {
1495                 nb_rx_desc = dev_info.default_rxportconf.ring_size;
1496                 /* If driver default is also zero, fall back on EAL default */
1497                 if (nb_rx_desc == 0)
1498                         nb_rx_desc = RTE_ETH_DEV_FALLBACK_RX_RINGSIZE;
1499         }
1500
1501         if (nb_rx_desc > dev_info.rx_desc_lim.nb_max ||
1502                         nb_rx_desc < dev_info.rx_desc_lim.nb_min ||
1503                         nb_rx_desc % dev_info.rx_desc_lim.nb_align != 0) {
1504
1505                 RTE_ETHDEV_LOG(ERR,
1506                         "Invalid value for nb_rx_desc(=%hu), should be: <= %hu, = %hu, and a product of %hu\n",
1507                         nb_rx_desc, dev_info.rx_desc_lim.nb_max,
1508                         dev_info.rx_desc_lim.nb_min,
1509                         dev_info.rx_desc_lim.nb_align);
1510                 return -EINVAL;
1511         }
1512
1513         if (dev->data->dev_started &&
1514                 !(dev_info.dev_capa &
1515                         RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP))
1516                 return -EBUSY;
1517
1518         if (dev->data->dev_started &&
1519                 (dev->data->rx_queue_state[rx_queue_id] !=
1520                         RTE_ETH_QUEUE_STATE_STOPPED))
1521                 return -EBUSY;
1522
1523         rxq = dev->data->rx_queues;
1524         if (rxq[rx_queue_id]) {
1525                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release,
1526                                         -ENOTSUP);
1527                 (*dev->dev_ops->rx_queue_release)(rxq[rx_queue_id]);
1528                 rxq[rx_queue_id] = NULL;
1529         }
1530
1531         if (rx_conf == NULL)
1532                 rx_conf = &dev_info.default_rxconf;
1533
1534         local_conf = *rx_conf;
1535
1536         /*
1537          * If an offloading has already been enabled in
1538          * rte_eth_dev_configure(), it has been enabled on all queues,
1539          * so there is no need to enable it in this queue again.
1540          * The local_conf.offloads input to underlying PMD only carries
1541          * those offloadings which are only enabled on this queue and
1542          * not enabled on all queues.
1543          */
1544         local_conf.offloads &= ~dev->data->dev_conf.rxmode.offloads;
1545
1546         /*
1547          * New added offloadings for this queue are those not enabled in
1548          * rte_eth_dev_configure() and they must be per-queue type.
1549          * A pure per-port offloading can't be enabled on a queue while
1550          * disabled on another queue. A pure per-port offloading can't
1551          * be enabled for any queue as new added one if it hasn't been
1552          * enabled in rte_eth_dev_configure().
1553          */
1554         if ((local_conf.offloads & dev_info.rx_queue_offload_capa) !=
1555              local_conf.offloads) {
1556                 RTE_ETHDEV_LOG(ERR,
1557                         "Ethdev port_id=%d rx_queue_id=%d, new added offloads 0x%"PRIx64" must be "
1558                         "within pre-queue offload capabilities 0x%"PRIx64" in %s()\n",
1559                         port_id, rx_queue_id, local_conf.offloads,
1560                         dev_info.rx_queue_offload_capa,
1561                         __func__);
1562                 return -EINVAL;
1563         }
1564
1565         ret = (*dev->dev_ops->rx_queue_setup)(dev, rx_queue_id, nb_rx_desc,
1566                                               socket_id, &local_conf, mp);
1567         if (!ret) {
1568                 if (!dev->data->min_rx_buf_size ||
1569                     dev->data->min_rx_buf_size > mbp_buf_size)
1570                         dev->data->min_rx_buf_size = mbp_buf_size;
1571         }
1572
1573         return eth_err(port_id, ret);
1574 }
1575
1576 int
1577 rte_eth_tx_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
1578                        uint16_t nb_tx_desc, unsigned int socket_id,
1579                        const struct rte_eth_txconf *tx_conf)
1580 {
1581         struct rte_eth_dev *dev;
1582         struct rte_eth_dev_info dev_info;
1583         struct rte_eth_txconf local_conf;
1584         void **txq;
1585
1586         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1587
1588         dev = &rte_eth_devices[port_id];
1589         if (tx_queue_id >= dev->data->nb_tx_queues) {
1590                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
1591                 return -EINVAL;
1592         }
1593
1594         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
1595         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_setup, -ENOTSUP);
1596
1597         rte_eth_dev_info_get(port_id, &dev_info);
1598
1599         /* Use default specified by driver, if nb_tx_desc is zero */
1600         if (nb_tx_desc == 0) {
1601                 nb_tx_desc = dev_info.default_txportconf.ring_size;
1602                 /* If driver default is zero, fall back on EAL default */
1603                 if (nb_tx_desc == 0)
1604                         nb_tx_desc = RTE_ETH_DEV_FALLBACK_TX_RINGSIZE;
1605         }
1606         if (nb_tx_desc > dev_info.tx_desc_lim.nb_max ||
1607             nb_tx_desc < dev_info.tx_desc_lim.nb_min ||
1608             nb_tx_desc % dev_info.tx_desc_lim.nb_align != 0) {
1609                 RTE_ETHDEV_LOG(ERR,
1610                         "Invalid value for nb_tx_desc(=%hu), should be: <= %hu, = %hu, and a product of %hu\n",
1611                         nb_tx_desc, dev_info.tx_desc_lim.nb_max,
1612                         dev_info.tx_desc_lim.nb_min,
1613                         dev_info.tx_desc_lim.nb_align);
1614                 return -EINVAL;
1615         }
1616
1617         if (dev->data->dev_started &&
1618                 !(dev_info.dev_capa &
1619                         RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP))
1620                 return -EBUSY;
1621
1622         if (dev->data->dev_started &&
1623                 (dev->data->tx_queue_state[tx_queue_id] !=
1624                         RTE_ETH_QUEUE_STATE_STOPPED))
1625                 return -EBUSY;
1626
1627         txq = dev->data->tx_queues;
1628         if (txq[tx_queue_id]) {
1629                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release,
1630                                         -ENOTSUP);
1631                 (*dev->dev_ops->tx_queue_release)(txq[tx_queue_id]);
1632                 txq[tx_queue_id] = NULL;
1633         }
1634
1635         if (tx_conf == NULL)
1636                 tx_conf = &dev_info.default_txconf;
1637
1638         local_conf = *tx_conf;
1639
1640         /*
1641          * If an offloading has already been enabled in
1642          * rte_eth_dev_configure(), it has been enabled on all queues,
1643          * so there is no need to enable it in this queue again.
1644          * The local_conf.offloads input to underlying PMD only carries
1645          * those offloadings which are only enabled on this queue and
1646          * not enabled on all queues.
1647          */
1648         local_conf.offloads &= ~dev->data->dev_conf.txmode.offloads;
1649
1650         /*
1651          * New added offloadings for this queue are those not enabled in
1652          * rte_eth_dev_configure() and they must be per-queue type.
1653          * A pure per-port offloading can't be enabled on a queue while
1654          * disabled on another queue. A pure per-port offloading can't
1655          * be enabled for any queue as new added one if it hasn't been
1656          * enabled in rte_eth_dev_configure().
1657          */
1658         if ((local_conf.offloads & dev_info.tx_queue_offload_capa) !=
1659              local_conf.offloads) {
1660                 RTE_ETHDEV_LOG(ERR,
1661                         "Ethdev port_id=%d tx_queue_id=%d, new added offloads 0x%"PRIx64" must be "
1662                         "within pre-queue offload capabilities 0x%"PRIx64" in %s()\n",
1663                         port_id, tx_queue_id, local_conf.offloads,
1664                         dev_info.tx_queue_offload_capa,
1665                         __func__);
1666                 return -EINVAL;
1667         }
1668
1669         return eth_err(port_id, (*dev->dev_ops->tx_queue_setup)(dev,
1670                        tx_queue_id, nb_tx_desc, socket_id, &local_conf));
1671 }
1672
1673 void
1674 rte_eth_tx_buffer_drop_callback(struct rte_mbuf **pkts, uint16_t unsent,
1675                 void *userdata __rte_unused)
1676 {
1677         unsigned i;
1678
1679         for (i = 0; i < unsent; i++)
1680                 rte_pktmbuf_free(pkts[i]);
1681 }
1682
1683 void
1684 rte_eth_tx_buffer_count_callback(struct rte_mbuf **pkts, uint16_t unsent,
1685                 void *userdata)
1686 {
1687         uint64_t *count = userdata;
1688         unsigned i;
1689
1690         for (i = 0; i < unsent; i++)
1691                 rte_pktmbuf_free(pkts[i]);
1692
1693         *count += unsent;
1694 }
1695
1696 int
1697 rte_eth_tx_buffer_set_err_callback(struct rte_eth_dev_tx_buffer *buffer,
1698                 buffer_tx_error_fn cbfn, void *userdata)
1699 {
1700         buffer->error_callback = cbfn;
1701         buffer->error_userdata = userdata;
1702         return 0;
1703 }
1704
1705 int
1706 rte_eth_tx_buffer_init(struct rte_eth_dev_tx_buffer *buffer, uint16_t size)
1707 {
1708         int ret = 0;
1709
1710         if (buffer == NULL)
1711                 return -EINVAL;
1712
1713         buffer->size = size;
1714         if (buffer->error_callback == NULL) {
1715                 ret = rte_eth_tx_buffer_set_err_callback(
1716                         buffer, rte_eth_tx_buffer_drop_callback, NULL);
1717         }
1718
1719         return ret;
1720 }
1721
1722 int
1723 rte_eth_tx_done_cleanup(uint16_t port_id, uint16_t queue_id, uint32_t free_cnt)
1724 {
1725         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
1726         int ret;
1727
1728         /* Validate Input Data. Bail if not valid or not supported. */
1729         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1730         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_done_cleanup, -ENOTSUP);
1731
1732         /* Call driver to free pending mbufs. */
1733         ret = (*dev->dev_ops->tx_done_cleanup)(dev->data->tx_queues[queue_id],
1734                                                free_cnt);
1735         return eth_err(port_id, ret);
1736 }
1737
1738 void
1739 rte_eth_promiscuous_enable(uint16_t port_id)
1740 {
1741         struct rte_eth_dev *dev;
1742
1743         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1744         dev = &rte_eth_devices[port_id];
1745
1746         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->promiscuous_enable);
1747         (*dev->dev_ops->promiscuous_enable)(dev);
1748         dev->data->promiscuous = 1;
1749 }
1750
1751 void
1752 rte_eth_promiscuous_disable(uint16_t port_id)
1753 {
1754         struct rte_eth_dev *dev;
1755
1756         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1757         dev = &rte_eth_devices[port_id];
1758
1759         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->promiscuous_disable);
1760         dev->data->promiscuous = 0;
1761         (*dev->dev_ops->promiscuous_disable)(dev);
1762 }
1763
1764 int
1765 rte_eth_promiscuous_get(uint16_t port_id)
1766 {
1767         struct rte_eth_dev *dev;
1768
1769         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1770
1771         dev = &rte_eth_devices[port_id];
1772         return dev->data->promiscuous;
1773 }
1774
1775 void
1776 rte_eth_allmulticast_enable(uint16_t port_id)
1777 {
1778         struct rte_eth_dev *dev;
1779
1780         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1781         dev = &rte_eth_devices[port_id];
1782
1783         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->allmulticast_enable);
1784         (*dev->dev_ops->allmulticast_enable)(dev);
1785         dev->data->all_multicast = 1;
1786 }
1787
1788 void
1789 rte_eth_allmulticast_disable(uint16_t port_id)
1790 {
1791         struct rte_eth_dev *dev;
1792
1793         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1794         dev = &rte_eth_devices[port_id];
1795
1796         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->allmulticast_disable);
1797         dev->data->all_multicast = 0;
1798         (*dev->dev_ops->allmulticast_disable)(dev);
1799 }
1800
1801 int
1802 rte_eth_allmulticast_get(uint16_t port_id)
1803 {
1804         struct rte_eth_dev *dev;
1805
1806         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1807
1808         dev = &rte_eth_devices[port_id];
1809         return dev->data->all_multicast;
1810 }
1811
1812 void
1813 rte_eth_link_get(uint16_t port_id, struct rte_eth_link *eth_link)
1814 {
1815         struct rte_eth_dev *dev;
1816
1817         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1818         dev = &rte_eth_devices[port_id];
1819
1820         if (dev->data->dev_conf.intr_conf.lsc &&
1821             dev->data->dev_started)
1822                 rte_eth_linkstatus_get(dev, eth_link);
1823         else {
1824                 RTE_FUNC_PTR_OR_RET(*dev->dev_ops->link_update);
1825                 (*dev->dev_ops->link_update)(dev, 1);
1826                 *eth_link = dev->data->dev_link;
1827         }
1828 }
1829
1830 void
1831 rte_eth_link_get_nowait(uint16_t port_id, struct rte_eth_link *eth_link)
1832 {
1833         struct rte_eth_dev *dev;
1834
1835         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1836         dev = &rte_eth_devices[port_id];
1837
1838         if (dev->data->dev_conf.intr_conf.lsc &&
1839             dev->data->dev_started)
1840                 rte_eth_linkstatus_get(dev, eth_link);
1841         else {
1842                 RTE_FUNC_PTR_OR_RET(*dev->dev_ops->link_update);
1843                 (*dev->dev_ops->link_update)(dev, 0);
1844                 *eth_link = dev->data->dev_link;
1845         }
1846 }
1847
1848 int
1849 rte_eth_stats_get(uint16_t port_id, struct rte_eth_stats *stats)
1850 {
1851         struct rte_eth_dev *dev;
1852
1853         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1854
1855         dev = &rte_eth_devices[port_id];
1856         memset(stats, 0, sizeof(*stats));
1857
1858         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
1859         stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
1860         return eth_err(port_id, (*dev->dev_ops->stats_get)(dev, stats));
1861 }
1862
1863 int
1864 rte_eth_stats_reset(uint16_t port_id)
1865 {
1866         struct rte_eth_dev *dev;
1867
1868         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1869         dev = &rte_eth_devices[port_id];
1870
1871         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_reset, -ENOTSUP);
1872         (*dev->dev_ops->stats_reset)(dev);
1873         dev->data->rx_mbuf_alloc_failed = 0;
1874
1875         return 0;
1876 }
1877
1878 static inline int
1879 get_xstats_basic_count(struct rte_eth_dev *dev)
1880 {
1881         uint16_t nb_rxqs, nb_txqs;
1882         int count;
1883
1884         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
1885         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
1886
1887         count = RTE_NB_STATS;
1888         count += nb_rxqs * RTE_NB_RXQ_STATS;
1889         count += nb_txqs * RTE_NB_TXQ_STATS;
1890
1891         return count;
1892 }
1893
1894 static int
1895 get_xstats_count(uint16_t port_id)
1896 {
1897         struct rte_eth_dev *dev;
1898         int count;
1899
1900         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1901         dev = &rte_eth_devices[port_id];
1902         if (dev->dev_ops->xstats_get_names_by_id != NULL) {
1903                 count = (*dev->dev_ops->xstats_get_names_by_id)(dev, NULL,
1904                                 NULL, 0);
1905                 if (count < 0)
1906                         return eth_err(port_id, count);
1907         }
1908         if (dev->dev_ops->xstats_get_names != NULL) {
1909                 count = (*dev->dev_ops->xstats_get_names)(dev, NULL, 0);
1910                 if (count < 0)
1911                         return eth_err(port_id, count);
1912         } else
1913                 count = 0;
1914
1915
1916         count += get_xstats_basic_count(dev);
1917
1918         return count;
1919 }
1920
1921 int
1922 rte_eth_xstats_get_id_by_name(uint16_t port_id, const char *xstat_name,
1923                 uint64_t *id)
1924 {
1925         int cnt_xstats, idx_xstat;
1926
1927         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1928
1929         if (!id) {
1930                 RTE_ETHDEV_LOG(ERR, "Id pointer is NULL\n");
1931                 return -ENOMEM;
1932         }
1933
1934         if (!xstat_name) {
1935                 RTE_ETHDEV_LOG(ERR, "xstat_name pointer is NULL\n");
1936                 return -ENOMEM;
1937         }
1938
1939         /* Get count */
1940         cnt_xstats = rte_eth_xstats_get_names_by_id(port_id, NULL, 0, NULL);
1941         if (cnt_xstats  < 0) {
1942                 RTE_ETHDEV_LOG(ERR, "Cannot get count of xstats\n");
1943                 return -ENODEV;
1944         }
1945
1946         /* Get id-name lookup table */
1947         struct rte_eth_xstat_name xstats_names[cnt_xstats];
1948
1949         if (cnt_xstats != rte_eth_xstats_get_names_by_id(
1950                         port_id, xstats_names, cnt_xstats, NULL)) {
1951                 RTE_ETHDEV_LOG(ERR, "Cannot get xstats lookup\n");
1952                 return -1;
1953         }
1954
1955         for (idx_xstat = 0; idx_xstat < cnt_xstats; idx_xstat++) {
1956                 if (!strcmp(xstats_names[idx_xstat].name, xstat_name)) {
1957                         *id = idx_xstat;
1958                         return 0;
1959                 };
1960         }
1961
1962         return -EINVAL;
1963 }
1964
1965 /* retrieve basic stats names */
1966 static int
1967 rte_eth_basic_stats_get_names(struct rte_eth_dev *dev,
1968         struct rte_eth_xstat_name *xstats_names)
1969 {
1970         int cnt_used_entries = 0;
1971         uint32_t idx, id_queue;
1972         uint16_t num_q;
1973
1974         for (idx = 0; idx < RTE_NB_STATS; idx++) {
1975                 snprintf(xstats_names[cnt_used_entries].name,
1976                         sizeof(xstats_names[0].name),
1977                         "%s", rte_stats_strings[idx].name);
1978                 cnt_used_entries++;
1979         }
1980         num_q = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
1981         for (id_queue = 0; id_queue < num_q; id_queue++) {
1982                 for (idx = 0; idx < RTE_NB_RXQ_STATS; idx++) {
1983                         snprintf(xstats_names[cnt_used_entries].name,
1984                                 sizeof(xstats_names[0].name),
1985                                 "rx_q%u%s",
1986                                 id_queue, rte_rxq_stats_strings[idx].name);
1987                         cnt_used_entries++;
1988                 }
1989
1990         }
1991         num_q = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
1992         for (id_queue = 0; id_queue < num_q; id_queue++) {
1993                 for (idx = 0; idx < RTE_NB_TXQ_STATS; idx++) {
1994                         snprintf(xstats_names[cnt_used_entries].name,
1995                                 sizeof(xstats_names[0].name),
1996                                 "tx_q%u%s",
1997                                 id_queue, rte_txq_stats_strings[idx].name);
1998                         cnt_used_entries++;
1999                 }
2000         }
2001         return cnt_used_entries;
2002 }
2003
2004 /* retrieve ethdev extended statistics names */
2005 int
2006 rte_eth_xstats_get_names_by_id(uint16_t port_id,
2007         struct rte_eth_xstat_name *xstats_names, unsigned int size,
2008         uint64_t *ids)
2009 {
2010         struct rte_eth_xstat_name *xstats_names_copy;
2011         unsigned int no_basic_stat_requested = 1;
2012         unsigned int no_ext_stat_requested = 1;
2013         unsigned int expected_entries;
2014         unsigned int basic_count;
2015         struct rte_eth_dev *dev;
2016         unsigned int i;
2017         int ret;
2018
2019         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2020         dev = &rte_eth_devices[port_id];
2021
2022         basic_count = get_xstats_basic_count(dev);
2023         ret = get_xstats_count(port_id);
2024         if (ret < 0)
2025                 return ret;
2026         expected_entries = (unsigned int)ret;
2027
2028         /* Return max number of stats if no ids given */
2029         if (!ids) {
2030                 if (!xstats_names)
2031                         return expected_entries;
2032                 else if (xstats_names && size < expected_entries)
2033                         return expected_entries;
2034         }
2035
2036         if (ids && !xstats_names)
2037                 return -EINVAL;
2038
2039         if (ids && dev->dev_ops->xstats_get_names_by_id != NULL && size > 0) {
2040                 uint64_t ids_copy[size];
2041
2042                 for (i = 0; i < size; i++) {
2043                         if (ids[i] < basic_count) {
2044                                 no_basic_stat_requested = 0;
2045                                 break;
2046                         }
2047
2048                         /*
2049                          * Convert ids to xstats ids that PMD knows.
2050                          * ids known by user are basic + extended stats.
2051                          */
2052                         ids_copy[i] = ids[i] - basic_count;
2053                 }
2054
2055                 if (no_basic_stat_requested)
2056                         return (*dev->dev_ops->xstats_get_names_by_id)(dev,
2057                                         xstats_names, ids_copy, size);
2058         }
2059
2060         /* Retrieve all stats */
2061         if (!ids) {
2062                 int num_stats = rte_eth_xstats_get_names(port_id, xstats_names,
2063                                 expected_entries);
2064                 if (num_stats < 0 || num_stats > (int)expected_entries)
2065                         return num_stats;
2066                 else
2067                         return expected_entries;
2068         }
2069
2070         xstats_names_copy = calloc(expected_entries,
2071                 sizeof(struct rte_eth_xstat_name));
2072
2073         if (!xstats_names_copy) {
2074                 RTE_ETHDEV_LOG(ERR, "Can't allocate memory\n");
2075                 return -ENOMEM;
2076         }
2077
2078         if (ids) {
2079                 for (i = 0; i < size; i++) {
2080                         if (ids[i] >= basic_count) {
2081                                 no_ext_stat_requested = 0;
2082                                 break;
2083                         }
2084                 }
2085         }
2086
2087         /* Fill xstats_names_copy structure */
2088         if (ids && no_ext_stat_requested) {
2089                 rte_eth_basic_stats_get_names(dev, xstats_names_copy);
2090         } else {
2091                 ret = rte_eth_xstats_get_names(port_id, xstats_names_copy,
2092                         expected_entries);
2093                 if (ret < 0) {
2094                         free(xstats_names_copy);
2095                         return ret;
2096                 }
2097         }
2098
2099         /* Filter stats */
2100         for (i = 0; i < size; i++) {
2101                 if (ids[i] >= expected_entries) {
2102                         RTE_ETHDEV_LOG(ERR, "Id value isn't valid\n");
2103                         free(xstats_names_copy);
2104                         return -1;
2105                 }
2106                 xstats_names[i] = xstats_names_copy[ids[i]];
2107         }
2108
2109         free(xstats_names_copy);
2110         return size;
2111 }
2112
2113 int
2114 rte_eth_xstats_get_names(uint16_t port_id,
2115         struct rte_eth_xstat_name *xstats_names,
2116         unsigned int size)
2117 {
2118         struct rte_eth_dev *dev;
2119         int cnt_used_entries;
2120         int cnt_expected_entries;
2121         int cnt_driver_entries;
2122
2123         cnt_expected_entries = get_xstats_count(port_id);
2124         if (xstats_names == NULL || cnt_expected_entries < 0 ||
2125                         (int)size < cnt_expected_entries)
2126                 return cnt_expected_entries;
2127
2128         /* port_id checked in get_xstats_count() */
2129         dev = &rte_eth_devices[port_id];
2130
2131         cnt_used_entries = rte_eth_basic_stats_get_names(
2132                 dev, xstats_names);
2133
2134         if (dev->dev_ops->xstats_get_names != NULL) {
2135                 /* If there are any driver-specific xstats, append them
2136                  * to end of list.
2137                  */
2138                 cnt_driver_entries = (*dev->dev_ops->xstats_get_names)(
2139                         dev,
2140                         xstats_names + cnt_used_entries,
2141                         size - cnt_used_entries);
2142                 if (cnt_driver_entries < 0)
2143                         return eth_err(port_id, cnt_driver_entries);
2144                 cnt_used_entries += cnt_driver_entries;
2145         }
2146
2147         return cnt_used_entries;
2148 }
2149
2150
2151 static int
2152 rte_eth_basic_stats_get(uint16_t port_id, struct rte_eth_xstat *xstats)
2153 {
2154         struct rte_eth_dev *dev;
2155         struct rte_eth_stats eth_stats;
2156         unsigned int count = 0, i, q;
2157         uint64_t val, *stats_ptr;
2158         uint16_t nb_rxqs, nb_txqs;
2159         int ret;
2160
2161         ret = rte_eth_stats_get(port_id, &eth_stats);
2162         if (ret < 0)
2163                 return ret;
2164
2165         dev = &rte_eth_devices[port_id];
2166
2167         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2168         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2169
2170         /* global stats */
2171         for (i = 0; i < RTE_NB_STATS; i++) {
2172                 stats_ptr = RTE_PTR_ADD(&eth_stats,
2173                                         rte_stats_strings[i].offset);
2174                 val = *stats_ptr;
2175                 xstats[count++].value = val;
2176         }
2177
2178         /* per-rxq stats */
2179         for (q = 0; q < nb_rxqs; q++) {
2180                 for (i = 0; i < RTE_NB_RXQ_STATS; i++) {
2181                         stats_ptr = RTE_PTR_ADD(&eth_stats,
2182                                         rte_rxq_stats_strings[i].offset +
2183                                         q * sizeof(uint64_t));
2184                         val = *stats_ptr;
2185                         xstats[count++].value = val;
2186                 }
2187         }
2188
2189         /* per-txq stats */
2190         for (q = 0; q < nb_txqs; q++) {
2191                 for (i = 0; i < RTE_NB_TXQ_STATS; i++) {
2192                         stats_ptr = RTE_PTR_ADD(&eth_stats,
2193                                         rte_txq_stats_strings[i].offset +
2194                                         q * sizeof(uint64_t));
2195                         val = *stats_ptr;
2196                         xstats[count++].value = val;
2197                 }
2198         }
2199         return count;
2200 }
2201
2202 /* retrieve ethdev extended statistics */
2203 int
2204 rte_eth_xstats_get_by_id(uint16_t port_id, const uint64_t *ids,
2205                          uint64_t *values, unsigned int size)
2206 {
2207         unsigned int no_basic_stat_requested = 1;
2208         unsigned int no_ext_stat_requested = 1;
2209         unsigned int num_xstats_filled;
2210         unsigned int basic_count;
2211         uint16_t expected_entries;
2212         struct rte_eth_dev *dev;
2213         unsigned int i;
2214         int ret;
2215
2216         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2217         ret = get_xstats_count(port_id);
2218         if (ret < 0)
2219                 return ret;
2220         expected_entries = (uint16_t)ret;
2221         struct rte_eth_xstat xstats[expected_entries];
2222         dev = &rte_eth_devices[port_id];
2223         basic_count = get_xstats_basic_count(dev);
2224
2225         /* Return max number of stats if no ids given */
2226         if (!ids) {
2227                 if (!values)
2228                         return expected_entries;
2229                 else if (values && size < expected_entries)
2230                         return expected_entries;
2231         }
2232
2233         if (ids && !values)
2234                 return -EINVAL;
2235
2236         if (ids && dev->dev_ops->xstats_get_by_id != NULL && size) {
2237                 unsigned int basic_count = get_xstats_basic_count(dev);
2238                 uint64_t ids_copy[size];
2239
2240                 for (i = 0; i < size; i++) {
2241                         if (ids[i] < basic_count) {
2242                                 no_basic_stat_requested = 0;
2243                                 break;
2244                         }
2245
2246                         /*
2247                          * Convert ids to xstats ids that PMD knows.
2248                          * ids known by user are basic + extended stats.
2249                          */
2250                         ids_copy[i] = ids[i] - basic_count;
2251                 }
2252
2253                 if (no_basic_stat_requested)
2254                         return (*dev->dev_ops->xstats_get_by_id)(dev, ids_copy,
2255                                         values, size);
2256         }
2257
2258         if (ids) {
2259                 for (i = 0; i < size; i++) {
2260                         if (ids[i] >= basic_count) {
2261                                 no_ext_stat_requested = 0;
2262                                 break;
2263                         }
2264                 }
2265         }
2266
2267         /* Fill the xstats structure */
2268         if (ids && no_ext_stat_requested)
2269                 ret = rte_eth_basic_stats_get(port_id, xstats);
2270         else
2271                 ret = rte_eth_xstats_get(port_id, xstats, expected_entries);
2272
2273         if (ret < 0)
2274                 return ret;
2275         num_xstats_filled = (unsigned int)ret;
2276
2277         /* Return all stats */
2278         if (!ids) {
2279                 for (i = 0; i < num_xstats_filled; i++)
2280                         values[i] = xstats[i].value;
2281                 return expected_entries;
2282         }
2283
2284         /* Filter stats */
2285         for (i = 0; i < size; i++) {
2286                 if (ids[i] >= expected_entries) {
2287                         RTE_ETHDEV_LOG(ERR, "Id value isn't valid\n");
2288                         return -1;
2289                 }
2290                 values[i] = xstats[ids[i]].value;
2291         }
2292         return size;
2293 }
2294
2295 int
2296 rte_eth_xstats_get(uint16_t port_id, struct rte_eth_xstat *xstats,
2297         unsigned int n)
2298 {
2299         struct rte_eth_dev *dev;
2300         unsigned int count = 0, i;
2301         signed int xcount = 0;
2302         uint16_t nb_rxqs, nb_txqs;
2303         int ret;
2304
2305         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2306
2307         dev = &rte_eth_devices[port_id];
2308
2309         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2310         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2311
2312         /* Return generic statistics */
2313         count = RTE_NB_STATS + (nb_rxqs * RTE_NB_RXQ_STATS) +
2314                 (nb_txqs * RTE_NB_TXQ_STATS);
2315
2316         /* implemented by the driver */
2317         if (dev->dev_ops->xstats_get != NULL) {
2318                 /* Retrieve the xstats from the driver at the end of the
2319                  * xstats struct.
2320                  */
2321                 xcount = (*dev->dev_ops->xstats_get)(dev,
2322                                      xstats ? xstats + count : NULL,
2323                                      (n > count) ? n - count : 0);
2324
2325                 if (xcount < 0)
2326                         return eth_err(port_id, xcount);
2327         }
2328
2329         if (n < count + xcount || xstats == NULL)
2330                 return count + xcount;
2331
2332         /* now fill the xstats structure */
2333         ret = rte_eth_basic_stats_get(port_id, xstats);
2334         if (ret < 0)
2335                 return ret;
2336         count = ret;
2337
2338         for (i = 0; i < count; i++)
2339                 xstats[i].id = i;
2340         /* add an offset to driver-specific stats */
2341         for ( ; i < count + xcount; i++)
2342                 xstats[i].id += count;
2343
2344         return count + xcount;
2345 }
2346
2347 /* reset ethdev extended statistics */
2348 void
2349 rte_eth_xstats_reset(uint16_t port_id)
2350 {
2351         struct rte_eth_dev *dev;
2352
2353         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2354         dev = &rte_eth_devices[port_id];
2355
2356         /* implemented by the driver */
2357         if (dev->dev_ops->xstats_reset != NULL) {
2358                 (*dev->dev_ops->xstats_reset)(dev);
2359                 return;
2360         }
2361
2362         /* fallback to default */
2363         rte_eth_stats_reset(port_id);
2364 }
2365
2366 static int
2367 set_queue_stats_mapping(uint16_t port_id, uint16_t queue_id, uint8_t stat_idx,
2368                 uint8_t is_rx)
2369 {
2370         struct rte_eth_dev *dev;
2371
2372         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2373
2374         dev = &rte_eth_devices[port_id];
2375
2376         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_stats_mapping_set, -ENOTSUP);
2377
2378         if (is_rx && (queue_id >= dev->data->nb_rx_queues))
2379                 return -EINVAL;
2380
2381         if (!is_rx && (queue_id >= dev->data->nb_tx_queues))
2382                 return -EINVAL;
2383
2384         if (stat_idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
2385                 return -EINVAL;
2386
2387         return (*dev->dev_ops->queue_stats_mapping_set)
2388                         (dev, queue_id, stat_idx, is_rx);
2389 }
2390
2391
2392 int
2393 rte_eth_dev_set_tx_queue_stats_mapping(uint16_t port_id, uint16_t tx_queue_id,
2394                 uint8_t stat_idx)
2395 {
2396         return eth_err(port_id, set_queue_stats_mapping(port_id, tx_queue_id,
2397                                                 stat_idx, STAT_QMAP_TX));
2398 }
2399
2400
2401 int
2402 rte_eth_dev_set_rx_queue_stats_mapping(uint16_t port_id, uint16_t rx_queue_id,
2403                 uint8_t stat_idx)
2404 {
2405         return eth_err(port_id, set_queue_stats_mapping(port_id, rx_queue_id,
2406                                                 stat_idx, STAT_QMAP_RX));
2407 }
2408
2409 int
2410 rte_eth_dev_fw_version_get(uint16_t port_id, char *fw_version, size_t fw_size)
2411 {
2412         struct rte_eth_dev *dev;
2413
2414         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2415         dev = &rte_eth_devices[port_id];
2416
2417         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fw_version_get, -ENOTSUP);
2418         return eth_err(port_id, (*dev->dev_ops->fw_version_get)(dev,
2419                                                         fw_version, fw_size));
2420 }
2421
2422 void
2423 rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info)
2424 {
2425         struct rte_eth_dev *dev;
2426         const struct rte_eth_desc_lim lim = {
2427                 .nb_max = UINT16_MAX,
2428                 .nb_min = 0,
2429                 .nb_align = 1,
2430         };
2431
2432         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2433         dev = &rte_eth_devices[port_id];
2434
2435         memset(dev_info, 0, sizeof(struct rte_eth_dev_info));
2436         dev_info->rx_desc_lim = lim;
2437         dev_info->tx_desc_lim = lim;
2438         dev_info->device = dev->device;
2439
2440         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_infos_get);
2441         (*dev->dev_ops->dev_infos_get)(dev, dev_info);
2442         dev_info->driver_name = dev->device->driver->name;
2443         dev_info->nb_rx_queues = dev->data->nb_rx_queues;
2444         dev_info->nb_tx_queues = dev->data->nb_tx_queues;
2445
2446         dev_info->dev_flags = &dev->data->dev_flags;
2447 }
2448
2449 int
2450 rte_eth_dev_get_supported_ptypes(uint16_t port_id, uint32_t ptype_mask,
2451                                  uint32_t *ptypes, int num)
2452 {
2453         int i, j;
2454         struct rte_eth_dev *dev;
2455         const uint32_t *all_ptypes;
2456
2457         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2458         dev = &rte_eth_devices[port_id];
2459         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_supported_ptypes_get, 0);
2460         all_ptypes = (*dev->dev_ops->dev_supported_ptypes_get)(dev);
2461
2462         if (!all_ptypes)
2463                 return 0;
2464
2465         for (i = 0, j = 0; all_ptypes[i] != RTE_PTYPE_UNKNOWN; ++i)
2466                 if (all_ptypes[i] & ptype_mask) {
2467                         if (j < num)
2468                                 ptypes[j] = all_ptypes[i];
2469                         j++;
2470                 }
2471
2472         return j;
2473 }
2474
2475 void
2476 rte_eth_macaddr_get(uint16_t port_id, struct ether_addr *mac_addr)
2477 {
2478         struct rte_eth_dev *dev;
2479
2480         RTE_ETH_VALID_PORTID_OR_RET(port_id);
2481         dev = &rte_eth_devices[port_id];
2482         ether_addr_copy(&dev->data->mac_addrs[0], mac_addr);
2483 }
2484
2485
2486 int
2487 rte_eth_dev_get_mtu(uint16_t port_id, uint16_t *mtu)
2488 {
2489         struct rte_eth_dev *dev;
2490
2491         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2492
2493         dev = &rte_eth_devices[port_id];
2494         *mtu = dev->data->mtu;
2495         return 0;
2496 }
2497
2498 int
2499 rte_eth_dev_set_mtu(uint16_t port_id, uint16_t mtu)
2500 {
2501         int ret;
2502         struct rte_eth_dev *dev;
2503
2504         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2505         dev = &rte_eth_devices[port_id];
2506         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mtu_set, -ENOTSUP);
2507
2508         ret = (*dev->dev_ops->mtu_set)(dev, mtu);
2509         if (!ret)
2510                 dev->data->mtu = mtu;
2511
2512         return eth_err(port_id, ret);
2513 }
2514
2515 int
2516 rte_eth_dev_vlan_filter(uint16_t port_id, uint16_t vlan_id, int on)
2517 {
2518         struct rte_eth_dev *dev;
2519         int ret;
2520
2521         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2522         dev = &rte_eth_devices[port_id];
2523         if (!(dev->data->dev_conf.rxmode.offloads &
2524               DEV_RX_OFFLOAD_VLAN_FILTER)) {
2525                 RTE_ETHDEV_LOG(ERR, "Port %u: vlan-filtering disabled\n",
2526                         port_id);
2527                 return -ENOSYS;
2528         }
2529
2530         if (vlan_id > 4095) {
2531                 RTE_ETHDEV_LOG(ERR, "Port_id=%u invalid vlan_id=%u > 4095\n",
2532                         port_id, vlan_id);
2533                 return -EINVAL;
2534         }
2535         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_filter_set, -ENOTSUP);
2536
2537         ret = (*dev->dev_ops->vlan_filter_set)(dev, vlan_id, on);
2538         if (ret == 0) {
2539                 struct rte_vlan_filter_conf *vfc;
2540                 int vidx;
2541                 int vbit;
2542
2543                 vfc = &dev->data->vlan_filter_conf;
2544                 vidx = vlan_id / 64;
2545                 vbit = vlan_id % 64;
2546
2547                 if (on)
2548                         vfc->ids[vidx] |= UINT64_C(1) << vbit;
2549                 else
2550                         vfc->ids[vidx] &= ~(UINT64_C(1) << vbit);
2551         }
2552
2553         return eth_err(port_id, ret);
2554 }
2555
2556 int
2557 rte_eth_dev_set_vlan_strip_on_queue(uint16_t port_id, uint16_t rx_queue_id,
2558                                     int on)
2559 {
2560         struct rte_eth_dev *dev;
2561
2562         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2563         dev = &rte_eth_devices[port_id];
2564         if (rx_queue_id >= dev->data->nb_rx_queues) {
2565                 RTE_ETHDEV_LOG(ERR, "Invalid rx_queue_id=%u\n", rx_queue_id);
2566                 return -EINVAL;
2567         }
2568
2569         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_strip_queue_set, -ENOTSUP);
2570         (*dev->dev_ops->vlan_strip_queue_set)(dev, rx_queue_id, on);
2571
2572         return 0;
2573 }
2574
2575 int
2576 rte_eth_dev_set_vlan_ether_type(uint16_t port_id,
2577                                 enum rte_vlan_type vlan_type,
2578                                 uint16_t tpid)
2579 {
2580         struct rte_eth_dev *dev;
2581
2582         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2583         dev = &rte_eth_devices[port_id];
2584         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_tpid_set, -ENOTSUP);
2585
2586         return eth_err(port_id, (*dev->dev_ops->vlan_tpid_set)(dev, vlan_type,
2587                                                                tpid));
2588 }
2589
2590 int
2591 rte_eth_dev_set_vlan_offload(uint16_t port_id, int offload_mask)
2592 {
2593         struct rte_eth_dev *dev;
2594         int ret = 0;
2595         int mask = 0;
2596         int cur, org = 0;
2597         uint64_t orig_offloads;
2598
2599         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2600         dev = &rte_eth_devices[port_id];
2601
2602         /* save original values in case of failure */
2603         orig_offloads = dev->data->dev_conf.rxmode.offloads;
2604
2605         /*check which option changed by application*/
2606         cur = !!(offload_mask & ETH_VLAN_STRIP_OFFLOAD);
2607         org = !!(dev->data->dev_conf.rxmode.offloads &
2608                  DEV_RX_OFFLOAD_VLAN_STRIP);
2609         if (cur != org) {
2610                 if (cur)
2611                         dev->data->dev_conf.rxmode.offloads |=
2612                                 DEV_RX_OFFLOAD_VLAN_STRIP;
2613                 else
2614                         dev->data->dev_conf.rxmode.offloads &=
2615                                 ~DEV_RX_OFFLOAD_VLAN_STRIP;
2616                 mask |= ETH_VLAN_STRIP_MASK;
2617         }
2618
2619         cur = !!(offload_mask & ETH_VLAN_FILTER_OFFLOAD);
2620         org = !!(dev->data->dev_conf.rxmode.offloads &
2621                  DEV_RX_OFFLOAD_VLAN_FILTER);
2622         if (cur != org) {
2623                 if (cur)
2624                         dev->data->dev_conf.rxmode.offloads |=
2625                                 DEV_RX_OFFLOAD_VLAN_FILTER;
2626                 else
2627                         dev->data->dev_conf.rxmode.offloads &=
2628                                 ~DEV_RX_OFFLOAD_VLAN_FILTER;
2629                 mask |= ETH_VLAN_FILTER_MASK;
2630         }
2631
2632         cur = !!(offload_mask & ETH_VLAN_EXTEND_OFFLOAD);
2633         org = !!(dev->data->dev_conf.rxmode.offloads &
2634                  DEV_RX_OFFLOAD_VLAN_EXTEND);
2635         if (cur != org) {
2636                 if (cur)
2637                         dev->data->dev_conf.rxmode.offloads |=
2638                                 DEV_RX_OFFLOAD_VLAN_EXTEND;
2639                 else
2640                         dev->data->dev_conf.rxmode.offloads &=
2641                                 ~DEV_RX_OFFLOAD_VLAN_EXTEND;
2642                 mask |= ETH_VLAN_EXTEND_MASK;
2643         }
2644
2645         /*no change*/
2646         if (mask == 0)
2647                 return ret;
2648
2649         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_offload_set, -ENOTSUP);
2650         ret = (*dev->dev_ops->vlan_offload_set)(dev, mask);
2651         if (ret) {
2652                 /* hit an error restore  original values */
2653                 dev->data->dev_conf.rxmode.offloads = orig_offloads;
2654         }
2655
2656         return eth_err(port_id, ret);
2657 }
2658
2659 int
2660 rte_eth_dev_get_vlan_offload(uint16_t port_id)
2661 {
2662         struct rte_eth_dev *dev;
2663         int ret = 0;
2664
2665         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2666         dev = &rte_eth_devices[port_id];
2667
2668         if (dev->data->dev_conf.rxmode.offloads &
2669             DEV_RX_OFFLOAD_VLAN_STRIP)
2670                 ret |= ETH_VLAN_STRIP_OFFLOAD;
2671
2672         if (dev->data->dev_conf.rxmode.offloads &
2673             DEV_RX_OFFLOAD_VLAN_FILTER)
2674                 ret |= ETH_VLAN_FILTER_OFFLOAD;
2675
2676         if (dev->data->dev_conf.rxmode.offloads &
2677             DEV_RX_OFFLOAD_VLAN_EXTEND)
2678                 ret |= ETH_VLAN_EXTEND_OFFLOAD;
2679
2680         return ret;
2681 }
2682
2683 int
2684 rte_eth_dev_set_vlan_pvid(uint16_t port_id, uint16_t pvid, int on)
2685 {
2686         struct rte_eth_dev *dev;
2687
2688         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2689         dev = &rte_eth_devices[port_id];
2690         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_pvid_set, -ENOTSUP);
2691
2692         return eth_err(port_id, (*dev->dev_ops->vlan_pvid_set)(dev, pvid, on));
2693 }
2694
2695 int
2696 rte_eth_dev_flow_ctrl_get(uint16_t port_id, struct rte_eth_fc_conf *fc_conf)
2697 {
2698         struct rte_eth_dev *dev;
2699
2700         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2701         dev = &rte_eth_devices[port_id];
2702         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_get, -ENOTSUP);
2703         memset(fc_conf, 0, sizeof(*fc_conf));
2704         return eth_err(port_id, (*dev->dev_ops->flow_ctrl_get)(dev, fc_conf));
2705 }
2706
2707 int
2708 rte_eth_dev_flow_ctrl_set(uint16_t port_id, struct rte_eth_fc_conf *fc_conf)
2709 {
2710         struct rte_eth_dev *dev;
2711
2712         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2713         if ((fc_conf->send_xon != 0) && (fc_conf->send_xon != 1)) {
2714                 RTE_ETHDEV_LOG(ERR, "Invalid send_xon, only 0/1 allowed\n");
2715                 return -EINVAL;
2716         }
2717
2718         dev = &rte_eth_devices[port_id];
2719         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_set, -ENOTSUP);
2720         return eth_err(port_id, (*dev->dev_ops->flow_ctrl_set)(dev, fc_conf));
2721 }
2722
2723 int
2724 rte_eth_dev_priority_flow_ctrl_set(uint16_t port_id,
2725                                    struct rte_eth_pfc_conf *pfc_conf)
2726 {
2727         struct rte_eth_dev *dev;
2728
2729         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2730         if (pfc_conf->priority > (ETH_DCB_NUM_USER_PRIORITIES - 1)) {
2731                 RTE_ETHDEV_LOG(ERR, "Invalid priority, only 0-7 allowed\n");
2732                 return -EINVAL;
2733         }
2734
2735         dev = &rte_eth_devices[port_id];
2736         /* High water, low water validation are device specific */
2737         if  (*dev->dev_ops->priority_flow_ctrl_set)
2738                 return eth_err(port_id, (*dev->dev_ops->priority_flow_ctrl_set)
2739                                         (dev, pfc_conf));
2740         return -ENOTSUP;
2741 }
2742
2743 static int
2744 rte_eth_check_reta_mask(struct rte_eth_rss_reta_entry64 *reta_conf,
2745                         uint16_t reta_size)
2746 {
2747         uint16_t i, num;
2748
2749         if (!reta_conf)
2750                 return -EINVAL;
2751
2752         num = (reta_size + RTE_RETA_GROUP_SIZE - 1) / RTE_RETA_GROUP_SIZE;
2753         for (i = 0; i < num; i++) {
2754                 if (reta_conf[i].mask)
2755                         return 0;
2756         }
2757
2758         return -EINVAL;
2759 }
2760
2761 static int
2762 rte_eth_check_reta_entry(struct rte_eth_rss_reta_entry64 *reta_conf,
2763                          uint16_t reta_size,
2764                          uint16_t max_rxq)
2765 {
2766         uint16_t i, idx, shift;
2767
2768         if (!reta_conf)
2769                 return -EINVAL;
2770
2771         if (max_rxq == 0) {
2772                 RTE_ETHDEV_LOG(ERR, "No receive queue is available\n");
2773                 return -EINVAL;
2774         }
2775
2776         for (i = 0; i < reta_size; i++) {
2777                 idx = i / RTE_RETA_GROUP_SIZE;
2778                 shift = i % RTE_RETA_GROUP_SIZE;
2779                 if ((reta_conf[idx].mask & (1ULL << shift)) &&
2780                         (reta_conf[idx].reta[shift] >= max_rxq)) {
2781                         RTE_ETHDEV_LOG(ERR,
2782                                 "reta_conf[%u]->reta[%u]: %u exceeds the maximum rxq index: %u\n",
2783                                 idx, shift,
2784                                 reta_conf[idx].reta[shift], max_rxq);
2785                         return -EINVAL;
2786                 }
2787         }
2788
2789         return 0;
2790 }
2791
2792 int
2793 rte_eth_dev_rss_reta_update(uint16_t port_id,
2794                             struct rte_eth_rss_reta_entry64 *reta_conf,
2795                             uint16_t reta_size)
2796 {
2797         struct rte_eth_dev *dev;
2798         int ret;
2799
2800         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2801         /* Check mask bits */
2802         ret = rte_eth_check_reta_mask(reta_conf, reta_size);
2803         if (ret < 0)
2804                 return ret;
2805
2806         dev = &rte_eth_devices[port_id];
2807
2808         /* Check entry value */
2809         ret = rte_eth_check_reta_entry(reta_conf, reta_size,
2810                                 dev->data->nb_rx_queues);
2811         if (ret < 0)
2812                 return ret;
2813
2814         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_update, -ENOTSUP);
2815         return eth_err(port_id, (*dev->dev_ops->reta_update)(dev, reta_conf,
2816                                                              reta_size));
2817 }
2818
2819 int
2820 rte_eth_dev_rss_reta_query(uint16_t port_id,
2821                            struct rte_eth_rss_reta_entry64 *reta_conf,
2822                            uint16_t reta_size)
2823 {
2824         struct rte_eth_dev *dev;
2825         int ret;
2826
2827         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2828
2829         /* Check mask bits */
2830         ret = rte_eth_check_reta_mask(reta_conf, reta_size);
2831         if (ret < 0)
2832                 return ret;
2833
2834         dev = &rte_eth_devices[port_id];
2835         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_query, -ENOTSUP);
2836         return eth_err(port_id, (*dev->dev_ops->reta_query)(dev, reta_conf,
2837                                                             reta_size));
2838 }
2839
2840 int
2841 rte_eth_dev_rss_hash_update(uint16_t port_id,
2842                             struct rte_eth_rss_conf *rss_conf)
2843 {
2844         struct rte_eth_dev *dev;
2845         struct rte_eth_dev_info dev_info = { .flow_type_rss_offloads = 0, };
2846
2847         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2848         dev = &rte_eth_devices[port_id];
2849         rte_eth_dev_info_get(port_id, &dev_info);
2850         if ((dev_info.flow_type_rss_offloads | rss_conf->rss_hf) !=
2851             dev_info.flow_type_rss_offloads) {
2852                 RTE_ETHDEV_LOG(ERR,
2853                         "Ethdev port_id=%u invalid rss_hf: 0x%"PRIx64", valid value: 0x%"PRIx64"\n",
2854                         port_id, rss_conf->rss_hf,
2855                         dev_info.flow_type_rss_offloads);
2856                 return -EINVAL;
2857         }
2858         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_update, -ENOTSUP);
2859         return eth_err(port_id, (*dev->dev_ops->rss_hash_update)(dev,
2860                                                                  rss_conf));
2861 }
2862
2863 int
2864 rte_eth_dev_rss_hash_conf_get(uint16_t port_id,
2865                               struct rte_eth_rss_conf *rss_conf)
2866 {
2867         struct rte_eth_dev *dev;
2868
2869         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2870         dev = &rte_eth_devices[port_id];
2871         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_conf_get, -ENOTSUP);
2872         return eth_err(port_id, (*dev->dev_ops->rss_hash_conf_get)(dev,
2873                                                                    rss_conf));
2874 }
2875
2876 int
2877 rte_eth_dev_udp_tunnel_port_add(uint16_t port_id,
2878                                 struct rte_eth_udp_tunnel *udp_tunnel)
2879 {
2880         struct rte_eth_dev *dev;
2881
2882         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2883         if (udp_tunnel == NULL) {
2884                 RTE_ETHDEV_LOG(ERR, "Invalid udp_tunnel parameter\n");
2885                 return -EINVAL;
2886         }
2887
2888         if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
2889                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
2890                 return -EINVAL;
2891         }
2892
2893         dev = &rte_eth_devices[port_id];
2894         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_port_add, -ENOTSUP);
2895         return eth_err(port_id, (*dev->dev_ops->udp_tunnel_port_add)(dev,
2896                                                                 udp_tunnel));
2897 }
2898
2899 int
2900 rte_eth_dev_udp_tunnel_port_delete(uint16_t port_id,
2901                                    struct rte_eth_udp_tunnel *udp_tunnel)
2902 {
2903         struct rte_eth_dev *dev;
2904
2905         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2906         dev = &rte_eth_devices[port_id];
2907
2908         if (udp_tunnel == NULL) {
2909                 RTE_ETHDEV_LOG(ERR, "Invalid udp_tunnel parameter\n");
2910                 return -EINVAL;
2911         }
2912
2913         if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
2914                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
2915                 return -EINVAL;
2916         }
2917
2918         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_port_del, -ENOTSUP);
2919         return eth_err(port_id, (*dev->dev_ops->udp_tunnel_port_del)(dev,
2920                                                                 udp_tunnel));
2921 }
2922
2923 int
2924 rte_eth_led_on(uint16_t port_id)
2925 {
2926         struct rte_eth_dev *dev;
2927
2928         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2929         dev = &rte_eth_devices[port_id];
2930         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_on, -ENOTSUP);
2931         return eth_err(port_id, (*dev->dev_ops->dev_led_on)(dev));
2932 }
2933
2934 int
2935 rte_eth_led_off(uint16_t port_id)
2936 {
2937         struct rte_eth_dev *dev;
2938
2939         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2940         dev = &rte_eth_devices[port_id];
2941         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_off, -ENOTSUP);
2942         return eth_err(port_id, (*dev->dev_ops->dev_led_off)(dev));
2943 }
2944
2945 /*
2946  * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find
2947  * an empty spot.
2948  */
2949 static int
2950 get_mac_addr_index(uint16_t port_id, const struct ether_addr *addr)
2951 {
2952         struct rte_eth_dev_info dev_info;
2953         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
2954         unsigned i;
2955
2956         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2957         rte_eth_dev_info_get(port_id, &dev_info);
2958
2959         for (i = 0; i < dev_info.max_mac_addrs; i++)
2960                 if (memcmp(addr, &dev->data->mac_addrs[i], ETHER_ADDR_LEN) == 0)
2961                         return i;
2962
2963         return -1;
2964 }
2965
2966 static const struct ether_addr null_mac_addr;
2967
2968 int
2969 rte_eth_dev_mac_addr_add(uint16_t port_id, struct ether_addr *addr,
2970                         uint32_t pool)
2971 {
2972         struct rte_eth_dev *dev;
2973         int index;
2974         uint64_t pool_mask;
2975         int ret;
2976
2977         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2978         dev = &rte_eth_devices[port_id];
2979         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_add, -ENOTSUP);
2980
2981         if (is_zero_ether_addr(addr)) {
2982                 RTE_ETHDEV_LOG(ERR, "Port %u: Cannot add NULL MAC address\n",
2983                         port_id);
2984                 return -EINVAL;
2985         }
2986         if (pool >= ETH_64_POOLS) {
2987                 RTE_ETHDEV_LOG(ERR, "Pool id must be 0-%d\n", ETH_64_POOLS - 1);
2988                 return -EINVAL;
2989         }
2990
2991         index = get_mac_addr_index(port_id, addr);
2992         if (index < 0) {
2993                 index = get_mac_addr_index(port_id, &null_mac_addr);
2994                 if (index < 0) {
2995                         RTE_ETHDEV_LOG(ERR, "Port %u: MAC address array full\n",
2996                                 port_id);
2997                         return -ENOSPC;
2998                 }
2999         } else {
3000                 pool_mask = dev->data->mac_pool_sel[index];
3001
3002                 /* Check if both MAC address and pool is already there, and do nothing */
3003                 if (pool_mask & (1ULL << pool))
3004                         return 0;
3005         }
3006
3007         /* Update NIC */
3008         ret = (*dev->dev_ops->mac_addr_add)(dev, addr, index, pool);
3009
3010         if (ret == 0) {
3011                 /* Update address in NIC data structure */
3012                 ether_addr_copy(addr, &dev->data->mac_addrs[index]);
3013
3014                 /* Update pool bitmap in NIC data structure */
3015                 dev->data->mac_pool_sel[index] |= (1ULL << pool);
3016         }
3017
3018         return eth_err(port_id, ret);
3019 }
3020
3021 int
3022 rte_eth_dev_mac_addr_remove(uint16_t port_id, struct ether_addr *addr)
3023 {
3024         struct rte_eth_dev *dev;
3025         int index;
3026
3027         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3028         dev = &rte_eth_devices[port_id];
3029         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_remove, -ENOTSUP);
3030
3031         index = get_mac_addr_index(port_id, addr);
3032         if (index == 0) {
3033                 RTE_ETHDEV_LOG(ERR,
3034                         "Port %u: Cannot remove default MAC address\n",
3035                         port_id);
3036                 return -EADDRINUSE;
3037         } else if (index < 0)
3038                 return 0;  /* Do nothing if address wasn't found */
3039
3040         /* Update NIC */
3041         (*dev->dev_ops->mac_addr_remove)(dev, index);
3042
3043         /* Update address in NIC data structure */
3044         ether_addr_copy(&null_mac_addr, &dev->data->mac_addrs[index]);
3045
3046         /* reset pool bitmap */
3047         dev->data->mac_pool_sel[index] = 0;
3048
3049         return 0;
3050 }
3051
3052 int
3053 rte_eth_dev_default_mac_addr_set(uint16_t port_id, struct ether_addr *addr)
3054 {
3055         struct rte_eth_dev *dev;
3056         int ret;
3057
3058         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3059
3060         if (!is_valid_assigned_ether_addr(addr))
3061                 return -EINVAL;
3062
3063         dev = &rte_eth_devices[port_id];
3064         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_set, -ENOTSUP);
3065
3066         ret = (*dev->dev_ops->mac_addr_set)(dev, addr);
3067         if (ret < 0)
3068                 return ret;
3069
3070         /* Update default address in NIC data structure */
3071         ether_addr_copy(addr, &dev->data->mac_addrs[0]);
3072
3073         return 0;
3074 }
3075
3076
3077 /*
3078  * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find
3079  * an empty spot.
3080  */
3081 static int
3082 get_hash_mac_addr_index(uint16_t port_id, const struct ether_addr *addr)
3083 {
3084         struct rte_eth_dev_info dev_info;
3085         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3086         unsigned i;
3087
3088         rte_eth_dev_info_get(port_id, &dev_info);
3089         if (!dev->data->hash_mac_addrs)
3090                 return -1;
3091
3092         for (i = 0; i < dev_info.max_hash_mac_addrs; i++)
3093                 if (memcmp(addr, &dev->data->hash_mac_addrs[i],
3094                         ETHER_ADDR_LEN) == 0)
3095                         return i;
3096
3097         return -1;
3098 }
3099
3100 int
3101 rte_eth_dev_uc_hash_table_set(uint16_t port_id, struct ether_addr *addr,
3102                                 uint8_t on)
3103 {
3104         int index;
3105         int ret;
3106         struct rte_eth_dev *dev;
3107
3108         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3109
3110         dev = &rte_eth_devices[port_id];
3111         if (is_zero_ether_addr(addr)) {
3112                 RTE_ETHDEV_LOG(ERR, "Port %u: Cannot add NULL MAC address\n",
3113                         port_id);
3114                 return -EINVAL;
3115         }
3116
3117         index = get_hash_mac_addr_index(port_id, addr);
3118         /* Check if it's already there, and do nothing */
3119         if ((index >= 0) && on)
3120                 return 0;
3121
3122         if (index < 0) {
3123                 if (!on) {
3124                         RTE_ETHDEV_LOG(ERR,
3125                                 "Port %u: the MAC address was not set in UTA\n",
3126                                 port_id);
3127                         return -EINVAL;
3128                 }
3129
3130                 index = get_hash_mac_addr_index(port_id, &null_mac_addr);
3131                 if (index < 0) {
3132                         RTE_ETHDEV_LOG(ERR, "Port %u: MAC address array full\n",
3133                                 port_id);
3134                         return -ENOSPC;
3135                 }
3136         }
3137
3138         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_hash_table_set, -ENOTSUP);
3139         ret = (*dev->dev_ops->uc_hash_table_set)(dev, addr, on);
3140         if (ret == 0) {
3141                 /* Update address in NIC data structure */
3142                 if (on)
3143                         ether_addr_copy(addr,
3144                                         &dev->data->hash_mac_addrs[index]);
3145                 else
3146                         ether_addr_copy(&null_mac_addr,
3147                                         &dev->data->hash_mac_addrs[index]);
3148         }
3149
3150         return eth_err(port_id, ret);
3151 }
3152
3153 int
3154 rte_eth_dev_uc_all_hash_table_set(uint16_t port_id, uint8_t on)
3155 {
3156         struct rte_eth_dev *dev;
3157
3158         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3159
3160         dev = &rte_eth_devices[port_id];
3161
3162         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_all_hash_table_set, -ENOTSUP);
3163         return eth_err(port_id, (*dev->dev_ops->uc_all_hash_table_set)(dev,
3164                                                                        on));
3165 }
3166
3167 int rte_eth_set_queue_rate_limit(uint16_t port_id, uint16_t queue_idx,
3168                                         uint16_t tx_rate)
3169 {
3170         struct rte_eth_dev *dev;
3171         struct rte_eth_dev_info dev_info;
3172         struct rte_eth_link link;
3173
3174         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3175
3176         dev = &rte_eth_devices[port_id];
3177         rte_eth_dev_info_get(port_id, &dev_info);
3178         link = dev->data->dev_link;
3179
3180         if (queue_idx > dev_info.max_tx_queues) {
3181                 RTE_ETHDEV_LOG(ERR,
3182                         "Set queue rate limit:port %u: invalid queue id=%u\n",
3183                         port_id, queue_idx);
3184                 return -EINVAL;
3185         }
3186
3187         if (tx_rate > link.link_speed) {
3188                 RTE_ETHDEV_LOG(ERR,
3189                         "Set queue rate limit:invalid tx_rate=%u, bigger than link speed= %d\n",
3190                         tx_rate, link.link_speed);
3191                 return -EINVAL;
3192         }
3193
3194         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_queue_rate_limit, -ENOTSUP);
3195         return eth_err(port_id, (*dev->dev_ops->set_queue_rate_limit)(dev,
3196                                                         queue_idx, tx_rate));
3197 }
3198
3199 int
3200 rte_eth_mirror_rule_set(uint16_t port_id,
3201                         struct rte_eth_mirror_conf *mirror_conf,
3202                         uint8_t rule_id, uint8_t on)
3203 {
3204         struct rte_eth_dev *dev;
3205
3206         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3207         if (mirror_conf->rule_type == 0) {
3208                 RTE_ETHDEV_LOG(ERR, "Mirror rule type can not be 0\n");
3209                 return -EINVAL;
3210         }
3211
3212         if (mirror_conf->dst_pool >= ETH_64_POOLS) {
3213                 RTE_ETHDEV_LOG(ERR, "Invalid dst pool, pool id must be 0-%d\n",
3214                         ETH_64_POOLS - 1);
3215                 return -EINVAL;
3216         }
3217
3218         if ((mirror_conf->rule_type & (ETH_MIRROR_VIRTUAL_POOL_UP |
3219              ETH_MIRROR_VIRTUAL_POOL_DOWN)) &&
3220             (mirror_conf->pool_mask == 0)) {
3221                 RTE_ETHDEV_LOG(ERR,
3222                         "Invalid mirror pool, pool mask can not be 0\n");
3223                 return -EINVAL;
3224         }
3225
3226         if ((mirror_conf->rule_type & ETH_MIRROR_VLAN) &&
3227             mirror_conf->vlan.vlan_mask == 0) {
3228                 RTE_ETHDEV_LOG(ERR,
3229                         "Invalid vlan mask, vlan mask can not be 0\n");
3230                 return -EINVAL;
3231         }
3232
3233         dev = &rte_eth_devices[port_id];
3234         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mirror_rule_set, -ENOTSUP);
3235
3236         return eth_err(port_id, (*dev->dev_ops->mirror_rule_set)(dev,
3237                                                 mirror_conf, rule_id, on));
3238 }
3239
3240 int
3241 rte_eth_mirror_rule_reset(uint16_t port_id, uint8_t rule_id)
3242 {
3243         struct rte_eth_dev *dev;
3244
3245         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3246
3247         dev = &rte_eth_devices[port_id];
3248         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mirror_rule_reset, -ENOTSUP);
3249
3250         return eth_err(port_id, (*dev->dev_ops->mirror_rule_reset)(dev,
3251                                                                    rule_id));
3252 }
3253
3254 RTE_INIT(eth_dev_init_cb_lists)
3255 {
3256         int i;
3257
3258         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3259                 TAILQ_INIT(&rte_eth_devices[i].link_intr_cbs);
3260 }
3261
3262 int
3263 rte_eth_dev_callback_register(uint16_t port_id,
3264                         enum rte_eth_event_type event,
3265                         rte_eth_dev_cb_fn cb_fn, void *cb_arg)
3266 {
3267         struct rte_eth_dev *dev;
3268         struct rte_eth_dev_callback *user_cb;
3269         uint32_t next_port; /* size is 32-bit to prevent loop wrap-around */
3270         uint16_t last_port;
3271
3272         if (!cb_fn)
3273                 return -EINVAL;
3274
3275         if (!rte_eth_dev_is_valid_port(port_id) && port_id != RTE_ETH_ALL) {
3276                 RTE_ETHDEV_LOG(ERR, "Invalid port_id=%d\n", port_id);
3277                 return -EINVAL;
3278         }
3279
3280         if (port_id == RTE_ETH_ALL) {
3281                 next_port = 0;
3282                 last_port = RTE_MAX_ETHPORTS - 1;
3283         } else {
3284                 next_port = last_port = port_id;
3285         }
3286
3287         rte_spinlock_lock(&rte_eth_dev_cb_lock);
3288
3289         do {
3290                 dev = &rte_eth_devices[next_port];
3291
3292                 TAILQ_FOREACH(user_cb, &(dev->link_intr_cbs), next) {
3293                         if (user_cb->cb_fn == cb_fn &&
3294                                 user_cb->cb_arg == cb_arg &&
3295                                 user_cb->event == event) {
3296                                 break;
3297                         }
3298                 }
3299
3300                 /* create a new callback. */
3301                 if (user_cb == NULL) {
3302                         user_cb = rte_zmalloc("INTR_USER_CALLBACK",
3303                                 sizeof(struct rte_eth_dev_callback), 0);
3304                         if (user_cb != NULL) {
3305                                 user_cb->cb_fn = cb_fn;
3306                                 user_cb->cb_arg = cb_arg;
3307                                 user_cb->event = event;
3308                                 TAILQ_INSERT_TAIL(&(dev->link_intr_cbs),
3309                                                   user_cb, next);
3310                         } else {
3311                                 rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3312                                 rte_eth_dev_callback_unregister(port_id, event,
3313                                                                 cb_fn, cb_arg);
3314                                 return -ENOMEM;
3315                         }
3316
3317                 }
3318         } while (++next_port <= last_port);
3319
3320         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3321         return 0;
3322 }
3323
3324 int
3325 rte_eth_dev_callback_unregister(uint16_t port_id,
3326                         enum rte_eth_event_type event,
3327                         rte_eth_dev_cb_fn cb_fn, void *cb_arg)
3328 {
3329         int ret;
3330         struct rte_eth_dev *dev;
3331         struct rte_eth_dev_callback *cb, *next;
3332         uint32_t next_port; /* size is 32-bit to prevent loop wrap-around */
3333         uint16_t last_port;
3334
3335         if (!cb_fn)
3336                 return -EINVAL;
3337
3338         if (!rte_eth_dev_is_valid_port(port_id) && port_id != RTE_ETH_ALL) {
3339                 RTE_ETHDEV_LOG(ERR, "Invalid port_id=%d\n", port_id);
3340                 return -EINVAL;
3341         }
3342
3343         if (port_id == RTE_ETH_ALL) {
3344                 next_port = 0;
3345                 last_port = RTE_MAX_ETHPORTS - 1;
3346         } else {
3347                 next_port = last_port = port_id;
3348         }
3349
3350         rte_spinlock_lock(&rte_eth_dev_cb_lock);
3351
3352         do {
3353                 dev = &rte_eth_devices[next_port];
3354                 ret = 0;
3355                 for (cb = TAILQ_FIRST(&dev->link_intr_cbs); cb != NULL;
3356                      cb = next) {
3357
3358                         next = TAILQ_NEXT(cb, next);
3359
3360                         if (cb->cb_fn != cb_fn || cb->event != event ||
3361                             (cb->cb_arg != (void *)-1 && cb->cb_arg != cb_arg))
3362                                 continue;
3363
3364                         /*
3365                          * if this callback is not executing right now,
3366                          * then remove it.
3367                          */
3368                         if (cb->active == 0) {
3369                                 TAILQ_REMOVE(&(dev->link_intr_cbs), cb, next);
3370                                 rte_free(cb);
3371                         } else {
3372                                 ret = -EAGAIN;
3373                         }
3374                 }
3375         } while (++next_port <= last_port);
3376
3377         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3378         return ret;
3379 }
3380
3381 int
3382 _rte_eth_dev_callback_process(struct rte_eth_dev *dev,
3383         enum rte_eth_event_type event, void *ret_param)
3384 {
3385         struct rte_eth_dev_callback *cb_lst;
3386         struct rte_eth_dev_callback dev_cb;
3387         int rc = 0;
3388
3389         rte_spinlock_lock(&rte_eth_dev_cb_lock);
3390         TAILQ_FOREACH(cb_lst, &(dev->link_intr_cbs), next) {
3391                 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
3392                         continue;
3393                 dev_cb = *cb_lst;
3394                 cb_lst->active = 1;
3395                 if (ret_param != NULL)
3396                         dev_cb.ret_param = ret_param;
3397
3398                 rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3399                 rc = dev_cb.cb_fn(dev->data->port_id, dev_cb.event,
3400                                 dev_cb.cb_arg, dev_cb.ret_param);
3401                 rte_spinlock_lock(&rte_eth_dev_cb_lock);
3402                 cb_lst->active = 0;
3403         }
3404         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3405         return rc;
3406 }
3407
3408 void
3409 rte_eth_dev_probing_finish(struct rte_eth_dev *dev)
3410 {
3411         if (dev == NULL)
3412                 return;
3413
3414         _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_NEW, NULL);
3415
3416         dev->state = RTE_ETH_DEV_ATTACHED;
3417 }
3418
3419 int
3420 rte_eth_dev_rx_intr_ctl(uint16_t port_id, int epfd, int op, void *data)
3421 {
3422         uint32_t vec;
3423         struct rte_eth_dev *dev;
3424         struct rte_intr_handle *intr_handle;
3425         uint16_t qid;
3426         int rc;
3427
3428         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3429
3430         dev = &rte_eth_devices[port_id];
3431
3432         if (!dev->intr_handle) {
3433                 RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
3434                 return -ENOTSUP;
3435         }
3436
3437         intr_handle = dev->intr_handle;
3438         if (!intr_handle->intr_vec) {
3439                 RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
3440                 return -EPERM;
3441         }
3442
3443         for (qid = 0; qid < dev->data->nb_rx_queues; qid++) {
3444                 vec = intr_handle->intr_vec[qid];
3445                 rc = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
3446                 if (rc && rc != -EEXIST) {
3447                         RTE_ETHDEV_LOG(ERR,
3448                                 "p %u q %u rx ctl error op %d epfd %d vec %u\n",
3449                                 port_id, qid, op, epfd, vec);
3450                 }
3451         }
3452
3453         return 0;
3454 }
3455
3456 int __rte_experimental
3457 rte_eth_dev_rx_intr_ctl_q_get_fd(uint16_t port_id, uint16_t queue_id)
3458 {
3459         struct rte_intr_handle *intr_handle;
3460         struct rte_eth_dev *dev;
3461         unsigned int efd_idx;
3462         uint32_t vec;
3463         int fd;
3464
3465         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -1);
3466
3467         dev = &rte_eth_devices[port_id];
3468
3469         if (queue_id >= dev->data->nb_rx_queues) {
3470                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
3471                 return -1;
3472         }
3473
3474         if (!dev->intr_handle) {
3475                 RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
3476                 return -1;
3477         }
3478
3479         intr_handle = dev->intr_handle;
3480         if (!intr_handle->intr_vec) {
3481                 RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
3482                 return -1;
3483         }
3484
3485         vec = intr_handle->intr_vec[queue_id];
3486         efd_idx = (vec >= RTE_INTR_VEC_RXTX_OFFSET) ?
3487                 (vec - RTE_INTR_VEC_RXTX_OFFSET) : vec;
3488         fd = intr_handle->efds[efd_idx];
3489
3490         return fd;
3491 }
3492
3493 const struct rte_memzone *
3494 rte_eth_dma_zone_reserve(const struct rte_eth_dev *dev, const char *ring_name,
3495                          uint16_t queue_id, size_t size, unsigned align,
3496                          int socket_id)
3497 {
3498         char z_name[RTE_MEMZONE_NAMESIZE];
3499         const struct rte_memzone *mz;
3500
3501         snprintf(z_name, sizeof(z_name), "eth_p%d_q%d_%s",
3502                  dev->data->port_id, queue_id, ring_name);
3503
3504         mz = rte_memzone_lookup(z_name);
3505         if (mz)
3506                 return mz;
3507
3508         return rte_memzone_reserve_aligned(z_name, size, socket_id,
3509                         RTE_MEMZONE_IOVA_CONTIG, align);
3510 }
3511
3512 int __rte_experimental
3513 rte_eth_dev_create(struct rte_device *device, const char *name,
3514         size_t priv_data_size,
3515         ethdev_bus_specific_init ethdev_bus_specific_init,
3516         void *bus_init_params,
3517         ethdev_init_t ethdev_init, void *init_params)
3518 {
3519         struct rte_eth_dev *ethdev;
3520         int retval;
3521
3522         RTE_FUNC_PTR_OR_ERR_RET(*ethdev_init, -EINVAL);
3523
3524         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
3525                 ethdev = rte_eth_dev_allocate(name);
3526                 if (!ethdev)
3527                         return -ENODEV;
3528
3529                 if (priv_data_size) {
3530                         ethdev->data->dev_private = rte_zmalloc_socket(
3531                                 name, priv_data_size, RTE_CACHE_LINE_SIZE,
3532                                 device->numa_node);
3533
3534                         if (!ethdev->data->dev_private) {
3535                                 RTE_LOG(ERR, EAL, "failed to allocate private data");
3536                                 retval = -ENOMEM;
3537                                 goto probe_failed;
3538                         }
3539                 }
3540         } else {
3541                 ethdev = rte_eth_dev_attach_secondary(name);
3542                 if (!ethdev) {
3543                         RTE_LOG(ERR, EAL, "secondary process attach failed, "
3544                                 "ethdev doesn't exist");
3545                         return  -ENODEV;
3546                 }
3547         }
3548
3549         ethdev->device = device;
3550
3551         if (ethdev_bus_specific_init) {
3552                 retval = ethdev_bus_specific_init(ethdev, bus_init_params);
3553                 if (retval) {
3554                         RTE_LOG(ERR, EAL,
3555                                 "ethdev bus specific initialisation failed");
3556                         goto probe_failed;
3557                 }
3558         }
3559
3560         retval = ethdev_init(ethdev, init_params);
3561         if (retval) {
3562                 RTE_LOG(ERR, EAL, "ethdev initialisation failed");
3563                 goto probe_failed;
3564         }
3565
3566         rte_eth_dev_probing_finish(ethdev);
3567
3568         return retval;
3569
3570 probe_failed:
3571         rte_eth_dev_release_port(ethdev);
3572         return retval;
3573 }
3574
3575 int  __rte_experimental
3576 rte_eth_dev_destroy(struct rte_eth_dev *ethdev,
3577         ethdev_uninit_t ethdev_uninit)
3578 {
3579         int ret;
3580
3581         ethdev = rte_eth_dev_allocated(ethdev->data->name);
3582         if (!ethdev)
3583                 return -ENODEV;
3584
3585         RTE_FUNC_PTR_OR_ERR_RET(*ethdev_uninit, -EINVAL);
3586         if (ethdev_uninit) {
3587                 ret = ethdev_uninit(ethdev);
3588                 if (ret)
3589                         return ret;
3590         }
3591
3592         return rte_eth_dev_release_port(ethdev);
3593 }
3594
3595 int
3596 rte_eth_dev_rx_intr_ctl_q(uint16_t port_id, uint16_t queue_id,
3597                           int epfd, int op, void *data)
3598 {
3599         uint32_t vec;
3600         struct rte_eth_dev *dev;
3601         struct rte_intr_handle *intr_handle;
3602         int rc;
3603
3604         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3605
3606         dev = &rte_eth_devices[port_id];
3607         if (queue_id >= dev->data->nb_rx_queues) {
3608                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
3609                 return -EINVAL;
3610         }
3611
3612         if (!dev->intr_handle) {
3613                 RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
3614                 return -ENOTSUP;
3615         }
3616
3617         intr_handle = dev->intr_handle;
3618         if (!intr_handle->intr_vec) {
3619                 RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
3620                 return -EPERM;
3621         }
3622
3623         vec = intr_handle->intr_vec[queue_id];
3624         rc = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
3625         if (rc && rc != -EEXIST) {
3626                 RTE_ETHDEV_LOG(ERR,
3627                         "p %u q %u rx ctl error op %d epfd %d vec %u\n",
3628                         port_id, queue_id, op, epfd, vec);
3629                 return rc;
3630         }
3631
3632         return 0;
3633 }
3634
3635 int
3636 rte_eth_dev_rx_intr_enable(uint16_t port_id,
3637                            uint16_t queue_id)
3638 {
3639         struct rte_eth_dev *dev;
3640
3641         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3642
3643         dev = &rte_eth_devices[port_id];
3644
3645         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_intr_enable, -ENOTSUP);
3646         return eth_err(port_id, (*dev->dev_ops->rx_queue_intr_enable)(dev,
3647                                                                 queue_id));
3648 }
3649
3650 int
3651 rte_eth_dev_rx_intr_disable(uint16_t port_id,
3652                             uint16_t queue_id)
3653 {
3654         struct rte_eth_dev *dev;
3655
3656         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3657
3658         dev = &rte_eth_devices[port_id];
3659
3660         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_intr_disable, -ENOTSUP);
3661         return eth_err(port_id, (*dev->dev_ops->rx_queue_intr_disable)(dev,
3662                                                                 queue_id));
3663 }
3664
3665
3666 int
3667 rte_eth_dev_filter_supported(uint16_t port_id,
3668                              enum rte_filter_type filter_type)
3669 {
3670         struct rte_eth_dev *dev;
3671
3672         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3673
3674         dev = &rte_eth_devices[port_id];
3675         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->filter_ctrl, -ENOTSUP);
3676         return (*dev->dev_ops->filter_ctrl)(dev, filter_type,
3677                                 RTE_ETH_FILTER_NOP, NULL);
3678 }
3679
3680 int
3681 rte_eth_dev_filter_ctrl(uint16_t port_id, enum rte_filter_type filter_type,
3682                         enum rte_filter_op filter_op, void *arg)
3683 {
3684         struct rte_eth_dev *dev;
3685
3686         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3687
3688         dev = &rte_eth_devices[port_id];
3689         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->filter_ctrl, -ENOTSUP);
3690         return eth_err(port_id, (*dev->dev_ops->filter_ctrl)(dev, filter_type,
3691                                                              filter_op, arg));
3692 }
3693
3694 const struct rte_eth_rxtx_callback *
3695 rte_eth_add_rx_callback(uint16_t port_id, uint16_t queue_id,
3696                 rte_rx_callback_fn fn, void *user_param)
3697 {
3698 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3699         rte_errno = ENOTSUP;
3700         return NULL;
3701 #endif
3702         /* check input parameters */
3703         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
3704                     queue_id >= rte_eth_devices[port_id].data->nb_rx_queues) {
3705                 rte_errno = EINVAL;
3706                 return NULL;
3707         }
3708         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
3709
3710         if (cb == NULL) {
3711                 rte_errno = ENOMEM;
3712                 return NULL;
3713         }
3714
3715         cb->fn.rx = fn;
3716         cb->param = user_param;
3717
3718         rte_spinlock_lock(&rte_eth_rx_cb_lock);
3719         /* Add the callbacks in fifo order. */
3720         struct rte_eth_rxtx_callback *tail =
3721                 rte_eth_devices[port_id].post_rx_burst_cbs[queue_id];
3722
3723         if (!tail) {
3724                 rte_eth_devices[port_id].post_rx_burst_cbs[queue_id] = cb;
3725
3726         } else {
3727                 while (tail->next)
3728                         tail = tail->next;
3729                 tail->next = cb;
3730         }
3731         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
3732
3733         return cb;
3734 }
3735
3736 const struct rte_eth_rxtx_callback *
3737 rte_eth_add_first_rx_callback(uint16_t port_id, uint16_t queue_id,
3738                 rte_rx_callback_fn fn, void *user_param)
3739 {
3740 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3741         rte_errno = ENOTSUP;
3742         return NULL;
3743 #endif
3744         /* check input parameters */
3745         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
3746                 queue_id >= rte_eth_devices[port_id].data->nb_rx_queues) {
3747                 rte_errno = EINVAL;
3748                 return NULL;
3749         }
3750
3751         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
3752
3753         if (cb == NULL) {
3754                 rte_errno = ENOMEM;
3755                 return NULL;
3756         }
3757
3758         cb->fn.rx = fn;
3759         cb->param = user_param;
3760
3761         rte_spinlock_lock(&rte_eth_rx_cb_lock);
3762         /* Add the callbacks at fisrt position*/
3763         cb->next = rte_eth_devices[port_id].post_rx_burst_cbs[queue_id];
3764         rte_smp_wmb();
3765         rte_eth_devices[port_id].post_rx_burst_cbs[queue_id] = cb;
3766         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
3767
3768         return cb;
3769 }
3770
3771 const struct rte_eth_rxtx_callback *
3772 rte_eth_add_tx_callback(uint16_t port_id, uint16_t queue_id,
3773                 rte_tx_callback_fn fn, void *user_param)
3774 {
3775 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3776         rte_errno = ENOTSUP;
3777         return NULL;
3778 #endif
3779         /* check input parameters */
3780         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
3781                     queue_id >= rte_eth_devices[port_id].data->nb_tx_queues) {
3782                 rte_errno = EINVAL;
3783                 return NULL;
3784         }
3785
3786         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
3787
3788         if (cb == NULL) {
3789                 rte_errno = ENOMEM;
3790                 return NULL;
3791         }
3792
3793         cb->fn.tx = fn;
3794         cb->param = user_param;
3795
3796         rte_spinlock_lock(&rte_eth_tx_cb_lock);
3797         /* Add the callbacks in fifo order. */
3798         struct rte_eth_rxtx_callback *tail =
3799                 rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id];
3800
3801         if (!tail) {
3802                 rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id] = cb;
3803
3804         } else {
3805                 while (tail->next)
3806                         tail = tail->next;
3807                 tail->next = cb;
3808         }
3809         rte_spinlock_unlock(&rte_eth_tx_cb_lock);
3810
3811         return cb;
3812 }
3813
3814 int
3815 rte_eth_remove_rx_callback(uint16_t port_id, uint16_t queue_id,
3816                 const struct rte_eth_rxtx_callback *user_cb)
3817 {
3818 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3819         return -ENOTSUP;
3820 #endif
3821         /* Check input parameters. */
3822         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
3823         if (user_cb == NULL ||
3824                         queue_id >= rte_eth_devices[port_id].data->nb_rx_queues)
3825                 return -EINVAL;
3826
3827         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3828         struct rte_eth_rxtx_callback *cb;
3829         struct rte_eth_rxtx_callback **prev_cb;
3830         int ret = -EINVAL;
3831
3832         rte_spinlock_lock(&rte_eth_rx_cb_lock);
3833         prev_cb = &dev->post_rx_burst_cbs[queue_id];
3834         for (; *prev_cb != NULL; prev_cb = &cb->next) {
3835                 cb = *prev_cb;
3836                 if (cb == user_cb) {
3837                         /* Remove the user cb from the callback list. */
3838                         *prev_cb = cb->next;
3839                         ret = 0;
3840                         break;
3841                 }
3842         }
3843         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
3844
3845         return ret;
3846 }
3847
3848 int
3849 rte_eth_remove_tx_callback(uint16_t port_id, uint16_t queue_id,
3850                 const struct rte_eth_rxtx_callback *user_cb)
3851 {
3852 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
3853         return -ENOTSUP;
3854 #endif
3855         /* Check input parameters. */
3856         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
3857         if (user_cb == NULL ||
3858                         queue_id >= rte_eth_devices[port_id].data->nb_tx_queues)
3859                 return -EINVAL;
3860
3861         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3862         int ret = -EINVAL;
3863         struct rte_eth_rxtx_callback *cb;
3864         struct rte_eth_rxtx_callback **prev_cb;
3865
3866         rte_spinlock_lock(&rte_eth_tx_cb_lock);
3867         prev_cb = &dev->pre_tx_burst_cbs[queue_id];
3868         for (; *prev_cb != NULL; prev_cb = &cb->next) {
3869                 cb = *prev_cb;
3870                 if (cb == user_cb) {
3871                         /* Remove the user cb from the callback list. */
3872                         *prev_cb = cb->next;
3873                         ret = 0;
3874                         break;
3875                 }
3876         }
3877         rte_spinlock_unlock(&rte_eth_tx_cb_lock);
3878
3879         return ret;
3880 }
3881
3882 int
3883 rte_eth_rx_queue_info_get(uint16_t port_id, uint16_t queue_id,
3884         struct rte_eth_rxq_info *qinfo)
3885 {
3886         struct rte_eth_dev *dev;
3887
3888         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3889
3890         if (qinfo == NULL)
3891                 return -EINVAL;
3892
3893         dev = &rte_eth_devices[port_id];
3894         if (queue_id >= dev->data->nb_rx_queues) {
3895                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
3896                 return -EINVAL;
3897         }
3898
3899         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rxq_info_get, -ENOTSUP);
3900
3901         memset(qinfo, 0, sizeof(*qinfo));
3902         dev->dev_ops->rxq_info_get(dev, queue_id, qinfo);
3903         return 0;
3904 }
3905
3906 int
3907 rte_eth_tx_queue_info_get(uint16_t port_id, uint16_t queue_id,
3908         struct rte_eth_txq_info *qinfo)
3909 {
3910         struct rte_eth_dev *dev;
3911
3912         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3913
3914         if (qinfo == NULL)
3915                 return -EINVAL;
3916
3917         dev = &rte_eth_devices[port_id];
3918         if (queue_id >= dev->data->nb_tx_queues) {
3919                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", queue_id);
3920                 return -EINVAL;
3921         }
3922
3923         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->txq_info_get, -ENOTSUP);
3924
3925         memset(qinfo, 0, sizeof(*qinfo));
3926         dev->dev_ops->txq_info_get(dev, queue_id, qinfo);
3927
3928         return 0;
3929 }
3930
3931 int
3932 rte_eth_dev_set_mc_addr_list(uint16_t port_id,
3933                              struct ether_addr *mc_addr_set,
3934                              uint32_t nb_mc_addr)
3935 {
3936         struct rte_eth_dev *dev;
3937
3938         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3939
3940         dev = &rte_eth_devices[port_id];
3941         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_mc_addr_list, -ENOTSUP);
3942         return eth_err(port_id, dev->dev_ops->set_mc_addr_list(dev,
3943                                                 mc_addr_set, nb_mc_addr));
3944 }
3945
3946 int
3947 rte_eth_timesync_enable(uint16_t port_id)
3948 {
3949         struct rte_eth_dev *dev;
3950
3951         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3952         dev = &rte_eth_devices[port_id];
3953
3954         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_enable, -ENOTSUP);
3955         return eth_err(port_id, (*dev->dev_ops->timesync_enable)(dev));
3956 }
3957
3958 int
3959 rte_eth_timesync_disable(uint16_t port_id)
3960 {
3961         struct rte_eth_dev *dev;
3962
3963         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3964         dev = &rte_eth_devices[port_id];
3965
3966         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_disable, -ENOTSUP);
3967         return eth_err(port_id, (*dev->dev_ops->timesync_disable)(dev));
3968 }
3969
3970 int
3971 rte_eth_timesync_read_rx_timestamp(uint16_t port_id, struct timespec *timestamp,
3972                                    uint32_t flags)
3973 {
3974         struct rte_eth_dev *dev;
3975
3976         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3977         dev = &rte_eth_devices[port_id];
3978
3979         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_rx_timestamp, -ENOTSUP);
3980         return eth_err(port_id, (*dev->dev_ops->timesync_read_rx_timestamp)
3981                                 (dev, timestamp, flags));
3982 }
3983
3984 int
3985 rte_eth_timesync_read_tx_timestamp(uint16_t port_id,
3986                                    struct timespec *timestamp)
3987 {
3988         struct rte_eth_dev *dev;
3989
3990         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3991         dev = &rte_eth_devices[port_id];
3992
3993         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_tx_timestamp, -ENOTSUP);
3994         return eth_err(port_id, (*dev->dev_ops->timesync_read_tx_timestamp)
3995                                 (dev, timestamp));
3996 }
3997
3998 int
3999 rte_eth_timesync_adjust_time(uint16_t port_id, int64_t delta)
4000 {
4001         struct rte_eth_dev *dev;
4002
4003         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4004         dev = &rte_eth_devices[port_id];
4005
4006         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_adjust_time, -ENOTSUP);
4007         return eth_err(port_id, (*dev->dev_ops->timesync_adjust_time)(dev,
4008                                                                       delta));
4009 }
4010
4011 int
4012 rte_eth_timesync_read_time(uint16_t port_id, struct timespec *timestamp)
4013 {
4014         struct rte_eth_dev *dev;
4015
4016         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4017         dev = &rte_eth_devices[port_id];
4018
4019         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_time, -ENOTSUP);
4020         return eth_err(port_id, (*dev->dev_ops->timesync_read_time)(dev,
4021                                                                 timestamp));
4022 }
4023
4024 int
4025 rte_eth_timesync_write_time(uint16_t port_id, const struct timespec *timestamp)
4026 {
4027         struct rte_eth_dev *dev;
4028
4029         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4030         dev = &rte_eth_devices[port_id];
4031
4032         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_write_time, -ENOTSUP);
4033         return eth_err(port_id, (*dev->dev_ops->timesync_write_time)(dev,
4034                                                                 timestamp));
4035 }
4036
4037 int
4038 rte_eth_dev_get_reg_info(uint16_t port_id, struct rte_dev_reg_info *info)
4039 {
4040         struct rte_eth_dev *dev;
4041
4042         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4043
4044         dev = &rte_eth_devices[port_id];
4045         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_reg, -ENOTSUP);
4046         return eth_err(port_id, (*dev->dev_ops->get_reg)(dev, info));
4047 }
4048
4049 int
4050 rte_eth_dev_get_eeprom_length(uint16_t port_id)
4051 {
4052         struct rte_eth_dev *dev;
4053
4054         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4055
4056         dev = &rte_eth_devices[port_id];
4057         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom_length, -ENOTSUP);
4058         return eth_err(port_id, (*dev->dev_ops->get_eeprom_length)(dev));
4059 }
4060
4061 int
4062 rte_eth_dev_get_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info)
4063 {
4064         struct rte_eth_dev *dev;
4065
4066         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4067
4068         dev = &rte_eth_devices[port_id];
4069         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom, -ENOTSUP);
4070         return eth_err(port_id, (*dev->dev_ops->get_eeprom)(dev, info));
4071 }
4072
4073 int
4074 rte_eth_dev_set_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info)
4075 {
4076         struct rte_eth_dev *dev;
4077
4078         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4079
4080         dev = &rte_eth_devices[port_id];
4081         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_eeprom, -ENOTSUP);
4082         return eth_err(port_id, (*dev->dev_ops->set_eeprom)(dev, info));
4083 }
4084
4085 int __rte_experimental
4086 rte_eth_dev_get_module_info(uint16_t port_id,
4087                             struct rte_eth_dev_module_info *modinfo)
4088 {
4089         struct rte_eth_dev *dev;
4090
4091         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4092
4093         dev = &rte_eth_devices[port_id];
4094         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_module_info, -ENOTSUP);
4095         return (*dev->dev_ops->get_module_info)(dev, modinfo);
4096 }
4097
4098 int __rte_experimental
4099 rte_eth_dev_get_module_eeprom(uint16_t port_id,
4100                               struct rte_dev_eeprom_info *info)
4101 {
4102         struct rte_eth_dev *dev;
4103
4104         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4105
4106         dev = &rte_eth_devices[port_id];
4107         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_module_eeprom, -ENOTSUP);
4108         return (*dev->dev_ops->get_module_eeprom)(dev, info);
4109 }
4110
4111 int
4112 rte_eth_dev_get_dcb_info(uint16_t port_id,
4113                              struct rte_eth_dcb_info *dcb_info)
4114 {
4115         struct rte_eth_dev *dev;
4116
4117         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4118
4119         dev = &rte_eth_devices[port_id];
4120         memset(dcb_info, 0, sizeof(struct rte_eth_dcb_info));
4121
4122         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_dcb_info, -ENOTSUP);
4123         return eth_err(port_id, (*dev->dev_ops->get_dcb_info)(dev, dcb_info));
4124 }
4125
4126 int
4127 rte_eth_dev_l2_tunnel_eth_type_conf(uint16_t port_id,
4128                                     struct rte_eth_l2_tunnel_conf *l2_tunnel)
4129 {
4130         struct rte_eth_dev *dev;
4131
4132         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4133         if (l2_tunnel == NULL) {
4134                 RTE_ETHDEV_LOG(ERR, "Invalid l2_tunnel parameter\n");
4135                 return -EINVAL;
4136         }
4137
4138         if (l2_tunnel->l2_tunnel_type >= RTE_TUNNEL_TYPE_MAX) {
4139                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
4140                 return -EINVAL;
4141         }
4142
4143         dev = &rte_eth_devices[port_id];
4144         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->l2_tunnel_eth_type_conf,
4145                                 -ENOTSUP);
4146         return eth_err(port_id, (*dev->dev_ops->l2_tunnel_eth_type_conf)(dev,
4147                                                                 l2_tunnel));
4148 }
4149
4150 int
4151 rte_eth_dev_l2_tunnel_offload_set(uint16_t port_id,
4152                                   struct rte_eth_l2_tunnel_conf *l2_tunnel,
4153                                   uint32_t mask,
4154                                   uint8_t en)
4155 {
4156         struct rte_eth_dev *dev;
4157
4158         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4159
4160         if (l2_tunnel == NULL) {
4161                 RTE_ETHDEV_LOG(ERR, "Invalid l2_tunnel parameter\n");
4162                 return -EINVAL;
4163         }
4164
4165         if (l2_tunnel->l2_tunnel_type >= RTE_TUNNEL_TYPE_MAX) {
4166                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
4167                 return -EINVAL;
4168         }
4169
4170         if (mask == 0) {
4171                 RTE_ETHDEV_LOG(ERR, "Mask should have a value\n");
4172                 return -EINVAL;
4173         }
4174
4175         dev = &rte_eth_devices[port_id];
4176         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->l2_tunnel_offload_set,
4177                                 -ENOTSUP);
4178         return eth_err(port_id, (*dev->dev_ops->l2_tunnel_offload_set)(dev,
4179                                                         l2_tunnel, mask, en));
4180 }
4181
4182 static void
4183 rte_eth_dev_adjust_nb_desc(uint16_t *nb_desc,
4184                            const struct rte_eth_desc_lim *desc_lim)
4185 {
4186         if (desc_lim->nb_align != 0)
4187                 *nb_desc = RTE_ALIGN_CEIL(*nb_desc, desc_lim->nb_align);
4188
4189         if (desc_lim->nb_max != 0)
4190                 *nb_desc = RTE_MIN(*nb_desc, desc_lim->nb_max);
4191
4192         *nb_desc = RTE_MAX(*nb_desc, desc_lim->nb_min);
4193 }
4194
4195 int
4196 rte_eth_dev_adjust_nb_rx_tx_desc(uint16_t port_id,
4197                                  uint16_t *nb_rx_desc,
4198                                  uint16_t *nb_tx_desc)
4199 {
4200         struct rte_eth_dev *dev;
4201         struct rte_eth_dev_info dev_info;
4202
4203         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4204
4205         dev = &rte_eth_devices[port_id];
4206         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
4207
4208         rte_eth_dev_info_get(port_id, &dev_info);
4209
4210         if (nb_rx_desc != NULL)
4211                 rte_eth_dev_adjust_nb_desc(nb_rx_desc, &dev_info.rx_desc_lim);
4212
4213         if (nb_tx_desc != NULL)
4214                 rte_eth_dev_adjust_nb_desc(nb_tx_desc, &dev_info.tx_desc_lim);
4215
4216         return 0;
4217 }
4218
4219 int
4220 rte_eth_dev_pool_ops_supported(uint16_t port_id, const char *pool)
4221 {
4222         struct rte_eth_dev *dev;
4223
4224         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4225
4226         if (pool == NULL)
4227                 return -EINVAL;
4228
4229         dev = &rte_eth_devices[port_id];
4230
4231         if (*dev->dev_ops->pool_ops_supported == NULL)
4232                 return 1; /* all pools are supported */
4233
4234         return (*dev->dev_ops->pool_ops_supported)(dev, pool);
4235 }
4236
4237 /**
4238  * A set of values to describe the possible states of a switch domain.
4239  */
4240 enum rte_eth_switch_domain_state {
4241         RTE_ETH_SWITCH_DOMAIN_UNUSED = 0,
4242         RTE_ETH_SWITCH_DOMAIN_ALLOCATED
4243 };
4244
4245 /**
4246  * Array of switch domains available for allocation. Array is sized to
4247  * RTE_MAX_ETHPORTS elements as there cannot be more active switch domains than
4248  * ethdev ports in a single process.
4249  */
4250 struct rte_eth_dev_switch {
4251         enum rte_eth_switch_domain_state state;
4252 } rte_eth_switch_domains[RTE_MAX_ETHPORTS];
4253
4254 int __rte_experimental
4255 rte_eth_switch_domain_alloc(uint16_t *domain_id)
4256 {
4257         unsigned int i;
4258
4259         *domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
4260
4261         for (i = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID + 1;
4262                 i < RTE_MAX_ETHPORTS; i++) {
4263                 if (rte_eth_switch_domains[i].state ==
4264                         RTE_ETH_SWITCH_DOMAIN_UNUSED) {
4265                         rte_eth_switch_domains[i].state =
4266                                 RTE_ETH_SWITCH_DOMAIN_ALLOCATED;
4267                         *domain_id = i;
4268                         return 0;
4269                 }
4270         }
4271
4272         return -ENOSPC;
4273 }
4274
4275 int __rte_experimental
4276 rte_eth_switch_domain_free(uint16_t domain_id)
4277 {
4278         if (domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID ||
4279                 domain_id >= RTE_MAX_ETHPORTS)
4280                 return -EINVAL;
4281
4282         if (rte_eth_switch_domains[domain_id].state !=
4283                 RTE_ETH_SWITCH_DOMAIN_ALLOCATED)
4284                 return -EINVAL;
4285
4286         rte_eth_switch_domains[domain_id].state = RTE_ETH_SWITCH_DOMAIN_UNUSED;
4287
4288         return 0;
4289 }
4290
4291 typedef int (*rte_eth_devargs_callback_t)(char *str, void *data);
4292
4293 static int
4294 rte_eth_devargs_tokenise(struct rte_kvargs *arglist, const char *str_in)
4295 {
4296         int state;
4297         struct rte_kvargs_pair *pair;
4298         char *letter;
4299
4300         arglist->str = strdup(str_in);
4301         if (arglist->str == NULL)
4302                 return -ENOMEM;
4303
4304         letter = arglist->str;
4305         state = 0;
4306         arglist->count = 0;
4307         pair = &arglist->pairs[0];
4308         while (1) {
4309                 switch (state) {
4310                 case 0: /* Initial */
4311                         if (*letter == '=')
4312                                 return -EINVAL;
4313                         else if (*letter == '\0')
4314                                 return 0;
4315
4316                         state = 1;
4317                         pair->key = letter;
4318                         /* fall-thru */
4319
4320                 case 1: /* Parsing key */
4321                         if (*letter == '=') {
4322                                 *letter = '\0';
4323                                 pair->value = letter + 1;
4324                                 state = 2;
4325                         } else if (*letter == ',' || *letter == '\0')
4326                                 return -EINVAL;
4327                         break;
4328
4329
4330                 case 2: /* Parsing value */
4331                         if (*letter == '[')
4332                                 state = 3;
4333                         else if (*letter == ',') {
4334                                 *letter = '\0';
4335                                 arglist->count++;
4336                                 pair = &arglist->pairs[arglist->count];
4337                                 state = 0;
4338                         } else if (*letter == '\0') {
4339                                 letter--;
4340                                 arglist->count++;
4341                                 pair = &arglist->pairs[arglist->count];
4342                                 state = 0;
4343                         }
4344                         break;
4345
4346                 case 3: /* Parsing list */
4347                         if (*letter == ']')
4348                                 state = 2;
4349                         else if (*letter == '\0')
4350                                 return -EINVAL;
4351                         break;
4352                 }
4353                 letter++;
4354         }
4355 }
4356
4357 static int
4358 rte_eth_devargs_parse_list(char *str, rte_eth_devargs_callback_t callback,
4359         void *data)
4360 {
4361         char *str_start;
4362         int state;
4363         int result;
4364
4365         if (*str != '[')
4366                 /* Single element, not a list */
4367                 return callback(str, data);
4368
4369         /* Sanity check, then strip the brackets */
4370         str_start = &str[strlen(str) - 1];
4371         if (*str_start != ']') {
4372                 RTE_LOG(ERR, EAL, "(%s): List does not end with ']'", str);
4373                 return -EINVAL;
4374         }
4375         str++;
4376         *str_start = '\0';
4377
4378         /* Process list elements */
4379         state = 0;
4380         while (1) {
4381                 if (state == 0) {
4382                         if (*str == '\0')
4383                                 break;
4384                         if (*str != ',') {
4385                                 str_start = str;
4386                                 state = 1;
4387                         }
4388                 } else if (state == 1) {
4389                         if (*str == ',' || *str == '\0') {
4390                                 if (str > str_start) {
4391                                         /* Non-empty string fragment */
4392                                         *str = '\0';
4393                                         result = callback(str_start, data);
4394                                         if (result < 0)
4395                                                 return result;
4396                                 }
4397                                 state = 0;
4398                         }
4399                 }
4400                 str++;
4401         }
4402         return 0;
4403 }
4404
4405 static int
4406 rte_eth_devargs_process_range(char *str, uint16_t *list, uint16_t *len_list,
4407         const uint16_t max_list)
4408 {
4409         uint16_t lo, hi, val;
4410         int result;
4411
4412         result = sscanf(str, "%hu-%hu", &lo, &hi);
4413         if (result == 1) {
4414                 if (*len_list >= max_list)
4415                         return -ENOMEM;
4416                 list[(*len_list)++] = lo;
4417         } else if (result == 2) {
4418                 if (lo >= hi || lo > RTE_MAX_ETHPORTS || hi > RTE_MAX_ETHPORTS)
4419                         return -EINVAL;
4420                 for (val = lo; val <= hi; val++) {
4421                         if (*len_list >= max_list)
4422                                 return -ENOMEM;
4423                         list[(*len_list)++] = val;
4424                 }
4425         } else
4426                 return -EINVAL;
4427         return 0;
4428 }
4429
4430
4431 static int
4432 rte_eth_devargs_parse_representor_ports(char *str, void *data)
4433 {
4434         struct rte_eth_devargs *eth_da = data;
4435
4436         return rte_eth_devargs_process_range(str, eth_da->representor_ports,
4437                 &eth_da->nb_representor_ports, RTE_MAX_ETHPORTS);
4438 }
4439
4440 int __rte_experimental
4441 rte_eth_devargs_parse(const char *dargs, struct rte_eth_devargs *eth_da)
4442 {
4443         struct rte_kvargs args;
4444         struct rte_kvargs_pair *pair;
4445         unsigned int i;
4446         int result = 0;
4447
4448         memset(eth_da, 0, sizeof(*eth_da));
4449
4450         result = rte_eth_devargs_tokenise(&args, dargs);
4451         if (result < 0)
4452                 goto parse_cleanup;
4453
4454         for (i = 0; i < args.count; i++) {
4455                 pair = &args.pairs[i];
4456                 if (strcmp("representor", pair->key) == 0) {
4457                         result = rte_eth_devargs_parse_list(pair->value,
4458                                 rte_eth_devargs_parse_representor_ports,
4459                                 eth_da);
4460                         if (result < 0)
4461                                 goto parse_cleanup;
4462                 }
4463         }
4464
4465 parse_cleanup:
4466         if (args.str)
4467                 free(args.str);
4468
4469         return result;
4470 }
4471
4472 RTE_INIT(ethdev_init_log)
4473 {
4474         rte_eth_dev_logtype = rte_log_register("lib.ethdev");
4475         if (rte_eth_dev_logtype >= 0)
4476                 rte_log_set_level(rte_eth_dev_logtype, RTE_LOG_INFO);
4477 }