lib/ethdev/rte_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <ctype.h>
   6 #include <errno.h>
   7 #include <inttypes.h>
   8 #include <stdbool.h>
   9 #include <stdint.h>
  10 #include <stdlib.h>
  11 #include <string.h>
  12 #include <sys/queue.h>
  13
  14 #include <rte_byteorder.h>
  15 #include <rte_log.h>
  16 #include <rte_debug.h>
  17 #include <rte_interrupts.h>
  18 #include <rte_memory.h>
  19 #include <rte_memcpy.h>
  20 #include <rte_memzone.h>
  21 #include <rte_launch.h>
  22 #include <rte_eal.h>
  23 #include <rte_per_lcore.h>
  24 #include <rte_lcore.h>
  25 #include <rte_branch_prediction.h>
  26 #include <rte_common.h>
  27 #include <rte_mempool.h>
  28 #include <rte_malloc.h>
  29 #include <rte_mbuf.h>
  30 #include <rte_errno.h>
  31 #include <rte_spinlock.h>
  32 #include <rte_string_fns.h>
  33 #include <rte_kvargs.h>
  34 #include <rte_class.h>
  35 #include <rte_ether.h>
  36 #include <rte_telemetry.h>
  37
  38 #include "rte_ethdev_trace.h"
  39 #include "rte_ethdev.h"
  40 #include "ethdev_driver.h"
  41 #include "ethdev_profile.h"
  42 #include "ethdev_private.h"
  43
  44 static const char *MZ_RTE_ETH_DEV_DATA = "rte_eth_dev_data";
  45 struct rte_eth_dev rte_eth_devices[RTE_MAX_ETHPORTS];
  46
  47 /* public fast-path API */
  48 struct rte_eth_fp_ops rte_eth_fp_ops[RTE_MAX_ETHPORTS];
  49
  50 /* spinlock for eth device callbacks */
  51 static rte_spinlock_t eth_dev_cb_lock = RTE_SPINLOCK_INITIALIZER;
  52
  53 /* spinlock for add/remove Rx callbacks */
  54 static rte_spinlock_t eth_dev_rx_cb_lock = RTE_SPINLOCK_INITIALIZER;
  55
  56 /* spinlock for add/remove Tx callbacks */
  57 static rte_spinlock_t eth_dev_tx_cb_lock = RTE_SPINLOCK_INITIALIZER;
  58
  59 /* spinlock for shared data allocation */
  60 static rte_spinlock_t eth_dev_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
  61
  62 /* store statistics names and its offset in stats structure  */
  63 struct rte_eth_xstats_name_off {
  64         char name[RTE_ETH_XSTATS_NAME_SIZE];
  65         unsigned offset;
  66 };
  67
  68 /* Shared memory between primary and secondary processes. */
  69 static struct {
  70         uint64_t next_owner_id;
  71         rte_spinlock_t ownership_lock;
  72         struct rte_eth_dev_data data[RTE_MAX_ETHPORTS];
  73 } *eth_dev_shared_data;
  74
  75 static const struct rte_eth_xstats_name_off eth_dev_stats_strings[] = {
  76         {"rx_good_packets", offsetof(struct rte_eth_stats, ipackets)},
  77         {"tx_good_packets", offsetof(struct rte_eth_stats, opackets)},
  78         {"rx_good_bytes", offsetof(struct rte_eth_stats, ibytes)},
  79         {"tx_good_bytes", offsetof(struct rte_eth_stats, obytes)},
  80         {"rx_missed_errors", offsetof(struct rte_eth_stats, imissed)},
  81         {"rx_errors", offsetof(struct rte_eth_stats, ierrors)},
  82         {"tx_errors", offsetof(struct rte_eth_stats, oerrors)},
  83         {"rx_mbuf_allocation_errors", offsetof(struct rte_eth_stats,
  84                 rx_nombuf)},
  85 };
  86
  87 #define RTE_NB_STATS RTE_DIM(eth_dev_stats_strings)
  88
  89 static const struct rte_eth_xstats_name_off eth_dev_rxq_stats_strings[] = {
  90         {"packets", offsetof(struct rte_eth_stats, q_ipackets)},
  91         {"bytes", offsetof(struct rte_eth_stats, q_ibytes)},
  92         {"errors", offsetof(struct rte_eth_stats, q_errors)},
  93 };
  94
  95 #define RTE_NB_RXQ_STATS RTE_DIM(eth_dev_rxq_stats_strings)
  96
  97 static const struct rte_eth_xstats_name_off eth_dev_txq_stats_strings[] = {
  98         {"packets", offsetof(struct rte_eth_stats, q_opackets)},
  99         {"bytes", offsetof(struct rte_eth_stats, q_obytes)},
 100 };
 101 #define RTE_NB_TXQ_STATS RTE_DIM(eth_dev_txq_stats_strings)
 102
 103 #define RTE_RX_OFFLOAD_BIT2STR(_name)   \
 104         { DEV_RX_OFFLOAD_##_name, #_name }
 105
 106 #define RTE_ETH_RX_OFFLOAD_BIT2STR(_name)       \
 107         { RTE_ETH_RX_OFFLOAD_##_name, #_name }
 108
 109 static const struct {
 110         uint64_t offload;
 111         const char *name;
 112 } eth_dev_rx_offload_names[] = {
 113         RTE_RX_OFFLOAD_BIT2STR(VLAN_STRIP),
 114         RTE_RX_OFFLOAD_BIT2STR(IPV4_CKSUM),
 115         RTE_RX_OFFLOAD_BIT2STR(UDP_CKSUM),
 116         RTE_RX_OFFLOAD_BIT2STR(TCP_CKSUM),
 117         RTE_RX_OFFLOAD_BIT2STR(TCP_LRO),
 118         RTE_RX_OFFLOAD_BIT2STR(QINQ_STRIP),
 119         RTE_RX_OFFLOAD_BIT2STR(OUTER_IPV4_CKSUM),
 120         RTE_RX_OFFLOAD_BIT2STR(MACSEC_STRIP),
 121         RTE_RX_OFFLOAD_BIT2STR(HEADER_SPLIT),
 122         RTE_RX_OFFLOAD_BIT2STR(VLAN_FILTER),
 123         RTE_RX_OFFLOAD_BIT2STR(VLAN_EXTEND),
 124         RTE_RX_OFFLOAD_BIT2STR(SCATTER),
 125         RTE_RX_OFFLOAD_BIT2STR(TIMESTAMP),
 126         RTE_RX_OFFLOAD_BIT2STR(SECURITY),
 127         RTE_RX_OFFLOAD_BIT2STR(KEEP_CRC),
 128         RTE_RX_OFFLOAD_BIT2STR(SCTP_CKSUM),
 129         RTE_RX_OFFLOAD_BIT2STR(OUTER_UDP_CKSUM),
 130         RTE_RX_OFFLOAD_BIT2STR(RSS_HASH),
 131         RTE_ETH_RX_OFFLOAD_BIT2STR(BUFFER_SPLIT),
 132 };
 133
 134 #undef RTE_RX_OFFLOAD_BIT2STR
 135 #undef RTE_ETH_RX_OFFLOAD_BIT2STR
 136
 137 #define RTE_TX_OFFLOAD_BIT2STR(_name)   \
 138         { DEV_TX_OFFLOAD_##_name, #_name }
 139
 140 static const struct {
 141         uint64_t offload;
 142         const char *name;
 143 } eth_dev_tx_offload_names[] = {
 144         RTE_TX_OFFLOAD_BIT2STR(VLAN_INSERT),
 145         RTE_TX_OFFLOAD_BIT2STR(IPV4_CKSUM),
 146         RTE_TX_OFFLOAD_BIT2STR(UDP_CKSUM),
 147         RTE_TX_OFFLOAD_BIT2STR(TCP_CKSUM),
 148         RTE_TX_OFFLOAD_BIT2STR(SCTP_CKSUM),
 149         RTE_TX_OFFLOAD_BIT2STR(TCP_TSO),
 150         RTE_TX_OFFLOAD_BIT2STR(UDP_TSO),
 151         RTE_TX_OFFLOAD_BIT2STR(OUTER_IPV4_CKSUM),
 152         RTE_TX_OFFLOAD_BIT2STR(QINQ_INSERT),
 153         RTE_TX_OFFLOAD_BIT2STR(VXLAN_TNL_TSO),
 154         RTE_TX_OFFLOAD_BIT2STR(GRE_TNL_TSO),
 155         RTE_TX_OFFLOAD_BIT2STR(IPIP_TNL_TSO),
 156         RTE_TX_OFFLOAD_BIT2STR(GENEVE_TNL_TSO),
 157         RTE_TX_OFFLOAD_BIT2STR(MACSEC_INSERT),
 158         RTE_TX_OFFLOAD_BIT2STR(MT_LOCKFREE),
 159         RTE_TX_OFFLOAD_BIT2STR(MULTI_SEGS),
 160         RTE_TX_OFFLOAD_BIT2STR(MBUF_FAST_FREE),
 161         RTE_TX_OFFLOAD_BIT2STR(SECURITY),
 162         RTE_TX_OFFLOAD_BIT2STR(UDP_TNL_TSO),
 163         RTE_TX_OFFLOAD_BIT2STR(IP_TNL_TSO),
 164         RTE_TX_OFFLOAD_BIT2STR(OUTER_UDP_CKSUM),
 165         RTE_TX_OFFLOAD_BIT2STR(SEND_ON_TIMESTAMP),
 166 };
 167
 168 #undef RTE_TX_OFFLOAD_BIT2STR
 169
 170 static const struct {
 171         uint64_t offload;
 172         const char *name;
 173 } rte_eth_dev_capa_names[] = {
 174         {RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP, "RUNTIME_RX_QUEUE_SETUP"},
 175         {RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP, "RUNTIME_TX_QUEUE_SETUP"},
 176         {RTE_ETH_DEV_CAPA_RXQ_SHARE, "RXQ_SHARE"},
 177 };
 178
 179 /**
 180  * The user application callback description.
 181  *
 182  * It contains callback address to be registered by user application,
 183  * the pointer to the parameters for callback, and the event type.
 184  */
 185 struct rte_eth_dev_callback {
 186         TAILQ_ENTRY(rte_eth_dev_callback) next; /**< Callbacks list */
 187         rte_eth_dev_cb_fn cb_fn;                /**< Callback address */
 188         void *cb_arg;                           /**< Parameter for callback */
 189         void *ret_param;                        /**< Return parameter */
 190         enum rte_eth_event_type event;          /**< Interrupt event type */
 191         uint32_t active;                        /**< Callback is executing */
 192 };
 193
 194 enum {
 195         STAT_QMAP_TX = 0,
 196         STAT_QMAP_RX
 197 };
 198
 199 int
 200 rte_eth_iterator_init(struct rte_dev_iterator *iter, const char *devargs_str)
 201 {
 202         int ret;
 203         struct rte_devargs devargs;
 204         const char *bus_param_key;
 205         char *bus_str = NULL;
 206         char *cls_str = NULL;
 207         int str_size;
 208
 209         if (iter == NULL) {
 210                 RTE_ETHDEV_LOG(ERR, "Cannot initialize NULL iterator\n");
 211                 return -EINVAL;
 212         }
 213
 214         if (devargs_str == NULL) {
 215                 RTE_ETHDEV_LOG(ERR,
 216                         "Cannot initialize iterator from NULL device description string\n");
 217                 return -EINVAL;
 218         }
 219
 220         memset(iter, 0, sizeof(*iter));
 221         memset(&devargs, 0, sizeof(devargs));
 222
 223         /*
 224          * The devargs string may use various syntaxes:
 225          *   - 0000:08:00.0,representor=[1-3]
 226          *   - pci:0000:06:00.0,representor=[0,5]
 227          *   - class=eth,mac=00:11:22:33:44:55
 228          *   - bus=X,paramX=x/class=Y,paramY=y/driver=Z,paramZ=z
 229          */
 230
 231         /*
 232          * Handle pure class filter (i.e. without any bus-level argument),
 233          * from future new syntax.
 234          * rte_devargs_parse() is not yet supporting the new syntax,
 235          * that's why this simple case is temporarily parsed here.
 236          */
 237 #define iter_anybus_str "class=eth,"
 238         if (strncmp(devargs_str, iter_anybus_str,
 239                         strlen(iter_anybus_str)) == 0) {
 240                 iter->cls_str = devargs_str + strlen(iter_anybus_str);
 241                 goto end;
 242         }
 243
 244         /* Split bus, device and parameters. */
 245         ret = rte_devargs_parse(&devargs, devargs_str);
 246         if (ret != 0)
 247                 goto error;
 248
 249         /*
 250          * Assume parameters of old syntax can match only at ethdev level.
 251          * Extra parameters will be ignored, thanks to "+" prefix.
 252          */
 253         str_size = strlen(devargs.args) + 2;
 254         cls_str = malloc(str_size);
 255         if (cls_str == NULL) {
 256                 ret = -ENOMEM;
 257                 goto error;
 258         }
 259         ret = snprintf(cls_str, str_size, "+%s", devargs.args);
 260         if (ret != str_size - 1) {
 261                 ret = -EINVAL;
 262                 goto error;
 263         }
 264         iter->cls_str = cls_str;
 265
 266         iter->bus = devargs.bus;
 267         if (iter->bus->dev_iterate == NULL) {
 268                 ret = -ENOTSUP;
 269                 goto error;
 270         }
 271
 272         /* Convert bus args to new syntax for use with new API dev_iterate. */
 273         if ((strcmp(iter->bus->name, "vdev") == 0) ||
 274                 (strcmp(iter->bus->name, "fslmc") == 0) ||
 275                 (strcmp(iter->bus->name, "dpaa_bus") == 0)) {
 276                 bus_param_key = "name";
 277         } else if (strcmp(iter->bus->name, "pci") == 0) {
 278                 bus_param_key = "addr";
 279         } else {
 280                 ret = -ENOTSUP;
 281                 goto error;
 282         }
 283         str_size = strlen(bus_param_key) + strlen(devargs.name) + 2;
 284         bus_str = malloc(str_size);
 285         if (bus_str == NULL) {
 286                 ret = -ENOMEM;
 287                 goto error;
 288         }
 289         ret = snprintf(bus_str, str_size, "%s=%s",
 290                         bus_param_key, devargs.name);
 291         if (ret != str_size - 1) {
 292                 ret = -EINVAL;
 293                 goto error;
 294         }
 295         iter->bus_str = bus_str;
 296
 297 end:
 298         iter->cls = rte_class_find_by_name("eth");
 299         rte_devargs_reset(&devargs);
 300         return 0;
 301
 302 error:
 303         if (ret == -ENOTSUP)
 304                 RTE_ETHDEV_LOG(ERR, "Bus %s does not support iterating.\n",
 305                                 iter->bus->name);
 306         rte_devargs_reset(&devargs);
 307         free(bus_str);
 308         free(cls_str);
 309         return ret;
 310 }
 311
 312 uint16_t
 313 rte_eth_iterator_next(struct rte_dev_iterator *iter)
 314 {
 315         if (iter == NULL) {
 316                 RTE_ETHDEV_LOG(ERR,
 317                         "Cannot get next device from NULL iterator\n");
 318                 return RTE_MAX_ETHPORTS;
 319         }
 320
 321         if (iter->cls == NULL) /* invalid ethdev iterator */
 322                 return RTE_MAX_ETHPORTS;
 323
 324         do { /* loop to try all matching rte_device */
 325                 /* If not pure ethdev filter and */
 326                 if (iter->bus != NULL &&
 327                                 /* not in middle of rte_eth_dev iteration, */
 328                                 iter->class_device == NULL) {
 329                         /* get next rte_device to try. */
 330                         iter->device = iter->bus->dev_iterate(
 331                                         iter->device, iter->bus_str, iter);
 332                         if (iter->device == NULL)
 333                                 break; /* no more rte_device candidate */
 334                 }
 335                 /* A device is matching bus part, need to check ethdev part. */
 336                 iter->class_device = iter->cls->dev_iterate(
 337                                 iter->class_device, iter->cls_str, iter);
 338                 if (iter->class_device != NULL)
 339                         return eth_dev_to_id(iter->class_device); /* match */
 340         } while (iter->bus != NULL); /* need to try next rte_device */
 341
 342         /* No more ethdev port to iterate. */
 343         rte_eth_iterator_cleanup(iter);
 344         return RTE_MAX_ETHPORTS;
 345 }
 346
 347 void
 348 rte_eth_iterator_cleanup(struct rte_dev_iterator *iter)
 349 {
 350         if (iter == NULL) {
 351                 RTE_ETHDEV_LOG(ERR, "Cannot do clean up from NULL iterator\n");
 352                 return;
 353         }
 354
 355         if (iter->bus_str == NULL)
 356                 return; /* nothing to free in pure class filter */
 357         free(RTE_CAST_FIELD(iter, bus_str, char *)); /* workaround const */
 358         free(RTE_CAST_FIELD(iter, cls_str, char *)); /* workaround const */
 359         memset(iter, 0, sizeof(*iter));
 360 }
 361
 362 uint16_t
 363 rte_eth_find_next(uint16_t port_id)
 364 {
 365         while (port_id < RTE_MAX_ETHPORTS &&
 366                         rte_eth_devices[port_id].state == RTE_ETH_DEV_UNUSED)
 367                 port_id++;
 368
 369         if (port_id >= RTE_MAX_ETHPORTS)
 370                 return RTE_MAX_ETHPORTS;
 371
 372         return port_id;
 373 }
 374
 375 /*
 376  * Macro to iterate over all valid ports for internal usage.
 377  * Note: RTE_ETH_FOREACH_DEV is different because filtering owned ports.
 378  */
 379 #define RTE_ETH_FOREACH_VALID_DEV(port_id) \
 380         for (port_id = rte_eth_find_next(0); \
 381              port_id < RTE_MAX_ETHPORTS; \
 382              port_id = rte_eth_find_next(port_id + 1))
 383
 384 uint16_t
 385 rte_eth_find_next_of(uint16_t port_id, const struct rte_device *parent)
 386 {
 387         port_id = rte_eth_find_next(port_id);
 388         while (port_id < RTE_MAX_ETHPORTS &&
 389                         rte_eth_devices[port_id].device != parent)
 390                 port_id = rte_eth_find_next(port_id + 1);
 391
 392         return port_id;
 393 }
 394
 395 uint16_t
 396 rte_eth_find_next_sibling(uint16_t port_id, uint16_t ref_port_id)
 397 {
 398         RTE_ETH_VALID_PORTID_OR_ERR_RET(ref_port_id, RTE_MAX_ETHPORTS);
 399         return rte_eth_find_next_of(port_id,
 400                         rte_eth_devices[ref_port_id].device);
 401 }
 402
 403 static void
 404 eth_dev_shared_data_prepare(void)
 405 {
 406         const unsigned flags = 0;
 407         const struct rte_memzone *mz;
 408
 409         rte_spinlock_lock(&eth_dev_shared_data_lock);
 410
 411         if (eth_dev_shared_data == NULL) {
 412                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 413                         /* Allocate port data and ownership shared memory. */
 414                         mz = rte_memzone_reserve(MZ_RTE_ETH_DEV_DATA,
 415                                         sizeof(*eth_dev_shared_data),
 416                                         rte_socket_id(), flags);
 417                 } else
 418                         mz = rte_memzone_lookup(MZ_RTE_ETH_DEV_DATA);
 419                 if (mz == NULL)
 420                         rte_panic("Cannot allocate ethdev shared data\n");
 421
 422                 eth_dev_shared_data = mz->addr;
 423                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 424                         eth_dev_shared_data->next_owner_id =
 425                                         RTE_ETH_DEV_NO_OWNER + 1;
 426                         rte_spinlock_init(&eth_dev_shared_data->ownership_lock);
 427                         memset(eth_dev_shared_data->data, 0,
 428                                sizeof(eth_dev_shared_data->data));
 429                 }
 430         }
 431
 432         rte_spinlock_unlock(&eth_dev_shared_data_lock);
 433 }
 434
 435 static bool
 436 eth_dev_is_allocated(const struct rte_eth_dev *ethdev)
 437 {
 438         return ethdev->data->name[0] != '\0';
 439 }
 440
 441 static struct rte_eth_dev *
 442 eth_dev_allocated(const char *name)
 443 {
 444         uint16_t i;
 445
 446         RTE_BUILD_BUG_ON(RTE_MAX_ETHPORTS >= UINT16_MAX);
 447
 448         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 449                 if (rte_eth_devices[i].data != NULL &&
 450                     strcmp(rte_eth_devices[i].data->name, name) == 0)
 451                         return &rte_eth_devices[i];
 452         }
 453         return NULL;
 454 }
 455
 456 struct rte_eth_dev *
 457 rte_eth_dev_allocated(const char *name)
 458 {
 459         struct rte_eth_dev *ethdev;
 460
 461         eth_dev_shared_data_prepare();
 462
 463         rte_spinlock_lock(&eth_dev_shared_data->ownership_lock);
 464
 465         ethdev = eth_dev_allocated(name);
 466
 467         rte_spinlock_unlock(&eth_dev_shared_data->ownership_lock);
 468
 469         return ethdev;
 470 }
 471
 472 static uint16_t
 473 eth_dev_find_free_port(void)
 474 {
 475         uint16_t i;
 476
 477         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 478                 /* Using shared name field to find a free port. */
 479                 if (eth_dev_shared_data->data[i].name[0] == '\0') {
 480                         RTE_ASSERT(rte_eth_devices[i].state ==
 481                                    RTE_ETH_DEV_UNUSED);
 482                         return i;
 483                 }
 484         }
 485         return RTE_MAX_ETHPORTS;
 486 }
 487
 488 static struct rte_eth_dev *
 489 eth_dev_get(uint16_t port_id)
 490 {
 491         struct rte_eth_dev *eth_dev = &rte_eth_devices[port_id];
 492
 493         eth_dev->data = &eth_dev_shared_data->data[port_id];
 494
 495         return eth_dev;
 496 }
 497
 498 struct rte_eth_dev *
 499 rte_eth_dev_allocate(const char *name)
 500 {
 501         uint16_t port_id;
 502         struct rte_eth_dev *eth_dev = NULL;
 503         size_t name_len;
 504
 505         name_len = strnlen(name, RTE_ETH_NAME_MAX_LEN);
 506         if (name_len == 0) {
 507                 RTE_ETHDEV_LOG(ERR, "Zero length Ethernet device name\n");
 508                 return NULL;
 509         }
 510
 511         if (name_len >= RTE_ETH_NAME_MAX_LEN) {
 512                 RTE_ETHDEV_LOG(ERR, "Ethernet device name is too long\n");
 513                 return NULL;
 514         }
 515
 516         eth_dev_shared_data_prepare();
 517
 518         /* Synchronize port creation between primary and secondary threads. */
 519         rte_spinlock_lock(&eth_dev_shared_data->ownership_lock);
 520
 521         if (eth_dev_allocated(name) != NULL) {
 522                 RTE_ETHDEV_LOG(ERR,
 523                         "Ethernet device with name %s already allocated\n",
 524                         name);
 525                 goto unlock;
 526         }
 527
 528         port_id = eth_dev_find_free_port();
 529         if (port_id == RTE_MAX_ETHPORTS) {
 530                 RTE_ETHDEV_LOG(ERR,
 531                         "Reached maximum number of Ethernet ports\n");
 532                 goto unlock;
 533         }
 534
 535         eth_dev = eth_dev_get(port_id);
 536         strlcpy(eth_dev->data->name, name, sizeof(eth_dev->data->name));
 537         eth_dev->data->port_id = port_id;
 538         eth_dev->data->backer_port_id = RTE_MAX_ETHPORTS;
 539         eth_dev->data->mtu = RTE_ETHER_MTU;
 540         pthread_mutex_init(&eth_dev->data->flow_ops_mutex, NULL);
 541
 542 unlock:
 543         rte_spinlock_unlock(&eth_dev_shared_data->ownership_lock);
 544
 545         return eth_dev;
 546 }
 547
 548 /*
 549  * Attach to a port already registered by the primary process, which
 550  * makes sure that the same device would have the same port ID both
 551  * in the primary and secondary process.
 552  */
 553 struct rte_eth_dev *
 554 rte_eth_dev_attach_secondary(const char *name)
 555 {
 556         uint16_t i;
 557         struct rte_eth_dev *eth_dev = NULL;
 558
 559         eth_dev_shared_data_prepare();
 560
 561         /* Synchronize port attachment to primary port creation and release. */
 562         rte_spinlock_lock(&eth_dev_shared_data->ownership_lock);
 563
 564         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 565                 if (strcmp(eth_dev_shared_data->data[i].name, name) == 0)
 566                         break;
 567         }
 568         if (i == RTE_MAX_ETHPORTS) {
 569                 RTE_ETHDEV_LOG(ERR,
 570                         "Device %s is not driven by the primary process\n",
 571                         name);
 572         } else {
 573                 eth_dev = eth_dev_get(i);
 574                 RTE_ASSERT(eth_dev->data->port_id == i);
 575         }
 576
 577         rte_spinlock_unlock(&eth_dev_shared_data->ownership_lock);
 578         return eth_dev;
 579 }
 580
 581 int
 582 rte_eth_dev_release_port(struct rte_eth_dev *eth_dev)
 583 {
 584         if (eth_dev == NULL)
 585                 return -EINVAL;
 586
 587         eth_dev_shared_data_prepare();
 588
 589         if (eth_dev->state != RTE_ETH_DEV_UNUSED)
 590                 rte_eth_dev_callback_process(eth_dev,
 591                                 RTE_ETH_EVENT_DESTROY, NULL);
 592
 593         eth_dev_fp_ops_reset(rte_eth_fp_ops + eth_dev->data->port_id);
 594
 595         rte_spinlock_lock(&eth_dev_shared_data->ownership_lock);
 596
 597         eth_dev->state = RTE_ETH_DEV_UNUSED;
 598         eth_dev->device = NULL;
 599         eth_dev->process_private = NULL;
 600         eth_dev->intr_handle = NULL;
 601         eth_dev->rx_pkt_burst = NULL;
 602         eth_dev->tx_pkt_burst = NULL;
 603         eth_dev->tx_pkt_prepare = NULL;
 604         eth_dev->rx_queue_count = NULL;
 605         eth_dev->rx_descriptor_status = NULL;
 606         eth_dev->tx_descriptor_status = NULL;
 607         eth_dev->dev_ops = NULL;
 608
 609         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 610                 rte_free(eth_dev->data->rx_queues);
 611                 rte_free(eth_dev->data->tx_queues);
 612                 rte_free(eth_dev->data->mac_addrs);
 613                 rte_free(eth_dev->data->hash_mac_addrs);
 614                 rte_free(eth_dev->data->dev_private);
 615                 pthread_mutex_destroy(&eth_dev->data->flow_ops_mutex);
 616                 memset(eth_dev->data, 0, sizeof(struct rte_eth_dev_data));
 617         }
 618
 619         rte_spinlock_unlock(&eth_dev_shared_data->ownership_lock);
 620
 621         return 0;
 622 }
 623
 624 int
 625 rte_eth_dev_is_valid_port(uint16_t port_id)
 626 {
 627         if (port_id >= RTE_MAX_ETHPORTS ||
 628             (rte_eth_devices[port_id].state == RTE_ETH_DEV_UNUSED))
 629                 return 0;
 630         else
 631                 return 1;
 632 }
 633
 634 static int
 635 eth_is_valid_owner_id(uint64_t owner_id)
 636 {
 637         if (owner_id == RTE_ETH_DEV_NO_OWNER ||
 638             eth_dev_shared_data->next_owner_id <= owner_id)
 639                 return 0;
 640         return 1;
 641 }
 642
 643 uint64_t
 644 rte_eth_find_next_owned_by(uint16_t port_id, const uint64_t owner_id)
 645 {
 646         port_id = rte_eth_find_next(port_id);
 647         while (port_id < RTE_MAX_ETHPORTS &&
 648                         rte_eth_devices[port_id].data->owner.id != owner_id)
 649                 port_id = rte_eth_find_next(port_id + 1);
 650
 651         return port_id;
 652 }
 653
 654 int
 655 rte_eth_dev_owner_new(uint64_t *owner_id)
 656 {
 657         if (owner_id == NULL) {
 658                 RTE_ETHDEV_LOG(ERR, "Cannot get new owner ID to NULL\n");
 659                 return -EINVAL;
 660         }
 661
 662         eth_dev_shared_data_prepare();
 663
 664         rte_spinlock_lock(&eth_dev_shared_data->ownership_lock);
 665
 666         *owner_id = eth_dev_shared_data->next_owner_id++;
 667
 668         rte_spinlock_unlock(&eth_dev_shared_data->ownership_lock);
 669         return 0;
 670 }
 671
 672 static int
 673 eth_dev_owner_set(const uint16_t port_id, const uint64_t old_owner_id,
 674                        const struct rte_eth_dev_owner *new_owner)
 675 {
 676         struct rte_eth_dev *ethdev = &rte_eth_devices[port_id];
 677         struct rte_eth_dev_owner *port_owner;
 678
 679         if (port_id >= RTE_MAX_ETHPORTS || !eth_dev_is_allocated(ethdev)) {
 680                 RTE_ETHDEV_LOG(ERR, "Port ID %"PRIu16" is not allocated\n",
 681                         port_id);
 682                 return -ENODEV;
 683         }
 684
 685         if (new_owner == NULL) {
 686                 RTE_ETHDEV_LOG(ERR,
 687                         "Cannot set ethdev port %u owner from NULL owner\n",
 688                         port_id);
 689                 return -EINVAL;
 690         }
 691
 692         if (!eth_is_valid_owner_id(new_owner->id) &&
 693             !eth_is_valid_owner_id(old_owner_id)) {
 694                 RTE_ETHDEV_LOG(ERR,
 695                         "Invalid owner old_id=%016"PRIx64" new_id=%016"PRIx64"\n",
 696                        old_owner_id, new_owner->id);
 697                 return -EINVAL;
 698         }
 699
 700         port_owner = &rte_eth_devices[port_id].data->owner;
 701         if (port_owner->id != old_owner_id) {
 702                 RTE_ETHDEV_LOG(ERR,
 703                         "Cannot set owner to port %u already owned by %s_%016"PRIX64"\n",
 704                         port_id, port_owner->name, port_owner->id);
 705                 return -EPERM;
 706         }
 707
 708         /* can not truncate (same structure) */
 709         strlcpy(port_owner->name, new_owner->name, RTE_ETH_MAX_OWNER_NAME_LEN);
 710
 711         port_owner->id = new_owner->id;
 712
 713         RTE_ETHDEV_LOG(DEBUG, "Port %u owner is %s_%016"PRIx64"\n",
 714                 port_id, new_owner->name, new_owner->id);
 715
 716         return 0;
 717 }
 718
 719 int
 720 rte_eth_dev_owner_set(const uint16_t port_id,
 721                       const struct rte_eth_dev_owner *owner)
 722 {
 723         int ret;
 724
 725         eth_dev_shared_data_prepare();
 726
 727         rte_spinlock_lock(&eth_dev_shared_data->ownership_lock);
 728
 729         ret = eth_dev_owner_set(port_id, RTE_ETH_DEV_NO_OWNER, owner);
 730
 731         rte_spinlock_unlock(&eth_dev_shared_data->ownership_lock);
 732         return ret;
 733 }
 734
 735 int
 736 rte_eth_dev_owner_unset(const uint16_t port_id, const uint64_t owner_id)
 737 {
 738         const struct rte_eth_dev_owner new_owner = (struct rte_eth_dev_owner)
 739                         {.id = RTE_ETH_DEV_NO_OWNER, .name = ""};
 740         int ret;
 741
 742         eth_dev_shared_data_prepare();
 743
 744         rte_spinlock_lock(&eth_dev_shared_data->ownership_lock);
 745
 746         ret = eth_dev_owner_set(port_id, owner_id, &new_owner);
 747
 748         rte_spinlock_unlock(&eth_dev_shared_data->ownership_lock);
 749         return ret;
 750 }
 751
 752 int
 753 rte_eth_dev_owner_delete(const uint64_t owner_id)
 754 {
 755         uint16_t port_id;
 756         int ret = 0;
 757
 758         eth_dev_shared_data_prepare();
 759
 760         rte_spinlock_lock(&eth_dev_shared_data->ownership_lock);
 761
 762         if (eth_is_valid_owner_id(owner_id)) {
 763                 for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++)
 764                         if (rte_eth_devices[port_id].data->owner.id == owner_id)
 765                                 memset(&rte_eth_devices[port_id].data->owner, 0,
 766                                        sizeof(struct rte_eth_dev_owner));
 767                 RTE_ETHDEV_LOG(NOTICE,
 768                         "All port owners owned by %016"PRIx64" identifier have removed\n",
 769                         owner_id);
 770         } else {
 771                 RTE_ETHDEV_LOG(ERR,
 772                                "Invalid owner ID=%016"PRIx64"\n",
 773                                owner_id);
 774                 ret = -EINVAL;
 775         }
 776
 777         rte_spinlock_unlock(&eth_dev_shared_data->ownership_lock);
 778
 779         return ret;
 780 }
 781
 782 int
 783 rte_eth_dev_owner_get(const uint16_t port_id, struct rte_eth_dev_owner *owner)
 784 {
 785         struct rte_eth_dev *ethdev;
 786
 787         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
 788         ethdev = &rte_eth_devices[port_id];
 789
 790         if (!eth_dev_is_allocated(ethdev)) {
 791                 RTE_ETHDEV_LOG(ERR, "Port ID %"PRIu16" is not allocated\n",
 792                         port_id);
 793                 return -ENODEV;
 794         }
 795
 796         if (owner == NULL) {
 797                 RTE_ETHDEV_LOG(ERR, "Cannot get ethdev port %u owner to NULL\n",
 798                         port_id);
 799                 return -EINVAL;
 800         }
 801
 802         eth_dev_shared_data_prepare();
 803
 804         rte_spinlock_lock(&eth_dev_shared_data->ownership_lock);
 805         rte_memcpy(owner, &ethdev->data->owner, sizeof(*owner));
 806         rte_spinlock_unlock(&eth_dev_shared_data->ownership_lock);
 807
 808         return 0;
 809 }
 810
 811 int
 812 rte_eth_dev_socket_id(uint16_t port_id)
 813 {
 814         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -1);
 815         return rte_eth_devices[port_id].data->numa_node;
 816 }
 817
 818 void *
 819 rte_eth_dev_get_sec_ctx(uint16_t port_id)
 820 {
 821         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, NULL);
 822         return rte_eth_devices[port_id].security_ctx;
 823 }
 824
 825 uint16_t
 826 rte_eth_dev_count_avail(void)
 827 {
 828         uint16_t p;
 829         uint16_t count;
 830
 831         count = 0;
 832
 833         RTE_ETH_FOREACH_DEV(p)
 834                 count++;
 835
 836         return count;
 837 }
 838
 839 uint16_t
 840 rte_eth_dev_count_total(void)
 841 {
 842         uint16_t port, count = 0;
 843
 844         RTE_ETH_FOREACH_VALID_DEV(port)
 845                 count++;
 846
 847         return count;
 848 }
 849
 850 int
 851 rte_eth_dev_get_name_by_port(uint16_t port_id, char *name)
 852 {
 853         char *tmp;
 854
 855         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
 856
 857         if (name == NULL) {
 858                 RTE_ETHDEV_LOG(ERR, "Cannot get ethdev port %u name to NULL\n",
 859                         port_id);
 860                 return -EINVAL;
 861         }
 862
 863         /* shouldn't check 'rte_eth_devices[i].data',
 864          * because it might be overwritten by VDEV PMD */
 865         tmp = eth_dev_shared_data->data[port_id].name;
 866         strcpy(name, tmp);
 867         return 0;
 868 }
 869
 870 int
 871 rte_eth_dev_get_port_by_name(const char *name, uint16_t *port_id)
 872 {
 873         uint16_t pid;
 874
 875         if (name == NULL) {
 876                 RTE_ETHDEV_LOG(ERR, "Cannot get port ID from NULL name");
 877                 return -EINVAL;
 878         }
 879
 880         if (port_id == NULL) {
 881                 RTE_ETHDEV_LOG(ERR,
 882                         "Cannot get port ID to NULL for %s\n", name);
 883                 return -EINVAL;
 884         }
 885
 886         RTE_ETH_FOREACH_VALID_DEV(pid)
 887                 if (!strcmp(name, eth_dev_shared_data->data[pid].name)) {
 888                         *port_id = pid;
 889                         return 0;
 890                 }
 891
 892         return -ENODEV;
 893 }
 894
 895 static int
 896 eth_err(uint16_t port_id, int ret)
 897 {
 898         if (ret == 0)
 899                 return 0;
 900         if (rte_eth_dev_is_removed(port_id))
 901                 return -EIO;
 902         return ret;
 903 }
 904
 905 static void
 906 eth_dev_rxq_release(struct rte_eth_dev *dev, uint16_t qid)
 907 {
 908         void **rxq = dev->data->rx_queues;
 909
 910         if (rxq[qid] == NULL)
 911                 return;
 912
 913         if (dev->dev_ops->rx_queue_release != NULL)
 914                 (*dev->dev_ops->rx_queue_release)(dev, qid);
 915         rxq[qid] = NULL;
 916 }
 917
 918 static void
 919 eth_dev_txq_release(struct rte_eth_dev *dev, uint16_t qid)
 920 {
 921         void **txq = dev->data->tx_queues;
 922
 923         if (txq[qid] == NULL)
 924                 return;
 925
 926         if (dev->dev_ops->tx_queue_release != NULL)
 927                 (*dev->dev_ops->tx_queue_release)(dev, qid);
 928         txq[qid] = NULL;
 929 }
 930
 931 static int
 932 eth_dev_rx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
 933 {
 934         uint16_t old_nb_queues = dev->data->nb_rx_queues;
 935         unsigned i;
 936
 937         if (dev->data->rx_queues == NULL && nb_queues != 0) { /* first time configuration */
 938                 dev->data->rx_queues = rte_zmalloc("ethdev->rx_queues",
 939                                 sizeof(dev->data->rx_queues[0]) *
 940                                 RTE_MAX_QUEUES_PER_PORT,
 941                                 RTE_CACHE_LINE_SIZE);
 942                 if (dev->data->rx_queues == NULL) {
 943                         dev->data->nb_rx_queues = 0;
 944                         return -(ENOMEM);
 945                 }
 946         } else if (dev->data->rx_queues != NULL && nb_queues != 0) { /* re-configure */
 947                 for (i = nb_queues; i < old_nb_queues; i++)
 948                         eth_dev_rxq_release(dev, i);
 949
 950         } else if (dev->data->rx_queues != NULL && nb_queues == 0) {
 951                 for (i = nb_queues; i < old_nb_queues; i++)
 952                         eth_dev_rxq_release(dev, i);
 953
 954                 rte_free(dev->data->rx_queues);
 955                 dev->data->rx_queues = NULL;
 956         }
 957         dev->data->nb_rx_queues = nb_queues;
 958         return 0;
 959 }
 960
 961 static int
 962 eth_dev_validate_rx_queue(const struct rte_eth_dev *dev, uint16_t rx_queue_id)
 963 {
 964         uint16_t port_id;
 965
 966         if (rx_queue_id >= dev->data->nb_rx_queues) {
 967                 port_id = dev->data->port_id;
 968                 RTE_ETHDEV_LOG(ERR,
 969                                "Invalid Rx queue_id=%u of device with port_id=%u\n",
 970                                rx_queue_id, port_id);
 971                 return -EINVAL;
 972         }
 973
 974         if (dev->data->rx_queues[rx_queue_id] == NULL) {
 975                 port_id = dev->data->port_id;
 976                 RTE_ETHDEV_LOG(ERR,
 977                                "Queue %u of device with port_id=%u has not been setup\n",
 978                                rx_queue_id, port_id);
 979                 return -EINVAL;
 980         }
 981
 982         return 0;
 983 }
 984
 985 static int
 986 eth_dev_validate_tx_queue(const struct rte_eth_dev *dev, uint16_t tx_queue_id)
 987 {
 988         uint16_t port_id;
 989
 990         if (tx_queue_id >= dev->data->nb_tx_queues) {
 991                 port_id = dev->data->port_id;
 992                 RTE_ETHDEV_LOG(ERR,
 993                                "Invalid Tx queue_id=%u of device with port_id=%u\n",
 994                                tx_queue_id, port_id);
 995                 return -EINVAL;
 996         }
 997
 998         if (dev->data->tx_queues[tx_queue_id] == NULL) {
 999                 port_id = dev->data->port_id;
1000                 RTE_ETHDEV_LOG(ERR,
1001                                "Queue %u of device with port_id=%u has not been setup\n",
1002                                tx_queue_id, port_id);
1003                 return -EINVAL;
1004         }
1005
1006         return 0;
1007 }
1008
1009 int
1010 rte_eth_dev_rx_queue_start(uint16_t port_id, uint16_t rx_queue_id)
1011 {
1012         struct rte_eth_dev *dev;
1013         int ret;
1014
1015         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1016         dev = &rte_eth_devices[port_id];
1017
1018         if (!dev->data->dev_started) {
1019                 RTE_ETHDEV_LOG(ERR,
1020                         "Port %u must be started before start any queue\n",
1021                         port_id);
1022                 return -EINVAL;
1023         }
1024
1025         ret = eth_dev_validate_rx_queue(dev, rx_queue_id);
1026         if (ret != 0)
1027                 return ret;
1028
1029         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_start, -ENOTSUP);
1030
1031         if (rte_eth_dev_is_rx_hairpin_queue(dev, rx_queue_id)) {
1032                 RTE_ETHDEV_LOG(INFO,
1033                         "Can't start Rx hairpin queue %"PRIu16" of device with port_id=%"PRIu16"\n",
1034                         rx_queue_id, port_id);
1035                 return -EINVAL;
1036         }
1037
1038         if (dev->data->rx_queue_state[rx_queue_id] != RTE_ETH_QUEUE_STATE_STOPPED) {
1039                 RTE_ETHDEV_LOG(INFO,
1040                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already started\n",
1041                         rx_queue_id, port_id);
1042                 return 0;
1043         }
1044
1045         return eth_err(port_id, dev->dev_ops->rx_queue_start(dev, rx_queue_id));
1046 }
1047
1048 int
1049 rte_eth_dev_rx_queue_stop(uint16_t port_id, uint16_t rx_queue_id)
1050 {
1051         struct rte_eth_dev *dev;
1052         int ret;
1053
1054         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1055         dev = &rte_eth_devices[port_id];
1056
1057         ret = eth_dev_validate_rx_queue(dev, rx_queue_id);
1058         if (ret != 0)
1059                 return ret;
1060
1061         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_stop, -ENOTSUP);
1062
1063         if (rte_eth_dev_is_rx_hairpin_queue(dev, rx_queue_id)) {
1064                 RTE_ETHDEV_LOG(INFO,
1065                         "Can't stop Rx hairpin queue %"PRIu16" of device with port_id=%"PRIu16"\n",
1066                         rx_queue_id, port_id);
1067                 return -EINVAL;
1068         }
1069
1070         if (dev->data->rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STOPPED) {
1071                 RTE_ETHDEV_LOG(INFO,
1072                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already stopped\n",
1073                         rx_queue_id, port_id);
1074                 return 0;
1075         }
1076
1077         return eth_err(port_id, dev->dev_ops->rx_queue_stop(dev, rx_queue_id));
1078 }
1079
1080 int
1081 rte_eth_dev_tx_queue_start(uint16_t port_id, uint16_t tx_queue_id)
1082 {
1083         struct rte_eth_dev *dev;
1084         int ret;
1085
1086         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1087         dev = &rte_eth_devices[port_id];
1088
1089         if (!dev->data->dev_started) {
1090                 RTE_ETHDEV_LOG(ERR,
1091                         "Port %u must be started before start any queue\n",
1092                         port_id);
1093                 return -EINVAL;
1094         }
1095
1096         ret = eth_dev_validate_tx_queue(dev, tx_queue_id);
1097         if (ret != 0)
1098                 return ret;
1099
1100         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_start, -ENOTSUP);
1101
1102         if (rte_eth_dev_is_tx_hairpin_queue(dev, tx_queue_id)) {
1103                 RTE_ETHDEV_LOG(INFO,
1104                         "Can't start Tx hairpin queue %"PRIu16" of device with port_id=%"PRIu16"\n",
1105                         tx_queue_id, port_id);
1106                 return -EINVAL;
1107         }
1108
1109         if (dev->data->tx_queue_state[tx_queue_id] != RTE_ETH_QUEUE_STATE_STOPPED) {
1110                 RTE_ETHDEV_LOG(INFO,
1111                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already started\n",
1112                         tx_queue_id, port_id);
1113                 return 0;
1114         }
1115
1116         return eth_err(port_id, dev->dev_ops->tx_queue_start(dev, tx_queue_id));
1117 }
1118
1119 int
1120 rte_eth_dev_tx_queue_stop(uint16_t port_id, uint16_t tx_queue_id)
1121 {
1122         struct rte_eth_dev *dev;
1123         int ret;
1124
1125         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1126         dev = &rte_eth_devices[port_id];
1127
1128         ret = eth_dev_validate_tx_queue(dev, tx_queue_id);
1129         if (ret != 0)
1130                 return ret;
1131
1132         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_stop, -ENOTSUP);
1133
1134         if (rte_eth_dev_is_tx_hairpin_queue(dev, tx_queue_id)) {
1135                 RTE_ETHDEV_LOG(INFO,
1136                         "Can't stop Tx hairpin queue %"PRIu16" of device with port_id=%"PRIu16"\n",
1137                         tx_queue_id, port_id);
1138                 return -EINVAL;
1139         }
1140
1141         if (dev->data->tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STOPPED) {
1142                 RTE_ETHDEV_LOG(INFO,
1143                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already stopped\n",
1144                         tx_queue_id, port_id);
1145                 return 0;
1146         }
1147
1148         return eth_err(port_id, dev->dev_ops->tx_queue_stop(dev, tx_queue_id));
1149 }
1150
1151 static int
1152 eth_dev_tx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
1153 {
1154         uint16_t old_nb_queues = dev->data->nb_tx_queues;
1155         unsigned i;
1156
1157         if (dev->data->tx_queues == NULL && nb_queues != 0) { /* first time configuration */
1158                 dev->data->tx_queues = rte_zmalloc("ethdev->tx_queues",
1159                                 sizeof(dev->data->tx_queues[0]) *
1160                                 RTE_MAX_QUEUES_PER_PORT,
1161                                 RTE_CACHE_LINE_SIZE);
1162                 if (dev->data->tx_queues == NULL) {
1163                         dev->data->nb_tx_queues = 0;
1164                         return -(ENOMEM);
1165                 }
1166         } else if (dev->data->tx_queues != NULL && nb_queues != 0) { /* re-configure */
1167                 for (i = nb_queues; i < old_nb_queues; i++)
1168                         eth_dev_txq_release(dev, i);
1169
1170         } else if (dev->data->tx_queues != NULL && nb_queues == 0) {
1171                 for (i = nb_queues; i < old_nb_queues; i++)
1172                         eth_dev_txq_release(dev, i);
1173
1174                 rte_free(dev->data->tx_queues);
1175                 dev->data->tx_queues = NULL;
1176         }
1177         dev->data->nb_tx_queues = nb_queues;
1178         return 0;
1179 }
1180
1181 uint32_t
1182 rte_eth_speed_bitflag(uint32_t speed, int duplex)
1183 {
1184         switch (speed) {
1185         case ETH_SPEED_NUM_10M:
1186                 return duplex ? ETH_LINK_SPEED_10M : ETH_LINK_SPEED_10M_HD;
1187         case ETH_SPEED_NUM_100M:
1188                 return duplex ? ETH_LINK_SPEED_100M : ETH_LINK_SPEED_100M_HD;
1189         case ETH_SPEED_NUM_1G:
1190                 return ETH_LINK_SPEED_1G;
1191         case ETH_SPEED_NUM_2_5G:
1192                 return ETH_LINK_SPEED_2_5G;
1193         case ETH_SPEED_NUM_5G:
1194                 return ETH_LINK_SPEED_5G;
1195         case ETH_SPEED_NUM_10G:
1196                 return ETH_LINK_SPEED_10G;
1197         case ETH_SPEED_NUM_20G:
1198                 return ETH_LINK_SPEED_20G;
1199         case ETH_SPEED_NUM_25G:
1200                 return ETH_LINK_SPEED_25G;
1201         case ETH_SPEED_NUM_40G:
1202                 return ETH_LINK_SPEED_40G;
1203         case ETH_SPEED_NUM_50G:
1204                 return ETH_LINK_SPEED_50G;
1205         case ETH_SPEED_NUM_56G:
1206                 return ETH_LINK_SPEED_56G;
1207         case ETH_SPEED_NUM_100G:
1208                 return ETH_LINK_SPEED_100G;
1209         case ETH_SPEED_NUM_200G:
1210                 return ETH_LINK_SPEED_200G;
1211         default:
1212                 return 0;
1213         }
1214 }
1215
1216 const char *
1217 rte_eth_dev_rx_offload_name(uint64_t offload)
1218 {
1219         const char *name = "UNKNOWN";
1220         unsigned int i;
1221
1222         for (i = 0; i < RTE_DIM(eth_dev_rx_offload_names); ++i) {
1223                 if (offload == eth_dev_rx_offload_names[i].offload) {
1224                         name = eth_dev_rx_offload_names[i].name;
1225                         break;
1226                 }
1227         }
1228
1229         return name;
1230 }
1231
1232 const char *
1233 rte_eth_dev_tx_offload_name(uint64_t offload)
1234 {
1235         const char *name = "UNKNOWN";
1236         unsigned int i;
1237
1238         for (i = 0; i < RTE_DIM(eth_dev_tx_offload_names); ++i) {
1239                 if (offload == eth_dev_tx_offload_names[i].offload) {
1240                         name = eth_dev_tx_offload_names[i].name;
1241                         break;
1242                 }
1243         }
1244
1245         return name;
1246 }
1247
1248 const char *
1249 rte_eth_dev_capability_name(uint64_t capability)
1250 {
1251         const char *name = "UNKNOWN";
1252         unsigned int i;
1253
1254         for (i = 0; i < RTE_DIM(rte_eth_dev_capa_names); ++i) {
1255                 if (capability == rte_eth_dev_capa_names[i].offload) {
1256                         name = rte_eth_dev_capa_names[i].name;
1257                         break;
1258                 }
1259         }
1260
1261         return name;
1262 }
1263
1264 static inline int
1265 eth_dev_check_lro_pkt_size(uint16_t port_id, uint32_t config_size,
1266                    uint32_t max_rx_pkt_len, uint32_t dev_info_size)
1267 {
1268         int ret = 0;
1269
1270         if (dev_info_size == 0) {
1271                 if (config_size != max_rx_pkt_len) {
1272                         RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%d max_lro_pkt_size"
1273                                        " %u != %u is not allowed\n",
1274                                        port_id, config_size, max_rx_pkt_len);
1275                         ret = -EINVAL;
1276                 }
1277         } else if (config_size > dev_info_size) {
1278                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%d max_lro_pkt_size %u "
1279                                "> max allowed value %u\n", port_id, config_size,
1280                                dev_info_size);
1281                 ret = -EINVAL;
1282         } else if (config_size < RTE_ETHER_MIN_LEN) {
1283                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%d max_lro_pkt_size %u "
1284                                "< min allowed value %u\n", port_id, config_size,
1285                                (unsigned int)RTE_ETHER_MIN_LEN);
1286                 ret = -EINVAL;
1287         }
1288         return ret;
1289 }
1290
1291 /*
1292  * Validate offloads that are requested through rte_eth_dev_configure against
1293  * the offloads successfully set by the Ethernet device.
1294  *
1295  * @param port_id
1296  *   The port identifier of the Ethernet device.
1297  * @param req_offloads
1298  *   The offloads that have been requested through `rte_eth_dev_configure`.
1299  * @param set_offloads
1300  *   The offloads successfully set by the Ethernet device.
1301  * @param offload_type
1302  *   The offload type i.e. Rx/Tx string.
1303  * @param offload_name
1304  *   The function that prints the offload name.
1305  * @return
1306  *   - (0) if validation successful.
1307  *   - (-EINVAL) if requested offload has been silently disabled.
1308  *
1309  */
1310 static int
1311 eth_dev_validate_offloads(uint16_t port_id, uint64_t req_offloads,
1312                   uint64_t set_offloads, const char *offload_type,
1313                   const char *(*offload_name)(uint64_t))
1314 {
1315         uint64_t offloads_diff = req_offloads ^ set_offloads;
1316         uint64_t offload;
1317         int ret = 0;
1318
1319         while (offloads_diff != 0) {
1320                 /* Check if any offload is requested but not enabled. */
1321                 offload = RTE_BIT64(__builtin_ctzll(offloads_diff));
1322                 if (offload & req_offloads) {
1323                         RTE_ETHDEV_LOG(ERR,
1324                                 "Port %u failed to enable %s offload %s\n",
1325                                 port_id, offload_type, offload_name(offload));
1326                         ret = -EINVAL;
1327                 }
1328
1329                 /* Check if offload couldn't be disabled. */
1330                 if (offload & set_offloads) {
1331                         RTE_ETHDEV_LOG(DEBUG,
1332                                 "Port %u %s offload %s is not requested but enabled\n",
1333                                 port_id, offload_type, offload_name(offload));
1334                 }
1335
1336                 offloads_diff &= ~offload;
1337         }
1338
1339         return ret;
1340 }
1341
1342 static uint32_t
1343 eth_dev_get_overhead_len(uint32_t max_rx_pktlen, uint16_t max_mtu)
1344 {
1345         uint32_t overhead_len;
1346
1347         if (max_mtu != UINT16_MAX && max_rx_pktlen > max_mtu)
1348                 overhead_len = max_rx_pktlen - max_mtu;
1349         else
1350                 overhead_len = RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN;
1351
1352         return overhead_len;
1353 }
1354
1355 /* rte_eth_dev_info_get() should be called prior to this function */
1356 static int
1357 eth_dev_validate_mtu(uint16_t port_id, struct rte_eth_dev_info *dev_info,
1358                 uint16_t mtu)
1359 {
1360         uint32_t overhead_len;
1361         uint32_t frame_size;
1362
1363         if (mtu < dev_info->min_mtu) {
1364                 RTE_ETHDEV_LOG(ERR,
1365                         "MTU (%u) < device min MTU (%u) for port_id %u\n",
1366                         mtu, dev_info->min_mtu, port_id);
1367                 return -EINVAL;
1368         }
1369         if (mtu > dev_info->max_mtu) {
1370                 RTE_ETHDEV_LOG(ERR,
1371                         "MTU (%u) > device max MTU (%u) for port_id %u\n",
1372                         mtu, dev_info->max_mtu, port_id);
1373                 return -EINVAL;
1374         }
1375
1376         overhead_len = eth_dev_get_overhead_len(dev_info->max_rx_pktlen,
1377                         dev_info->max_mtu);
1378         frame_size = mtu + overhead_len;
1379         if (frame_size < RTE_ETHER_MIN_LEN) {
1380                 RTE_ETHDEV_LOG(ERR,
1381                         "Frame size (%u) < min frame size (%u) for port_id %u\n",
1382                         frame_size, RTE_ETHER_MIN_LEN, port_id);
1383                 return -EINVAL;
1384         }
1385
1386         if (frame_size > dev_info->max_rx_pktlen) {
1387                 RTE_ETHDEV_LOG(ERR,
1388                         "Frame size (%u) > device max frame size (%u) for port_id %u\n",
1389                         frame_size, dev_info->max_rx_pktlen, port_id);
1390                 return -EINVAL;
1391         }
1392
1393         return 0;
1394 }
1395
1396 int
1397 rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
1398                       const struct rte_eth_conf *dev_conf)
1399 {
1400         struct rte_eth_dev *dev;
1401         struct rte_eth_dev_info dev_info;
1402         struct rte_eth_conf orig_conf;
1403         int diag;
1404         int ret;
1405         uint16_t old_mtu;
1406
1407         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1408         dev = &rte_eth_devices[port_id];
1409
1410         if (dev_conf == NULL) {
1411                 RTE_ETHDEV_LOG(ERR,
1412                         "Cannot configure ethdev port %u from NULL config\n",
1413                         port_id);
1414                 return -EINVAL;
1415         }
1416
1417         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
1418
1419         if (dev->data->dev_started) {
1420                 RTE_ETHDEV_LOG(ERR,
1421                         "Port %u must be stopped to allow configuration\n",
1422                         port_id);
1423                 return -EBUSY;
1424         }
1425
1426         /*
1427          * Ensure that "dev_configured" is always 0 each time prepare to do
1428          * dev_configure() to avoid any non-anticipated behaviour.
1429          * And set to 1 when dev_configure() is executed successfully.
1430          */
1431         dev->data->dev_configured = 0;
1432
1433          /* Store original config, as rollback required on failure */
1434         memcpy(&orig_conf, &dev->data->dev_conf, sizeof(dev->data->dev_conf));
1435
1436         /*
1437          * Copy the dev_conf parameter into the dev structure.
1438          * rte_eth_dev_info_get() requires dev_conf, copy it before dev_info get
1439          */
1440         if (dev_conf != &dev->data->dev_conf)
1441                 memcpy(&dev->data->dev_conf, dev_conf,
1442                        sizeof(dev->data->dev_conf));
1443
1444         /* Backup mtu for rollback */
1445         old_mtu = dev->data->mtu;
1446
1447         ret = rte_eth_dev_info_get(port_id, &dev_info);
1448         if (ret != 0)
1449                 goto rollback;
1450
1451         /* If number of queues specified by application for both Rx and Tx is
1452          * zero, use driver preferred values. This cannot be done individually
1453          * as it is valid for either Tx or Rx (but not both) to be zero.
1454          * If driver does not provide any preferred valued, fall back on
1455          * EAL defaults.
1456          */
1457         if (nb_rx_q == 0 && nb_tx_q == 0) {
1458                 nb_rx_q = dev_info.default_rxportconf.nb_queues;
1459                 if (nb_rx_q == 0)
1460                         nb_rx_q = RTE_ETH_DEV_FALLBACK_RX_NBQUEUES;
1461                 nb_tx_q = dev_info.default_txportconf.nb_queues;
1462                 if (nb_tx_q == 0)
1463                         nb_tx_q = RTE_ETH_DEV_FALLBACK_TX_NBQUEUES;
1464         }
1465
1466         if (nb_rx_q > RTE_MAX_QUEUES_PER_PORT) {
1467                 RTE_ETHDEV_LOG(ERR,
1468                         "Number of Rx queues requested (%u) is greater than max supported(%d)\n",
1469                         nb_rx_q, RTE_MAX_QUEUES_PER_PORT);
1470                 ret = -EINVAL;
1471                 goto rollback;
1472         }
1473
1474         if (nb_tx_q > RTE_MAX_QUEUES_PER_PORT) {
1475                 RTE_ETHDEV_LOG(ERR,
1476                         "Number of Tx queues requested (%u) is greater than max supported(%d)\n",
1477                         nb_tx_q, RTE_MAX_QUEUES_PER_PORT);
1478                 ret = -EINVAL;
1479                 goto rollback;
1480         }
1481
1482         /*
1483          * Check that the numbers of Rx and Tx queues are not greater
1484          * than the maximum number of Rx and Tx queues supported by the
1485          * configured device.
1486          */
1487         if (nb_rx_q > dev_info.max_rx_queues) {
1488                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u nb_rx_queues=%u > %u\n",
1489                         port_id, nb_rx_q, dev_info.max_rx_queues);
1490                 ret = -EINVAL;
1491                 goto rollback;
1492         }
1493
1494         if (nb_tx_q > dev_info.max_tx_queues) {
1495                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u nb_tx_queues=%u > %u\n",
1496                         port_id, nb_tx_q, dev_info.max_tx_queues);
1497                 ret = -EINVAL;
1498                 goto rollback;
1499         }
1500
1501         /* Check that the device supports requested interrupts */
1502         if ((dev_conf->intr_conf.lsc == 1) &&
1503                         (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC))) {
1504                 RTE_ETHDEV_LOG(ERR, "Driver %s does not support lsc\n",
1505                         dev->device->driver->name);
1506                 ret = -EINVAL;
1507                 goto rollback;
1508         }
1509         if ((dev_conf->intr_conf.rmv == 1) &&
1510                         (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_RMV))) {
1511                 RTE_ETHDEV_LOG(ERR, "Driver %s does not support rmv\n",
1512                         dev->device->driver->name);
1513                 ret = -EINVAL;
1514                 goto rollback;
1515         }
1516
1517         if (dev_conf->rxmode.mtu == 0)
1518                 dev->data->dev_conf.rxmode.mtu = RTE_ETHER_MTU;
1519
1520         ret = eth_dev_validate_mtu(port_id, &dev_info,
1521                         dev->data->dev_conf.rxmode.mtu);
1522         if (ret != 0)
1523                 goto rollback;
1524
1525         dev->data->mtu = dev->data->dev_conf.rxmode.mtu;
1526
1527         /*
1528          * If LRO is enabled, check that the maximum aggregated packet
1529          * size is supported by the configured device.
1530          */
1531         if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_TCP_LRO) {
1532                 uint32_t max_rx_pktlen;
1533                 uint32_t overhead_len;
1534
1535                 overhead_len = eth_dev_get_overhead_len(dev_info.max_rx_pktlen,
1536                                 dev_info.max_mtu);
1537                 max_rx_pktlen = dev->data->dev_conf.rxmode.mtu + overhead_len;
1538                 if (dev_conf->rxmode.max_lro_pkt_size == 0)
1539                         dev->data->dev_conf.rxmode.max_lro_pkt_size = max_rx_pktlen;
1540                 ret = eth_dev_check_lro_pkt_size(port_id,
1541                                 dev->data->dev_conf.rxmode.max_lro_pkt_size,
1542                                 max_rx_pktlen,
1543                                 dev_info.max_lro_pkt_size);
1544                 if (ret != 0)
1545                         goto rollback;
1546         }
1547
1548         /* Any requested offloading must be within its device capabilities */
1549         if ((dev_conf->rxmode.offloads & dev_info.rx_offload_capa) !=
1550              dev_conf->rxmode.offloads) {
1551                 RTE_ETHDEV_LOG(ERR,
1552                         "Ethdev port_id=%u requested Rx offloads 0x%"PRIx64" doesn't match Rx offloads "
1553                         "capabilities 0x%"PRIx64" in %s()\n",
1554                         port_id, dev_conf->rxmode.offloads,
1555                         dev_info.rx_offload_capa,
1556                         __func__);
1557                 ret = -EINVAL;
1558                 goto rollback;
1559         }
1560         if ((dev_conf->txmode.offloads & dev_info.tx_offload_capa) !=
1561              dev_conf->txmode.offloads) {
1562                 RTE_ETHDEV_LOG(ERR,
1563                         "Ethdev port_id=%u requested Tx offloads 0x%"PRIx64" doesn't match Tx offloads "
1564                         "capabilities 0x%"PRIx64" in %s()\n",
1565                         port_id, dev_conf->txmode.offloads,
1566                         dev_info.tx_offload_capa,
1567                         __func__);
1568                 ret = -EINVAL;
1569                 goto rollback;
1570         }
1571
1572         dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf =
1573                 rte_eth_rss_hf_refine(dev_conf->rx_adv_conf.rss_conf.rss_hf);
1574
1575         /* Check that device supports requested rss hash functions. */
1576         if ((dev_info.flow_type_rss_offloads |
1577              dev_conf->rx_adv_conf.rss_conf.rss_hf) !=
1578             dev_info.flow_type_rss_offloads) {
1579                 RTE_ETHDEV_LOG(ERR,
1580                         "Ethdev port_id=%u invalid rss_hf: 0x%"PRIx64", valid value: 0x%"PRIx64"\n",
1581                         port_id, dev_conf->rx_adv_conf.rss_conf.rss_hf,
1582                         dev_info.flow_type_rss_offloads);
1583                 ret = -EINVAL;
1584                 goto rollback;
1585         }
1586
1587         /* Check if Rx RSS distribution is disabled but RSS hash is enabled. */
1588         if (((dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) == 0) &&
1589             (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_RSS_HASH)) {
1590                 RTE_ETHDEV_LOG(ERR,
1591                         "Ethdev port_id=%u config invalid Rx mq_mode without RSS but %s offload is requested\n",
1592                         port_id,
1593                         rte_eth_dev_rx_offload_name(DEV_RX_OFFLOAD_RSS_HASH));
1594                 ret = -EINVAL;
1595                 goto rollback;
1596         }
1597
1598         /*
1599          * Setup new number of Rx/Tx queues and reconfigure device.
1600          */
1601         diag = eth_dev_rx_queue_config(dev, nb_rx_q);
1602         if (diag != 0) {
1603                 RTE_ETHDEV_LOG(ERR,
1604                         "Port%u eth_dev_rx_queue_config = %d\n",
1605                         port_id, diag);
1606                 ret = diag;
1607                 goto rollback;
1608         }
1609
1610         diag = eth_dev_tx_queue_config(dev, nb_tx_q);
1611         if (diag != 0) {
1612                 RTE_ETHDEV_LOG(ERR,
1613                         "Port%u eth_dev_tx_queue_config = %d\n",
1614                         port_id, diag);
1615                 eth_dev_rx_queue_config(dev, 0);
1616                 ret = diag;
1617                 goto rollback;
1618         }
1619
1620         diag = (*dev->dev_ops->dev_configure)(dev);
1621         if (diag != 0) {
1622                 RTE_ETHDEV_LOG(ERR, "Port%u dev_configure = %d\n",
1623                         port_id, diag);
1624                 ret = eth_err(port_id, diag);
1625                 goto reset_queues;
1626         }
1627
1628         /* Initialize Rx profiling if enabled at compilation time. */
1629         diag = __rte_eth_dev_profile_init(port_id, dev);
1630         if (diag != 0) {
1631                 RTE_ETHDEV_LOG(ERR, "Port%u __rte_eth_dev_profile_init = %d\n",
1632                         port_id, diag);
1633                 ret = eth_err(port_id, diag);
1634                 goto reset_queues;
1635         }
1636
1637         /* Validate Rx offloads. */
1638         diag = eth_dev_validate_offloads(port_id,
1639                         dev_conf->rxmode.offloads,
1640                         dev->data->dev_conf.rxmode.offloads, "Rx",
1641                         rte_eth_dev_rx_offload_name);
1642         if (diag != 0) {
1643                 ret = diag;
1644                 goto reset_queues;
1645         }
1646
1647         /* Validate Tx offloads. */
1648         diag = eth_dev_validate_offloads(port_id,
1649                         dev_conf->txmode.offloads,
1650                         dev->data->dev_conf.txmode.offloads, "Tx",
1651                         rte_eth_dev_tx_offload_name);
1652         if (diag != 0) {
1653                 ret = diag;
1654                 goto reset_queues;
1655         }
1656
1657         dev->data->dev_configured = 1;
1658         rte_ethdev_trace_configure(port_id, nb_rx_q, nb_tx_q, dev_conf, 0);
1659         return 0;
1660 reset_queues:
1661         eth_dev_rx_queue_config(dev, 0);
1662         eth_dev_tx_queue_config(dev, 0);
1663 rollback:
1664         memcpy(&dev->data->dev_conf, &orig_conf, sizeof(dev->data->dev_conf));
1665         if (old_mtu != dev->data->mtu)
1666                 dev->data->mtu = old_mtu;
1667
1668         rte_ethdev_trace_configure(port_id, nb_rx_q, nb_tx_q, dev_conf, ret);
1669         return ret;
1670 }
1671
1672 void
1673 rte_eth_dev_internal_reset(struct rte_eth_dev *dev)
1674 {
1675         if (dev->data->dev_started) {
1676                 RTE_ETHDEV_LOG(ERR, "Port %u must be stopped to allow reset\n",
1677                         dev->data->port_id);
1678                 return;
1679         }
1680
1681         eth_dev_rx_queue_config(dev, 0);
1682         eth_dev_tx_queue_config(dev, 0);
1683
1684         memset(&dev->data->dev_conf, 0, sizeof(dev->data->dev_conf));
1685 }
1686
1687 static void
1688 eth_dev_mac_restore(struct rte_eth_dev *dev,
1689                         struct rte_eth_dev_info *dev_info)
1690 {
1691         struct rte_ether_addr *addr;
1692         uint16_t i;
1693         uint32_t pool = 0;
1694         uint64_t pool_mask;
1695
1696         /* replay MAC address configuration including default MAC */
1697         addr = &dev->data->mac_addrs[0];
1698         if (*dev->dev_ops->mac_addr_set != NULL)
1699                 (*dev->dev_ops->mac_addr_set)(dev, addr);
1700         else if (*dev->dev_ops->mac_addr_add != NULL)
1701                 (*dev->dev_ops->mac_addr_add)(dev, addr, 0, pool);
1702
1703         if (*dev->dev_ops->mac_addr_add != NULL) {
1704                 for (i = 1; i < dev_info->max_mac_addrs; i++) {
1705                         addr = &dev->data->mac_addrs[i];
1706
1707                         /* skip zero address */
1708                         if (rte_is_zero_ether_addr(addr))
1709                                 continue;
1710
1711                         pool = 0;
1712                         pool_mask = dev->data->mac_pool_sel[i];
1713
1714                         do {
1715                                 if (pool_mask & UINT64_C(1))
1716                                         (*dev->dev_ops->mac_addr_add)(dev,
1717                                                 addr, i, pool);
1718                                 pool_mask >>= 1;
1719                                 pool++;
1720                         } while (pool_mask);
1721                 }
1722         }
1723 }
1724
1725 static int
1726 eth_dev_config_restore(struct rte_eth_dev *dev,
1727                 struct rte_eth_dev_info *dev_info, uint16_t port_id)
1728 {
1729         int ret;
1730
1731         if (!(*dev_info->dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR))
1732                 eth_dev_mac_restore(dev, dev_info);
1733
1734         /* replay promiscuous configuration */
1735         /*
1736          * use callbacks directly since we don't need port_id check and
1737          * would like to bypass the same value set
1738          */
1739         if (rte_eth_promiscuous_get(port_id) == 1 &&
1740             *dev->dev_ops->promiscuous_enable != NULL) {
1741                 ret = eth_err(port_id,
1742                               (*dev->dev_ops->promiscuous_enable)(dev));
1743                 if (ret != 0 && ret != -ENOTSUP) {
1744                         RTE_ETHDEV_LOG(ERR,
1745                                 "Failed to enable promiscuous mode for device (port %u): %s\n",
1746                                 port_id, rte_strerror(-ret));
1747                         return ret;
1748                 }
1749         } else if (rte_eth_promiscuous_get(port_id) == 0 &&
1750                    *dev->dev_ops->promiscuous_disable != NULL) {
1751                 ret = eth_err(port_id,
1752                               (*dev->dev_ops->promiscuous_disable)(dev));
1753                 if (ret != 0 && ret != -ENOTSUP) {
1754                         RTE_ETHDEV_LOG(ERR,
1755                                 "Failed to disable promiscuous mode for device (port %u): %s\n",
1756                                 port_id, rte_strerror(-ret));
1757                         return ret;
1758                 }
1759         }
1760
1761         /* replay all multicast configuration */
1762         /*
1763          * use callbacks directly since we don't need port_id check and
1764          * would like to bypass the same value set
1765          */
1766         if (rte_eth_allmulticast_get(port_id) == 1 &&
1767             *dev->dev_ops->allmulticast_enable != NULL) {
1768                 ret = eth_err(port_id,
1769                               (*dev->dev_ops->allmulticast_enable)(dev));
1770                 if (ret != 0 && ret != -ENOTSUP) {
1771                         RTE_ETHDEV_LOG(ERR,
1772                                 "Failed to enable allmulticast mode for device (port %u): %s\n",
1773                                 port_id, rte_strerror(-ret));
1774                         return ret;
1775                 }
1776         } else if (rte_eth_allmulticast_get(port_id) == 0 &&
1777                    *dev->dev_ops->allmulticast_disable != NULL) {
1778                 ret = eth_err(port_id,
1779                               (*dev->dev_ops->allmulticast_disable)(dev));
1780                 if (ret != 0 && ret != -ENOTSUP) {
1781                         RTE_ETHDEV_LOG(ERR,
1782                                 "Failed to disable allmulticast mode for device (port %u): %s\n",
1783                                 port_id, rte_strerror(-ret));
1784                         return ret;
1785                 }
1786         }
1787
1788         return 0;
1789 }
1790
1791 int
1792 rte_eth_dev_start(uint16_t port_id)
1793 {
1794         struct rte_eth_dev *dev;
1795         struct rte_eth_dev_info dev_info;
1796         int diag;
1797         int ret, ret_stop;
1798
1799         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1800         dev = &rte_eth_devices[port_id];
1801
1802         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_start, -ENOTSUP);
1803
1804         if (dev->data->dev_configured == 0) {
1805                 RTE_ETHDEV_LOG(INFO,
1806                         "Device with port_id=%"PRIu16" is not configured.\n",
1807                         port_id);
1808                 return -EINVAL;
1809         }
1810
1811         if (dev->data->dev_started != 0) {
1812                 RTE_ETHDEV_LOG(INFO,
1813                         "Device with port_id=%"PRIu16" already started\n",
1814                         port_id);
1815                 return 0;
1816         }
1817
1818         ret = rte_eth_dev_info_get(port_id, &dev_info);
1819         if (ret != 0)
1820                 return ret;
1821
1822         /* Lets restore MAC now if device does not support live change */
1823         if (*dev_info.dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR)
1824                 eth_dev_mac_restore(dev, &dev_info);
1825
1826         diag = (*dev->dev_ops->dev_start)(dev);
1827         if (diag == 0)
1828                 dev->data->dev_started = 1;
1829         else
1830                 return eth_err(port_id, diag);
1831
1832         ret = eth_dev_config_restore(dev, &dev_info, port_id);
1833         if (ret != 0) {
1834                 RTE_ETHDEV_LOG(ERR,
1835                         "Error during restoring configuration for device (port %u): %s\n",
1836                         port_id, rte_strerror(-ret));
1837                 ret_stop = rte_eth_dev_stop(port_id);
1838                 if (ret_stop != 0) {
1839                         RTE_ETHDEV_LOG(ERR,
1840                                 "Failed to stop device (port %u): %s\n",
1841                                 port_id, rte_strerror(-ret_stop));
1842                 }
1843
1844                 return ret;
1845         }
1846
1847         if (dev->data->dev_conf.intr_conf.lsc == 0) {
1848                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
1849                 (*dev->dev_ops->link_update)(dev, 0);
1850         }
1851
1852         /* expose selection of PMD fast-path functions */
1853         eth_dev_fp_ops_setup(rte_eth_fp_ops + port_id, dev);
1854
1855         rte_ethdev_trace_start(port_id);
1856         return 0;
1857 }
1858
1859 int
1860 rte_eth_dev_stop(uint16_t port_id)
1861 {
1862         struct rte_eth_dev *dev;
1863         int ret;
1864
1865         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1866         dev = &rte_eth_devices[port_id];
1867
1868         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_stop, -ENOTSUP);
1869
1870         if (dev->data->dev_started == 0) {
1871                 RTE_ETHDEV_LOG(INFO,
1872                         "Device with port_id=%"PRIu16" already stopped\n",
1873                         port_id);
1874                 return 0;
1875         }
1876
1877         /* point fast-path functions to dummy ones */
1878         eth_dev_fp_ops_reset(rte_eth_fp_ops + port_id);
1879
1880         dev->data->dev_started = 0;
1881         ret = (*dev->dev_ops->dev_stop)(dev);
1882         rte_ethdev_trace_stop(port_id, ret);
1883
1884         return ret;
1885 }
1886
1887 int
1888 rte_eth_dev_set_link_up(uint16_t port_id)
1889 {
1890         struct rte_eth_dev *dev;
1891
1892         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1893         dev = &rte_eth_devices[port_id];
1894
1895         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_up, -ENOTSUP);
1896         return eth_err(port_id, (*dev->dev_ops->dev_set_link_up)(dev));
1897 }
1898
1899 int
1900 rte_eth_dev_set_link_down(uint16_t port_id)
1901 {
1902         struct rte_eth_dev *dev;
1903
1904         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1905         dev = &rte_eth_devices[port_id];
1906
1907         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_down, -ENOTSUP);
1908         return eth_err(port_id, (*dev->dev_ops->dev_set_link_down)(dev));
1909 }
1910
1911 int
1912 rte_eth_dev_close(uint16_t port_id)
1913 {
1914         struct rte_eth_dev *dev;
1915         int firsterr, binerr;
1916         int *lasterr = &firsterr;
1917
1918         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1919         dev = &rte_eth_devices[port_id];
1920
1921         if (dev->data->dev_started) {
1922                 RTE_ETHDEV_LOG(ERR, "Cannot close started device (port %u)\n",
1923                                port_id);
1924                 return -EINVAL;
1925         }
1926
1927         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_close, -ENOTSUP);
1928         *lasterr = (*dev->dev_ops->dev_close)(dev);
1929         if (*lasterr != 0)
1930                 lasterr = &binerr;
1931
1932         rte_ethdev_trace_close(port_id);
1933         *lasterr = rte_eth_dev_release_port(dev);
1934
1935         return firsterr;
1936 }
1937
1938 int
1939 rte_eth_dev_reset(uint16_t port_id)
1940 {
1941         struct rte_eth_dev *dev;
1942         int ret;
1943
1944         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
1945         dev = &rte_eth_devices[port_id];
1946
1947         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_reset, -ENOTSUP);
1948
1949         ret = rte_eth_dev_stop(port_id);
1950         if (ret != 0) {
1951                 RTE_ETHDEV_LOG(ERR,
1952                         "Failed to stop device (port %u) before reset: %s - ignore\n",
1953                         port_id, rte_strerror(-ret));
1954         }
1955         ret = dev->dev_ops->dev_reset(dev);
1956
1957         return eth_err(port_id, ret);
1958 }
1959
1960 int
1961 rte_eth_dev_is_removed(uint16_t port_id)
1962 {
1963         struct rte_eth_dev *dev;
1964         int ret;
1965
1966         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, 0);
1967         dev = &rte_eth_devices[port_id];
1968
1969         if (dev->state == RTE_ETH_DEV_REMOVED)
1970                 return 1;
1971
1972         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->is_removed, 0);
1973
1974         ret = dev->dev_ops->is_removed(dev);
1975         if (ret != 0)
1976                 /* Device is physically removed. */
1977                 dev->state = RTE_ETH_DEV_REMOVED;
1978
1979         return ret;
1980 }
1981
1982 static int
1983 rte_eth_rx_queue_check_split(const struct rte_eth_rxseg_split *rx_seg,
1984                              uint16_t n_seg, uint32_t *mbp_buf_size,
1985                              const struct rte_eth_dev_info *dev_info)
1986 {
1987         const struct rte_eth_rxseg_capa *seg_capa = &dev_info->rx_seg_capa;
1988         struct rte_mempool *mp_first;
1989         uint32_t offset_mask;
1990         uint16_t seg_idx;
1991
1992         if (n_seg > seg_capa->max_nseg) {
1993                 RTE_ETHDEV_LOG(ERR,
1994                                "Requested Rx segments %u exceed supported %u\n",
1995                                n_seg, seg_capa->max_nseg);
1996                 return -EINVAL;
1997         }
1998         /*
1999          * Check the sizes and offsets against buffer sizes
2000          * for each segment specified in extended configuration.
2001          */
2002         mp_first = rx_seg[0].mp;
2003         offset_mask = RTE_BIT32(seg_capa->offset_align_log2) - 1;
2004         for (seg_idx = 0; seg_idx < n_seg; seg_idx++) {
2005                 struct rte_mempool *mpl = rx_seg[seg_idx].mp;
2006                 uint32_t length = rx_seg[seg_idx].length;
2007                 uint32_t offset = rx_seg[seg_idx].offset;
2008
2009                 if (mpl == NULL) {
2010                         RTE_ETHDEV_LOG(ERR, "null mempool pointer\n");
2011                         return -EINVAL;
2012                 }
2013                 if (seg_idx != 0 && mp_first != mpl &&
2014                     seg_capa->multi_pools == 0) {
2015                         RTE_ETHDEV_LOG(ERR, "Receiving to multiple pools is not supported\n");
2016                         return -ENOTSUP;
2017                 }
2018                 if (offset != 0) {
2019                         if (seg_capa->offset_allowed == 0) {
2020                                 RTE_ETHDEV_LOG(ERR, "Rx segmentation with offset is not supported\n");
2021                                 return -ENOTSUP;
2022                         }
2023                         if (offset & offset_mask) {
2024                                 RTE_ETHDEV_LOG(ERR, "Rx segmentation invalid offset alignment %u, %u\n",
2025                                                offset,
2026                                                seg_capa->offset_align_log2);
2027                                 return -EINVAL;
2028                         }
2029                 }
2030                 if (mpl->private_data_size <
2031                         sizeof(struct rte_pktmbuf_pool_private)) {
2032                         RTE_ETHDEV_LOG(ERR,
2033                                        "%s private_data_size %u < %u\n",
2034                                        mpl->name, mpl->private_data_size,
2035                                        (unsigned int)sizeof
2036                                         (struct rte_pktmbuf_pool_private));
2037                         return -ENOSPC;
2038                 }
2039                 offset += seg_idx != 0 ? 0 : RTE_PKTMBUF_HEADROOM;
2040                 *mbp_buf_size = rte_pktmbuf_data_room_size(mpl);
2041                 length = length != 0 ? length : *mbp_buf_size;
2042                 if (*mbp_buf_size < length + offset) {
2043                         RTE_ETHDEV_LOG(ERR,
2044                                        "%s mbuf_data_room_size %u < %u (segment length=%u + segment offset=%u)\n",
2045                                        mpl->name, *mbp_buf_size,
2046                                        length + offset, length, offset);
2047                         return -EINVAL;
2048                 }
2049         }
2050         return 0;
2051 }
2052
2053 int
2054 rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2055                        uint16_t nb_rx_desc, unsigned int socket_id,
2056                        const struct rte_eth_rxconf *rx_conf,
2057                        struct rte_mempool *mp)
2058 {
2059         int ret;
2060         uint32_t mbp_buf_size;
2061         struct rte_eth_dev *dev;
2062         struct rte_eth_dev_info dev_info;
2063         struct rte_eth_rxconf local_conf;
2064
2065         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2066         dev = &rte_eth_devices[port_id];
2067
2068         if (rx_queue_id >= dev->data->nb_rx_queues) {
2069                 RTE_ETHDEV_LOG(ERR, "Invalid Rx queue_id=%u\n", rx_queue_id);
2070                 return -EINVAL;
2071         }
2072
2073         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_setup, -ENOTSUP);
2074
2075         ret = rte_eth_dev_info_get(port_id, &dev_info);
2076         if (ret != 0)
2077                 return ret;
2078
2079         if (mp != NULL) {
2080                 /* Single pool configuration check. */
2081                 if (rx_conf != NULL && rx_conf->rx_nseg != 0) {
2082                         RTE_ETHDEV_LOG(ERR,
2083                                        "Ambiguous segment configuration\n");
2084                         return -EINVAL;
2085                 }
2086                 /*
2087                  * Check the size of the mbuf data buffer, this value
2088                  * must be provided in the private data of the memory pool.
2089                  * First check that the memory pool(s) has a valid private data.
2090                  */
2091                 if (mp->private_data_size <
2092                                 sizeof(struct rte_pktmbuf_pool_private)) {
2093                         RTE_ETHDEV_LOG(ERR, "%s private_data_size %u < %u\n",
2094                                 mp->name, mp->private_data_size,
2095                                 (unsigned int)
2096                                 sizeof(struct rte_pktmbuf_pool_private));
2097                         return -ENOSPC;
2098                 }
2099                 mbp_buf_size = rte_pktmbuf_data_room_size(mp);
2100                 if (mbp_buf_size < dev_info.min_rx_bufsize +
2101                                    RTE_PKTMBUF_HEADROOM) {
2102                         RTE_ETHDEV_LOG(ERR,
2103                                        "%s mbuf_data_room_size %u < %u (RTE_PKTMBUF_HEADROOM=%u + min_rx_bufsize(dev)=%u)\n",
2104                                        mp->name, mbp_buf_size,
2105                                        RTE_PKTMBUF_HEADROOM +
2106                                        dev_info.min_rx_bufsize,
2107                                        RTE_PKTMBUF_HEADROOM,
2108                                        dev_info.min_rx_bufsize);
2109                         return -EINVAL;
2110                 }
2111         } else {
2112                 const struct rte_eth_rxseg_split *rx_seg;
2113                 uint16_t n_seg;
2114
2115                 /* Extended multi-segment configuration check. */
2116                 if (rx_conf == NULL || rx_conf->rx_seg == NULL || rx_conf->rx_nseg == 0) {
2117                         RTE_ETHDEV_LOG(ERR,
2118                                        "Memory pool is null and no extended configuration provided\n");
2119                         return -EINVAL;
2120                 }
2121
2122                 rx_seg = (const struct rte_eth_rxseg_split *)rx_conf->rx_seg;
2123                 n_seg = rx_conf->rx_nseg;
2124
2125                 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) {
2126                         ret = rte_eth_rx_queue_check_split(rx_seg, n_seg,
2127                                                            &mbp_buf_size,
2128                                                            &dev_info);
2129                         if (ret != 0)
2130                                 return ret;
2131                 } else {
2132                         RTE_ETHDEV_LOG(ERR, "No Rx segmentation offload configured\n");
2133                         return -EINVAL;
2134                 }
2135         }
2136
2137         /* Use default specified by driver, if nb_rx_desc is zero */
2138         if (nb_rx_desc == 0) {
2139                 nb_rx_desc = dev_info.default_rxportconf.ring_size;
2140                 /* If driver default is also zero, fall back on EAL default */
2141                 if (nb_rx_desc == 0)
2142                         nb_rx_desc = RTE_ETH_DEV_FALLBACK_RX_RINGSIZE;
2143         }
2144
2145         if (nb_rx_desc > dev_info.rx_desc_lim.nb_max ||
2146                         nb_rx_desc < dev_info.rx_desc_lim.nb_min ||
2147                         nb_rx_desc % dev_info.rx_desc_lim.nb_align != 0) {
2148
2149                 RTE_ETHDEV_LOG(ERR,
2150                         "Invalid value for nb_rx_desc(=%hu), should be: <= %hu, >= %hu, and a product of %hu\n",
2151                         nb_rx_desc, dev_info.rx_desc_lim.nb_max,
2152                         dev_info.rx_desc_lim.nb_min,
2153                         dev_info.rx_desc_lim.nb_align);
2154                 return -EINVAL;
2155         }
2156
2157         if (dev->data->dev_started &&
2158                 !(dev_info.dev_capa &
2159                         RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP))
2160                 return -EBUSY;
2161
2162         if (dev->data->dev_started &&
2163                 (dev->data->rx_queue_state[rx_queue_id] !=
2164                         RTE_ETH_QUEUE_STATE_STOPPED))
2165                 return -EBUSY;
2166
2167         eth_dev_rxq_release(dev, rx_queue_id);
2168
2169         if (rx_conf == NULL)
2170                 rx_conf = &dev_info.default_rxconf;
2171
2172         local_conf = *rx_conf;
2173
2174         /*
2175          * If an offloading has already been enabled in
2176          * rte_eth_dev_configure(), it has been enabled on all queues,
2177          * so there is no need to enable it in this queue again.
2178          * The local_conf.offloads input to underlying PMD only carries
2179          * those offloadings which are only enabled on this queue and
2180          * not enabled on all queues.
2181          */
2182         local_conf.offloads &= ~dev->data->dev_conf.rxmode.offloads;
2183
2184         /*
2185          * New added offloadings for this queue are those not enabled in
2186          * rte_eth_dev_configure() and they must be per-queue type.
2187          * A pure per-port offloading can't be enabled on a queue while
2188          * disabled on another queue. A pure per-port offloading can't
2189          * be enabled for any queue as new added one if it hasn't been
2190          * enabled in rte_eth_dev_configure().
2191          */
2192         if ((local_conf.offloads & dev_info.rx_queue_offload_capa) !=
2193              local_conf.offloads) {
2194                 RTE_ETHDEV_LOG(ERR,
2195                         "Ethdev port_id=%d rx_queue_id=%d, new added offloads 0x%"PRIx64" must be "
2196                         "within per-queue offload capabilities 0x%"PRIx64" in %s()\n",
2197                         port_id, rx_queue_id, local_conf.offloads,
2198                         dev_info.rx_queue_offload_capa,
2199                         __func__);
2200                 return -EINVAL;
2201         }
2202
2203         if (local_conf.share_group > 0 &&
2204             (dev_info.dev_capa & RTE_ETH_DEV_CAPA_RXQ_SHARE) == 0) {
2205                 RTE_ETHDEV_LOG(ERR,
2206                         "Ethdev port_id=%d rx_queue_id=%d, enabled share_group=%hu while device doesn't support Rx queue share\n",
2207                         port_id, rx_queue_id, local_conf.share_group);
2208                 return -EINVAL;
2209         }
2210
2211         /*
2212          * If LRO is enabled, check that the maximum aggregated packet
2213          * size is supported by the configured device.
2214          */
2215         /* Get the real Ethernet overhead length */
2216         if (local_conf.offloads & DEV_RX_OFFLOAD_TCP_LRO) {
2217                 uint32_t overhead_len;
2218                 uint32_t max_rx_pktlen;
2219                 int ret;
2220
2221                 overhead_len = eth_dev_get_overhead_len(dev_info.max_rx_pktlen,
2222                                 dev_info.max_mtu);
2223                 max_rx_pktlen = dev->data->mtu + overhead_len;
2224                 if (dev->data->dev_conf.rxmode.max_lro_pkt_size == 0)
2225                         dev->data->dev_conf.rxmode.max_lro_pkt_size = max_rx_pktlen;
2226                 ret = eth_dev_check_lro_pkt_size(port_id,
2227                                 dev->data->dev_conf.rxmode.max_lro_pkt_size,
2228                                 max_rx_pktlen,
2229                                 dev_info.max_lro_pkt_size);
2230                 if (ret != 0)
2231                         return ret;
2232         }
2233
2234         ret = (*dev->dev_ops->rx_queue_setup)(dev, rx_queue_id, nb_rx_desc,
2235                                               socket_id, &local_conf, mp);
2236         if (!ret) {
2237                 if (!dev->data->min_rx_buf_size ||
2238                     dev->data->min_rx_buf_size > mbp_buf_size)
2239                         dev->data->min_rx_buf_size = mbp_buf_size;
2240         }
2241
2242         rte_ethdev_trace_rxq_setup(port_id, rx_queue_id, nb_rx_desc, mp,
2243                 rx_conf, ret);
2244         return eth_err(port_id, ret);
2245 }
2246
2247 int
2248 rte_eth_rx_hairpin_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2249                                uint16_t nb_rx_desc,
2250                                const struct rte_eth_hairpin_conf *conf)
2251 {
2252         int ret;
2253         struct rte_eth_dev *dev;
2254         struct rte_eth_hairpin_cap cap;
2255         int i;
2256         int count;
2257
2258         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2259         dev = &rte_eth_devices[port_id];
2260
2261         if (rx_queue_id >= dev->data->nb_rx_queues) {
2262                 RTE_ETHDEV_LOG(ERR, "Invalid Rx queue_id=%u\n", rx_queue_id);
2263                 return -EINVAL;
2264         }
2265
2266         if (conf == NULL) {
2267                 RTE_ETHDEV_LOG(ERR,
2268                         "Cannot setup ethdev port %u Rx hairpin queue from NULL config\n",
2269                         port_id);
2270                 return -EINVAL;
2271         }
2272
2273         ret = rte_eth_dev_hairpin_capability_get(port_id, &cap);
2274         if (ret != 0)
2275                 return ret;
2276         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_hairpin_queue_setup,
2277                                 -ENOTSUP);
2278         /* if nb_rx_desc is zero use max number of desc from the driver. */
2279         if (nb_rx_desc == 0)
2280                 nb_rx_desc = cap.max_nb_desc;
2281         if (nb_rx_desc > cap.max_nb_desc) {
2282                 RTE_ETHDEV_LOG(ERR,
2283                         "Invalid value for nb_rx_desc(=%hu), should be: <= %hu",
2284                         nb_rx_desc, cap.max_nb_desc);
2285                 return -EINVAL;
2286         }
2287         if (conf->peer_count > cap.max_rx_2_tx) {
2288                 RTE_ETHDEV_LOG(ERR,
2289                         "Invalid value for number of peers for Rx queue(=%u), should be: <= %hu",
2290                         conf->peer_count, cap.max_rx_2_tx);
2291                 return -EINVAL;
2292         }
2293         if (conf->peer_count == 0) {
2294                 RTE_ETHDEV_LOG(ERR,
2295                         "Invalid value for number of peers for Rx queue(=%u), should be: > 0",
2296                         conf->peer_count);
2297                 return -EINVAL;
2298         }
2299         for (i = 0, count = 0; i < dev->data->nb_rx_queues &&
2300              cap.max_nb_queues != UINT16_MAX; i++) {
2301                 if (i == rx_queue_id || rte_eth_dev_is_rx_hairpin_queue(dev, i))
2302                         count++;
2303         }
2304         if (count > cap.max_nb_queues) {
2305                 RTE_ETHDEV_LOG(ERR, "To many Rx hairpin queues max is %d",
2306                 cap.max_nb_queues);
2307                 return -EINVAL;
2308         }
2309         if (dev->data->dev_started)
2310                 return -EBUSY;
2311         eth_dev_rxq_release(dev, rx_queue_id);
2312         ret = (*dev->dev_ops->rx_hairpin_queue_setup)(dev, rx_queue_id,
2313                                                       nb_rx_desc, conf);
2314         if (ret == 0)
2315                 dev->data->rx_queue_state[rx_queue_id] =
2316                         RTE_ETH_QUEUE_STATE_HAIRPIN;
2317         return eth_err(port_id, ret);
2318 }
2319
2320 int
2321 rte_eth_tx_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
2322                        uint16_t nb_tx_desc, unsigned int socket_id,
2323                        const struct rte_eth_txconf *tx_conf)
2324 {
2325         struct rte_eth_dev *dev;
2326         struct rte_eth_dev_info dev_info;
2327         struct rte_eth_txconf local_conf;
2328         int ret;
2329
2330         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2331         dev = &rte_eth_devices[port_id];
2332
2333         if (tx_queue_id >= dev->data->nb_tx_queues) {
2334                 RTE_ETHDEV_LOG(ERR, "Invalid Tx queue_id=%u\n", tx_queue_id);
2335                 return -EINVAL;
2336         }
2337
2338         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_setup, -ENOTSUP);
2339
2340         ret = rte_eth_dev_info_get(port_id, &dev_info);
2341         if (ret != 0)
2342                 return ret;
2343
2344         /* Use default specified by driver, if nb_tx_desc is zero */
2345         if (nb_tx_desc == 0) {
2346                 nb_tx_desc = dev_info.default_txportconf.ring_size;
2347                 /* If driver default is zero, fall back on EAL default */
2348                 if (nb_tx_desc == 0)
2349                         nb_tx_desc = RTE_ETH_DEV_FALLBACK_TX_RINGSIZE;
2350         }
2351         if (nb_tx_desc > dev_info.tx_desc_lim.nb_max ||
2352             nb_tx_desc < dev_info.tx_desc_lim.nb_min ||
2353             nb_tx_desc % dev_info.tx_desc_lim.nb_align != 0) {
2354                 RTE_ETHDEV_LOG(ERR,
2355                         "Invalid value for nb_tx_desc(=%hu), should be: <= %hu, >= %hu, and a product of %hu\n",
2356                         nb_tx_desc, dev_info.tx_desc_lim.nb_max,
2357                         dev_info.tx_desc_lim.nb_min,
2358                         dev_info.tx_desc_lim.nb_align);
2359                 return -EINVAL;
2360         }
2361
2362         if (dev->data->dev_started &&
2363                 !(dev_info.dev_capa &
2364                         RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP))
2365                 return -EBUSY;
2366
2367         if (dev->data->dev_started &&
2368                 (dev->data->tx_queue_state[tx_queue_id] !=
2369                         RTE_ETH_QUEUE_STATE_STOPPED))
2370                 return -EBUSY;
2371
2372         eth_dev_txq_release(dev, tx_queue_id);
2373
2374         if (tx_conf == NULL)
2375                 tx_conf = &dev_info.default_txconf;
2376
2377         local_conf = *tx_conf;
2378
2379         /*
2380          * If an offloading has already been enabled in
2381          * rte_eth_dev_configure(), it has been enabled on all queues,
2382          * so there is no need to enable it in this queue again.
2383          * The local_conf.offloads input to underlying PMD only carries
2384          * those offloadings which are only enabled on this queue and
2385          * not enabled on all queues.
2386          */
2387         local_conf.offloads &= ~dev->data->dev_conf.txmode.offloads;
2388
2389         /*
2390          * New added offloadings for this queue are those not enabled in
2391          * rte_eth_dev_configure() and they must be per-queue type.
2392          * A pure per-port offloading can't be enabled on a queue while
2393          * disabled on another queue. A pure per-port offloading can't
2394          * be enabled for any queue as new added one if it hasn't been
2395          * enabled in rte_eth_dev_configure().
2396          */
2397         if ((local_conf.offloads & dev_info.tx_queue_offload_capa) !=
2398              local_conf.offloads) {
2399                 RTE_ETHDEV_LOG(ERR,
2400                         "Ethdev port_id=%d tx_queue_id=%d, new added offloads 0x%"PRIx64" must be "
2401                         "within per-queue offload capabilities 0x%"PRIx64" in %s()\n",
2402                         port_id, tx_queue_id, local_conf.offloads,
2403                         dev_info.tx_queue_offload_capa,
2404                         __func__);
2405                 return -EINVAL;
2406         }
2407
2408         rte_ethdev_trace_txq_setup(port_id, tx_queue_id, nb_tx_desc, tx_conf);
2409         return eth_err(port_id, (*dev->dev_ops->tx_queue_setup)(dev,
2410                        tx_queue_id, nb_tx_desc, socket_id, &local_conf));
2411 }
2412
2413 int
2414 rte_eth_tx_hairpin_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
2415                                uint16_t nb_tx_desc,
2416                                const struct rte_eth_hairpin_conf *conf)
2417 {
2418         struct rte_eth_dev *dev;
2419         struct rte_eth_hairpin_cap cap;
2420         int i;
2421         int count;
2422         int ret;
2423
2424         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2425         dev = &rte_eth_devices[port_id];
2426
2427         if (tx_queue_id >= dev->data->nb_tx_queues) {
2428                 RTE_ETHDEV_LOG(ERR, "Invalid Tx queue_id=%u\n", tx_queue_id);
2429                 return -EINVAL;
2430         }
2431
2432         if (conf == NULL) {
2433                 RTE_ETHDEV_LOG(ERR,
2434                         "Cannot setup ethdev port %u Tx hairpin queue from NULL config\n",
2435                         port_id);
2436                 return -EINVAL;
2437         }
2438
2439         ret = rte_eth_dev_hairpin_capability_get(port_id, &cap);
2440         if (ret != 0)
2441                 return ret;
2442         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_hairpin_queue_setup,
2443                                 -ENOTSUP);
2444         /* if nb_rx_desc is zero use max number of desc from the driver. */
2445         if (nb_tx_desc == 0)
2446                 nb_tx_desc = cap.max_nb_desc;
2447         if (nb_tx_desc > cap.max_nb_desc) {
2448                 RTE_ETHDEV_LOG(ERR,
2449                         "Invalid value for nb_tx_desc(=%hu), should be: <= %hu",
2450                         nb_tx_desc, cap.max_nb_desc);
2451                 return -EINVAL;
2452         }
2453         if (conf->peer_count > cap.max_tx_2_rx) {
2454                 RTE_ETHDEV_LOG(ERR,
2455                         "Invalid value for number of peers for Tx queue(=%u), should be: <= %hu",
2456                         conf->peer_count, cap.max_tx_2_rx);
2457                 return -EINVAL;
2458         }
2459         if (conf->peer_count == 0) {
2460                 RTE_ETHDEV_LOG(ERR,
2461                         "Invalid value for number of peers for Tx queue(=%u), should be: > 0",
2462                         conf->peer_count);
2463                 return -EINVAL;
2464         }
2465         for (i = 0, count = 0; i < dev->data->nb_tx_queues &&
2466              cap.max_nb_queues != UINT16_MAX; i++) {
2467                 if (i == tx_queue_id || rte_eth_dev_is_tx_hairpin_queue(dev, i))
2468                         count++;
2469         }
2470         if (count > cap.max_nb_queues) {
2471                 RTE_ETHDEV_LOG(ERR, "To many Tx hairpin queues max is %d",
2472                 cap.max_nb_queues);
2473                 return -EINVAL;
2474         }
2475         if (dev->data->dev_started)
2476                 return -EBUSY;
2477         eth_dev_txq_release(dev, tx_queue_id);
2478         ret = (*dev->dev_ops->tx_hairpin_queue_setup)
2479                 (dev, tx_queue_id, nb_tx_desc, conf);
2480         if (ret == 0)
2481                 dev->data->tx_queue_state[tx_queue_id] =
2482                         RTE_ETH_QUEUE_STATE_HAIRPIN;
2483         return eth_err(port_id, ret);
2484 }
2485
2486 int
2487 rte_eth_hairpin_bind(uint16_t tx_port, uint16_t rx_port)
2488 {
2489         struct rte_eth_dev *dev;
2490         int ret;
2491
2492         RTE_ETH_VALID_PORTID_OR_ERR_RET(tx_port, -ENODEV);
2493         dev = &rte_eth_devices[tx_port];
2494
2495         if (dev->data->dev_started == 0) {
2496                 RTE_ETHDEV_LOG(ERR, "Tx port %d is not started\n", tx_port);
2497                 return -EBUSY;
2498         }
2499
2500         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->hairpin_bind, -ENOTSUP);
2501         ret = (*dev->dev_ops->hairpin_bind)(dev, rx_port);
2502         if (ret != 0)
2503                 RTE_ETHDEV_LOG(ERR, "Failed to bind hairpin Tx %d"
2504                                " to Rx %d (%d - all ports)\n",
2505                                tx_port, rx_port, RTE_MAX_ETHPORTS);
2506
2507         return ret;
2508 }
2509
2510 int
2511 rte_eth_hairpin_unbind(uint16_t tx_port, uint16_t rx_port)
2512 {
2513         struct rte_eth_dev *dev;
2514         int ret;
2515
2516         RTE_ETH_VALID_PORTID_OR_ERR_RET(tx_port, -ENODEV);
2517         dev = &rte_eth_devices[tx_port];
2518
2519         if (dev->data->dev_started == 0) {
2520                 RTE_ETHDEV_LOG(ERR, "Tx port %d is already stopped\n", tx_port);
2521                 return -EBUSY;
2522         }
2523
2524         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->hairpin_unbind, -ENOTSUP);
2525         ret = (*dev->dev_ops->hairpin_unbind)(dev, rx_port);
2526         if (ret != 0)
2527                 RTE_ETHDEV_LOG(ERR, "Failed to unbind hairpin Tx %d"
2528                                " from Rx %d (%d - all ports)\n",
2529                                tx_port, rx_port, RTE_MAX_ETHPORTS);
2530
2531         return ret;
2532 }
2533
2534 int
2535 rte_eth_hairpin_get_peer_ports(uint16_t port_id, uint16_t *peer_ports,
2536                                size_t len, uint32_t direction)
2537 {
2538         struct rte_eth_dev *dev;
2539         int ret;
2540
2541         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2542         dev = &rte_eth_devices[port_id];
2543
2544         if (peer_ports == NULL) {
2545                 RTE_ETHDEV_LOG(ERR,
2546                         "Cannot get ethdev port %u hairpin peer ports to NULL\n",
2547                         port_id);
2548                 return -EINVAL;
2549         }
2550
2551         if (len == 0) {
2552                 RTE_ETHDEV_LOG(ERR,
2553                         "Cannot get ethdev port %u hairpin peer ports to array with zero size\n",
2554                         port_id);
2555                 return -EINVAL;
2556         }
2557
2558         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->hairpin_get_peer_ports,
2559                                 -ENOTSUP);
2560
2561         ret = (*dev->dev_ops->hairpin_get_peer_ports)(dev, peer_ports,
2562                                                       len, direction);
2563         if (ret < 0)
2564                 RTE_ETHDEV_LOG(ERR, "Failed to get %d hairpin peer %s ports\n",
2565                                port_id, direction ? "Rx" : "Tx");
2566
2567         return ret;
2568 }
2569
2570 void
2571 rte_eth_tx_buffer_drop_callback(struct rte_mbuf **pkts, uint16_t unsent,
2572                 void *userdata __rte_unused)
2573 {
2574         rte_pktmbuf_free_bulk(pkts, unsent);
2575 }
2576
2577 void
2578 rte_eth_tx_buffer_count_callback(struct rte_mbuf **pkts, uint16_t unsent,
2579                 void *userdata)
2580 {
2581         uint64_t *count = userdata;
2582
2583         rte_pktmbuf_free_bulk(pkts, unsent);
2584         *count += unsent;
2585 }
2586
2587 int
2588 rte_eth_tx_buffer_set_err_callback(struct rte_eth_dev_tx_buffer *buffer,
2589                 buffer_tx_error_fn cbfn, void *userdata)
2590 {
2591         if (buffer == NULL) {
2592                 RTE_ETHDEV_LOG(ERR,
2593                         "Cannot set Tx buffer error callback to NULL buffer\n");
2594                 return -EINVAL;
2595         }
2596
2597         buffer->error_callback = cbfn;
2598         buffer->error_userdata = userdata;
2599         return 0;
2600 }
2601
2602 int
2603 rte_eth_tx_buffer_init(struct rte_eth_dev_tx_buffer *buffer, uint16_t size)
2604 {
2605         int ret = 0;
2606
2607         if (buffer == NULL) {
2608                 RTE_ETHDEV_LOG(ERR, "Cannot initialize NULL buffer\n");
2609                 return -EINVAL;
2610         }
2611
2612         buffer->size = size;
2613         if (buffer->error_callback == NULL) {
2614                 ret = rte_eth_tx_buffer_set_err_callback(
2615                         buffer, rte_eth_tx_buffer_drop_callback, NULL);
2616         }
2617
2618         return ret;
2619 }
2620
2621 int
2622 rte_eth_tx_done_cleanup(uint16_t port_id, uint16_t queue_id, uint32_t free_cnt)
2623 {
2624         struct rte_eth_dev *dev;
2625         int ret;
2626
2627         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2628         dev = &rte_eth_devices[port_id];
2629
2630         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_done_cleanup, -ENOTSUP);
2631
2632         /* Call driver to free pending mbufs. */
2633         ret = (*dev->dev_ops->tx_done_cleanup)(dev->data->tx_queues[queue_id],
2634                                                free_cnt);
2635         return eth_err(port_id, ret);
2636 }
2637
2638 int
2639 rte_eth_promiscuous_enable(uint16_t port_id)
2640 {
2641         struct rte_eth_dev *dev;
2642         int diag = 0;
2643
2644         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2645         dev = &rte_eth_devices[port_id];
2646
2647         if (dev->data->promiscuous == 1)
2648                 return 0;
2649
2650         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->promiscuous_enable, -ENOTSUP);
2651
2652         diag = (*dev->dev_ops->promiscuous_enable)(dev);
2653         dev->data->promiscuous = (diag == 0) ? 1 : 0;
2654
2655         return eth_err(port_id, diag);
2656 }
2657
2658 int
2659 rte_eth_promiscuous_disable(uint16_t port_id)
2660 {
2661         struct rte_eth_dev *dev;
2662         int diag = 0;
2663
2664         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2665         dev = &rte_eth_devices[port_id];
2666
2667         if (dev->data->promiscuous == 0)
2668                 return 0;
2669
2670         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->promiscuous_disable, -ENOTSUP);
2671
2672         dev->data->promiscuous = 0;
2673         diag = (*dev->dev_ops->promiscuous_disable)(dev);
2674         if (diag != 0)
2675                 dev->data->promiscuous = 1;
2676
2677         return eth_err(port_id, diag);
2678 }
2679
2680 int
2681 rte_eth_promiscuous_get(uint16_t port_id)
2682 {
2683         struct rte_eth_dev *dev;
2684
2685         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2686         dev = &rte_eth_devices[port_id];
2687
2688         return dev->data->promiscuous;
2689 }
2690
2691 int
2692 rte_eth_allmulticast_enable(uint16_t port_id)
2693 {
2694         struct rte_eth_dev *dev;
2695         int diag;
2696
2697         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2698         dev = &rte_eth_devices[port_id];
2699
2700         if (dev->data->all_multicast == 1)
2701                 return 0;
2702
2703         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->allmulticast_enable, -ENOTSUP);
2704         diag = (*dev->dev_ops->allmulticast_enable)(dev);
2705         dev->data->all_multicast = (diag == 0) ? 1 : 0;
2706
2707         return eth_err(port_id, diag);
2708 }
2709
2710 int
2711 rte_eth_allmulticast_disable(uint16_t port_id)
2712 {
2713         struct rte_eth_dev *dev;
2714         int diag;
2715
2716         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2717         dev = &rte_eth_devices[port_id];
2718
2719         if (dev->data->all_multicast == 0)
2720                 return 0;
2721
2722         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->allmulticast_disable, -ENOTSUP);
2723         dev->data->all_multicast = 0;
2724         diag = (*dev->dev_ops->allmulticast_disable)(dev);
2725         if (diag != 0)
2726                 dev->data->all_multicast = 1;
2727
2728         return eth_err(port_id, diag);
2729 }
2730
2731 int
2732 rte_eth_allmulticast_get(uint16_t port_id)
2733 {
2734         struct rte_eth_dev *dev;
2735
2736         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2737         dev = &rte_eth_devices[port_id];
2738
2739         return dev->data->all_multicast;
2740 }
2741
2742 int
2743 rte_eth_link_get(uint16_t port_id, struct rte_eth_link *eth_link)
2744 {
2745         struct rte_eth_dev *dev;
2746
2747         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2748         dev = &rte_eth_devices[port_id];
2749
2750         if (eth_link == NULL) {
2751                 RTE_ETHDEV_LOG(ERR, "Cannot get ethdev port %u link to NULL\n",
2752                         port_id);
2753                 return -EINVAL;
2754         }
2755
2756         if (dev->data->dev_conf.intr_conf.lsc && dev->data->dev_started)
2757                 rte_eth_linkstatus_get(dev, eth_link);
2758         else {
2759                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
2760                 (*dev->dev_ops->link_update)(dev, 1);
2761                 *eth_link = dev->data->dev_link;
2762         }
2763
2764         return 0;
2765 }
2766
2767 int
2768 rte_eth_link_get_nowait(uint16_t port_id, struct rte_eth_link *eth_link)
2769 {
2770         struct rte_eth_dev *dev;
2771
2772         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2773         dev = &rte_eth_devices[port_id];
2774
2775         if (eth_link == NULL) {
2776                 RTE_ETHDEV_LOG(ERR, "Cannot get ethdev port %u link to NULL\n",
2777                         port_id);
2778                 return -EINVAL;
2779         }
2780
2781         if (dev->data->dev_conf.intr_conf.lsc && dev->data->dev_started)
2782                 rte_eth_linkstatus_get(dev, eth_link);
2783         else {
2784                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
2785                 (*dev->dev_ops->link_update)(dev, 0);
2786                 *eth_link = dev->data->dev_link;
2787         }
2788
2789         return 0;
2790 }
2791
2792 const char *
2793 rte_eth_link_speed_to_str(uint32_t link_speed)
2794 {
2795         switch (link_speed) {
2796         case ETH_SPEED_NUM_NONE: return "None";
2797         case ETH_SPEED_NUM_10M:  return "10 Mbps";
2798         case ETH_SPEED_NUM_100M: return "100 Mbps";
2799         case ETH_SPEED_NUM_1G:   return "1 Gbps";
2800         case ETH_SPEED_NUM_2_5G: return "2.5 Gbps";
2801         case ETH_SPEED_NUM_5G:   return "5 Gbps";
2802         case ETH_SPEED_NUM_10G:  return "10 Gbps";
2803         case ETH_SPEED_NUM_20G:  return "20 Gbps";
2804         case ETH_SPEED_NUM_25G:  return "25 Gbps";
2805         case ETH_SPEED_NUM_40G:  return "40 Gbps";
2806         case ETH_SPEED_NUM_50G:  return "50 Gbps";
2807         case ETH_SPEED_NUM_56G:  return "56 Gbps";
2808         case ETH_SPEED_NUM_100G: return "100 Gbps";
2809         case ETH_SPEED_NUM_200G: return "200 Gbps";
2810         case ETH_SPEED_NUM_UNKNOWN: return "Unknown";
2811         default: return "Invalid";
2812         }
2813 }
2814
2815 int
2816 rte_eth_link_to_str(char *str, size_t len, const struct rte_eth_link *eth_link)
2817 {
2818         if (str == NULL) {
2819                 RTE_ETHDEV_LOG(ERR, "Cannot convert link to NULL string\n");
2820                 return -EINVAL;
2821         }
2822
2823         if (len == 0) {
2824                 RTE_ETHDEV_LOG(ERR,
2825                         "Cannot convert link to string with zero size\n");
2826                 return -EINVAL;
2827         }
2828
2829         if (eth_link == NULL) {
2830                 RTE_ETHDEV_LOG(ERR, "Cannot convert to string from NULL link\n");
2831                 return -EINVAL;
2832         }
2833
2834         if (eth_link->link_status == ETH_LINK_DOWN)
2835                 return snprintf(str, len, "Link down");
2836         else
2837                 return snprintf(str, len, "Link up at %s %s %s",
2838                         rte_eth_link_speed_to_str(eth_link->link_speed),
2839                         (eth_link->link_duplex == ETH_LINK_FULL_DUPLEX) ?
2840                         "FDX" : "HDX",
2841                         (eth_link->link_autoneg == ETH_LINK_AUTONEG) ?
2842                         "Autoneg" : "Fixed");
2843 }
2844
2845 int
2846 rte_eth_stats_get(uint16_t port_id, struct rte_eth_stats *stats)
2847 {
2848         struct rte_eth_dev *dev;
2849
2850         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2851         dev = &rte_eth_devices[port_id];
2852
2853         if (stats == NULL) {
2854                 RTE_ETHDEV_LOG(ERR, "Cannot get ethdev port %u stats to NULL\n",
2855                         port_id);
2856                 return -EINVAL;
2857         }
2858
2859         memset(stats, 0, sizeof(*stats));
2860
2861         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
2862         stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
2863         return eth_err(port_id, (*dev->dev_ops->stats_get)(dev, stats));
2864 }
2865
2866 int
2867 rte_eth_stats_reset(uint16_t port_id)
2868 {
2869         struct rte_eth_dev *dev;
2870         int ret;
2871
2872         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2873         dev = &rte_eth_devices[port_id];
2874
2875         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_reset, -ENOTSUP);
2876         ret = (*dev->dev_ops->stats_reset)(dev);
2877         if (ret != 0)
2878                 return eth_err(port_id, ret);
2879
2880         dev->data->rx_mbuf_alloc_failed = 0;
2881
2882         return 0;
2883 }
2884
2885 static inline int
2886 eth_dev_get_xstats_basic_count(struct rte_eth_dev *dev)
2887 {
2888         uint16_t nb_rxqs, nb_txqs;
2889         int count;
2890
2891         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2892         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2893
2894         count = RTE_NB_STATS;
2895         if (dev->data->dev_flags & RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS) {
2896                 count += nb_rxqs * RTE_NB_RXQ_STATS;
2897                 count += nb_txqs * RTE_NB_TXQ_STATS;
2898         }
2899
2900         return count;
2901 }
2902
2903 static int
2904 eth_dev_get_xstats_count(uint16_t port_id)
2905 {
2906         struct rte_eth_dev *dev;
2907         int count;
2908
2909         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2910         dev = &rte_eth_devices[port_id];
2911         if (dev->dev_ops->xstats_get_names != NULL) {
2912                 count = (*dev->dev_ops->xstats_get_names)(dev, NULL, 0);
2913                 if (count < 0)
2914                         return eth_err(port_id, count);
2915         } else
2916                 count = 0;
2917
2918
2919         count += eth_dev_get_xstats_basic_count(dev);
2920
2921         return count;
2922 }
2923
2924 int
2925 rte_eth_xstats_get_id_by_name(uint16_t port_id, const char *xstat_name,
2926                 uint64_t *id)
2927 {
2928         int cnt_xstats, idx_xstat;
2929
2930         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2931
2932         if (xstat_name == NULL) {
2933                 RTE_ETHDEV_LOG(ERR,
2934                         "Cannot get ethdev port %u xstats ID from NULL xstat name\n",
2935                         port_id);
2936                 return -ENOMEM;
2937         }
2938
2939         if (id == NULL) {
2940                 RTE_ETHDEV_LOG(ERR,
2941                         "Cannot get ethdev port %u xstats ID to NULL\n",
2942                         port_id);
2943                 return -ENOMEM;
2944         }
2945
2946         /* Get count */
2947         cnt_xstats = rte_eth_xstats_get_names_by_id(port_id, NULL, 0, NULL);
2948         if (cnt_xstats  < 0) {
2949                 RTE_ETHDEV_LOG(ERR, "Cannot get count of xstats\n");
2950                 return -ENODEV;
2951         }
2952
2953         /* Get id-name lookup table */
2954         struct rte_eth_xstat_name xstats_names[cnt_xstats];
2955
2956         if (cnt_xstats != rte_eth_xstats_get_names_by_id(
2957                         port_id, xstats_names, cnt_xstats, NULL)) {
2958                 RTE_ETHDEV_LOG(ERR, "Cannot get xstats lookup\n");
2959                 return -1;
2960         }
2961
2962         for (idx_xstat = 0; idx_xstat < cnt_xstats; idx_xstat++) {
2963                 if (!strcmp(xstats_names[idx_xstat].name, xstat_name)) {
2964                         *id = idx_xstat;
2965                         return 0;
2966                 };
2967         }
2968
2969         return -EINVAL;
2970 }
2971
2972 /* retrieve basic stats names */
2973 static int
2974 eth_basic_stats_get_names(struct rte_eth_dev *dev,
2975         struct rte_eth_xstat_name *xstats_names)
2976 {
2977         int cnt_used_entries = 0;
2978         uint32_t idx, id_queue;
2979         uint16_t num_q;
2980
2981         for (idx = 0; idx < RTE_NB_STATS; idx++) {
2982                 strlcpy(xstats_names[cnt_used_entries].name,
2983                         eth_dev_stats_strings[idx].name,
2984                         sizeof(xstats_names[0].name));
2985                 cnt_used_entries++;
2986         }
2987
2988         if ((dev->data->dev_flags & RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS) == 0)
2989                 return cnt_used_entries;
2990
2991         num_q = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2992         for (id_queue = 0; id_queue < num_q; id_queue++) {
2993                 for (idx = 0; idx < RTE_NB_RXQ_STATS; idx++) {
2994                         snprintf(xstats_names[cnt_used_entries].name,
2995                                 sizeof(xstats_names[0].name),
2996                                 "rx_q%u_%s",
2997                                 id_queue, eth_dev_rxq_stats_strings[idx].name);
2998                         cnt_used_entries++;
2999                 }
3000
3001         }
3002         num_q = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
3003         for (id_queue = 0; id_queue < num_q; id_queue++) {
3004                 for (idx = 0; idx < RTE_NB_TXQ_STATS; idx++) {
3005                         snprintf(xstats_names[cnt_used_entries].name,
3006                                 sizeof(xstats_names[0].name),
3007                                 "tx_q%u_%s",
3008                                 id_queue, eth_dev_txq_stats_strings[idx].name);
3009                         cnt_used_entries++;
3010                 }
3011         }
3012         return cnt_used_entries;
3013 }
3014
3015 /* retrieve ethdev extended statistics names */
3016 int
3017 rte_eth_xstats_get_names_by_id(uint16_t port_id,
3018         struct rte_eth_xstat_name *xstats_names, unsigned int size,
3019         uint64_t *ids)
3020 {
3021         struct rte_eth_xstat_name *xstats_names_copy;
3022         unsigned int no_basic_stat_requested = 1;
3023         unsigned int no_ext_stat_requested = 1;
3024         unsigned int expected_entries;
3025         unsigned int basic_count;
3026         struct rte_eth_dev *dev;
3027         unsigned int i;
3028         int ret;
3029
3030         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3031         dev = &rte_eth_devices[port_id];
3032
3033         basic_count = eth_dev_get_xstats_basic_count(dev);
3034         ret = eth_dev_get_xstats_count(port_id);
3035         if (ret < 0)
3036                 return ret;
3037         expected_entries = (unsigned int)ret;
3038
3039         /* Return max number of stats if no ids given */
3040         if (!ids) {
3041                 if (!xstats_names)
3042                         return expected_entries;
3043                 else if (xstats_names && size < expected_entries)
3044                         return expected_entries;
3045         }
3046
3047         if (ids && !xstats_names)
3048                 return -EINVAL;
3049
3050         if (ids && dev->dev_ops->xstats_get_names_by_id != NULL && size > 0) {
3051                 uint64_t ids_copy[size];
3052
3053                 for (i = 0; i < size; i++) {
3054                         if (ids[i] < basic_count) {
3055                                 no_basic_stat_requested = 0;
3056                                 break;
3057                         }
3058
3059                         /*
3060                          * Convert ids to xstats ids that PMD knows.
3061                          * ids known by user are basic + extended stats.
3062                          */
3063                         ids_copy[i] = ids[i] - basic_count;
3064                 }
3065
3066                 if (no_basic_stat_requested)
3067                         return (*dev->dev_ops->xstats_get_names_by_id)(dev,
3068                                         ids_copy, xstats_names, size);
3069         }
3070
3071         /* Retrieve all stats */
3072         if (!ids) {
3073                 int num_stats = rte_eth_xstats_get_names(port_id, xstats_names,
3074                                 expected_entries);
3075                 if (num_stats < 0 || num_stats > (int)expected_entries)
3076                         return num_stats;
3077                 else
3078                         return expected_entries;
3079         }
3080
3081         xstats_names_copy = calloc(expected_entries,
3082                 sizeof(struct rte_eth_xstat_name));
3083
3084         if (!xstats_names_copy) {
3085                 RTE_ETHDEV_LOG(ERR, "Can't allocate memory\n");
3086                 return -ENOMEM;
3087         }
3088
3089         if (ids) {
3090                 for (i = 0; i < size; i++) {
3091                         if (ids[i] >= basic_count) {
3092                                 no_ext_stat_requested = 0;
3093                                 break;
3094                         }
3095                 }
3096         }
3097
3098         /* Fill xstats_names_copy structure */
3099         if (ids && no_ext_stat_requested) {
3100                 eth_basic_stats_get_names(dev, xstats_names_copy);
3101         } else {
3102                 ret = rte_eth_xstats_get_names(port_id, xstats_names_copy,
3103                         expected_entries);
3104                 if (ret < 0) {
3105                         free(xstats_names_copy);
3106                         return ret;
3107                 }
3108         }
3109
3110         /* Filter stats */
3111         for (i = 0; i < size; i++) {
3112                 if (ids[i] >= expected_entries) {
3113                         RTE_ETHDEV_LOG(ERR, "Id value isn't valid\n");
3114                         free(xstats_names_copy);
3115                         return -1;
3116                 }
3117                 xstats_names[i] = xstats_names_copy[ids[i]];
3118         }
3119
3120         free(xstats_names_copy);
3121         return size;
3122 }
3123
3124 int
3125 rte_eth_xstats_get_names(uint16_t port_id,
3126         struct rte_eth_xstat_name *xstats_names,
3127         unsigned int size)
3128 {
3129         struct rte_eth_dev *dev;
3130         int cnt_used_entries;
3131         int cnt_expected_entries;
3132         int cnt_driver_entries;
3133
3134         cnt_expected_entries = eth_dev_get_xstats_count(port_id);
3135         if (xstats_names == NULL || cnt_expected_entries < 0 ||
3136                         (int)size < cnt_expected_entries)
3137                 return cnt_expected_entries;
3138
3139         /* port_id checked in eth_dev_get_xstats_count() */
3140         dev = &rte_eth_devices[port_id];
3141
3142         cnt_used_entries = eth_basic_stats_get_names(dev, xstats_names);
3143
3144         if (dev->dev_ops->xstats_get_names != NULL) {
3145                 /* If there are any driver-specific xstats, append them
3146                  * to end of list.
3147                  */
3148                 cnt_driver_entries = (*dev->dev_ops->xstats_get_names)(
3149                         dev,
3150                         xstats_names + cnt_used_entries,
3151                         size - cnt_used_entries);
3152                 if (cnt_driver_entries < 0)
3153                         return eth_err(port_id, cnt_driver_entries);
3154                 cnt_used_entries += cnt_driver_entries;
3155         }
3156
3157         return cnt_used_entries;
3158 }
3159
3160
3161 static int
3162 eth_basic_stats_get(uint16_t port_id, struct rte_eth_xstat *xstats)
3163 {
3164         struct rte_eth_dev *dev;
3165         struct rte_eth_stats eth_stats;
3166         unsigned int count = 0, i, q;
3167         uint64_t val, *stats_ptr;
3168         uint16_t nb_rxqs, nb_txqs;
3169         int ret;
3170
3171         ret = rte_eth_stats_get(port_id, &eth_stats);
3172         if (ret < 0)
3173                 return ret;
3174
3175         dev = &rte_eth_devices[port_id];
3176
3177         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
3178         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
3179
3180         /* global stats */
3181         for (i = 0; i < RTE_NB_STATS; i++) {
3182                 stats_ptr = RTE_PTR_ADD(&eth_stats,
3183                                         eth_dev_stats_strings[i].offset);
3184                 val = *stats_ptr;
3185                 xstats[count++].value = val;
3186         }
3187
3188         if ((dev->data->dev_flags & RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS) == 0)
3189                 return count;
3190
3191         /* per-rxq stats */
3192         for (q = 0; q < nb_rxqs; q++) {
3193                 for (i = 0; i < RTE_NB_RXQ_STATS; i++) {
3194                         stats_ptr = RTE_PTR_ADD(&eth_stats,
3195                                         eth_dev_rxq_stats_strings[i].offset +
3196                                         q * sizeof(uint64_t));
3197                         val = *stats_ptr;
3198                         xstats[count++].value = val;
3199                 }
3200         }
3201
3202         /* per-txq stats */
3203         for (q = 0; q < nb_txqs; q++) {
3204                 for (i = 0; i < RTE_NB_TXQ_STATS; i++) {
3205                         stats_ptr = RTE_PTR_ADD(&eth_stats,
3206                                         eth_dev_txq_stats_strings[i].offset +
3207                                         q * sizeof(uint64_t));
3208                         val = *stats_ptr;
3209                         xstats[count++].value = val;
3210                 }
3211         }
3212         return count;
3213 }
3214
3215 /* retrieve ethdev extended statistics */
3216 int
3217 rte_eth_xstats_get_by_id(uint16_t port_id, const uint64_t *ids,
3218                          uint64_t *values, unsigned int size)
3219 {
3220         unsigned int no_basic_stat_requested = 1;
3221         unsigned int no_ext_stat_requested = 1;
3222         unsigned int num_xstats_filled;
3223         unsigned int basic_count;
3224         uint16_t expected_entries;
3225         struct rte_eth_dev *dev;
3226         unsigned int i;
3227         int ret;
3228
3229         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3230         dev = &rte_eth_devices[port_id];
3231
3232         ret = eth_dev_get_xstats_count(port_id);
3233         if (ret < 0)
3234                 return ret;
3235         expected_entries = (uint16_t)ret;
3236         struct rte_eth_xstat xstats[expected_entries];
3237         basic_count = eth_dev_get_xstats_basic_count(dev);
3238
3239         /* Return max number of stats if no ids given */
3240         if (!ids) {
3241                 if (!values)
3242                         return expected_entries;
3243                 else if (values && size < expected_entries)
3244                         return expected_entries;
3245         }
3246
3247         if (ids && !values)
3248                 return -EINVAL;
3249
3250         if (ids && dev->dev_ops->xstats_get_by_id != NULL && size) {
3251                 unsigned int basic_count = eth_dev_get_xstats_basic_count(dev);
3252                 uint64_t ids_copy[size];
3253
3254                 for (i = 0; i < size; i++) {
3255                         if (ids[i] < basic_count) {
3256                                 no_basic_stat_requested = 0;
3257                                 break;
3258                         }
3259
3260                         /*
3261                          * Convert ids to xstats ids that PMD knows.
3262                          * ids known by user are basic + extended stats.
3263                          */
3264                         ids_copy[i] = ids[i] - basic_count;
3265                 }
3266
3267                 if (no_basic_stat_requested)
3268                         return (*dev->dev_ops->xstats_get_by_id)(dev, ids_copy,
3269                                         values, size);
3270         }
3271
3272         if (ids) {
3273                 for (i = 0; i < size; i++) {
3274                         if (ids[i] >= basic_count) {
3275                                 no_ext_stat_requested = 0;
3276                                 break;
3277                         }
3278                 }
3279         }
3280
3281         /* Fill the xstats structure */
3282         if (ids && no_ext_stat_requested)
3283                 ret = eth_basic_stats_get(port_id, xstats);
3284         else
3285                 ret = rte_eth_xstats_get(port_id, xstats, expected_entries);
3286
3287         if (ret < 0)
3288                 return ret;
3289         num_xstats_filled = (unsigned int)ret;
3290
3291         /* Return all stats */
3292         if (!ids) {
3293                 for (i = 0; i < num_xstats_filled; i++)
3294                         values[i] = xstats[i].value;
3295                 return expected_entries;
3296         }
3297
3298         /* Filter stats */
3299         for (i = 0; i < size; i++) {
3300                 if (ids[i] >= expected_entries) {
3301                         RTE_ETHDEV_LOG(ERR, "Id value isn't valid\n");
3302                         return -1;
3303                 }
3304                 values[i] = xstats[ids[i]].value;
3305         }
3306         return size;
3307 }
3308
3309 int
3310 rte_eth_xstats_get(uint16_t port_id, struct rte_eth_xstat *xstats,
3311         unsigned int n)
3312 {
3313         struct rte_eth_dev *dev;
3314         unsigned int count = 0, i;
3315         signed int xcount = 0;
3316         uint16_t nb_rxqs, nb_txqs;
3317         int ret;
3318
3319         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3320         dev = &rte_eth_devices[port_id];
3321
3322         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
3323         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
3324
3325         /* Return generic statistics */
3326         count = RTE_NB_STATS;
3327         if (dev->data->dev_flags & RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS)
3328                 count += (nb_rxqs * RTE_NB_RXQ_STATS) + (nb_txqs * RTE_NB_TXQ_STATS);
3329
3330         /* implemented by the driver */
3331         if (dev->dev_ops->xstats_get != NULL) {
3332                 /* Retrieve the xstats from the driver at the end of the
3333                  * xstats struct.
3334                  */
3335                 xcount = (*dev->dev_ops->xstats_get)(dev,
3336                                      xstats ? xstats + count : NULL,
3337                                      (n > count) ? n - count : 0);
3338
3339                 if (xcount < 0)
3340                         return eth_err(port_id, xcount);
3341         }
3342
3343         if (n < count + xcount || xstats == NULL)
3344                 return count + xcount;
3345
3346         /* now fill the xstats structure */
3347         ret = eth_basic_stats_get(port_id, xstats);
3348         if (ret < 0)
3349                 return ret;
3350         count = ret;
3351
3352         for (i = 0; i < count; i++)
3353                 xstats[i].id = i;
3354         /* add an offset to driver-specific stats */
3355         for ( ; i < count + xcount; i++)
3356                 xstats[i].id += count;
3357
3358         return count + xcount;
3359 }
3360
3361 /* reset ethdev extended statistics */
3362 int
3363 rte_eth_xstats_reset(uint16_t port_id)
3364 {
3365         struct rte_eth_dev *dev;
3366
3367         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3368         dev = &rte_eth_devices[port_id];
3369
3370         /* implemented by the driver */
3371         if (dev->dev_ops->xstats_reset != NULL)
3372                 return eth_err(port_id, (*dev->dev_ops->xstats_reset)(dev));
3373
3374         /* fallback to default */
3375         return rte_eth_stats_reset(port_id);
3376 }
3377
3378 static int
3379 eth_dev_set_queue_stats_mapping(uint16_t port_id, uint16_t queue_id,
3380                 uint8_t stat_idx, uint8_t is_rx)
3381 {
3382         struct rte_eth_dev *dev;
3383
3384         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3385         dev = &rte_eth_devices[port_id];
3386
3387         if (is_rx && (queue_id >= dev->data->nb_rx_queues))
3388                 return -EINVAL;
3389
3390         if (!is_rx && (queue_id >= dev->data->nb_tx_queues))
3391                 return -EINVAL;
3392
3393         if (stat_idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
3394                 return -EINVAL;
3395
3396         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_stats_mapping_set, -ENOTSUP);
3397         return (*dev->dev_ops->queue_stats_mapping_set) (dev, queue_id, stat_idx, is_rx);
3398 }
3399
3400 int
3401 rte_eth_dev_set_tx_queue_stats_mapping(uint16_t port_id, uint16_t tx_queue_id,
3402                 uint8_t stat_idx)
3403 {
3404         return eth_err(port_id, eth_dev_set_queue_stats_mapping(port_id,
3405                                                 tx_queue_id,
3406                                                 stat_idx, STAT_QMAP_TX));
3407 }
3408
3409 int
3410 rte_eth_dev_set_rx_queue_stats_mapping(uint16_t port_id, uint16_t rx_queue_id,
3411                 uint8_t stat_idx)
3412 {
3413         return eth_err(port_id, eth_dev_set_queue_stats_mapping(port_id,
3414                                                 rx_queue_id,
3415                                                 stat_idx, STAT_QMAP_RX));
3416 }
3417
3418 int
3419 rte_eth_dev_fw_version_get(uint16_t port_id, char *fw_version, size_t fw_size)
3420 {
3421         struct rte_eth_dev *dev;
3422
3423         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3424         dev = &rte_eth_devices[port_id];
3425
3426         if (fw_version == NULL && fw_size > 0) {
3427                 RTE_ETHDEV_LOG(ERR,
3428                         "Cannot get ethdev port %u FW version to NULL when string size is non zero\n",
3429                         port_id);
3430                 return -EINVAL;
3431         }
3432
3433         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fw_version_get, -ENOTSUP);
3434         return eth_err(port_id, (*dev->dev_ops->fw_version_get)(dev,
3435                                                         fw_version, fw_size));
3436 }
3437
3438 int
3439 rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info)
3440 {
3441         struct rte_eth_dev *dev;
3442         const struct rte_eth_desc_lim lim = {
3443                 .nb_max = UINT16_MAX,
3444                 .nb_min = 0,
3445                 .nb_align = 1,
3446                 .nb_seg_max = UINT16_MAX,
3447                 .nb_mtu_seg_max = UINT16_MAX,
3448         };
3449         int diag;
3450
3451         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3452         dev = &rte_eth_devices[port_id];
3453
3454         if (dev_info == NULL) {
3455                 RTE_ETHDEV_LOG(ERR, "Cannot get ethdev port %u info to NULL\n",
3456                         port_id);
3457                 return -EINVAL;
3458         }
3459
3460         /*
3461          * Init dev_info before port_id check since caller does not have
3462          * return status and does not know if get is successful or not.
3463          */
3464         memset(dev_info, 0, sizeof(struct rte_eth_dev_info));
3465         dev_info->switch_info.domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
3466
3467         dev_info->rx_desc_lim = lim;
3468         dev_info->tx_desc_lim = lim;
3469         dev_info->device = dev->device;
3470         dev_info->min_mtu = RTE_ETHER_MIN_LEN - RTE_ETHER_HDR_LEN -
3471                 RTE_ETHER_CRC_LEN;
3472         dev_info->max_mtu = UINT16_MAX;
3473
3474         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
3475         diag = (*dev->dev_ops->dev_infos_get)(dev, dev_info);
3476         if (diag != 0) {
3477                 /* Cleanup already filled in device information */
3478                 memset(dev_info, 0, sizeof(struct rte_eth_dev_info));
3479                 return eth_err(port_id, diag);
3480         }
3481
3482         /* Maximum number of queues should be <= RTE_MAX_QUEUES_PER_PORT */
3483         dev_info->max_rx_queues = RTE_MIN(dev_info->max_rx_queues,
3484                         RTE_MAX_QUEUES_PER_PORT);
3485         dev_info->max_tx_queues = RTE_MIN(dev_info->max_tx_queues,
3486                         RTE_MAX_QUEUES_PER_PORT);
3487
3488         dev_info->driver_name = dev->device->driver->name;
3489         dev_info->nb_rx_queues = dev->data->nb_rx_queues;
3490         dev_info->nb_tx_queues = dev->data->nb_tx_queues;
3491
3492         dev_info->dev_flags = &dev->data->dev_flags;
3493
3494         return 0;
3495 }
3496
3497 int
3498 rte_eth_dev_conf_get(uint16_t port_id, struct rte_eth_conf *dev_conf)
3499 {
3500         struct rte_eth_dev *dev;
3501
3502         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3503         dev = &rte_eth_devices[port_id];
3504
3505         if (dev_conf == NULL) {
3506                 RTE_ETHDEV_LOG(ERR,
3507                         "Cannot get ethdev port %u configuration to NULL\n",
3508                         port_id);
3509                 return -EINVAL;
3510         }
3511
3512         memcpy(dev_conf, &dev->data->dev_conf, sizeof(struct rte_eth_conf));
3513
3514         return 0;
3515 }
3516
3517 int
3518 rte_eth_dev_get_supported_ptypes(uint16_t port_id, uint32_t ptype_mask,
3519                                  uint32_t *ptypes, int num)
3520 {
3521         int i, j;
3522         struct rte_eth_dev *dev;
3523         const uint32_t *all_ptypes;
3524
3525         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3526         dev = &rte_eth_devices[port_id];
3527
3528         if (ptypes == NULL && num > 0) {
3529                 RTE_ETHDEV_LOG(ERR,
3530                         "Cannot get ethdev port %u supported packet types to NULL when array size is non zero\n",
3531                         port_id);
3532                 return -EINVAL;
3533         }
3534
3535         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_supported_ptypes_get, 0);
3536         all_ptypes = (*dev->dev_ops->dev_supported_ptypes_get)(dev);
3537
3538         if (!all_ptypes)
3539                 return 0;
3540
3541         for (i = 0, j = 0; all_ptypes[i] != RTE_PTYPE_UNKNOWN; ++i)
3542                 if (all_ptypes[i] & ptype_mask) {
3543                         if (j < num)
3544                                 ptypes[j] = all_ptypes[i];
3545                         j++;
3546                 }
3547
3548         return j;
3549 }
3550
3551 int
3552 rte_eth_dev_set_ptypes(uint16_t port_id, uint32_t ptype_mask,
3553                                  uint32_t *set_ptypes, unsigned int num)
3554 {
3555         const uint32_t valid_ptype_masks[] = {
3556                 RTE_PTYPE_L2_MASK,
3557                 RTE_PTYPE_L3_MASK,
3558                 RTE_PTYPE_L4_MASK,
3559                 RTE_PTYPE_TUNNEL_MASK,
3560                 RTE_PTYPE_INNER_L2_MASK,
3561                 RTE_PTYPE_INNER_L3_MASK,
3562                 RTE_PTYPE_INNER_L4_MASK,
3563         };
3564         const uint32_t *all_ptypes;
3565         struct rte_eth_dev *dev;
3566         uint32_t unused_mask;
3567         unsigned int i, j;
3568         int ret;
3569
3570         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3571         dev = &rte_eth_devices[port_id];
3572
3573         if (num > 0 && set_ptypes == NULL) {
3574                 RTE_ETHDEV_LOG(ERR,
3575                         "Cannot get ethdev port %u set packet types to NULL when array size is non zero\n",
3576                         port_id);
3577                 return -EINVAL;
3578         }
3579
3580         if (*dev->dev_ops->dev_supported_ptypes_get == NULL ||
3581                         *dev->dev_ops->dev_ptypes_set == NULL) {
3582                 ret = 0;
3583                 goto ptype_unknown;
3584         }
3585
3586         if (ptype_mask == 0) {
3587                 ret = (*dev->dev_ops->dev_ptypes_set)(dev,
3588                                 ptype_mask);
3589                 goto ptype_unknown;
3590         }
3591
3592         unused_mask = ptype_mask;
3593         for (i = 0; i < RTE_DIM(valid_ptype_masks); i++) {
3594                 uint32_t mask = ptype_mask & valid_ptype_masks[i];
3595                 if (mask && mask != valid_ptype_masks[i]) {
3596                         ret = -EINVAL;
3597                         goto ptype_unknown;
3598                 }
3599                 unused_mask &= ~valid_ptype_masks[i];
3600         }
3601
3602         if (unused_mask) {
3603                 ret = -EINVAL;
3604                 goto ptype_unknown;
3605         }
3606
3607         all_ptypes = (*dev->dev_ops->dev_supported_ptypes_get)(dev);
3608         if (all_ptypes == NULL) {
3609                 ret = 0;
3610                 goto ptype_unknown;
3611         }
3612
3613         /*
3614          * Accommodate as many set_ptypes as possible. If the supplied
3615          * set_ptypes array is insufficient fill it partially.
3616          */
3617         for (i = 0, j = 0; set_ptypes != NULL &&
3618                                 (all_ptypes[i] != RTE_PTYPE_UNKNOWN); ++i) {
3619                 if (ptype_mask & all_ptypes[i]) {
3620                         if (j < num - 1) {
3621                                 set_ptypes[j] = all_ptypes[i];
3622                                 j++;
3623                                 continue;
3624                         }
3625                         break;
3626                 }
3627         }
3628
3629         if (set_ptypes != NULL && j < num)
3630                 set_ptypes[j] = RTE_PTYPE_UNKNOWN;
3631
3632         return (*dev->dev_ops->dev_ptypes_set)(dev, ptype_mask);
3633
3634 ptype_unknown:
3635         if (num > 0)
3636                 set_ptypes[0] = RTE_PTYPE_UNKNOWN;
3637
3638         return ret;
3639 }
3640
3641 int
3642 rte_eth_macaddrs_get(uint16_t port_id, struct rte_ether_addr *ma,
3643         unsigned int num)
3644 {
3645         int32_t ret;
3646         struct rte_eth_dev *dev;
3647         struct rte_eth_dev_info dev_info;
3648
3649         if (ma == NULL) {
3650                 RTE_ETHDEV_LOG(ERR, "%s: invalid parameters\n", __func__);
3651                 return -EINVAL;
3652         }
3653
3654         /* will check for us that port_id is a valid one */
3655         ret = rte_eth_dev_info_get(port_id, &dev_info);
3656         if (ret != 0)
3657                 return ret;
3658
3659         dev = &rte_eth_devices[port_id];
3660         num = RTE_MIN(dev_info.max_mac_addrs, num);
3661         memcpy(ma, dev->data->mac_addrs, num * sizeof(ma[0]));
3662
3663         return num;
3664 }
3665
3666 int
3667 rte_eth_macaddr_get(uint16_t port_id, struct rte_ether_addr *mac_addr)
3668 {
3669         struct rte_eth_dev *dev;
3670
3671         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3672         dev = &rte_eth_devices[port_id];
3673
3674         if (mac_addr == NULL) {
3675                 RTE_ETHDEV_LOG(ERR,
3676                         "Cannot get ethdev port %u MAC address to NULL\n",
3677                         port_id);
3678                 return -EINVAL;
3679         }
3680
3681         rte_ether_addr_copy(&dev->data->mac_addrs[0], mac_addr);
3682
3683         return 0;
3684 }
3685
3686 int
3687 rte_eth_dev_get_mtu(uint16_t port_id, uint16_t *mtu)
3688 {
3689         struct rte_eth_dev *dev;
3690
3691         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3692         dev = &rte_eth_devices[port_id];
3693
3694         if (mtu == NULL) {
3695                 RTE_ETHDEV_LOG(ERR, "Cannot get ethdev port %u MTU to NULL\n",
3696                         port_id);
3697                 return -EINVAL;
3698         }
3699
3700         *mtu = dev->data->mtu;
3701         return 0;
3702 }
3703
3704 int
3705 rte_eth_dev_set_mtu(uint16_t port_id, uint16_t mtu)
3706 {
3707         int ret;
3708         struct rte_eth_dev_info dev_info;
3709         struct rte_eth_dev *dev;
3710
3711         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3712         dev = &rte_eth_devices[port_id];
3713         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mtu_set, -ENOTSUP);
3714
3715         /*
3716          * Check if the device supports dev_infos_get, if it does not
3717          * skip min_mtu/max_mtu validation here as this requires values
3718          * that are populated within the call to rte_eth_dev_info_get()
3719          * which relies on dev->dev_ops->dev_infos_get.
3720          */
3721         if (*dev->dev_ops->dev_infos_get != NULL) {
3722                 ret = rte_eth_dev_info_get(port_id, &dev_info);
3723                 if (ret != 0)
3724                         return ret;
3725
3726                 ret = eth_dev_validate_mtu(port_id, &dev_info, mtu);
3727                 if (ret != 0)
3728                         return ret;
3729         }
3730
3731         ret = (*dev->dev_ops->mtu_set)(dev, mtu);
3732         if (ret == 0)
3733                 dev->data->mtu = mtu;
3734
3735         return eth_err(port_id, ret);
3736 }
3737
3738 int
3739 rte_eth_dev_vlan_filter(uint16_t port_id, uint16_t vlan_id, int on)
3740 {
3741         struct rte_eth_dev *dev;
3742         int ret;
3743
3744         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3745         dev = &rte_eth_devices[port_id];
3746
3747         if (!(dev->data->dev_conf.rxmode.offloads &
3748               DEV_RX_OFFLOAD_VLAN_FILTER)) {
3749                 RTE_ETHDEV_LOG(ERR, "Port %u: VLAN-filtering disabled\n",
3750                         port_id);
3751                 return -ENOSYS;
3752         }
3753
3754         if (vlan_id > 4095) {
3755                 RTE_ETHDEV_LOG(ERR, "Port_id=%u invalid vlan_id=%u > 4095\n",
3756                         port_id, vlan_id);
3757                 return -EINVAL;
3758         }
3759         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_filter_set, -ENOTSUP);
3760
3761         ret = (*dev->dev_ops->vlan_filter_set)(dev, vlan_id, on);
3762         if (ret == 0) {
3763                 struct rte_vlan_filter_conf *vfc;
3764                 int vidx;
3765                 int vbit;
3766
3767                 vfc = &dev->data->vlan_filter_conf;
3768                 vidx = vlan_id / 64;
3769                 vbit = vlan_id % 64;
3770
3771                 if (on)
3772                         vfc->ids[vidx] |= RTE_BIT64(vbit);
3773                 else
3774                         vfc->ids[vidx] &= ~RTE_BIT64(vbit);
3775         }
3776
3777         return eth_err(port_id, ret);
3778 }
3779
3780 int
3781 rte_eth_dev_set_vlan_strip_on_queue(uint16_t port_id, uint16_t rx_queue_id,
3782                                     int on)
3783 {
3784         struct rte_eth_dev *dev;
3785
3786         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3787         dev = &rte_eth_devices[port_id];
3788
3789         if (rx_queue_id >= dev->data->nb_rx_queues) {
3790                 RTE_ETHDEV_LOG(ERR, "Invalid rx_queue_id=%u\n", rx_queue_id);
3791                 return -EINVAL;
3792         }
3793
3794         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_strip_queue_set, -ENOTSUP);
3795         (*dev->dev_ops->vlan_strip_queue_set)(dev, rx_queue_id, on);
3796
3797         return 0;
3798 }
3799
3800 int
3801 rte_eth_dev_set_vlan_ether_type(uint16_t port_id,
3802                                 enum rte_vlan_type vlan_type,
3803                                 uint16_t tpid)
3804 {
3805         struct rte_eth_dev *dev;
3806
3807         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3808         dev = &rte_eth_devices[port_id];
3809
3810         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_tpid_set, -ENOTSUP);
3811         return eth_err(port_id, (*dev->dev_ops->vlan_tpid_set)(dev, vlan_type,
3812                                                                tpid));
3813 }
3814
3815 int
3816 rte_eth_dev_set_vlan_offload(uint16_t port_id, int offload_mask)
3817 {
3818         struct rte_eth_dev_info dev_info;
3819         struct rte_eth_dev *dev;
3820         int ret = 0;
3821         int mask = 0;
3822         int cur, org = 0;
3823         uint64_t orig_offloads;
3824         uint64_t dev_offloads;
3825         uint64_t new_offloads;
3826
3827         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3828         dev = &rte_eth_devices[port_id];
3829
3830         /* save original values in case of failure */
3831         orig_offloads = dev->data->dev_conf.rxmode.offloads;
3832         dev_offloads = orig_offloads;
3833
3834         /* check which option changed by application */
3835         cur = !!(offload_mask & ETH_VLAN_STRIP_OFFLOAD);
3836         org = !!(dev_offloads & DEV_RX_OFFLOAD_VLAN_STRIP);
3837         if (cur != org) {
3838                 if (cur)
3839                         dev_offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
3840                 else
3841                         dev_offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP;
3842                 mask |= ETH_VLAN_STRIP_MASK;
3843         }
3844
3845         cur = !!(offload_mask & ETH_VLAN_FILTER_OFFLOAD);
3846         org = !!(dev_offloads & DEV_RX_OFFLOAD_VLAN_FILTER);
3847         if (cur != org) {
3848                 if (cur)
3849                         dev_offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3850                 else
3851                         dev_offloads &= ~DEV_RX_OFFLOAD_VLAN_FILTER;
3852                 mask |= ETH_VLAN_FILTER_MASK;
3853         }
3854
3855         cur = !!(offload_mask & ETH_VLAN_EXTEND_OFFLOAD);
3856         org = !!(dev_offloads & DEV_RX_OFFLOAD_VLAN_EXTEND);
3857         if (cur != org) {
3858                 if (cur)
3859                         dev_offloads |= DEV_RX_OFFLOAD_VLAN_EXTEND;
3860                 else
3861                         dev_offloads &= ~DEV_RX_OFFLOAD_VLAN_EXTEND;
3862                 mask |= ETH_VLAN_EXTEND_MASK;
3863         }
3864
3865         cur = !!(offload_mask & ETH_QINQ_STRIP_OFFLOAD);
3866         org = !!(dev_offloads & DEV_RX_OFFLOAD_QINQ_STRIP);
3867         if (cur != org) {
3868                 if (cur)
3869                         dev_offloads |= DEV_RX_OFFLOAD_QINQ_STRIP;
3870                 else
3871                         dev_offloads &= ~DEV_RX_OFFLOAD_QINQ_STRIP;
3872                 mask |= ETH_QINQ_STRIP_MASK;
3873         }
3874
3875         /*no change*/
3876         if (mask == 0)
3877                 return ret;
3878
3879         ret = rte_eth_dev_info_get(port_id, &dev_info);
3880         if (ret != 0)
3881                 return ret;
3882
3883         /* Rx VLAN offloading must be within its device capabilities */
3884         if ((dev_offloads & dev_info.rx_offload_capa) != dev_offloads) {
3885                 new_offloads = dev_offloads & ~orig_offloads;
3886                 RTE_ETHDEV_LOG(ERR,
3887                         "Ethdev port_id=%u requested new added VLAN offloads "
3888                         "0x%" PRIx64 " must be within Rx offloads capabilities "
3889                         "0x%" PRIx64 " in %s()\n",
3890                         port_id, new_offloads, dev_info.rx_offload_capa,
3891                         __func__);
3892                 return -EINVAL;
3893         }
3894
3895         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_offload_set, -ENOTSUP);
3896         dev->data->dev_conf.rxmode.offloads = dev_offloads;
3897         ret = (*dev->dev_ops->vlan_offload_set)(dev, mask);
3898         if (ret) {
3899                 /* hit an error restore  original values */
3900                 dev->data->dev_conf.rxmode.offloads = orig_offloads;
3901         }
3902
3903         return eth_err(port_id, ret);
3904 }
3905
3906 int
3907 rte_eth_dev_get_vlan_offload(uint16_t port_id)
3908 {
3909         struct rte_eth_dev *dev;
3910         uint64_t *dev_offloads;
3911         int ret = 0;
3912
3913         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3914         dev = &rte_eth_devices[port_id];
3915         dev_offloads = &dev->data->dev_conf.rxmode.offloads;
3916
3917         if (*dev_offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
3918                 ret |= ETH_VLAN_STRIP_OFFLOAD;
3919
3920         if (*dev_offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
3921                 ret |= ETH_VLAN_FILTER_OFFLOAD;
3922
3923         if (*dev_offloads & DEV_RX_OFFLOAD_VLAN_EXTEND)
3924                 ret |= ETH_VLAN_EXTEND_OFFLOAD;
3925
3926         if (*dev_offloads & DEV_RX_OFFLOAD_QINQ_STRIP)
3927                 ret |= ETH_QINQ_STRIP_OFFLOAD;
3928
3929         return ret;
3930 }
3931
3932 int
3933 rte_eth_dev_set_vlan_pvid(uint16_t port_id, uint16_t pvid, int on)
3934 {
3935         struct rte_eth_dev *dev;
3936
3937         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3938         dev = &rte_eth_devices[port_id];
3939
3940         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_pvid_set, -ENOTSUP);
3941         return eth_err(port_id, (*dev->dev_ops->vlan_pvid_set)(dev, pvid, on));
3942 }
3943
3944 int
3945 rte_eth_dev_flow_ctrl_get(uint16_t port_id, struct rte_eth_fc_conf *fc_conf)
3946 {
3947         struct rte_eth_dev *dev;
3948
3949         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3950         dev = &rte_eth_devices[port_id];
3951
3952         if (fc_conf == NULL) {
3953                 RTE_ETHDEV_LOG(ERR,
3954                         "Cannot get ethdev port %u flow control config to NULL\n",
3955                         port_id);
3956                 return -EINVAL;
3957         }
3958
3959         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_get, -ENOTSUP);
3960         memset(fc_conf, 0, sizeof(*fc_conf));
3961         return eth_err(port_id, (*dev->dev_ops->flow_ctrl_get)(dev, fc_conf));
3962 }
3963
3964 int
3965 rte_eth_dev_flow_ctrl_set(uint16_t port_id, struct rte_eth_fc_conf *fc_conf)
3966 {
3967         struct rte_eth_dev *dev;
3968
3969         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3970         dev = &rte_eth_devices[port_id];
3971
3972         if (fc_conf == NULL) {
3973                 RTE_ETHDEV_LOG(ERR,
3974                         "Cannot set ethdev port %u flow control from NULL config\n",
3975                         port_id);
3976                 return -EINVAL;
3977         }
3978
3979         if ((fc_conf->send_xon != 0) && (fc_conf->send_xon != 1)) {
3980                 RTE_ETHDEV_LOG(ERR, "Invalid send_xon, only 0/1 allowed\n");
3981                 return -EINVAL;
3982         }
3983
3984         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_set, -ENOTSUP);
3985         return eth_err(port_id, (*dev->dev_ops->flow_ctrl_set)(dev, fc_conf));
3986 }
3987
3988 int
3989 rte_eth_dev_priority_flow_ctrl_set(uint16_t port_id,
3990                                    struct rte_eth_pfc_conf *pfc_conf)
3991 {
3992         struct rte_eth_dev *dev;
3993
3994         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3995         dev = &rte_eth_devices[port_id];
3996
3997         if (pfc_conf == NULL) {
3998                 RTE_ETHDEV_LOG(ERR,
3999                         "Cannot set ethdev port %u priority flow control from NULL config\n",
4000                         port_id);
4001                 return -EINVAL;
4002         }
4003
4004         if (pfc_conf->priority > (ETH_DCB_NUM_USER_PRIORITIES - 1)) {
4005                 RTE_ETHDEV_LOG(ERR, "Invalid priority, only 0-7 allowed\n");
4006                 return -EINVAL;
4007         }
4008
4009         /* High water, low water validation are device specific */
4010         if  (*dev->dev_ops->priority_flow_ctrl_set)
4011                 return eth_err(port_id, (*dev->dev_ops->priority_flow_ctrl_set)
4012                                         (dev, pfc_conf));
4013         return -ENOTSUP;
4014 }
4015
4016 static int
4017 eth_check_reta_mask(struct rte_eth_rss_reta_entry64 *reta_conf,
4018                         uint16_t reta_size)
4019 {
4020         uint16_t i, num;
4021
4022         num = (reta_size + RTE_RETA_GROUP_SIZE - 1) / RTE_RETA_GROUP_SIZE;
4023         for (i = 0; i < num; i++) {
4024                 if (reta_conf[i].mask)
4025                         return 0;
4026         }
4027
4028         return -EINVAL;
4029 }
4030
4031 static int
4032 eth_check_reta_entry(struct rte_eth_rss_reta_entry64 *reta_conf,
4033                          uint16_t reta_size,
4034                          uint16_t max_rxq)
4035 {
4036         uint16_t i, idx, shift;
4037
4038         if (max_rxq == 0) {
4039                 RTE_ETHDEV_LOG(ERR, "No receive queue is available\n");
4040                 return -EINVAL;
4041         }
4042
4043         for (i = 0; i < reta_size; i++) {
4044                 idx = i / RTE_RETA_GROUP_SIZE;
4045                 shift = i % RTE_RETA_GROUP_SIZE;
4046                 if ((reta_conf[idx].mask & RTE_BIT64(shift)) &&
4047                         (reta_conf[idx].reta[shift] >= max_rxq)) {
4048                         RTE_ETHDEV_LOG(ERR,
4049                                 "reta_conf[%u]->reta[%u]: %u exceeds the maximum rxq index: %u\n",
4050                                 idx, shift,
4051                                 reta_conf[idx].reta[shift], max_rxq);
4052                         return -EINVAL;
4053                 }
4054         }
4055
4056         return 0;
4057 }
4058
4059 int
4060 rte_eth_dev_rss_reta_update(uint16_t port_id,
4061                             struct rte_eth_rss_reta_entry64 *reta_conf,
4062                             uint16_t reta_size)
4063 {
4064         struct rte_eth_dev *dev;
4065         int ret;
4066
4067         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4068         dev = &rte_eth_devices[port_id];
4069
4070         if (reta_conf == NULL) {
4071                 RTE_ETHDEV_LOG(ERR,
4072                         "Cannot update ethdev port %u RSS RETA to NULL\n",
4073                         port_id);
4074                 return -EINVAL;
4075         }
4076
4077         if (reta_size == 0) {
4078                 RTE_ETHDEV_LOG(ERR,
4079                         "Cannot update ethdev port %u RSS RETA with zero size\n",
4080                         port_id);
4081                 return -EINVAL;
4082         }
4083
4084         /* Check mask bits */
4085         ret = eth_check_reta_mask(reta_conf, reta_size);
4086         if (ret < 0)
4087                 return ret;
4088
4089         /* Check entry value */
4090         ret = eth_check_reta_entry(reta_conf, reta_size,
4091                                 dev->data->nb_rx_queues);
4092         if (ret < 0)
4093                 return ret;
4094
4095         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_update, -ENOTSUP);
4096         return eth_err(port_id, (*dev->dev_ops->reta_update)(dev, reta_conf,
4097                                                              reta_size));
4098 }
4099
4100 int
4101 rte_eth_dev_rss_reta_query(uint16_t port_id,
4102                            struct rte_eth_rss_reta_entry64 *reta_conf,
4103                            uint16_t reta_size)
4104 {
4105         struct rte_eth_dev *dev;
4106         int ret;
4107
4108         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4109         dev = &rte_eth_devices[port_id];
4110
4111         if (reta_conf == NULL) {
4112                 RTE_ETHDEV_LOG(ERR,
4113                         "Cannot query ethdev port %u RSS RETA from NULL config\n",
4114                         port_id);
4115                 return -EINVAL;
4116         }
4117
4118         /* Check mask bits */
4119         ret = eth_check_reta_mask(reta_conf, reta_size);
4120         if (ret < 0)
4121                 return ret;
4122
4123         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_query, -ENOTSUP);
4124         return eth_err(port_id, (*dev->dev_ops->reta_query)(dev, reta_conf,
4125                                                             reta_size));
4126 }
4127
4128 int
4129 rte_eth_dev_rss_hash_update(uint16_t port_id,
4130                             struct rte_eth_rss_conf *rss_conf)
4131 {
4132         struct rte_eth_dev *dev;
4133         struct rte_eth_dev_info dev_info = { .flow_type_rss_offloads = 0, };
4134         int ret;
4135
4136         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4137         dev = &rte_eth_devices[port_id];
4138
4139         if (rss_conf == NULL) {
4140                 RTE_ETHDEV_LOG(ERR,
4141                         "Cannot update ethdev port %u RSS hash from NULL config\n",
4142                         port_id);
4143                 return -EINVAL;
4144         }
4145
4146         ret = rte_eth_dev_info_get(port_id, &dev_info);
4147         if (ret != 0)
4148                 return ret;
4149
4150         rss_conf->rss_hf = rte_eth_rss_hf_refine(rss_conf->rss_hf);
4151         if ((dev_info.flow_type_rss_offloads | rss_conf->rss_hf) !=
4152             dev_info.flow_type_rss_offloads) {
4153                 RTE_ETHDEV_LOG(ERR,
4154                         "Ethdev port_id=%u invalid rss_hf: 0x%"PRIx64", valid value: 0x%"PRIx64"\n",
4155                         port_id, rss_conf->rss_hf,
4156                         dev_info.flow_type_rss_offloads);
4157                 return -EINVAL;
4158         }
4159         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_update, -ENOTSUP);
4160         return eth_err(port_id, (*dev->dev_ops->rss_hash_update)(dev,
4161                                                                  rss_conf));
4162 }
4163
4164 int
4165 rte_eth_dev_rss_hash_conf_get(uint16_t port_id,
4166                               struct rte_eth_rss_conf *rss_conf)
4167 {
4168         struct rte_eth_dev *dev;
4169
4170         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4171         dev = &rte_eth_devices[port_id];
4172
4173         if (rss_conf == NULL) {
4174                 RTE_ETHDEV_LOG(ERR,
4175                         "Cannot get ethdev port %u RSS hash config to NULL\n",
4176                         port_id);
4177                 return -EINVAL;
4178         }
4179
4180         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_conf_get, -ENOTSUP);
4181         return eth_err(port_id, (*dev->dev_ops->rss_hash_conf_get)(dev,
4182                                                                    rss_conf));
4183 }
4184
4185 int
4186 rte_eth_dev_udp_tunnel_port_add(uint16_t port_id,
4187                                 struct rte_eth_udp_tunnel *udp_tunnel)
4188 {
4189         struct rte_eth_dev *dev;
4190
4191         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4192         dev = &rte_eth_devices[port_id];
4193
4194         if (udp_tunnel == NULL) {
4195                 RTE_ETHDEV_LOG(ERR,
4196                         "Cannot add ethdev port %u UDP tunnel port from NULL UDP tunnel\n",
4197                         port_id);
4198                 return -EINVAL;
4199         }
4200
4201         if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
4202                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
4203                 return -EINVAL;
4204         }
4205
4206         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_port_add, -ENOTSUP);
4207         return eth_err(port_id, (*dev->dev_ops->udp_tunnel_port_add)(dev,
4208                                                                 udp_tunnel));
4209 }
4210
4211 int
4212 rte_eth_dev_udp_tunnel_port_delete(uint16_t port_id,
4213                                    struct rte_eth_udp_tunnel *udp_tunnel)
4214 {
4215         struct rte_eth_dev *dev;
4216
4217         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4218         dev = &rte_eth_devices[port_id];
4219
4220         if (udp_tunnel == NULL) {
4221                 RTE_ETHDEV_LOG(ERR,
4222                         "Cannot delete ethdev port %u UDP tunnel port from NULL UDP tunnel\n",
4223                         port_id);
4224                 return -EINVAL;
4225         }
4226
4227         if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
4228                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
4229                 return -EINVAL;
4230         }
4231
4232         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_port_del, -ENOTSUP);
4233         return eth_err(port_id, (*dev->dev_ops->udp_tunnel_port_del)(dev,
4234                                                                 udp_tunnel));
4235 }
4236
4237 int
4238 rte_eth_led_on(uint16_t port_id)
4239 {
4240         struct rte_eth_dev *dev;
4241
4242         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4243         dev = &rte_eth_devices[port_id];
4244
4245         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_on, -ENOTSUP);
4246         return eth_err(port_id, (*dev->dev_ops->dev_led_on)(dev));
4247 }
4248
4249 int
4250 rte_eth_led_off(uint16_t port_id)
4251 {
4252         struct rte_eth_dev *dev;
4253
4254         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4255         dev = &rte_eth_devices[port_id];
4256
4257         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_off, -ENOTSUP);
4258         return eth_err(port_id, (*dev->dev_ops->dev_led_off)(dev));
4259 }
4260
4261 int
4262 rte_eth_fec_get_capability(uint16_t port_id,
4263                            struct rte_eth_fec_capa *speed_fec_capa,
4264                            unsigned int num)
4265 {
4266         struct rte_eth_dev *dev;
4267         int ret;
4268
4269         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4270         dev = &rte_eth_devices[port_id];
4271
4272         if (speed_fec_capa == NULL && num > 0) {
4273                 RTE_ETHDEV_LOG(ERR,
4274                         "Cannot get ethdev port %u FEC capability to NULL when array size is non zero\n",
4275                         port_id);
4276                 return -EINVAL;
4277         }
4278
4279         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fec_get_capability, -ENOTSUP);
4280         ret = (*dev->dev_ops->fec_get_capability)(dev, speed_fec_capa, num);
4281
4282         return ret;
4283 }
4284
4285 int
4286 rte_eth_fec_get(uint16_t port_id, uint32_t *fec_capa)
4287 {
4288         struct rte_eth_dev *dev;
4289
4290         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4291         dev = &rte_eth_devices[port_id];
4292
4293         if (fec_capa == NULL) {
4294                 RTE_ETHDEV_LOG(ERR,
4295                         "Cannot get ethdev port %u current FEC mode to NULL\n",
4296                         port_id);
4297                 return -EINVAL;
4298         }
4299
4300         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fec_get, -ENOTSUP);
4301         return eth_err(port_id, (*dev->dev_ops->fec_get)(dev, fec_capa));
4302 }
4303
4304 int
4305 rte_eth_fec_set(uint16_t port_id, uint32_t fec_capa)
4306 {
4307         struct rte_eth_dev *dev;
4308
4309         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4310         dev = &rte_eth_devices[port_id];
4311
4312         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fec_set, -ENOTSUP);
4313         return eth_err(port_id, (*dev->dev_ops->fec_set)(dev, fec_capa));
4314 }
4315
4316 /*
4317  * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find
4318  * an empty spot.
4319  */
4320 static int
4321 eth_dev_get_mac_addr_index(uint16_t port_id, const struct rte_ether_addr *addr)
4322 {
4323         struct rte_eth_dev_info dev_info;
4324         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
4325         unsigned i;
4326         int ret;
4327
4328         ret = rte_eth_dev_info_get(port_id, &dev_info);
4329         if (ret != 0)
4330                 return -1;
4331
4332         for (i = 0; i < dev_info.max_mac_addrs; i++)
4333                 if (memcmp(addr, &dev->data->mac_addrs[i],
4334                                 RTE_ETHER_ADDR_LEN) == 0)
4335                         return i;
4336
4337         return -1;
4338 }
4339
4340 static const struct rte_ether_addr null_mac_addr;
4341
4342 int
4343 rte_eth_dev_mac_addr_add(uint16_t port_id, struct rte_ether_addr *addr,
4344                         uint32_t pool)
4345 {
4346         struct rte_eth_dev *dev;
4347         int index;
4348         uint64_t pool_mask;
4349         int ret;
4350
4351         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4352         dev = &rte_eth_devices[port_id];
4353
4354         if (addr == NULL) {
4355                 RTE_ETHDEV_LOG(ERR,
4356                         "Cannot add ethdev port %u MAC address from NULL address\n",
4357                         port_id);
4358                 return -EINVAL;
4359         }
4360
4361         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_add, -ENOTSUP);
4362
4363         if (rte_is_zero_ether_addr(addr)) {
4364                 RTE_ETHDEV_LOG(ERR, "Port %u: Cannot add NULL MAC address\n",
4365                         port_id);
4366                 return -EINVAL;
4367         }
4368         if (pool >= ETH_64_POOLS) {
4369                 RTE_ETHDEV_LOG(ERR, "Pool ID must be 0-%d\n", ETH_64_POOLS - 1);
4370                 return -EINVAL;
4371         }
4372
4373         index = eth_dev_get_mac_addr_index(port_id, addr);
4374         if (index < 0) {
4375                 index = eth_dev_get_mac_addr_index(port_id, &null_mac_addr);
4376                 if (index < 0) {
4377                         RTE_ETHDEV_LOG(ERR, "Port %u: MAC address array full\n",
4378                                 port_id);
4379                         return -ENOSPC;
4380                 }
4381         } else {
4382                 pool_mask = dev->data->mac_pool_sel[index];
4383
4384                 /* Check if both MAC address and pool is already there, and do nothing */
4385                 if (pool_mask & RTE_BIT64(pool))
4386                         return 0;
4387         }
4388
4389         /* Update NIC */
4390         ret = (*dev->dev_ops->mac_addr_add)(dev, addr, index, pool);
4391
4392         if (ret == 0) {
4393                 /* Update address in NIC data structure */
4394                 rte_ether_addr_copy(addr, &dev->data->mac_addrs[index]);
4395
4396                 /* Update pool bitmap in NIC data structure */
4397                 dev->data->mac_pool_sel[index] |= RTE_BIT64(pool);
4398         }
4399
4400         return eth_err(port_id, ret);
4401 }
4402
4403 int
4404 rte_eth_dev_mac_addr_remove(uint16_t port_id, struct rte_ether_addr *addr)
4405 {
4406         struct rte_eth_dev *dev;
4407         int index;
4408
4409         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4410         dev = &rte_eth_devices[port_id];
4411
4412         if (addr == NULL) {
4413                 RTE_ETHDEV_LOG(ERR,
4414                         "Cannot remove ethdev port %u MAC address from NULL address\n",
4415                         port_id);
4416                 return -EINVAL;
4417         }
4418
4419         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_remove, -ENOTSUP);
4420
4421         index = eth_dev_get_mac_addr_index(port_id, addr);
4422         if (index == 0) {
4423                 RTE_ETHDEV_LOG(ERR,
4424                         "Port %u: Cannot remove default MAC address\n",
4425                         port_id);
4426                 return -EADDRINUSE;
4427         } else if (index < 0)
4428                 return 0;  /* Do nothing if address wasn't found */
4429
4430         /* Update NIC */
4431         (*dev->dev_ops->mac_addr_remove)(dev, index);
4432
4433         /* Update address in NIC data structure */
4434         rte_ether_addr_copy(&null_mac_addr, &dev->data->mac_addrs[index]);
4435
4436         /* reset pool bitmap */
4437         dev->data->mac_pool_sel[index] = 0;
4438
4439         return 0;
4440 }
4441
4442 int
4443 rte_eth_dev_default_mac_addr_set(uint16_t port_id, struct rte_ether_addr *addr)
4444 {
4445         struct rte_eth_dev *dev;
4446         int ret;
4447
4448         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4449         dev = &rte_eth_devices[port_id];
4450
4451         if (addr == NULL) {
4452                 RTE_ETHDEV_LOG(ERR,
4453                         "Cannot set ethdev port %u default MAC address from NULL address\n",
4454                         port_id);
4455                 return -EINVAL;
4456         }
4457
4458         if (!rte_is_valid_assigned_ether_addr(addr))
4459                 return -EINVAL;
4460
4461         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_set, -ENOTSUP);
4462
4463         ret = (*dev->dev_ops->mac_addr_set)(dev, addr);
4464         if (ret < 0)
4465                 return ret;
4466
4467         /* Update default address in NIC data structure */
4468         rte_ether_addr_copy(addr, &dev->data->mac_addrs[0]);
4469
4470         return 0;
4471 }
4472
4473
4474 /*
4475  * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find
4476  * an empty spot.
4477  */
4478 static int
4479 eth_dev_get_hash_mac_addr_index(uint16_t port_id,
4480                 const struct rte_ether_addr *addr)
4481 {
4482         struct rte_eth_dev_info dev_info;
4483         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
4484         unsigned i;
4485         int ret;
4486
4487         ret = rte_eth_dev_info_get(port_id, &dev_info);
4488         if (ret != 0)
4489                 return -1;
4490
4491         if (!dev->data->hash_mac_addrs)
4492                 return -1;
4493
4494         for (i = 0; i < dev_info.max_hash_mac_addrs; i++)
4495                 if (memcmp(addr, &dev->data->hash_mac_addrs[i],
4496                         RTE_ETHER_ADDR_LEN) == 0)
4497                         return i;
4498
4499         return -1;
4500 }
4501
4502 int
4503 rte_eth_dev_uc_hash_table_set(uint16_t port_id, struct rte_ether_addr *addr,
4504                                 uint8_t on)
4505 {
4506         int index;
4507         int ret;
4508         struct rte_eth_dev *dev;
4509
4510         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4511         dev = &rte_eth_devices[port_id];
4512
4513         if (addr == NULL) {
4514                 RTE_ETHDEV_LOG(ERR,
4515                         "Cannot set ethdev port %u unicast hash table from NULL address\n",
4516                         port_id);
4517                 return -EINVAL;
4518         }
4519
4520         if (rte_is_zero_ether_addr(addr)) {
4521                 RTE_ETHDEV_LOG(ERR, "Port %u: Cannot add NULL MAC address\n",
4522                         port_id);
4523                 return -EINVAL;
4524         }
4525
4526         index = eth_dev_get_hash_mac_addr_index(port_id, addr);
4527         /* Check if it's already there, and do nothing */
4528         if ((index >= 0) && on)
4529                 return 0;
4530
4531         if (index < 0) {
4532                 if (!on) {
4533                         RTE_ETHDEV_LOG(ERR,
4534                                 "Port %u: the MAC address was not set in UTA\n",
4535                                 port_id);
4536                         return -EINVAL;
4537                 }
4538
4539                 index = eth_dev_get_hash_mac_addr_index(port_id, &null_mac_addr);
4540                 if (index < 0) {
4541                         RTE_ETHDEV_LOG(ERR, "Port %u: MAC address array full\n",
4542                                 port_id);
4543                         return -ENOSPC;
4544                 }
4545         }
4546
4547         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_hash_table_set, -ENOTSUP);
4548         ret = (*dev->dev_ops->uc_hash_table_set)(dev, addr, on);
4549         if (ret == 0) {
4550                 /* Update address in NIC data structure */
4551                 if (on)
4552                         rte_ether_addr_copy(addr,
4553                                         &dev->data->hash_mac_addrs[index]);
4554                 else
4555                         rte_ether_addr_copy(&null_mac_addr,
4556                                         &dev->data->hash_mac_addrs[index]);
4557         }
4558
4559         return eth_err(port_id, ret);
4560 }
4561
4562 int
4563 rte_eth_dev_uc_all_hash_table_set(uint16_t port_id, uint8_t on)
4564 {
4565         struct rte_eth_dev *dev;
4566
4567         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4568         dev = &rte_eth_devices[port_id];
4569
4570         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_all_hash_table_set, -ENOTSUP);
4571         return eth_err(port_id, (*dev->dev_ops->uc_all_hash_table_set)(dev,
4572                                                                        on));
4573 }
4574
4575 int rte_eth_set_queue_rate_limit(uint16_t port_id, uint16_t queue_idx,
4576                                         uint16_t tx_rate)
4577 {
4578         struct rte_eth_dev *dev;
4579         struct rte_eth_dev_info dev_info;
4580         struct rte_eth_link link;
4581         int ret;
4582
4583         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4584         dev = &rte_eth_devices[port_id];
4585
4586         ret = rte_eth_dev_info_get(port_id, &dev_info);
4587         if (ret != 0)
4588                 return ret;
4589
4590         link = dev->data->dev_link;
4591
4592         if (queue_idx > dev_info.max_tx_queues) {
4593                 RTE_ETHDEV_LOG(ERR,
4594                         "Set queue rate limit:port %u: invalid queue ID=%u\n",
4595                         port_id, queue_idx);
4596                 return -EINVAL;
4597         }
4598
4599         if (tx_rate > link.link_speed) {
4600                 RTE_ETHDEV_LOG(ERR,
4601                         "Set queue rate limit:invalid tx_rate=%u, bigger than link speed= %d\n",
4602                         tx_rate, link.link_speed);
4603                 return -EINVAL;
4604         }
4605
4606         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_queue_rate_limit, -ENOTSUP);
4607         return eth_err(port_id, (*dev->dev_ops->set_queue_rate_limit)(dev,
4608                                                         queue_idx, tx_rate));
4609 }
4610
4611 RTE_INIT(eth_dev_init_fp_ops)
4612 {
4613         uint32_t i;
4614
4615         for (i = 0; i != RTE_DIM(rte_eth_fp_ops); i++)
4616                 eth_dev_fp_ops_reset(rte_eth_fp_ops + i);
4617 }
4618
4619 RTE_INIT(eth_dev_init_cb_lists)
4620 {
4621         uint16_t i;
4622
4623         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
4624                 TAILQ_INIT(&rte_eth_devices[i].link_intr_cbs);
4625 }
4626
4627 int
4628 rte_eth_dev_callback_register(uint16_t port_id,
4629                         enum rte_eth_event_type event,
4630                         rte_eth_dev_cb_fn cb_fn, void *cb_arg)
4631 {
4632         struct rte_eth_dev *dev;
4633         struct rte_eth_dev_callback *user_cb;
4634         uint16_t next_port;
4635         uint16_t last_port;
4636
4637         if (cb_fn == NULL) {
4638                 RTE_ETHDEV_LOG(ERR,
4639                         "Cannot register ethdev port %u callback from NULL\n",
4640                         port_id);
4641                 return -EINVAL;
4642         }
4643
4644         if (!rte_eth_dev_is_valid_port(port_id) && port_id != RTE_ETH_ALL) {
4645                 RTE_ETHDEV_LOG(ERR, "Invalid port_id=%d\n", port_id);
4646                 return -EINVAL;
4647         }
4648
4649         if (port_id == RTE_ETH_ALL) {
4650                 next_port = 0;
4651                 last_port = RTE_MAX_ETHPORTS - 1;
4652         } else {
4653                 next_port = last_port = port_id;
4654         }
4655
4656         rte_spinlock_lock(&eth_dev_cb_lock);
4657
4658         do {
4659                 dev = &rte_eth_devices[next_port];
4660
4661                 TAILQ_FOREACH(user_cb, &(dev->link_intr_cbs), next) {
4662                         if (user_cb->cb_fn == cb_fn &&
4663                                 user_cb->cb_arg == cb_arg &&
4664                                 user_cb->event == event) {
4665                                 break;
4666                         }
4667                 }
4668
4669                 /* create a new callback. */
4670                 if (user_cb == NULL) {
4671                         user_cb = rte_zmalloc("INTR_USER_CALLBACK",
4672                                 sizeof(struct rte_eth_dev_callback), 0);
4673                         if (user_cb != NULL) {
4674                                 user_cb->cb_fn = cb_fn;
4675                                 user_cb->cb_arg = cb_arg;
4676                                 user_cb->event = event;
4677                                 TAILQ_INSERT_TAIL(&(dev->link_intr_cbs),
4678                                                   user_cb, next);
4679                         } else {
4680                                 rte_spinlock_unlock(&eth_dev_cb_lock);
4681                                 rte_eth_dev_callback_unregister(port_id, event,
4682                                                                 cb_fn, cb_arg);
4683                                 return -ENOMEM;
4684                         }
4685
4686                 }
4687         } while (++next_port <= last_port);
4688
4689         rte_spinlock_unlock(&eth_dev_cb_lock);
4690         return 0;
4691 }
4692
4693 int
4694 rte_eth_dev_callback_unregister(uint16_t port_id,
4695                         enum rte_eth_event_type event,
4696                         rte_eth_dev_cb_fn cb_fn, void *cb_arg)
4697 {
4698         int ret;
4699         struct rte_eth_dev *dev;
4700         struct rte_eth_dev_callback *cb, *next;
4701         uint16_t next_port;
4702         uint16_t last_port;
4703
4704         if (cb_fn == NULL) {
4705                 RTE_ETHDEV_LOG(ERR,
4706                         "Cannot unregister ethdev port %u callback from NULL\n",
4707                         port_id);
4708                 return -EINVAL;
4709         }
4710
4711         if (!rte_eth_dev_is_valid_port(port_id) && port_id != RTE_ETH_ALL) {
4712                 RTE_ETHDEV_LOG(ERR, "Invalid port_id=%d\n", port_id);
4713                 return -EINVAL;
4714         }
4715
4716         if (port_id == RTE_ETH_ALL) {
4717                 next_port = 0;
4718                 last_port = RTE_MAX_ETHPORTS - 1;
4719         } else {
4720                 next_port = last_port = port_id;
4721         }
4722
4723         rte_spinlock_lock(&eth_dev_cb_lock);
4724
4725         do {
4726                 dev = &rte_eth_devices[next_port];
4727                 ret = 0;
4728                 for (cb = TAILQ_FIRST(&dev->link_intr_cbs); cb != NULL;
4729                      cb = next) {
4730
4731                         next = TAILQ_NEXT(cb, next);
4732
4733                         if (cb->cb_fn != cb_fn || cb->event != event ||
4734                             (cb_arg != (void *)-1 && cb->cb_arg != cb_arg))
4735                                 continue;
4736
4737                         /*
4738                          * if this callback is not executing right now,
4739                          * then remove it.
4740                          */
4741                         if (cb->active == 0) {
4742                                 TAILQ_REMOVE(&(dev->link_intr_cbs), cb, next);
4743                                 rte_free(cb);
4744                         } else {
4745                                 ret = -EAGAIN;
4746                         }
4747                 }
4748         } while (++next_port <= last_port);
4749
4750         rte_spinlock_unlock(&eth_dev_cb_lock);
4751         return ret;
4752 }
4753
4754 int
4755 rte_eth_dev_callback_process(struct rte_eth_dev *dev,
4756         enum rte_eth_event_type event, void *ret_param)
4757 {
4758         struct rte_eth_dev_callback *cb_lst;
4759         struct rte_eth_dev_callback dev_cb;
4760         int rc = 0;
4761
4762         rte_spinlock_lock(&eth_dev_cb_lock);
4763         TAILQ_FOREACH(cb_lst, &(dev->link_intr_cbs), next) {
4764                 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
4765                         continue;
4766                 dev_cb = *cb_lst;
4767                 cb_lst->active = 1;
4768                 if (ret_param != NULL)
4769                         dev_cb.ret_param = ret_param;
4770
4771                 rte_spinlock_unlock(&eth_dev_cb_lock);
4772                 rc = dev_cb.cb_fn(dev->data->port_id, dev_cb.event,
4773                                 dev_cb.cb_arg, dev_cb.ret_param);
4774                 rte_spinlock_lock(&eth_dev_cb_lock);
4775                 cb_lst->active = 0;
4776         }
4777         rte_spinlock_unlock(&eth_dev_cb_lock);
4778         return rc;
4779 }
4780
4781 void
4782 rte_eth_dev_probing_finish(struct rte_eth_dev *dev)
4783 {
4784         if (dev == NULL)
4785                 return;
4786
4787         /*
4788          * for secondary process, at that point we expect device
4789          * to be already 'usable', so shared data and all function pointers
4790          * for fast-path devops have to be setup properly inside rte_eth_dev.
4791          */
4792         if (rte_eal_process_type() == RTE_PROC_SECONDARY)
4793                 eth_dev_fp_ops_setup(rte_eth_fp_ops + dev->data->port_id, dev);
4794
4795         rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_NEW, NULL);
4796
4797         dev->state = RTE_ETH_DEV_ATTACHED;
4798 }
4799
4800 int
4801 rte_eth_dev_rx_intr_ctl(uint16_t port_id, int epfd, int op, void *data)
4802 {
4803         uint32_t vec;
4804         struct rte_eth_dev *dev;
4805         struct rte_intr_handle *intr_handle;
4806         uint16_t qid;
4807         int rc;
4808
4809         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4810         dev = &rte_eth_devices[port_id];
4811
4812         if (!dev->intr_handle) {
4813                 RTE_ETHDEV_LOG(ERR, "Rx Intr handle unset\n");
4814                 return -ENOTSUP;
4815         }
4816
4817         intr_handle = dev->intr_handle;
4818         if (!intr_handle->intr_vec) {
4819                 RTE_ETHDEV_LOG(ERR, "Rx Intr vector unset\n");
4820                 return -EPERM;
4821         }
4822
4823         for (qid = 0; qid < dev->data->nb_rx_queues; qid++) {
4824                 vec = intr_handle->intr_vec[qid];
4825                 rc = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
4826                 if (rc && rc != -EEXIST) {
4827                         RTE_ETHDEV_LOG(ERR,
4828                                 "p %u q %u Rx ctl error op %d epfd %d vec %u\n",
4829                                 port_id, qid, op, epfd, vec);
4830                 }
4831         }
4832
4833         return 0;
4834 }
4835
4836 int
4837 rte_eth_dev_rx_intr_ctl_q_get_fd(uint16_t port_id, uint16_t queue_id)
4838 {
4839         struct rte_intr_handle *intr_handle;
4840         struct rte_eth_dev *dev;
4841         unsigned int efd_idx;
4842         uint32_t vec;
4843         int fd;
4844
4845         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -1);
4846         dev = &rte_eth_devices[port_id];
4847
4848         if (queue_id >= dev->data->nb_rx_queues) {
4849                 RTE_ETHDEV_LOG(ERR, "Invalid Rx queue_id=%u\n", queue_id);
4850                 return -1;
4851         }
4852
4853         if (!dev->intr_handle) {
4854                 RTE_ETHDEV_LOG(ERR, "Rx Intr handle unset\n");
4855                 return -1;
4856         }
4857
4858         intr_handle = dev->intr_handle;
4859         if (!intr_handle->intr_vec) {
4860                 RTE_ETHDEV_LOG(ERR, "Rx Intr vector unset\n");
4861                 return -1;
4862         }
4863
4864         vec = intr_handle->intr_vec[queue_id];
4865         efd_idx = (vec >= RTE_INTR_VEC_RXTX_OFFSET) ?
4866                 (vec - RTE_INTR_VEC_RXTX_OFFSET) : vec;
4867         fd = intr_handle->efds[efd_idx];
4868
4869         return fd;
4870 }
4871
4872 static inline int
4873 eth_dev_dma_mzone_name(char *name, size_t len, uint16_t port_id, uint16_t queue_id,
4874                 const char *ring_name)
4875 {
4876         return snprintf(name, len, "eth_p%d_q%d_%s",
4877                         port_id, queue_id, ring_name);
4878 }
4879
4880 const struct rte_memzone *
4881 rte_eth_dma_zone_reserve(const struct rte_eth_dev *dev, const char *ring_name,
4882                          uint16_t queue_id, size_t size, unsigned align,
4883                          int socket_id)
4884 {
4885         char z_name[RTE_MEMZONE_NAMESIZE];
4886         const struct rte_memzone *mz;
4887         int rc;
4888
4889         rc = eth_dev_dma_mzone_name(z_name, sizeof(z_name), dev->data->port_id,
4890                         queue_id, ring_name);
4891         if (rc >= RTE_MEMZONE_NAMESIZE) {
4892                 RTE_ETHDEV_LOG(ERR, "ring name too long\n");
4893                 rte_errno = ENAMETOOLONG;
4894                 return NULL;
4895         }
4896
4897         mz = rte_memzone_lookup(z_name);
4898         if (mz) {
4899                 if ((socket_id != SOCKET_ID_ANY && socket_id != mz->socket_id) ||
4900                                 size > mz->len ||
4901                                 ((uintptr_t)mz->addr & (align - 1)) != 0) {
4902                         RTE_ETHDEV_LOG(ERR,
4903                                 "memzone %s does not justify the requested attributes\n",
4904                                 mz->name);
4905                         return NULL;
4906                 }
4907
4908                 return mz;
4909         }
4910
4911         return rte_memzone_reserve_aligned(z_name, size, socket_id,
4912                         RTE_MEMZONE_IOVA_CONTIG, align);
4913 }
4914
4915 int
4916 rte_eth_dma_zone_free(const struct rte_eth_dev *dev, const char *ring_name,
4917                 uint16_t queue_id)
4918 {
4919         char z_name[RTE_MEMZONE_NAMESIZE];
4920         const struct rte_memzone *mz;
4921         int rc = 0;
4922
4923         rc = eth_dev_dma_mzone_name(z_name, sizeof(z_name), dev->data->port_id,
4924                         queue_id, ring_name);
4925         if (rc >= RTE_MEMZONE_NAMESIZE) {
4926                 RTE_ETHDEV_LOG(ERR, "ring name too long\n");
4927                 return -ENAMETOOLONG;
4928         }
4929
4930         mz = rte_memzone_lookup(z_name);
4931         if (mz)
4932                 rc = rte_memzone_free(mz);
4933         else
4934                 rc = -ENOENT;
4935
4936         return rc;
4937 }
4938
4939 int
4940 rte_eth_dev_create(struct rte_device *device, const char *name,
4941         size_t priv_data_size,
4942         ethdev_bus_specific_init ethdev_bus_specific_init,
4943         void *bus_init_params,
4944         ethdev_init_t ethdev_init, void *init_params)
4945 {
4946         struct rte_eth_dev *ethdev;
4947         int retval;
4948
4949         RTE_FUNC_PTR_OR_ERR_RET(*ethdev_init, -EINVAL);
4950
4951         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
4952                 ethdev = rte_eth_dev_allocate(name);
4953                 if (!ethdev)
4954                         return -ENODEV;
4955
4956                 if (priv_data_size) {
4957                         ethdev->data->dev_private = rte_zmalloc_socket(
4958                                 name, priv_data_size, RTE_CACHE_LINE_SIZE,
4959                                 device->numa_node);
4960
4961                         if (!ethdev->data->dev_private) {
4962                                 RTE_ETHDEV_LOG(ERR,
4963                                         "failed to allocate private data\n");
4964                                 retval = -ENOMEM;
4965                                 goto probe_failed;
4966                         }
4967                 }
4968         } else {
4969                 ethdev = rte_eth_dev_attach_secondary(name);
4970                 if (!ethdev) {
4971                         RTE_ETHDEV_LOG(ERR,
4972                                 "secondary process attach failed, ethdev doesn't exist\n");
4973                         return  -ENODEV;
4974                 }
4975         }
4976
4977         ethdev->device = device;
4978
4979         if (ethdev_bus_specific_init) {
4980                 retval = ethdev_bus_specific_init(ethdev, bus_init_params);
4981                 if (retval) {
4982                         RTE_ETHDEV_LOG(ERR,
4983                                 "ethdev bus specific initialisation failed\n");
4984                         goto probe_failed;
4985                 }
4986         }
4987
4988         retval = ethdev_init(ethdev, init_params);
4989         if (retval) {
4990                 RTE_ETHDEV_LOG(ERR, "ethdev initialisation failed\n");
4991                 goto probe_failed;
4992         }
4993
4994         rte_eth_dev_probing_finish(ethdev);
4995
4996         return retval;
4997
4998 probe_failed:
4999         rte_eth_dev_release_port(ethdev);
5000         return retval;
5001 }
5002
5003 int
5004 rte_eth_dev_destroy(struct rte_eth_dev *ethdev,
5005         ethdev_uninit_t ethdev_uninit)
5006 {
5007         int ret;
5008
5009         ethdev = rte_eth_dev_allocated(ethdev->data->name);
5010         if (!ethdev)
5011                 return -ENODEV;
5012
5013         RTE_FUNC_PTR_OR_ERR_RET(*ethdev_uninit, -EINVAL);
5014
5015         ret = ethdev_uninit(ethdev);
5016         if (ret)
5017                 return ret;
5018
5019         return rte_eth_dev_release_port(ethdev);
5020 }
5021
5022 int
5023 rte_eth_dev_rx_intr_ctl_q(uint16_t port_id, uint16_t queue_id,
5024                           int epfd, int op, void *data)
5025 {
5026         uint32_t vec;
5027         struct rte_eth_dev *dev;
5028         struct rte_intr_handle *intr_handle;
5029         int rc;
5030
5031         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5032         dev = &rte_eth_devices[port_id];
5033
5034         if (queue_id >= dev->data->nb_rx_queues) {
5035                 RTE_ETHDEV_LOG(ERR, "Invalid Rx queue_id=%u\n", queue_id);
5036                 return -EINVAL;
5037         }
5038
5039         if (!dev->intr_handle) {
5040                 RTE_ETHDEV_LOG(ERR, "Rx Intr handle unset\n");
5041                 return -ENOTSUP;
5042         }
5043
5044         intr_handle = dev->intr_handle;
5045         if (!intr_handle->intr_vec) {
5046                 RTE_ETHDEV_LOG(ERR, "Rx Intr vector unset\n");
5047                 return -EPERM;
5048         }
5049
5050         vec = intr_handle->intr_vec[queue_id];
5051         rc = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
5052         if (rc && rc != -EEXIST) {
5053                 RTE_ETHDEV_LOG(ERR,
5054                         "p %u q %u Rx ctl error op %d epfd %d vec %u\n",
5055                         port_id, queue_id, op, epfd, vec);
5056                 return rc;
5057         }
5058
5059         return 0;
5060 }
5061
5062 int
5063 rte_eth_dev_rx_intr_enable(uint16_t port_id,
5064                            uint16_t queue_id)
5065 {
5066         struct rte_eth_dev *dev;
5067         int ret;
5068
5069         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5070         dev = &rte_eth_devices[port_id];
5071
5072         ret = eth_dev_validate_rx_queue(dev, queue_id);
5073         if (ret != 0)
5074                 return ret;
5075
5076         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_intr_enable, -ENOTSUP);
5077         return eth_err(port_id, (*dev->dev_ops->rx_queue_intr_enable)(dev, queue_id));
5078 }
5079
5080 int
5081 rte_eth_dev_rx_intr_disable(uint16_t port_id,
5082                             uint16_t queue_id)
5083 {
5084         struct rte_eth_dev *dev;
5085         int ret;
5086
5087         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5088         dev = &rte_eth_devices[port_id];
5089
5090         ret = eth_dev_validate_rx_queue(dev, queue_id);
5091         if (ret != 0)
5092                 return ret;
5093
5094         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_intr_disable, -ENOTSUP);
5095         return eth_err(port_id, (*dev->dev_ops->rx_queue_intr_disable)(dev, queue_id));
5096 }
5097
5098
5099 const struct rte_eth_rxtx_callback *
5100 rte_eth_add_rx_callback(uint16_t port_id, uint16_t queue_id,
5101                 rte_rx_callback_fn fn, void *user_param)
5102 {
5103 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
5104         rte_errno = ENOTSUP;
5105         return NULL;
5106 #endif
5107         struct rte_eth_dev *dev;
5108
5109         /* check input parameters */
5110         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
5111                     queue_id >= rte_eth_devices[port_id].data->nb_rx_queues) {
5112                 rte_errno = EINVAL;
5113                 return NULL;
5114         }
5115         dev = &rte_eth_devices[port_id];
5116         if (rte_eth_dev_is_rx_hairpin_queue(dev, queue_id)) {
5117                 rte_errno = EINVAL;
5118                 return NULL;
5119         }
5120         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
5121
5122         if (cb == NULL) {
5123                 rte_errno = ENOMEM;
5124                 return NULL;
5125         }
5126
5127         cb->fn.rx = fn;
5128         cb->param = user_param;
5129
5130         rte_spinlock_lock(&eth_dev_rx_cb_lock);
5131         /* Add the callbacks in fifo order. */
5132         struct rte_eth_rxtx_callback *tail =
5133                 rte_eth_devices[port_id].post_rx_burst_cbs[queue_id];
5134
5135         if (!tail) {
5136                 /* Stores to cb->fn and cb->param should complete before
5137                  * cb is visible to data plane.
5138                  */
5139                 __atomic_store_n(
5140                         &rte_eth_devices[port_id].post_rx_burst_cbs[queue_id],
5141                         cb, __ATOMIC_RELEASE);
5142
5143         } else {
5144                 while (tail->next)
5145                         tail = tail->next;
5146                 /* Stores to cb->fn and cb->param should complete before
5147                  * cb is visible to data plane.
5148                  */
5149                 __atomic_store_n(&tail->next, cb, __ATOMIC_RELEASE);
5150         }
5151         rte_spinlock_unlock(&eth_dev_rx_cb_lock);
5152
5153         return cb;
5154 }
5155
5156 const struct rte_eth_rxtx_callback *
5157 rte_eth_add_first_rx_callback(uint16_t port_id, uint16_t queue_id,
5158                 rte_rx_callback_fn fn, void *user_param)
5159 {
5160 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
5161         rte_errno = ENOTSUP;
5162         return NULL;
5163 #endif
5164         /* check input parameters */
5165         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
5166                 queue_id >= rte_eth_devices[port_id].data->nb_rx_queues) {
5167                 rte_errno = EINVAL;
5168                 return NULL;
5169         }
5170
5171         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
5172
5173         if (cb == NULL) {
5174                 rte_errno = ENOMEM;
5175                 return NULL;
5176         }
5177
5178         cb->fn.rx = fn;
5179         cb->param = user_param;
5180
5181         rte_spinlock_lock(&eth_dev_rx_cb_lock);
5182         /* Add the callbacks at first position */
5183         cb->next = rte_eth_devices[port_id].post_rx_burst_cbs[queue_id];
5184         /* Stores to cb->fn, cb->param and cb->next should complete before
5185          * cb is visible to data plane threads.
5186          */
5187         __atomic_store_n(
5188                 &rte_eth_devices[port_id].post_rx_burst_cbs[queue_id],
5189                 cb, __ATOMIC_RELEASE);
5190         rte_spinlock_unlock(&eth_dev_rx_cb_lock);
5191
5192         return cb;
5193 }
5194
5195 const struct rte_eth_rxtx_callback *
5196 rte_eth_add_tx_callback(uint16_t port_id, uint16_t queue_id,
5197                 rte_tx_callback_fn fn, void *user_param)
5198 {
5199 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
5200         rte_errno = ENOTSUP;
5201         return NULL;
5202 #endif
5203         struct rte_eth_dev *dev;
5204
5205         /* check input parameters */
5206         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
5207                     queue_id >= rte_eth_devices[port_id].data->nb_tx_queues) {
5208                 rte_errno = EINVAL;
5209                 return NULL;
5210         }
5211
5212         dev = &rte_eth_devices[port_id];
5213         if (rte_eth_dev_is_tx_hairpin_queue(dev, queue_id)) {
5214                 rte_errno = EINVAL;
5215                 return NULL;
5216         }
5217
5218         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
5219
5220         if (cb == NULL) {
5221                 rte_errno = ENOMEM;
5222                 return NULL;
5223         }
5224
5225         cb->fn.tx = fn;
5226         cb->param = user_param;
5227
5228         rte_spinlock_lock(&eth_dev_tx_cb_lock);
5229         /* Add the callbacks in fifo order. */
5230         struct rte_eth_rxtx_callback *tail =
5231                 rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id];
5232
5233         if (!tail) {
5234                 /* Stores to cb->fn and cb->param should complete before
5235                  * cb is visible to data plane.
5236                  */
5237                 __atomic_store_n(
5238                         &rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id],
5239                         cb, __ATOMIC_RELEASE);
5240
5241         } else {
5242                 while (tail->next)
5243                         tail = tail->next;
5244                 /* Stores to cb->fn and cb->param should complete before
5245                  * cb is visible to data plane.
5246                  */
5247                 __atomic_store_n(&tail->next, cb, __ATOMIC_RELEASE);
5248         }
5249         rte_spinlock_unlock(&eth_dev_tx_cb_lock);
5250
5251         return cb;
5252 }
5253
5254 int
5255 rte_eth_remove_rx_callback(uint16_t port_id, uint16_t queue_id,
5256                 const struct rte_eth_rxtx_callback *user_cb)
5257 {
5258 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
5259         return -ENOTSUP;
5260 #endif
5261         /* Check input parameters. */
5262         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5263         if (user_cb == NULL ||
5264                         queue_id >= rte_eth_devices[port_id].data->nb_rx_queues)
5265                 return -EINVAL;
5266
5267         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
5268         struct rte_eth_rxtx_callback *cb;
5269         struct rte_eth_rxtx_callback **prev_cb;
5270         int ret = -EINVAL;
5271
5272         rte_spinlock_lock(&eth_dev_rx_cb_lock);
5273         prev_cb = &dev->post_rx_burst_cbs[queue_id];
5274         for (; *prev_cb != NULL; prev_cb = &cb->next) {
5275                 cb = *prev_cb;
5276                 if (cb == user_cb) {
5277                         /* Remove the user cb from the callback list. */
5278                         __atomic_store_n(prev_cb, cb->next, __ATOMIC_RELAXED);
5279                         ret = 0;
5280                         break;
5281                 }
5282         }
5283         rte_spinlock_unlock(&eth_dev_rx_cb_lock);
5284
5285         return ret;
5286 }
5287
5288 int
5289 rte_eth_remove_tx_callback(uint16_t port_id, uint16_t queue_id,
5290                 const struct rte_eth_rxtx_callback *user_cb)
5291 {
5292 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
5293         return -ENOTSUP;
5294 #endif
5295         /* Check input parameters. */
5296         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5297         if (user_cb == NULL ||
5298                         queue_id >= rte_eth_devices[port_id].data->nb_tx_queues)
5299                 return -EINVAL;
5300
5301         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
5302         int ret = -EINVAL;
5303         struct rte_eth_rxtx_callback *cb;
5304         struct rte_eth_rxtx_callback **prev_cb;
5305
5306         rte_spinlock_lock(&eth_dev_tx_cb_lock);
5307         prev_cb = &dev->pre_tx_burst_cbs[queue_id];
5308         for (; *prev_cb != NULL; prev_cb = &cb->next) {
5309                 cb = *prev_cb;
5310                 if (cb == user_cb) {
5311                         /* Remove the user cb from the callback list. */
5312                         __atomic_store_n(prev_cb, cb->next, __ATOMIC_RELAXED);
5313                         ret = 0;
5314                         break;
5315                 }
5316         }
5317         rte_spinlock_unlock(&eth_dev_tx_cb_lock);
5318
5319         return ret;
5320 }
5321
5322 int
5323 rte_eth_rx_queue_info_get(uint16_t port_id, uint16_t queue_id,
5324         struct rte_eth_rxq_info *qinfo)
5325 {
5326         struct rte_eth_dev *dev;
5327
5328         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5329         dev = &rte_eth_devices[port_id];
5330
5331         if (queue_id >= dev->data->nb_rx_queues) {
5332                 RTE_ETHDEV_LOG(ERR, "Invalid Rx queue_id=%u\n", queue_id);
5333                 return -EINVAL;
5334         }
5335
5336         if (qinfo == NULL) {
5337                 RTE_ETHDEV_LOG(ERR, "Cannot get ethdev port %u Rx queue %u info to NULL\n",
5338                         port_id, queue_id);
5339                 return -EINVAL;
5340         }
5341
5342         if (dev->data->rx_queues == NULL ||
5343                         dev->data->rx_queues[queue_id] == NULL) {
5344                 RTE_ETHDEV_LOG(ERR,
5345                                "Rx queue %"PRIu16" of device with port_id=%"
5346                                PRIu16" has not been setup\n",
5347                                queue_id, port_id);
5348                 return -EINVAL;
5349         }
5350
5351         if (rte_eth_dev_is_rx_hairpin_queue(dev, queue_id)) {
5352                 RTE_ETHDEV_LOG(INFO,
5353                         "Can't get hairpin Rx queue %"PRIu16" info of device with port_id=%"PRIu16"\n",
5354                         queue_id, port_id);
5355                 return -EINVAL;
5356         }
5357
5358         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rxq_info_get, -ENOTSUP);
5359
5360         memset(qinfo, 0, sizeof(*qinfo));
5361         dev->dev_ops->rxq_info_get(dev, queue_id, qinfo);
5362         qinfo->queue_state = dev->data->rx_queue_state[queue_id];
5363
5364         return 0;
5365 }
5366
5367 int
5368 rte_eth_tx_queue_info_get(uint16_t port_id, uint16_t queue_id,
5369         struct rte_eth_txq_info *qinfo)
5370 {
5371         struct rte_eth_dev *dev;
5372
5373         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5374         dev = &rte_eth_devices[port_id];
5375
5376         if (queue_id >= dev->data->nb_tx_queues) {
5377                 RTE_ETHDEV_LOG(ERR, "Invalid Tx queue_id=%u\n", queue_id);
5378                 return -EINVAL;
5379         }
5380
5381         if (qinfo == NULL) {
5382                 RTE_ETHDEV_LOG(ERR, "Cannot get ethdev port %u Tx queue %u info to NULL\n",
5383                         port_id, queue_id);
5384                 return -EINVAL;
5385         }
5386
5387         if (dev->data->tx_queues == NULL ||
5388                         dev->data->tx_queues[queue_id] == NULL) {
5389                 RTE_ETHDEV_LOG(ERR,
5390                                "Tx queue %"PRIu16" of device with port_id=%"
5391                                PRIu16" has not been setup\n",
5392                                queue_id, port_id);
5393                 return -EINVAL;
5394         }
5395
5396         if (rte_eth_dev_is_tx_hairpin_queue(dev, queue_id)) {
5397                 RTE_ETHDEV_LOG(INFO,
5398                         "Can't get hairpin Tx queue %"PRIu16" info of device with port_id=%"PRIu16"\n",
5399                         queue_id, port_id);
5400                 return -EINVAL;
5401         }
5402
5403         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->txq_info_get, -ENOTSUP);
5404
5405         memset(qinfo, 0, sizeof(*qinfo));
5406         dev->dev_ops->txq_info_get(dev, queue_id, qinfo);
5407         qinfo->queue_state = dev->data->tx_queue_state[queue_id];
5408
5409         return 0;
5410 }
5411
5412 int
5413 rte_eth_rx_burst_mode_get(uint16_t port_id, uint16_t queue_id,
5414                           struct rte_eth_burst_mode *mode)
5415 {
5416         struct rte_eth_dev *dev;
5417
5418         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5419         dev = &rte_eth_devices[port_id];
5420
5421         if (queue_id >= dev->data->nb_rx_queues) {
5422                 RTE_ETHDEV_LOG(ERR, "Invalid Rx queue_id=%u\n", queue_id);
5423                 return -EINVAL;
5424         }
5425
5426         if (mode == NULL) {
5427                 RTE_ETHDEV_LOG(ERR,
5428                         "Cannot get ethdev port %u Rx queue %u burst mode to NULL\n",
5429                         port_id, queue_id);
5430                 return -EINVAL;
5431         }
5432
5433         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_burst_mode_get, -ENOTSUP);
5434         memset(mode, 0, sizeof(*mode));
5435         return eth_err(port_id,
5436                        dev->dev_ops->rx_burst_mode_get(dev, queue_id, mode));
5437 }
5438
5439 int
5440 rte_eth_tx_burst_mode_get(uint16_t port_id, uint16_t queue_id,
5441                           struct rte_eth_burst_mode *mode)
5442 {
5443         struct rte_eth_dev *dev;
5444
5445         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5446         dev = &rte_eth_devices[port_id];
5447
5448         if (queue_id >= dev->data->nb_tx_queues) {
5449                 RTE_ETHDEV_LOG(ERR, "Invalid Tx queue_id=%u\n", queue_id);
5450                 return -EINVAL;
5451         }
5452
5453         if (mode == NULL) {
5454                 RTE_ETHDEV_LOG(ERR,
5455                         "Cannot get ethdev port %u Tx queue %u burst mode to NULL\n",
5456                         port_id, queue_id);
5457                 return -EINVAL;
5458         }
5459
5460         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_burst_mode_get, -ENOTSUP);
5461         memset(mode, 0, sizeof(*mode));
5462         return eth_err(port_id,
5463                        dev->dev_ops->tx_burst_mode_get(dev, queue_id, mode));
5464 }
5465
5466 int
5467 rte_eth_get_monitor_addr(uint16_t port_id, uint16_t queue_id,
5468                 struct rte_power_monitor_cond *pmc)
5469 {
5470         struct rte_eth_dev *dev;
5471
5472         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5473         dev = &rte_eth_devices[port_id];
5474
5475         if (queue_id >= dev->data->nb_rx_queues) {
5476                 RTE_ETHDEV_LOG(ERR, "Invalid Rx queue_id=%u\n", queue_id);
5477                 return -EINVAL;
5478         }
5479
5480         if (pmc == NULL) {
5481                 RTE_ETHDEV_LOG(ERR,
5482                         "Cannot get ethdev port %u Rx queue %u power monitor condition to NULL\n",
5483                         port_id, queue_id);
5484                 return -EINVAL;
5485         }
5486
5487         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_monitor_addr, -ENOTSUP);
5488         return eth_err(port_id,
5489                 dev->dev_ops->get_monitor_addr(dev->data->rx_queues[queue_id], pmc));
5490 }
5491
5492 int
5493 rte_eth_dev_set_mc_addr_list(uint16_t port_id,
5494                              struct rte_ether_addr *mc_addr_set,
5495                              uint32_t nb_mc_addr)
5496 {
5497         struct rte_eth_dev *dev;
5498
5499         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5500         dev = &rte_eth_devices[port_id];
5501
5502         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_mc_addr_list, -ENOTSUP);
5503         return eth_err(port_id, dev->dev_ops->set_mc_addr_list(dev,
5504                                                 mc_addr_set, nb_mc_addr));
5505 }
5506
5507 int
5508 rte_eth_timesync_enable(uint16_t port_id)
5509 {
5510         struct rte_eth_dev *dev;
5511
5512         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5513         dev = &rte_eth_devices[port_id];
5514
5515         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_enable, -ENOTSUP);
5516         return eth_err(port_id, (*dev->dev_ops->timesync_enable)(dev));
5517 }
5518
5519 int
5520 rte_eth_timesync_disable(uint16_t port_id)
5521 {
5522         struct rte_eth_dev *dev;
5523
5524         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5525         dev = &rte_eth_devices[port_id];
5526
5527         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_disable, -ENOTSUP);
5528         return eth_err(port_id, (*dev->dev_ops->timesync_disable)(dev));
5529 }
5530
5531 int
5532 rte_eth_timesync_read_rx_timestamp(uint16_t port_id, struct timespec *timestamp,
5533                                    uint32_t flags)
5534 {
5535         struct rte_eth_dev *dev;
5536
5537         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5538         dev = &rte_eth_devices[port_id];
5539
5540         if (timestamp == NULL) {
5541                 RTE_ETHDEV_LOG(ERR,
5542                         "Cannot read ethdev port %u Rx timestamp to NULL\n",
5543                         port_id);
5544                 return -EINVAL;
5545         }
5546
5547         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_rx_timestamp, -ENOTSUP);
5548         return eth_err(port_id, (*dev->dev_ops->timesync_read_rx_timestamp)
5549                                 (dev, timestamp, flags));
5550 }
5551
5552 int
5553 rte_eth_timesync_read_tx_timestamp(uint16_t port_id,
5554                                    struct timespec *timestamp)
5555 {
5556         struct rte_eth_dev *dev;
5557
5558         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5559         dev = &rte_eth_devices[port_id];
5560
5561         if (timestamp == NULL) {
5562                 RTE_ETHDEV_LOG(ERR,
5563                         "Cannot read ethdev port %u Tx timestamp to NULL\n",
5564                         port_id);
5565                 return -EINVAL;
5566         }
5567
5568         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_tx_timestamp, -ENOTSUP);
5569         return eth_err(port_id, (*dev->dev_ops->timesync_read_tx_timestamp)
5570                                 (dev, timestamp));
5571 }
5572
5573 int
5574 rte_eth_timesync_adjust_time(uint16_t port_id, int64_t delta)
5575 {
5576         struct rte_eth_dev *dev;
5577
5578         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5579         dev = &rte_eth_devices[port_id];
5580
5581         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_adjust_time, -ENOTSUP);
5582         return eth_err(port_id, (*dev->dev_ops->timesync_adjust_time)(dev, delta));
5583 }
5584
5585 int
5586 rte_eth_timesync_read_time(uint16_t port_id, struct timespec *timestamp)
5587 {
5588         struct rte_eth_dev *dev;
5589
5590         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5591         dev = &rte_eth_devices[port_id];
5592
5593         if (timestamp == NULL) {
5594                 RTE_ETHDEV_LOG(ERR,
5595                         "Cannot read ethdev port %u timesync time to NULL\n",
5596                         port_id);
5597                 return -EINVAL;
5598         }
5599
5600         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_time, -ENOTSUP);
5601         return eth_err(port_id, (*dev->dev_ops->timesync_read_time)(dev,
5602                                                                 timestamp));
5603 }
5604
5605 int
5606 rte_eth_timesync_write_time(uint16_t port_id, const struct timespec *timestamp)
5607 {
5608         struct rte_eth_dev *dev;
5609
5610         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5611         dev = &rte_eth_devices[port_id];
5612
5613         if (timestamp == NULL) {
5614                 RTE_ETHDEV_LOG(ERR,
5615                         "Cannot write ethdev port %u timesync from NULL time\n",
5616                         port_id);
5617                 return -EINVAL;
5618         }
5619
5620         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_write_time, -ENOTSUP);
5621         return eth_err(port_id, (*dev->dev_ops->timesync_write_time)(dev,
5622                                                                 timestamp));
5623 }
5624
5625 int
5626 rte_eth_read_clock(uint16_t port_id, uint64_t *clock)
5627 {
5628         struct rte_eth_dev *dev;
5629
5630         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5631         dev = &rte_eth_devices[port_id];
5632
5633         if (clock == NULL) {
5634                 RTE_ETHDEV_LOG(ERR, "Cannot read ethdev port %u clock to NULL\n",
5635                         port_id);
5636                 return -EINVAL;
5637         }
5638
5639         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->read_clock, -ENOTSUP);
5640         return eth_err(port_id, (*dev->dev_ops->read_clock)(dev, clock));
5641 }
5642
5643 int
5644 rte_eth_dev_get_reg_info(uint16_t port_id, struct rte_dev_reg_info *info)
5645 {
5646         struct rte_eth_dev *dev;
5647
5648         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5649         dev = &rte_eth_devices[port_id];
5650
5651         if (info == NULL) {
5652                 RTE_ETHDEV_LOG(ERR,
5653                         "Cannot get ethdev port %u register info to NULL\n",
5654                         port_id);
5655                 return -EINVAL;
5656         }
5657
5658         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_reg, -ENOTSUP);
5659         return eth_err(port_id, (*dev->dev_ops->get_reg)(dev, info));
5660 }
5661
5662 int
5663 rte_eth_dev_get_eeprom_length(uint16_t port_id)
5664 {
5665         struct rte_eth_dev *dev;
5666
5667         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5668         dev = &rte_eth_devices[port_id];
5669
5670         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom_length, -ENOTSUP);
5671         return eth_err(port_id, (*dev->dev_ops->get_eeprom_length)(dev));
5672 }
5673
5674 int
5675 rte_eth_dev_get_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info)
5676 {
5677         struct rte_eth_dev *dev;
5678
5679         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5680         dev = &rte_eth_devices[port_id];
5681
5682         if (info == NULL) {
5683                 RTE_ETHDEV_LOG(ERR,
5684                         "Cannot get ethdev port %u EEPROM info to NULL\n",
5685                         port_id);
5686                 return -EINVAL;
5687         }
5688
5689         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom, -ENOTSUP);
5690         return eth_err(port_id, (*dev->dev_ops->get_eeprom)(dev, info));
5691 }
5692
5693 int
5694 rte_eth_dev_set_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info)
5695 {
5696         struct rte_eth_dev *dev;
5697
5698         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5699         dev = &rte_eth_devices[port_id];
5700
5701         if (info == NULL) {
5702                 RTE_ETHDEV_LOG(ERR,
5703                         "Cannot set ethdev port %u EEPROM from NULL info\n",
5704                         port_id);
5705                 return -EINVAL;
5706         }
5707
5708         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_eeprom, -ENOTSUP);
5709         return eth_err(port_id, (*dev->dev_ops->set_eeprom)(dev, info));
5710 }
5711
5712 int
5713 rte_eth_dev_get_module_info(uint16_t port_id,
5714                             struct rte_eth_dev_module_info *modinfo)
5715 {
5716         struct rte_eth_dev *dev;
5717
5718         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5719         dev = &rte_eth_devices[port_id];
5720
5721         if (modinfo == NULL) {
5722                 RTE_ETHDEV_LOG(ERR,
5723                         "Cannot get ethdev port %u EEPROM module info to NULL\n",
5724                         port_id);
5725                 return -EINVAL;
5726         }
5727
5728         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_module_info, -ENOTSUP);
5729         return (*dev->dev_ops->get_module_info)(dev, modinfo);
5730 }
5731
5732 int
5733 rte_eth_dev_get_module_eeprom(uint16_t port_id,
5734                               struct rte_dev_eeprom_info *info)
5735 {
5736         struct rte_eth_dev *dev;
5737
5738         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5739         dev = &rte_eth_devices[port_id];
5740
5741         if (info == NULL) {
5742                 RTE_ETHDEV_LOG(ERR,
5743                         "Cannot get ethdev port %u module EEPROM info to NULL\n",
5744                         port_id);
5745                 return -EINVAL;
5746         }
5747
5748         if (info->data == NULL) {
5749                 RTE_ETHDEV_LOG(ERR,
5750                         "Cannot get ethdev port %u module EEPROM data to NULL\n",
5751                         port_id);
5752                 return -EINVAL;
5753         }
5754
5755         if (info->length == 0) {
5756                 RTE_ETHDEV_LOG(ERR,
5757                         "Cannot get ethdev port %u module EEPROM to data with zero size\n",
5758                         port_id);
5759                 return -EINVAL;
5760         }
5761
5762         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_module_eeprom, -ENOTSUP);
5763         return (*dev->dev_ops->get_module_eeprom)(dev, info);
5764 }
5765
5766 int
5767 rte_eth_dev_get_dcb_info(uint16_t port_id,
5768                              struct rte_eth_dcb_info *dcb_info)
5769 {
5770         struct rte_eth_dev *dev;
5771
5772         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5773         dev = &rte_eth_devices[port_id];
5774
5775         if (dcb_info == NULL) {
5776                 RTE_ETHDEV_LOG(ERR,
5777                         "Cannot get ethdev port %u DCB info to NULL\n",
5778                         port_id);
5779                 return -EINVAL;
5780         }
5781
5782         memset(dcb_info, 0, sizeof(struct rte_eth_dcb_info));
5783
5784         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_dcb_info, -ENOTSUP);
5785         return eth_err(port_id, (*dev->dev_ops->get_dcb_info)(dev, dcb_info));
5786 }
5787
5788 static void
5789 eth_dev_adjust_nb_desc(uint16_t *nb_desc,
5790                 const struct rte_eth_desc_lim *desc_lim)
5791 {
5792         if (desc_lim->nb_align != 0)
5793                 *nb_desc = RTE_ALIGN_CEIL(*nb_desc, desc_lim->nb_align);
5794
5795         if (desc_lim->nb_max != 0)
5796                 *nb_desc = RTE_MIN(*nb_desc, desc_lim->nb_max);
5797
5798         *nb_desc = RTE_MAX(*nb_desc, desc_lim->nb_min);
5799 }
5800
5801 int
5802 rte_eth_dev_adjust_nb_rx_tx_desc(uint16_t port_id,
5803                                  uint16_t *nb_rx_desc,
5804                                  uint16_t *nb_tx_desc)
5805 {
5806         struct rte_eth_dev_info dev_info;
5807         int ret;
5808
5809         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5810
5811         ret = rte_eth_dev_info_get(port_id, &dev_info);
5812         if (ret != 0)
5813                 return ret;
5814
5815         if (nb_rx_desc != NULL)
5816                 eth_dev_adjust_nb_desc(nb_rx_desc, &dev_info.rx_desc_lim);
5817
5818         if (nb_tx_desc != NULL)
5819                 eth_dev_adjust_nb_desc(nb_tx_desc, &dev_info.tx_desc_lim);
5820
5821         return 0;
5822 }
5823
5824 int
5825 rte_eth_dev_hairpin_capability_get(uint16_t port_id,
5826                                    struct rte_eth_hairpin_cap *cap)
5827 {
5828         struct rte_eth_dev *dev;
5829
5830         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5831         dev = &rte_eth_devices[port_id];
5832
5833         if (cap == NULL) {
5834                 RTE_ETHDEV_LOG(ERR,
5835                         "Cannot get ethdev port %u hairpin capability to NULL\n",
5836                         port_id);
5837                 return -EINVAL;
5838         }
5839
5840         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->hairpin_cap_get, -ENOTSUP);
5841         memset(cap, 0, sizeof(*cap));
5842         return eth_err(port_id, (*dev->dev_ops->hairpin_cap_get)(dev, cap));
5843 }
5844
5845 int
5846 rte_eth_dev_is_rx_hairpin_queue(struct rte_eth_dev *dev, uint16_t queue_id)
5847 {
5848         if (dev->data->rx_queue_state[queue_id] == RTE_ETH_QUEUE_STATE_HAIRPIN)
5849                 return 1;
5850         return 0;
5851 }
5852
5853 int
5854 rte_eth_dev_is_tx_hairpin_queue(struct rte_eth_dev *dev, uint16_t queue_id)
5855 {
5856         if (dev->data->tx_queue_state[queue_id] == RTE_ETH_QUEUE_STATE_HAIRPIN)
5857                 return 1;
5858         return 0;
5859 }
5860
5861 int
5862 rte_eth_dev_pool_ops_supported(uint16_t port_id, const char *pool)
5863 {
5864         struct rte_eth_dev *dev;
5865
5866         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5867         dev = &rte_eth_devices[port_id];
5868
5869         if (pool == NULL) {
5870                 RTE_ETHDEV_LOG(ERR,
5871                         "Cannot test ethdev port %u mempool operation from NULL pool\n",
5872                         port_id);
5873                 return -EINVAL;
5874         }
5875
5876         if (*dev->dev_ops->pool_ops_supported == NULL)
5877                 return 1; /* all pools are supported */
5878
5879         return (*dev->dev_ops->pool_ops_supported)(dev, pool);
5880 }
5881
5882 /**
5883  * A set of values to describe the possible states of a switch domain.
5884  */
5885 enum rte_eth_switch_domain_state {
5886         RTE_ETH_SWITCH_DOMAIN_UNUSED = 0,
5887         RTE_ETH_SWITCH_DOMAIN_ALLOCATED
5888 };
5889
5890 /**
5891  * Array of switch domains available for allocation. Array is sized to
5892  * RTE_MAX_ETHPORTS elements as there cannot be more active switch domains than
5893  * ethdev ports in a single process.
5894  */
5895 static struct rte_eth_dev_switch {
5896         enum rte_eth_switch_domain_state state;
5897 } eth_dev_switch_domains[RTE_MAX_ETHPORTS];
5898
5899 int
5900 rte_eth_switch_domain_alloc(uint16_t *domain_id)
5901 {
5902         uint16_t i;
5903
5904         *domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
5905
5906         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
5907                 if (eth_dev_switch_domains[i].state ==
5908                         RTE_ETH_SWITCH_DOMAIN_UNUSED) {
5909                         eth_dev_switch_domains[i].state =
5910                                 RTE_ETH_SWITCH_DOMAIN_ALLOCATED;
5911                         *domain_id = i;
5912                         return 0;
5913                 }
5914         }
5915
5916         return -ENOSPC;
5917 }
5918
5919 int
5920 rte_eth_switch_domain_free(uint16_t domain_id)
5921 {
5922         if (domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID ||
5923                 domain_id >= RTE_MAX_ETHPORTS)
5924                 return -EINVAL;
5925
5926         if (eth_dev_switch_domains[domain_id].state !=
5927                 RTE_ETH_SWITCH_DOMAIN_ALLOCATED)
5928                 return -EINVAL;
5929
5930         eth_dev_switch_domains[domain_id].state = RTE_ETH_SWITCH_DOMAIN_UNUSED;
5931
5932         return 0;
5933 }
5934
5935 static int
5936 eth_dev_devargs_tokenise(struct rte_kvargs *arglist, const char *str_in)
5937 {
5938         int state;
5939         struct rte_kvargs_pair *pair;
5940         char *letter;
5941
5942         arglist->str = strdup(str_in);
5943         if (arglist->str == NULL)
5944                 return -ENOMEM;
5945
5946         letter = arglist->str;
5947         state = 0;
5948         arglist->count = 0;
5949         pair = &arglist->pairs[0];
5950         while (1) {
5951                 switch (state) {
5952                 case 0: /* Initial */
5953                         if (*letter == '=')
5954                                 return -EINVAL;
5955                         else if (*letter == '\0')
5956                                 return 0;
5957
5958                         state = 1;
5959                         pair->key = letter;
5960                         /* fall-thru */
5961
5962                 case 1: /* Parsing key */
5963                         if (*letter == '=') {
5964                                 *letter = '\0';
5965                                 pair->value = letter + 1;
5966                                 state = 2;
5967                         } else if (*letter == ',' || *letter == '\0')
5968                                 return -EINVAL;
5969                         break;
5970
5971
5972                 case 2: /* Parsing value */
5973                         if (*letter == '[')
5974                                 state = 3;
5975                         else if (*letter == ',') {
5976                                 *letter = '\0';
5977                                 arglist->count++;
5978                                 pair = &arglist->pairs[arglist->count];
5979                                 state = 0;
5980                         } else if (*letter == '\0') {
5981                                 letter--;
5982                                 arglist->count++;
5983                                 pair = &arglist->pairs[arglist->count];
5984                                 state = 0;
5985                         }
5986                         break;
5987
5988                 case 3: /* Parsing list */
5989                         if (*letter == ']')
5990                                 state = 2;
5991                         else if (*letter == '\0')
5992                                 return -EINVAL;
5993                         break;
5994                 }
5995                 letter++;
5996         }
5997 }
5998
5999 int
6000 rte_eth_devargs_parse(const char *dargs, struct rte_eth_devargs *eth_da)
6001 {
6002         struct rte_kvargs args;
6003         struct rte_kvargs_pair *pair;
6004         unsigned int i;
6005         int result = 0;
6006
6007         memset(eth_da, 0, sizeof(*eth_da));
6008
6009         result = eth_dev_devargs_tokenise(&args, dargs);
6010         if (result < 0)
6011                 goto parse_cleanup;
6012
6013         for (i = 0; i < args.count; i++) {
6014                 pair = &args.pairs[i];
6015                 if (strcmp("representor", pair->key) == 0) {
6016                         if (eth_da->type != RTE_ETH_REPRESENTOR_NONE) {
6017                                 RTE_LOG(ERR, EAL, "duplicated representor key: %s\n",
6018                                         dargs);
6019                                 result = -1;
6020                                 goto parse_cleanup;
6021                         }
6022                         result = rte_eth_devargs_parse_representor_ports(
6023                                         pair->value, eth_da);
6024                         if (result < 0)
6025                                 goto parse_cleanup;
6026                 }
6027         }
6028
6029 parse_cleanup:
6030         if (args.str)
6031                 free(args.str);
6032
6033         return result;
6034 }
6035
6036 int
6037 rte_eth_representor_id_get(uint16_t port_id,
6038                            enum rte_eth_representor_type type,
6039                            int controller, int pf, int representor_port,
6040                            uint16_t *repr_id)
6041 {
6042         int ret, n, count;
6043         uint32_t i;
6044         struct rte_eth_representor_info *info = NULL;
6045         size_t size;
6046
6047         if (type == RTE_ETH_REPRESENTOR_NONE)
6048                 return 0;
6049         if (repr_id == NULL)
6050                 return -EINVAL;
6051
6052         /* Get PMD representor range info. */
6053         ret = rte_eth_representor_info_get(port_id, NULL);
6054         if (ret == -ENOTSUP && type == RTE_ETH_REPRESENTOR_VF &&
6055             controller == -1 && pf == -1) {
6056                 /* Direct mapping for legacy VF representor. */
6057                 *repr_id = representor_port;
6058                 return 0;
6059         } else if (ret < 0) {
6060                 return ret;
6061         }
6062         n = ret;
6063         size = sizeof(*info) + n * sizeof(info->ranges[0]);
6064         info = calloc(1, size);
6065         if (info == NULL)
6066                 return -ENOMEM;
6067         info->nb_ranges_alloc = n;
6068         ret = rte_eth_representor_info_get(port_id, info);
6069         if (ret < 0)
6070                 goto out;
6071
6072         /* Default controller and pf to caller. */
6073         if (controller == -1)
6074                 controller = info->controller;
6075         if (pf == -1)
6076                 pf = info->pf;
6077
6078         /* Locate representor ID. */
6079         ret = -ENOENT;
6080         for (i = 0; i < info->nb_ranges; ++i) {
6081                 if (info->ranges[i].type != type)
6082                         continue;
6083                 if (info->ranges[i].controller != controller)
6084                         continue;
6085                 if (info->ranges[i].id_end < info->ranges[i].id_base) {
6086                         RTE_LOG(WARNING, EAL, "Port %hu invalid representor ID Range %u - %u, entry %d\n",
6087                                 port_id, info->ranges[i].id_base,
6088                                 info->ranges[i].id_end, i);
6089                         continue;
6090
6091                 }
6092                 count = info->ranges[i].id_end - info->ranges[i].id_base + 1;
6093                 switch (info->ranges[i].type) {
6094                 case RTE_ETH_REPRESENTOR_PF:
6095                         if (pf < info->ranges[i].pf ||
6096                             pf >= info->ranges[i].pf + count)
6097                                 continue;
6098                         *repr_id = info->ranges[i].id_base +
6099                                    (pf - info->ranges[i].pf);
6100                         ret = 0;
6101                         goto out;
6102                 case RTE_ETH_REPRESENTOR_VF:
6103                         if (info->ranges[i].pf != pf)
6104                                 continue;
6105                         if (representor_port < info->ranges[i].vf ||
6106                             representor_port >= info->ranges[i].vf + count)
6107                                 continue;
6108                         *repr_id = info->ranges[i].id_base +
6109                                    (representor_port - info->ranges[i].vf);
6110                         ret = 0;
6111                         goto out;
6112                 case RTE_ETH_REPRESENTOR_SF:
6113                         if (info->ranges[i].pf != pf)
6114                                 continue;
6115                         if (representor_port < info->ranges[i].sf ||
6116                             representor_port >= info->ranges[i].sf + count)
6117                                 continue;
6118                         *repr_id = info->ranges[i].id_base +
6119                               (representor_port - info->ranges[i].sf);
6120                         ret = 0;
6121                         goto out;
6122                 default:
6123                         break;
6124                 }
6125         }
6126 out:
6127         free(info);
6128         return ret;
6129 }
6130
6131 static int
6132 eth_dev_handle_port_list(const char *cmd __rte_unused,
6133                 const char *params __rte_unused,
6134                 struct rte_tel_data *d)
6135 {
6136         int port_id;
6137
6138         rte_tel_data_start_array(d, RTE_TEL_INT_VAL);
6139         RTE_ETH_FOREACH_DEV(port_id)
6140                 rte_tel_data_add_array_int(d, port_id);
6141         return 0;
6142 }
6143
6144 static void
6145 eth_dev_add_port_queue_stats(struct rte_tel_data *d, uint64_t *q_stats,
6146                 const char *stat_name)
6147 {
6148         int q;
6149         struct rte_tel_data *q_data = rte_tel_data_alloc();
6150         rte_tel_data_start_array(q_data, RTE_TEL_U64_VAL);
6151         for (q = 0; q < RTE_ETHDEV_QUEUE_STAT_CNTRS; q++)
6152                 rte_tel_data_add_array_u64(q_data, q_stats[q]);
6153         rte_tel_data_add_dict_container(d, stat_name, q_data, 0);
6154 }
6155
6156 #define ADD_DICT_STAT(stats, s) rte_tel_data_add_dict_u64(d, #s, stats.s)
6157
6158 static int
6159 eth_dev_handle_port_stats(const char *cmd __rte_unused,
6160                 const char *params,
6161                 struct rte_tel_data *d)
6162 {
6163         struct rte_eth_stats stats;
6164         int port_id, ret;
6165
6166         if (params == NULL || strlen(params) == 0 || !isdigit(*params))
6167                 return -1;
6168
6169         port_id = atoi(params);
6170         if (!rte_eth_dev_is_valid_port(port_id))
6171                 return -1;
6172
6173         ret = rte_eth_stats_get(port_id, &stats);
6174         if (ret < 0)
6175                 return -1;
6176
6177         rte_tel_data_start_dict(d);
6178         ADD_DICT_STAT(stats, ipackets);
6179         ADD_DICT_STAT(stats, opackets);
6180         ADD_DICT_STAT(stats, ibytes);
6181         ADD_DICT_STAT(stats, obytes);
6182         ADD_DICT_STAT(stats, imissed);
6183         ADD_DICT_STAT(stats, ierrors);
6184         ADD_DICT_STAT(stats, oerrors);
6185         ADD_DICT_STAT(stats, rx_nombuf);
6186         eth_dev_add_port_queue_stats(d, stats.q_ipackets, "q_ipackets");
6187         eth_dev_add_port_queue_stats(d, stats.q_opackets, "q_opackets");
6188         eth_dev_add_port_queue_stats(d, stats.q_ibytes, "q_ibytes");
6189         eth_dev_add_port_queue_stats(d, stats.q_obytes, "q_obytes");
6190         eth_dev_add_port_queue_stats(d, stats.q_errors, "q_errors");
6191
6192         return 0;
6193 }
6194
6195 static int
6196 eth_dev_handle_port_xstats(const char *cmd __rte_unused,
6197                 const char *params,
6198                 struct rte_tel_data *d)
6199 {
6200         struct rte_eth_xstat *eth_xstats;
6201         struct rte_eth_xstat_name *xstat_names;
6202         int port_id, num_xstats;
6203         int i, ret;
6204         char *end_param;
6205
6206         if (params == NULL || strlen(params) == 0 || !isdigit(*params))
6207                 return -1;
6208
6209         port_id = strtoul(params, &end_param, 0);
6210         if (*end_param != '\0')
6211                 RTE_ETHDEV_LOG(NOTICE,
6212                         "Extra parameters passed to ethdev telemetry command, ignoring");
6213         if (!rte_eth_dev_is_valid_port(port_id))
6214                 return -1;
6215
6216         num_xstats = rte_eth_xstats_get(port_id, NULL, 0);
6217         if (num_xstats < 0)
6218                 return -1;
6219
6220         /* use one malloc for both names and stats */
6221         eth_xstats = malloc((sizeof(struct rte_eth_xstat) +
6222                         sizeof(struct rte_eth_xstat_name)) * num_xstats);
6223         if (eth_xstats == NULL)
6224                 return -1;
6225         xstat_names = (void *)&eth_xstats[num_xstats];
6226
6227         ret = rte_eth_xstats_get_names(port_id, xstat_names, num_xstats);
6228         if (ret < 0 || ret > num_xstats) {
6229                 free(eth_xstats);
6230                 return -1;
6231         }
6232
6233         ret = rte_eth_xstats_get(port_id, eth_xstats, num_xstats);
6234         if (ret < 0 || ret > num_xstats) {
6235                 free(eth_xstats);
6236                 return -1;
6237         }
6238
6239         rte_tel_data_start_dict(d);
6240         for (i = 0; i < num_xstats; i++)
6241                 rte_tel_data_add_dict_u64(d, xstat_names[i].name,
6242                                 eth_xstats[i].value);
6243         return 0;
6244 }
6245
6246 static int
6247 eth_dev_handle_port_link_status(const char *cmd __rte_unused,
6248                 const char *params,
6249                 struct rte_tel_data *d)
6250 {
6251         static const char *status_str = "status";
6252         int ret, port_id;
6253         struct rte_eth_link link;
6254         char *end_param;
6255
6256         if (params == NULL || strlen(params) == 0 || !isdigit(*params))
6257                 return -1;
6258
6259         port_id = strtoul(params, &end_param, 0);
6260         if (*end_param != '\0')
6261                 RTE_ETHDEV_LOG(NOTICE,
6262                         "Extra parameters passed to ethdev telemetry command, ignoring");
6263         if (!rte_eth_dev_is_valid_port(port_id))
6264                 return -1;
6265
6266         ret = rte_eth_link_get_nowait(port_id, &link);
6267         if (ret < 0)
6268                 return -1;
6269
6270         rte_tel_data_start_dict(d);
6271         if (!link.link_status) {
6272                 rte_tel_data_add_dict_string(d, status_str, "DOWN");
6273                 return 0;
6274         }
6275         rte_tel_data_add_dict_string(d, status_str, "UP");
6276         rte_tel_data_add_dict_u64(d, "speed", link.link_speed);
6277         rte_tel_data_add_dict_string(d, "duplex",
6278                         (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
6279                                 "full-duplex" : "half-duplex");
6280         return 0;
6281 }
6282
6283 static int
6284 eth_dev_handle_port_info(const char *cmd __rte_unused,
6285                 const char *params,
6286                 struct rte_tel_data *d)
6287 {
6288         struct rte_tel_data *rxq_state, *txq_state;
6289         char mac_addr[RTE_ETHER_ADDR_LEN];
6290         struct rte_eth_dev *eth_dev;
6291         char *end_param;
6292         int port_id, i;
6293
6294         if (params == NULL || strlen(params) == 0 || !isdigit(*params))
6295                 return -1;
6296
6297         port_id = strtoul(params, &end_param, 0);
6298         if (*end_param != '\0')
6299                 RTE_ETHDEV_LOG(NOTICE,
6300                         "Extra parameters passed to ethdev telemetry command, ignoring");
6301
6302         if (!rte_eth_dev_is_valid_port(port_id))
6303                 return -EINVAL;
6304
6305         eth_dev = &rte_eth_devices[port_id];
6306         if (!eth_dev)
6307                 return -EINVAL;
6308
6309         rxq_state = rte_tel_data_alloc();
6310         if (!rxq_state)
6311                 return -ENOMEM;
6312
6313         txq_state = rte_tel_data_alloc();
6314         if (!txq_state)
6315                 return -ENOMEM;
6316
6317         rte_tel_data_start_dict(d);
6318         rte_tel_data_add_dict_string(d, "name", eth_dev->data->name);
6319         rte_tel_data_add_dict_int(d, "state", eth_dev->state);
6320         rte_tel_data_add_dict_int(d, "nb_rx_queues",
6321                         eth_dev->data->nb_rx_queues);
6322         rte_tel_data_add_dict_int(d, "nb_tx_queues",
6323                         eth_dev->data->nb_tx_queues);
6324         rte_tel_data_add_dict_int(d, "port_id", eth_dev->data->port_id);
6325         rte_tel_data_add_dict_int(d, "mtu", eth_dev->data->mtu);
6326         rte_tel_data_add_dict_int(d, "rx_mbuf_size_min",
6327                         eth_dev->data->min_rx_buf_size);
6328         rte_tel_data_add_dict_int(d, "rx_mbuf_alloc_fail",
6329                         eth_dev->data->rx_mbuf_alloc_failed);
6330         snprintf(mac_addr, RTE_ETHER_ADDR_LEN, "%02x:%02x:%02x:%02x:%02x:%02x",
6331                          eth_dev->data->mac_addrs->addr_bytes[0],
6332                          eth_dev->data->mac_addrs->addr_bytes[1],
6333                          eth_dev->data->mac_addrs->addr_bytes[2],
6334                          eth_dev->data->mac_addrs->addr_bytes[3],
6335                          eth_dev->data->mac_addrs->addr_bytes[4],
6336                          eth_dev->data->mac_addrs->addr_bytes[5]);
6337         rte_tel_data_add_dict_string(d, "mac_addr", mac_addr);
6338         rte_tel_data_add_dict_int(d, "promiscuous",
6339                         eth_dev->data->promiscuous);
6340         rte_tel_data_add_dict_int(d, "scattered_rx",
6341                         eth_dev->data->scattered_rx);
6342         rte_tel_data_add_dict_int(d, "all_multicast",
6343                         eth_dev->data->all_multicast);
6344         rte_tel_data_add_dict_int(d, "dev_started", eth_dev->data->dev_started);
6345         rte_tel_data_add_dict_int(d, "lro", eth_dev->data->lro);
6346         rte_tel_data_add_dict_int(d, "dev_configured",
6347                         eth_dev->data->dev_configured);
6348
6349         rte_tel_data_start_array(rxq_state, RTE_TEL_INT_VAL);
6350         for (i = 0; i < eth_dev->data->nb_rx_queues; i++)
6351                 rte_tel_data_add_array_int(rxq_state,
6352                                 eth_dev->data->rx_queue_state[i]);
6353
6354         rte_tel_data_start_array(txq_state, RTE_TEL_INT_VAL);
6355         for (i = 0; i < eth_dev->data->nb_tx_queues; i++)
6356                 rte_tel_data_add_array_int(txq_state,
6357                                 eth_dev->data->tx_queue_state[i]);
6358
6359         rte_tel_data_add_dict_container(d, "rxq_state", rxq_state, 0);
6360         rte_tel_data_add_dict_container(d, "txq_state", txq_state, 0);
6361         rte_tel_data_add_dict_int(d, "numa_node", eth_dev->data->numa_node);
6362         rte_tel_data_add_dict_int(d, "dev_flags", eth_dev->data->dev_flags);
6363         rte_tel_data_add_dict_int(d, "rx_offloads",
6364                         eth_dev->data->dev_conf.rxmode.offloads);
6365         rte_tel_data_add_dict_int(d, "tx_offloads",
6366                         eth_dev->data->dev_conf.txmode.offloads);
6367         rte_tel_data_add_dict_int(d, "ethdev_rss_hf",
6368                         eth_dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf);
6369
6370         return 0;
6371 }
6372
6373 int
6374 rte_eth_hairpin_queue_peer_update(uint16_t peer_port, uint16_t peer_queue,
6375                                   struct rte_hairpin_peer_info *cur_info,
6376                                   struct rte_hairpin_peer_info *peer_info,
6377                                   uint32_t direction)
6378 {
6379         struct rte_eth_dev *dev;
6380
6381         /* Current queue information is not mandatory. */
6382         if (peer_info == NULL)
6383                 return -EINVAL;
6384
6385         /* No need to check the validity again. */
6386         dev = &rte_eth_devices[peer_port];
6387         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->hairpin_queue_peer_update,
6388                                 -ENOTSUP);
6389
6390         return (*dev->dev_ops->hairpin_queue_peer_update)(dev, peer_queue,
6391                                         cur_info, peer_info, direction);
6392 }
6393
6394 int
6395 rte_eth_hairpin_queue_peer_bind(uint16_t cur_port, uint16_t cur_queue,
6396                                 struct rte_hairpin_peer_info *peer_info,
6397                                 uint32_t direction)
6398 {
6399         struct rte_eth_dev *dev;
6400
6401         if (peer_info == NULL)
6402                 return -EINVAL;
6403
6404         /* No need to check the validity again. */
6405         dev = &rte_eth_devices[cur_port];
6406         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->hairpin_queue_peer_bind,
6407                                 -ENOTSUP);
6408
6409         return (*dev->dev_ops->hairpin_queue_peer_bind)(dev, cur_queue,
6410                                                         peer_info, direction);
6411 }
6412
6413 int
6414 rte_eth_hairpin_queue_peer_unbind(uint16_t cur_port, uint16_t cur_queue,
6415                                   uint32_t direction)
6416 {
6417         struct rte_eth_dev *dev;
6418
6419         /* No need to check the validity again. */
6420         dev = &rte_eth_devices[cur_port];
6421         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->hairpin_queue_peer_unbind,
6422                                 -ENOTSUP);
6423
6424         return (*dev->dev_ops->hairpin_queue_peer_unbind)(dev, cur_queue,
6425                                                           direction);
6426 }
6427
6428 int
6429 rte_eth_representor_info_get(uint16_t port_id,
6430                              struct rte_eth_representor_info *info)
6431 {
6432         struct rte_eth_dev *dev;
6433
6434         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
6435         dev = &rte_eth_devices[port_id];
6436
6437         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->representor_info_get, -ENOTSUP);
6438         return eth_err(port_id, (*dev->dev_ops->representor_info_get)(dev, info));
6439 }
6440
6441 int
6442 rte_eth_rx_metadata_negotiate(uint16_t port_id, uint64_t *features)
6443 {
6444         struct rte_eth_dev *dev;
6445
6446         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
6447         dev = &rte_eth_devices[port_id];
6448
6449         if (dev->data->dev_configured != 0) {
6450                 RTE_ETHDEV_LOG(ERR,
6451                         "The port (ID=%"PRIu16") is already configured\n",
6452                         port_id);
6453                 return -EBUSY;
6454         }
6455
6456         if (features == NULL) {
6457                 RTE_ETHDEV_LOG(ERR, "Invalid features (NULL)\n");
6458                 return -EINVAL;
6459         }
6460
6461         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_metadata_negotiate, -ENOTSUP);
6462         return eth_err(port_id,
6463                        (*dev->dev_ops->rx_metadata_negotiate)(dev, features));
6464 }
6465
6466 RTE_LOG_REGISTER_DEFAULT(rte_eth_dev_logtype, INFO);
6467
6468 RTE_INIT(ethdev_init_telemetry)
6469 {
6470         rte_telemetry_register_cmd("/ethdev/list", eth_dev_handle_port_list,
6471                         "Returns list of available ethdev ports. Takes no parameters");
6472         rte_telemetry_register_cmd("/ethdev/stats", eth_dev_handle_port_stats,
6473                         "Returns the common stats for a port. Parameters: int port_id");
6474         rte_telemetry_register_cmd("/ethdev/xstats", eth_dev_handle_port_xstats,
6475                         "Returns the extended stats for a port. Parameters: int port_id");
6476         rte_telemetry_register_cmd("/ethdev/link_status",
6477                         eth_dev_handle_port_link_status,
6478                         "Returns the link status for a port. Parameters: int port_id");
6479         rte_telemetry_register_cmd("/ethdev/info", eth_dev_handle_port_info,
6480                         "Returns the device info for a port. Parameters: int port_id");
6481 }