lib/librte_ethdev/rte_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <sys/types.h>
   6 #include <sys/queue.h>
   7 #include <ctype.h>
   8 #include <stdio.h>
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <stdarg.h>
  12 #include <errno.h>
  13 #include <stdbool.h>
  14 #include <stdint.h>
  15 #include <inttypes.h>
  16 #include <netinet/in.h>
  17
  18 #include <rte_byteorder.h>
  19 #include <rte_log.h>
  20 #include <rte_debug.h>
  21 #include <rte_interrupts.h>
  22 #include <rte_memory.h>
  23 #include <rte_memcpy.h>
  24 #include <rte_memzone.h>
  25 #include <rte_launch.h>
  26 #include <rte_eal.h>
  27 #include <rte_per_lcore.h>
  28 #include <rte_lcore.h>
  29 #include <rte_atomic.h>
  30 #include <rte_branch_prediction.h>
  31 #include <rte_common.h>
  32 #include <rte_mempool.h>
  33 #include <rte_malloc.h>
  34 #include <rte_mbuf.h>
  35 #include <rte_errno.h>
  36 #include <rte_spinlock.h>
  37 #include <rte_string_fns.h>
  38 #include <rte_kvargs.h>
  39 #include <rte_class.h>
  40 #include <rte_ether.h>
  41
  42 #include "rte_ethdev.h"
  43 #include "rte_ethdev_driver.h"
  44 #include "ethdev_profile.h"
  45 #include "ethdev_private.h"
  46
  47 int rte_eth_dev_logtype;
  48
  49 static const char *MZ_RTE_ETH_DEV_DATA = "rte_eth_dev_data";
  50 struct rte_eth_dev rte_eth_devices[RTE_MAX_ETHPORTS];
  51
  52 /* spinlock for eth device callbacks */
  53 static rte_spinlock_t rte_eth_dev_cb_lock = RTE_SPINLOCK_INITIALIZER;
  54
  55 /* spinlock for add/remove rx callbacks */
  56 static rte_spinlock_t rte_eth_rx_cb_lock = RTE_SPINLOCK_INITIALIZER;
  57
  58 /* spinlock for add/remove tx callbacks */
  59 static rte_spinlock_t rte_eth_tx_cb_lock = RTE_SPINLOCK_INITIALIZER;
  60
  61 /* spinlock for shared data allocation */
  62 static rte_spinlock_t rte_eth_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
  63
  64 /* store statistics names and its offset in stats structure  */
  65 struct rte_eth_xstats_name_off {
  66         char name[RTE_ETH_XSTATS_NAME_SIZE];
  67         unsigned offset;
  68 };
  69
  70 /* Shared memory between primary and secondary processes. */
  71 static struct {
  72         uint64_t next_owner_id;
  73         rte_spinlock_t ownership_lock;
  74         struct rte_eth_dev_data data[RTE_MAX_ETHPORTS];
  75 } *rte_eth_dev_shared_data;
  76
  77 static const struct rte_eth_xstats_name_off rte_stats_strings[] = {
  78         {"rx_good_packets", offsetof(struct rte_eth_stats, ipackets)},
  79         {"tx_good_packets", offsetof(struct rte_eth_stats, opackets)},
  80         {"rx_good_bytes", offsetof(struct rte_eth_stats, ibytes)},
  81         {"tx_good_bytes", offsetof(struct rte_eth_stats, obytes)},
  82         {"rx_missed_errors", offsetof(struct rte_eth_stats, imissed)},
  83         {"rx_errors", offsetof(struct rte_eth_stats, ierrors)},
  84         {"tx_errors", offsetof(struct rte_eth_stats, oerrors)},
  85         {"rx_mbuf_allocation_errors", offsetof(struct rte_eth_stats,
  86                 rx_nombuf)},
  87 };
  88
  89 #define RTE_NB_STATS RTE_DIM(rte_stats_strings)
  90
  91 static const struct rte_eth_xstats_name_off rte_rxq_stats_strings[] = {
  92         {"packets", offsetof(struct rte_eth_stats, q_ipackets)},
  93         {"bytes", offsetof(struct rte_eth_stats, q_ibytes)},
  94         {"errors", offsetof(struct rte_eth_stats, q_errors)},
  95 };
  96
  97 #define RTE_NB_RXQ_STATS RTE_DIM(rte_rxq_stats_strings)
  98
  99 static const struct rte_eth_xstats_name_off rte_txq_stats_strings[] = {
 100         {"packets", offsetof(struct rte_eth_stats, q_opackets)},
 101         {"bytes", offsetof(struct rte_eth_stats, q_obytes)},
 102 };
 103 #define RTE_NB_TXQ_STATS RTE_DIM(rte_txq_stats_strings)
 104
 105 #define RTE_RX_OFFLOAD_BIT2STR(_name)   \
 106         { DEV_RX_OFFLOAD_##_name, #_name }
 107
 108 static const struct {
 109         uint64_t offload;
 110         const char *name;
 111 } rte_rx_offload_names[] = {
 112         RTE_RX_OFFLOAD_BIT2STR(VLAN_STRIP),
 113         RTE_RX_OFFLOAD_BIT2STR(IPV4_CKSUM),
 114         RTE_RX_OFFLOAD_BIT2STR(UDP_CKSUM),
 115         RTE_RX_OFFLOAD_BIT2STR(TCP_CKSUM),
 116         RTE_RX_OFFLOAD_BIT2STR(TCP_LRO),
 117         RTE_RX_OFFLOAD_BIT2STR(QINQ_STRIP),
 118         RTE_RX_OFFLOAD_BIT2STR(OUTER_IPV4_CKSUM),
 119         RTE_RX_OFFLOAD_BIT2STR(MACSEC_STRIP),
 120         RTE_RX_OFFLOAD_BIT2STR(HEADER_SPLIT),
 121         RTE_RX_OFFLOAD_BIT2STR(VLAN_FILTER),
 122         RTE_RX_OFFLOAD_BIT2STR(VLAN_EXTEND),
 123         RTE_RX_OFFLOAD_BIT2STR(JUMBO_FRAME),
 124         RTE_RX_OFFLOAD_BIT2STR(SCATTER),
 125         RTE_RX_OFFLOAD_BIT2STR(TIMESTAMP),
 126         RTE_RX_OFFLOAD_BIT2STR(SECURITY),
 127         RTE_RX_OFFLOAD_BIT2STR(KEEP_CRC),
 128         RTE_RX_OFFLOAD_BIT2STR(SCTP_CKSUM),
 129         RTE_RX_OFFLOAD_BIT2STR(OUTER_UDP_CKSUM),
 130         RTE_RX_OFFLOAD_BIT2STR(RSS_HASH),
 131 };
 132
 133 #undef RTE_RX_OFFLOAD_BIT2STR
 134
 135 #define RTE_TX_OFFLOAD_BIT2STR(_name)   \
 136         { DEV_TX_OFFLOAD_##_name, #_name }
 137
 138 static const struct {
 139         uint64_t offload;
 140         const char *name;
 141 } rte_tx_offload_names[] = {
 142         RTE_TX_OFFLOAD_BIT2STR(VLAN_INSERT),
 143         RTE_TX_OFFLOAD_BIT2STR(IPV4_CKSUM),
 144         RTE_TX_OFFLOAD_BIT2STR(UDP_CKSUM),
 145         RTE_TX_OFFLOAD_BIT2STR(TCP_CKSUM),
 146         RTE_TX_OFFLOAD_BIT2STR(SCTP_CKSUM),
 147         RTE_TX_OFFLOAD_BIT2STR(TCP_TSO),
 148         RTE_TX_OFFLOAD_BIT2STR(UDP_TSO),
 149         RTE_TX_OFFLOAD_BIT2STR(OUTER_IPV4_CKSUM),
 150         RTE_TX_OFFLOAD_BIT2STR(QINQ_INSERT),
 151         RTE_TX_OFFLOAD_BIT2STR(VXLAN_TNL_TSO),
 152         RTE_TX_OFFLOAD_BIT2STR(GRE_TNL_TSO),
 153         RTE_TX_OFFLOAD_BIT2STR(IPIP_TNL_TSO),
 154         RTE_TX_OFFLOAD_BIT2STR(GENEVE_TNL_TSO),
 155         RTE_TX_OFFLOAD_BIT2STR(MACSEC_INSERT),
 156         RTE_TX_OFFLOAD_BIT2STR(MT_LOCKFREE),
 157         RTE_TX_OFFLOAD_BIT2STR(MULTI_SEGS),
 158         RTE_TX_OFFLOAD_BIT2STR(MBUF_FAST_FREE),
 159         RTE_TX_OFFLOAD_BIT2STR(SECURITY),
 160         RTE_TX_OFFLOAD_BIT2STR(UDP_TNL_TSO),
 161         RTE_TX_OFFLOAD_BIT2STR(IP_TNL_TSO),
 162         RTE_TX_OFFLOAD_BIT2STR(OUTER_UDP_CKSUM),
 163 };
 164
 165 #undef RTE_TX_OFFLOAD_BIT2STR
 166
 167 /**
 168  * The user application callback description.
 169  *
 170  * It contains callback address to be registered by user application,
 171  * the pointer to the parameters for callback, and the event type.
 172  */
 173 struct rte_eth_dev_callback {
 174         TAILQ_ENTRY(rte_eth_dev_callback) next; /**< Callbacks list */
 175         rte_eth_dev_cb_fn cb_fn;                /**< Callback address */
 176         void *cb_arg;                           /**< Parameter for callback */
 177         void *ret_param;                        /**< Return parameter */
 178         enum rte_eth_event_type event;          /**< Interrupt event type */
 179         uint32_t active;                        /**< Callback is executing */
 180 };
 181
 182 enum {
 183         STAT_QMAP_TX = 0,
 184         STAT_QMAP_RX
 185 };
 186
 187 int
 188 rte_eth_iterator_init(struct rte_dev_iterator *iter, const char *devargs_str)
 189 {
 190         int ret;
 191         struct rte_devargs devargs = {.args = NULL};
 192         const char *bus_param_key;
 193         char *bus_str = NULL;
 194         char *cls_str = NULL;
 195         int str_size;
 196
 197         memset(iter, 0, sizeof(*iter));
 198
 199         /*
 200          * The devargs string may use various syntaxes:
 201          *   - 0000:08:00.0,representor=[1-3]
 202          *   - pci:0000:06:00.0,representor=[0,5]
 203          *   - class=eth,mac=00:11:22:33:44:55
 204          * A new syntax is in development (not yet supported):
 205          *   - bus=X,paramX=x/class=Y,paramY=y/driver=Z,paramZ=z
 206          */
 207
 208         /*
 209          * Handle pure class filter (i.e. without any bus-level argument),
 210          * from future new syntax.
 211          * rte_devargs_parse() is not yet supporting the new syntax,
 212          * that's why this simple case is temporarily parsed here.
 213          */
 214 #define iter_anybus_str "class=eth,"
 215         if (strncmp(devargs_str, iter_anybus_str,
 216                         strlen(iter_anybus_str)) == 0) {
 217                 iter->cls_str = devargs_str + strlen(iter_anybus_str);
 218                 goto end;
 219         }
 220
 221         /* Split bus, device and parameters. */
 222         ret = rte_devargs_parse(&devargs, devargs_str);
 223         if (ret != 0)
 224                 goto error;
 225
 226         /*
 227          * Assume parameters of old syntax can match only at ethdev level.
 228          * Extra parameters will be ignored, thanks to "+" prefix.
 229          */
 230         str_size = strlen(devargs.args) + 2;
 231         cls_str = malloc(str_size);
 232         if (cls_str == NULL) {
 233                 ret = -ENOMEM;
 234                 goto error;
 235         }
 236         ret = snprintf(cls_str, str_size, "+%s", devargs.args);
 237         if (ret != str_size - 1) {
 238                 ret = -EINVAL;
 239                 goto error;
 240         }
 241         iter->cls_str = cls_str;
 242         free(devargs.args); /* allocated by rte_devargs_parse() */
 243         devargs.args = NULL;
 244
 245         iter->bus = devargs.bus;
 246         if (iter->bus->dev_iterate == NULL) {
 247                 ret = -ENOTSUP;
 248                 goto error;
 249         }
 250
 251         /* Convert bus args to new syntax for use with new API dev_iterate. */
 252         if (strcmp(iter->bus->name, "vdev") == 0) {
 253                 bus_param_key = "name";
 254         } else if (strcmp(iter->bus->name, "pci") == 0) {
 255                 bus_param_key = "addr";
 256         } else {
 257                 ret = -ENOTSUP;
 258                 goto error;
 259         }
 260         str_size = strlen(bus_param_key) + strlen(devargs.name) + 2;
 261         bus_str = malloc(str_size);
 262         if (bus_str == NULL) {
 263                 ret = -ENOMEM;
 264                 goto error;
 265         }
 266         ret = snprintf(bus_str, str_size, "%s=%s",
 267                         bus_param_key, devargs.name);
 268         if (ret != str_size - 1) {
 269                 ret = -EINVAL;
 270                 goto error;
 271         }
 272         iter->bus_str = bus_str;
 273
 274 end:
 275         iter->cls = rte_class_find_by_name("eth");
 276         return 0;
 277
 278 error:
 279         if (ret == -ENOTSUP)
 280                 RTE_LOG(ERR, EAL, "Bus %s does not support iterating.\n",
 281                                 iter->bus->name);
 282         free(devargs.args);
 283         free(bus_str);
 284         free(cls_str);
 285         return ret;
 286 }
 287
 288 uint16_t
 289 rte_eth_iterator_next(struct rte_dev_iterator *iter)
 290 {
 291         if (iter->cls == NULL) /* invalid ethdev iterator */
 292                 return RTE_MAX_ETHPORTS;
 293
 294         do { /* loop to try all matching rte_device */
 295                 /* If not pure ethdev filter and */
 296                 if (iter->bus != NULL &&
 297                                 /* not in middle of rte_eth_dev iteration, */
 298                                 iter->class_device == NULL) {
 299                         /* get next rte_device to try. */
 300                         iter->device = iter->bus->dev_iterate(
 301                                         iter->device, iter->bus_str, iter);
 302                         if (iter->device == NULL)
 303                                 break; /* no more rte_device candidate */
 304                 }
 305                 /* A device is matching bus part, need to check ethdev part. */
 306                 iter->class_device = iter->cls->dev_iterate(
 307                                 iter->class_device, iter->cls_str, iter);
 308                 if (iter->class_device != NULL)
 309                         return eth_dev_to_id(iter->class_device); /* match */
 310         } while (iter->bus != NULL); /* need to try next rte_device */
 311
 312         /* No more ethdev port to iterate. */
 313         rte_eth_iterator_cleanup(iter);
 314         return RTE_MAX_ETHPORTS;
 315 }
 316
 317 void
 318 rte_eth_iterator_cleanup(struct rte_dev_iterator *iter)
 319 {
 320         if (iter->bus_str == NULL)
 321                 return; /* nothing to free in pure class filter */
 322         free(RTE_CAST_FIELD(iter, bus_str, char *)); /* workaround const */
 323         free(RTE_CAST_FIELD(iter, cls_str, char *)); /* workaround const */
 324         memset(iter, 0, sizeof(*iter));
 325 }
 326
 327 uint16_t
 328 rte_eth_find_next(uint16_t port_id)
 329 {
 330         while (port_id < RTE_MAX_ETHPORTS &&
 331                         rte_eth_devices[port_id].state == RTE_ETH_DEV_UNUSED)
 332                 port_id++;
 333
 334         if (port_id >= RTE_MAX_ETHPORTS)
 335                 return RTE_MAX_ETHPORTS;
 336
 337         return port_id;
 338 }
 339
 340 /*
 341  * Macro to iterate over all valid ports for internal usage.
 342  * Note: RTE_ETH_FOREACH_DEV is different because filtering owned ports.
 343  */
 344 #define RTE_ETH_FOREACH_VALID_DEV(port_id) \
 345         for (port_id = rte_eth_find_next(0); \
 346              port_id < RTE_MAX_ETHPORTS; \
 347              port_id = rte_eth_find_next(port_id + 1))
 348
 349 uint16_t
 350 rte_eth_find_next_of(uint16_t port_id, const struct rte_device *parent)
 351 {
 352         port_id = rte_eth_find_next(port_id);
 353         while (port_id < RTE_MAX_ETHPORTS &&
 354                         rte_eth_devices[port_id].device != parent)
 355                 port_id = rte_eth_find_next(port_id + 1);
 356
 357         return port_id;
 358 }
 359
 360 uint16_t
 361 rte_eth_find_next_sibling(uint16_t port_id, uint16_t ref_port_id)
 362 {
 363         RTE_ETH_VALID_PORTID_OR_ERR_RET(ref_port_id, RTE_MAX_ETHPORTS);
 364         return rte_eth_find_next_of(port_id,
 365                         rte_eth_devices[ref_port_id].device);
 366 }
 367
 368 static void
 369 rte_eth_dev_shared_data_prepare(void)
 370 {
 371         const unsigned flags = 0;
 372         const struct rte_memzone *mz;
 373
 374         rte_spinlock_lock(&rte_eth_shared_data_lock);
 375
 376         if (rte_eth_dev_shared_data == NULL) {
 377                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 378                         /* Allocate port data and ownership shared memory. */
 379                         mz = rte_memzone_reserve(MZ_RTE_ETH_DEV_DATA,
 380                                         sizeof(*rte_eth_dev_shared_data),
 381                                         rte_socket_id(), flags);
 382                 } else
 383                         mz = rte_memzone_lookup(MZ_RTE_ETH_DEV_DATA);
 384                 if (mz == NULL)
 385                         rte_panic("Cannot allocate ethdev shared data\n");
 386
 387                 rte_eth_dev_shared_data = mz->addr;
 388                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 389                         rte_eth_dev_shared_data->next_owner_id =
 390                                         RTE_ETH_DEV_NO_OWNER + 1;
 391                         rte_spinlock_init(&rte_eth_dev_shared_data->ownership_lock);
 392                         memset(rte_eth_dev_shared_data->data, 0,
 393                                sizeof(rte_eth_dev_shared_data->data));
 394                 }
 395         }
 396
 397         rte_spinlock_unlock(&rte_eth_shared_data_lock);
 398 }
 399
 400 static bool
 401 is_allocated(const struct rte_eth_dev *ethdev)
 402 {
 403         return ethdev->data->name[0] != '\0';
 404 }
 405
 406 static struct rte_eth_dev *
 407 _rte_eth_dev_allocated(const char *name)
 408 {
 409         unsigned i;
 410
 411         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 412                 if (rte_eth_devices[i].data != NULL &&
 413                     strcmp(rte_eth_devices[i].data->name, name) == 0)
 414                         return &rte_eth_devices[i];
 415         }
 416         return NULL;
 417 }
 418
 419 struct rte_eth_dev *
 420 rte_eth_dev_allocated(const char *name)
 421 {
 422         struct rte_eth_dev *ethdev;
 423
 424         rte_eth_dev_shared_data_prepare();
 425
 426         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 427
 428         ethdev = _rte_eth_dev_allocated(name);
 429
 430         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 431
 432         return ethdev;
 433 }
 434
 435 static uint16_t
 436 rte_eth_dev_find_free_port(void)
 437 {
 438         unsigned i;
 439
 440         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 441                 /* Using shared name field to find a free port. */
 442                 if (rte_eth_dev_shared_data->data[i].name[0] == '\0') {
 443                         RTE_ASSERT(rte_eth_devices[i].state ==
 444                                    RTE_ETH_DEV_UNUSED);
 445                         return i;
 446                 }
 447         }
 448         return RTE_MAX_ETHPORTS;
 449 }
 450
 451 static struct rte_eth_dev *
 452 eth_dev_get(uint16_t port_id)
 453 {
 454         struct rte_eth_dev *eth_dev = &rte_eth_devices[port_id];
 455
 456         eth_dev->data = &rte_eth_dev_shared_data->data[port_id];
 457
 458         return eth_dev;
 459 }
 460
 461 struct rte_eth_dev *
 462 rte_eth_dev_allocate(const char *name)
 463 {
 464         uint16_t port_id;
 465         struct rte_eth_dev *eth_dev = NULL;
 466         size_t name_len;
 467
 468         name_len = strnlen(name, RTE_ETH_NAME_MAX_LEN);
 469         if (name_len == 0) {
 470                 RTE_ETHDEV_LOG(ERR, "Zero length Ethernet device name\n");
 471                 return NULL;
 472         }
 473
 474         if (name_len >= RTE_ETH_NAME_MAX_LEN) {
 475                 RTE_ETHDEV_LOG(ERR, "Ethernet device name is too long\n");
 476                 return NULL;
 477         }
 478
 479         rte_eth_dev_shared_data_prepare();
 480
 481         /* Synchronize port creation between primary and secondary threads. */
 482         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 483
 484         if (_rte_eth_dev_allocated(name) != NULL) {
 485                 RTE_ETHDEV_LOG(ERR,
 486                         "Ethernet device with name %s already allocated\n",
 487                         name);
 488                 goto unlock;
 489         }
 490
 491         port_id = rte_eth_dev_find_free_port();
 492         if (port_id == RTE_MAX_ETHPORTS) {
 493                 RTE_ETHDEV_LOG(ERR,
 494                         "Reached maximum number of Ethernet ports\n");
 495                 goto unlock;
 496         }
 497
 498         eth_dev = eth_dev_get(port_id);
 499         strlcpy(eth_dev->data->name, name, sizeof(eth_dev->data->name));
 500         eth_dev->data->port_id = port_id;
 501         eth_dev->data->mtu = RTE_ETHER_MTU;
 502
 503 unlock:
 504         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 505
 506         return eth_dev;
 507 }
 508
 509 /*
 510  * Attach to a port already registered by the primary process, which
 511  * makes sure that the same device would have the same port id both
 512  * in the primary and secondary process.
 513  */
 514 struct rte_eth_dev *
 515 rte_eth_dev_attach_secondary(const char *name)
 516 {
 517         uint16_t i;
 518         struct rte_eth_dev *eth_dev = NULL;
 519
 520         rte_eth_dev_shared_data_prepare();
 521
 522         /* Synchronize port attachment to primary port creation and release. */
 523         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 524
 525         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 526                 if (strcmp(rte_eth_dev_shared_data->data[i].name, name) == 0)
 527                         break;
 528         }
 529         if (i == RTE_MAX_ETHPORTS) {
 530                 RTE_ETHDEV_LOG(ERR,
 531                         "Device %s is not driven by the primary process\n",
 532                         name);
 533         } else {
 534                 eth_dev = eth_dev_get(i);
 535                 RTE_ASSERT(eth_dev->data->port_id == i);
 536         }
 537
 538         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 539         return eth_dev;
 540 }
 541
 542 int
 543 rte_eth_dev_release_port(struct rte_eth_dev *eth_dev)
 544 {
 545         if (eth_dev == NULL)
 546                 return -EINVAL;
 547
 548         rte_eth_dev_shared_data_prepare();
 549
 550         if (eth_dev->state != RTE_ETH_DEV_UNUSED)
 551                 _rte_eth_dev_callback_process(eth_dev,
 552                                 RTE_ETH_EVENT_DESTROY, NULL);
 553
 554         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 555
 556         eth_dev->state = RTE_ETH_DEV_UNUSED;
 557
 558         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 559                 rte_free(eth_dev->data->rx_queues);
 560                 rte_free(eth_dev->data->tx_queues);
 561                 rte_free(eth_dev->data->mac_addrs);
 562                 rte_free(eth_dev->data->hash_mac_addrs);
 563                 rte_free(eth_dev->data->dev_private);
 564                 memset(eth_dev->data, 0, sizeof(struct rte_eth_dev_data));
 565         }
 566
 567         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 568
 569         return 0;
 570 }
 571
 572 int
 573 rte_eth_dev_is_valid_port(uint16_t port_id)
 574 {
 575         if (port_id >= RTE_MAX_ETHPORTS ||
 576             (rte_eth_devices[port_id].state == RTE_ETH_DEV_UNUSED))
 577                 return 0;
 578         else
 579                 return 1;
 580 }
 581
 582 static int
 583 rte_eth_is_valid_owner_id(uint64_t owner_id)
 584 {
 585         if (owner_id == RTE_ETH_DEV_NO_OWNER ||
 586             rte_eth_dev_shared_data->next_owner_id <= owner_id)
 587                 return 0;
 588         return 1;
 589 }
 590
 591 uint64_t
 592 rte_eth_find_next_owned_by(uint16_t port_id, const uint64_t owner_id)
 593 {
 594         port_id = rte_eth_find_next(port_id);
 595         while (port_id < RTE_MAX_ETHPORTS &&
 596                         rte_eth_devices[port_id].data->owner.id != owner_id)
 597                 port_id = rte_eth_find_next(port_id + 1);
 598
 599         return port_id;
 600 }
 601
 602 int
 603 rte_eth_dev_owner_new(uint64_t *owner_id)
 604 {
 605         rte_eth_dev_shared_data_prepare();
 606
 607         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 608
 609         *owner_id = rte_eth_dev_shared_data->next_owner_id++;
 610
 611         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 612         return 0;
 613 }
 614
 615 static int
 616 _rte_eth_dev_owner_set(const uint16_t port_id, const uint64_t old_owner_id,
 617                        const struct rte_eth_dev_owner *new_owner)
 618 {
 619         struct rte_eth_dev *ethdev = &rte_eth_devices[port_id];
 620         struct rte_eth_dev_owner *port_owner;
 621
 622         if (port_id >= RTE_MAX_ETHPORTS || !is_allocated(ethdev)) {
 623                 RTE_ETHDEV_LOG(ERR, "Port id %"PRIu16" is not allocated\n",
 624                         port_id);
 625                 return -ENODEV;
 626         }
 627
 628         if (!rte_eth_is_valid_owner_id(new_owner->id) &&
 629             !rte_eth_is_valid_owner_id(old_owner_id)) {
 630                 RTE_ETHDEV_LOG(ERR,
 631                         "Invalid owner old_id=%016"PRIx64" new_id=%016"PRIx64"\n",
 632                        old_owner_id, new_owner->id);
 633                 return -EINVAL;
 634         }
 635
 636         port_owner = &rte_eth_devices[port_id].data->owner;
 637         if (port_owner->id != old_owner_id) {
 638                 RTE_ETHDEV_LOG(ERR,
 639                         "Cannot set owner to port %u already owned by %s_%016"PRIX64"\n",
 640                         port_id, port_owner->name, port_owner->id);
 641                 return -EPERM;
 642         }
 643
 644         /* can not truncate (same structure) */
 645         strlcpy(port_owner->name, new_owner->name, RTE_ETH_MAX_OWNER_NAME_LEN);
 646
 647         port_owner->id = new_owner->id;
 648
 649         RTE_ETHDEV_LOG(DEBUG, "Port %u owner is %s_%016"PRIx64"\n",
 650                 port_id, new_owner->name, new_owner->id);
 651
 652         return 0;
 653 }
 654
 655 int
 656 rte_eth_dev_owner_set(const uint16_t port_id,
 657                       const struct rte_eth_dev_owner *owner)
 658 {
 659         int ret;
 660
 661         rte_eth_dev_shared_data_prepare();
 662
 663         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 664
 665         ret = _rte_eth_dev_owner_set(port_id, RTE_ETH_DEV_NO_OWNER, owner);
 666
 667         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 668         return ret;
 669 }
 670
 671 int
 672 rte_eth_dev_owner_unset(const uint16_t port_id, const uint64_t owner_id)
 673 {
 674         const struct rte_eth_dev_owner new_owner = (struct rte_eth_dev_owner)
 675                         {.id = RTE_ETH_DEV_NO_OWNER, .name = ""};
 676         int ret;
 677
 678         rte_eth_dev_shared_data_prepare();
 679
 680         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 681
 682         ret = _rte_eth_dev_owner_set(port_id, owner_id, &new_owner);
 683
 684         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 685         return ret;
 686 }
 687
 688 int
 689 rte_eth_dev_owner_delete(const uint64_t owner_id)
 690 {
 691         uint16_t port_id;
 692         int ret = 0;
 693
 694         rte_eth_dev_shared_data_prepare();
 695
 696         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 697
 698         if (rte_eth_is_valid_owner_id(owner_id)) {
 699                 for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++)
 700                         if (rte_eth_devices[port_id].data->owner.id == owner_id)
 701                                 memset(&rte_eth_devices[port_id].data->owner, 0,
 702                                        sizeof(struct rte_eth_dev_owner));
 703                 RTE_ETHDEV_LOG(NOTICE,
 704                         "All port owners owned by %016"PRIx64" identifier have removed\n",
 705                         owner_id);
 706         } else {
 707                 RTE_ETHDEV_LOG(ERR,
 708                                "Invalid owner id=%016"PRIx64"\n",
 709                                owner_id);
 710                 ret = -EINVAL;
 711         }
 712
 713         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 714
 715         return ret;
 716 }
 717
 718 int
 719 rte_eth_dev_owner_get(const uint16_t port_id, struct rte_eth_dev_owner *owner)
 720 {
 721         int ret = 0;
 722         struct rte_eth_dev *ethdev = &rte_eth_devices[port_id];
 723
 724         rte_eth_dev_shared_data_prepare();
 725
 726         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 727
 728         if (port_id >= RTE_MAX_ETHPORTS || !is_allocated(ethdev)) {
 729                 RTE_ETHDEV_LOG(ERR, "Port id %"PRIu16" is not allocated\n",
 730                         port_id);
 731                 ret = -ENODEV;
 732         } else {
 733                 rte_memcpy(owner, &ethdev->data->owner, sizeof(*owner));
 734         }
 735
 736         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 737         return ret;
 738 }
 739
 740 int
 741 rte_eth_dev_socket_id(uint16_t port_id)
 742 {
 743         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -1);
 744         return rte_eth_devices[port_id].data->numa_node;
 745 }
 746
 747 void *
 748 rte_eth_dev_get_sec_ctx(uint16_t port_id)
 749 {
 750         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, NULL);
 751         return rte_eth_devices[port_id].security_ctx;
 752 }
 753
 754 uint16_t
 755 rte_eth_dev_count_avail(void)
 756 {
 757         uint16_t p;
 758         uint16_t count;
 759
 760         count = 0;
 761
 762         RTE_ETH_FOREACH_DEV(p)
 763                 count++;
 764
 765         return count;
 766 }
 767
 768 uint16_t
 769 rte_eth_dev_count_total(void)
 770 {
 771         uint16_t port, count = 0;
 772
 773         RTE_ETH_FOREACH_VALID_DEV(port)
 774                 count++;
 775
 776         return count;
 777 }
 778
 779 int
 780 rte_eth_dev_get_name_by_port(uint16_t port_id, char *name)
 781 {
 782         char *tmp;
 783
 784         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 785
 786         if (name == NULL) {
 787                 RTE_ETHDEV_LOG(ERR, "Null pointer is specified\n");
 788                 return -EINVAL;
 789         }
 790
 791         /* shouldn't check 'rte_eth_devices[i].data',
 792          * because it might be overwritten by VDEV PMD */
 793         tmp = rte_eth_dev_shared_data->data[port_id].name;
 794         strcpy(name, tmp);
 795         return 0;
 796 }
 797
 798 int
 799 rte_eth_dev_get_port_by_name(const char *name, uint16_t *port_id)
 800 {
 801         uint32_t pid;
 802
 803         if (name == NULL) {
 804                 RTE_ETHDEV_LOG(ERR, "Null pointer is specified\n");
 805                 return -EINVAL;
 806         }
 807
 808         RTE_ETH_FOREACH_VALID_DEV(pid)
 809                 if (!strcmp(name, rte_eth_dev_shared_data->data[pid].name)) {
 810                         *port_id = pid;
 811                         return 0;
 812                 }
 813
 814         return -ENODEV;
 815 }
 816
 817 static int
 818 eth_err(uint16_t port_id, int ret)
 819 {
 820         if (ret == 0)
 821                 return 0;
 822         if (rte_eth_dev_is_removed(port_id))
 823                 return -EIO;
 824         return ret;
 825 }
 826
 827 static int
 828 rte_eth_dev_rx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
 829 {
 830         uint16_t old_nb_queues = dev->data->nb_rx_queues;
 831         void **rxq;
 832         unsigned i;
 833
 834         if (dev->data->rx_queues == NULL && nb_queues != 0) { /* first time configuration */
 835                 dev->data->rx_queues = rte_zmalloc("ethdev->rx_queues",
 836                                 sizeof(dev->data->rx_queues[0]) * nb_queues,
 837                                 RTE_CACHE_LINE_SIZE);
 838                 if (dev->data->rx_queues == NULL) {
 839                         dev->data->nb_rx_queues = 0;
 840                         return -(ENOMEM);
 841                 }
 842         } else if (dev->data->rx_queues != NULL && nb_queues != 0) { /* re-configure */
 843                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release, -ENOTSUP);
 844
 845                 rxq = dev->data->rx_queues;
 846
 847                 for (i = nb_queues; i < old_nb_queues; i++)
 848                         (*dev->dev_ops->rx_queue_release)(rxq[i]);
 849                 rxq = rte_realloc(rxq, sizeof(rxq[0]) * nb_queues,
 850                                 RTE_CACHE_LINE_SIZE);
 851                 if (rxq == NULL)
 852                         return -(ENOMEM);
 853                 if (nb_queues > old_nb_queues) {
 854                         uint16_t new_qs = nb_queues - old_nb_queues;
 855
 856                         memset(rxq + old_nb_queues, 0,
 857                                 sizeof(rxq[0]) * new_qs);
 858                 }
 859
 860                 dev->data->rx_queues = rxq;
 861
 862         } else if (dev->data->rx_queues != NULL && nb_queues == 0) {
 863                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release, -ENOTSUP);
 864
 865                 rxq = dev->data->rx_queues;
 866
 867                 for (i = nb_queues; i < old_nb_queues; i++)
 868                         (*dev->dev_ops->rx_queue_release)(rxq[i]);
 869
 870                 rte_free(dev->data->rx_queues);
 871                 dev->data->rx_queues = NULL;
 872         }
 873         dev->data->nb_rx_queues = nb_queues;
 874         return 0;
 875 }
 876
 877 int
 878 rte_eth_dev_rx_queue_start(uint16_t port_id, uint16_t rx_queue_id)
 879 {
 880         struct rte_eth_dev *dev;
 881
 882         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 883
 884         dev = &rte_eth_devices[port_id];
 885         if (!dev->data->dev_started) {
 886                 RTE_ETHDEV_LOG(ERR,
 887                         "Port %u must be started before start any queue\n",
 888                         port_id);
 889                 return -EINVAL;
 890         }
 891
 892         if (rx_queue_id >= dev->data->nb_rx_queues) {
 893                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
 894                 return -EINVAL;
 895         }
 896
 897         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_start, -ENOTSUP);
 898
 899         if (rte_eth_dev_is_rx_hairpin_queue(dev, rx_queue_id)) {
 900                 RTE_ETHDEV_LOG(INFO,
 901                         "Can't start Rx hairpin queue %"PRIu16" of device with port_id=%"PRIu16"\n",
 902                         rx_queue_id, port_id);
 903                 return -EINVAL;
 904         }
 905
 906         if (dev->data->rx_queue_state[rx_queue_id] != RTE_ETH_QUEUE_STATE_STOPPED) {
 907                 RTE_ETHDEV_LOG(INFO,
 908                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already started\n",
 909                         rx_queue_id, port_id);
 910                 return 0;
 911         }
 912
 913         return eth_err(port_id, dev->dev_ops->rx_queue_start(dev,
 914                                                              rx_queue_id));
 915
 916 }
 917
 918 int
 919 rte_eth_dev_rx_queue_stop(uint16_t port_id, uint16_t rx_queue_id)
 920 {
 921         struct rte_eth_dev *dev;
 922
 923         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 924
 925         dev = &rte_eth_devices[port_id];
 926         if (rx_queue_id >= dev->data->nb_rx_queues) {
 927                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
 928                 return -EINVAL;
 929         }
 930
 931         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_stop, -ENOTSUP);
 932
 933         if (rte_eth_dev_is_rx_hairpin_queue(dev, rx_queue_id)) {
 934                 RTE_ETHDEV_LOG(INFO,
 935                         "Can't stop Rx hairpin queue %"PRIu16" of device with port_id=%"PRIu16"\n",
 936                         rx_queue_id, port_id);
 937                 return -EINVAL;
 938         }
 939
 940         if (dev->data->rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STOPPED) {
 941                 RTE_ETHDEV_LOG(INFO,
 942                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already stopped\n",
 943                         rx_queue_id, port_id);
 944                 return 0;
 945         }
 946
 947         return eth_err(port_id, dev->dev_ops->rx_queue_stop(dev, rx_queue_id));
 948
 949 }
 950
 951 int
 952 rte_eth_dev_tx_queue_start(uint16_t port_id, uint16_t tx_queue_id)
 953 {
 954         struct rte_eth_dev *dev;
 955
 956         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 957
 958         dev = &rte_eth_devices[port_id];
 959         if (!dev->data->dev_started) {
 960                 RTE_ETHDEV_LOG(ERR,
 961                         "Port %u must be started before start any queue\n",
 962                         port_id);
 963                 return -EINVAL;
 964         }
 965
 966         if (tx_queue_id >= dev->data->nb_tx_queues) {
 967                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
 968                 return -EINVAL;
 969         }
 970
 971         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_start, -ENOTSUP);
 972
 973         if (rte_eth_dev_is_tx_hairpin_queue(dev, tx_queue_id)) {
 974                 RTE_ETHDEV_LOG(INFO,
 975                         "Can't start Tx hairpin queue %"PRIu16" of device with port_id=%"PRIu16"\n",
 976                         tx_queue_id, port_id);
 977                 return -EINVAL;
 978         }
 979
 980         if (dev->data->tx_queue_state[tx_queue_id] != RTE_ETH_QUEUE_STATE_STOPPED) {
 981                 RTE_ETHDEV_LOG(INFO,
 982                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already started\n",
 983                         tx_queue_id, port_id);
 984                 return 0;
 985         }
 986
 987         return eth_err(port_id, dev->dev_ops->tx_queue_start(dev, tx_queue_id));
 988 }
 989
 990 int
 991 rte_eth_dev_tx_queue_stop(uint16_t port_id, uint16_t tx_queue_id)
 992 {
 993         struct rte_eth_dev *dev;
 994
 995         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 996
 997         dev = &rte_eth_devices[port_id];
 998         if (tx_queue_id >= dev->data->nb_tx_queues) {
 999                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
1000                 return -EINVAL;
1001         }
1002
1003         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_stop, -ENOTSUP);
1004
1005         if (rte_eth_dev_is_tx_hairpin_queue(dev, tx_queue_id)) {
1006                 RTE_ETHDEV_LOG(INFO,
1007                         "Can't stop Tx hairpin queue %"PRIu16" of device with port_id=%"PRIu16"\n",
1008                         tx_queue_id, port_id);
1009                 return -EINVAL;
1010         }
1011
1012         if (dev->data->tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STOPPED) {
1013                 RTE_ETHDEV_LOG(INFO,
1014                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already stopped\n",
1015                         tx_queue_id, port_id);
1016                 return 0;
1017         }
1018
1019         return eth_err(port_id, dev->dev_ops->tx_queue_stop(dev, tx_queue_id));
1020
1021 }
1022
1023 static int
1024 rte_eth_dev_tx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
1025 {
1026         uint16_t old_nb_queues = dev->data->nb_tx_queues;
1027         void **txq;
1028         unsigned i;
1029
1030         if (dev->data->tx_queues == NULL && nb_queues != 0) { /* first time configuration */
1031                 dev->data->tx_queues = rte_zmalloc("ethdev->tx_queues",
1032                                                    sizeof(dev->data->tx_queues[0]) * nb_queues,
1033                                                    RTE_CACHE_LINE_SIZE);
1034                 if (dev->data->tx_queues == NULL) {
1035                         dev->data->nb_tx_queues = 0;
1036                         return -(ENOMEM);
1037                 }
1038         } else if (dev->data->tx_queues != NULL && nb_queues != 0) { /* re-configure */
1039                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release, -ENOTSUP);
1040
1041                 txq = dev->data->tx_queues;
1042
1043                 for (i = nb_queues; i < old_nb_queues; i++)
1044                         (*dev->dev_ops->tx_queue_release)(txq[i]);
1045                 txq = rte_realloc(txq, sizeof(txq[0]) * nb_queues,
1046                                   RTE_CACHE_LINE_SIZE);
1047                 if (txq == NULL)
1048                         return -ENOMEM;
1049                 if (nb_queues > old_nb_queues) {
1050                         uint16_t new_qs = nb_queues - old_nb_queues;
1051
1052                         memset(txq + old_nb_queues, 0,
1053                                sizeof(txq[0]) * new_qs);
1054                 }
1055
1056                 dev->data->tx_queues = txq;
1057
1058         } else if (dev->data->tx_queues != NULL && nb_queues == 0) {
1059                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release, -ENOTSUP);
1060
1061                 txq = dev->data->tx_queues;
1062
1063                 for (i = nb_queues; i < old_nb_queues; i++)
1064                         (*dev->dev_ops->tx_queue_release)(txq[i]);
1065
1066                 rte_free(dev->data->tx_queues);
1067                 dev->data->tx_queues = NULL;
1068         }
1069         dev->data->nb_tx_queues = nb_queues;
1070         return 0;
1071 }
1072
1073 uint32_t
1074 rte_eth_speed_bitflag(uint32_t speed, int duplex)
1075 {
1076         switch (speed) {
1077         case ETH_SPEED_NUM_10M:
1078                 return duplex ? ETH_LINK_SPEED_10M : ETH_LINK_SPEED_10M_HD;
1079         case ETH_SPEED_NUM_100M:
1080                 return duplex ? ETH_LINK_SPEED_100M : ETH_LINK_SPEED_100M_HD;
1081         case ETH_SPEED_NUM_1G:
1082                 return ETH_LINK_SPEED_1G;
1083         case ETH_SPEED_NUM_2_5G:
1084                 return ETH_LINK_SPEED_2_5G;
1085         case ETH_SPEED_NUM_5G:
1086                 return ETH_LINK_SPEED_5G;
1087         case ETH_SPEED_NUM_10G:
1088                 return ETH_LINK_SPEED_10G;
1089         case ETH_SPEED_NUM_20G:
1090                 return ETH_LINK_SPEED_20G;
1091         case ETH_SPEED_NUM_25G:
1092                 return ETH_LINK_SPEED_25G;
1093         case ETH_SPEED_NUM_40G:
1094                 return ETH_LINK_SPEED_40G;
1095         case ETH_SPEED_NUM_50G:
1096                 return ETH_LINK_SPEED_50G;
1097         case ETH_SPEED_NUM_56G:
1098                 return ETH_LINK_SPEED_56G;
1099         case ETH_SPEED_NUM_100G:
1100                 return ETH_LINK_SPEED_100G;
1101         default:
1102                 return 0;
1103         }
1104 }
1105
1106 const char *
1107 rte_eth_dev_rx_offload_name(uint64_t offload)
1108 {
1109         const char *name = "UNKNOWN";
1110         unsigned int i;
1111
1112         for (i = 0; i < RTE_DIM(rte_rx_offload_names); ++i) {
1113                 if (offload == rte_rx_offload_names[i].offload) {
1114                         name = rte_rx_offload_names[i].name;
1115                         break;
1116                 }
1117         }
1118
1119         return name;
1120 }
1121
1122 const char *
1123 rte_eth_dev_tx_offload_name(uint64_t offload)
1124 {
1125         const char *name = "UNKNOWN";
1126         unsigned int i;
1127
1128         for (i = 0; i < RTE_DIM(rte_tx_offload_names); ++i) {
1129                 if (offload == rte_tx_offload_names[i].offload) {
1130                         name = rte_tx_offload_names[i].name;
1131                         break;
1132                 }
1133         }
1134
1135         return name;
1136 }
1137
1138 static inline int
1139 check_lro_pkt_size(uint16_t port_id, uint32_t config_size,
1140                    uint32_t max_rx_pkt_len, uint32_t dev_info_size)
1141 {
1142         int ret = 0;
1143
1144         if (dev_info_size == 0) {
1145                 if (config_size != max_rx_pkt_len) {
1146                         RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%d max_lro_pkt_size"
1147                                        " %u != %u is not allowed\n",
1148                                        port_id, config_size, max_rx_pkt_len);
1149                         ret = -EINVAL;
1150                 }
1151         } else if (config_size > dev_info_size) {
1152                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%d max_lro_pkt_size %u "
1153                                "> max allowed value %u\n", port_id, config_size,
1154                                dev_info_size);
1155                 ret = -EINVAL;
1156         } else if (config_size < RTE_ETHER_MIN_LEN) {
1157                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%d max_lro_pkt_size %u "
1158                                "< min allowed value %u\n", port_id, config_size,
1159                                (unsigned int)RTE_ETHER_MIN_LEN);
1160                 ret = -EINVAL;
1161         }
1162         return ret;
1163 }
1164
1165 /*
1166  * Validate offloads that are requested through rte_eth_dev_configure against
1167  * the offloads successfully set by the ethernet device.
1168  *
1169  * @param port_id
1170  *   The port identifier of the Ethernet device.
1171  * @param req_offloads
1172  *   The offloads that have been requested through `rte_eth_dev_configure`.
1173  * @param set_offloads
1174  *   The offloads successfully set by the ethernet device.
1175  * @param offload_type
1176  *   The offload type i.e. Rx/Tx string.
1177  * @param offload_name
1178  *   The function that prints the offload name.
1179  * @return
1180  *   - (0) if validation successful.
1181  *   - (-EINVAL) if requested offload has been silently disabled.
1182  *
1183  */
1184 static int
1185 validate_offloads(uint16_t port_id, uint64_t req_offloads,
1186                   uint64_t set_offloads, const char *offload_type,
1187                   const char *(*offload_name)(uint64_t))
1188 {
1189         uint64_t offloads_diff = req_offloads ^ set_offloads;
1190         uint64_t offload;
1191         int ret = 0;
1192
1193         while (offloads_diff != 0) {
1194                 /* Check if any offload is requested but not enabled. */
1195                 offload = 1ULL << __builtin_ctzll(offloads_diff);
1196                 if (offload & req_offloads) {
1197                         RTE_ETHDEV_LOG(ERR,
1198                                 "Port %u failed to enable %s offload %s\n",
1199                                 port_id, offload_type, offload_name(offload));
1200                         ret = -EINVAL;
1201                 }
1202
1203                 /* Check if offload couldn't be disabled. */
1204                 if (offload & set_offloads) {
1205                         RTE_ETHDEV_LOG(DEBUG,
1206                                 "Port %u %s offload %s is not requested but enabled\n",
1207                                 port_id, offload_type, offload_name(offload));
1208                 }
1209
1210                 offloads_diff &= ~offload;
1211         }
1212
1213         return ret;
1214 }
1215
1216 int
1217 rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
1218                       const struct rte_eth_conf *dev_conf)
1219 {
1220         struct rte_eth_dev *dev;
1221         struct rte_eth_dev_info dev_info;
1222         struct rte_eth_conf orig_conf;
1223         int diag;
1224         int ret;
1225
1226         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1227
1228         dev = &rte_eth_devices[port_id];
1229
1230         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
1231
1232         if (dev->data->dev_started) {
1233                 RTE_ETHDEV_LOG(ERR,
1234                         "Port %u must be stopped to allow configuration\n",
1235                         port_id);
1236                 return -EBUSY;
1237         }
1238
1239          /* Store original config, as rollback required on failure */
1240         memcpy(&orig_conf, &dev->data->dev_conf, sizeof(dev->data->dev_conf));
1241
1242         /*
1243          * Copy the dev_conf parameter into the dev structure.
1244          * rte_eth_dev_info_get() requires dev_conf, copy it before dev_info get
1245          */
1246         if (dev_conf != &dev->data->dev_conf)
1247                 memcpy(&dev->data->dev_conf, dev_conf,
1248                        sizeof(dev->data->dev_conf));
1249
1250         ret = rte_eth_dev_info_get(port_id, &dev_info);
1251         if (ret != 0)
1252                 goto rollback;
1253
1254         /* If number of queues specified by application for both Rx and Tx is
1255          * zero, use driver preferred values. This cannot be done individually
1256          * as it is valid for either Tx or Rx (but not both) to be zero.
1257          * If driver does not provide any preferred valued, fall back on
1258          * EAL defaults.
1259          */
1260         if (nb_rx_q == 0 && nb_tx_q == 0) {
1261                 nb_rx_q = dev_info.default_rxportconf.nb_queues;
1262                 if (nb_rx_q == 0)
1263                         nb_rx_q = RTE_ETH_DEV_FALLBACK_RX_NBQUEUES;
1264                 nb_tx_q = dev_info.default_txportconf.nb_queues;
1265                 if (nb_tx_q == 0)
1266                         nb_tx_q = RTE_ETH_DEV_FALLBACK_TX_NBQUEUES;
1267         }
1268
1269         if (nb_rx_q > RTE_MAX_QUEUES_PER_PORT) {
1270                 RTE_ETHDEV_LOG(ERR,
1271                         "Number of RX queues requested (%u) is greater than max supported(%d)\n",
1272                         nb_rx_q, RTE_MAX_QUEUES_PER_PORT);
1273                 ret = -EINVAL;
1274                 goto rollback;
1275         }
1276
1277         if (nb_tx_q > RTE_MAX_QUEUES_PER_PORT) {
1278                 RTE_ETHDEV_LOG(ERR,
1279                         "Number of TX queues requested (%u) is greater than max supported(%d)\n",
1280                         nb_tx_q, RTE_MAX_QUEUES_PER_PORT);
1281                 ret = -EINVAL;
1282                 goto rollback;
1283         }
1284
1285         /*
1286          * Check that the numbers of RX and TX queues are not greater
1287          * than the maximum number of RX and TX queues supported by the
1288          * configured device.
1289          */
1290         if (nb_rx_q > dev_info.max_rx_queues) {
1291                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u nb_rx_queues=%u > %u\n",
1292                         port_id, nb_rx_q, dev_info.max_rx_queues);
1293                 ret = -EINVAL;
1294                 goto rollback;
1295         }
1296
1297         if (nb_tx_q > dev_info.max_tx_queues) {
1298                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u nb_tx_queues=%u > %u\n",
1299                         port_id, nb_tx_q, dev_info.max_tx_queues);
1300                 ret = -EINVAL;
1301                 goto rollback;
1302         }
1303
1304         /* Check that the device supports requested interrupts */
1305         if ((dev_conf->intr_conf.lsc == 1) &&
1306                         (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC))) {
1307                 RTE_ETHDEV_LOG(ERR, "Driver %s does not support lsc\n",
1308                         dev->device->driver->name);
1309                 ret = -EINVAL;
1310                 goto rollback;
1311         }
1312         if ((dev_conf->intr_conf.rmv == 1) &&
1313                         (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_RMV))) {
1314                 RTE_ETHDEV_LOG(ERR, "Driver %s does not support rmv\n",
1315                         dev->device->driver->name);
1316                 ret = -EINVAL;
1317                 goto rollback;
1318         }
1319
1320         /*
1321          * If jumbo frames are enabled, check that the maximum RX packet
1322          * length is supported by the configured device.
1323          */
1324         if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
1325                 if (dev_conf->rxmode.max_rx_pkt_len > dev_info.max_rx_pktlen) {
1326                         RTE_ETHDEV_LOG(ERR,
1327                                 "Ethdev port_id=%u max_rx_pkt_len %u > max valid value %u\n",
1328                                 port_id, dev_conf->rxmode.max_rx_pkt_len,
1329                                 dev_info.max_rx_pktlen);
1330                         ret = -EINVAL;
1331                         goto rollback;
1332                 } else if (dev_conf->rxmode.max_rx_pkt_len < RTE_ETHER_MIN_LEN) {
1333                         RTE_ETHDEV_LOG(ERR,
1334                                 "Ethdev port_id=%u max_rx_pkt_len %u < min valid value %u\n",
1335                                 port_id, dev_conf->rxmode.max_rx_pkt_len,
1336                                 (unsigned int)RTE_ETHER_MIN_LEN);
1337                         ret = -EINVAL;
1338                         goto rollback;
1339                 }
1340         } else {
1341                 if (dev_conf->rxmode.max_rx_pkt_len < RTE_ETHER_MIN_LEN ||
1342                         dev_conf->rxmode.max_rx_pkt_len > RTE_ETHER_MAX_LEN)
1343                         /* Use default value */
1344                         dev->data->dev_conf.rxmode.max_rx_pkt_len =
1345                                                         RTE_ETHER_MAX_LEN;
1346         }
1347
1348         /*
1349          * If LRO is enabled, check that the maximum aggregated packet
1350          * size is supported by the configured device.
1351          */
1352         if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_TCP_LRO) {
1353                 if (dev_conf->rxmode.max_lro_pkt_size == 0)
1354                         dev->data->dev_conf.rxmode.max_lro_pkt_size =
1355                                 dev->data->dev_conf.rxmode.max_rx_pkt_len;
1356                 ret = check_lro_pkt_size(port_id,
1357                                 dev->data->dev_conf.rxmode.max_lro_pkt_size,
1358                                 dev->data->dev_conf.rxmode.max_rx_pkt_len,
1359                                 dev_info.max_lro_pkt_size);
1360                 if (ret != 0)
1361                         goto rollback;
1362         }
1363
1364         /* Any requested offloading must be within its device capabilities */
1365         if ((dev_conf->rxmode.offloads & dev_info.rx_offload_capa) !=
1366              dev_conf->rxmode.offloads) {
1367                 RTE_ETHDEV_LOG(ERR,
1368                         "Ethdev port_id=%u requested Rx offloads 0x%"PRIx64" doesn't match Rx offloads "
1369                         "capabilities 0x%"PRIx64" in %s()\n",
1370                         port_id, dev_conf->rxmode.offloads,
1371                         dev_info.rx_offload_capa,
1372                         __func__);
1373                 ret = -EINVAL;
1374                 goto rollback;
1375         }
1376         if ((dev_conf->txmode.offloads & dev_info.tx_offload_capa) !=
1377              dev_conf->txmode.offloads) {
1378                 RTE_ETHDEV_LOG(ERR,
1379                         "Ethdev port_id=%u requested Tx offloads 0x%"PRIx64" doesn't match Tx offloads "
1380                         "capabilities 0x%"PRIx64" in %s()\n",
1381                         port_id, dev_conf->txmode.offloads,
1382                         dev_info.tx_offload_capa,
1383                         __func__);
1384                 ret = -EINVAL;
1385                 goto rollback;
1386         }
1387
1388         dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf =
1389                 rte_eth_rss_hf_refine(dev_conf->rx_adv_conf.rss_conf.rss_hf);
1390
1391         /* Check that device supports requested rss hash functions. */
1392         if ((dev_info.flow_type_rss_offloads |
1393              dev_conf->rx_adv_conf.rss_conf.rss_hf) !=
1394             dev_info.flow_type_rss_offloads) {
1395                 RTE_ETHDEV_LOG(ERR,
1396                         "Ethdev port_id=%u invalid rss_hf: 0x%"PRIx64", valid value: 0x%"PRIx64"\n",
1397                         port_id, dev_conf->rx_adv_conf.rss_conf.rss_hf,
1398                         dev_info.flow_type_rss_offloads);
1399                 ret = -EINVAL;
1400                 goto rollback;
1401         }
1402
1403         /* Check if Rx RSS distribution is disabled but RSS hash is enabled. */
1404         if (((dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) == 0) &&
1405             (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_RSS_HASH)) {
1406                 RTE_ETHDEV_LOG(ERR,
1407                         "Ethdev port_id=%u config invalid Rx mq_mode without RSS but %s offload is requested\n",
1408                         port_id,
1409                         rte_eth_dev_rx_offload_name(DEV_RX_OFFLOAD_RSS_HASH));
1410                 ret = -EINVAL;
1411                 goto rollback;
1412         }
1413
1414         /*
1415          * Setup new number of RX/TX queues and reconfigure device.
1416          */
1417         diag = rte_eth_dev_rx_queue_config(dev, nb_rx_q);
1418         if (diag != 0) {
1419                 RTE_ETHDEV_LOG(ERR,
1420                         "Port%u rte_eth_dev_rx_queue_config = %d\n",
1421                         port_id, diag);
1422                 ret = diag;
1423                 goto rollback;
1424         }
1425
1426         diag = rte_eth_dev_tx_queue_config(dev, nb_tx_q);
1427         if (diag != 0) {
1428                 RTE_ETHDEV_LOG(ERR,
1429                         "Port%u rte_eth_dev_tx_queue_config = %d\n",
1430                         port_id, diag);
1431                 rte_eth_dev_rx_queue_config(dev, 0);
1432                 ret = diag;
1433                 goto rollback;
1434         }
1435
1436         diag = (*dev->dev_ops->dev_configure)(dev);
1437         if (diag != 0) {
1438                 RTE_ETHDEV_LOG(ERR, "Port%u dev_configure = %d\n",
1439                         port_id, diag);
1440                 ret = eth_err(port_id, diag);
1441                 goto reset_queues;
1442         }
1443
1444         /* Initialize Rx profiling if enabled at compilation time. */
1445         diag = __rte_eth_dev_profile_init(port_id, dev);
1446         if (diag != 0) {
1447                 RTE_ETHDEV_LOG(ERR, "Port%u __rte_eth_dev_profile_init = %d\n",
1448                         port_id, diag);
1449                 ret = eth_err(port_id, diag);
1450                 goto reset_queues;
1451         }
1452
1453         /* Validate Rx offloads. */
1454         diag = validate_offloads(port_id,
1455                         dev_conf->rxmode.offloads,
1456                         dev->data->dev_conf.rxmode.offloads, "Rx",
1457                         rte_eth_dev_rx_offload_name);
1458         if (diag != 0) {
1459                 ret = diag;
1460                 goto reset_queues;
1461         }
1462
1463         /* Validate Tx offloads. */
1464         diag = validate_offloads(port_id,
1465                         dev_conf->txmode.offloads,
1466                         dev->data->dev_conf.txmode.offloads, "Tx",
1467                         rte_eth_dev_tx_offload_name);
1468         if (diag != 0) {
1469                 ret = diag;
1470                 goto reset_queues;
1471         }
1472
1473         return 0;
1474 reset_queues:
1475         rte_eth_dev_rx_queue_config(dev, 0);
1476         rte_eth_dev_tx_queue_config(dev, 0);
1477 rollback:
1478         memcpy(&dev->data->dev_conf, &orig_conf, sizeof(dev->data->dev_conf));
1479
1480         return ret;
1481 }
1482
1483 void
1484 _rte_eth_dev_reset(struct rte_eth_dev *dev)
1485 {
1486         if (dev->data->dev_started) {
1487                 RTE_ETHDEV_LOG(ERR, "Port %u must be stopped to allow reset\n",
1488                         dev->data->port_id);
1489                 return;
1490         }
1491
1492         rte_eth_dev_rx_queue_config(dev, 0);
1493         rte_eth_dev_tx_queue_config(dev, 0);
1494
1495         memset(&dev->data->dev_conf, 0, sizeof(dev->data->dev_conf));
1496 }
1497
1498 static void
1499 rte_eth_dev_mac_restore(struct rte_eth_dev *dev,
1500                         struct rte_eth_dev_info *dev_info)
1501 {
1502         struct rte_ether_addr *addr;
1503         uint16_t i;
1504         uint32_t pool = 0;
1505         uint64_t pool_mask;
1506
1507         /* replay MAC address configuration including default MAC */
1508         addr = &dev->data->mac_addrs[0];
1509         if (*dev->dev_ops->mac_addr_set != NULL)
1510                 (*dev->dev_ops->mac_addr_set)(dev, addr);
1511         else if (*dev->dev_ops->mac_addr_add != NULL)
1512                 (*dev->dev_ops->mac_addr_add)(dev, addr, 0, pool);
1513
1514         if (*dev->dev_ops->mac_addr_add != NULL) {
1515                 for (i = 1; i < dev_info->max_mac_addrs; i++) {
1516                         addr = &dev->data->mac_addrs[i];
1517
1518                         /* skip zero address */
1519                         if (rte_is_zero_ether_addr(addr))
1520                                 continue;
1521
1522                         pool = 0;
1523                         pool_mask = dev->data->mac_pool_sel[i];
1524
1525                         do {
1526                                 if (pool_mask & 1ULL)
1527                                         (*dev->dev_ops->mac_addr_add)(dev,
1528                                                 addr, i, pool);
1529                                 pool_mask >>= 1;
1530                                 pool++;
1531                         } while (pool_mask);
1532                 }
1533         }
1534 }
1535
1536 static int
1537 rte_eth_dev_config_restore(struct rte_eth_dev *dev,
1538                            struct rte_eth_dev_info *dev_info, uint16_t port_id)
1539 {
1540         int ret;
1541
1542         if (!(*dev_info->dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR))
1543                 rte_eth_dev_mac_restore(dev, dev_info);
1544
1545         /* replay promiscuous configuration */
1546         /*
1547          * use callbacks directly since we don't need port_id check and
1548          * would like to bypass the same value set
1549          */
1550         if (rte_eth_promiscuous_get(port_id) == 1 &&
1551             *dev->dev_ops->promiscuous_enable != NULL) {
1552                 ret = eth_err(port_id,
1553                               (*dev->dev_ops->promiscuous_enable)(dev));
1554                 if (ret != 0 && ret != -ENOTSUP) {
1555                         RTE_ETHDEV_LOG(ERR,
1556                                 "Failed to enable promiscuous mode for device (port %u): %s\n",
1557                                 port_id, rte_strerror(-ret));
1558                         return ret;
1559                 }
1560         } else if (rte_eth_promiscuous_get(port_id) == 0 &&
1561                    *dev->dev_ops->promiscuous_disable != NULL) {
1562                 ret = eth_err(port_id,
1563                               (*dev->dev_ops->promiscuous_disable)(dev));
1564                 if (ret != 0 && ret != -ENOTSUP) {
1565                         RTE_ETHDEV_LOG(ERR,
1566                                 "Failed to disable promiscuous mode for device (port %u): %s\n",
1567                                 port_id, rte_strerror(-ret));
1568                         return ret;
1569                 }
1570         }
1571
1572         /* replay all multicast configuration */
1573         /*
1574          * use callbacks directly since we don't need port_id check and
1575          * would like to bypass the same value set
1576          */
1577         if (rte_eth_allmulticast_get(port_id) == 1 &&
1578             *dev->dev_ops->allmulticast_enable != NULL) {
1579                 ret = eth_err(port_id,
1580                               (*dev->dev_ops->allmulticast_enable)(dev));
1581                 if (ret != 0 && ret != -ENOTSUP) {
1582                         RTE_ETHDEV_LOG(ERR,
1583                                 "Failed to enable allmulticast mode for device (port %u): %s\n",
1584                                 port_id, rte_strerror(-ret));
1585                         return ret;
1586                 }
1587         } else if (rte_eth_allmulticast_get(port_id) == 0 &&
1588                    *dev->dev_ops->allmulticast_disable != NULL) {
1589                 ret = eth_err(port_id,
1590                               (*dev->dev_ops->allmulticast_disable)(dev));
1591                 if (ret != 0 && ret != -ENOTSUP) {
1592                         RTE_ETHDEV_LOG(ERR,
1593                                 "Failed to disable allmulticast mode for device (port %u): %s\n",
1594                                 port_id, rte_strerror(-ret));
1595                         return ret;
1596                 }
1597         }
1598
1599         return 0;
1600 }
1601
1602 int
1603 rte_eth_dev_start(uint16_t port_id)
1604 {
1605         struct rte_eth_dev *dev;
1606         struct rte_eth_dev_info dev_info;
1607         int diag;
1608         int ret;
1609
1610         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1611
1612         dev = &rte_eth_devices[port_id];
1613
1614         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_start, -ENOTSUP);
1615
1616         if (dev->data->dev_started != 0) {
1617                 RTE_ETHDEV_LOG(INFO,
1618                         "Device with port_id=%"PRIu16" already started\n",
1619                         port_id);
1620                 return 0;
1621         }
1622
1623         ret = rte_eth_dev_info_get(port_id, &dev_info);
1624         if (ret != 0)
1625                 return ret;
1626
1627         /* Lets restore MAC now if device does not support live change */
1628         if (*dev_info.dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR)
1629                 rte_eth_dev_mac_restore(dev, &dev_info);
1630
1631         diag = (*dev->dev_ops->dev_start)(dev);
1632         if (diag == 0)
1633                 dev->data->dev_started = 1;
1634         else
1635                 return eth_err(port_id, diag);
1636
1637         ret = rte_eth_dev_config_restore(dev, &dev_info, port_id);
1638         if (ret != 0) {
1639                 RTE_ETHDEV_LOG(ERR,
1640                         "Error during restoring configuration for device (port %u): %s\n",
1641                         port_id, rte_strerror(-ret));
1642                 rte_eth_dev_stop(port_id);
1643                 return ret;
1644         }
1645
1646         if (dev->data->dev_conf.intr_conf.lsc == 0) {
1647                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
1648                 (*dev->dev_ops->link_update)(dev, 0);
1649         }
1650         return 0;
1651 }
1652
1653 void
1654 rte_eth_dev_stop(uint16_t port_id)
1655 {
1656         struct rte_eth_dev *dev;
1657
1658         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1659         dev = &rte_eth_devices[port_id];
1660
1661         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_stop);
1662
1663         if (dev->data->dev_started == 0) {
1664                 RTE_ETHDEV_LOG(INFO,
1665                         "Device with port_id=%"PRIu16" already stopped\n",
1666                         port_id);
1667                 return;
1668         }
1669
1670         dev->data->dev_started = 0;
1671         (*dev->dev_ops->dev_stop)(dev);
1672 }
1673
1674 int
1675 rte_eth_dev_set_link_up(uint16_t port_id)
1676 {
1677         struct rte_eth_dev *dev;
1678
1679         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1680
1681         dev = &rte_eth_devices[port_id];
1682
1683         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_up, -ENOTSUP);
1684         return eth_err(port_id, (*dev->dev_ops->dev_set_link_up)(dev));
1685 }
1686
1687 int
1688 rte_eth_dev_set_link_down(uint16_t port_id)
1689 {
1690         struct rte_eth_dev *dev;
1691
1692         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1693
1694         dev = &rte_eth_devices[port_id];
1695
1696         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_down, -ENOTSUP);
1697         return eth_err(port_id, (*dev->dev_ops->dev_set_link_down)(dev));
1698 }
1699
1700 void
1701 rte_eth_dev_close(uint16_t port_id)
1702 {
1703         struct rte_eth_dev *dev;
1704
1705         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1706         dev = &rte_eth_devices[port_id];
1707
1708         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_close);
1709         dev->data->dev_started = 0;
1710         (*dev->dev_ops->dev_close)(dev);
1711
1712         /* check behaviour flag - temporary for PMD migration */
1713         if ((dev->data->dev_flags & RTE_ETH_DEV_CLOSE_REMOVE) != 0) {
1714                 /* new behaviour: send event + reset state + free all data */
1715                 rte_eth_dev_release_port(dev);
1716                 return;
1717         }
1718         RTE_ETHDEV_LOG(DEBUG, "Port closing is using an old behaviour.\n"
1719                         "The driver %s should migrate to the new behaviour.\n",
1720                         dev->device->driver->name);
1721         /* old behaviour: only free queue arrays */
1722         dev->data->nb_rx_queues = 0;
1723         rte_free(dev->data->rx_queues);
1724         dev->data->rx_queues = NULL;
1725         dev->data->nb_tx_queues = 0;
1726         rte_free(dev->data->tx_queues);
1727         dev->data->tx_queues = NULL;
1728 }
1729
1730 int
1731 rte_eth_dev_reset(uint16_t port_id)
1732 {
1733         struct rte_eth_dev *dev;
1734         int ret;
1735
1736         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1737         dev = &rte_eth_devices[port_id];
1738
1739         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_reset, -ENOTSUP);
1740
1741         rte_eth_dev_stop(port_id);
1742         ret = dev->dev_ops->dev_reset(dev);
1743
1744         return eth_err(port_id, ret);
1745 }
1746
1747 int
1748 rte_eth_dev_is_removed(uint16_t port_id)
1749 {
1750         struct rte_eth_dev *dev;
1751         int ret;
1752
1753         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, 0);
1754
1755         dev = &rte_eth_devices[port_id];
1756
1757         if (dev->state == RTE_ETH_DEV_REMOVED)
1758                 return 1;
1759
1760         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->is_removed, 0);
1761
1762         ret = dev->dev_ops->is_removed(dev);
1763         if (ret != 0)
1764                 /* Device is physically removed. */
1765                 dev->state = RTE_ETH_DEV_REMOVED;
1766
1767         return ret;
1768 }
1769
1770 int
1771 rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
1772                        uint16_t nb_rx_desc, unsigned int socket_id,
1773                        const struct rte_eth_rxconf *rx_conf,
1774                        struct rte_mempool *mp)
1775 {
1776         int ret;
1777         uint32_t mbp_buf_size;
1778         struct rte_eth_dev *dev;
1779         struct rte_eth_dev_info dev_info;
1780         struct rte_eth_rxconf local_conf;
1781         void **rxq;
1782
1783         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1784
1785         dev = &rte_eth_devices[port_id];
1786         if (rx_queue_id >= dev->data->nb_rx_queues) {
1787                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
1788                 return -EINVAL;
1789         }
1790
1791         if (mp == NULL) {
1792                 RTE_ETHDEV_LOG(ERR, "Invalid null mempool pointer\n");
1793                 return -EINVAL;
1794         }
1795
1796         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_setup, -ENOTSUP);
1797
1798         /*
1799          * Check the size of the mbuf data buffer.
1800          * This value must be provided in the private data of the memory pool.
1801          * First check that the memory pool has a valid private data.
1802          */
1803         ret = rte_eth_dev_info_get(port_id, &dev_info);
1804         if (ret != 0)
1805                 return ret;
1806
1807         if (mp->private_data_size < sizeof(struct rte_pktmbuf_pool_private)) {
1808                 RTE_ETHDEV_LOG(ERR, "%s private_data_size %d < %d\n",
1809                         mp->name, (int)mp->private_data_size,
1810                         (int)sizeof(struct rte_pktmbuf_pool_private));
1811                 return -ENOSPC;
1812         }
1813         mbp_buf_size = rte_pktmbuf_data_room_size(mp);
1814
1815         if ((mbp_buf_size - RTE_PKTMBUF_HEADROOM) < dev_info.min_rx_bufsize) {
1816                 RTE_ETHDEV_LOG(ERR,
1817                         "%s mbuf_data_room_size %d < %d (RTE_PKTMBUF_HEADROOM=%d + min_rx_bufsize(dev)=%d)\n",
1818                         mp->name, (int)mbp_buf_size,
1819                         (int)(RTE_PKTMBUF_HEADROOM + dev_info.min_rx_bufsize),
1820                         (int)RTE_PKTMBUF_HEADROOM,
1821                         (int)dev_info.min_rx_bufsize);
1822                 return -EINVAL;
1823         }
1824
1825         /* Use default specified by driver, if nb_rx_desc is zero */
1826         if (nb_rx_desc == 0) {
1827                 nb_rx_desc = dev_info.default_rxportconf.ring_size;
1828                 /* If driver default is also zero, fall back on EAL default */
1829                 if (nb_rx_desc == 0)
1830                         nb_rx_desc = RTE_ETH_DEV_FALLBACK_RX_RINGSIZE;
1831         }
1832
1833         if (nb_rx_desc > dev_info.rx_desc_lim.nb_max ||
1834                         nb_rx_desc < dev_info.rx_desc_lim.nb_min ||
1835                         nb_rx_desc % dev_info.rx_desc_lim.nb_align != 0) {
1836
1837                 RTE_ETHDEV_LOG(ERR,
1838                         "Invalid value for nb_rx_desc(=%hu), should be: <= %hu, >= %hu, and a product of %hu\n",
1839                         nb_rx_desc, dev_info.rx_desc_lim.nb_max,
1840                         dev_info.rx_desc_lim.nb_min,
1841                         dev_info.rx_desc_lim.nb_align);
1842                 return -EINVAL;
1843         }
1844
1845         if (dev->data->dev_started &&
1846                 !(dev_info.dev_capa &
1847                         RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP))
1848                 return -EBUSY;
1849
1850         if (dev->data->dev_started &&
1851                 (dev->data->rx_queue_state[rx_queue_id] !=
1852                         RTE_ETH_QUEUE_STATE_STOPPED))
1853                 return -EBUSY;
1854
1855         rxq = dev->data->rx_queues;
1856         if (rxq[rx_queue_id]) {
1857                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release,
1858                                         -ENOTSUP);
1859                 (*dev->dev_ops->rx_queue_release)(rxq[rx_queue_id]);
1860                 rxq[rx_queue_id] = NULL;
1861         }
1862
1863         if (rx_conf == NULL)
1864                 rx_conf = &dev_info.default_rxconf;
1865
1866         local_conf = *rx_conf;
1867
1868         /*
1869          * If an offloading has already been enabled in
1870          * rte_eth_dev_configure(), it has been enabled on all queues,
1871          * so there is no need to enable it in this queue again.
1872          * The local_conf.offloads input to underlying PMD only carries
1873          * those offloadings which are only enabled on this queue and
1874          * not enabled on all queues.
1875          */
1876         local_conf.offloads &= ~dev->data->dev_conf.rxmode.offloads;
1877
1878         /*
1879          * New added offloadings for this queue are those not enabled in
1880          * rte_eth_dev_configure() and they must be per-queue type.
1881          * A pure per-port offloading can't be enabled on a queue while
1882          * disabled on another queue. A pure per-port offloading can't
1883          * be enabled for any queue as new added one if it hasn't been
1884          * enabled in rte_eth_dev_configure().
1885          */
1886         if ((local_conf.offloads & dev_info.rx_queue_offload_capa) !=
1887              local_conf.offloads) {
1888                 RTE_ETHDEV_LOG(ERR,
1889                         "Ethdev port_id=%d rx_queue_id=%d, new added offloads 0x%"PRIx64" must be "
1890                         "within per-queue offload capabilities 0x%"PRIx64" in %s()\n",
1891                         port_id, rx_queue_id, local_conf.offloads,
1892                         dev_info.rx_queue_offload_capa,
1893                         __func__);
1894                 return -EINVAL;
1895         }
1896
1897         /*
1898          * If LRO is enabled, check that the maximum aggregated packet
1899          * size is supported by the configured device.
1900          */
1901         if (local_conf.offloads & DEV_RX_OFFLOAD_TCP_LRO) {
1902                 if (dev->data->dev_conf.rxmode.max_lro_pkt_size == 0)
1903                         dev->data->dev_conf.rxmode.max_lro_pkt_size =
1904                                 dev->data->dev_conf.rxmode.max_rx_pkt_len;
1905                 int ret = check_lro_pkt_size(port_id,
1906                                 dev->data->dev_conf.rxmode.max_lro_pkt_size,
1907                                 dev->data->dev_conf.rxmode.max_rx_pkt_len,
1908                                 dev_info.max_lro_pkt_size);
1909                 if (ret != 0)
1910                         return ret;
1911         }
1912
1913         ret = (*dev->dev_ops->rx_queue_setup)(dev, rx_queue_id, nb_rx_desc,
1914                                               socket_id, &local_conf, mp);
1915         if (!ret) {
1916                 if (!dev->data->min_rx_buf_size ||
1917                     dev->data->min_rx_buf_size > mbp_buf_size)
1918                         dev->data->min_rx_buf_size = mbp_buf_size;
1919         }
1920
1921         return eth_err(port_id, ret);
1922 }
1923
1924 int
1925 rte_eth_rx_hairpin_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
1926                                uint16_t nb_rx_desc,
1927                                const struct rte_eth_hairpin_conf *conf)
1928 {
1929         int ret;
1930         struct rte_eth_dev *dev;
1931         struct rte_eth_hairpin_cap cap;
1932         void **rxq;
1933         int i;
1934         int count;
1935
1936         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1937
1938         dev = &rte_eth_devices[port_id];
1939         if (rx_queue_id >= dev->data->nb_rx_queues) {
1940                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
1941                 return -EINVAL;
1942         }
1943         ret = rte_eth_dev_hairpin_capability_get(port_id, &cap);
1944         if (ret != 0)
1945                 return ret;
1946         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_hairpin_queue_setup,
1947                                 -ENOTSUP);
1948         /* if nb_rx_desc is zero use max number of desc from the driver. */
1949         if (nb_rx_desc == 0)
1950                 nb_rx_desc = cap.max_nb_desc;
1951         if (nb_rx_desc > cap.max_nb_desc) {
1952                 RTE_ETHDEV_LOG(ERR,
1953                         "Invalid value for nb_rx_desc(=%hu), should be: <= %hu",
1954                         nb_rx_desc, cap.max_nb_desc);
1955                 return -EINVAL;
1956         }
1957         if (conf->peer_count > cap.max_rx_2_tx) {
1958                 RTE_ETHDEV_LOG(ERR,
1959                         "Invalid value for number of peers for Rx queue(=%hu), should be: <= %hu",
1960                         conf->peer_count, cap.max_rx_2_tx);
1961                 return -EINVAL;
1962         }
1963         if (conf->peer_count == 0) {
1964                 RTE_ETHDEV_LOG(ERR,
1965                         "Invalid value for number of peers for Rx queue(=%hu), should be: > 0",
1966                         conf->peer_count);
1967                 return -EINVAL;
1968         }
1969         for (i = 0, count = 0; i < dev->data->nb_rx_queues &&
1970              cap.max_nb_queues != UINT16_MAX; i++) {
1971                 if (i == rx_queue_id || rte_eth_dev_is_rx_hairpin_queue(dev, i))
1972                         count++;
1973         }
1974         if (count > cap.max_nb_queues) {
1975                 RTE_ETHDEV_LOG(ERR, "To many Rx hairpin queues max is %d",
1976                 cap.max_nb_queues);
1977                 return -EINVAL;
1978         }
1979         if (dev->data->dev_started)
1980                 return -EBUSY;
1981         rxq = dev->data->rx_queues;
1982         if (rxq[rx_queue_id] != NULL) {
1983                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release,
1984                                         -ENOTSUP);
1985                 (*dev->dev_ops->rx_queue_release)(rxq[rx_queue_id]);
1986                 rxq[rx_queue_id] = NULL;
1987         }
1988         ret = (*dev->dev_ops->rx_hairpin_queue_setup)(dev, rx_queue_id,
1989                                                       nb_rx_desc, conf);
1990         if (ret == 0)
1991                 dev->data->rx_queue_state[rx_queue_id] =
1992                         RTE_ETH_QUEUE_STATE_HAIRPIN;
1993         return eth_err(port_id, ret);
1994 }
1995
1996 int
1997 rte_eth_tx_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
1998                        uint16_t nb_tx_desc, unsigned int socket_id,
1999                        const struct rte_eth_txconf *tx_conf)
2000 {
2001         struct rte_eth_dev *dev;
2002         struct rte_eth_dev_info dev_info;
2003         struct rte_eth_txconf local_conf;
2004         void **txq;
2005         int ret;
2006
2007         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2008
2009         dev = &rte_eth_devices[port_id];
2010         if (tx_queue_id >= dev->data->nb_tx_queues) {
2011                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
2012                 return -EINVAL;
2013         }
2014
2015         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_setup, -ENOTSUP);
2016
2017         ret = rte_eth_dev_info_get(port_id, &dev_info);
2018         if (ret != 0)
2019                 return ret;
2020
2021         /* Use default specified by driver, if nb_tx_desc is zero */
2022         if (nb_tx_desc == 0) {
2023                 nb_tx_desc = dev_info.default_txportconf.ring_size;
2024                 /* If driver default is zero, fall back on EAL default */
2025                 if (nb_tx_desc == 0)
2026                         nb_tx_desc = RTE_ETH_DEV_FALLBACK_TX_RINGSIZE;
2027         }
2028         if (nb_tx_desc > dev_info.tx_desc_lim.nb_max ||
2029             nb_tx_desc < dev_info.tx_desc_lim.nb_min ||
2030             nb_tx_desc % dev_info.tx_desc_lim.nb_align != 0) {
2031                 RTE_ETHDEV_LOG(ERR,
2032                         "Invalid value for nb_tx_desc(=%hu), should be: <= %hu, >= %hu, and a product of %hu\n",
2033                         nb_tx_desc, dev_info.tx_desc_lim.nb_max,
2034                         dev_info.tx_desc_lim.nb_min,
2035                         dev_info.tx_desc_lim.nb_align);
2036                 return -EINVAL;
2037         }
2038
2039         if (dev->data->dev_started &&
2040                 !(dev_info.dev_capa &
2041                         RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP))
2042                 return -EBUSY;
2043
2044         if (dev->data->dev_started &&
2045                 (dev->data->tx_queue_state[tx_queue_id] !=
2046                         RTE_ETH_QUEUE_STATE_STOPPED))
2047                 return -EBUSY;
2048
2049         txq = dev->data->tx_queues;
2050         if (txq[tx_queue_id]) {
2051                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release,
2052                                         -ENOTSUP);
2053                 (*dev->dev_ops->tx_queue_release)(txq[tx_queue_id]);
2054                 txq[tx_queue_id] = NULL;
2055         }
2056
2057         if (tx_conf == NULL)
2058                 tx_conf = &dev_info.default_txconf;
2059
2060         local_conf = *tx_conf;
2061
2062         /*
2063          * If an offloading has already been enabled in
2064          * rte_eth_dev_configure(), it has been enabled on all queues,
2065          * so there is no need to enable it in this queue again.
2066          * The local_conf.offloads input to underlying PMD only carries
2067          * those offloadings which are only enabled on this queue and
2068          * not enabled on all queues.
2069          */
2070         local_conf.offloads &= ~dev->data->dev_conf.txmode.offloads;
2071
2072         /*
2073          * New added offloadings for this queue are those not enabled in
2074          * rte_eth_dev_configure() and they must be per-queue type.
2075          * A pure per-port offloading can't be enabled on a queue while
2076          * disabled on another queue. A pure per-port offloading can't
2077          * be enabled for any queue as new added one if it hasn't been
2078          * enabled in rte_eth_dev_configure().
2079          */
2080         if ((local_conf.offloads & dev_info.tx_queue_offload_capa) !=
2081              local_conf.offloads) {
2082                 RTE_ETHDEV_LOG(ERR,
2083                         "Ethdev port_id=%d tx_queue_id=%d, new added offloads 0x%"PRIx64" must be "
2084                         "within per-queue offload capabilities 0x%"PRIx64" in %s()\n",
2085                         port_id, tx_queue_id, local_conf.offloads,
2086                         dev_info.tx_queue_offload_capa,
2087                         __func__);
2088                 return -EINVAL;
2089         }
2090
2091         return eth_err(port_id, (*dev->dev_ops->tx_queue_setup)(dev,
2092                        tx_queue_id, nb_tx_desc, socket_id, &local_conf));
2093 }
2094
2095 int
2096 rte_eth_tx_hairpin_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
2097                                uint16_t nb_tx_desc,
2098                                const struct rte_eth_hairpin_conf *conf)
2099 {
2100         struct rte_eth_dev *dev;
2101         struct rte_eth_hairpin_cap cap;
2102         void **txq;
2103         int i;
2104         int count;
2105         int ret;
2106
2107         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2108         dev = &rte_eth_devices[port_id];
2109         if (tx_queue_id >= dev->data->nb_tx_queues) {
2110                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
2111                 return -EINVAL;
2112         }
2113         ret = rte_eth_dev_hairpin_capability_get(port_id, &cap);
2114         if (ret != 0)
2115                 return ret;
2116         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_hairpin_queue_setup,
2117                                 -ENOTSUP);
2118         /* if nb_rx_desc is zero use max number of desc from the driver. */
2119         if (nb_tx_desc == 0)
2120                 nb_tx_desc = cap.max_nb_desc;
2121         if (nb_tx_desc > cap.max_nb_desc) {
2122                 RTE_ETHDEV_LOG(ERR,
2123                         "Invalid value for nb_tx_desc(=%hu), should be: <= %hu",
2124                         nb_tx_desc, cap.max_nb_desc);
2125                 return -EINVAL;
2126         }
2127         if (conf->peer_count > cap.max_tx_2_rx) {
2128                 RTE_ETHDEV_LOG(ERR,
2129                         "Invalid value for number of peers for Tx queue(=%hu), should be: <= %hu",
2130                         conf->peer_count, cap.max_tx_2_rx);
2131                 return -EINVAL;
2132         }
2133         if (conf->peer_count == 0) {
2134                 RTE_ETHDEV_LOG(ERR,
2135                         "Invalid value for number of peers for Tx queue(=%hu), should be: > 0",
2136                         conf->peer_count);
2137                 return -EINVAL;
2138         }
2139         for (i = 0, count = 0; i < dev->data->nb_tx_queues &&
2140              cap.max_nb_queues != UINT16_MAX; i++) {
2141                 if (i == tx_queue_id || rte_eth_dev_is_tx_hairpin_queue(dev, i))
2142                         count++;
2143         }
2144         if (count > cap.max_nb_queues) {
2145                 RTE_ETHDEV_LOG(ERR, "To many Tx hairpin queues max is %d",
2146                 cap.max_nb_queues);
2147                 return -EINVAL;
2148         }
2149         if (dev->data->dev_started)
2150                 return -EBUSY;
2151         txq = dev->data->tx_queues;
2152         if (txq[tx_queue_id] != NULL) {
2153                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release,
2154                                         -ENOTSUP);
2155                 (*dev->dev_ops->tx_queue_release)(txq[tx_queue_id]);
2156                 txq[tx_queue_id] = NULL;
2157         }
2158         ret = (*dev->dev_ops->tx_hairpin_queue_setup)
2159                 (dev, tx_queue_id, nb_tx_desc, conf);
2160         if (ret == 0)
2161                 dev->data->tx_queue_state[tx_queue_id] =
2162                         RTE_ETH_QUEUE_STATE_HAIRPIN;
2163         return eth_err(port_id, ret);
2164 }
2165
2166 void
2167 rte_eth_tx_buffer_drop_callback(struct rte_mbuf **pkts, uint16_t unsent,
2168                 void *userdata __rte_unused)
2169 {
2170         unsigned i;
2171
2172         for (i = 0; i < unsent; i++)
2173                 rte_pktmbuf_free(pkts[i]);
2174 }
2175
2176 void
2177 rte_eth_tx_buffer_count_callback(struct rte_mbuf **pkts, uint16_t unsent,
2178                 void *userdata)
2179 {
2180         uint64_t *count = userdata;
2181         unsigned i;
2182
2183         for (i = 0; i < unsent; i++)
2184                 rte_pktmbuf_free(pkts[i]);
2185
2186         *count += unsent;
2187 }
2188
2189 int
2190 rte_eth_tx_buffer_set_err_callback(struct rte_eth_dev_tx_buffer *buffer,
2191                 buffer_tx_error_fn cbfn, void *userdata)
2192 {
2193         buffer->error_callback = cbfn;
2194         buffer->error_userdata = userdata;
2195         return 0;
2196 }
2197
2198 int
2199 rte_eth_tx_buffer_init(struct rte_eth_dev_tx_buffer *buffer, uint16_t size)
2200 {
2201         int ret = 0;
2202
2203         if (buffer == NULL)
2204                 return -EINVAL;
2205
2206         buffer->size = size;
2207         if (buffer->error_callback == NULL) {
2208                 ret = rte_eth_tx_buffer_set_err_callback(
2209                         buffer, rte_eth_tx_buffer_drop_callback, NULL);
2210         }
2211
2212         return ret;
2213 }
2214
2215 int
2216 rte_eth_tx_done_cleanup(uint16_t port_id, uint16_t queue_id, uint32_t free_cnt)
2217 {
2218         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
2219         int ret;
2220
2221         /* Validate Input Data. Bail if not valid or not supported. */
2222         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2223         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_done_cleanup, -ENOTSUP);
2224
2225         /* Call driver to free pending mbufs. */
2226         ret = (*dev->dev_ops->tx_done_cleanup)(dev->data->tx_queues[queue_id],
2227                                                free_cnt);
2228         return eth_err(port_id, ret);
2229 }
2230
2231 int
2232 rte_eth_promiscuous_enable(uint16_t port_id)
2233 {
2234         struct rte_eth_dev *dev;
2235         int diag = 0;
2236
2237         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2238         dev = &rte_eth_devices[port_id];
2239
2240         if (dev->data->promiscuous == 1)
2241                 return 0;
2242
2243         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->promiscuous_enable, -ENOTSUP);
2244
2245         diag = (*dev->dev_ops->promiscuous_enable)(dev);
2246         dev->data->promiscuous = (diag == 0) ? 1 : 0;
2247
2248         return eth_err(port_id, diag);
2249 }
2250
2251 int
2252 rte_eth_promiscuous_disable(uint16_t port_id)
2253 {
2254         struct rte_eth_dev *dev;
2255         int diag = 0;
2256
2257         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2258         dev = &rte_eth_devices[port_id];
2259
2260         if (dev->data->promiscuous == 0)
2261                 return 0;
2262
2263         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->promiscuous_disable, -ENOTSUP);
2264
2265         dev->data->promiscuous = 0;
2266         diag = (*dev->dev_ops->promiscuous_disable)(dev);
2267         if (diag != 0)
2268                 dev->data->promiscuous = 1;
2269
2270         return eth_err(port_id, diag);
2271 }
2272
2273 int
2274 rte_eth_promiscuous_get(uint16_t port_id)
2275 {
2276         struct rte_eth_dev *dev;
2277
2278         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2279
2280         dev = &rte_eth_devices[port_id];
2281         return dev->data->promiscuous;
2282 }
2283
2284 int
2285 rte_eth_allmulticast_enable(uint16_t port_id)
2286 {
2287         struct rte_eth_dev *dev;
2288         int diag;
2289
2290         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2291         dev = &rte_eth_devices[port_id];
2292
2293         if (dev->data->all_multicast == 1)
2294                 return 0;
2295
2296         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->allmulticast_enable, -ENOTSUP);
2297         diag = (*dev->dev_ops->allmulticast_enable)(dev);
2298         dev->data->all_multicast = (diag == 0) ? 1 : 0;
2299
2300         return eth_err(port_id, diag);
2301 }
2302
2303 int
2304 rte_eth_allmulticast_disable(uint16_t port_id)
2305 {
2306         struct rte_eth_dev *dev;
2307         int diag;
2308
2309         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2310         dev = &rte_eth_devices[port_id];
2311
2312         if (dev->data->all_multicast == 0)
2313                 return 0;
2314
2315         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->allmulticast_disable, -ENOTSUP);
2316         dev->data->all_multicast = 0;
2317         diag = (*dev->dev_ops->allmulticast_disable)(dev);
2318         if (diag != 0)
2319                 dev->data->all_multicast = 1;
2320
2321         return eth_err(port_id, diag);
2322 }
2323
2324 int
2325 rte_eth_allmulticast_get(uint16_t port_id)
2326 {
2327         struct rte_eth_dev *dev;
2328
2329         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2330
2331         dev = &rte_eth_devices[port_id];
2332         return dev->data->all_multicast;
2333 }
2334
2335 int
2336 rte_eth_link_get(uint16_t port_id, struct rte_eth_link *eth_link)
2337 {
2338         struct rte_eth_dev *dev;
2339
2340         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2341         dev = &rte_eth_devices[port_id];
2342
2343         if (dev->data->dev_conf.intr_conf.lsc &&
2344             dev->data->dev_started)
2345                 rte_eth_linkstatus_get(dev, eth_link);
2346         else {
2347                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
2348                 (*dev->dev_ops->link_update)(dev, 1);
2349                 *eth_link = dev->data->dev_link;
2350         }
2351
2352         return 0;
2353 }
2354
2355 int
2356 rte_eth_link_get_nowait(uint16_t port_id, struct rte_eth_link *eth_link)
2357 {
2358         struct rte_eth_dev *dev;
2359
2360         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2361         dev = &rte_eth_devices[port_id];
2362
2363         if (dev->data->dev_conf.intr_conf.lsc &&
2364             dev->data->dev_started)
2365                 rte_eth_linkstatus_get(dev, eth_link);
2366         else {
2367                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
2368                 (*dev->dev_ops->link_update)(dev, 0);
2369                 *eth_link = dev->data->dev_link;
2370         }
2371
2372         return 0;
2373 }
2374
2375 int
2376 rte_eth_stats_get(uint16_t port_id, struct rte_eth_stats *stats)
2377 {
2378         struct rte_eth_dev *dev;
2379
2380         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2381
2382         dev = &rte_eth_devices[port_id];
2383         memset(stats, 0, sizeof(*stats));
2384
2385         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
2386         stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
2387         return eth_err(port_id, (*dev->dev_ops->stats_get)(dev, stats));
2388 }
2389
2390 int
2391 rte_eth_stats_reset(uint16_t port_id)
2392 {
2393         struct rte_eth_dev *dev;
2394         int ret;
2395
2396         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2397         dev = &rte_eth_devices[port_id];
2398
2399         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_reset, -ENOTSUP);
2400         ret = (*dev->dev_ops->stats_reset)(dev);
2401         if (ret != 0)
2402                 return eth_err(port_id, ret);
2403
2404         dev->data->rx_mbuf_alloc_failed = 0;
2405
2406         return 0;
2407 }
2408
2409 static inline int
2410 get_xstats_basic_count(struct rte_eth_dev *dev)
2411 {
2412         uint16_t nb_rxqs, nb_txqs;
2413         int count;
2414
2415         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2416         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2417
2418         count = RTE_NB_STATS;
2419         count += nb_rxqs * RTE_NB_RXQ_STATS;
2420         count += nb_txqs * RTE_NB_TXQ_STATS;
2421
2422         return count;
2423 }
2424
2425 static int
2426 get_xstats_count(uint16_t port_id)
2427 {
2428         struct rte_eth_dev *dev;
2429         int count;
2430
2431         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2432         dev = &rte_eth_devices[port_id];
2433         if (dev->dev_ops->xstats_get_names_by_id != NULL) {
2434                 count = (*dev->dev_ops->xstats_get_names_by_id)(dev, NULL,
2435                                 NULL, 0);
2436                 if (count < 0)
2437                         return eth_err(port_id, count);
2438         }
2439         if (dev->dev_ops->xstats_get_names != NULL) {
2440                 count = (*dev->dev_ops->xstats_get_names)(dev, NULL, 0);
2441                 if (count < 0)
2442                         return eth_err(port_id, count);
2443         } else
2444                 count = 0;
2445
2446
2447         count += get_xstats_basic_count(dev);
2448
2449         return count;
2450 }
2451
2452 int
2453 rte_eth_xstats_get_id_by_name(uint16_t port_id, const char *xstat_name,
2454                 uint64_t *id)
2455 {
2456         int cnt_xstats, idx_xstat;
2457
2458         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2459
2460         if (!id) {
2461                 RTE_ETHDEV_LOG(ERR, "Id pointer is NULL\n");
2462                 return -ENOMEM;
2463         }
2464
2465         if (!xstat_name) {
2466                 RTE_ETHDEV_LOG(ERR, "xstat_name pointer is NULL\n");
2467                 return -ENOMEM;
2468         }
2469
2470         /* Get count */
2471         cnt_xstats = rte_eth_xstats_get_names_by_id(port_id, NULL, 0, NULL);
2472         if (cnt_xstats  < 0) {
2473                 RTE_ETHDEV_LOG(ERR, "Cannot get count of xstats\n");
2474                 return -ENODEV;
2475         }
2476
2477         /* Get id-name lookup table */
2478         struct rte_eth_xstat_name xstats_names[cnt_xstats];
2479
2480         if (cnt_xstats != rte_eth_xstats_get_names_by_id(
2481                         port_id, xstats_names, cnt_xstats, NULL)) {
2482                 RTE_ETHDEV_LOG(ERR, "Cannot get xstats lookup\n");
2483                 return -1;
2484         }
2485
2486         for (idx_xstat = 0; idx_xstat < cnt_xstats; idx_xstat++) {
2487                 if (!strcmp(xstats_names[idx_xstat].name, xstat_name)) {
2488                         *id = idx_xstat;
2489                         return 0;
2490                 };
2491         }
2492
2493         return -EINVAL;
2494 }
2495
2496 /* retrieve basic stats names */
2497 static int
2498 rte_eth_basic_stats_get_names(struct rte_eth_dev *dev,
2499         struct rte_eth_xstat_name *xstats_names)
2500 {
2501         int cnt_used_entries = 0;
2502         uint32_t idx, id_queue;
2503         uint16_t num_q;
2504
2505         for (idx = 0; idx < RTE_NB_STATS; idx++) {
2506                 strlcpy(xstats_names[cnt_used_entries].name,
2507                         rte_stats_strings[idx].name,
2508                         sizeof(xstats_names[0].name));
2509                 cnt_used_entries++;
2510         }
2511         num_q = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2512         for (id_queue = 0; id_queue < num_q; id_queue++) {
2513                 for (idx = 0; idx < RTE_NB_RXQ_STATS; idx++) {
2514                         snprintf(xstats_names[cnt_used_entries].name,
2515                                 sizeof(xstats_names[0].name),
2516                                 "rx_q%u%s",
2517                                 id_queue, rte_rxq_stats_strings[idx].name);
2518                         cnt_used_entries++;
2519                 }
2520
2521         }
2522         num_q = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2523         for (id_queue = 0; id_queue < num_q; id_queue++) {
2524                 for (idx = 0; idx < RTE_NB_TXQ_STATS; idx++) {
2525                         snprintf(xstats_names[cnt_used_entries].name,
2526                                 sizeof(xstats_names[0].name),
2527                                 "tx_q%u%s",
2528                                 id_queue, rte_txq_stats_strings[idx].name);
2529                         cnt_used_entries++;
2530                 }
2531         }
2532         return cnt_used_entries;
2533 }
2534
2535 /* retrieve ethdev extended statistics names */
2536 int
2537 rte_eth_xstats_get_names_by_id(uint16_t port_id,
2538         struct rte_eth_xstat_name *xstats_names, unsigned int size,
2539         uint64_t *ids)
2540 {
2541         struct rte_eth_xstat_name *xstats_names_copy;
2542         unsigned int no_basic_stat_requested = 1;
2543         unsigned int no_ext_stat_requested = 1;
2544         unsigned int expected_entries;
2545         unsigned int basic_count;
2546         struct rte_eth_dev *dev;
2547         unsigned int i;
2548         int ret;
2549
2550         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2551         dev = &rte_eth_devices[port_id];
2552
2553         basic_count = get_xstats_basic_count(dev);
2554         ret = get_xstats_count(port_id);
2555         if (ret < 0)
2556                 return ret;
2557         expected_entries = (unsigned int)ret;
2558
2559         /* Return max number of stats if no ids given */
2560         if (!ids) {
2561                 if (!xstats_names)
2562                         return expected_entries;
2563                 else if (xstats_names && size < expected_entries)
2564                         return expected_entries;
2565         }
2566
2567         if (ids && !xstats_names)
2568                 return -EINVAL;
2569
2570         if (ids && dev->dev_ops->xstats_get_names_by_id != NULL && size > 0) {
2571                 uint64_t ids_copy[size];
2572
2573                 for (i = 0; i < size; i++) {
2574                         if (ids[i] < basic_count) {
2575                                 no_basic_stat_requested = 0;
2576                                 break;
2577                         }
2578
2579                         /*
2580                          * Convert ids to xstats ids that PMD knows.
2581                          * ids known by user are basic + extended stats.
2582                          */
2583                         ids_copy[i] = ids[i] - basic_count;
2584                 }
2585
2586                 if (no_basic_stat_requested)
2587                         return (*dev->dev_ops->xstats_get_names_by_id)(dev,
2588                                         xstats_names, ids_copy, size);
2589         }
2590
2591         /* Retrieve all stats */
2592         if (!ids) {
2593                 int num_stats = rte_eth_xstats_get_names(port_id, xstats_names,
2594                                 expected_entries);
2595                 if (num_stats < 0 || num_stats > (int)expected_entries)
2596                         return num_stats;
2597                 else
2598                         return expected_entries;
2599         }
2600
2601         xstats_names_copy = calloc(expected_entries,
2602                 sizeof(struct rte_eth_xstat_name));
2603
2604         if (!xstats_names_copy) {
2605                 RTE_ETHDEV_LOG(ERR, "Can't allocate memory\n");
2606                 return -ENOMEM;
2607         }
2608
2609         if (ids) {
2610                 for (i = 0; i < size; i++) {
2611                         if (ids[i] >= basic_count) {
2612                                 no_ext_stat_requested = 0;
2613                                 break;
2614                         }
2615                 }
2616         }
2617
2618         /* Fill xstats_names_copy structure */
2619         if (ids && no_ext_stat_requested) {
2620                 rte_eth_basic_stats_get_names(dev, xstats_names_copy);
2621         } else {
2622                 ret = rte_eth_xstats_get_names(port_id, xstats_names_copy,
2623                         expected_entries);
2624                 if (ret < 0) {
2625                         free(xstats_names_copy);
2626                         return ret;
2627                 }
2628         }
2629
2630         /* Filter stats */
2631         for (i = 0; i < size; i++) {
2632                 if (ids[i] >= expected_entries) {
2633                         RTE_ETHDEV_LOG(ERR, "Id value isn't valid\n");
2634                         free(xstats_names_copy);
2635                         return -1;
2636                 }
2637                 xstats_names[i] = xstats_names_copy[ids[i]];
2638         }
2639
2640         free(xstats_names_copy);
2641         return size;
2642 }
2643
2644 int
2645 rte_eth_xstats_get_names(uint16_t port_id,
2646         struct rte_eth_xstat_name *xstats_names,
2647         unsigned int size)
2648 {
2649         struct rte_eth_dev *dev;
2650         int cnt_used_entries;
2651         int cnt_expected_entries;
2652         int cnt_driver_entries;
2653
2654         cnt_expected_entries = get_xstats_count(port_id);
2655         if (xstats_names == NULL || cnt_expected_entries < 0 ||
2656                         (int)size < cnt_expected_entries)
2657                 return cnt_expected_entries;
2658
2659         /* port_id checked in get_xstats_count() */
2660         dev = &rte_eth_devices[port_id];
2661
2662         cnt_used_entries = rte_eth_basic_stats_get_names(
2663                 dev, xstats_names);
2664
2665         if (dev->dev_ops->xstats_get_names != NULL) {
2666                 /* If there are any driver-specific xstats, append them
2667                  * to end of list.
2668                  */
2669                 cnt_driver_entries = (*dev->dev_ops->xstats_get_names)(
2670                         dev,
2671                         xstats_names + cnt_used_entries,
2672                         size - cnt_used_entries);
2673                 if (cnt_driver_entries < 0)
2674                         return eth_err(port_id, cnt_driver_entries);
2675                 cnt_used_entries += cnt_driver_entries;
2676         }
2677
2678         return cnt_used_entries;
2679 }
2680
2681
2682 static int
2683 rte_eth_basic_stats_get(uint16_t port_id, struct rte_eth_xstat *xstats)
2684 {
2685         struct rte_eth_dev *dev;
2686         struct rte_eth_stats eth_stats;
2687         unsigned int count = 0, i, q;
2688         uint64_t val, *stats_ptr;
2689         uint16_t nb_rxqs, nb_txqs;
2690         int ret;
2691
2692         ret = rte_eth_stats_get(port_id, &eth_stats);
2693         if (ret < 0)
2694                 return ret;
2695
2696         dev = &rte_eth_devices[port_id];
2697
2698         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2699         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2700
2701         /* global stats */
2702         for (i = 0; i < RTE_NB_STATS; i++) {
2703                 stats_ptr = RTE_PTR_ADD(&eth_stats,
2704                                         rte_stats_strings[i].offset);
2705                 val = *stats_ptr;
2706                 xstats[count++].value = val;
2707         }
2708
2709         /* per-rxq stats */
2710         for (q = 0; q < nb_rxqs; q++) {
2711                 for (i = 0; i < RTE_NB_RXQ_STATS; i++) {
2712                         stats_ptr = RTE_PTR_ADD(&eth_stats,
2713                                         rte_rxq_stats_strings[i].offset +
2714                                         q * sizeof(uint64_t));
2715                         val = *stats_ptr;
2716                         xstats[count++].value = val;
2717                 }
2718         }
2719
2720         /* per-txq stats */
2721         for (q = 0; q < nb_txqs; q++) {
2722                 for (i = 0; i < RTE_NB_TXQ_STATS; i++) {
2723                         stats_ptr = RTE_PTR_ADD(&eth_stats,
2724                                         rte_txq_stats_strings[i].offset +
2725                                         q * sizeof(uint64_t));
2726                         val = *stats_ptr;
2727                         xstats[count++].value = val;
2728                 }
2729         }
2730         return count;
2731 }
2732
2733 /* retrieve ethdev extended statistics */
2734 int
2735 rte_eth_xstats_get_by_id(uint16_t port_id, const uint64_t *ids,
2736                          uint64_t *values, unsigned int size)
2737 {
2738         unsigned int no_basic_stat_requested = 1;
2739         unsigned int no_ext_stat_requested = 1;
2740         unsigned int num_xstats_filled;
2741         unsigned int basic_count;
2742         uint16_t expected_entries;
2743         struct rte_eth_dev *dev;
2744         unsigned int i;
2745         int ret;
2746
2747         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2748         ret = get_xstats_count(port_id);
2749         if (ret < 0)
2750                 return ret;
2751         expected_entries = (uint16_t)ret;
2752         struct rte_eth_xstat xstats[expected_entries];
2753         dev = &rte_eth_devices[port_id];
2754         basic_count = get_xstats_basic_count(dev);
2755
2756         /* Return max number of stats if no ids given */
2757         if (!ids) {
2758                 if (!values)
2759                         return expected_entries;
2760                 else if (values && size < expected_entries)
2761                         return expected_entries;
2762         }
2763
2764         if (ids && !values)
2765                 return -EINVAL;
2766
2767         if (ids && dev->dev_ops->xstats_get_by_id != NULL && size) {
2768                 unsigned int basic_count = get_xstats_basic_count(dev);
2769                 uint64_t ids_copy[size];
2770
2771                 for (i = 0; i < size; i++) {
2772                         if (ids[i] < basic_count) {
2773                                 no_basic_stat_requested = 0;
2774                                 break;
2775                         }
2776
2777                         /*
2778                          * Convert ids to xstats ids that PMD knows.
2779                          * ids known by user are basic + extended stats.
2780                          */
2781                         ids_copy[i] = ids[i] - basic_count;
2782                 }
2783
2784                 if (no_basic_stat_requested)
2785                         return (*dev->dev_ops->xstats_get_by_id)(dev, ids_copy,
2786                                         values, size);
2787         }
2788
2789         if (ids) {
2790                 for (i = 0; i < size; i++) {
2791                         if (ids[i] >= basic_count) {
2792                                 no_ext_stat_requested = 0;
2793                                 break;
2794                         }
2795                 }
2796         }
2797
2798         /* Fill the xstats structure */
2799         if (ids && no_ext_stat_requested)
2800                 ret = rte_eth_basic_stats_get(port_id, xstats);
2801         else
2802                 ret = rte_eth_xstats_get(port_id, xstats, expected_entries);
2803
2804         if (ret < 0)
2805                 return ret;
2806         num_xstats_filled = (unsigned int)ret;
2807
2808         /* Return all stats */
2809         if (!ids) {
2810                 for (i = 0; i < num_xstats_filled; i++)
2811                         values[i] = xstats[i].value;
2812                 return expected_entries;
2813         }
2814
2815         /* Filter stats */
2816         for (i = 0; i < size; i++) {
2817                 if (ids[i] >= expected_entries) {
2818                         RTE_ETHDEV_LOG(ERR, "Id value isn't valid\n");
2819                         return -1;
2820                 }
2821                 values[i] = xstats[ids[i]].value;
2822         }
2823         return size;
2824 }
2825
2826 int
2827 rte_eth_xstats_get(uint16_t port_id, struct rte_eth_xstat *xstats,
2828         unsigned int n)
2829 {
2830         struct rte_eth_dev *dev;
2831         unsigned int count = 0, i;
2832         signed int xcount = 0;
2833         uint16_t nb_rxqs, nb_txqs;
2834         int ret;
2835
2836         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2837
2838         dev = &rte_eth_devices[port_id];
2839
2840         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2841         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2842
2843         /* Return generic statistics */
2844         count = RTE_NB_STATS + (nb_rxqs * RTE_NB_RXQ_STATS) +
2845                 (nb_txqs * RTE_NB_TXQ_STATS);
2846
2847         /* implemented by the driver */
2848         if (dev->dev_ops->xstats_get != NULL) {
2849                 /* Retrieve the xstats from the driver at the end of the
2850                  * xstats struct.
2851                  */
2852                 xcount = (*dev->dev_ops->xstats_get)(dev,
2853                                      xstats ? xstats + count : NULL,
2854                                      (n > count) ? n - count : 0);
2855
2856                 if (xcount < 0)
2857                         return eth_err(port_id, xcount);
2858         }
2859
2860         if (n < count + xcount || xstats == NULL)
2861                 return count + xcount;
2862
2863         /* now fill the xstats structure */
2864         ret = rte_eth_basic_stats_get(port_id, xstats);
2865         if (ret < 0)
2866                 return ret;
2867         count = ret;
2868
2869         for (i = 0; i < count; i++)
2870                 xstats[i].id = i;
2871         /* add an offset to driver-specific stats */
2872         for ( ; i < count + xcount; i++)
2873                 xstats[i].id += count;
2874
2875         return count + xcount;
2876 }
2877
2878 /* reset ethdev extended statistics */
2879 int
2880 rte_eth_xstats_reset(uint16_t port_id)
2881 {
2882         struct rte_eth_dev *dev;
2883
2884         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2885         dev = &rte_eth_devices[port_id];
2886
2887         /* implemented by the driver */
2888         if (dev->dev_ops->xstats_reset != NULL)
2889                 return eth_err(port_id, (*dev->dev_ops->xstats_reset)(dev));
2890
2891         /* fallback to default */
2892         return rte_eth_stats_reset(port_id);
2893 }
2894
2895 static int
2896 set_queue_stats_mapping(uint16_t port_id, uint16_t queue_id, uint8_t stat_idx,
2897                 uint8_t is_rx)
2898 {
2899         struct rte_eth_dev *dev;
2900
2901         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2902
2903         dev = &rte_eth_devices[port_id];
2904
2905         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_stats_mapping_set, -ENOTSUP);
2906
2907         if (is_rx && (queue_id >= dev->data->nb_rx_queues))
2908                 return -EINVAL;
2909
2910         if (!is_rx && (queue_id >= dev->data->nb_tx_queues))
2911                 return -EINVAL;
2912
2913         if (stat_idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
2914                 return -EINVAL;
2915
2916         return (*dev->dev_ops->queue_stats_mapping_set)
2917                         (dev, queue_id, stat_idx, is_rx);
2918 }
2919
2920
2921 int
2922 rte_eth_dev_set_tx_queue_stats_mapping(uint16_t port_id, uint16_t tx_queue_id,
2923                 uint8_t stat_idx)
2924 {
2925         return eth_err(port_id, set_queue_stats_mapping(port_id, tx_queue_id,
2926                                                 stat_idx, STAT_QMAP_TX));
2927 }
2928
2929
2930 int
2931 rte_eth_dev_set_rx_queue_stats_mapping(uint16_t port_id, uint16_t rx_queue_id,
2932                 uint8_t stat_idx)
2933 {
2934         return eth_err(port_id, set_queue_stats_mapping(port_id, rx_queue_id,
2935                                                 stat_idx, STAT_QMAP_RX));
2936 }
2937
2938 int
2939 rte_eth_dev_fw_version_get(uint16_t port_id, char *fw_version, size_t fw_size)
2940 {
2941         struct rte_eth_dev *dev;
2942
2943         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2944         dev = &rte_eth_devices[port_id];
2945
2946         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fw_version_get, -ENOTSUP);
2947         return eth_err(port_id, (*dev->dev_ops->fw_version_get)(dev,
2948                                                         fw_version, fw_size));
2949 }
2950
2951 int
2952 rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info)
2953 {
2954         struct rte_eth_dev *dev;
2955         const struct rte_eth_desc_lim lim = {
2956                 .nb_max = UINT16_MAX,
2957                 .nb_min = 0,
2958                 .nb_align = 1,
2959                 .nb_seg_max = UINT16_MAX,
2960                 .nb_mtu_seg_max = UINT16_MAX,
2961         };
2962         int diag;
2963
2964         /*
2965          * Init dev_info before port_id check since caller does not have
2966          * return status and does not know if get is successful or not.
2967          */
2968         memset(dev_info, 0, sizeof(struct rte_eth_dev_info));
2969         dev_info->switch_info.domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
2970
2971         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2972         dev = &rte_eth_devices[port_id];
2973
2974         dev_info->rx_desc_lim = lim;
2975         dev_info->tx_desc_lim = lim;
2976         dev_info->device = dev->device;
2977         dev_info->min_mtu = RTE_ETHER_MIN_MTU;
2978         dev_info->max_mtu = UINT16_MAX;
2979
2980         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
2981         diag = (*dev->dev_ops->dev_infos_get)(dev, dev_info);
2982         if (diag != 0) {
2983                 /* Cleanup already filled in device information */
2984                 memset(dev_info, 0, sizeof(struct rte_eth_dev_info));
2985                 return eth_err(port_id, diag);
2986         }
2987
2988         /* Maximum number of queues should be <= RTE_MAX_QUEUES_PER_PORT */
2989         dev_info->max_rx_queues = RTE_MIN(dev_info->max_rx_queues,
2990                         RTE_MAX_QUEUES_PER_PORT);
2991         dev_info->max_tx_queues = RTE_MIN(dev_info->max_tx_queues,
2992                         RTE_MAX_QUEUES_PER_PORT);
2993
2994         dev_info->driver_name = dev->device->driver->name;
2995         dev_info->nb_rx_queues = dev->data->nb_rx_queues;
2996         dev_info->nb_tx_queues = dev->data->nb_tx_queues;
2997
2998         dev_info->dev_flags = &dev->data->dev_flags;
2999
3000         return 0;
3001 }
3002
3003 int
3004 rte_eth_dev_get_supported_ptypes(uint16_t port_id, uint32_t ptype_mask,
3005                                  uint32_t *ptypes, int num)
3006 {
3007         int i, j;
3008         struct rte_eth_dev *dev;
3009         const uint32_t *all_ptypes;
3010
3011         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3012         dev = &rte_eth_devices[port_id];
3013         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_supported_ptypes_get, 0);
3014         all_ptypes = (*dev->dev_ops->dev_supported_ptypes_get)(dev);
3015
3016         if (!all_ptypes)
3017                 return 0;
3018
3019         for (i = 0, j = 0; all_ptypes[i] != RTE_PTYPE_UNKNOWN; ++i)
3020                 if (all_ptypes[i] & ptype_mask) {
3021                         if (j < num)
3022                                 ptypes[j] = all_ptypes[i];
3023                         j++;
3024                 }
3025
3026         return j;
3027 }
3028
3029 int
3030 rte_eth_dev_set_ptypes(uint16_t port_id, uint32_t ptype_mask,
3031                                  uint32_t *set_ptypes, unsigned int num)
3032 {
3033         const uint32_t valid_ptype_masks[] = {
3034                 RTE_PTYPE_L2_MASK,
3035                 RTE_PTYPE_L3_MASK,
3036                 RTE_PTYPE_L4_MASK,
3037                 RTE_PTYPE_TUNNEL_MASK,
3038                 RTE_PTYPE_INNER_L2_MASK,
3039                 RTE_PTYPE_INNER_L3_MASK,
3040                 RTE_PTYPE_INNER_L4_MASK,
3041         };
3042         const uint32_t *all_ptypes;
3043         struct rte_eth_dev *dev;
3044         uint32_t unused_mask;
3045         unsigned int i, j;
3046         int ret;
3047
3048         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3049         dev = &rte_eth_devices[port_id];
3050
3051         if (num > 0 && set_ptypes == NULL)
3052                 return -EINVAL;
3053
3054         if (*dev->dev_ops->dev_supported_ptypes_get == NULL ||
3055                         *dev->dev_ops->dev_ptypes_set == NULL) {
3056                 ret = 0;
3057                 goto ptype_unknown;
3058         }
3059
3060         if (ptype_mask == 0) {
3061                 ret = (*dev->dev_ops->dev_ptypes_set)(dev,
3062                                 ptype_mask);
3063                 goto ptype_unknown;
3064         }
3065
3066         unused_mask = ptype_mask;
3067         for (i = 0; i < RTE_DIM(valid_ptype_masks); i++) {
3068                 uint32_t mask = ptype_mask & valid_ptype_masks[i];
3069                 if (mask && mask != valid_ptype_masks[i]) {
3070                         ret = -EINVAL;
3071                         goto ptype_unknown;
3072                 }
3073                 unused_mask &= ~valid_ptype_masks[i];
3074         }
3075
3076         if (unused_mask) {
3077                 ret = -EINVAL;
3078                 goto ptype_unknown;
3079         }
3080
3081         all_ptypes = (*dev->dev_ops->dev_supported_ptypes_get)(dev);
3082         if (all_ptypes == NULL) {
3083                 ret = 0;
3084                 goto ptype_unknown;
3085         }
3086
3087         /*
3088          * Accommodate as many set_ptypes as possible. If the supplied
3089          * set_ptypes array is insufficient fill it partially.
3090          */
3091         for (i = 0, j = 0; set_ptypes != NULL &&
3092                                 (all_ptypes[i] != RTE_PTYPE_UNKNOWN); ++i) {
3093                 if (ptype_mask & all_ptypes[i]) {
3094                         if (j < num - 1) {
3095                                 set_ptypes[j] = all_ptypes[i];
3096                                 j++;
3097                                 continue;
3098                         }
3099                         break;
3100                 }
3101         }
3102
3103         if (set_ptypes != NULL && j < num)
3104                 set_ptypes[j] = RTE_PTYPE_UNKNOWN;
3105
3106         return (*dev->dev_ops->dev_ptypes_set)(dev, ptype_mask);
3107
3108 ptype_unknown:
3109         if (num > 0)
3110                 set_ptypes[0] = RTE_PTYPE_UNKNOWN;
3111
3112         return ret;
3113 }
3114
3115 int
3116 rte_eth_macaddr_get(uint16_t port_id, struct rte_ether_addr *mac_addr)
3117 {
3118         struct rte_eth_dev *dev;
3119
3120         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3121         dev = &rte_eth_devices[port_id];
3122         rte_ether_addr_copy(&dev->data->mac_addrs[0], mac_addr);
3123
3124         return 0;
3125 }
3126
3127 int
3128 rte_eth_dev_get_mtu(uint16_t port_id, uint16_t *mtu)
3129 {
3130         struct rte_eth_dev *dev;
3131
3132         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3133
3134         dev = &rte_eth_devices[port_id];
3135         *mtu = dev->data->mtu;
3136         return 0;
3137 }
3138
3139 int
3140 rte_eth_dev_set_mtu(uint16_t port_id, uint16_t mtu)
3141 {
3142         int ret;
3143         struct rte_eth_dev_info dev_info;
3144         struct rte_eth_dev *dev;
3145
3146         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3147         dev = &rte_eth_devices[port_id];
3148         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mtu_set, -ENOTSUP);
3149
3150         /*
3151          * Check if the device supports dev_infos_get, if it does not
3152          * skip min_mtu/max_mtu validation here as this requires values
3153          * that are populated within the call to rte_eth_dev_info_get()
3154          * which relies on dev->dev_ops->dev_infos_get.
3155          */
3156         if (*dev->dev_ops->dev_infos_get != NULL) {
3157                 ret = rte_eth_dev_info_get(port_id, &dev_info);
3158                 if (ret != 0)
3159                         return ret;
3160
3161                 if (mtu < dev_info.min_mtu || mtu > dev_info.max_mtu)
3162                         return -EINVAL;
3163         }
3164
3165         ret = (*dev->dev_ops->mtu_set)(dev, mtu);
3166         if (!ret)
3167                 dev->data->mtu = mtu;
3168
3169         return eth_err(port_id, ret);
3170 }
3171
3172 int
3173 rte_eth_dev_vlan_filter(uint16_t port_id, uint16_t vlan_id, int on)
3174 {
3175         struct rte_eth_dev *dev;
3176         int ret;
3177
3178         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3179         dev = &rte_eth_devices[port_id];
3180         if (!(dev->data->dev_conf.rxmode.offloads &
3181               DEV_RX_OFFLOAD_VLAN_FILTER)) {
3182                 RTE_ETHDEV_LOG(ERR, "Port %u: vlan-filtering disabled\n",
3183                         port_id);
3184                 return -ENOSYS;
3185         }
3186
3187         if (vlan_id > 4095) {
3188                 RTE_ETHDEV_LOG(ERR, "Port_id=%u invalid vlan_id=%u > 4095\n",
3189                         port_id, vlan_id);
3190                 return -EINVAL;
3191         }
3192         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_filter_set, -ENOTSUP);
3193
3194         ret = (*dev->dev_ops->vlan_filter_set)(dev, vlan_id, on);
3195         if (ret == 0) {
3196                 struct rte_vlan_filter_conf *vfc;
3197                 int vidx;
3198                 int vbit;
3199
3200                 vfc = &dev->data->vlan_filter_conf;
3201                 vidx = vlan_id / 64;
3202                 vbit = vlan_id % 64;
3203
3204                 if (on)
3205                         vfc->ids[vidx] |= UINT64_C(1) << vbit;
3206                 else
3207                         vfc->ids[vidx] &= ~(UINT64_C(1) << vbit);
3208         }
3209
3210         return eth_err(port_id, ret);
3211 }
3212
3213 int
3214 rte_eth_dev_set_vlan_strip_on_queue(uint16_t port_id, uint16_t rx_queue_id,
3215                                     int on)
3216 {
3217         struct rte_eth_dev *dev;
3218
3219         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3220         dev = &rte_eth_devices[port_id];
3221         if (rx_queue_id >= dev->data->nb_rx_queues) {
3222                 RTE_ETHDEV_LOG(ERR, "Invalid rx_queue_id=%u\n", rx_queue_id);
3223                 return -EINVAL;
3224         }
3225
3226         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_strip_queue_set, -ENOTSUP);
3227         (*dev->dev_ops->vlan_strip_queue_set)(dev, rx_queue_id, on);
3228
3229         return 0;
3230 }
3231
3232 int
3233 rte_eth_dev_set_vlan_ether_type(uint16_t port_id,
3234                                 enum rte_vlan_type vlan_type,
3235                                 uint16_t tpid)
3236 {
3237         struct rte_eth_dev *dev;
3238
3239         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3240         dev = &rte_eth_devices[port_id];
3241         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_tpid_set, -ENOTSUP);
3242
3243         return eth_err(port_id, (*dev->dev_ops->vlan_tpid_set)(dev, vlan_type,
3244                                                                tpid));
3245 }
3246
3247 int
3248 rte_eth_dev_set_vlan_offload(uint16_t port_id, int offload_mask)
3249 {
3250         struct rte_eth_dev *dev;
3251         int ret = 0;
3252         int mask = 0;
3253         int cur, org = 0;
3254         uint64_t orig_offloads;
3255         uint64_t dev_offloads;
3256
3257         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3258         dev = &rte_eth_devices[port_id];
3259
3260         /* save original values in case of failure */
3261         orig_offloads = dev->data->dev_conf.rxmode.offloads;
3262         dev_offloads = orig_offloads;
3263
3264         /* check which option changed by application */
3265         cur = !!(offload_mask & ETH_VLAN_STRIP_OFFLOAD);
3266         org = !!(dev_offloads & DEV_RX_OFFLOAD_VLAN_STRIP);
3267         if (cur != org) {
3268                 if (cur)
3269                         dev_offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
3270                 else
3271                         dev_offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP;
3272                 mask |= ETH_VLAN_STRIP_MASK;
3273         }
3274
3275         cur = !!(offload_mask & ETH_VLAN_FILTER_OFFLOAD);
3276         org = !!(dev_offloads & DEV_RX_OFFLOAD_VLAN_FILTER);
3277         if (cur != org) {
3278                 if (cur)
3279                         dev_offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3280                 else
3281                         dev_offloads &= ~DEV_RX_OFFLOAD_VLAN_FILTER;
3282                 mask |= ETH_VLAN_FILTER_MASK;
3283         }
3284
3285         cur = !!(offload_mask & ETH_VLAN_EXTEND_OFFLOAD);
3286         org = !!(dev_offloads & DEV_RX_OFFLOAD_VLAN_EXTEND);
3287         if (cur != org) {
3288                 if (cur)
3289                         dev_offloads |= DEV_RX_OFFLOAD_VLAN_EXTEND;
3290                 else
3291                         dev_offloads &= ~DEV_RX_OFFLOAD_VLAN_EXTEND;
3292                 mask |= ETH_VLAN_EXTEND_MASK;
3293         }
3294
3295         cur = !!(offload_mask & ETH_QINQ_STRIP_OFFLOAD);
3296         org = !!(dev_offloads & DEV_RX_OFFLOAD_QINQ_STRIP);
3297         if (cur != org) {
3298                 if (cur)
3299                         dev_offloads |= DEV_RX_OFFLOAD_QINQ_STRIP;
3300                 else
3301                         dev_offloads &= ~DEV_RX_OFFLOAD_QINQ_STRIP;
3302                 mask |= ETH_QINQ_STRIP_MASK;
3303         }
3304
3305         /*no change*/
3306         if (mask == 0)
3307                 return ret;
3308
3309         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_offload_set, -ENOTSUP);
3310         dev->data->dev_conf.rxmode.offloads = dev_offloads;
3311         ret = (*dev->dev_ops->vlan_offload_set)(dev, mask);
3312         if (ret) {
3313                 /* hit an error restore  original values */
3314                 dev->data->dev_conf.rxmode.offloads = orig_offloads;
3315         }
3316
3317         return eth_err(port_id, ret);
3318 }
3319
3320 int
3321 rte_eth_dev_get_vlan_offload(uint16_t port_id)
3322 {
3323         struct rte_eth_dev *dev;
3324         uint64_t *dev_offloads;
3325         int ret = 0;
3326
3327         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3328         dev = &rte_eth_devices[port_id];
3329         dev_offloads = &dev->data->dev_conf.rxmode.offloads;
3330
3331         if (*dev_offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
3332                 ret |= ETH_VLAN_STRIP_OFFLOAD;
3333
3334         if (*dev_offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
3335                 ret |= ETH_VLAN_FILTER_OFFLOAD;
3336
3337         if (*dev_offloads & DEV_RX_OFFLOAD_VLAN_EXTEND)
3338                 ret |= ETH_VLAN_EXTEND_OFFLOAD;
3339
3340         if (*dev_offloads & DEV_RX_OFFLOAD_QINQ_STRIP)
3341                 ret |= ETH_QINQ_STRIP_OFFLOAD;
3342
3343         return ret;
3344 }
3345
3346 int
3347 rte_eth_dev_set_vlan_pvid(uint16_t port_id, uint16_t pvid, int on)
3348 {
3349         struct rte_eth_dev *dev;
3350
3351         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3352         dev = &rte_eth_devices[port_id];
3353         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_pvid_set, -ENOTSUP);
3354
3355         return eth_err(port_id, (*dev->dev_ops->vlan_pvid_set)(dev, pvid, on));
3356 }
3357
3358 int
3359 rte_eth_dev_flow_ctrl_get(uint16_t port_id, struct rte_eth_fc_conf *fc_conf)
3360 {
3361         struct rte_eth_dev *dev;
3362
3363         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3364         dev = &rte_eth_devices[port_id];
3365         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_get, -ENOTSUP);
3366         memset(fc_conf, 0, sizeof(*fc_conf));
3367         return eth_err(port_id, (*dev->dev_ops->flow_ctrl_get)(dev, fc_conf));
3368 }
3369
3370 int
3371 rte_eth_dev_flow_ctrl_set(uint16_t port_id, struct rte_eth_fc_conf *fc_conf)
3372 {
3373         struct rte_eth_dev *dev;
3374
3375         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3376         if ((fc_conf->send_xon != 0) && (fc_conf->send_xon != 1)) {
3377                 RTE_ETHDEV_LOG(ERR, "Invalid send_xon, only 0/1 allowed\n");
3378                 return -EINVAL;
3379         }
3380
3381         dev = &rte_eth_devices[port_id];
3382         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_set, -ENOTSUP);
3383         return eth_err(port_id, (*dev->dev_ops->flow_ctrl_set)(dev, fc_conf));
3384 }
3385
3386 int
3387 rte_eth_dev_priority_flow_ctrl_set(uint16_t port_id,
3388                                    struct rte_eth_pfc_conf *pfc_conf)
3389 {
3390         struct rte_eth_dev *dev;
3391
3392         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3393         if (pfc_conf->priority > (ETH_DCB_NUM_USER_PRIORITIES - 1)) {
3394                 RTE_ETHDEV_LOG(ERR, "Invalid priority, only 0-7 allowed\n");
3395                 return -EINVAL;
3396         }
3397
3398         dev = &rte_eth_devices[port_id];
3399         /* High water, low water validation are device specific */
3400         if  (*dev->dev_ops->priority_flow_ctrl_set)
3401                 return eth_err(port_id, (*dev->dev_ops->priority_flow_ctrl_set)
3402                                         (dev, pfc_conf));
3403         return -ENOTSUP;
3404 }
3405
3406 static int
3407 rte_eth_check_reta_mask(struct rte_eth_rss_reta_entry64 *reta_conf,
3408                         uint16_t reta_size)
3409 {
3410         uint16_t i, num;
3411
3412         if (!reta_conf)
3413                 return -EINVAL;
3414
3415         num = (reta_size + RTE_RETA_GROUP_SIZE - 1) / RTE_RETA_GROUP_SIZE;
3416         for (i = 0; i < num; i++) {
3417                 if (reta_conf[i].mask)
3418                         return 0;
3419         }
3420
3421         return -EINVAL;
3422 }
3423
3424 static int
3425 rte_eth_check_reta_entry(struct rte_eth_rss_reta_entry64 *reta_conf,
3426                          uint16_t reta_size,
3427                          uint16_t max_rxq)
3428 {
3429         uint16_t i, idx, shift;
3430
3431         if (!reta_conf)
3432                 return -EINVAL;
3433
3434         if (max_rxq == 0) {
3435                 RTE_ETHDEV_LOG(ERR, "No receive queue is available\n");
3436                 return -EINVAL;
3437         }
3438
3439         for (i = 0; i < reta_size; i++) {
3440                 idx = i / RTE_RETA_GROUP_SIZE;
3441                 shift = i % RTE_RETA_GROUP_SIZE;
3442                 if ((reta_conf[idx].mask & (1ULL << shift)) &&
3443                         (reta_conf[idx].reta[shift] >= max_rxq)) {
3444                         RTE_ETHDEV_LOG(ERR,
3445                                 "reta_conf[%u]->reta[%u]: %u exceeds the maximum rxq index: %u\n",
3446                                 idx, shift,
3447                                 reta_conf[idx].reta[shift], max_rxq);
3448                         return -EINVAL;
3449                 }
3450         }
3451
3452         return 0;
3453 }
3454
3455 int
3456 rte_eth_dev_rss_reta_update(uint16_t port_id,
3457                             struct rte_eth_rss_reta_entry64 *reta_conf,
3458                             uint16_t reta_size)
3459 {
3460         struct rte_eth_dev *dev;
3461         int ret;
3462
3463         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3464         /* Check mask bits */
3465         ret = rte_eth_check_reta_mask(reta_conf, reta_size);
3466         if (ret < 0)
3467                 return ret;
3468
3469         dev = &rte_eth_devices[port_id];
3470
3471         /* Check entry value */
3472         ret = rte_eth_check_reta_entry(reta_conf, reta_size,
3473                                 dev->data->nb_rx_queues);
3474         if (ret < 0)
3475                 return ret;
3476
3477         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_update, -ENOTSUP);
3478         return eth_err(port_id, (*dev->dev_ops->reta_update)(dev, reta_conf,
3479                                                              reta_size));
3480 }
3481
3482 int
3483 rte_eth_dev_rss_reta_query(uint16_t port_id,
3484                            struct rte_eth_rss_reta_entry64 *reta_conf,
3485                            uint16_t reta_size)
3486 {
3487         struct rte_eth_dev *dev;
3488         int ret;
3489
3490         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3491
3492         /* Check mask bits */
3493         ret = rte_eth_check_reta_mask(reta_conf, reta_size);
3494         if (ret < 0)
3495                 return ret;
3496
3497         dev = &rte_eth_devices[port_id];
3498         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_query, -ENOTSUP);
3499         return eth_err(port_id, (*dev->dev_ops->reta_query)(dev, reta_conf,
3500                                                             reta_size));
3501 }
3502
3503 int
3504 rte_eth_dev_rss_hash_update(uint16_t port_id,
3505                             struct rte_eth_rss_conf *rss_conf)
3506 {
3507         struct rte_eth_dev *dev;
3508         struct rte_eth_dev_info dev_info = { .flow_type_rss_offloads = 0, };
3509         int ret;
3510
3511         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3512
3513         ret = rte_eth_dev_info_get(port_id, &dev_info);
3514         if (ret != 0)
3515                 return ret;
3516
3517         rss_conf->rss_hf = rte_eth_rss_hf_refine(rss_conf->rss_hf);
3518
3519         dev = &rte_eth_devices[port_id];
3520         if ((dev_info.flow_type_rss_offloads | rss_conf->rss_hf) !=
3521             dev_info.flow_type_rss_offloads) {
3522                 RTE_ETHDEV_LOG(ERR,
3523                         "Ethdev port_id=%u invalid rss_hf: 0x%"PRIx64", valid value: 0x%"PRIx64"\n",
3524                         port_id, rss_conf->rss_hf,
3525                         dev_info.flow_type_rss_offloads);
3526                 return -EINVAL;
3527         }
3528         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_update, -ENOTSUP);
3529         return eth_err(port_id, (*dev->dev_ops->rss_hash_update)(dev,
3530                                                                  rss_conf));
3531 }
3532
3533 int
3534 rte_eth_dev_rss_hash_conf_get(uint16_t port_id,
3535                               struct rte_eth_rss_conf *rss_conf)
3536 {
3537         struct rte_eth_dev *dev;
3538
3539         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3540         dev = &rte_eth_devices[port_id];
3541         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_conf_get, -ENOTSUP);
3542         return eth_err(port_id, (*dev->dev_ops->rss_hash_conf_get)(dev,
3543                                                                    rss_conf));
3544 }
3545
3546 int
3547 rte_eth_dev_udp_tunnel_port_add(uint16_t port_id,
3548                                 struct rte_eth_udp_tunnel *udp_tunnel)
3549 {
3550         struct rte_eth_dev *dev;
3551
3552         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3553         if (udp_tunnel == NULL) {
3554                 RTE_ETHDEV_LOG(ERR, "Invalid udp_tunnel parameter\n");
3555                 return -EINVAL;
3556         }
3557
3558         if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
3559                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
3560                 return -EINVAL;
3561         }
3562
3563         dev = &rte_eth_devices[port_id];
3564         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_port_add, -ENOTSUP);
3565         return eth_err(port_id, (*dev->dev_ops->udp_tunnel_port_add)(dev,
3566                                                                 udp_tunnel));
3567 }
3568
3569 int
3570 rte_eth_dev_udp_tunnel_port_delete(uint16_t port_id,
3571                                    struct rte_eth_udp_tunnel *udp_tunnel)
3572 {
3573         struct rte_eth_dev *dev;
3574
3575         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3576         dev = &rte_eth_devices[port_id];
3577
3578         if (udp_tunnel == NULL) {
3579                 RTE_ETHDEV_LOG(ERR, "Invalid udp_tunnel parameter\n");
3580                 return -EINVAL;
3581         }
3582
3583         if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
3584                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
3585                 return -EINVAL;
3586         }
3587
3588         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_port_del, -ENOTSUP);
3589         return eth_err(port_id, (*dev->dev_ops->udp_tunnel_port_del)(dev,
3590                                                                 udp_tunnel));
3591 }
3592
3593 int
3594 rte_eth_led_on(uint16_t port_id)
3595 {
3596         struct rte_eth_dev *dev;
3597
3598         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3599         dev = &rte_eth_devices[port_id];
3600         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_on, -ENOTSUP);
3601         return eth_err(port_id, (*dev->dev_ops->dev_led_on)(dev));
3602 }
3603
3604 int
3605 rte_eth_led_off(uint16_t port_id)
3606 {
3607         struct rte_eth_dev *dev;
3608
3609         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3610         dev = &rte_eth_devices[port_id];
3611         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_off, -ENOTSUP);
3612         return eth_err(port_id, (*dev->dev_ops->dev_led_off)(dev));
3613 }
3614
3615 /*
3616  * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find
3617  * an empty spot.
3618  */
3619 static int
3620 get_mac_addr_index(uint16_t port_id, const struct rte_ether_addr *addr)
3621 {
3622         struct rte_eth_dev_info dev_info;
3623         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3624         unsigned i;
3625         int ret;
3626
3627         ret = rte_eth_dev_info_get(port_id, &dev_info);
3628         if (ret != 0)
3629                 return -1;
3630
3631         for (i = 0; i < dev_info.max_mac_addrs; i++)
3632                 if (memcmp(addr, &dev->data->mac_addrs[i],
3633                                 RTE_ETHER_ADDR_LEN) == 0)
3634                         return i;
3635
3636         return -1;
3637 }
3638
3639 static const struct rte_ether_addr null_mac_addr;
3640
3641 int
3642 rte_eth_dev_mac_addr_add(uint16_t port_id, struct rte_ether_addr *addr,
3643                         uint32_t pool)
3644 {
3645         struct rte_eth_dev *dev;
3646         int index;
3647         uint64_t pool_mask;
3648         int ret;
3649
3650         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3651         dev = &rte_eth_devices[port_id];
3652         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_add, -ENOTSUP);
3653
3654         if (rte_is_zero_ether_addr(addr)) {
3655                 RTE_ETHDEV_LOG(ERR, "Port %u: Cannot add NULL MAC address\n",
3656                         port_id);
3657                 return -EINVAL;
3658         }
3659         if (pool >= ETH_64_POOLS) {
3660                 RTE_ETHDEV_LOG(ERR, "Pool id must be 0-%d\n", ETH_64_POOLS - 1);
3661                 return -EINVAL;
3662         }
3663
3664         index = get_mac_addr_index(port_id, addr);
3665         if (index < 0) {
3666                 index = get_mac_addr_index(port_id, &null_mac_addr);
3667                 if (index < 0) {
3668                         RTE_ETHDEV_LOG(ERR, "Port %u: MAC address array full\n",
3669                                 port_id);
3670                         return -ENOSPC;
3671                 }
3672         } else {
3673                 pool_mask = dev->data->mac_pool_sel[index];
3674
3675                 /* Check if both MAC address and pool is already there, and do nothing */
3676                 if (pool_mask & (1ULL << pool))
3677                         return 0;
3678         }
3679
3680         /* Update NIC */
3681         ret = (*dev->dev_ops->mac_addr_add)(dev, addr, index, pool);
3682
3683         if (ret == 0) {
3684                 /* Update address in NIC data structure */
3685                 rte_ether_addr_copy(addr, &dev->data->mac_addrs[index]);
3686
3687                 /* Update pool bitmap in NIC data structure */
3688                 dev->data->mac_pool_sel[index] |= (1ULL << pool);
3689         }
3690
3691         return eth_err(port_id, ret);
3692 }
3693
3694 int
3695 rte_eth_dev_mac_addr_remove(uint16_t port_id, struct rte_ether_addr *addr)
3696 {
3697         struct rte_eth_dev *dev;
3698         int index;
3699
3700         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3701         dev = &rte_eth_devices[port_id];
3702         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_remove, -ENOTSUP);
3703
3704         index = get_mac_addr_index(port_id, addr);
3705         if (index == 0) {
3706                 RTE_ETHDEV_LOG(ERR,
3707                         "Port %u: Cannot remove default MAC address\n",
3708                         port_id);
3709                 return -EADDRINUSE;
3710         } else if (index < 0)
3711                 return 0;  /* Do nothing if address wasn't found */
3712
3713         /* Update NIC */
3714         (*dev->dev_ops->mac_addr_remove)(dev, index);
3715
3716         /* Update address in NIC data structure */
3717         rte_ether_addr_copy(&null_mac_addr, &dev->data->mac_addrs[index]);
3718
3719         /* reset pool bitmap */
3720         dev->data->mac_pool_sel[index] = 0;
3721
3722         return 0;
3723 }
3724
3725 int
3726 rte_eth_dev_default_mac_addr_set(uint16_t port_id, struct rte_ether_addr *addr)
3727 {
3728         struct rte_eth_dev *dev;
3729         int ret;
3730
3731         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3732
3733         if (!rte_is_valid_assigned_ether_addr(addr))
3734                 return -EINVAL;
3735
3736         dev = &rte_eth_devices[port_id];
3737         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_set, -ENOTSUP);
3738
3739         ret = (*dev->dev_ops->mac_addr_set)(dev, addr);
3740         if (ret < 0)
3741                 return ret;
3742
3743         /* Update default address in NIC data structure */
3744         rte_ether_addr_copy(addr, &dev->data->mac_addrs[0]);
3745
3746         return 0;
3747 }
3748
3749
3750 /*
3751  * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find
3752  * an empty spot.
3753  */
3754 static int
3755 get_hash_mac_addr_index(uint16_t port_id, const struct rte_ether_addr *addr)
3756 {
3757         struct rte_eth_dev_info dev_info;
3758         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3759         unsigned i;
3760         int ret;
3761
3762         ret = rte_eth_dev_info_get(port_id, &dev_info);
3763         if (ret != 0)
3764                 return -1;
3765
3766         if (!dev->data->hash_mac_addrs)
3767                 return -1;
3768
3769         for (i = 0; i < dev_info.max_hash_mac_addrs; i++)
3770                 if (memcmp(addr, &dev->data->hash_mac_addrs[i],
3771                         RTE_ETHER_ADDR_LEN) == 0)
3772                         return i;
3773
3774         return -1;
3775 }
3776
3777 int
3778 rte_eth_dev_uc_hash_table_set(uint16_t port_id, struct rte_ether_addr *addr,
3779                                 uint8_t on)
3780 {
3781         int index;
3782         int ret;
3783         struct rte_eth_dev *dev;
3784
3785         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3786
3787         dev = &rte_eth_devices[port_id];
3788         if (rte_is_zero_ether_addr(addr)) {
3789                 RTE_ETHDEV_LOG(ERR, "Port %u: Cannot add NULL MAC address\n",
3790                         port_id);
3791                 return -EINVAL;
3792         }
3793
3794         index = get_hash_mac_addr_index(port_id, addr);
3795         /* Check if it's already there, and do nothing */
3796         if ((index >= 0) && on)
3797                 return 0;
3798
3799         if (index < 0) {
3800                 if (!on) {
3801                         RTE_ETHDEV_LOG(ERR,
3802                                 "Port %u: the MAC address was not set in UTA\n",
3803                                 port_id);
3804                         return -EINVAL;
3805                 }
3806
3807                 index = get_hash_mac_addr_index(port_id, &null_mac_addr);
3808                 if (index < 0) {
3809                         RTE_ETHDEV_LOG(ERR, "Port %u: MAC address array full\n",
3810                                 port_id);
3811                         return -ENOSPC;
3812                 }
3813         }
3814
3815         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_hash_table_set, -ENOTSUP);
3816         ret = (*dev->dev_ops->uc_hash_table_set)(dev, addr, on);
3817         if (ret == 0) {
3818                 /* Update address in NIC data structure */
3819                 if (on)
3820                         rte_ether_addr_copy(addr,
3821                                         &dev->data->hash_mac_addrs[index]);
3822                 else
3823                         rte_ether_addr_copy(&null_mac_addr,
3824                                         &dev->data->hash_mac_addrs[index]);
3825         }
3826
3827         return eth_err(port_id, ret);
3828 }
3829
3830 int
3831 rte_eth_dev_uc_all_hash_table_set(uint16_t port_id, uint8_t on)
3832 {
3833         struct rte_eth_dev *dev;
3834
3835         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3836
3837         dev = &rte_eth_devices[port_id];
3838
3839         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_all_hash_table_set, -ENOTSUP);
3840         return eth_err(port_id, (*dev->dev_ops->uc_all_hash_table_set)(dev,
3841                                                                        on));
3842 }
3843
3844 int rte_eth_set_queue_rate_limit(uint16_t port_id, uint16_t queue_idx,
3845                                         uint16_t tx_rate)
3846 {
3847         struct rte_eth_dev *dev;
3848         struct rte_eth_dev_info dev_info;
3849         struct rte_eth_link link;
3850         int ret;
3851
3852         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3853
3854         ret = rte_eth_dev_info_get(port_id, &dev_info);
3855         if (ret != 0)
3856                 return ret;
3857
3858         dev = &rte_eth_devices[port_id];
3859         link = dev->data->dev_link;
3860
3861         if (queue_idx > dev_info.max_tx_queues) {
3862                 RTE_ETHDEV_LOG(ERR,
3863                         "Set queue rate limit:port %u: invalid queue id=%u\n",
3864                         port_id, queue_idx);
3865                 return -EINVAL;
3866         }
3867
3868         if (tx_rate > link.link_speed) {
3869                 RTE_ETHDEV_LOG(ERR,
3870                         "Set queue rate limit:invalid tx_rate=%u, bigger than link speed= %d\n",
3871                         tx_rate, link.link_speed);
3872                 return -EINVAL;
3873         }
3874
3875         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_queue_rate_limit, -ENOTSUP);
3876         return eth_err(port_id, (*dev->dev_ops->set_queue_rate_limit)(dev,
3877                                                         queue_idx, tx_rate));
3878 }
3879
3880 int
3881 rte_eth_mirror_rule_set(uint16_t port_id,
3882                         struct rte_eth_mirror_conf *mirror_conf,
3883                         uint8_t rule_id, uint8_t on)
3884 {
3885         struct rte_eth_dev *dev;
3886
3887         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3888         if (mirror_conf->rule_type == 0) {
3889                 RTE_ETHDEV_LOG(ERR, "Mirror rule type can not be 0\n");
3890                 return -EINVAL;
3891         }
3892
3893         if (mirror_conf->dst_pool >= ETH_64_POOLS) {
3894                 RTE_ETHDEV_LOG(ERR, "Invalid dst pool, pool id must be 0-%d\n",
3895                         ETH_64_POOLS - 1);
3896                 return -EINVAL;
3897         }
3898
3899         if ((mirror_conf->rule_type & (ETH_MIRROR_VIRTUAL_POOL_UP |
3900              ETH_MIRROR_VIRTUAL_POOL_DOWN)) &&
3901             (mirror_conf->pool_mask == 0)) {
3902                 RTE_ETHDEV_LOG(ERR,
3903                         "Invalid mirror pool, pool mask can not be 0\n");
3904                 return -EINVAL;
3905         }
3906
3907         if ((mirror_conf->rule_type & ETH_MIRROR_VLAN) &&
3908             mirror_conf->vlan.vlan_mask == 0) {
3909                 RTE_ETHDEV_LOG(ERR,
3910                         "Invalid vlan mask, vlan mask can not be 0\n");
3911                 return -EINVAL;
3912         }
3913
3914         dev = &rte_eth_devices[port_id];
3915         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mirror_rule_set, -ENOTSUP);
3916
3917         return eth_err(port_id, (*dev->dev_ops->mirror_rule_set)(dev,
3918                                                 mirror_conf, rule_id, on));
3919 }
3920
3921 int
3922 rte_eth_mirror_rule_reset(uint16_t port_id, uint8_t rule_id)
3923 {
3924         struct rte_eth_dev *dev;
3925
3926         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3927
3928         dev = &rte_eth_devices[port_id];
3929         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mirror_rule_reset, -ENOTSUP);
3930
3931         return eth_err(port_id, (*dev->dev_ops->mirror_rule_reset)(dev,
3932                                                                    rule_id));
3933 }
3934
3935 RTE_INIT(eth_dev_init_cb_lists)
3936 {
3937         int i;
3938
3939         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
3940                 TAILQ_INIT(&rte_eth_devices[i].link_intr_cbs);
3941 }
3942
3943 int
3944 rte_eth_dev_callback_register(uint16_t port_id,
3945                         enum rte_eth_event_type event,
3946                         rte_eth_dev_cb_fn cb_fn, void *cb_arg)
3947 {
3948         struct rte_eth_dev *dev;
3949         struct rte_eth_dev_callback *user_cb;
3950         uint32_t next_port; /* size is 32-bit to prevent loop wrap-around */
3951         uint16_t last_port;
3952
3953         if (!cb_fn)
3954                 return -EINVAL;
3955
3956         if (!rte_eth_dev_is_valid_port(port_id) && port_id != RTE_ETH_ALL) {
3957                 RTE_ETHDEV_LOG(ERR, "Invalid port_id=%d\n", port_id);
3958                 return -EINVAL;
3959         }
3960
3961         if (port_id == RTE_ETH_ALL) {
3962                 next_port = 0;
3963                 last_port = RTE_MAX_ETHPORTS - 1;
3964         } else {
3965                 next_port = last_port = port_id;
3966         }
3967
3968         rte_spinlock_lock(&rte_eth_dev_cb_lock);
3969
3970         do {
3971                 dev = &rte_eth_devices[next_port];
3972
3973                 TAILQ_FOREACH(user_cb, &(dev->link_intr_cbs), next) {
3974                         if (user_cb->cb_fn == cb_fn &&
3975                                 user_cb->cb_arg == cb_arg &&
3976                                 user_cb->event == event) {
3977                                 break;
3978                         }
3979                 }
3980
3981                 /* create a new callback. */
3982                 if (user_cb == NULL) {
3983                         user_cb = rte_zmalloc("INTR_USER_CALLBACK",
3984                                 sizeof(struct rte_eth_dev_callback), 0);
3985                         if (user_cb != NULL) {
3986                                 user_cb->cb_fn = cb_fn;
3987                                 user_cb->cb_arg = cb_arg;
3988                                 user_cb->event = event;
3989                                 TAILQ_INSERT_TAIL(&(dev->link_intr_cbs),
3990                                                   user_cb, next);
3991                         } else {
3992                                 rte_spinlock_unlock(&rte_eth_dev_cb_lock);
3993                                 rte_eth_dev_callback_unregister(port_id, event,
3994                                                                 cb_fn, cb_arg);
3995                                 return -ENOMEM;
3996                         }
3997
3998                 }
3999         } while (++next_port <= last_port);
4000
4001         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
4002         return 0;
4003 }
4004
4005 int
4006 rte_eth_dev_callback_unregister(uint16_t port_id,
4007                         enum rte_eth_event_type event,
4008                         rte_eth_dev_cb_fn cb_fn, void *cb_arg)
4009 {
4010         int ret;
4011         struct rte_eth_dev *dev;
4012         struct rte_eth_dev_callback *cb, *next;
4013         uint32_t next_port; /* size is 32-bit to prevent loop wrap-around */
4014         uint16_t last_port;
4015
4016         if (!cb_fn)
4017                 return -EINVAL;
4018
4019         if (!rte_eth_dev_is_valid_port(port_id) && port_id != RTE_ETH_ALL) {
4020                 RTE_ETHDEV_LOG(ERR, "Invalid port_id=%d\n", port_id);
4021                 return -EINVAL;
4022         }
4023
4024         if (port_id == RTE_ETH_ALL) {
4025                 next_port = 0;
4026                 last_port = RTE_MAX_ETHPORTS - 1;
4027         } else {
4028                 next_port = last_port = port_id;
4029         }
4030
4031         rte_spinlock_lock(&rte_eth_dev_cb_lock);
4032
4033         do {
4034                 dev = &rte_eth_devices[next_port];
4035                 ret = 0;
4036                 for (cb = TAILQ_FIRST(&dev->link_intr_cbs); cb != NULL;
4037                      cb = next) {
4038
4039                         next = TAILQ_NEXT(cb, next);
4040
4041                         if (cb->cb_fn != cb_fn || cb->event != event ||
4042                             (cb_arg != (void *)-1 && cb->cb_arg != cb_arg))
4043                                 continue;
4044
4045                         /*
4046                          * if this callback is not executing right now,
4047                          * then remove it.
4048                          */
4049                         if (cb->active == 0) {
4050                                 TAILQ_REMOVE(&(dev->link_intr_cbs), cb, next);
4051                                 rte_free(cb);
4052                         } else {
4053                                 ret = -EAGAIN;
4054                         }
4055                 }
4056         } while (++next_port <= last_port);
4057
4058         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
4059         return ret;
4060 }
4061
4062 int
4063 _rte_eth_dev_callback_process(struct rte_eth_dev *dev,
4064         enum rte_eth_event_type event, void *ret_param)
4065 {
4066         struct rte_eth_dev_callback *cb_lst;
4067         struct rte_eth_dev_callback dev_cb;
4068         int rc = 0;
4069
4070         rte_spinlock_lock(&rte_eth_dev_cb_lock);
4071         TAILQ_FOREACH(cb_lst, &(dev->link_intr_cbs), next) {
4072                 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
4073                         continue;
4074                 dev_cb = *cb_lst;
4075                 cb_lst->active = 1;
4076                 if (ret_param != NULL)
4077                         dev_cb.ret_param = ret_param;
4078
4079                 rte_spinlock_unlock(&rte_eth_dev_cb_lock);
4080                 rc = dev_cb.cb_fn(dev->data->port_id, dev_cb.event,
4081                                 dev_cb.cb_arg, dev_cb.ret_param);
4082                 rte_spinlock_lock(&rte_eth_dev_cb_lock);
4083                 cb_lst->active = 0;
4084         }
4085         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
4086         return rc;
4087 }
4088
4089 void
4090 rte_eth_dev_probing_finish(struct rte_eth_dev *dev)
4091 {
4092         if (dev == NULL)
4093                 return;
4094
4095         _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_NEW, NULL);
4096
4097         dev->state = RTE_ETH_DEV_ATTACHED;
4098 }
4099
4100 int
4101 rte_eth_dev_rx_intr_ctl(uint16_t port_id, int epfd, int op, void *data)
4102 {
4103         uint32_t vec;
4104         struct rte_eth_dev *dev;
4105         struct rte_intr_handle *intr_handle;
4106         uint16_t qid;
4107         int rc;
4108
4109         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4110
4111         dev = &rte_eth_devices[port_id];
4112
4113         if (!dev->intr_handle) {
4114                 RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
4115                 return -ENOTSUP;
4116         }
4117
4118         intr_handle = dev->intr_handle;
4119         if (!intr_handle->intr_vec) {
4120                 RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
4121                 return -EPERM;
4122         }
4123
4124         for (qid = 0; qid < dev->data->nb_rx_queues; qid++) {
4125                 vec = intr_handle->intr_vec[qid];
4126                 rc = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
4127                 if (rc && rc != -EEXIST) {
4128                         RTE_ETHDEV_LOG(ERR,
4129                                 "p %u q %u rx ctl error op %d epfd %d vec %u\n",
4130                                 port_id, qid, op, epfd, vec);
4131                 }
4132         }
4133
4134         return 0;
4135 }
4136
4137 int
4138 rte_eth_dev_rx_intr_ctl_q_get_fd(uint16_t port_id, uint16_t queue_id)
4139 {
4140         struct rte_intr_handle *intr_handle;
4141         struct rte_eth_dev *dev;
4142         unsigned int efd_idx;
4143         uint32_t vec;
4144         int fd;
4145
4146         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -1);
4147
4148         dev = &rte_eth_devices[port_id];
4149
4150         if (queue_id >= dev->data->nb_rx_queues) {
4151                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
4152                 return -1;
4153         }
4154
4155         if (!dev->intr_handle) {
4156                 RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
4157                 return -1;
4158         }
4159
4160         intr_handle = dev->intr_handle;
4161         if (!intr_handle->intr_vec) {
4162                 RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
4163                 return -1;
4164         }
4165
4166         vec = intr_handle->intr_vec[queue_id];
4167         efd_idx = (vec >= RTE_INTR_VEC_RXTX_OFFSET) ?
4168                 (vec - RTE_INTR_VEC_RXTX_OFFSET) : vec;
4169         fd = intr_handle->efds[efd_idx];
4170
4171         return fd;
4172 }
4173
4174 const struct rte_memzone *
4175 rte_eth_dma_zone_reserve(const struct rte_eth_dev *dev, const char *ring_name,
4176                          uint16_t queue_id, size_t size, unsigned align,
4177                          int socket_id)
4178 {
4179         char z_name[RTE_MEMZONE_NAMESIZE];
4180         const struct rte_memzone *mz;
4181         int rc;
4182
4183         rc = snprintf(z_name, sizeof(z_name), "eth_p%d_q%d_%s",
4184                       dev->data->port_id, queue_id, ring_name);
4185         if (rc >= RTE_MEMZONE_NAMESIZE) {
4186                 RTE_ETHDEV_LOG(ERR, "ring name too long\n");
4187                 rte_errno = ENAMETOOLONG;
4188                 return NULL;
4189         }
4190
4191         mz = rte_memzone_lookup(z_name);
4192         if (mz)
4193                 return mz;
4194
4195         return rte_memzone_reserve_aligned(z_name, size, socket_id,
4196                         RTE_MEMZONE_IOVA_CONTIG, align);
4197 }
4198
4199 int
4200 rte_eth_dev_create(struct rte_device *device, const char *name,
4201         size_t priv_data_size,
4202         ethdev_bus_specific_init ethdev_bus_specific_init,
4203         void *bus_init_params,
4204         ethdev_init_t ethdev_init, void *init_params)
4205 {
4206         struct rte_eth_dev *ethdev;
4207         int retval;
4208
4209         RTE_FUNC_PTR_OR_ERR_RET(*ethdev_init, -EINVAL);
4210
4211         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
4212                 ethdev = rte_eth_dev_allocate(name);
4213                 if (!ethdev)
4214                         return -ENODEV;
4215
4216                 if (priv_data_size) {
4217                         ethdev->data->dev_private = rte_zmalloc_socket(
4218                                 name, priv_data_size, RTE_CACHE_LINE_SIZE,
4219                                 device->numa_node);
4220
4221                         if (!ethdev->data->dev_private) {
4222                                 RTE_LOG(ERR, EAL, "failed to allocate private data");
4223                                 retval = -ENOMEM;
4224                                 goto probe_failed;
4225                         }
4226                 }
4227         } else {
4228                 ethdev = rte_eth_dev_attach_secondary(name);
4229                 if (!ethdev) {
4230                         RTE_LOG(ERR, EAL, "secondary process attach failed, "
4231                                 "ethdev doesn't exist");
4232                         return  -ENODEV;
4233                 }
4234         }
4235
4236         ethdev->device = device;
4237
4238         if (ethdev_bus_specific_init) {
4239                 retval = ethdev_bus_specific_init(ethdev, bus_init_params);
4240                 if (retval) {
4241                         RTE_LOG(ERR, EAL,
4242                                 "ethdev bus specific initialisation failed");
4243                         goto probe_failed;
4244                 }
4245         }
4246
4247         retval = ethdev_init(ethdev, init_params);
4248         if (retval) {
4249                 RTE_LOG(ERR, EAL, "ethdev initialisation failed");
4250                 goto probe_failed;
4251         }
4252
4253         rte_eth_dev_probing_finish(ethdev);
4254
4255         return retval;
4256
4257 probe_failed:
4258         rte_eth_dev_release_port(ethdev);
4259         return retval;
4260 }
4261
4262 int
4263 rte_eth_dev_destroy(struct rte_eth_dev *ethdev,
4264         ethdev_uninit_t ethdev_uninit)
4265 {
4266         int ret;
4267
4268         ethdev = rte_eth_dev_allocated(ethdev->data->name);
4269         if (!ethdev)
4270                 return -ENODEV;
4271
4272         RTE_FUNC_PTR_OR_ERR_RET(*ethdev_uninit, -EINVAL);
4273
4274         ret = ethdev_uninit(ethdev);
4275         if (ret)
4276                 return ret;
4277
4278         return rte_eth_dev_release_port(ethdev);
4279 }
4280
4281 int
4282 rte_eth_dev_rx_intr_ctl_q(uint16_t port_id, uint16_t queue_id,
4283                           int epfd, int op, void *data)
4284 {
4285         uint32_t vec;
4286         struct rte_eth_dev *dev;
4287         struct rte_intr_handle *intr_handle;
4288         int rc;
4289
4290         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4291
4292         dev = &rte_eth_devices[port_id];
4293         if (queue_id >= dev->data->nb_rx_queues) {
4294                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
4295                 return -EINVAL;
4296         }
4297
4298         if (!dev->intr_handle) {
4299                 RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
4300                 return -ENOTSUP;
4301         }
4302
4303         intr_handle = dev->intr_handle;
4304         if (!intr_handle->intr_vec) {
4305                 RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
4306                 return -EPERM;
4307         }
4308
4309         vec = intr_handle->intr_vec[queue_id];
4310         rc = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
4311         if (rc && rc != -EEXIST) {
4312                 RTE_ETHDEV_LOG(ERR,
4313                         "p %u q %u rx ctl error op %d epfd %d vec %u\n",
4314                         port_id, queue_id, op, epfd, vec);
4315                 return rc;
4316         }
4317
4318         return 0;
4319 }
4320
4321 int
4322 rte_eth_dev_rx_intr_enable(uint16_t port_id,
4323                            uint16_t queue_id)
4324 {
4325         struct rte_eth_dev *dev;
4326
4327         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4328
4329         dev = &rte_eth_devices[port_id];
4330
4331         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_intr_enable, -ENOTSUP);
4332         return eth_err(port_id, (*dev->dev_ops->rx_queue_intr_enable)(dev,
4333                                                                 queue_id));
4334 }
4335
4336 int
4337 rte_eth_dev_rx_intr_disable(uint16_t port_id,
4338                             uint16_t queue_id)
4339 {
4340         struct rte_eth_dev *dev;
4341
4342         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4343
4344         dev = &rte_eth_devices[port_id];
4345
4346         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_intr_disable, -ENOTSUP);
4347         return eth_err(port_id, (*dev->dev_ops->rx_queue_intr_disable)(dev,
4348                                                                 queue_id));
4349 }
4350
4351
4352 int
4353 rte_eth_dev_filter_supported(uint16_t port_id,
4354                              enum rte_filter_type filter_type)
4355 {
4356         struct rte_eth_dev *dev;
4357
4358         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4359
4360         dev = &rte_eth_devices[port_id];
4361         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->filter_ctrl, -ENOTSUP);
4362         return (*dev->dev_ops->filter_ctrl)(dev, filter_type,
4363                                 RTE_ETH_FILTER_NOP, NULL);
4364 }
4365
4366 int
4367 rte_eth_dev_filter_ctrl(uint16_t port_id, enum rte_filter_type filter_type,
4368                         enum rte_filter_op filter_op, void *arg)
4369 {
4370         struct rte_eth_dev *dev;
4371
4372         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4373
4374         dev = &rte_eth_devices[port_id];
4375         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->filter_ctrl, -ENOTSUP);
4376         return eth_err(port_id, (*dev->dev_ops->filter_ctrl)(dev, filter_type,
4377                                                              filter_op, arg));
4378 }
4379
4380 const struct rte_eth_rxtx_callback *
4381 rte_eth_add_rx_callback(uint16_t port_id, uint16_t queue_id,
4382                 rte_rx_callback_fn fn, void *user_param)
4383 {
4384 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
4385         rte_errno = ENOTSUP;
4386         return NULL;
4387 #endif
4388         struct rte_eth_dev *dev;
4389
4390         /* check input parameters */
4391         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
4392                     queue_id >= rte_eth_devices[port_id].data->nb_rx_queues) {
4393                 rte_errno = EINVAL;
4394                 return NULL;
4395         }
4396         dev = &rte_eth_devices[port_id];
4397         if (rte_eth_dev_is_rx_hairpin_queue(dev, queue_id)) {
4398                 rte_errno = EINVAL;
4399                 return NULL;
4400         }
4401         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
4402
4403         if (cb == NULL) {
4404                 rte_errno = ENOMEM;
4405                 return NULL;
4406         }
4407
4408         cb->fn.rx = fn;
4409         cb->param = user_param;
4410
4411         rte_spinlock_lock(&rte_eth_rx_cb_lock);
4412         /* Add the callbacks in fifo order. */
4413         struct rte_eth_rxtx_callback *tail =
4414                 rte_eth_devices[port_id].post_rx_burst_cbs[queue_id];
4415
4416         if (!tail) {
4417                 rte_eth_devices[port_id].post_rx_burst_cbs[queue_id] = cb;
4418
4419         } else {
4420                 while (tail->next)
4421                         tail = tail->next;
4422                 tail->next = cb;
4423         }
4424         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
4425
4426         return cb;
4427 }
4428
4429 const struct rte_eth_rxtx_callback *
4430 rte_eth_add_first_rx_callback(uint16_t port_id, uint16_t queue_id,
4431                 rte_rx_callback_fn fn, void *user_param)
4432 {
4433 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
4434         rte_errno = ENOTSUP;
4435         return NULL;
4436 #endif
4437         /* check input parameters */
4438         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
4439                 queue_id >= rte_eth_devices[port_id].data->nb_rx_queues) {
4440                 rte_errno = EINVAL;
4441                 return NULL;
4442         }
4443
4444         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
4445
4446         if (cb == NULL) {
4447                 rte_errno = ENOMEM;
4448                 return NULL;
4449         }
4450
4451         cb->fn.rx = fn;
4452         cb->param = user_param;
4453
4454         rte_spinlock_lock(&rte_eth_rx_cb_lock);
4455         /* Add the callbacks at first position */
4456         cb->next = rte_eth_devices[port_id].post_rx_burst_cbs[queue_id];
4457         rte_smp_wmb();
4458         rte_eth_devices[port_id].post_rx_burst_cbs[queue_id] = cb;
4459         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
4460
4461         return cb;
4462 }
4463
4464 const struct rte_eth_rxtx_callback *
4465 rte_eth_add_tx_callback(uint16_t port_id, uint16_t queue_id,
4466                 rte_tx_callback_fn fn, void *user_param)
4467 {
4468 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
4469         rte_errno = ENOTSUP;
4470         return NULL;
4471 #endif
4472         struct rte_eth_dev *dev;
4473
4474         /* check input parameters */
4475         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
4476                     queue_id >= rte_eth_devices[port_id].data->nb_tx_queues) {
4477                 rte_errno = EINVAL;
4478                 return NULL;
4479         }
4480
4481         dev = &rte_eth_devices[port_id];
4482         if (rte_eth_dev_is_tx_hairpin_queue(dev, queue_id)) {
4483                 rte_errno = EINVAL;
4484                 return NULL;
4485         }
4486
4487         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
4488
4489         if (cb == NULL) {
4490                 rte_errno = ENOMEM;
4491                 return NULL;
4492         }
4493
4494         cb->fn.tx = fn;
4495         cb->param = user_param;
4496
4497         rte_spinlock_lock(&rte_eth_tx_cb_lock);
4498         /* Add the callbacks in fifo order. */
4499         struct rte_eth_rxtx_callback *tail =
4500                 rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id];
4501
4502         if (!tail) {
4503                 rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id] = cb;
4504
4505         } else {
4506                 while (tail->next)
4507                         tail = tail->next;
4508                 tail->next = cb;
4509         }
4510         rte_spinlock_unlock(&rte_eth_tx_cb_lock);
4511
4512         return cb;
4513 }
4514
4515 int
4516 rte_eth_remove_rx_callback(uint16_t port_id, uint16_t queue_id,
4517                 const struct rte_eth_rxtx_callback *user_cb)
4518 {
4519 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
4520         return -ENOTSUP;
4521 #endif
4522         /* Check input parameters. */
4523         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
4524         if (user_cb == NULL ||
4525                         queue_id >= rte_eth_devices[port_id].data->nb_rx_queues)
4526                 return -EINVAL;
4527
4528         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
4529         struct rte_eth_rxtx_callback *cb;
4530         struct rte_eth_rxtx_callback **prev_cb;
4531         int ret = -EINVAL;
4532
4533         rte_spinlock_lock(&rte_eth_rx_cb_lock);
4534         prev_cb = &dev->post_rx_burst_cbs[queue_id];
4535         for (; *prev_cb != NULL; prev_cb = &cb->next) {
4536                 cb = *prev_cb;
4537                 if (cb == user_cb) {
4538                         /* Remove the user cb from the callback list. */
4539                         *prev_cb = cb->next;
4540                         ret = 0;
4541                         break;
4542                 }
4543         }
4544         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
4545
4546         return ret;
4547 }
4548
4549 int
4550 rte_eth_remove_tx_callback(uint16_t port_id, uint16_t queue_id,
4551                 const struct rte_eth_rxtx_callback *user_cb)
4552 {
4553 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
4554         return -ENOTSUP;
4555 #endif
4556         /* Check input parameters. */
4557         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
4558         if (user_cb == NULL ||
4559                         queue_id >= rte_eth_devices[port_id].data->nb_tx_queues)
4560                 return -EINVAL;
4561
4562         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
4563         int ret = -EINVAL;
4564         struct rte_eth_rxtx_callback *cb;
4565         struct rte_eth_rxtx_callback **prev_cb;
4566
4567         rte_spinlock_lock(&rte_eth_tx_cb_lock);
4568         prev_cb = &dev->pre_tx_burst_cbs[queue_id];
4569         for (; *prev_cb != NULL; prev_cb = &cb->next) {
4570                 cb = *prev_cb;
4571                 if (cb == user_cb) {
4572                         /* Remove the user cb from the callback list. */
4573                         *prev_cb = cb->next;
4574                         ret = 0;
4575                         break;
4576                 }
4577         }
4578         rte_spinlock_unlock(&rte_eth_tx_cb_lock);
4579
4580         return ret;
4581 }
4582
4583 int
4584 rte_eth_rx_queue_info_get(uint16_t port_id, uint16_t queue_id,
4585         struct rte_eth_rxq_info *qinfo)
4586 {
4587         struct rte_eth_dev *dev;
4588
4589         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4590
4591         if (qinfo == NULL)
4592                 return -EINVAL;
4593
4594         dev = &rte_eth_devices[port_id];
4595         if (queue_id >= dev->data->nb_rx_queues) {
4596                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
4597                 return -EINVAL;
4598         }
4599
4600         if (rte_eth_dev_is_rx_hairpin_queue(dev, queue_id)) {
4601                 RTE_ETHDEV_LOG(INFO,
4602                         "Can't get hairpin Rx queue %"PRIu16" info of device with port_id=%"PRIu16"\n",
4603                         queue_id, port_id);
4604                 return -EINVAL;
4605         }
4606
4607         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rxq_info_get, -ENOTSUP);
4608
4609         memset(qinfo, 0, sizeof(*qinfo));
4610         dev->dev_ops->rxq_info_get(dev, queue_id, qinfo);
4611         return 0;
4612 }
4613
4614 int
4615 rte_eth_tx_queue_info_get(uint16_t port_id, uint16_t queue_id,
4616         struct rte_eth_txq_info *qinfo)
4617 {
4618         struct rte_eth_dev *dev;
4619
4620         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4621
4622         if (qinfo == NULL)
4623                 return -EINVAL;
4624
4625         dev = &rte_eth_devices[port_id];
4626         if (queue_id >= dev->data->nb_tx_queues) {
4627                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", queue_id);
4628                 return -EINVAL;
4629         }
4630
4631         if (rte_eth_dev_is_tx_hairpin_queue(dev, queue_id)) {
4632                 RTE_ETHDEV_LOG(INFO,
4633                         "Can't get hairpin Tx queue %"PRIu16" info of device with port_id=%"PRIu16"\n",
4634                         queue_id, port_id);
4635                 return -EINVAL;
4636         }
4637
4638         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->txq_info_get, -ENOTSUP);
4639
4640         memset(qinfo, 0, sizeof(*qinfo));
4641         dev->dev_ops->txq_info_get(dev, queue_id, qinfo);
4642
4643         return 0;
4644 }
4645
4646 int
4647 rte_eth_rx_burst_mode_get(uint16_t port_id, uint16_t queue_id,
4648                           struct rte_eth_burst_mode *mode)
4649 {
4650         struct rte_eth_dev *dev;
4651
4652         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4653
4654         if (mode == NULL)
4655                 return -EINVAL;
4656
4657         dev = &rte_eth_devices[port_id];
4658
4659         if (queue_id >= dev->data->nb_rx_queues) {
4660                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
4661                 return -EINVAL;
4662         }
4663
4664         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_burst_mode_get, -ENOTSUP);
4665         memset(mode, 0, sizeof(*mode));
4666         return eth_err(port_id,
4667                        dev->dev_ops->rx_burst_mode_get(dev, queue_id, mode));
4668 }
4669
4670 int
4671 rte_eth_tx_burst_mode_get(uint16_t port_id, uint16_t queue_id,
4672                           struct rte_eth_burst_mode *mode)
4673 {
4674         struct rte_eth_dev *dev;
4675
4676         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4677
4678         if (mode == NULL)
4679                 return -EINVAL;
4680
4681         dev = &rte_eth_devices[port_id];
4682
4683         if (queue_id >= dev->data->nb_tx_queues) {
4684                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", queue_id);
4685                 return -EINVAL;
4686         }
4687
4688         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_burst_mode_get, -ENOTSUP);
4689         memset(mode, 0, sizeof(*mode));
4690         return eth_err(port_id,
4691                        dev->dev_ops->tx_burst_mode_get(dev, queue_id, mode));
4692 }
4693
4694 int
4695 rte_eth_dev_set_mc_addr_list(uint16_t port_id,
4696                              struct rte_ether_addr *mc_addr_set,
4697                              uint32_t nb_mc_addr)
4698 {
4699         struct rte_eth_dev *dev;
4700
4701         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4702
4703         dev = &rte_eth_devices[port_id];
4704         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_mc_addr_list, -ENOTSUP);
4705         return eth_err(port_id, dev->dev_ops->set_mc_addr_list(dev,
4706                                                 mc_addr_set, nb_mc_addr));
4707 }
4708
4709 int
4710 rte_eth_timesync_enable(uint16_t port_id)
4711 {
4712         struct rte_eth_dev *dev;
4713
4714         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4715         dev = &rte_eth_devices[port_id];
4716
4717         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_enable, -ENOTSUP);
4718         return eth_err(port_id, (*dev->dev_ops->timesync_enable)(dev));
4719 }
4720
4721 int
4722 rte_eth_timesync_disable(uint16_t port_id)
4723 {
4724         struct rte_eth_dev *dev;
4725
4726         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4727         dev = &rte_eth_devices[port_id];
4728
4729         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_disable, -ENOTSUP);
4730         return eth_err(port_id, (*dev->dev_ops->timesync_disable)(dev));
4731 }
4732
4733 int
4734 rte_eth_timesync_read_rx_timestamp(uint16_t port_id, struct timespec *timestamp,
4735                                    uint32_t flags)
4736 {
4737         struct rte_eth_dev *dev;
4738
4739         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4740         dev = &rte_eth_devices[port_id];
4741
4742         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_rx_timestamp, -ENOTSUP);
4743         return eth_err(port_id, (*dev->dev_ops->timesync_read_rx_timestamp)
4744                                 (dev, timestamp, flags));
4745 }
4746
4747 int
4748 rte_eth_timesync_read_tx_timestamp(uint16_t port_id,
4749                                    struct timespec *timestamp)
4750 {
4751         struct rte_eth_dev *dev;
4752
4753         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4754         dev = &rte_eth_devices[port_id];
4755
4756         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_tx_timestamp, -ENOTSUP);
4757         return eth_err(port_id, (*dev->dev_ops->timesync_read_tx_timestamp)
4758                                 (dev, timestamp));
4759 }
4760
4761 int
4762 rte_eth_timesync_adjust_time(uint16_t port_id, int64_t delta)
4763 {
4764         struct rte_eth_dev *dev;
4765
4766         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4767         dev = &rte_eth_devices[port_id];
4768
4769         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_adjust_time, -ENOTSUP);
4770         return eth_err(port_id, (*dev->dev_ops->timesync_adjust_time)(dev,
4771                                                                       delta));
4772 }
4773
4774 int
4775 rte_eth_timesync_read_time(uint16_t port_id, struct timespec *timestamp)
4776 {
4777         struct rte_eth_dev *dev;
4778
4779         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4780         dev = &rte_eth_devices[port_id];
4781
4782         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_time, -ENOTSUP);
4783         return eth_err(port_id, (*dev->dev_ops->timesync_read_time)(dev,
4784                                                                 timestamp));
4785 }
4786
4787 int
4788 rte_eth_timesync_write_time(uint16_t port_id, const struct timespec *timestamp)
4789 {
4790         struct rte_eth_dev *dev;
4791
4792         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4793         dev = &rte_eth_devices[port_id];
4794
4795         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_write_time, -ENOTSUP);
4796         return eth_err(port_id, (*dev->dev_ops->timesync_write_time)(dev,
4797                                                                 timestamp));
4798 }
4799
4800 int
4801 rte_eth_read_clock(uint16_t port_id, uint64_t *clock)
4802 {
4803         struct rte_eth_dev *dev;
4804
4805         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4806         dev = &rte_eth_devices[port_id];
4807
4808         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->read_clock, -ENOTSUP);
4809         return eth_err(port_id, (*dev->dev_ops->read_clock)(dev, clock));
4810 }
4811
4812 int
4813 rte_eth_dev_get_reg_info(uint16_t port_id, struct rte_dev_reg_info *info)
4814 {
4815         struct rte_eth_dev *dev;
4816
4817         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4818
4819         dev = &rte_eth_devices[port_id];
4820         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_reg, -ENOTSUP);
4821         return eth_err(port_id, (*dev->dev_ops->get_reg)(dev, info));
4822 }
4823
4824 int
4825 rte_eth_dev_get_eeprom_length(uint16_t port_id)
4826 {
4827         struct rte_eth_dev *dev;
4828
4829         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4830
4831         dev = &rte_eth_devices[port_id];
4832         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom_length, -ENOTSUP);
4833         return eth_err(port_id, (*dev->dev_ops->get_eeprom_length)(dev));
4834 }
4835
4836 int
4837 rte_eth_dev_get_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info)
4838 {
4839         struct rte_eth_dev *dev;
4840
4841         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4842
4843         dev = &rte_eth_devices[port_id];
4844         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom, -ENOTSUP);
4845         return eth_err(port_id, (*dev->dev_ops->get_eeprom)(dev, info));
4846 }
4847
4848 int
4849 rte_eth_dev_set_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info)
4850 {
4851         struct rte_eth_dev *dev;
4852
4853         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4854
4855         dev = &rte_eth_devices[port_id];
4856         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_eeprom, -ENOTSUP);
4857         return eth_err(port_id, (*dev->dev_ops->set_eeprom)(dev, info));
4858 }
4859
4860 int
4861 rte_eth_dev_get_module_info(uint16_t port_id,
4862                             struct rte_eth_dev_module_info *modinfo)
4863 {
4864         struct rte_eth_dev *dev;
4865
4866         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4867
4868         dev = &rte_eth_devices[port_id];
4869         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_module_info, -ENOTSUP);
4870         return (*dev->dev_ops->get_module_info)(dev, modinfo);
4871 }
4872
4873 int
4874 rte_eth_dev_get_module_eeprom(uint16_t port_id,
4875                               struct rte_dev_eeprom_info *info)
4876 {
4877         struct rte_eth_dev *dev;
4878
4879         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4880
4881         dev = &rte_eth_devices[port_id];
4882         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_module_eeprom, -ENOTSUP);
4883         return (*dev->dev_ops->get_module_eeprom)(dev, info);
4884 }
4885
4886 int
4887 rte_eth_dev_get_dcb_info(uint16_t port_id,
4888                              struct rte_eth_dcb_info *dcb_info)
4889 {
4890         struct rte_eth_dev *dev;
4891
4892         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4893
4894         dev = &rte_eth_devices[port_id];
4895         memset(dcb_info, 0, sizeof(struct rte_eth_dcb_info));
4896
4897         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_dcb_info, -ENOTSUP);
4898         return eth_err(port_id, (*dev->dev_ops->get_dcb_info)(dev, dcb_info));
4899 }
4900
4901 int
4902 rte_eth_dev_l2_tunnel_eth_type_conf(uint16_t port_id,
4903                                     struct rte_eth_l2_tunnel_conf *l2_tunnel)
4904 {
4905         struct rte_eth_dev *dev;
4906
4907         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4908         if (l2_tunnel == NULL) {
4909                 RTE_ETHDEV_LOG(ERR, "Invalid l2_tunnel parameter\n");
4910                 return -EINVAL;
4911         }
4912
4913         if (l2_tunnel->l2_tunnel_type >= RTE_TUNNEL_TYPE_MAX) {
4914                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
4915                 return -EINVAL;
4916         }
4917
4918         dev = &rte_eth_devices[port_id];
4919         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->l2_tunnel_eth_type_conf,
4920                                 -ENOTSUP);
4921         return eth_err(port_id, (*dev->dev_ops->l2_tunnel_eth_type_conf)(dev,
4922                                                                 l2_tunnel));
4923 }
4924
4925 int
4926 rte_eth_dev_l2_tunnel_offload_set(uint16_t port_id,
4927                                   struct rte_eth_l2_tunnel_conf *l2_tunnel,
4928                                   uint32_t mask,
4929                                   uint8_t en)
4930 {
4931         struct rte_eth_dev *dev;
4932
4933         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4934
4935         if (l2_tunnel == NULL) {
4936                 RTE_ETHDEV_LOG(ERR, "Invalid l2_tunnel parameter\n");
4937                 return -EINVAL;
4938         }
4939
4940         if (l2_tunnel->l2_tunnel_type >= RTE_TUNNEL_TYPE_MAX) {
4941                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
4942                 return -EINVAL;
4943         }
4944
4945         if (mask == 0) {
4946                 RTE_ETHDEV_LOG(ERR, "Mask should have a value\n");
4947                 return -EINVAL;
4948         }
4949
4950         dev = &rte_eth_devices[port_id];
4951         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->l2_tunnel_offload_set,
4952                                 -ENOTSUP);
4953         return eth_err(port_id, (*dev->dev_ops->l2_tunnel_offload_set)(dev,
4954                                                         l2_tunnel, mask, en));
4955 }
4956
4957 static void
4958 rte_eth_dev_adjust_nb_desc(uint16_t *nb_desc,
4959                            const struct rte_eth_desc_lim *desc_lim)
4960 {
4961         if (desc_lim->nb_align != 0)
4962                 *nb_desc = RTE_ALIGN_CEIL(*nb_desc, desc_lim->nb_align);
4963
4964         if (desc_lim->nb_max != 0)
4965                 *nb_desc = RTE_MIN(*nb_desc, desc_lim->nb_max);
4966
4967         *nb_desc = RTE_MAX(*nb_desc, desc_lim->nb_min);
4968 }
4969
4970 int
4971 rte_eth_dev_adjust_nb_rx_tx_desc(uint16_t port_id,
4972                                  uint16_t *nb_rx_desc,
4973                                  uint16_t *nb_tx_desc)
4974 {
4975         struct rte_eth_dev_info dev_info;
4976         int ret;
4977
4978         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4979
4980         ret = rte_eth_dev_info_get(port_id, &dev_info);
4981         if (ret != 0)
4982                 return ret;
4983
4984         if (nb_rx_desc != NULL)
4985                 rte_eth_dev_adjust_nb_desc(nb_rx_desc, &dev_info.rx_desc_lim);
4986
4987         if (nb_tx_desc != NULL)
4988                 rte_eth_dev_adjust_nb_desc(nb_tx_desc, &dev_info.tx_desc_lim);
4989
4990         return 0;
4991 }
4992
4993 int
4994 rte_eth_dev_hairpin_capability_get(uint16_t port_id,
4995                                    struct rte_eth_hairpin_cap *cap)
4996 {
4997         struct rte_eth_dev *dev;
4998
4999         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
5000
5001         dev = &rte_eth_devices[port_id];
5002         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->hairpin_cap_get, -ENOTSUP);
5003         memset(cap, 0, sizeof(*cap));
5004         return eth_err(port_id, (*dev->dev_ops->hairpin_cap_get)(dev, cap));
5005 }
5006
5007 int
5008 rte_eth_dev_is_rx_hairpin_queue(struct rte_eth_dev *dev, uint16_t queue_id)
5009 {
5010         if (dev->data->rx_queue_state[queue_id] ==
5011             RTE_ETH_QUEUE_STATE_HAIRPIN)
5012                 return 1;
5013         return 0;
5014 }
5015
5016 int
5017 rte_eth_dev_is_tx_hairpin_queue(struct rte_eth_dev *dev, uint16_t queue_id)
5018 {
5019         if (dev->data->tx_queue_state[queue_id] ==
5020             RTE_ETH_QUEUE_STATE_HAIRPIN)
5021                 return 1;
5022         return 0;
5023 }
5024
5025 int
5026 rte_eth_dev_pool_ops_supported(uint16_t port_id, const char *pool)
5027 {
5028         struct rte_eth_dev *dev;
5029
5030         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5031
5032         if (pool == NULL)
5033                 return -EINVAL;
5034
5035         dev = &rte_eth_devices[port_id];
5036
5037         if (*dev->dev_ops->pool_ops_supported == NULL)
5038                 return 1; /* all pools are supported */
5039
5040         return (*dev->dev_ops->pool_ops_supported)(dev, pool);
5041 }
5042
5043 /**
5044  * A set of values to describe the possible states of a switch domain.
5045  */
5046 enum rte_eth_switch_domain_state {
5047         RTE_ETH_SWITCH_DOMAIN_UNUSED = 0,
5048         RTE_ETH_SWITCH_DOMAIN_ALLOCATED
5049 };
5050
5051 /**
5052  * Array of switch domains available for allocation. Array is sized to
5053  * RTE_MAX_ETHPORTS elements as there cannot be more active switch domains than
5054  * ethdev ports in a single process.
5055  */
5056 static struct rte_eth_dev_switch {
5057         enum rte_eth_switch_domain_state state;
5058 } rte_eth_switch_domains[RTE_MAX_ETHPORTS];
5059
5060 int
5061 rte_eth_switch_domain_alloc(uint16_t *domain_id)
5062 {
5063         unsigned int i;
5064
5065         *domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
5066
5067         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
5068                 if (rte_eth_switch_domains[i].state ==
5069                         RTE_ETH_SWITCH_DOMAIN_UNUSED) {
5070                         rte_eth_switch_domains[i].state =
5071                                 RTE_ETH_SWITCH_DOMAIN_ALLOCATED;
5072                         *domain_id = i;
5073                         return 0;
5074                 }
5075         }
5076
5077         return -ENOSPC;
5078 }
5079
5080 int
5081 rte_eth_switch_domain_free(uint16_t domain_id)
5082 {
5083         if (domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID ||
5084                 domain_id >= RTE_MAX_ETHPORTS)
5085                 return -EINVAL;
5086
5087         if (rte_eth_switch_domains[domain_id].state !=
5088                 RTE_ETH_SWITCH_DOMAIN_ALLOCATED)
5089                 return -EINVAL;
5090
5091         rte_eth_switch_domains[domain_id].state = RTE_ETH_SWITCH_DOMAIN_UNUSED;
5092
5093         return 0;
5094 }
5095
5096 static int
5097 rte_eth_devargs_tokenise(struct rte_kvargs *arglist, const char *str_in)
5098 {
5099         int state;
5100         struct rte_kvargs_pair *pair;
5101         char *letter;
5102
5103         arglist->str = strdup(str_in);
5104         if (arglist->str == NULL)
5105                 return -ENOMEM;
5106
5107         letter = arglist->str;
5108         state = 0;
5109         arglist->count = 0;
5110         pair = &arglist->pairs[0];
5111         while (1) {
5112                 switch (state) {
5113                 case 0: /* Initial */
5114                         if (*letter == '=')
5115                                 return -EINVAL;
5116                         else if (*letter == '\0')
5117                                 return 0;
5118
5119                         state = 1;
5120                         pair->key = letter;
5121                         /* fall-thru */
5122
5123                 case 1: /* Parsing key */
5124                         if (*letter == '=') {
5125                                 *letter = '\0';
5126                                 pair->value = letter + 1;
5127                                 state = 2;
5128                         } else if (*letter == ',' || *letter == '\0')
5129                                 return -EINVAL;
5130                         break;
5131
5132
5133                 case 2: /* Parsing value */
5134                         if (*letter == '[')
5135                                 state = 3;
5136                         else if (*letter == ',') {
5137                                 *letter = '\0';
5138                                 arglist->count++;
5139                                 pair = &arglist->pairs[arglist->count];
5140                                 state = 0;
5141                         } else if (*letter == '\0') {
5142                                 letter--;
5143                                 arglist->count++;
5144                                 pair = &arglist->pairs[arglist->count];
5145                                 state = 0;
5146                         }
5147                         break;
5148
5149                 case 3: /* Parsing list */
5150                         if (*letter == ']')
5151                                 state = 2;
5152                         else if (*letter == '\0')
5153                                 return -EINVAL;
5154                         break;
5155                 }
5156                 letter++;
5157         }
5158 }
5159
5160 int
5161 rte_eth_devargs_parse(const char *dargs, struct rte_eth_devargs *eth_da)
5162 {
5163         struct rte_kvargs args;
5164         struct rte_kvargs_pair *pair;
5165         unsigned int i;
5166         int result = 0;
5167
5168         memset(eth_da, 0, sizeof(*eth_da));
5169
5170         result = rte_eth_devargs_tokenise(&args, dargs);
5171         if (result < 0)
5172                 goto parse_cleanup;
5173
5174         for (i = 0; i < args.count; i++) {
5175                 pair = &args.pairs[i];
5176                 if (strcmp("representor", pair->key) == 0) {
5177                         result = rte_eth_devargs_parse_list(pair->value,
5178                                 rte_eth_devargs_parse_representor_ports,
5179                                 eth_da);
5180                         if (result < 0)
5181                                 goto parse_cleanup;
5182                 }
5183         }
5184
5185 parse_cleanup:
5186         if (args.str)
5187                 free(args.str);
5188
5189         return result;
5190 }
5191
5192 RTE_INIT(ethdev_init_log)
5193 {
5194         rte_eth_dev_logtype = rte_log_register("lib.ethdev");
5195         if (rte_eth_dev_logtype >= 0)
5196                 rte_log_set_level(rte_eth_dev_logtype, RTE_LOG_INFO);
5197 }