lib/librte_ethdev/rte_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2010-2017 Intel Corporation
   3  */
   4
   5 #include <sys/types.h>
   6 #include <sys/queue.h>
   7 #include <ctype.h>
   8 #include <stdio.h>
   9 #include <stdlib.h>
  10 #include <string.h>
  11 #include <stdarg.h>
  12 #include <errno.h>
  13 #include <stdbool.h>
  14 #include <stdint.h>
  15 #include <inttypes.h>
  16 #include <netinet/in.h>
  17
  18 #include <rte_byteorder.h>
  19 #include <rte_log.h>
  20 #include <rte_debug.h>
  21 #include <rte_interrupts.h>
  22 #include <rte_memory.h>
  23 #include <rte_memcpy.h>
  24 #include <rte_memzone.h>
  25 #include <rte_launch.h>
  26 #include <rte_eal.h>
  27 #include <rte_per_lcore.h>
  28 #include <rte_lcore.h>
  29 #include <rte_branch_prediction.h>
  30 #include <rte_common.h>
  31 #include <rte_mempool.h>
  32 #include <rte_malloc.h>
  33 #include <rte_mbuf.h>
  34 #include <rte_errno.h>
  35 #include <rte_spinlock.h>
  36 #include <rte_string_fns.h>
  37 #include <rte_kvargs.h>
  38 #include <rte_class.h>
  39 #include <rte_ether.h>
  40 #include <rte_telemetry.h>
  41
  42 #include "rte_ethdev_trace.h"
  43 #include "rte_ethdev.h"
  44 #include "rte_ethdev_driver.h"
  45 #include "ethdev_profile.h"
  46 #include "ethdev_private.h"
  47
  48 static const char *MZ_RTE_ETH_DEV_DATA = "rte_eth_dev_data";
  49 struct rte_eth_dev rte_eth_devices[RTE_MAX_ETHPORTS];
  50
  51 /* spinlock for eth device callbacks */
  52 static rte_spinlock_t rte_eth_dev_cb_lock = RTE_SPINLOCK_INITIALIZER;
  53
  54 /* spinlock for add/remove rx callbacks */
  55 static rte_spinlock_t rte_eth_rx_cb_lock = RTE_SPINLOCK_INITIALIZER;
  56
  57 /* spinlock for add/remove tx callbacks */
  58 static rte_spinlock_t rte_eth_tx_cb_lock = RTE_SPINLOCK_INITIALIZER;
  59
  60 /* spinlock for shared data allocation */
  61 static rte_spinlock_t rte_eth_shared_data_lock = RTE_SPINLOCK_INITIALIZER;
  62
  63 /* store statistics names and its offset in stats structure  */
  64 struct rte_eth_xstats_name_off {
  65         char name[RTE_ETH_XSTATS_NAME_SIZE];
  66         unsigned offset;
  67 };
  68
  69 /* Shared memory between primary and secondary processes. */
  70 static struct {
  71         uint64_t next_owner_id;
  72         rte_spinlock_t ownership_lock;
  73         struct rte_eth_dev_data data[RTE_MAX_ETHPORTS];
  74 } *rte_eth_dev_shared_data;
  75
  76 static const struct rte_eth_xstats_name_off rte_stats_strings[] = {
  77         {"rx_good_packets", offsetof(struct rte_eth_stats, ipackets)},
  78         {"tx_good_packets", offsetof(struct rte_eth_stats, opackets)},
  79         {"rx_good_bytes", offsetof(struct rte_eth_stats, ibytes)},
  80         {"tx_good_bytes", offsetof(struct rte_eth_stats, obytes)},
  81         {"rx_missed_errors", offsetof(struct rte_eth_stats, imissed)},
  82         {"rx_errors", offsetof(struct rte_eth_stats, ierrors)},
  83         {"tx_errors", offsetof(struct rte_eth_stats, oerrors)},
  84         {"rx_mbuf_allocation_errors", offsetof(struct rte_eth_stats,
  85                 rx_nombuf)},
  86 };
  87
  88 #define RTE_NB_STATS RTE_DIM(rte_stats_strings)
  89
  90 static const struct rte_eth_xstats_name_off rte_rxq_stats_strings[] = {
  91         {"packets", offsetof(struct rte_eth_stats, q_ipackets)},
  92         {"bytes", offsetof(struct rte_eth_stats, q_ibytes)},
  93         {"errors", offsetof(struct rte_eth_stats, q_errors)},
  94 };
  95
  96 #define RTE_NB_RXQ_STATS RTE_DIM(rte_rxq_stats_strings)
  97
  98 static const struct rte_eth_xstats_name_off rte_txq_stats_strings[] = {
  99         {"packets", offsetof(struct rte_eth_stats, q_opackets)},
 100         {"bytes", offsetof(struct rte_eth_stats, q_obytes)},
 101 };
 102 #define RTE_NB_TXQ_STATS RTE_DIM(rte_txq_stats_strings)
 103
 104 #define RTE_RX_OFFLOAD_BIT2STR(_name)   \
 105         { DEV_RX_OFFLOAD_##_name, #_name }
 106
 107 #define RTE_ETH_RX_OFFLOAD_BIT2STR(_name)       \
 108         { RTE_ETH_RX_OFFLOAD_##_name, #_name }
 109
 110 static const struct {
 111         uint64_t offload;
 112         const char *name;
 113 } rte_rx_offload_names[] = {
 114         RTE_RX_OFFLOAD_BIT2STR(VLAN_STRIP),
 115         RTE_RX_OFFLOAD_BIT2STR(IPV4_CKSUM),
 116         RTE_RX_OFFLOAD_BIT2STR(UDP_CKSUM),
 117         RTE_RX_OFFLOAD_BIT2STR(TCP_CKSUM),
 118         RTE_RX_OFFLOAD_BIT2STR(TCP_LRO),
 119         RTE_RX_OFFLOAD_BIT2STR(QINQ_STRIP),
 120         RTE_RX_OFFLOAD_BIT2STR(OUTER_IPV4_CKSUM),
 121         RTE_RX_OFFLOAD_BIT2STR(MACSEC_STRIP),
 122         RTE_RX_OFFLOAD_BIT2STR(HEADER_SPLIT),
 123         RTE_RX_OFFLOAD_BIT2STR(VLAN_FILTER),
 124         RTE_RX_OFFLOAD_BIT2STR(VLAN_EXTEND),
 125         RTE_RX_OFFLOAD_BIT2STR(JUMBO_FRAME),
 126         RTE_RX_OFFLOAD_BIT2STR(SCATTER),
 127         RTE_RX_OFFLOAD_BIT2STR(TIMESTAMP),
 128         RTE_RX_OFFLOAD_BIT2STR(SECURITY),
 129         RTE_RX_OFFLOAD_BIT2STR(KEEP_CRC),
 130         RTE_RX_OFFLOAD_BIT2STR(SCTP_CKSUM),
 131         RTE_RX_OFFLOAD_BIT2STR(OUTER_UDP_CKSUM),
 132         RTE_RX_OFFLOAD_BIT2STR(RSS_HASH),
 133         RTE_ETH_RX_OFFLOAD_BIT2STR(BUFFER_SPLIT),
 134 };
 135
 136 #undef RTE_RX_OFFLOAD_BIT2STR
 137 #undef RTE_ETH_RX_OFFLOAD_BIT2STR
 138
 139 #define RTE_TX_OFFLOAD_BIT2STR(_name)   \
 140         { DEV_TX_OFFLOAD_##_name, #_name }
 141
 142 static const struct {
 143         uint64_t offload;
 144         const char *name;
 145 } rte_tx_offload_names[] = {
 146         RTE_TX_OFFLOAD_BIT2STR(VLAN_INSERT),
 147         RTE_TX_OFFLOAD_BIT2STR(IPV4_CKSUM),
 148         RTE_TX_OFFLOAD_BIT2STR(UDP_CKSUM),
 149         RTE_TX_OFFLOAD_BIT2STR(TCP_CKSUM),
 150         RTE_TX_OFFLOAD_BIT2STR(SCTP_CKSUM),
 151         RTE_TX_OFFLOAD_BIT2STR(TCP_TSO),
 152         RTE_TX_OFFLOAD_BIT2STR(UDP_TSO),
 153         RTE_TX_OFFLOAD_BIT2STR(OUTER_IPV4_CKSUM),
 154         RTE_TX_OFFLOAD_BIT2STR(QINQ_INSERT),
 155         RTE_TX_OFFLOAD_BIT2STR(VXLAN_TNL_TSO),
 156         RTE_TX_OFFLOAD_BIT2STR(GRE_TNL_TSO),
 157         RTE_TX_OFFLOAD_BIT2STR(IPIP_TNL_TSO),
 158         RTE_TX_OFFLOAD_BIT2STR(GENEVE_TNL_TSO),
 159         RTE_TX_OFFLOAD_BIT2STR(MACSEC_INSERT),
 160         RTE_TX_OFFLOAD_BIT2STR(MT_LOCKFREE),
 161         RTE_TX_OFFLOAD_BIT2STR(MULTI_SEGS),
 162         RTE_TX_OFFLOAD_BIT2STR(MBUF_FAST_FREE),
 163         RTE_TX_OFFLOAD_BIT2STR(SECURITY),
 164         RTE_TX_OFFLOAD_BIT2STR(UDP_TNL_TSO),
 165         RTE_TX_OFFLOAD_BIT2STR(IP_TNL_TSO),
 166         RTE_TX_OFFLOAD_BIT2STR(OUTER_UDP_CKSUM),
 167         RTE_TX_OFFLOAD_BIT2STR(SEND_ON_TIMESTAMP),
 168 };
 169
 170 #undef RTE_TX_OFFLOAD_BIT2STR
 171
 172 /**
 173  * The user application callback description.
 174  *
 175  * It contains callback address to be registered by user application,
 176  * the pointer to the parameters for callback, and the event type.
 177  */
 178 struct rte_eth_dev_callback {
 179         TAILQ_ENTRY(rte_eth_dev_callback) next; /**< Callbacks list */
 180         rte_eth_dev_cb_fn cb_fn;                /**< Callback address */
 181         void *cb_arg;                           /**< Parameter for callback */
 182         void *ret_param;                        /**< Return parameter */
 183         enum rte_eth_event_type event;          /**< Interrupt event type */
 184         uint32_t active;                        /**< Callback is executing */
 185 };
 186
 187 enum {
 188         STAT_QMAP_TX = 0,
 189         STAT_QMAP_RX
 190 };
 191
 192 int
 193 rte_eth_iterator_init(struct rte_dev_iterator *iter, const char *devargs_str)
 194 {
 195         int ret;
 196         struct rte_devargs devargs = {.args = NULL};
 197         const char *bus_param_key;
 198         char *bus_str = NULL;
 199         char *cls_str = NULL;
 200         int str_size;
 201
 202         memset(iter, 0, sizeof(*iter));
 203
 204         /*
 205          * The devargs string may use various syntaxes:
 206          *   - 0000:08:00.0,representor=[1-3]
 207          *   - pci:0000:06:00.0,representor=[0,5]
 208          *   - class=eth,mac=00:11:22:33:44:55
 209          * A new syntax is in development (not yet supported):
 210          *   - bus=X,paramX=x/class=Y,paramY=y/driver=Z,paramZ=z
 211          */
 212
 213         /*
 214          * Handle pure class filter (i.e. without any bus-level argument),
 215          * from future new syntax.
 216          * rte_devargs_parse() is not yet supporting the new syntax,
 217          * that's why this simple case is temporarily parsed here.
 218          */
 219 #define iter_anybus_str "class=eth,"
 220         if (strncmp(devargs_str, iter_anybus_str,
 221                         strlen(iter_anybus_str)) == 0) {
 222                 iter->cls_str = devargs_str + strlen(iter_anybus_str);
 223                 goto end;
 224         }
 225
 226         /* Split bus, device and parameters. */
 227         ret = rte_devargs_parse(&devargs, devargs_str);
 228         if (ret != 0)
 229                 goto error;
 230
 231         /*
 232          * Assume parameters of old syntax can match only at ethdev level.
 233          * Extra parameters will be ignored, thanks to "+" prefix.
 234          */
 235         str_size = strlen(devargs.args) + 2;
 236         cls_str = malloc(str_size);
 237         if (cls_str == NULL) {
 238                 ret = -ENOMEM;
 239                 goto error;
 240         }
 241         ret = snprintf(cls_str, str_size, "+%s", devargs.args);
 242         if (ret != str_size - 1) {
 243                 ret = -EINVAL;
 244                 goto error;
 245         }
 246         iter->cls_str = cls_str;
 247         free(devargs.args); /* allocated by rte_devargs_parse() */
 248         devargs.args = NULL;
 249
 250         iter->bus = devargs.bus;
 251         if (iter->bus->dev_iterate == NULL) {
 252                 ret = -ENOTSUP;
 253                 goto error;
 254         }
 255
 256         /* Convert bus args to new syntax for use with new API dev_iterate. */
 257         if (strcmp(iter->bus->name, "vdev") == 0) {
 258                 bus_param_key = "name";
 259         } else if (strcmp(iter->bus->name, "pci") == 0) {
 260                 bus_param_key = "addr";
 261         } else {
 262                 ret = -ENOTSUP;
 263                 goto error;
 264         }
 265         str_size = strlen(bus_param_key) + strlen(devargs.name) + 2;
 266         bus_str = malloc(str_size);
 267         if (bus_str == NULL) {
 268                 ret = -ENOMEM;
 269                 goto error;
 270         }
 271         ret = snprintf(bus_str, str_size, "%s=%s",
 272                         bus_param_key, devargs.name);
 273         if (ret != str_size - 1) {
 274                 ret = -EINVAL;
 275                 goto error;
 276         }
 277         iter->bus_str = bus_str;
 278
 279 end:
 280         iter->cls = rte_class_find_by_name("eth");
 281         return 0;
 282
 283 error:
 284         if (ret == -ENOTSUP)
 285                 RTE_ETHDEV_LOG(ERR, "Bus %s does not support iterating.\n",
 286                                 iter->bus->name);
 287         free(devargs.args);
 288         free(bus_str);
 289         free(cls_str);
 290         return ret;
 291 }
 292
 293 uint16_t
 294 rte_eth_iterator_next(struct rte_dev_iterator *iter)
 295 {
 296         if (iter->cls == NULL) /* invalid ethdev iterator */
 297                 return RTE_MAX_ETHPORTS;
 298
 299         do { /* loop to try all matching rte_device */
 300                 /* If not pure ethdev filter and */
 301                 if (iter->bus != NULL &&
 302                                 /* not in middle of rte_eth_dev iteration, */
 303                                 iter->class_device == NULL) {
 304                         /* get next rte_device to try. */
 305                         iter->device = iter->bus->dev_iterate(
 306                                         iter->device, iter->bus_str, iter);
 307                         if (iter->device == NULL)
 308                                 break; /* no more rte_device candidate */
 309                 }
 310                 /* A device is matching bus part, need to check ethdev part. */
 311                 iter->class_device = iter->cls->dev_iterate(
 312                                 iter->class_device, iter->cls_str, iter);
 313                 if (iter->class_device != NULL)
 314                         return eth_dev_to_id(iter->class_device); /* match */
 315         } while (iter->bus != NULL); /* need to try next rte_device */
 316
 317         /* No more ethdev port to iterate. */
 318         rte_eth_iterator_cleanup(iter);
 319         return RTE_MAX_ETHPORTS;
 320 }
 321
 322 void
 323 rte_eth_iterator_cleanup(struct rte_dev_iterator *iter)
 324 {
 325         if (iter->bus_str == NULL)
 326                 return; /* nothing to free in pure class filter */
 327         free(RTE_CAST_FIELD(iter, bus_str, char *)); /* workaround const */
 328         free(RTE_CAST_FIELD(iter, cls_str, char *)); /* workaround const */
 329         memset(iter, 0, sizeof(*iter));
 330 }
 331
 332 uint16_t
 333 rte_eth_find_next(uint16_t port_id)
 334 {
 335         while (port_id < RTE_MAX_ETHPORTS &&
 336                         rte_eth_devices[port_id].state == RTE_ETH_DEV_UNUSED)
 337                 port_id++;
 338
 339         if (port_id >= RTE_MAX_ETHPORTS)
 340                 return RTE_MAX_ETHPORTS;
 341
 342         return port_id;
 343 }
 344
 345 /*
 346  * Macro to iterate over all valid ports for internal usage.
 347  * Note: RTE_ETH_FOREACH_DEV is different because filtering owned ports.
 348  */
 349 #define RTE_ETH_FOREACH_VALID_DEV(port_id) \
 350         for (port_id = rte_eth_find_next(0); \
 351              port_id < RTE_MAX_ETHPORTS; \
 352              port_id = rte_eth_find_next(port_id + 1))
 353
 354 uint16_t
 355 rte_eth_find_next_of(uint16_t port_id, const struct rte_device *parent)
 356 {
 357         port_id = rte_eth_find_next(port_id);
 358         while (port_id < RTE_MAX_ETHPORTS &&
 359                         rte_eth_devices[port_id].device != parent)
 360                 port_id = rte_eth_find_next(port_id + 1);
 361
 362         return port_id;
 363 }
 364
 365 uint16_t
 366 rte_eth_find_next_sibling(uint16_t port_id, uint16_t ref_port_id)
 367 {
 368         RTE_ETH_VALID_PORTID_OR_ERR_RET(ref_port_id, RTE_MAX_ETHPORTS);
 369         return rte_eth_find_next_of(port_id,
 370                         rte_eth_devices[ref_port_id].device);
 371 }
 372
 373 static void
 374 rte_eth_dev_shared_data_prepare(void)
 375 {
 376         const unsigned flags = 0;
 377         const struct rte_memzone *mz;
 378
 379         rte_spinlock_lock(&rte_eth_shared_data_lock);
 380
 381         if (rte_eth_dev_shared_data == NULL) {
 382                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 383                         /* Allocate port data and ownership shared memory. */
 384                         mz = rte_memzone_reserve(MZ_RTE_ETH_DEV_DATA,
 385                                         sizeof(*rte_eth_dev_shared_data),
 386                                         rte_socket_id(), flags);
 387                 } else
 388                         mz = rte_memzone_lookup(MZ_RTE_ETH_DEV_DATA);
 389                 if (mz == NULL)
 390                         rte_panic("Cannot allocate ethdev shared data\n");
 391
 392                 rte_eth_dev_shared_data = mz->addr;
 393                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 394                         rte_eth_dev_shared_data->next_owner_id =
 395                                         RTE_ETH_DEV_NO_OWNER + 1;
 396                         rte_spinlock_init(&rte_eth_dev_shared_data->ownership_lock);
 397                         memset(rte_eth_dev_shared_data->data, 0,
 398                                sizeof(rte_eth_dev_shared_data->data));
 399                 }
 400         }
 401
 402         rte_spinlock_unlock(&rte_eth_shared_data_lock);
 403 }
 404
 405 static bool
 406 is_allocated(const struct rte_eth_dev *ethdev)
 407 {
 408         return ethdev->data->name[0] != '\0';
 409 }
 410
 411 static struct rte_eth_dev *
 412 _rte_eth_dev_allocated(const char *name)
 413 {
 414         unsigned i;
 415
 416         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 417                 if (rte_eth_devices[i].data != NULL &&
 418                     strcmp(rte_eth_devices[i].data->name, name) == 0)
 419                         return &rte_eth_devices[i];
 420         }
 421         return NULL;
 422 }
 423
 424 struct rte_eth_dev *
 425 rte_eth_dev_allocated(const char *name)
 426 {
 427         struct rte_eth_dev *ethdev;
 428
 429         rte_eth_dev_shared_data_prepare();
 430
 431         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 432
 433         ethdev = _rte_eth_dev_allocated(name);
 434
 435         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 436
 437         return ethdev;
 438 }
 439
 440 static uint16_t
 441 rte_eth_dev_find_free_port(void)
 442 {
 443         unsigned i;
 444
 445         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 446                 /* Using shared name field to find a free port. */
 447                 if (rte_eth_dev_shared_data->data[i].name[0] == '\0') {
 448                         RTE_ASSERT(rte_eth_devices[i].state ==
 449                                    RTE_ETH_DEV_UNUSED);
 450                         return i;
 451                 }
 452         }
 453         return RTE_MAX_ETHPORTS;
 454 }
 455
 456 static struct rte_eth_dev *
 457 eth_dev_get(uint16_t port_id)
 458 {
 459         struct rte_eth_dev *eth_dev = &rte_eth_devices[port_id];
 460
 461         eth_dev->data = &rte_eth_dev_shared_data->data[port_id];
 462
 463         return eth_dev;
 464 }
 465
 466 struct rte_eth_dev *
 467 rte_eth_dev_allocate(const char *name)
 468 {
 469         uint16_t port_id;
 470         struct rte_eth_dev *eth_dev = NULL;
 471         size_t name_len;
 472
 473         name_len = strnlen(name, RTE_ETH_NAME_MAX_LEN);
 474         if (name_len == 0) {
 475                 RTE_ETHDEV_LOG(ERR, "Zero length Ethernet device name\n");
 476                 return NULL;
 477         }
 478
 479         if (name_len >= RTE_ETH_NAME_MAX_LEN) {
 480                 RTE_ETHDEV_LOG(ERR, "Ethernet device name is too long\n");
 481                 return NULL;
 482         }
 483
 484         rte_eth_dev_shared_data_prepare();
 485
 486         /* Synchronize port creation between primary and secondary threads. */
 487         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 488
 489         if (_rte_eth_dev_allocated(name) != NULL) {
 490                 RTE_ETHDEV_LOG(ERR,
 491                         "Ethernet device with name %s already allocated\n",
 492                         name);
 493                 goto unlock;
 494         }
 495
 496         port_id = rte_eth_dev_find_free_port();
 497         if (port_id == RTE_MAX_ETHPORTS) {
 498                 RTE_ETHDEV_LOG(ERR,
 499                         "Reached maximum number of Ethernet ports\n");
 500                 goto unlock;
 501         }
 502
 503         eth_dev = eth_dev_get(port_id);
 504         strlcpy(eth_dev->data->name, name, sizeof(eth_dev->data->name));
 505         eth_dev->data->port_id = port_id;
 506         eth_dev->data->mtu = RTE_ETHER_MTU;
 507         pthread_mutex_init(&eth_dev->data->flow_ops_mutex, NULL);
 508
 509 unlock:
 510         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 511
 512         return eth_dev;
 513 }
 514
 515 /*
 516  * Attach to a port already registered by the primary process, which
 517  * makes sure that the same device would have the same port id both
 518  * in the primary and secondary process.
 519  */
 520 struct rte_eth_dev *
 521 rte_eth_dev_attach_secondary(const char *name)
 522 {
 523         uint16_t i;
 524         struct rte_eth_dev *eth_dev = NULL;
 525
 526         rte_eth_dev_shared_data_prepare();
 527
 528         /* Synchronize port attachment to primary port creation and release. */
 529         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 530
 531         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
 532                 if (strcmp(rte_eth_dev_shared_data->data[i].name, name) == 0)
 533                         break;
 534         }
 535         if (i == RTE_MAX_ETHPORTS) {
 536                 RTE_ETHDEV_LOG(ERR,
 537                         "Device %s is not driven by the primary process\n",
 538                         name);
 539         } else {
 540                 eth_dev = eth_dev_get(i);
 541                 RTE_ASSERT(eth_dev->data->port_id == i);
 542         }
 543
 544         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 545         return eth_dev;
 546 }
 547
 548 int
 549 rte_eth_dev_release_port(struct rte_eth_dev *eth_dev)
 550 {
 551         if (eth_dev == NULL)
 552                 return -EINVAL;
 553
 554         rte_eth_dev_shared_data_prepare();
 555
 556         if (eth_dev->state != RTE_ETH_DEV_UNUSED)
 557                 rte_eth_dev_callback_process(eth_dev,
 558                                 RTE_ETH_EVENT_DESTROY, NULL);
 559
 560         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 561
 562         eth_dev->state = RTE_ETH_DEV_UNUSED;
 563         eth_dev->device = NULL;
 564         eth_dev->intr_handle = NULL;
 565
 566         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 567                 rte_free(eth_dev->data->rx_queues);
 568                 rte_free(eth_dev->data->tx_queues);
 569                 rte_free(eth_dev->data->mac_addrs);
 570                 rte_free(eth_dev->data->hash_mac_addrs);
 571                 rte_free(eth_dev->data->dev_private);
 572                 pthread_mutex_destroy(&eth_dev->data->flow_ops_mutex);
 573                 memset(eth_dev->data, 0, sizeof(struct rte_eth_dev_data));
 574         }
 575
 576         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 577
 578         return 0;
 579 }
 580
 581 int
 582 rte_eth_dev_is_valid_port(uint16_t port_id)
 583 {
 584         if (port_id >= RTE_MAX_ETHPORTS ||
 585             (rte_eth_devices[port_id].state == RTE_ETH_DEV_UNUSED))
 586                 return 0;
 587         else
 588                 return 1;
 589 }
 590
 591 static int
 592 rte_eth_is_valid_owner_id(uint64_t owner_id)
 593 {
 594         if (owner_id == RTE_ETH_DEV_NO_OWNER ||
 595             rte_eth_dev_shared_data->next_owner_id <= owner_id)
 596                 return 0;
 597         return 1;
 598 }
 599
 600 uint64_t
 601 rte_eth_find_next_owned_by(uint16_t port_id, const uint64_t owner_id)
 602 {
 603         port_id = rte_eth_find_next(port_id);
 604         while (port_id < RTE_MAX_ETHPORTS &&
 605                         rte_eth_devices[port_id].data->owner.id != owner_id)
 606                 port_id = rte_eth_find_next(port_id + 1);
 607
 608         return port_id;
 609 }
 610
 611 int
 612 rte_eth_dev_owner_new(uint64_t *owner_id)
 613 {
 614         rte_eth_dev_shared_data_prepare();
 615
 616         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 617
 618         *owner_id = rte_eth_dev_shared_data->next_owner_id++;
 619
 620         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 621         return 0;
 622 }
 623
 624 static int
 625 _rte_eth_dev_owner_set(const uint16_t port_id, const uint64_t old_owner_id,
 626                        const struct rte_eth_dev_owner *new_owner)
 627 {
 628         struct rte_eth_dev *ethdev = &rte_eth_devices[port_id];
 629         struct rte_eth_dev_owner *port_owner;
 630
 631         if (port_id >= RTE_MAX_ETHPORTS || !is_allocated(ethdev)) {
 632                 RTE_ETHDEV_LOG(ERR, "Port id %"PRIu16" is not allocated\n",
 633                         port_id);
 634                 return -ENODEV;
 635         }
 636
 637         if (!rte_eth_is_valid_owner_id(new_owner->id) &&
 638             !rte_eth_is_valid_owner_id(old_owner_id)) {
 639                 RTE_ETHDEV_LOG(ERR,
 640                         "Invalid owner old_id=%016"PRIx64" new_id=%016"PRIx64"\n",
 641                        old_owner_id, new_owner->id);
 642                 return -EINVAL;
 643         }
 644
 645         port_owner = &rte_eth_devices[port_id].data->owner;
 646         if (port_owner->id != old_owner_id) {
 647                 RTE_ETHDEV_LOG(ERR,
 648                         "Cannot set owner to port %u already owned by %s_%016"PRIX64"\n",
 649                         port_id, port_owner->name, port_owner->id);
 650                 return -EPERM;
 651         }
 652
 653         /* can not truncate (same structure) */
 654         strlcpy(port_owner->name, new_owner->name, RTE_ETH_MAX_OWNER_NAME_LEN);
 655
 656         port_owner->id = new_owner->id;
 657
 658         RTE_ETHDEV_LOG(DEBUG, "Port %u owner is %s_%016"PRIx64"\n",
 659                 port_id, new_owner->name, new_owner->id);
 660
 661         return 0;
 662 }
 663
 664 int
 665 rte_eth_dev_owner_set(const uint16_t port_id,
 666                       const struct rte_eth_dev_owner *owner)
 667 {
 668         int ret;
 669
 670         rte_eth_dev_shared_data_prepare();
 671
 672         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 673
 674         ret = _rte_eth_dev_owner_set(port_id, RTE_ETH_DEV_NO_OWNER, owner);
 675
 676         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 677         return ret;
 678 }
 679
 680 int
 681 rte_eth_dev_owner_unset(const uint16_t port_id, const uint64_t owner_id)
 682 {
 683         const struct rte_eth_dev_owner new_owner = (struct rte_eth_dev_owner)
 684                         {.id = RTE_ETH_DEV_NO_OWNER, .name = ""};
 685         int ret;
 686
 687         rte_eth_dev_shared_data_prepare();
 688
 689         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 690
 691         ret = _rte_eth_dev_owner_set(port_id, owner_id, &new_owner);
 692
 693         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 694         return ret;
 695 }
 696
 697 int
 698 rte_eth_dev_owner_delete(const uint64_t owner_id)
 699 {
 700         uint16_t port_id;
 701         int ret = 0;
 702
 703         rte_eth_dev_shared_data_prepare();
 704
 705         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 706
 707         if (rte_eth_is_valid_owner_id(owner_id)) {
 708                 for (port_id = 0; port_id < RTE_MAX_ETHPORTS; port_id++)
 709                         if (rte_eth_devices[port_id].data->owner.id == owner_id)
 710                                 memset(&rte_eth_devices[port_id].data->owner, 0,
 711                                        sizeof(struct rte_eth_dev_owner));
 712                 RTE_ETHDEV_LOG(NOTICE,
 713                         "All port owners owned by %016"PRIx64" identifier have removed\n",
 714                         owner_id);
 715         } else {
 716                 RTE_ETHDEV_LOG(ERR,
 717                                "Invalid owner id=%016"PRIx64"\n",
 718                                owner_id);
 719                 ret = -EINVAL;
 720         }
 721
 722         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 723
 724         return ret;
 725 }
 726
 727 int
 728 rte_eth_dev_owner_get(const uint16_t port_id, struct rte_eth_dev_owner *owner)
 729 {
 730         int ret = 0;
 731         struct rte_eth_dev *ethdev = &rte_eth_devices[port_id];
 732
 733         rte_eth_dev_shared_data_prepare();
 734
 735         rte_spinlock_lock(&rte_eth_dev_shared_data->ownership_lock);
 736
 737         if (port_id >= RTE_MAX_ETHPORTS || !is_allocated(ethdev)) {
 738                 RTE_ETHDEV_LOG(ERR, "Port id %"PRIu16" is not allocated\n",
 739                         port_id);
 740                 ret = -ENODEV;
 741         } else {
 742                 rte_memcpy(owner, &ethdev->data->owner, sizeof(*owner));
 743         }
 744
 745         rte_spinlock_unlock(&rte_eth_dev_shared_data->ownership_lock);
 746         return ret;
 747 }
 748
 749 int
 750 rte_eth_dev_socket_id(uint16_t port_id)
 751 {
 752         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -1);
 753         return rte_eth_devices[port_id].data->numa_node;
 754 }
 755
 756 void *
 757 rte_eth_dev_get_sec_ctx(uint16_t port_id)
 758 {
 759         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, NULL);
 760         return rte_eth_devices[port_id].security_ctx;
 761 }
 762
 763 uint16_t
 764 rte_eth_dev_count_avail(void)
 765 {
 766         uint16_t p;
 767         uint16_t count;
 768
 769         count = 0;
 770
 771         RTE_ETH_FOREACH_DEV(p)
 772                 count++;
 773
 774         return count;
 775 }
 776
 777 uint16_t
 778 rte_eth_dev_count_total(void)
 779 {
 780         uint16_t port, count = 0;
 781
 782         RTE_ETH_FOREACH_VALID_DEV(port)
 783                 count++;
 784
 785         return count;
 786 }
 787
 788 int
 789 rte_eth_dev_get_name_by_port(uint16_t port_id, char *name)
 790 {
 791         char *tmp;
 792
 793         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 794
 795         if (name == NULL) {
 796                 RTE_ETHDEV_LOG(ERR, "Null pointer is specified\n");
 797                 return -EINVAL;
 798         }
 799
 800         /* shouldn't check 'rte_eth_devices[i].data',
 801          * because it might be overwritten by VDEV PMD */
 802         tmp = rte_eth_dev_shared_data->data[port_id].name;
 803         strcpy(name, tmp);
 804         return 0;
 805 }
 806
 807 int
 808 rte_eth_dev_get_port_by_name(const char *name, uint16_t *port_id)
 809 {
 810         uint32_t pid;
 811
 812         if (name == NULL) {
 813                 RTE_ETHDEV_LOG(ERR, "Null pointer is specified\n");
 814                 return -EINVAL;
 815         }
 816
 817         RTE_ETH_FOREACH_VALID_DEV(pid)
 818                 if (!strcmp(name, rte_eth_dev_shared_data->data[pid].name)) {
 819                         *port_id = pid;
 820                         return 0;
 821                 }
 822
 823         return -ENODEV;
 824 }
 825
 826 static int
 827 eth_err(uint16_t port_id, int ret)
 828 {
 829         if (ret == 0)
 830                 return 0;
 831         if (rte_eth_dev_is_removed(port_id))
 832                 return -EIO;
 833         return ret;
 834 }
 835
 836 static int
 837 rte_eth_dev_rx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
 838 {
 839         uint16_t old_nb_queues = dev->data->nb_rx_queues;
 840         void **rxq;
 841         unsigned i;
 842
 843         if (dev->data->rx_queues == NULL && nb_queues != 0) { /* first time configuration */
 844                 dev->data->rx_queues = rte_zmalloc("ethdev->rx_queues",
 845                                 sizeof(dev->data->rx_queues[0]) * nb_queues,
 846                                 RTE_CACHE_LINE_SIZE);
 847                 if (dev->data->rx_queues == NULL) {
 848                         dev->data->nb_rx_queues = 0;
 849                         return -(ENOMEM);
 850                 }
 851         } else if (dev->data->rx_queues != NULL && nb_queues != 0) { /* re-configure */
 852                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release, -ENOTSUP);
 853
 854                 rxq = dev->data->rx_queues;
 855
 856                 for (i = nb_queues; i < old_nb_queues; i++)
 857                         (*dev->dev_ops->rx_queue_release)(rxq[i]);
 858                 rxq = rte_realloc(rxq, sizeof(rxq[0]) * nb_queues,
 859                                 RTE_CACHE_LINE_SIZE);
 860                 if (rxq == NULL)
 861                         return -(ENOMEM);
 862                 if (nb_queues > old_nb_queues) {
 863                         uint16_t new_qs = nb_queues - old_nb_queues;
 864
 865                         memset(rxq + old_nb_queues, 0,
 866                                 sizeof(rxq[0]) * new_qs);
 867                 }
 868
 869                 dev->data->rx_queues = rxq;
 870
 871         } else if (dev->data->rx_queues != NULL && nb_queues == 0) {
 872                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release, -ENOTSUP);
 873
 874                 rxq = dev->data->rx_queues;
 875
 876                 for (i = nb_queues; i < old_nb_queues; i++)
 877                         (*dev->dev_ops->rx_queue_release)(rxq[i]);
 878
 879                 rte_free(dev->data->rx_queues);
 880                 dev->data->rx_queues = NULL;
 881         }
 882         dev->data->nb_rx_queues = nb_queues;
 883         return 0;
 884 }
 885
 886 static int
 887 eth_dev_validate_rx_queue(const struct rte_eth_dev *dev, uint16_t rx_queue_id)
 888 {
 889         uint16_t port_id;
 890
 891         if (rx_queue_id >= dev->data->nb_rx_queues) {
 892                 port_id = dev->data->port_id;
 893                 RTE_ETHDEV_LOG(ERR,
 894                                "Invalid Rx queue_id=%u of device with port_id=%u\n",
 895                                rx_queue_id, port_id);
 896                 return -EINVAL;
 897         }
 898
 899         if (dev->data->rx_queues[rx_queue_id] == NULL) {
 900                 port_id = dev->data->port_id;
 901                 RTE_ETHDEV_LOG(ERR,
 902                                "Queue %u of device with port_id=%u has not been setup\n",
 903                                rx_queue_id, port_id);
 904                 return -EINVAL;
 905         }
 906
 907         return 0;
 908 }
 909
 910 static int
 911 eth_dev_validate_tx_queue(const struct rte_eth_dev *dev, uint16_t tx_queue_id)
 912 {
 913         uint16_t port_id;
 914
 915         if (tx_queue_id >= dev->data->nb_tx_queues) {
 916                 port_id = dev->data->port_id;
 917                 RTE_ETHDEV_LOG(ERR,
 918                                "Invalid Tx queue_id=%u of device with port_id=%u\n",
 919                                tx_queue_id, port_id);
 920                 return -EINVAL;
 921         }
 922
 923         if (dev->data->tx_queues[tx_queue_id] == NULL) {
 924                 port_id = dev->data->port_id;
 925                 RTE_ETHDEV_LOG(ERR,
 926                                "Queue %u of device with port_id=%u has not been setup\n",
 927                                tx_queue_id, port_id);
 928                 return -EINVAL;
 929         }
 930
 931         return 0;
 932 }
 933
 934 int
 935 rte_eth_dev_rx_queue_start(uint16_t port_id, uint16_t rx_queue_id)
 936 {
 937         struct rte_eth_dev *dev;
 938         int ret;
 939
 940         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 941
 942         dev = &rte_eth_devices[port_id];
 943         if (!dev->data->dev_started) {
 944                 RTE_ETHDEV_LOG(ERR,
 945                         "Port %u must be started before start any queue\n",
 946                         port_id);
 947                 return -EINVAL;
 948         }
 949
 950         ret = eth_dev_validate_rx_queue(dev, rx_queue_id);
 951         if (ret != 0)
 952                 return ret;
 953
 954         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_start, -ENOTSUP);
 955
 956         if (rte_eth_dev_is_rx_hairpin_queue(dev, rx_queue_id)) {
 957                 RTE_ETHDEV_LOG(INFO,
 958                         "Can't start Rx hairpin queue %"PRIu16" of device with port_id=%"PRIu16"\n",
 959                         rx_queue_id, port_id);
 960                 return -EINVAL;
 961         }
 962
 963         if (dev->data->rx_queue_state[rx_queue_id] != RTE_ETH_QUEUE_STATE_STOPPED) {
 964                 RTE_ETHDEV_LOG(INFO,
 965                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already started\n",
 966                         rx_queue_id, port_id);
 967                 return 0;
 968         }
 969
 970         return eth_err(port_id, dev->dev_ops->rx_queue_start(dev,
 971                                                              rx_queue_id));
 972
 973 }
 974
 975 int
 976 rte_eth_dev_rx_queue_stop(uint16_t port_id, uint16_t rx_queue_id)
 977 {
 978         struct rte_eth_dev *dev;
 979         int ret;
 980
 981         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
 982
 983         dev = &rte_eth_devices[port_id];
 984
 985         ret = eth_dev_validate_rx_queue(dev, rx_queue_id);
 986         if (ret != 0)
 987                 return ret;
 988
 989         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_stop, -ENOTSUP);
 990
 991         if (rte_eth_dev_is_rx_hairpin_queue(dev, rx_queue_id)) {
 992                 RTE_ETHDEV_LOG(INFO,
 993                         "Can't stop Rx hairpin queue %"PRIu16" of device with port_id=%"PRIu16"\n",
 994                         rx_queue_id, port_id);
 995                 return -EINVAL;
 996         }
 997
 998         if (dev->data->rx_queue_state[rx_queue_id] == RTE_ETH_QUEUE_STATE_STOPPED) {
 999                 RTE_ETHDEV_LOG(INFO,
1000                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already stopped\n",
1001                         rx_queue_id, port_id);
1002                 return 0;
1003         }
1004
1005         return eth_err(port_id, dev->dev_ops->rx_queue_stop(dev, rx_queue_id));
1006
1007 }
1008
1009 int
1010 rte_eth_dev_tx_queue_start(uint16_t port_id, uint16_t tx_queue_id)
1011 {
1012         struct rte_eth_dev *dev;
1013         int ret;
1014
1015         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1016
1017         dev = &rte_eth_devices[port_id];
1018         if (!dev->data->dev_started) {
1019                 RTE_ETHDEV_LOG(ERR,
1020                         "Port %u must be started before start any queue\n",
1021                         port_id);
1022                 return -EINVAL;
1023         }
1024
1025         ret = eth_dev_validate_tx_queue(dev, tx_queue_id);
1026         if (ret != 0)
1027                 return ret;
1028
1029         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_start, -ENOTSUP);
1030
1031         if (rte_eth_dev_is_tx_hairpin_queue(dev, tx_queue_id)) {
1032                 RTE_ETHDEV_LOG(INFO,
1033                         "Can't start Tx hairpin queue %"PRIu16" of device with port_id=%"PRIu16"\n",
1034                         tx_queue_id, port_id);
1035                 return -EINVAL;
1036         }
1037
1038         if (dev->data->tx_queue_state[tx_queue_id] != RTE_ETH_QUEUE_STATE_STOPPED) {
1039                 RTE_ETHDEV_LOG(INFO,
1040                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already started\n",
1041                         tx_queue_id, port_id);
1042                 return 0;
1043         }
1044
1045         return eth_err(port_id, dev->dev_ops->tx_queue_start(dev, tx_queue_id));
1046 }
1047
1048 int
1049 rte_eth_dev_tx_queue_stop(uint16_t port_id, uint16_t tx_queue_id)
1050 {
1051         struct rte_eth_dev *dev;
1052         int ret;
1053
1054         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1055
1056         dev = &rte_eth_devices[port_id];
1057
1058         ret = eth_dev_validate_tx_queue(dev, tx_queue_id);
1059         if (ret != 0)
1060                 return ret;
1061
1062         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_stop, -ENOTSUP);
1063
1064         if (rte_eth_dev_is_tx_hairpin_queue(dev, tx_queue_id)) {
1065                 RTE_ETHDEV_LOG(INFO,
1066                         "Can't stop Tx hairpin queue %"PRIu16" of device with port_id=%"PRIu16"\n",
1067                         tx_queue_id, port_id);
1068                 return -EINVAL;
1069         }
1070
1071         if (dev->data->tx_queue_state[tx_queue_id] == RTE_ETH_QUEUE_STATE_STOPPED) {
1072                 RTE_ETHDEV_LOG(INFO,
1073                         "Queue %"PRIu16" of device with port_id=%"PRIu16" already stopped\n",
1074                         tx_queue_id, port_id);
1075                 return 0;
1076         }
1077
1078         return eth_err(port_id, dev->dev_ops->tx_queue_stop(dev, tx_queue_id));
1079
1080 }
1081
1082 static int
1083 rte_eth_dev_tx_queue_config(struct rte_eth_dev *dev, uint16_t nb_queues)
1084 {
1085         uint16_t old_nb_queues = dev->data->nb_tx_queues;
1086         void **txq;
1087         unsigned i;
1088
1089         if (dev->data->tx_queues == NULL && nb_queues != 0) { /* first time configuration */
1090                 dev->data->tx_queues = rte_zmalloc("ethdev->tx_queues",
1091                                                    sizeof(dev->data->tx_queues[0]) * nb_queues,
1092                                                    RTE_CACHE_LINE_SIZE);
1093                 if (dev->data->tx_queues == NULL) {
1094                         dev->data->nb_tx_queues = 0;
1095                         return -(ENOMEM);
1096                 }
1097         } else if (dev->data->tx_queues != NULL && nb_queues != 0) { /* re-configure */
1098                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release, -ENOTSUP);
1099
1100                 txq = dev->data->tx_queues;
1101
1102                 for (i = nb_queues; i < old_nb_queues; i++)
1103                         (*dev->dev_ops->tx_queue_release)(txq[i]);
1104                 txq = rte_realloc(txq, sizeof(txq[0]) * nb_queues,
1105                                   RTE_CACHE_LINE_SIZE);
1106                 if (txq == NULL)
1107                         return -ENOMEM;
1108                 if (nb_queues > old_nb_queues) {
1109                         uint16_t new_qs = nb_queues - old_nb_queues;
1110
1111                         memset(txq + old_nb_queues, 0,
1112                                sizeof(txq[0]) * new_qs);
1113                 }
1114
1115                 dev->data->tx_queues = txq;
1116
1117         } else if (dev->data->tx_queues != NULL && nb_queues == 0) {
1118                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release, -ENOTSUP);
1119
1120                 txq = dev->data->tx_queues;
1121
1122                 for (i = nb_queues; i < old_nb_queues; i++)
1123                         (*dev->dev_ops->tx_queue_release)(txq[i]);
1124
1125                 rte_free(dev->data->tx_queues);
1126                 dev->data->tx_queues = NULL;
1127         }
1128         dev->data->nb_tx_queues = nb_queues;
1129         return 0;
1130 }
1131
1132 uint32_t
1133 rte_eth_speed_bitflag(uint32_t speed, int duplex)
1134 {
1135         switch (speed) {
1136         case ETH_SPEED_NUM_10M:
1137                 return duplex ? ETH_LINK_SPEED_10M : ETH_LINK_SPEED_10M_HD;
1138         case ETH_SPEED_NUM_100M:
1139                 return duplex ? ETH_LINK_SPEED_100M : ETH_LINK_SPEED_100M_HD;
1140         case ETH_SPEED_NUM_1G:
1141                 return ETH_LINK_SPEED_1G;
1142         case ETH_SPEED_NUM_2_5G:
1143                 return ETH_LINK_SPEED_2_5G;
1144         case ETH_SPEED_NUM_5G:
1145                 return ETH_LINK_SPEED_5G;
1146         case ETH_SPEED_NUM_10G:
1147                 return ETH_LINK_SPEED_10G;
1148         case ETH_SPEED_NUM_20G:
1149                 return ETH_LINK_SPEED_20G;
1150         case ETH_SPEED_NUM_25G:
1151                 return ETH_LINK_SPEED_25G;
1152         case ETH_SPEED_NUM_40G:
1153                 return ETH_LINK_SPEED_40G;
1154         case ETH_SPEED_NUM_50G:
1155                 return ETH_LINK_SPEED_50G;
1156         case ETH_SPEED_NUM_56G:
1157                 return ETH_LINK_SPEED_56G;
1158         case ETH_SPEED_NUM_100G:
1159                 return ETH_LINK_SPEED_100G;
1160         case ETH_SPEED_NUM_200G:
1161                 return ETH_LINK_SPEED_200G;
1162         default:
1163                 return 0;
1164         }
1165 }
1166
1167 const char *
1168 rte_eth_dev_rx_offload_name(uint64_t offload)
1169 {
1170         const char *name = "UNKNOWN";
1171         unsigned int i;
1172
1173         for (i = 0; i < RTE_DIM(rte_rx_offload_names); ++i) {
1174                 if (offload == rte_rx_offload_names[i].offload) {
1175                         name = rte_rx_offload_names[i].name;
1176                         break;
1177                 }
1178         }
1179
1180         return name;
1181 }
1182
1183 const char *
1184 rte_eth_dev_tx_offload_name(uint64_t offload)
1185 {
1186         const char *name = "UNKNOWN";
1187         unsigned int i;
1188
1189         for (i = 0; i < RTE_DIM(rte_tx_offload_names); ++i) {
1190                 if (offload == rte_tx_offload_names[i].offload) {
1191                         name = rte_tx_offload_names[i].name;
1192                         break;
1193                 }
1194         }
1195
1196         return name;
1197 }
1198
1199 static inline int
1200 check_lro_pkt_size(uint16_t port_id, uint32_t config_size,
1201                    uint32_t max_rx_pkt_len, uint32_t dev_info_size)
1202 {
1203         int ret = 0;
1204
1205         if (dev_info_size == 0) {
1206                 if (config_size != max_rx_pkt_len) {
1207                         RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%d max_lro_pkt_size"
1208                                        " %u != %u is not allowed\n",
1209                                        port_id, config_size, max_rx_pkt_len);
1210                         ret = -EINVAL;
1211                 }
1212         } else if (config_size > dev_info_size) {
1213                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%d max_lro_pkt_size %u "
1214                                "> max allowed value %u\n", port_id, config_size,
1215                                dev_info_size);
1216                 ret = -EINVAL;
1217         } else if (config_size < RTE_ETHER_MIN_LEN) {
1218                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%d max_lro_pkt_size %u "
1219                                "< min allowed value %u\n", port_id, config_size,
1220                                (unsigned int)RTE_ETHER_MIN_LEN);
1221                 ret = -EINVAL;
1222         }
1223         return ret;
1224 }
1225
1226 /*
1227  * Validate offloads that are requested through rte_eth_dev_configure against
1228  * the offloads successfully set by the ethernet device.
1229  *
1230  * @param port_id
1231  *   The port identifier of the Ethernet device.
1232  * @param req_offloads
1233  *   The offloads that have been requested through `rte_eth_dev_configure`.
1234  * @param set_offloads
1235  *   The offloads successfully set by the ethernet device.
1236  * @param offload_type
1237  *   The offload type i.e. Rx/Tx string.
1238  * @param offload_name
1239  *   The function that prints the offload name.
1240  * @return
1241  *   - (0) if validation successful.
1242  *   - (-EINVAL) if requested offload has been silently disabled.
1243  *
1244  */
1245 static int
1246 validate_offloads(uint16_t port_id, uint64_t req_offloads,
1247                   uint64_t set_offloads, const char *offload_type,
1248                   const char *(*offload_name)(uint64_t))
1249 {
1250         uint64_t offloads_diff = req_offloads ^ set_offloads;
1251         uint64_t offload;
1252         int ret = 0;
1253
1254         while (offloads_diff != 0) {
1255                 /* Check if any offload is requested but not enabled. */
1256                 offload = 1ULL << __builtin_ctzll(offloads_diff);
1257                 if (offload & req_offloads) {
1258                         RTE_ETHDEV_LOG(ERR,
1259                                 "Port %u failed to enable %s offload %s\n",
1260                                 port_id, offload_type, offload_name(offload));
1261                         ret = -EINVAL;
1262                 }
1263
1264                 /* Check if offload couldn't be disabled. */
1265                 if (offload & set_offloads) {
1266                         RTE_ETHDEV_LOG(DEBUG,
1267                                 "Port %u %s offload %s is not requested but enabled\n",
1268                                 port_id, offload_type, offload_name(offload));
1269                 }
1270
1271                 offloads_diff &= ~offload;
1272         }
1273
1274         return ret;
1275 }
1276
1277 int
1278 rte_eth_dev_configure(uint16_t port_id, uint16_t nb_rx_q, uint16_t nb_tx_q,
1279                       const struct rte_eth_conf *dev_conf)
1280 {
1281         struct rte_eth_dev *dev;
1282         struct rte_eth_dev_info dev_info;
1283         struct rte_eth_conf orig_conf;
1284         int diag;
1285         int ret;
1286
1287         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1288
1289         dev = &rte_eth_devices[port_id];
1290
1291         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_configure, -ENOTSUP);
1292
1293         if (dev->data->dev_started) {
1294                 RTE_ETHDEV_LOG(ERR,
1295                         "Port %u must be stopped to allow configuration\n",
1296                         port_id);
1297                 return -EBUSY;
1298         }
1299
1300          /* Store original config, as rollback required on failure */
1301         memcpy(&orig_conf, &dev->data->dev_conf, sizeof(dev->data->dev_conf));
1302
1303         /*
1304          * Copy the dev_conf parameter into the dev structure.
1305          * rte_eth_dev_info_get() requires dev_conf, copy it before dev_info get
1306          */
1307         if (dev_conf != &dev->data->dev_conf)
1308                 memcpy(&dev->data->dev_conf, dev_conf,
1309                        sizeof(dev->data->dev_conf));
1310
1311         ret = rte_eth_dev_info_get(port_id, &dev_info);
1312         if (ret != 0)
1313                 goto rollback;
1314
1315         /* If number of queues specified by application for both Rx and Tx is
1316          * zero, use driver preferred values. This cannot be done individually
1317          * as it is valid for either Tx or Rx (but not both) to be zero.
1318          * If driver does not provide any preferred valued, fall back on
1319          * EAL defaults.
1320          */
1321         if (nb_rx_q == 0 && nb_tx_q == 0) {
1322                 nb_rx_q = dev_info.default_rxportconf.nb_queues;
1323                 if (nb_rx_q == 0)
1324                         nb_rx_q = RTE_ETH_DEV_FALLBACK_RX_NBQUEUES;
1325                 nb_tx_q = dev_info.default_txportconf.nb_queues;
1326                 if (nb_tx_q == 0)
1327                         nb_tx_q = RTE_ETH_DEV_FALLBACK_TX_NBQUEUES;
1328         }
1329
1330         if (nb_rx_q > RTE_MAX_QUEUES_PER_PORT) {
1331                 RTE_ETHDEV_LOG(ERR,
1332                         "Number of RX queues requested (%u) is greater than max supported(%d)\n",
1333                         nb_rx_q, RTE_MAX_QUEUES_PER_PORT);
1334                 ret = -EINVAL;
1335                 goto rollback;
1336         }
1337
1338         if (nb_tx_q > RTE_MAX_QUEUES_PER_PORT) {
1339                 RTE_ETHDEV_LOG(ERR,
1340                         "Number of TX queues requested (%u) is greater than max supported(%d)\n",
1341                         nb_tx_q, RTE_MAX_QUEUES_PER_PORT);
1342                 ret = -EINVAL;
1343                 goto rollback;
1344         }
1345
1346         /*
1347          * Check that the numbers of RX and TX queues are not greater
1348          * than the maximum number of RX and TX queues supported by the
1349          * configured device.
1350          */
1351         if (nb_rx_q > dev_info.max_rx_queues) {
1352                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u nb_rx_queues=%u > %u\n",
1353                         port_id, nb_rx_q, dev_info.max_rx_queues);
1354                 ret = -EINVAL;
1355                 goto rollback;
1356         }
1357
1358         if (nb_tx_q > dev_info.max_tx_queues) {
1359                 RTE_ETHDEV_LOG(ERR, "Ethdev port_id=%u nb_tx_queues=%u > %u\n",
1360                         port_id, nb_tx_q, dev_info.max_tx_queues);
1361                 ret = -EINVAL;
1362                 goto rollback;
1363         }
1364
1365         /* Check that the device supports requested interrupts */
1366         if ((dev_conf->intr_conf.lsc == 1) &&
1367                         (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC))) {
1368                 RTE_ETHDEV_LOG(ERR, "Driver %s does not support lsc\n",
1369                         dev->device->driver->name);
1370                 ret = -EINVAL;
1371                 goto rollback;
1372         }
1373         if ((dev_conf->intr_conf.rmv == 1) &&
1374                         (!(dev->data->dev_flags & RTE_ETH_DEV_INTR_RMV))) {
1375                 RTE_ETHDEV_LOG(ERR, "Driver %s does not support rmv\n",
1376                         dev->device->driver->name);
1377                 ret = -EINVAL;
1378                 goto rollback;
1379         }
1380
1381         /*
1382          * If jumbo frames are enabled, check that the maximum RX packet
1383          * length is supported by the configured device.
1384          */
1385         if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
1386                 if (dev_conf->rxmode.max_rx_pkt_len > dev_info.max_rx_pktlen) {
1387                         RTE_ETHDEV_LOG(ERR,
1388                                 "Ethdev port_id=%u max_rx_pkt_len %u > max valid value %u\n",
1389                                 port_id, dev_conf->rxmode.max_rx_pkt_len,
1390                                 dev_info.max_rx_pktlen);
1391                         ret = -EINVAL;
1392                         goto rollback;
1393                 } else if (dev_conf->rxmode.max_rx_pkt_len < RTE_ETHER_MIN_LEN) {
1394                         RTE_ETHDEV_LOG(ERR,
1395                                 "Ethdev port_id=%u max_rx_pkt_len %u < min valid value %u\n",
1396                                 port_id, dev_conf->rxmode.max_rx_pkt_len,
1397                                 (unsigned int)RTE_ETHER_MIN_LEN);
1398                         ret = -EINVAL;
1399                         goto rollback;
1400                 }
1401         } else {
1402                 if (dev_conf->rxmode.max_rx_pkt_len < RTE_ETHER_MIN_LEN ||
1403                         dev_conf->rxmode.max_rx_pkt_len > RTE_ETHER_MAX_LEN)
1404                         /* Use default value */
1405                         dev->data->dev_conf.rxmode.max_rx_pkt_len =
1406                                                         RTE_ETHER_MAX_LEN;
1407         }
1408
1409         /*
1410          * If LRO is enabled, check that the maximum aggregated packet
1411          * size is supported by the configured device.
1412          */
1413         if (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_TCP_LRO) {
1414                 if (dev_conf->rxmode.max_lro_pkt_size == 0)
1415                         dev->data->dev_conf.rxmode.max_lro_pkt_size =
1416                                 dev->data->dev_conf.rxmode.max_rx_pkt_len;
1417                 ret = check_lro_pkt_size(port_id,
1418                                 dev->data->dev_conf.rxmode.max_lro_pkt_size,
1419                                 dev->data->dev_conf.rxmode.max_rx_pkt_len,
1420                                 dev_info.max_lro_pkt_size);
1421                 if (ret != 0)
1422                         goto rollback;
1423         }
1424
1425         /* Any requested offloading must be within its device capabilities */
1426         if ((dev_conf->rxmode.offloads & dev_info.rx_offload_capa) !=
1427              dev_conf->rxmode.offloads) {
1428                 RTE_ETHDEV_LOG(ERR,
1429                         "Ethdev port_id=%u requested Rx offloads 0x%"PRIx64" doesn't match Rx offloads "
1430                         "capabilities 0x%"PRIx64" in %s()\n",
1431                         port_id, dev_conf->rxmode.offloads,
1432                         dev_info.rx_offload_capa,
1433                         __func__);
1434                 ret = -EINVAL;
1435                 goto rollback;
1436         }
1437         if ((dev_conf->txmode.offloads & dev_info.tx_offload_capa) !=
1438              dev_conf->txmode.offloads) {
1439                 RTE_ETHDEV_LOG(ERR,
1440                         "Ethdev port_id=%u requested Tx offloads 0x%"PRIx64" doesn't match Tx offloads "
1441                         "capabilities 0x%"PRIx64" in %s()\n",
1442                         port_id, dev_conf->txmode.offloads,
1443                         dev_info.tx_offload_capa,
1444                         __func__);
1445                 ret = -EINVAL;
1446                 goto rollback;
1447         }
1448
1449         dev->data->dev_conf.rx_adv_conf.rss_conf.rss_hf =
1450                 rte_eth_rss_hf_refine(dev_conf->rx_adv_conf.rss_conf.rss_hf);
1451
1452         /* Check that device supports requested rss hash functions. */
1453         if ((dev_info.flow_type_rss_offloads |
1454              dev_conf->rx_adv_conf.rss_conf.rss_hf) !=
1455             dev_info.flow_type_rss_offloads) {
1456                 RTE_ETHDEV_LOG(ERR,
1457                         "Ethdev port_id=%u invalid rss_hf: 0x%"PRIx64", valid value: 0x%"PRIx64"\n",
1458                         port_id, dev_conf->rx_adv_conf.rss_conf.rss_hf,
1459                         dev_info.flow_type_rss_offloads);
1460                 ret = -EINVAL;
1461                 goto rollback;
1462         }
1463
1464         /* Check if Rx RSS distribution is disabled but RSS hash is enabled. */
1465         if (((dev_conf->rxmode.mq_mode & ETH_MQ_RX_RSS_FLAG) == 0) &&
1466             (dev_conf->rxmode.offloads & DEV_RX_OFFLOAD_RSS_HASH)) {
1467                 RTE_ETHDEV_LOG(ERR,
1468                         "Ethdev port_id=%u config invalid Rx mq_mode without RSS but %s offload is requested\n",
1469                         port_id,
1470                         rte_eth_dev_rx_offload_name(DEV_RX_OFFLOAD_RSS_HASH));
1471                 ret = -EINVAL;
1472                 goto rollback;
1473         }
1474
1475         /*
1476          * Setup new number of RX/TX queues and reconfigure device.
1477          */
1478         diag = rte_eth_dev_rx_queue_config(dev, nb_rx_q);
1479         if (diag != 0) {
1480                 RTE_ETHDEV_LOG(ERR,
1481                         "Port%u rte_eth_dev_rx_queue_config = %d\n",
1482                         port_id, diag);
1483                 ret = diag;
1484                 goto rollback;
1485         }
1486
1487         diag = rte_eth_dev_tx_queue_config(dev, nb_tx_q);
1488         if (diag != 0) {
1489                 RTE_ETHDEV_LOG(ERR,
1490                         "Port%u rte_eth_dev_tx_queue_config = %d\n",
1491                         port_id, diag);
1492                 rte_eth_dev_rx_queue_config(dev, 0);
1493                 ret = diag;
1494                 goto rollback;
1495         }
1496
1497         diag = (*dev->dev_ops->dev_configure)(dev);
1498         if (diag != 0) {
1499                 RTE_ETHDEV_LOG(ERR, "Port%u dev_configure = %d\n",
1500                         port_id, diag);
1501                 ret = eth_err(port_id, diag);
1502                 goto reset_queues;
1503         }
1504
1505         /* Initialize Rx profiling if enabled at compilation time. */
1506         diag = __rte_eth_dev_profile_init(port_id, dev);
1507         if (diag != 0) {
1508                 RTE_ETHDEV_LOG(ERR, "Port%u __rte_eth_dev_profile_init = %d\n",
1509                         port_id, diag);
1510                 ret = eth_err(port_id, diag);
1511                 goto reset_queues;
1512         }
1513
1514         /* Validate Rx offloads. */
1515         diag = validate_offloads(port_id,
1516                         dev_conf->rxmode.offloads,
1517                         dev->data->dev_conf.rxmode.offloads, "Rx",
1518                         rte_eth_dev_rx_offload_name);
1519         if (diag != 0) {
1520                 ret = diag;
1521                 goto reset_queues;
1522         }
1523
1524         /* Validate Tx offloads. */
1525         diag = validate_offloads(port_id,
1526                         dev_conf->txmode.offloads,
1527                         dev->data->dev_conf.txmode.offloads, "Tx",
1528                         rte_eth_dev_tx_offload_name);
1529         if (diag != 0) {
1530                 ret = diag;
1531                 goto reset_queues;
1532         }
1533
1534         rte_ethdev_trace_configure(port_id, nb_rx_q, nb_tx_q, dev_conf, 0);
1535         return 0;
1536 reset_queues:
1537         rte_eth_dev_rx_queue_config(dev, 0);
1538         rte_eth_dev_tx_queue_config(dev, 0);
1539 rollback:
1540         memcpy(&dev->data->dev_conf, &orig_conf, sizeof(dev->data->dev_conf));
1541
1542         rte_ethdev_trace_configure(port_id, nb_rx_q, nb_tx_q, dev_conf, ret);
1543         return ret;
1544 }
1545
1546 void
1547 rte_eth_dev_internal_reset(struct rte_eth_dev *dev)
1548 {
1549         if (dev->data->dev_started) {
1550                 RTE_ETHDEV_LOG(ERR, "Port %u must be stopped to allow reset\n",
1551                         dev->data->port_id);
1552                 return;
1553         }
1554
1555         rte_eth_dev_rx_queue_config(dev, 0);
1556         rte_eth_dev_tx_queue_config(dev, 0);
1557
1558         memset(&dev->data->dev_conf, 0, sizeof(dev->data->dev_conf));
1559 }
1560
1561 static void
1562 rte_eth_dev_mac_restore(struct rte_eth_dev *dev,
1563                         struct rte_eth_dev_info *dev_info)
1564 {
1565         struct rte_ether_addr *addr;
1566         uint16_t i;
1567         uint32_t pool = 0;
1568         uint64_t pool_mask;
1569
1570         /* replay MAC address configuration including default MAC */
1571         addr = &dev->data->mac_addrs[0];
1572         if (*dev->dev_ops->mac_addr_set != NULL)
1573                 (*dev->dev_ops->mac_addr_set)(dev, addr);
1574         else if (*dev->dev_ops->mac_addr_add != NULL)
1575                 (*dev->dev_ops->mac_addr_add)(dev, addr, 0, pool);
1576
1577         if (*dev->dev_ops->mac_addr_add != NULL) {
1578                 for (i = 1; i < dev_info->max_mac_addrs; i++) {
1579                         addr = &dev->data->mac_addrs[i];
1580
1581                         /* skip zero address */
1582                         if (rte_is_zero_ether_addr(addr))
1583                                 continue;
1584
1585                         pool = 0;
1586                         pool_mask = dev->data->mac_pool_sel[i];
1587
1588                         do {
1589                                 if (pool_mask & 1ULL)
1590                                         (*dev->dev_ops->mac_addr_add)(dev,
1591                                                 addr, i, pool);
1592                                 pool_mask >>= 1;
1593                                 pool++;
1594                         } while (pool_mask);
1595                 }
1596         }
1597 }
1598
1599 static int
1600 rte_eth_dev_config_restore(struct rte_eth_dev *dev,
1601                            struct rte_eth_dev_info *dev_info, uint16_t port_id)
1602 {
1603         int ret;
1604
1605         if (!(*dev_info->dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR))
1606                 rte_eth_dev_mac_restore(dev, dev_info);
1607
1608         /* replay promiscuous configuration */
1609         /*
1610          * use callbacks directly since we don't need port_id check and
1611          * would like to bypass the same value set
1612          */
1613         if (rte_eth_promiscuous_get(port_id) == 1 &&
1614             *dev->dev_ops->promiscuous_enable != NULL) {
1615                 ret = eth_err(port_id,
1616                               (*dev->dev_ops->promiscuous_enable)(dev));
1617                 if (ret != 0 && ret != -ENOTSUP) {
1618                         RTE_ETHDEV_LOG(ERR,
1619                                 "Failed to enable promiscuous mode for device (port %u): %s\n",
1620                                 port_id, rte_strerror(-ret));
1621                         return ret;
1622                 }
1623         } else if (rte_eth_promiscuous_get(port_id) == 0 &&
1624                    *dev->dev_ops->promiscuous_disable != NULL) {
1625                 ret = eth_err(port_id,
1626                               (*dev->dev_ops->promiscuous_disable)(dev));
1627                 if (ret != 0 && ret != -ENOTSUP) {
1628                         RTE_ETHDEV_LOG(ERR,
1629                                 "Failed to disable promiscuous mode for device (port %u): %s\n",
1630                                 port_id, rte_strerror(-ret));
1631                         return ret;
1632                 }
1633         }
1634
1635         /* replay all multicast configuration */
1636         /*
1637          * use callbacks directly since we don't need port_id check and
1638          * would like to bypass the same value set
1639          */
1640         if (rte_eth_allmulticast_get(port_id) == 1 &&
1641             *dev->dev_ops->allmulticast_enable != NULL) {
1642                 ret = eth_err(port_id,
1643                               (*dev->dev_ops->allmulticast_enable)(dev));
1644                 if (ret != 0 && ret != -ENOTSUP) {
1645                         RTE_ETHDEV_LOG(ERR,
1646                                 "Failed to enable allmulticast mode for device (port %u): %s\n",
1647                                 port_id, rte_strerror(-ret));
1648                         return ret;
1649                 }
1650         } else if (rte_eth_allmulticast_get(port_id) == 0 &&
1651                    *dev->dev_ops->allmulticast_disable != NULL) {
1652                 ret = eth_err(port_id,
1653                               (*dev->dev_ops->allmulticast_disable)(dev));
1654                 if (ret != 0 && ret != -ENOTSUP) {
1655                         RTE_ETHDEV_LOG(ERR,
1656                                 "Failed to disable allmulticast mode for device (port %u): %s\n",
1657                                 port_id, rte_strerror(-ret));
1658                         return ret;
1659                 }
1660         }
1661
1662         return 0;
1663 }
1664
1665 int
1666 rte_eth_dev_start(uint16_t port_id)
1667 {
1668         struct rte_eth_dev *dev;
1669         struct rte_eth_dev_info dev_info;
1670         int diag;
1671         int ret;
1672
1673         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1674
1675         dev = &rte_eth_devices[port_id];
1676
1677         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_start, -ENOTSUP);
1678
1679         if (dev->data->dev_started != 0) {
1680                 RTE_ETHDEV_LOG(INFO,
1681                         "Device with port_id=%"PRIu16" already started\n",
1682                         port_id);
1683                 return 0;
1684         }
1685
1686         ret = rte_eth_dev_info_get(port_id, &dev_info);
1687         if (ret != 0)
1688                 return ret;
1689
1690         /* Lets restore MAC now if device does not support live change */
1691         if (*dev_info.dev_flags & RTE_ETH_DEV_NOLIVE_MAC_ADDR)
1692                 rte_eth_dev_mac_restore(dev, &dev_info);
1693
1694         diag = (*dev->dev_ops->dev_start)(dev);
1695         if (diag == 0)
1696                 dev->data->dev_started = 1;
1697         else
1698                 return eth_err(port_id, diag);
1699
1700         ret = rte_eth_dev_config_restore(dev, &dev_info, port_id);
1701         if (ret != 0) {
1702                 RTE_ETHDEV_LOG(ERR,
1703                         "Error during restoring configuration for device (port %u): %s\n",
1704                         port_id, rte_strerror(-ret));
1705                 rte_eth_dev_stop(port_id);
1706                 return ret;
1707         }
1708
1709         if (dev->data->dev_conf.intr_conf.lsc == 0) {
1710                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
1711                 (*dev->dev_ops->link_update)(dev, 0);
1712         }
1713
1714         rte_ethdev_trace_start(port_id);
1715         return 0;
1716 }
1717
1718 void
1719 rte_eth_dev_stop(uint16_t port_id)
1720 {
1721         struct rte_eth_dev *dev;
1722
1723         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1724         dev = &rte_eth_devices[port_id];
1725
1726         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_stop);
1727
1728         if (dev->data->dev_started == 0) {
1729                 RTE_ETHDEV_LOG(INFO,
1730                         "Device with port_id=%"PRIu16" already stopped\n",
1731                         port_id);
1732                 return;
1733         }
1734
1735         dev->data->dev_started = 0;
1736         (*dev->dev_ops->dev_stop)(dev);
1737         rte_ethdev_trace_stop(port_id);
1738 }
1739
1740 int
1741 rte_eth_dev_set_link_up(uint16_t port_id)
1742 {
1743         struct rte_eth_dev *dev;
1744
1745         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1746
1747         dev = &rte_eth_devices[port_id];
1748
1749         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_up, -ENOTSUP);
1750         return eth_err(port_id, (*dev->dev_ops->dev_set_link_up)(dev));
1751 }
1752
1753 int
1754 rte_eth_dev_set_link_down(uint16_t port_id)
1755 {
1756         struct rte_eth_dev *dev;
1757
1758         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1759
1760         dev = &rte_eth_devices[port_id];
1761
1762         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_set_link_down, -ENOTSUP);
1763         return eth_err(port_id, (*dev->dev_ops->dev_set_link_down)(dev));
1764 }
1765
1766 void
1767 rte_eth_dev_close(uint16_t port_id)
1768 {
1769         struct rte_eth_dev *dev;
1770
1771         RTE_ETH_VALID_PORTID_OR_RET(port_id);
1772         dev = &rte_eth_devices[port_id];
1773
1774         RTE_FUNC_PTR_OR_RET(*dev->dev_ops->dev_close);
1775         dev->data->dev_started = 0;
1776         (*dev->dev_ops->dev_close)(dev);
1777
1778         rte_ethdev_trace_close(port_id);
1779         rte_eth_dev_release_port(dev);
1780 }
1781
1782 int
1783 rte_eth_dev_reset(uint16_t port_id)
1784 {
1785         struct rte_eth_dev *dev;
1786         int ret;
1787
1788         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1789         dev = &rte_eth_devices[port_id];
1790
1791         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_reset, -ENOTSUP);
1792
1793         rte_eth_dev_stop(port_id);
1794         ret = dev->dev_ops->dev_reset(dev);
1795
1796         return eth_err(port_id, ret);
1797 }
1798
1799 int
1800 rte_eth_dev_is_removed(uint16_t port_id)
1801 {
1802         struct rte_eth_dev *dev;
1803         int ret;
1804
1805         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, 0);
1806
1807         dev = &rte_eth_devices[port_id];
1808
1809         if (dev->state == RTE_ETH_DEV_REMOVED)
1810                 return 1;
1811
1812         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->is_removed, 0);
1813
1814         ret = dev->dev_ops->is_removed(dev);
1815         if (ret != 0)
1816                 /* Device is physically removed. */
1817                 dev->state = RTE_ETH_DEV_REMOVED;
1818
1819         return ret;
1820 }
1821
1822 static int
1823 rte_eth_rx_queue_check_split(const struct rte_eth_rxseg_split *rx_seg,
1824                              uint16_t n_seg, uint32_t *mbp_buf_size,
1825                              const struct rte_eth_dev_info *dev_info)
1826 {
1827         const struct rte_eth_rxseg_capa *seg_capa = &dev_info->rx_seg_capa;
1828         struct rte_mempool *mp_first;
1829         uint32_t offset_mask;
1830         uint16_t seg_idx;
1831
1832         if (n_seg > seg_capa->max_nseg) {
1833                 RTE_ETHDEV_LOG(ERR,
1834                                "Requested Rx segments %u exceed supported %u\n",
1835                                n_seg, seg_capa->max_nseg);
1836                 return -EINVAL;
1837         }
1838         /*
1839          * Check the sizes and offsets against buffer sizes
1840          * for each segment specified in extended configuration.
1841          */
1842         mp_first = rx_seg[0].mp;
1843         offset_mask = (1u << seg_capa->offset_align_log2) - 1;
1844         for (seg_idx = 0; seg_idx < n_seg; seg_idx++) {
1845                 struct rte_mempool *mpl = rx_seg[seg_idx].mp;
1846                 uint32_t length = rx_seg[seg_idx].length;
1847                 uint32_t offset = rx_seg[seg_idx].offset;
1848
1849                 if (mpl == NULL) {
1850                         RTE_ETHDEV_LOG(ERR, "null mempool pointer\n");
1851                         return -EINVAL;
1852                 }
1853                 if (seg_idx != 0 && mp_first != mpl &&
1854                     seg_capa->multi_pools == 0) {
1855                         RTE_ETHDEV_LOG(ERR, "Receiving to multiple pools is not supported\n");
1856                         return -ENOTSUP;
1857                 }
1858                 if (offset != 0) {
1859                         if (seg_capa->offset_allowed == 0) {
1860                                 RTE_ETHDEV_LOG(ERR, "Rx segmentation with offset is not supported\n");
1861                                 return -ENOTSUP;
1862                         }
1863                         if (offset & offset_mask) {
1864                                 RTE_ETHDEV_LOG(ERR, "Rx segmentation invalid offset alignment %u, %u\n",
1865                                                offset,
1866                                                seg_capa->offset_align_log2);
1867                                 return -EINVAL;
1868                         }
1869                 }
1870                 if (mpl->private_data_size <
1871                         sizeof(struct rte_pktmbuf_pool_private)) {
1872                         RTE_ETHDEV_LOG(ERR,
1873                                        "%s private_data_size %u < %u\n",
1874                                        mpl->name, mpl->private_data_size,
1875                                        (unsigned int)sizeof
1876                                         (struct rte_pktmbuf_pool_private));
1877                         return -ENOSPC;
1878                 }
1879                 offset += seg_idx != 0 ? 0 : RTE_PKTMBUF_HEADROOM;
1880                 *mbp_buf_size = rte_pktmbuf_data_room_size(mpl);
1881                 length = length != 0 ? length : *mbp_buf_size;
1882                 if (*mbp_buf_size < length + offset) {
1883                         RTE_ETHDEV_LOG(ERR,
1884                                        "%s mbuf_data_room_size %u < %u (segment length=%u + segment offset=%u)\n",
1885                                        mpl->name, *mbp_buf_size,
1886                                        length + offset, length, offset);
1887                         return -EINVAL;
1888                 }
1889         }
1890         return 0;
1891 }
1892
1893 int
1894 rte_eth_rx_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
1895                        uint16_t nb_rx_desc, unsigned int socket_id,
1896                        const struct rte_eth_rxconf *rx_conf,
1897                        struct rte_mempool *mp)
1898 {
1899         int ret;
1900         uint32_t mbp_buf_size;
1901         struct rte_eth_dev *dev;
1902         struct rte_eth_dev_info dev_info;
1903         struct rte_eth_rxconf local_conf;
1904         void **rxq;
1905
1906         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
1907
1908         dev = &rte_eth_devices[port_id];
1909         if (rx_queue_id >= dev->data->nb_rx_queues) {
1910                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
1911                 return -EINVAL;
1912         }
1913
1914         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_setup, -ENOTSUP);
1915
1916         ret = rte_eth_dev_info_get(port_id, &dev_info);
1917         if (ret != 0)
1918                 return ret;
1919
1920         if (mp != NULL) {
1921                 /* Single pool configuration check. */
1922                 if (rx_conf != NULL && rx_conf->rx_nseg != 0) {
1923                         RTE_ETHDEV_LOG(ERR,
1924                                        "Ambiguous segment configuration\n");
1925                         return -EINVAL;
1926                 }
1927                 /*
1928                  * Check the size of the mbuf data buffer, this value
1929                  * must be provided in the private data of the memory pool.
1930                  * First check that the memory pool(s) has a valid private data.
1931                  */
1932                 if (mp->private_data_size <
1933                                 sizeof(struct rte_pktmbuf_pool_private)) {
1934                         RTE_ETHDEV_LOG(ERR, "%s private_data_size %u < %u\n",
1935                                 mp->name, mp->private_data_size,
1936                                 (unsigned int)
1937                                 sizeof(struct rte_pktmbuf_pool_private));
1938                         return -ENOSPC;
1939                 }
1940                 mbp_buf_size = rte_pktmbuf_data_room_size(mp);
1941                 if (mbp_buf_size < dev_info.min_rx_bufsize +
1942                                    RTE_PKTMBUF_HEADROOM) {
1943                         RTE_ETHDEV_LOG(ERR,
1944                                        "%s mbuf_data_room_size %u < %u (RTE_PKTMBUF_HEADROOM=%u + min_rx_bufsize(dev)=%u)\n",
1945                                        mp->name, mbp_buf_size,
1946                                        RTE_PKTMBUF_HEADROOM +
1947                                        dev_info.min_rx_bufsize,
1948                                        RTE_PKTMBUF_HEADROOM,
1949                                        dev_info.min_rx_bufsize);
1950                         return -EINVAL;
1951                 }
1952         } else {
1953                 const struct rte_eth_rxseg_split *rx_seg =
1954                         (const struct rte_eth_rxseg_split *)rx_conf->rx_seg;
1955                 uint16_t n_seg = rx_conf->rx_nseg;
1956
1957                 /* Extended multi-segment configuration check. */
1958                 if (rx_conf == NULL || rx_conf->rx_seg == NULL || rx_conf->rx_nseg == 0) {
1959                         RTE_ETHDEV_LOG(ERR,
1960                                        "Memory pool is null and no extended configuration provided\n");
1961                         return -EINVAL;
1962                 }
1963                 if (rx_conf->offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) {
1964                         ret = rte_eth_rx_queue_check_split(rx_seg, n_seg,
1965                                                            &mbp_buf_size,
1966                                                            &dev_info);
1967                         if (ret != 0)
1968                                 return ret;
1969                 } else {
1970                         RTE_ETHDEV_LOG(ERR, "No Rx segmentation offload configured\n");
1971                         return -EINVAL;
1972                 }
1973         }
1974
1975         /* Use default specified by driver, if nb_rx_desc is zero */
1976         if (nb_rx_desc == 0) {
1977                 nb_rx_desc = dev_info.default_rxportconf.ring_size;
1978                 /* If driver default is also zero, fall back on EAL default */
1979                 if (nb_rx_desc == 0)
1980                         nb_rx_desc = RTE_ETH_DEV_FALLBACK_RX_RINGSIZE;
1981         }
1982
1983         if (nb_rx_desc > dev_info.rx_desc_lim.nb_max ||
1984                         nb_rx_desc < dev_info.rx_desc_lim.nb_min ||
1985                         nb_rx_desc % dev_info.rx_desc_lim.nb_align != 0) {
1986
1987                 RTE_ETHDEV_LOG(ERR,
1988                         "Invalid value for nb_rx_desc(=%hu), should be: <= %hu, >= %hu, and a product of %hu\n",
1989                         nb_rx_desc, dev_info.rx_desc_lim.nb_max,
1990                         dev_info.rx_desc_lim.nb_min,
1991                         dev_info.rx_desc_lim.nb_align);
1992                 return -EINVAL;
1993         }
1994
1995         if (dev->data->dev_started &&
1996                 !(dev_info.dev_capa &
1997                         RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP))
1998                 return -EBUSY;
1999
2000         if (dev->data->dev_started &&
2001                 (dev->data->rx_queue_state[rx_queue_id] !=
2002                         RTE_ETH_QUEUE_STATE_STOPPED))
2003                 return -EBUSY;
2004
2005         rxq = dev->data->rx_queues;
2006         if (rxq[rx_queue_id]) {
2007                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release,
2008                                         -ENOTSUP);
2009                 (*dev->dev_ops->rx_queue_release)(rxq[rx_queue_id]);
2010                 rxq[rx_queue_id] = NULL;
2011         }
2012
2013         if (rx_conf == NULL)
2014                 rx_conf = &dev_info.default_rxconf;
2015
2016         local_conf = *rx_conf;
2017
2018         /*
2019          * If an offloading has already been enabled in
2020          * rte_eth_dev_configure(), it has been enabled on all queues,
2021          * so there is no need to enable it in this queue again.
2022          * The local_conf.offloads input to underlying PMD only carries
2023          * those offloadings which are only enabled on this queue and
2024          * not enabled on all queues.
2025          */
2026         local_conf.offloads &= ~dev->data->dev_conf.rxmode.offloads;
2027
2028         /*
2029          * New added offloadings for this queue are those not enabled in
2030          * rte_eth_dev_configure() and they must be per-queue type.
2031          * A pure per-port offloading can't be enabled on a queue while
2032          * disabled on another queue. A pure per-port offloading can't
2033          * be enabled for any queue as new added one if it hasn't been
2034          * enabled in rte_eth_dev_configure().
2035          */
2036         if ((local_conf.offloads & dev_info.rx_queue_offload_capa) !=
2037              local_conf.offloads) {
2038                 RTE_ETHDEV_LOG(ERR,
2039                         "Ethdev port_id=%d rx_queue_id=%d, new added offloads 0x%"PRIx64" must be "
2040                         "within per-queue offload capabilities 0x%"PRIx64" in %s()\n",
2041                         port_id, rx_queue_id, local_conf.offloads,
2042                         dev_info.rx_queue_offload_capa,
2043                         __func__);
2044                 return -EINVAL;
2045         }
2046
2047         /*
2048          * If LRO is enabled, check that the maximum aggregated packet
2049          * size is supported by the configured device.
2050          */
2051         if (local_conf.offloads & DEV_RX_OFFLOAD_TCP_LRO) {
2052                 if (dev->data->dev_conf.rxmode.max_lro_pkt_size == 0)
2053                         dev->data->dev_conf.rxmode.max_lro_pkt_size =
2054                                 dev->data->dev_conf.rxmode.max_rx_pkt_len;
2055                 int ret = check_lro_pkt_size(port_id,
2056                                 dev->data->dev_conf.rxmode.max_lro_pkt_size,
2057                                 dev->data->dev_conf.rxmode.max_rx_pkt_len,
2058                                 dev_info.max_lro_pkt_size);
2059                 if (ret != 0)
2060                         return ret;
2061         }
2062
2063         ret = (*dev->dev_ops->rx_queue_setup)(dev, rx_queue_id, nb_rx_desc,
2064                                               socket_id, &local_conf, mp);
2065         if (!ret) {
2066                 if (!dev->data->min_rx_buf_size ||
2067                     dev->data->min_rx_buf_size > mbp_buf_size)
2068                         dev->data->min_rx_buf_size = mbp_buf_size;
2069         }
2070
2071         rte_ethdev_trace_rxq_setup(port_id, rx_queue_id, nb_rx_desc, mp,
2072                 rx_conf, ret);
2073         return eth_err(port_id, ret);
2074 }
2075
2076 int
2077 rte_eth_rx_hairpin_queue_setup(uint16_t port_id, uint16_t rx_queue_id,
2078                                uint16_t nb_rx_desc,
2079                                const struct rte_eth_hairpin_conf *conf)
2080 {
2081         int ret;
2082         struct rte_eth_dev *dev;
2083         struct rte_eth_hairpin_cap cap;
2084         void **rxq;
2085         int i;
2086         int count;
2087
2088         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2089
2090         dev = &rte_eth_devices[port_id];
2091         if (rx_queue_id >= dev->data->nb_rx_queues) {
2092                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", rx_queue_id);
2093                 return -EINVAL;
2094         }
2095         ret = rte_eth_dev_hairpin_capability_get(port_id, &cap);
2096         if (ret != 0)
2097                 return ret;
2098         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_hairpin_queue_setup,
2099                                 -ENOTSUP);
2100         /* if nb_rx_desc is zero use max number of desc from the driver. */
2101         if (nb_rx_desc == 0)
2102                 nb_rx_desc = cap.max_nb_desc;
2103         if (nb_rx_desc > cap.max_nb_desc) {
2104                 RTE_ETHDEV_LOG(ERR,
2105                         "Invalid value for nb_rx_desc(=%hu), should be: <= %hu",
2106                         nb_rx_desc, cap.max_nb_desc);
2107                 return -EINVAL;
2108         }
2109         if (conf->peer_count > cap.max_rx_2_tx) {
2110                 RTE_ETHDEV_LOG(ERR,
2111                         "Invalid value for number of peers for Rx queue(=%u), should be: <= %hu",
2112                         conf->peer_count, cap.max_rx_2_tx);
2113                 return -EINVAL;
2114         }
2115         if (conf->peer_count == 0) {
2116                 RTE_ETHDEV_LOG(ERR,
2117                         "Invalid value for number of peers for Rx queue(=%u), should be: > 0",
2118                         conf->peer_count);
2119                 return -EINVAL;
2120         }
2121         for (i = 0, count = 0; i < dev->data->nb_rx_queues &&
2122              cap.max_nb_queues != UINT16_MAX; i++) {
2123                 if (i == rx_queue_id || rte_eth_dev_is_rx_hairpin_queue(dev, i))
2124                         count++;
2125         }
2126         if (count > cap.max_nb_queues) {
2127                 RTE_ETHDEV_LOG(ERR, "To many Rx hairpin queues max is %d",
2128                 cap.max_nb_queues);
2129                 return -EINVAL;
2130         }
2131         if (dev->data->dev_started)
2132                 return -EBUSY;
2133         rxq = dev->data->rx_queues;
2134         if (rxq[rx_queue_id] != NULL) {
2135                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_release,
2136                                         -ENOTSUP);
2137                 (*dev->dev_ops->rx_queue_release)(rxq[rx_queue_id]);
2138                 rxq[rx_queue_id] = NULL;
2139         }
2140         ret = (*dev->dev_ops->rx_hairpin_queue_setup)(dev, rx_queue_id,
2141                                                       nb_rx_desc, conf);
2142         if (ret == 0)
2143                 dev->data->rx_queue_state[rx_queue_id] =
2144                         RTE_ETH_QUEUE_STATE_HAIRPIN;
2145         return eth_err(port_id, ret);
2146 }
2147
2148 int
2149 rte_eth_tx_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
2150                        uint16_t nb_tx_desc, unsigned int socket_id,
2151                        const struct rte_eth_txconf *tx_conf)
2152 {
2153         struct rte_eth_dev *dev;
2154         struct rte_eth_dev_info dev_info;
2155         struct rte_eth_txconf local_conf;
2156         void **txq;
2157         int ret;
2158
2159         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2160
2161         dev = &rte_eth_devices[port_id];
2162         if (tx_queue_id >= dev->data->nb_tx_queues) {
2163                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
2164                 return -EINVAL;
2165         }
2166
2167         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_setup, -ENOTSUP);
2168
2169         ret = rte_eth_dev_info_get(port_id, &dev_info);
2170         if (ret != 0)
2171                 return ret;
2172
2173         /* Use default specified by driver, if nb_tx_desc is zero */
2174         if (nb_tx_desc == 0) {
2175                 nb_tx_desc = dev_info.default_txportconf.ring_size;
2176                 /* If driver default is zero, fall back on EAL default */
2177                 if (nb_tx_desc == 0)
2178                         nb_tx_desc = RTE_ETH_DEV_FALLBACK_TX_RINGSIZE;
2179         }
2180         if (nb_tx_desc > dev_info.tx_desc_lim.nb_max ||
2181             nb_tx_desc < dev_info.tx_desc_lim.nb_min ||
2182             nb_tx_desc % dev_info.tx_desc_lim.nb_align != 0) {
2183                 RTE_ETHDEV_LOG(ERR,
2184                         "Invalid value for nb_tx_desc(=%hu), should be: <= %hu, >= %hu, and a product of %hu\n",
2185                         nb_tx_desc, dev_info.tx_desc_lim.nb_max,
2186                         dev_info.tx_desc_lim.nb_min,
2187                         dev_info.tx_desc_lim.nb_align);
2188                 return -EINVAL;
2189         }
2190
2191         if (dev->data->dev_started &&
2192                 !(dev_info.dev_capa &
2193                         RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP))
2194                 return -EBUSY;
2195
2196         if (dev->data->dev_started &&
2197                 (dev->data->tx_queue_state[tx_queue_id] !=
2198                         RTE_ETH_QUEUE_STATE_STOPPED))
2199                 return -EBUSY;
2200
2201         txq = dev->data->tx_queues;
2202         if (txq[tx_queue_id]) {
2203                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release,
2204                                         -ENOTSUP);
2205                 (*dev->dev_ops->tx_queue_release)(txq[tx_queue_id]);
2206                 txq[tx_queue_id] = NULL;
2207         }
2208
2209         if (tx_conf == NULL)
2210                 tx_conf = &dev_info.default_txconf;
2211
2212         local_conf = *tx_conf;
2213
2214         /*
2215          * If an offloading has already been enabled in
2216          * rte_eth_dev_configure(), it has been enabled on all queues,
2217          * so there is no need to enable it in this queue again.
2218          * The local_conf.offloads input to underlying PMD only carries
2219          * those offloadings which are only enabled on this queue and
2220          * not enabled on all queues.
2221          */
2222         local_conf.offloads &= ~dev->data->dev_conf.txmode.offloads;
2223
2224         /*
2225          * New added offloadings for this queue are those not enabled in
2226          * rte_eth_dev_configure() and they must be per-queue type.
2227          * A pure per-port offloading can't be enabled on a queue while
2228          * disabled on another queue. A pure per-port offloading can't
2229          * be enabled for any queue as new added one if it hasn't been
2230          * enabled in rte_eth_dev_configure().
2231          */
2232         if ((local_conf.offloads & dev_info.tx_queue_offload_capa) !=
2233              local_conf.offloads) {
2234                 RTE_ETHDEV_LOG(ERR,
2235                         "Ethdev port_id=%d tx_queue_id=%d, new added offloads 0x%"PRIx64" must be "
2236                         "within per-queue offload capabilities 0x%"PRIx64" in %s()\n",
2237                         port_id, tx_queue_id, local_conf.offloads,
2238                         dev_info.tx_queue_offload_capa,
2239                         __func__);
2240                 return -EINVAL;
2241         }
2242
2243         rte_ethdev_trace_txq_setup(port_id, tx_queue_id, nb_tx_desc, tx_conf);
2244         return eth_err(port_id, (*dev->dev_ops->tx_queue_setup)(dev,
2245                        tx_queue_id, nb_tx_desc, socket_id, &local_conf));
2246 }
2247
2248 int
2249 rte_eth_tx_hairpin_queue_setup(uint16_t port_id, uint16_t tx_queue_id,
2250                                uint16_t nb_tx_desc,
2251                                const struct rte_eth_hairpin_conf *conf)
2252 {
2253         struct rte_eth_dev *dev;
2254         struct rte_eth_hairpin_cap cap;
2255         void **txq;
2256         int i;
2257         int count;
2258         int ret;
2259
2260         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2261         dev = &rte_eth_devices[port_id];
2262         if (tx_queue_id >= dev->data->nb_tx_queues) {
2263                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", tx_queue_id);
2264                 return -EINVAL;
2265         }
2266         ret = rte_eth_dev_hairpin_capability_get(port_id, &cap);
2267         if (ret != 0)
2268                 return ret;
2269         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_hairpin_queue_setup,
2270                                 -ENOTSUP);
2271         /* if nb_rx_desc is zero use max number of desc from the driver. */
2272         if (nb_tx_desc == 0)
2273                 nb_tx_desc = cap.max_nb_desc;
2274         if (nb_tx_desc > cap.max_nb_desc) {
2275                 RTE_ETHDEV_LOG(ERR,
2276                         "Invalid value for nb_tx_desc(=%hu), should be: <= %hu",
2277                         nb_tx_desc, cap.max_nb_desc);
2278                 return -EINVAL;
2279         }
2280         if (conf->peer_count > cap.max_tx_2_rx) {
2281                 RTE_ETHDEV_LOG(ERR,
2282                         "Invalid value for number of peers for Tx queue(=%u), should be: <= %hu",
2283                         conf->peer_count, cap.max_tx_2_rx);
2284                 return -EINVAL;
2285         }
2286         if (conf->peer_count == 0) {
2287                 RTE_ETHDEV_LOG(ERR,
2288                         "Invalid value for number of peers for Tx queue(=%u), should be: > 0",
2289                         conf->peer_count);
2290                 return -EINVAL;
2291         }
2292         for (i = 0, count = 0; i < dev->data->nb_tx_queues &&
2293              cap.max_nb_queues != UINT16_MAX; i++) {
2294                 if (i == tx_queue_id || rte_eth_dev_is_tx_hairpin_queue(dev, i))
2295                         count++;
2296         }
2297         if (count > cap.max_nb_queues) {
2298                 RTE_ETHDEV_LOG(ERR, "To many Tx hairpin queues max is %d",
2299                 cap.max_nb_queues);
2300                 return -EINVAL;
2301         }
2302         if (dev->data->dev_started)
2303                 return -EBUSY;
2304         txq = dev->data->tx_queues;
2305         if (txq[tx_queue_id] != NULL) {
2306                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_queue_release,
2307                                         -ENOTSUP);
2308                 (*dev->dev_ops->tx_queue_release)(txq[tx_queue_id]);
2309                 txq[tx_queue_id] = NULL;
2310         }
2311         ret = (*dev->dev_ops->tx_hairpin_queue_setup)
2312                 (dev, tx_queue_id, nb_tx_desc, conf);
2313         if (ret == 0)
2314                 dev->data->tx_queue_state[tx_queue_id] =
2315                         RTE_ETH_QUEUE_STATE_HAIRPIN;
2316         return eth_err(port_id, ret);
2317 }
2318
2319 int
2320 rte_eth_hairpin_bind(uint16_t tx_port, uint16_t rx_port)
2321 {
2322         struct rte_eth_dev *dev;
2323         int ret;
2324
2325         RTE_ETH_VALID_PORTID_OR_ERR_RET(tx_port, -ENODEV);
2326         dev = &rte_eth_devices[tx_port];
2327         if (dev->data->dev_started == 0) {
2328                 RTE_ETHDEV_LOG(ERR, "Tx port %d is not started\n", tx_port);
2329                 return -EBUSY;
2330         }
2331
2332         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->hairpin_bind, -ENOTSUP);
2333         ret = (*dev->dev_ops->hairpin_bind)(dev, rx_port);
2334         if (ret != 0)
2335                 RTE_ETHDEV_LOG(ERR, "Failed to bind hairpin Tx %d"
2336                                " to Rx %d (%d - all ports)\n",
2337                                tx_port, rx_port, RTE_MAX_ETHPORTS);
2338
2339         return ret;
2340 }
2341
2342 int
2343 rte_eth_hairpin_unbind(uint16_t tx_port, uint16_t rx_port)
2344 {
2345         struct rte_eth_dev *dev;
2346         int ret;
2347
2348         RTE_ETH_VALID_PORTID_OR_ERR_RET(tx_port, -ENODEV);
2349         dev = &rte_eth_devices[tx_port];
2350         if (dev->data->dev_started == 0) {
2351                 RTE_ETHDEV_LOG(ERR, "Tx port %d is already stopped\n", tx_port);
2352                 return -EBUSY;
2353         }
2354
2355         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->hairpin_unbind, -ENOTSUP);
2356         ret = (*dev->dev_ops->hairpin_unbind)(dev, rx_port);
2357         if (ret != 0)
2358                 RTE_ETHDEV_LOG(ERR, "Failed to unbind hairpin Tx %d"
2359                                " from Rx %d (%d - all ports)\n",
2360                                tx_port, rx_port, RTE_MAX_ETHPORTS);
2361
2362         return ret;
2363 }
2364
2365 int
2366 rte_eth_hairpin_get_peer_ports(uint16_t port_id, uint16_t *peer_ports,
2367                                size_t len, uint32_t direction)
2368 {
2369         struct rte_eth_dev *dev;
2370         int ret;
2371
2372         if (peer_ports == NULL || len == 0)
2373                 return -EINVAL;
2374
2375         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2376         dev = &rte_eth_devices[port_id];
2377         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->hairpin_get_peer_ports,
2378                                 -ENOTSUP);
2379
2380         ret = (*dev->dev_ops->hairpin_get_peer_ports)(dev, peer_ports,
2381                                                       len, direction);
2382         if (ret < 0)
2383                 RTE_ETHDEV_LOG(ERR, "Failed to get %d hairpin peer %s ports\n",
2384                                port_id, direction ? "Rx" : "Tx");
2385
2386         return ret;
2387 }
2388
2389 void
2390 rte_eth_tx_buffer_drop_callback(struct rte_mbuf **pkts, uint16_t unsent,
2391                 void *userdata __rte_unused)
2392 {
2393         rte_pktmbuf_free_bulk(pkts, unsent);
2394 }
2395
2396 void
2397 rte_eth_tx_buffer_count_callback(struct rte_mbuf **pkts, uint16_t unsent,
2398                 void *userdata)
2399 {
2400         uint64_t *count = userdata;
2401
2402         rte_pktmbuf_free_bulk(pkts, unsent);
2403         *count += unsent;
2404 }
2405
2406 int
2407 rte_eth_tx_buffer_set_err_callback(struct rte_eth_dev_tx_buffer *buffer,
2408                 buffer_tx_error_fn cbfn, void *userdata)
2409 {
2410         buffer->error_callback = cbfn;
2411         buffer->error_userdata = userdata;
2412         return 0;
2413 }
2414
2415 int
2416 rte_eth_tx_buffer_init(struct rte_eth_dev_tx_buffer *buffer, uint16_t size)
2417 {
2418         int ret = 0;
2419
2420         if (buffer == NULL)
2421                 return -EINVAL;
2422
2423         buffer->size = size;
2424         if (buffer->error_callback == NULL) {
2425                 ret = rte_eth_tx_buffer_set_err_callback(
2426                         buffer, rte_eth_tx_buffer_drop_callback, NULL);
2427         }
2428
2429         return ret;
2430 }
2431
2432 int
2433 rte_eth_tx_done_cleanup(uint16_t port_id, uint16_t queue_id, uint32_t free_cnt)
2434 {
2435         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
2436         int ret;
2437
2438         /* Validate Input Data. Bail if not valid or not supported. */
2439         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2440         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_done_cleanup, -ENOTSUP);
2441
2442         /* Call driver to free pending mbufs. */
2443         ret = (*dev->dev_ops->tx_done_cleanup)(dev->data->tx_queues[queue_id],
2444                                                free_cnt);
2445         return eth_err(port_id, ret);
2446 }
2447
2448 int
2449 rte_eth_promiscuous_enable(uint16_t port_id)
2450 {
2451         struct rte_eth_dev *dev;
2452         int diag = 0;
2453
2454         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2455         dev = &rte_eth_devices[port_id];
2456
2457         if (dev->data->promiscuous == 1)
2458                 return 0;
2459
2460         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->promiscuous_enable, -ENOTSUP);
2461
2462         diag = (*dev->dev_ops->promiscuous_enable)(dev);
2463         dev->data->promiscuous = (diag == 0) ? 1 : 0;
2464
2465         return eth_err(port_id, diag);
2466 }
2467
2468 int
2469 rte_eth_promiscuous_disable(uint16_t port_id)
2470 {
2471         struct rte_eth_dev *dev;
2472         int diag = 0;
2473
2474         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2475         dev = &rte_eth_devices[port_id];
2476
2477         if (dev->data->promiscuous == 0)
2478                 return 0;
2479
2480         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->promiscuous_disable, -ENOTSUP);
2481
2482         dev->data->promiscuous = 0;
2483         diag = (*dev->dev_ops->promiscuous_disable)(dev);
2484         if (diag != 0)
2485                 dev->data->promiscuous = 1;
2486
2487         return eth_err(port_id, diag);
2488 }
2489
2490 int
2491 rte_eth_promiscuous_get(uint16_t port_id)
2492 {
2493         struct rte_eth_dev *dev;
2494
2495         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2496
2497         dev = &rte_eth_devices[port_id];
2498         return dev->data->promiscuous;
2499 }
2500
2501 int
2502 rte_eth_allmulticast_enable(uint16_t port_id)
2503 {
2504         struct rte_eth_dev *dev;
2505         int diag;
2506
2507         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2508         dev = &rte_eth_devices[port_id];
2509
2510         if (dev->data->all_multicast == 1)
2511                 return 0;
2512
2513         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->allmulticast_enable, -ENOTSUP);
2514         diag = (*dev->dev_ops->allmulticast_enable)(dev);
2515         dev->data->all_multicast = (diag == 0) ? 1 : 0;
2516
2517         return eth_err(port_id, diag);
2518 }
2519
2520 int
2521 rte_eth_allmulticast_disable(uint16_t port_id)
2522 {
2523         struct rte_eth_dev *dev;
2524         int diag;
2525
2526         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2527         dev = &rte_eth_devices[port_id];
2528
2529         if (dev->data->all_multicast == 0)
2530                 return 0;
2531
2532         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->allmulticast_disable, -ENOTSUP);
2533         dev->data->all_multicast = 0;
2534         diag = (*dev->dev_ops->allmulticast_disable)(dev);
2535         if (diag != 0)
2536                 dev->data->all_multicast = 1;
2537
2538         return eth_err(port_id, diag);
2539 }
2540
2541 int
2542 rte_eth_allmulticast_get(uint16_t port_id)
2543 {
2544         struct rte_eth_dev *dev;
2545
2546         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2547
2548         dev = &rte_eth_devices[port_id];
2549         return dev->data->all_multicast;
2550 }
2551
2552 int
2553 rte_eth_link_get(uint16_t port_id, struct rte_eth_link *eth_link)
2554 {
2555         struct rte_eth_dev *dev;
2556
2557         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2558         dev = &rte_eth_devices[port_id];
2559
2560         if (dev->data->dev_conf.intr_conf.lsc &&
2561             dev->data->dev_started)
2562                 rte_eth_linkstatus_get(dev, eth_link);
2563         else {
2564                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
2565                 (*dev->dev_ops->link_update)(dev, 1);
2566                 *eth_link = dev->data->dev_link;
2567         }
2568
2569         return 0;
2570 }
2571
2572 int
2573 rte_eth_link_get_nowait(uint16_t port_id, struct rte_eth_link *eth_link)
2574 {
2575         struct rte_eth_dev *dev;
2576
2577         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2578         dev = &rte_eth_devices[port_id];
2579
2580         if (dev->data->dev_conf.intr_conf.lsc &&
2581             dev->data->dev_started)
2582                 rte_eth_linkstatus_get(dev, eth_link);
2583         else {
2584                 RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->link_update, -ENOTSUP);
2585                 (*dev->dev_ops->link_update)(dev, 0);
2586                 *eth_link = dev->data->dev_link;
2587         }
2588
2589         return 0;
2590 }
2591
2592 const char *
2593 rte_eth_link_speed_to_str(uint32_t link_speed)
2594 {
2595         switch (link_speed) {
2596         case ETH_SPEED_NUM_NONE: return "None";
2597         case ETH_SPEED_NUM_10M:  return "10 Mbps";
2598         case ETH_SPEED_NUM_100M: return "100 Mbps";
2599         case ETH_SPEED_NUM_1G:   return "1 Gbps";
2600         case ETH_SPEED_NUM_2_5G: return "2.5 Gbps";
2601         case ETH_SPEED_NUM_5G:   return "5 Gbps";
2602         case ETH_SPEED_NUM_10G:  return "10 Gbps";
2603         case ETH_SPEED_NUM_20G:  return "20 Gbps";
2604         case ETH_SPEED_NUM_25G:  return "25 Gbps";
2605         case ETH_SPEED_NUM_40G:  return "40 Gbps";
2606         case ETH_SPEED_NUM_50G:  return "50 Gbps";
2607         case ETH_SPEED_NUM_56G:  return "56 Gbps";
2608         case ETH_SPEED_NUM_100G: return "100 Gbps";
2609         case ETH_SPEED_NUM_200G: return "200 Gbps";
2610         case ETH_SPEED_NUM_UNKNOWN: return "Unknown";
2611         default: return "Invalid";
2612         }
2613 }
2614
2615 int
2616 rte_eth_link_to_str(char *str, size_t len, const struct rte_eth_link *eth_link)
2617 {
2618         if (eth_link->link_status == ETH_LINK_DOWN)
2619                 return snprintf(str, len, "Link down");
2620         else
2621                 return snprintf(str, len, "Link up at %s %s %s",
2622                         rte_eth_link_speed_to_str(eth_link->link_speed),
2623                         (eth_link->link_duplex == ETH_LINK_FULL_DUPLEX) ?
2624                         "FDX" : "HDX",
2625                         (eth_link->link_autoneg == ETH_LINK_AUTONEG) ?
2626                         "Autoneg" : "Fixed");
2627 }
2628
2629 int
2630 rte_eth_stats_get(uint16_t port_id, struct rte_eth_stats *stats)
2631 {
2632         struct rte_eth_dev *dev;
2633
2634         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2635
2636         dev = &rte_eth_devices[port_id];
2637         memset(stats, 0, sizeof(*stats));
2638
2639         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_get, -ENOTSUP);
2640         stats->rx_nombuf = dev->data->rx_mbuf_alloc_failed;
2641         return eth_err(port_id, (*dev->dev_ops->stats_get)(dev, stats));
2642 }
2643
2644 int
2645 rte_eth_stats_reset(uint16_t port_id)
2646 {
2647         struct rte_eth_dev *dev;
2648         int ret;
2649
2650         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2651         dev = &rte_eth_devices[port_id];
2652
2653         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->stats_reset, -ENOTSUP);
2654         ret = (*dev->dev_ops->stats_reset)(dev);
2655         if (ret != 0)
2656                 return eth_err(port_id, ret);
2657
2658         dev->data->rx_mbuf_alloc_failed = 0;
2659
2660         return 0;
2661 }
2662
2663 static inline int
2664 get_xstats_basic_count(struct rte_eth_dev *dev)
2665 {
2666         uint16_t nb_rxqs, nb_txqs;
2667         int count;
2668
2669         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2670         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2671
2672         count = RTE_NB_STATS;
2673         count += nb_rxqs * RTE_NB_RXQ_STATS;
2674         count += nb_txqs * RTE_NB_TXQ_STATS;
2675
2676         return count;
2677 }
2678
2679 static int
2680 get_xstats_count(uint16_t port_id)
2681 {
2682         struct rte_eth_dev *dev;
2683         int count;
2684
2685         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
2686         dev = &rte_eth_devices[port_id];
2687         if (dev->dev_ops->xstats_get_names_by_id != NULL) {
2688                 count = (*dev->dev_ops->xstats_get_names_by_id)(dev, NULL,
2689                                 NULL, 0);
2690                 if (count < 0)
2691                         return eth_err(port_id, count);
2692         }
2693         if (dev->dev_ops->xstats_get_names != NULL) {
2694                 count = (*dev->dev_ops->xstats_get_names)(dev, NULL, 0);
2695                 if (count < 0)
2696                         return eth_err(port_id, count);
2697         } else
2698                 count = 0;
2699
2700
2701         count += get_xstats_basic_count(dev);
2702
2703         return count;
2704 }
2705
2706 int
2707 rte_eth_xstats_get_id_by_name(uint16_t port_id, const char *xstat_name,
2708                 uint64_t *id)
2709 {
2710         int cnt_xstats, idx_xstat;
2711
2712         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2713
2714         if (!id) {
2715                 RTE_ETHDEV_LOG(ERR, "Id pointer is NULL\n");
2716                 return -ENOMEM;
2717         }
2718
2719         if (!xstat_name) {
2720                 RTE_ETHDEV_LOG(ERR, "xstat_name pointer is NULL\n");
2721                 return -ENOMEM;
2722         }
2723
2724         /* Get count */
2725         cnt_xstats = rte_eth_xstats_get_names_by_id(port_id, NULL, 0, NULL);
2726         if (cnt_xstats  < 0) {
2727                 RTE_ETHDEV_LOG(ERR, "Cannot get count of xstats\n");
2728                 return -ENODEV;
2729         }
2730
2731         /* Get id-name lookup table */
2732         struct rte_eth_xstat_name xstats_names[cnt_xstats];
2733
2734         if (cnt_xstats != rte_eth_xstats_get_names_by_id(
2735                         port_id, xstats_names, cnt_xstats, NULL)) {
2736                 RTE_ETHDEV_LOG(ERR, "Cannot get xstats lookup\n");
2737                 return -1;
2738         }
2739
2740         for (idx_xstat = 0; idx_xstat < cnt_xstats; idx_xstat++) {
2741                 if (!strcmp(xstats_names[idx_xstat].name, xstat_name)) {
2742                         *id = idx_xstat;
2743                         return 0;
2744                 };
2745         }
2746
2747         return -EINVAL;
2748 }
2749
2750 /* retrieve basic stats names */
2751 static int
2752 rte_eth_basic_stats_get_names(struct rte_eth_dev *dev,
2753         struct rte_eth_xstat_name *xstats_names)
2754 {
2755         int cnt_used_entries = 0;
2756         uint32_t idx, id_queue;
2757         uint16_t num_q;
2758
2759         for (idx = 0; idx < RTE_NB_STATS; idx++) {
2760                 strlcpy(xstats_names[cnt_used_entries].name,
2761                         rte_stats_strings[idx].name,
2762                         sizeof(xstats_names[0].name));
2763                 cnt_used_entries++;
2764         }
2765         num_q = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2766         for (id_queue = 0; id_queue < num_q; id_queue++) {
2767                 for (idx = 0; idx < RTE_NB_RXQ_STATS; idx++) {
2768                         snprintf(xstats_names[cnt_used_entries].name,
2769                                 sizeof(xstats_names[0].name),
2770                                 "rx_q%u_%s",
2771                                 id_queue, rte_rxq_stats_strings[idx].name);
2772                         cnt_used_entries++;
2773                 }
2774
2775         }
2776         num_q = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2777         for (id_queue = 0; id_queue < num_q; id_queue++) {
2778                 for (idx = 0; idx < RTE_NB_TXQ_STATS; idx++) {
2779                         snprintf(xstats_names[cnt_used_entries].name,
2780                                 sizeof(xstats_names[0].name),
2781                                 "tx_q%u_%s",
2782                                 id_queue, rte_txq_stats_strings[idx].name);
2783                         cnt_used_entries++;
2784                 }
2785         }
2786         return cnt_used_entries;
2787 }
2788
2789 /* retrieve ethdev extended statistics names */
2790 int
2791 rte_eth_xstats_get_names_by_id(uint16_t port_id,
2792         struct rte_eth_xstat_name *xstats_names, unsigned int size,
2793         uint64_t *ids)
2794 {
2795         struct rte_eth_xstat_name *xstats_names_copy;
2796         unsigned int no_basic_stat_requested = 1;
2797         unsigned int no_ext_stat_requested = 1;
2798         unsigned int expected_entries;
2799         unsigned int basic_count;
2800         struct rte_eth_dev *dev;
2801         unsigned int i;
2802         int ret;
2803
2804         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
2805         dev = &rte_eth_devices[port_id];
2806
2807         basic_count = get_xstats_basic_count(dev);
2808         ret = get_xstats_count(port_id);
2809         if (ret < 0)
2810                 return ret;
2811         expected_entries = (unsigned int)ret;
2812
2813         /* Return max number of stats if no ids given */
2814         if (!ids) {
2815                 if (!xstats_names)
2816                         return expected_entries;
2817                 else if (xstats_names && size < expected_entries)
2818                         return expected_entries;
2819         }
2820
2821         if (ids && !xstats_names)
2822                 return -EINVAL;
2823
2824         if (ids && dev->dev_ops->xstats_get_names_by_id != NULL && size > 0) {
2825                 uint64_t ids_copy[size];
2826
2827                 for (i = 0; i < size; i++) {
2828                         if (ids[i] < basic_count) {
2829                                 no_basic_stat_requested = 0;
2830                                 break;
2831                         }
2832
2833                         /*
2834                          * Convert ids to xstats ids that PMD knows.
2835                          * ids known by user are basic + extended stats.
2836                          */
2837                         ids_copy[i] = ids[i] - basic_count;
2838                 }
2839
2840                 if (no_basic_stat_requested)
2841                         return (*dev->dev_ops->xstats_get_names_by_id)(dev,
2842                                         xstats_names, ids_copy, size);
2843         }
2844
2845         /* Retrieve all stats */
2846         if (!ids) {
2847                 int num_stats = rte_eth_xstats_get_names(port_id, xstats_names,
2848                                 expected_entries);
2849                 if (num_stats < 0 || num_stats > (int)expected_entries)
2850                         return num_stats;
2851                 else
2852                         return expected_entries;
2853         }
2854
2855         xstats_names_copy = calloc(expected_entries,
2856                 sizeof(struct rte_eth_xstat_name));
2857
2858         if (!xstats_names_copy) {
2859                 RTE_ETHDEV_LOG(ERR, "Can't allocate memory\n");
2860                 return -ENOMEM;
2861         }
2862
2863         if (ids) {
2864                 for (i = 0; i < size; i++) {
2865                         if (ids[i] >= basic_count) {
2866                                 no_ext_stat_requested = 0;
2867                                 break;
2868                         }
2869                 }
2870         }
2871
2872         /* Fill xstats_names_copy structure */
2873         if (ids && no_ext_stat_requested) {
2874                 rte_eth_basic_stats_get_names(dev, xstats_names_copy);
2875         } else {
2876                 ret = rte_eth_xstats_get_names(port_id, xstats_names_copy,
2877                         expected_entries);
2878                 if (ret < 0) {
2879                         free(xstats_names_copy);
2880                         return ret;
2881                 }
2882         }
2883
2884         /* Filter stats */
2885         for (i = 0; i < size; i++) {
2886                 if (ids[i] >= expected_entries) {
2887                         RTE_ETHDEV_LOG(ERR, "Id value isn't valid\n");
2888                         free(xstats_names_copy);
2889                         return -1;
2890                 }
2891                 xstats_names[i] = xstats_names_copy[ids[i]];
2892         }
2893
2894         free(xstats_names_copy);
2895         return size;
2896 }
2897
2898 int
2899 rte_eth_xstats_get_names(uint16_t port_id,
2900         struct rte_eth_xstat_name *xstats_names,
2901         unsigned int size)
2902 {
2903         struct rte_eth_dev *dev;
2904         int cnt_used_entries;
2905         int cnt_expected_entries;
2906         int cnt_driver_entries;
2907
2908         cnt_expected_entries = get_xstats_count(port_id);
2909         if (xstats_names == NULL || cnt_expected_entries < 0 ||
2910                         (int)size < cnt_expected_entries)
2911                 return cnt_expected_entries;
2912
2913         /* port_id checked in get_xstats_count() */
2914         dev = &rte_eth_devices[port_id];
2915
2916         cnt_used_entries = rte_eth_basic_stats_get_names(
2917                 dev, xstats_names);
2918
2919         if (dev->dev_ops->xstats_get_names != NULL) {
2920                 /* If there are any driver-specific xstats, append them
2921                  * to end of list.
2922                  */
2923                 cnt_driver_entries = (*dev->dev_ops->xstats_get_names)(
2924                         dev,
2925                         xstats_names + cnt_used_entries,
2926                         size - cnt_used_entries);
2927                 if (cnt_driver_entries < 0)
2928                         return eth_err(port_id, cnt_driver_entries);
2929                 cnt_used_entries += cnt_driver_entries;
2930         }
2931
2932         return cnt_used_entries;
2933 }
2934
2935
2936 static int
2937 rte_eth_basic_stats_get(uint16_t port_id, struct rte_eth_xstat *xstats)
2938 {
2939         struct rte_eth_dev *dev;
2940         struct rte_eth_stats eth_stats;
2941         unsigned int count = 0, i, q;
2942         uint64_t val, *stats_ptr;
2943         uint16_t nb_rxqs, nb_txqs;
2944         int ret;
2945
2946         ret = rte_eth_stats_get(port_id, &eth_stats);
2947         if (ret < 0)
2948                 return ret;
2949
2950         dev = &rte_eth_devices[port_id];
2951
2952         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2953         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
2954
2955         /* global stats */
2956         for (i = 0; i < RTE_NB_STATS; i++) {
2957                 stats_ptr = RTE_PTR_ADD(&eth_stats,
2958                                         rte_stats_strings[i].offset);
2959                 val = *stats_ptr;
2960                 xstats[count++].value = val;
2961         }
2962
2963         /* per-rxq stats */
2964         for (q = 0; q < nb_rxqs; q++) {
2965                 for (i = 0; i < RTE_NB_RXQ_STATS; i++) {
2966                         stats_ptr = RTE_PTR_ADD(&eth_stats,
2967                                         rte_rxq_stats_strings[i].offset +
2968                                         q * sizeof(uint64_t));
2969                         val = *stats_ptr;
2970                         xstats[count++].value = val;
2971                 }
2972         }
2973
2974         /* per-txq stats */
2975         for (q = 0; q < nb_txqs; q++) {
2976                 for (i = 0; i < RTE_NB_TXQ_STATS; i++) {
2977                         stats_ptr = RTE_PTR_ADD(&eth_stats,
2978                                         rte_txq_stats_strings[i].offset +
2979                                         q * sizeof(uint64_t));
2980                         val = *stats_ptr;
2981                         xstats[count++].value = val;
2982                 }
2983         }
2984         return count;
2985 }
2986
2987 /* retrieve ethdev extended statistics */
2988 int
2989 rte_eth_xstats_get_by_id(uint16_t port_id, const uint64_t *ids,
2990                          uint64_t *values, unsigned int size)
2991 {
2992         unsigned int no_basic_stat_requested = 1;
2993         unsigned int no_ext_stat_requested = 1;
2994         unsigned int num_xstats_filled;
2995         unsigned int basic_count;
2996         uint16_t expected_entries;
2997         struct rte_eth_dev *dev;
2998         unsigned int i;
2999         int ret;
3000
3001         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3002         ret = get_xstats_count(port_id);
3003         if (ret < 0)
3004                 return ret;
3005         expected_entries = (uint16_t)ret;
3006         struct rte_eth_xstat xstats[expected_entries];
3007         dev = &rte_eth_devices[port_id];
3008         basic_count = get_xstats_basic_count(dev);
3009
3010         /* Return max number of stats if no ids given */
3011         if (!ids) {
3012                 if (!values)
3013                         return expected_entries;
3014                 else if (values && size < expected_entries)
3015                         return expected_entries;
3016         }
3017
3018         if (ids && !values)
3019                 return -EINVAL;
3020
3021         if (ids && dev->dev_ops->xstats_get_by_id != NULL && size) {
3022                 unsigned int basic_count = get_xstats_basic_count(dev);
3023                 uint64_t ids_copy[size];
3024
3025                 for (i = 0; i < size; i++) {
3026                         if (ids[i] < basic_count) {
3027                                 no_basic_stat_requested = 0;
3028                                 break;
3029                         }
3030
3031                         /*
3032                          * Convert ids to xstats ids that PMD knows.
3033                          * ids known by user are basic + extended stats.
3034                          */
3035                         ids_copy[i] = ids[i] - basic_count;
3036                 }
3037
3038                 if (no_basic_stat_requested)
3039                         return (*dev->dev_ops->xstats_get_by_id)(dev, ids_copy,
3040                                         values, size);
3041         }
3042
3043         if (ids) {
3044                 for (i = 0; i < size; i++) {
3045                         if (ids[i] >= basic_count) {
3046                                 no_ext_stat_requested = 0;
3047                                 break;
3048                         }
3049                 }
3050         }
3051
3052         /* Fill the xstats structure */
3053         if (ids && no_ext_stat_requested)
3054                 ret = rte_eth_basic_stats_get(port_id, xstats);
3055         else
3056                 ret = rte_eth_xstats_get(port_id, xstats, expected_entries);
3057
3058         if (ret < 0)
3059                 return ret;
3060         num_xstats_filled = (unsigned int)ret;
3061
3062         /* Return all stats */
3063         if (!ids) {
3064                 for (i = 0; i < num_xstats_filled; i++)
3065                         values[i] = xstats[i].value;
3066                 return expected_entries;
3067         }
3068
3069         /* Filter stats */
3070         for (i = 0; i < size; i++) {
3071                 if (ids[i] >= expected_entries) {
3072                         RTE_ETHDEV_LOG(ERR, "Id value isn't valid\n");
3073                         return -1;
3074                 }
3075                 values[i] = xstats[ids[i]].value;
3076         }
3077         return size;
3078 }
3079
3080 int
3081 rte_eth_xstats_get(uint16_t port_id, struct rte_eth_xstat *xstats,
3082         unsigned int n)
3083 {
3084         struct rte_eth_dev *dev;
3085         unsigned int count = 0, i;
3086         signed int xcount = 0;
3087         uint16_t nb_rxqs, nb_txqs;
3088         int ret;
3089
3090         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
3091
3092         dev = &rte_eth_devices[port_id];
3093
3094         nb_rxqs = RTE_MIN(dev->data->nb_rx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
3095         nb_txqs = RTE_MIN(dev->data->nb_tx_queues, RTE_ETHDEV_QUEUE_STAT_CNTRS);
3096
3097         /* Return generic statistics */
3098         count = RTE_NB_STATS + (nb_rxqs * RTE_NB_RXQ_STATS) +
3099                 (nb_txqs * RTE_NB_TXQ_STATS);
3100
3101         /* implemented by the driver */
3102         if (dev->dev_ops->xstats_get != NULL) {
3103                 /* Retrieve the xstats from the driver at the end of the
3104                  * xstats struct.
3105                  */
3106                 xcount = (*dev->dev_ops->xstats_get)(dev,
3107                                      xstats ? xstats + count : NULL,
3108                                      (n > count) ? n - count : 0);
3109
3110                 if (xcount < 0)
3111                         return eth_err(port_id, xcount);
3112         }
3113
3114         if (n < count + xcount || xstats == NULL)
3115                 return count + xcount;
3116
3117         /* now fill the xstats structure */
3118         ret = rte_eth_basic_stats_get(port_id, xstats);
3119         if (ret < 0)
3120                 return ret;
3121         count = ret;
3122
3123         for (i = 0; i < count; i++)
3124                 xstats[i].id = i;
3125         /* add an offset to driver-specific stats */
3126         for ( ; i < count + xcount; i++)
3127                 xstats[i].id += count;
3128
3129         return count + xcount;
3130 }
3131
3132 /* reset ethdev extended statistics */
3133 int
3134 rte_eth_xstats_reset(uint16_t port_id)
3135 {
3136         struct rte_eth_dev *dev;
3137
3138         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3139         dev = &rte_eth_devices[port_id];
3140
3141         /* implemented by the driver */
3142         if (dev->dev_ops->xstats_reset != NULL)
3143                 return eth_err(port_id, (*dev->dev_ops->xstats_reset)(dev));
3144
3145         /* fallback to default */
3146         return rte_eth_stats_reset(port_id);
3147 }
3148
3149 static int
3150 set_queue_stats_mapping(uint16_t port_id, uint16_t queue_id, uint8_t stat_idx,
3151                 uint8_t is_rx)
3152 {
3153         struct rte_eth_dev *dev;
3154
3155         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3156
3157         dev = &rte_eth_devices[port_id];
3158
3159         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->queue_stats_mapping_set, -ENOTSUP);
3160
3161         if (is_rx && (queue_id >= dev->data->nb_rx_queues))
3162                 return -EINVAL;
3163
3164         if (!is_rx && (queue_id >= dev->data->nb_tx_queues))
3165                 return -EINVAL;
3166
3167         if (stat_idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)
3168                 return -EINVAL;
3169
3170         return (*dev->dev_ops->queue_stats_mapping_set)
3171                         (dev, queue_id, stat_idx, is_rx);
3172 }
3173
3174
3175 int
3176 rte_eth_dev_set_tx_queue_stats_mapping(uint16_t port_id, uint16_t tx_queue_id,
3177                 uint8_t stat_idx)
3178 {
3179         return eth_err(port_id, set_queue_stats_mapping(port_id, tx_queue_id,
3180                                                 stat_idx, STAT_QMAP_TX));
3181 }
3182
3183
3184 int
3185 rte_eth_dev_set_rx_queue_stats_mapping(uint16_t port_id, uint16_t rx_queue_id,
3186                 uint8_t stat_idx)
3187 {
3188         return eth_err(port_id, set_queue_stats_mapping(port_id, rx_queue_id,
3189                                                 stat_idx, STAT_QMAP_RX));
3190 }
3191
3192 int
3193 rte_eth_dev_fw_version_get(uint16_t port_id, char *fw_version, size_t fw_size)
3194 {
3195         struct rte_eth_dev *dev;
3196
3197         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3198         dev = &rte_eth_devices[port_id];
3199
3200         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fw_version_get, -ENOTSUP);
3201         return eth_err(port_id, (*dev->dev_ops->fw_version_get)(dev,
3202                                                         fw_version, fw_size));
3203 }
3204
3205 int
3206 rte_eth_dev_info_get(uint16_t port_id, struct rte_eth_dev_info *dev_info)
3207 {
3208         struct rte_eth_dev *dev;
3209         const struct rte_eth_desc_lim lim = {
3210                 .nb_max = UINT16_MAX,
3211                 .nb_min = 0,
3212                 .nb_align = 1,
3213                 .nb_seg_max = UINT16_MAX,
3214                 .nb_mtu_seg_max = UINT16_MAX,
3215         };
3216         int diag;
3217
3218         /*
3219          * Init dev_info before port_id check since caller does not have
3220          * return status and does not know if get is successful or not.
3221          */
3222         memset(dev_info, 0, sizeof(struct rte_eth_dev_info));
3223         dev_info->switch_info.domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
3224
3225         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3226         dev = &rte_eth_devices[port_id];
3227
3228         dev_info->rx_desc_lim = lim;
3229         dev_info->tx_desc_lim = lim;
3230         dev_info->device = dev->device;
3231         dev_info->min_mtu = RTE_ETHER_MIN_MTU;
3232         dev_info->max_mtu = UINT16_MAX;
3233
3234         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_infos_get, -ENOTSUP);
3235         diag = (*dev->dev_ops->dev_infos_get)(dev, dev_info);
3236         if (diag != 0) {
3237                 /* Cleanup already filled in device information */
3238                 memset(dev_info, 0, sizeof(struct rte_eth_dev_info));
3239                 return eth_err(port_id, diag);
3240         }
3241
3242         /* Maximum number of queues should be <= RTE_MAX_QUEUES_PER_PORT */
3243         dev_info->max_rx_queues = RTE_MIN(dev_info->max_rx_queues,
3244                         RTE_MAX_QUEUES_PER_PORT);
3245         dev_info->max_tx_queues = RTE_MIN(dev_info->max_tx_queues,
3246                         RTE_MAX_QUEUES_PER_PORT);
3247
3248         dev_info->driver_name = dev->device->driver->name;
3249         dev_info->nb_rx_queues = dev->data->nb_rx_queues;
3250         dev_info->nb_tx_queues = dev->data->nb_tx_queues;
3251
3252         dev_info->dev_flags = &dev->data->dev_flags;
3253
3254         return 0;
3255 }
3256
3257 int
3258 rte_eth_dev_get_supported_ptypes(uint16_t port_id, uint32_t ptype_mask,
3259                                  uint32_t *ptypes, int num)
3260 {
3261         int i, j;
3262         struct rte_eth_dev *dev;
3263         const uint32_t *all_ptypes;
3264
3265         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3266         dev = &rte_eth_devices[port_id];
3267         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_supported_ptypes_get, 0);
3268         all_ptypes = (*dev->dev_ops->dev_supported_ptypes_get)(dev);
3269
3270         if (!all_ptypes)
3271                 return 0;
3272
3273         for (i = 0, j = 0; all_ptypes[i] != RTE_PTYPE_UNKNOWN; ++i)
3274                 if (all_ptypes[i] & ptype_mask) {
3275                         if (j < num)
3276                                 ptypes[j] = all_ptypes[i];
3277                         j++;
3278                 }
3279
3280         return j;
3281 }
3282
3283 int
3284 rte_eth_dev_set_ptypes(uint16_t port_id, uint32_t ptype_mask,
3285                                  uint32_t *set_ptypes, unsigned int num)
3286 {
3287         const uint32_t valid_ptype_masks[] = {
3288                 RTE_PTYPE_L2_MASK,
3289                 RTE_PTYPE_L3_MASK,
3290                 RTE_PTYPE_L4_MASK,
3291                 RTE_PTYPE_TUNNEL_MASK,
3292                 RTE_PTYPE_INNER_L2_MASK,
3293                 RTE_PTYPE_INNER_L3_MASK,
3294                 RTE_PTYPE_INNER_L4_MASK,
3295         };
3296         const uint32_t *all_ptypes;
3297         struct rte_eth_dev *dev;
3298         uint32_t unused_mask;
3299         unsigned int i, j;
3300         int ret;
3301
3302         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3303         dev = &rte_eth_devices[port_id];
3304
3305         if (num > 0 && set_ptypes == NULL)
3306                 return -EINVAL;
3307
3308         if (*dev->dev_ops->dev_supported_ptypes_get == NULL ||
3309                         *dev->dev_ops->dev_ptypes_set == NULL) {
3310                 ret = 0;
3311                 goto ptype_unknown;
3312         }
3313
3314         if (ptype_mask == 0) {
3315                 ret = (*dev->dev_ops->dev_ptypes_set)(dev,
3316                                 ptype_mask);
3317                 goto ptype_unknown;
3318         }
3319
3320         unused_mask = ptype_mask;
3321         for (i = 0; i < RTE_DIM(valid_ptype_masks); i++) {
3322                 uint32_t mask = ptype_mask & valid_ptype_masks[i];
3323                 if (mask && mask != valid_ptype_masks[i]) {
3324                         ret = -EINVAL;
3325                         goto ptype_unknown;
3326                 }
3327                 unused_mask &= ~valid_ptype_masks[i];
3328         }
3329
3330         if (unused_mask) {
3331                 ret = -EINVAL;
3332                 goto ptype_unknown;
3333         }
3334
3335         all_ptypes = (*dev->dev_ops->dev_supported_ptypes_get)(dev);
3336         if (all_ptypes == NULL) {
3337                 ret = 0;
3338                 goto ptype_unknown;
3339         }
3340
3341         /*
3342          * Accommodate as many set_ptypes as possible. If the supplied
3343          * set_ptypes array is insufficient fill it partially.
3344          */
3345         for (i = 0, j = 0; set_ptypes != NULL &&
3346                                 (all_ptypes[i] != RTE_PTYPE_UNKNOWN); ++i) {
3347                 if (ptype_mask & all_ptypes[i]) {
3348                         if (j < num - 1) {
3349                                 set_ptypes[j] = all_ptypes[i];
3350                                 j++;
3351                                 continue;
3352                         }
3353                         break;
3354                 }
3355         }
3356
3357         if (set_ptypes != NULL && j < num)
3358                 set_ptypes[j] = RTE_PTYPE_UNKNOWN;
3359
3360         return (*dev->dev_ops->dev_ptypes_set)(dev, ptype_mask);
3361
3362 ptype_unknown:
3363         if (num > 0)
3364                 set_ptypes[0] = RTE_PTYPE_UNKNOWN;
3365
3366         return ret;
3367 }
3368
3369 int
3370 rte_eth_macaddr_get(uint16_t port_id, struct rte_ether_addr *mac_addr)
3371 {
3372         struct rte_eth_dev *dev;
3373
3374         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3375         dev = &rte_eth_devices[port_id];
3376         rte_ether_addr_copy(&dev->data->mac_addrs[0], mac_addr);
3377
3378         return 0;
3379 }
3380
3381 int
3382 rte_eth_dev_get_mtu(uint16_t port_id, uint16_t *mtu)
3383 {
3384         struct rte_eth_dev *dev;
3385
3386         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3387
3388         dev = &rte_eth_devices[port_id];
3389         *mtu = dev->data->mtu;
3390         return 0;
3391 }
3392
3393 int
3394 rte_eth_dev_set_mtu(uint16_t port_id, uint16_t mtu)
3395 {
3396         int ret;
3397         struct rte_eth_dev_info dev_info;
3398         struct rte_eth_dev *dev;
3399
3400         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3401         dev = &rte_eth_devices[port_id];
3402         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mtu_set, -ENOTSUP);
3403
3404         /*
3405          * Check if the device supports dev_infos_get, if it does not
3406          * skip min_mtu/max_mtu validation here as this requires values
3407          * that are populated within the call to rte_eth_dev_info_get()
3408          * which relies on dev->dev_ops->dev_infos_get.
3409          */
3410         if (*dev->dev_ops->dev_infos_get != NULL) {
3411                 ret = rte_eth_dev_info_get(port_id, &dev_info);
3412                 if (ret != 0)
3413                         return ret;
3414
3415                 if (mtu < dev_info.min_mtu || mtu > dev_info.max_mtu)
3416                         return -EINVAL;
3417         }
3418
3419         ret = (*dev->dev_ops->mtu_set)(dev, mtu);
3420         if (!ret)
3421                 dev->data->mtu = mtu;
3422
3423         return eth_err(port_id, ret);
3424 }
3425
3426 int
3427 rte_eth_dev_vlan_filter(uint16_t port_id, uint16_t vlan_id, int on)
3428 {
3429         struct rte_eth_dev *dev;
3430         int ret;
3431
3432         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3433         dev = &rte_eth_devices[port_id];
3434         if (!(dev->data->dev_conf.rxmode.offloads &
3435               DEV_RX_OFFLOAD_VLAN_FILTER)) {
3436                 RTE_ETHDEV_LOG(ERR, "Port %u: vlan-filtering disabled\n",
3437                         port_id);
3438                 return -ENOSYS;
3439         }
3440
3441         if (vlan_id > 4095) {
3442                 RTE_ETHDEV_LOG(ERR, "Port_id=%u invalid vlan_id=%u > 4095\n",
3443                         port_id, vlan_id);
3444                 return -EINVAL;
3445         }
3446         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_filter_set, -ENOTSUP);
3447
3448         ret = (*dev->dev_ops->vlan_filter_set)(dev, vlan_id, on);
3449         if (ret == 0) {
3450                 struct rte_vlan_filter_conf *vfc;
3451                 int vidx;
3452                 int vbit;
3453
3454                 vfc = &dev->data->vlan_filter_conf;
3455                 vidx = vlan_id / 64;
3456                 vbit = vlan_id % 64;
3457
3458                 if (on)
3459                         vfc->ids[vidx] |= UINT64_C(1) << vbit;
3460                 else
3461                         vfc->ids[vidx] &= ~(UINT64_C(1) << vbit);
3462         }
3463
3464         return eth_err(port_id, ret);
3465 }
3466
3467 int
3468 rte_eth_dev_set_vlan_strip_on_queue(uint16_t port_id, uint16_t rx_queue_id,
3469                                     int on)
3470 {
3471         struct rte_eth_dev *dev;
3472
3473         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3474         dev = &rte_eth_devices[port_id];
3475         if (rx_queue_id >= dev->data->nb_rx_queues) {
3476                 RTE_ETHDEV_LOG(ERR, "Invalid rx_queue_id=%u\n", rx_queue_id);
3477                 return -EINVAL;
3478         }
3479
3480         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_strip_queue_set, -ENOTSUP);
3481         (*dev->dev_ops->vlan_strip_queue_set)(dev, rx_queue_id, on);
3482
3483         return 0;
3484 }
3485
3486 int
3487 rte_eth_dev_set_vlan_ether_type(uint16_t port_id,
3488                                 enum rte_vlan_type vlan_type,
3489                                 uint16_t tpid)
3490 {
3491         struct rte_eth_dev *dev;
3492
3493         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3494         dev = &rte_eth_devices[port_id];
3495         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_tpid_set, -ENOTSUP);
3496
3497         return eth_err(port_id, (*dev->dev_ops->vlan_tpid_set)(dev, vlan_type,
3498                                                                tpid));
3499 }
3500
3501 int
3502 rte_eth_dev_set_vlan_offload(uint16_t port_id, int offload_mask)
3503 {
3504         struct rte_eth_dev_info dev_info;
3505         struct rte_eth_dev *dev;
3506         int ret = 0;
3507         int mask = 0;
3508         int cur, org = 0;
3509         uint64_t orig_offloads;
3510         uint64_t dev_offloads;
3511         uint64_t new_offloads;
3512
3513         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3514         dev = &rte_eth_devices[port_id];
3515
3516         /* save original values in case of failure */
3517         orig_offloads = dev->data->dev_conf.rxmode.offloads;
3518         dev_offloads = orig_offloads;
3519
3520         /* check which option changed by application */
3521         cur = !!(offload_mask & ETH_VLAN_STRIP_OFFLOAD);
3522         org = !!(dev_offloads & DEV_RX_OFFLOAD_VLAN_STRIP);
3523         if (cur != org) {
3524                 if (cur)
3525                         dev_offloads |= DEV_RX_OFFLOAD_VLAN_STRIP;
3526                 else
3527                         dev_offloads &= ~DEV_RX_OFFLOAD_VLAN_STRIP;
3528                 mask |= ETH_VLAN_STRIP_MASK;
3529         }
3530
3531         cur = !!(offload_mask & ETH_VLAN_FILTER_OFFLOAD);
3532         org = !!(dev_offloads & DEV_RX_OFFLOAD_VLAN_FILTER);
3533         if (cur != org) {
3534                 if (cur)
3535                         dev_offloads |= DEV_RX_OFFLOAD_VLAN_FILTER;
3536                 else
3537                         dev_offloads &= ~DEV_RX_OFFLOAD_VLAN_FILTER;
3538                 mask |= ETH_VLAN_FILTER_MASK;
3539         }
3540
3541         cur = !!(offload_mask & ETH_VLAN_EXTEND_OFFLOAD);
3542         org = !!(dev_offloads & DEV_RX_OFFLOAD_VLAN_EXTEND);
3543         if (cur != org) {
3544                 if (cur)
3545                         dev_offloads |= DEV_RX_OFFLOAD_VLAN_EXTEND;
3546                 else
3547                         dev_offloads &= ~DEV_RX_OFFLOAD_VLAN_EXTEND;
3548                 mask |= ETH_VLAN_EXTEND_MASK;
3549         }
3550
3551         cur = !!(offload_mask & ETH_QINQ_STRIP_OFFLOAD);
3552         org = !!(dev_offloads & DEV_RX_OFFLOAD_QINQ_STRIP);
3553         if (cur != org) {
3554                 if (cur)
3555                         dev_offloads |= DEV_RX_OFFLOAD_QINQ_STRIP;
3556                 else
3557                         dev_offloads &= ~DEV_RX_OFFLOAD_QINQ_STRIP;
3558                 mask |= ETH_QINQ_STRIP_MASK;
3559         }
3560
3561         /*no change*/
3562         if (mask == 0)
3563                 return ret;
3564
3565         ret = rte_eth_dev_info_get(port_id, &dev_info);
3566         if (ret != 0)
3567                 return ret;
3568
3569         /* Rx VLAN offloading must be within its device capabilities */
3570         if ((dev_offloads & dev_info.rx_offload_capa) != dev_offloads) {
3571                 new_offloads = dev_offloads & ~orig_offloads;
3572                 RTE_ETHDEV_LOG(ERR,
3573                         "Ethdev port_id=%u requested new added VLAN offloads "
3574                         "0x%" PRIx64 " must be within Rx offloads capabilities "
3575                         "0x%" PRIx64 " in %s()\n",
3576                         port_id, new_offloads, dev_info.rx_offload_capa,
3577                         __func__);
3578                 return -EINVAL;
3579         }
3580
3581         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_offload_set, -ENOTSUP);
3582         dev->data->dev_conf.rxmode.offloads = dev_offloads;
3583         ret = (*dev->dev_ops->vlan_offload_set)(dev, mask);
3584         if (ret) {
3585                 /* hit an error restore  original values */
3586                 dev->data->dev_conf.rxmode.offloads = orig_offloads;
3587         }
3588
3589         return eth_err(port_id, ret);
3590 }
3591
3592 int
3593 rte_eth_dev_get_vlan_offload(uint16_t port_id)
3594 {
3595         struct rte_eth_dev *dev;
3596         uint64_t *dev_offloads;
3597         int ret = 0;
3598
3599         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3600         dev = &rte_eth_devices[port_id];
3601         dev_offloads = &dev->data->dev_conf.rxmode.offloads;
3602
3603         if (*dev_offloads & DEV_RX_OFFLOAD_VLAN_STRIP)
3604                 ret |= ETH_VLAN_STRIP_OFFLOAD;
3605
3606         if (*dev_offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
3607                 ret |= ETH_VLAN_FILTER_OFFLOAD;
3608
3609         if (*dev_offloads & DEV_RX_OFFLOAD_VLAN_EXTEND)
3610                 ret |= ETH_VLAN_EXTEND_OFFLOAD;
3611
3612         if (*dev_offloads & DEV_RX_OFFLOAD_QINQ_STRIP)
3613                 ret |= ETH_QINQ_STRIP_OFFLOAD;
3614
3615         return ret;
3616 }
3617
3618 int
3619 rte_eth_dev_set_vlan_pvid(uint16_t port_id, uint16_t pvid, int on)
3620 {
3621         struct rte_eth_dev *dev;
3622
3623         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3624         dev = &rte_eth_devices[port_id];
3625         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->vlan_pvid_set, -ENOTSUP);
3626
3627         return eth_err(port_id, (*dev->dev_ops->vlan_pvid_set)(dev, pvid, on));
3628 }
3629
3630 int
3631 rte_eth_dev_flow_ctrl_get(uint16_t port_id, struct rte_eth_fc_conf *fc_conf)
3632 {
3633         struct rte_eth_dev *dev;
3634
3635         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3636         dev = &rte_eth_devices[port_id];
3637         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_get, -ENOTSUP);
3638         memset(fc_conf, 0, sizeof(*fc_conf));
3639         return eth_err(port_id, (*dev->dev_ops->flow_ctrl_get)(dev, fc_conf));
3640 }
3641
3642 int
3643 rte_eth_dev_flow_ctrl_set(uint16_t port_id, struct rte_eth_fc_conf *fc_conf)
3644 {
3645         struct rte_eth_dev *dev;
3646
3647         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3648         if ((fc_conf->send_xon != 0) && (fc_conf->send_xon != 1)) {
3649                 RTE_ETHDEV_LOG(ERR, "Invalid send_xon, only 0/1 allowed\n");
3650                 return -EINVAL;
3651         }
3652
3653         dev = &rte_eth_devices[port_id];
3654         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->flow_ctrl_set, -ENOTSUP);
3655         return eth_err(port_id, (*dev->dev_ops->flow_ctrl_set)(dev, fc_conf));
3656 }
3657
3658 int
3659 rte_eth_dev_priority_flow_ctrl_set(uint16_t port_id,
3660                                    struct rte_eth_pfc_conf *pfc_conf)
3661 {
3662         struct rte_eth_dev *dev;
3663
3664         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3665         if (pfc_conf->priority > (ETH_DCB_NUM_USER_PRIORITIES - 1)) {
3666                 RTE_ETHDEV_LOG(ERR, "Invalid priority, only 0-7 allowed\n");
3667                 return -EINVAL;
3668         }
3669
3670         dev = &rte_eth_devices[port_id];
3671         /* High water, low water validation are device specific */
3672         if  (*dev->dev_ops->priority_flow_ctrl_set)
3673                 return eth_err(port_id, (*dev->dev_ops->priority_flow_ctrl_set)
3674                                         (dev, pfc_conf));
3675         return -ENOTSUP;
3676 }
3677
3678 static int
3679 rte_eth_check_reta_mask(struct rte_eth_rss_reta_entry64 *reta_conf,
3680                         uint16_t reta_size)
3681 {
3682         uint16_t i, num;
3683
3684         if (!reta_conf)
3685                 return -EINVAL;
3686
3687         num = (reta_size + RTE_RETA_GROUP_SIZE - 1) / RTE_RETA_GROUP_SIZE;
3688         for (i = 0; i < num; i++) {
3689                 if (reta_conf[i].mask)
3690                         return 0;
3691         }
3692
3693         return -EINVAL;
3694 }
3695
3696 static int
3697 rte_eth_check_reta_entry(struct rte_eth_rss_reta_entry64 *reta_conf,
3698                          uint16_t reta_size,
3699                          uint16_t max_rxq)
3700 {
3701         uint16_t i, idx, shift;
3702
3703         if (!reta_conf)
3704                 return -EINVAL;
3705
3706         if (max_rxq == 0) {
3707                 RTE_ETHDEV_LOG(ERR, "No receive queue is available\n");
3708                 return -EINVAL;
3709         }
3710
3711         for (i = 0; i < reta_size; i++) {
3712                 idx = i / RTE_RETA_GROUP_SIZE;
3713                 shift = i % RTE_RETA_GROUP_SIZE;
3714                 if ((reta_conf[idx].mask & (1ULL << shift)) &&
3715                         (reta_conf[idx].reta[shift] >= max_rxq)) {
3716                         RTE_ETHDEV_LOG(ERR,
3717                                 "reta_conf[%u]->reta[%u]: %u exceeds the maximum rxq index: %u\n",
3718                                 idx, shift,
3719                                 reta_conf[idx].reta[shift], max_rxq);
3720                         return -EINVAL;
3721                 }
3722         }
3723
3724         return 0;
3725 }
3726
3727 int
3728 rte_eth_dev_rss_reta_update(uint16_t port_id,
3729                             struct rte_eth_rss_reta_entry64 *reta_conf,
3730                             uint16_t reta_size)
3731 {
3732         struct rte_eth_dev *dev;
3733         int ret;
3734
3735         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3736         /* Check mask bits */
3737         ret = rte_eth_check_reta_mask(reta_conf, reta_size);
3738         if (ret < 0)
3739                 return ret;
3740
3741         dev = &rte_eth_devices[port_id];
3742
3743         /* Check entry value */
3744         ret = rte_eth_check_reta_entry(reta_conf, reta_size,
3745                                 dev->data->nb_rx_queues);
3746         if (ret < 0)
3747                 return ret;
3748
3749         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_update, -ENOTSUP);
3750         return eth_err(port_id, (*dev->dev_ops->reta_update)(dev, reta_conf,
3751                                                              reta_size));
3752 }
3753
3754 int
3755 rte_eth_dev_rss_reta_query(uint16_t port_id,
3756                            struct rte_eth_rss_reta_entry64 *reta_conf,
3757                            uint16_t reta_size)
3758 {
3759         struct rte_eth_dev *dev;
3760         int ret;
3761
3762         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3763
3764         /* Check mask bits */
3765         ret = rte_eth_check_reta_mask(reta_conf, reta_size);
3766         if (ret < 0)
3767                 return ret;
3768
3769         dev = &rte_eth_devices[port_id];
3770         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->reta_query, -ENOTSUP);
3771         return eth_err(port_id, (*dev->dev_ops->reta_query)(dev, reta_conf,
3772                                                             reta_size));
3773 }
3774
3775 int
3776 rte_eth_dev_rss_hash_update(uint16_t port_id,
3777                             struct rte_eth_rss_conf *rss_conf)
3778 {
3779         struct rte_eth_dev *dev;
3780         struct rte_eth_dev_info dev_info = { .flow_type_rss_offloads = 0, };
3781         int ret;
3782
3783         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3784
3785         ret = rte_eth_dev_info_get(port_id, &dev_info);
3786         if (ret != 0)
3787                 return ret;
3788
3789         rss_conf->rss_hf = rte_eth_rss_hf_refine(rss_conf->rss_hf);
3790
3791         dev = &rte_eth_devices[port_id];
3792         if ((dev_info.flow_type_rss_offloads | rss_conf->rss_hf) !=
3793             dev_info.flow_type_rss_offloads) {
3794                 RTE_ETHDEV_LOG(ERR,
3795                         "Ethdev port_id=%u invalid rss_hf: 0x%"PRIx64", valid value: 0x%"PRIx64"\n",
3796                         port_id, rss_conf->rss_hf,
3797                         dev_info.flow_type_rss_offloads);
3798                 return -EINVAL;
3799         }
3800         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_update, -ENOTSUP);
3801         return eth_err(port_id, (*dev->dev_ops->rss_hash_update)(dev,
3802                                                                  rss_conf));
3803 }
3804
3805 int
3806 rte_eth_dev_rss_hash_conf_get(uint16_t port_id,
3807                               struct rte_eth_rss_conf *rss_conf)
3808 {
3809         struct rte_eth_dev *dev;
3810
3811         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3812         dev = &rte_eth_devices[port_id];
3813         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rss_hash_conf_get, -ENOTSUP);
3814         return eth_err(port_id, (*dev->dev_ops->rss_hash_conf_get)(dev,
3815                                                                    rss_conf));
3816 }
3817
3818 int
3819 rte_eth_dev_udp_tunnel_port_add(uint16_t port_id,
3820                                 struct rte_eth_udp_tunnel *udp_tunnel)
3821 {
3822         struct rte_eth_dev *dev;
3823
3824         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3825         if (udp_tunnel == NULL) {
3826                 RTE_ETHDEV_LOG(ERR, "Invalid udp_tunnel parameter\n");
3827                 return -EINVAL;
3828         }
3829
3830         if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
3831                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
3832                 return -EINVAL;
3833         }
3834
3835         dev = &rte_eth_devices[port_id];
3836         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_port_add, -ENOTSUP);
3837         return eth_err(port_id, (*dev->dev_ops->udp_tunnel_port_add)(dev,
3838                                                                 udp_tunnel));
3839 }
3840
3841 int
3842 rte_eth_dev_udp_tunnel_port_delete(uint16_t port_id,
3843                                    struct rte_eth_udp_tunnel *udp_tunnel)
3844 {
3845         struct rte_eth_dev *dev;
3846
3847         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3848         dev = &rte_eth_devices[port_id];
3849
3850         if (udp_tunnel == NULL) {
3851                 RTE_ETHDEV_LOG(ERR, "Invalid udp_tunnel parameter\n");
3852                 return -EINVAL;
3853         }
3854
3855         if (udp_tunnel->prot_type >= RTE_TUNNEL_TYPE_MAX) {
3856                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
3857                 return -EINVAL;
3858         }
3859
3860         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->udp_tunnel_port_del, -ENOTSUP);
3861         return eth_err(port_id, (*dev->dev_ops->udp_tunnel_port_del)(dev,
3862                                                                 udp_tunnel));
3863 }
3864
3865 int
3866 rte_eth_led_on(uint16_t port_id)
3867 {
3868         struct rte_eth_dev *dev;
3869
3870         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3871         dev = &rte_eth_devices[port_id];
3872         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_on, -ENOTSUP);
3873         return eth_err(port_id, (*dev->dev_ops->dev_led_on)(dev));
3874 }
3875
3876 int
3877 rte_eth_led_off(uint16_t port_id)
3878 {
3879         struct rte_eth_dev *dev;
3880
3881         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3882         dev = &rte_eth_devices[port_id];
3883         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->dev_led_off, -ENOTSUP);
3884         return eth_err(port_id, (*dev->dev_ops->dev_led_off)(dev));
3885 }
3886
3887 int
3888 rte_eth_fec_get_capability(uint16_t port_id,
3889                            struct rte_eth_fec_capa *speed_fec_capa,
3890                            unsigned int num)
3891 {
3892         struct rte_eth_dev *dev;
3893         int ret;
3894
3895         if (speed_fec_capa == NULL && num > 0)
3896                 return -EINVAL;
3897
3898         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3899         dev = &rte_eth_devices[port_id];
3900         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fec_get_capability, -ENOTSUP);
3901         ret = (*dev->dev_ops->fec_get_capability)(dev, speed_fec_capa, num);
3902
3903         return ret;
3904 }
3905
3906 int
3907 rte_eth_fec_get(uint16_t port_id, uint32_t *fec_capa)
3908 {
3909         struct rte_eth_dev *dev;
3910
3911         if (fec_capa == NULL)
3912                 return -EINVAL;
3913
3914         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3915         dev = &rte_eth_devices[port_id];
3916         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fec_get, -ENOTSUP);
3917         return eth_err(port_id, (*dev->dev_ops->fec_get)(dev, fec_capa));
3918 }
3919
3920 int
3921 rte_eth_fec_set(uint16_t port_id, uint32_t fec_capa)
3922 {
3923         struct rte_eth_dev *dev;
3924
3925         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3926         dev = &rte_eth_devices[port_id];
3927         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->fec_set, -ENOTSUP);
3928         return eth_err(port_id, (*dev->dev_ops->fec_set)(dev, fec_capa));
3929 }
3930
3931 /*
3932  * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find
3933  * an empty spot.
3934  */
3935 static int
3936 get_mac_addr_index(uint16_t port_id, const struct rte_ether_addr *addr)
3937 {
3938         struct rte_eth_dev_info dev_info;
3939         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
3940         unsigned i;
3941         int ret;
3942
3943         ret = rte_eth_dev_info_get(port_id, &dev_info);
3944         if (ret != 0)
3945                 return -1;
3946
3947         for (i = 0; i < dev_info.max_mac_addrs; i++)
3948                 if (memcmp(addr, &dev->data->mac_addrs[i],
3949                                 RTE_ETHER_ADDR_LEN) == 0)
3950                         return i;
3951
3952         return -1;
3953 }
3954
3955 static const struct rte_ether_addr null_mac_addr;
3956
3957 int
3958 rte_eth_dev_mac_addr_add(uint16_t port_id, struct rte_ether_addr *addr,
3959                         uint32_t pool)
3960 {
3961         struct rte_eth_dev *dev;
3962         int index;
3963         uint64_t pool_mask;
3964         int ret;
3965
3966         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
3967         dev = &rte_eth_devices[port_id];
3968         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_add, -ENOTSUP);
3969
3970         if (rte_is_zero_ether_addr(addr)) {
3971                 RTE_ETHDEV_LOG(ERR, "Port %u: Cannot add NULL MAC address\n",
3972                         port_id);
3973                 return -EINVAL;
3974         }
3975         if (pool >= ETH_64_POOLS) {
3976                 RTE_ETHDEV_LOG(ERR, "Pool id must be 0-%d\n", ETH_64_POOLS - 1);
3977                 return -EINVAL;
3978         }
3979
3980         index = get_mac_addr_index(port_id, addr);
3981         if (index < 0) {
3982                 index = get_mac_addr_index(port_id, &null_mac_addr);
3983                 if (index < 0) {
3984                         RTE_ETHDEV_LOG(ERR, "Port %u: MAC address array full\n",
3985                                 port_id);
3986                         return -ENOSPC;
3987                 }
3988         } else {
3989                 pool_mask = dev->data->mac_pool_sel[index];
3990
3991                 /* Check if both MAC address and pool is already there, and do nothing */
3992                 if (pool_mask & (1ULL << pool))
3993                         return 0;
3994         }
3995
3996         /* Update NIC */
3997         ret = (*dev->dev_ops->mac_addr_add)(dev, addr, index, pool);
3998
3999         if (ret == 0) {
4000                 /* Update address in NIC data structure */
4001                 rte_ether_addr_copy(addr, &dev->data->mac_addrs[index]);
4002
4003                 /* Update pool bitmap in NIC data structure */
4004                 dev->data->mac_pool_sel[index] |= (1ULL << pool);
4005         }
4006
4007         return eth_err(port_id, ret);
4008 }
4009
4010 int
4011 rte_eth_dev_mac_addr_remove(uint16_t port_id, struct rte_ether_addr *addr)
4012 {
4013         struct rte_eth_dev *dev;
4014         int index;
4015
4016         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4017         dev = &rte_eth_devices[port_id];
4018         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_remove, -ENOTSUP);
4019
4020         index = get_mac_addr_index(port_id, addr);
4021         if (index == 0) {
4022                 RTE_ETHDEV_LOG(ERR,
4023                         "Port %u: Cannot remove default MAC address\n",
4024                         port_id);
4025                 return -EADDRINUSE;
4026         } else if (index < 0)
4027                 return 0;  /* Do nothing if address wasn't found */
4028
4029         /* Update NIC */
4030         (*dev->dev_ops->mac_addr_remove)(dev, index);
4031
4032         /* Update address in NIC data structure */
4033         rte_ether_addr_copy(&null_mac_addr, &dev->data->mac_addrs[index]);
4034
4035         /* reset pool bitmap */
4036         dev->data->mac_pool_sel[index] = 0;
4037
4038         return 0;
4039 }
4040
4041 int
4042 rte_eth_dev_default_mac_addr_set(uint16_t port_id, struct rte_ether_addr *addr)
4043 {
4044         struct rte_eth_dev *dev;
4045         int ret;
4046
4047         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4048
4049         if (!rte_is_valid_assigned_ether_addr(addr))
4050                 return -EINVAL;
4051
4052         dev = &rte_eth_devices[port_id];
4053         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mac_addr_set, -ENOTSUP);
4054
4055         ret = (*dev->dev_ops->mac_addr_set)(dev, addr);
4056         if (ret < 0)
4057                 return ret;
4058
4059         /* Update default address in NIC data structure */
4060         rte_ether_addr_copy(addr, &dev->data->mac_addrs[0]);
4061
4062         return 0;
4063 }
4064
4065
4066 /*
4067  * Returns index into MAC address array of addr. Use 00:00:00:00:00:00 to find
4068  * an empty spot.
4069  */
4070 static int
4071 get_hash_mac_addr_index(uint16_t port_id, const struct rte_ether_addr *addr)
4072 {
4073         struct rte_eth_dev_info dev_info;
4074         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
4075         unsigned i;
4076         int ret;
4077
4078         ret = rte_eth_dev_info_get(port_id, &dev_info);
4079         if (ret != 0)
4080                 return -1;
4081
4082         if (!dev->data->hash_mac_addrs)
4083                 return -1;
4084
4085         for (i = 0; i < dev_info.max_hash_mac_addrs; i++)
4086                 if (memcmp(addr, &dev->data->hash_mac_addrs[i],
4087                         RTE_ETHER_ADDR_LEN) == 0)
4088                         return i;
4089
4090         return -1;
4091 }
4092
4093 int
4094 rte_eth_dev_uc_hash_table_set(uint16_t port_id, struct rte_ether_addr *addr,
4095                                 uint8_t on)
4096 {
4097         int index;
4098         int ret;
4099         struct rte_eth_dev *dev;
4100
4101         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4102
4103         dev = &rte_eth_devices[port_id];
4104         if (rte_is_zero_ether_addr(addr)) {
4105                 RTE_ETHDEV_LOG(ERR, "Port %u: Cannot add NULL MAC address\n",
4106                         port_id);
4107                 return -EINVAL;
4108         }
4109
4110         index = get_hash_mac_addr_index(port_id, addr);
4111         /* Check if it's already there, and do nothing */
4112         if ((index >= 0) && on)
4113                 return 0;
4114
4115         if (index < 0) {
4116                 if (!on) {
4117                         RTE_ETHDEV_LOG(ERR,
4118                                 "Port %u: the MAC address was not set in UTA\n",
4119                                 port_id);
4120                         return -EINVAL;
4121                 }
4122
4123                 index = get_hash_mac_addr_index(port_id, &null_mac_addr);
4124                 if (index < 0) {
4125                         RTE_ETHDEV_LOG(ERR, "Port %u: MAC address array full\n",
4126                                 port_id);
4127                         return -ENOSPC;
4128                 }
4129         }
4130
4131         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_hash_table_set, -ENOTSUP);
4132         ret = (*dev->dev_ops->uc_hash_table_set)(dev, addr, on);
4133         if (ret == 0) {
4134                 /* Update address in NIC data structure */
4135                 if (on)
4136                         rte_ether_addr_copy(addr,
4137                                         &dev->data->hash_mac_addrs[index]);
4138                 else
4139                         rte_ether_addr_copy(&null_mac_addr,
4140                                         &dev->data->hash_mac_addrs[index]);
4141         }
4142
4143         return eth_err(port_id, ret);
4144 }
4145
4146 int
4147 rte_eth_dev_uc_all_hash_table_set(uint16_t port_id, uint8_t on)
4148 {
4149         struct rte_eth_dev *dev;
4150
4151         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4152
4153         dev = &rte_eth_devices[port_id];
4154
4155         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->uc_all_hash_table_set, -ENOTSUP);
4156         return eth_err(port_id, (*dev->dev_ops->uc_all_hash_table_set)(dev,
4157                                                                        on));
4158 }
4159
4160 int rte_eth_set_queue_rate_limit(uint16_t port_id, uint16_t queue_idx,
4161                                         uint16_t tx_rate)
4162 {
4163         struct rte_eth_dev *dev;
4164         struct rte_eth_dev_info dev_info;
4165         struct rte_eth_link link;
4166         int ret;
4167
4168         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4169
4170         ret = rte_eth_dev_info_get(port_id, &dev_info);
4171         if (ret != 0)
4172                 return ret;
4173
4174         dev = &rte_eth_devices[port_id];
4175         link = dev->data->dev_link;
4176
4177         if (queue_idx > dev_info.max_tx_queues) {
4178                 RTE_ETHDEV_LOG(ERR,
4179                         "Set queue rate limit:port %u: invalid queue id=%u\n",
4180                         port_id, queue_idx);
4181                 return -EINVAL;
4182         }
4183
4184         if (tx_rate > link.link_speed) {
4185                 RTE_ETHDEV_LOG(ERR,
4186                         "Set queue rate limit:invalid tx_rate=%u, bigger than link speed= %d\n",
4187                         tx_rate, link.link_speed);
4188                 return -EINVAL;
4189         }
4190
4191         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_queue_rate_limit, -ENOTSUP);
4192         return eth_err(port_id, (*dev->dev_ops->set_queue_rate_limit)(dev,
4193                                                         queue_idx, tx_rate));
4194 }
4195
4196 int
4197 rte_eth_mirror_rule_set(uint16_t port_id,
4198                         struct rte_eth_mirror_conf *mirror_conf,
4199                         uint8_t rule_id, uint8_t on)
4200 {
4201         struct rte_eth_dev *dev;
4202
4203         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4204         if (mirror_conf->rule_type == 0) {
4205                 RTE_ETHDEV_LOG(ERR, "Mirror rule type can not be 0\n");
4206                 return -EINVAL;
4207         }
4208
4209         if (mirror_conf->dst_pool >= ETH_64_POOLS) {
4210                 RTE_ETHDEV_LOG(ERR, "Invalid dst pool, pool id must be 0-%d\n",
4211                         ETH_64_POOLS - 1);
4212                 return -EINVAL;
4213         }
4214
4215         if ((mirror_conf->rule_type & (ETH_MIRROR_VIRTUAL_POOL_UP |
4216              ETH_MIRROR_VIRTUAL_POOL_DOWN)) &&
4217             (mirror_conf->pool_mask == 0)) {
4218                 RTE_ETHDEV_LOG(ERR,
4219                         "Invalid mirror pool, pool mask can not be 0\n");
4220                 return -EINVAL;
4221         }
4222
4223         if ((mirror_conf->rule_type & ETH_MIRROR_VLAN) &&
4224             mirror_conf->vlan.vlan_mask == 0) {
4225                 RTE_ETHDEV_LOG(ERR,
4226                         "Invalid vlan mask, vlan mask can not be 0\n");
4227                 return -EINVAL;
4228         }
4229
4230         dev = &rte_eth_devices[port_id];
4231         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mirror_rule_set, -ENOTSUP);
4232
4233         return eth_err(port_id, (*dev->dev_ops->mirror_rule_set)(dev,
4234                                                 mirror_conf, rule_id, on));
4235 }
4236
4237 int
4238 rte_eth_mirror_rule_reset(uint16_t port_id, uint8_t rule_id)
4239 {
4240         struct rte_eth_dev *dev;
4241
4242         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4243
4244         dev = &rte_eth_devices[port_id];
4245         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->mirror_rule_reset, -ENOTSUP);
4246
4247         return eth_err(port_id, (*dev->dev_ops->mirror_rule_reset)(dev,
4248                                                                    rule_id));
4249 }
4250
4251 RTE_INIT(eth_dev_init_cb_lists)
4252 {
4253         int i;
4254
4255         for (i = 0; i < RTE_MAX_ETHPORTS; i++)
4256                 TAILQ_INIT(&rte_eth_devices[i].link_intr_cbs);
4257 }
4258
4259 int
4260 rte_eth_dev_callback_register(uint16_t port_id,
4261                         enum rte_eth_event_type event,
4262                         rte_eth_dev_cb_fn cb_fn, void *cb_arg)
4263 {
4264         struct rte_eth_dev *dev;
4265         struct rte_eth_dev_callback *user_cb;
4266         uint32_t next_port; /* size is 32-bit to prevent loop wrap-around */
4267         uint16_t last_port;
4268
4269         if (!cb_fn)
4270                 return -EINVAL;
4271
4272         if (!rte_eth_dev_is_valid_port(port_id) && port_id != RTE_ETH_ALL) {
4273                 RTE_ETHDEV_LOG(ERR, "Invalid port_id=%d\n", port_id);
4274                 return -EINVAL;
4275         }
4276
4277         if (port_id == RTE_ETH_ALL) {
4278                 next_port = 0;
4279                 last_port = RTE_MAX_ETHPORTS - 1;
4280         } else {
4281                 next_port = last_port = port_id;
4282         }
4283
4284         rte_spinlock_lock(&rte_eth_dev_cb_lock);
4285
4286         do {
4287                 dev = &rte_eth_devices[next_port];
4288
4289                 TAILQ_FOREACH(user_cb, &(dev->link_intr_cbs), next) {
4290                         if (user_cb->cb_fn == cb_fn &&
4291                                 user_cb->cb_arg == cb_arg &&
4292                                 user_cb->event == event) {
4293                                 break;
4294                         }
4295                 }
4296
4297                 /* create a new callback. */
4298                 if (user_cb == NULL) {
4299                         user_cb = rte_zmalloc("INTR_USER_CALLBACK",
4300                                 sizeof(struct rte_eth_dev_callback), 0);
4301                         if (user_cb != NULL) {
4302                                 user_cb->cb_fn = cb_fn;
4303                                 user_cb->cb_arg = cb_arg;
4304                                 user_cb->event = event;
4305                                 TAILQ_INSERT_TAIL(&(dev->link_intr_cbs),
4306                                                   user_cb, next);
4307                         } else {
4308                                 rte_spinlock_unlock(&rte_eth_dev_cb_lock);
4309                                 rte_eth_dev_callback_unregister(port_id, event,
4310                                                                 cb_fn, cb_arg);
4311                                 return -ENOMEM;
4312                         }
4313
4314                 }
4315         } while (++next_port <= last_port);
4316
4317         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
4318         return 0;
4319 }
4320
4321 int
4322 rte_eth_dev_callback_unregister(uint16_t port_id,
4323                         enum rte_eth_event_type event,
4324                         rte_eth_dev_cb_fn cb_fn, void *cb_arg)
4325 {
4326         int ret;
4327         struct rte_eth_dev *dev;
4328         struct rte_eth_dev_callback *cb, *next;
4329         uint32_t next_port; /* size is 32-bit to prevent loop wrap-around */
4330         uint16_t last_port;
4331
4332         if (!cb_fn)
4333                 return -EINVAL;
4334
4335         if (!rte_eth_dev_is_valid_port(port_id) && port_id != RTE_ETH_ALL) {
4336                 RTE_ETHDEV_LOG(ERR, "Invalid port_id=%d\n", port_id);
4337                 return -EINVAL;
4338         }
4339
4340         if (port_id == RTE_ETH_ALL) {
4341                 next_port = 0;
4342                 last_port = RTE_MAX_ETHPORTS - 1;
4343         } else {
4344                 next_port = last_port = port_id;
4345         }
4346
4347         rte_spinlock_lock(&rte_eth_dev_cb_lock);
4348
4349         do {
4350                 dev = &rte_eth_devices[next_port];
4351                 ret = 0;
4352                 for (cb = TAILQ_FIRST(&dev->link_intr_cbs); cb != NULL;
4353                      cb = next) {
4354
4355                         next = TAILQ_NEXT(cb, next);
4356
4357                         if (cb->cb_fn != cb_fn || cb->event != event ||
4358                             (cb_arg != (void *)-1 && cb->cb_arg != cb_arg))
4359                                 continue;
4360
4361                         /*
4362                          * if this callback is not executing right now,
4363                          * then remove it.
4364                          */
4365                         if (cb->active == 0) {
4366                                 TAILQ_REMOVE(&(dev->link_intr_cbs), cb, next);
4367                                 rte_free(cb);
4368                         } else {
4369                                 ret = -EAGAIN;
4370                         }
4371                 }
4372         } while (++next_port <= last_port);
4373
4374         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
4375         return ret;
4376 }
4377
4378 int
4379 rte_eth_dev_callback_process(struct rte_eth_dev *dev,
4380         enum rte_eth_event_type event, void *ret_param)
4381 {
4382         struct rte_eth_dev_callback *cb_lst;
4383         struct rte_eth_dev_callback dev_cb;
4384         int rc = 0;
4385
4386         rte_spinlock_lock(&rte_eth_dev_cb_lock);
4387         TAILQ_FOREACH(cb_lst, &(dev->link_intr_cbs), next) {
4388                 if (cb_lst->cb_fn == NULL || cb_lst->event != event)
4389                         continue;
4390                 dev_cb = *cb_lst;
4391                 cb_lst->active = 1;
4392                 if (ret_param != NULL)
4393                         dev_cb.ret_param = ret_param;
4394
4395                 rte_spinlock_unlock(&rte_eth_dev_cb_lock);
4396                 rc = dev_cb.cb_fn(dev->data->port_id, dev_cb.event,
4397                                 dev_cb.cb_arg, dev_cb.ret_param);
4398                 rte_spinlock_lock(&rte_eth_dev_cb_lock);
4399                 cb_lst->active = 0;
4400         }
4401         rte_spinlock_unlock(&rte_eth_dev_cb_lock);
4402         return rc;
4403 }
4404
4405 void
4406 rte_eth_dev_probing_finish(struct rte_eth_dev *dev)
4407 {
4408         if (dev == NULL)
4409                 return;
4410
4411         rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_NEW, NULL);
4412
4413         dev->state = RTE_ETH_DEV_ATTACHED;
4414 }
4415
4416 int
4417 rte_eth_dev_rx_intr_ctl(uint16_t port_id, int epfd, int op, void *data)
4418 {
4419         uint32_t vec;
4420         struct rte_eth_dev *dev;
4421         struct rte_intr_handle *intr_handle;
4422         uint16_t qid;
4423         int rc;
4424
4425         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4426
4427         dev = &rte_eth_devices[port_id];
4428
4429         if (!dev->intr_handle) {
4430                 RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
4431                 return -ENOTSUP;
4432         }
4433
4434         intr_handle = dev->intr_handle;
4435         if (!intr_handle->intr_vec) {
4436                 RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
4437                 return -EPERM;
4438         }
4439
4440         for (qid = 0; qid < dev->data->nb_rx_queues; qid++) {
4441                 vec = intr_handle->intr_vec[qid];
4442                 rc = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
4443                 if (rc && rc != -EEXIST) {
4444                         RTE_ETHDEV_LOG(ERR,
4445                                 "p %u q %u rx ctl error op %d epfd %d vec %u\n",
4446                                 port_id, qid, op, epfd, vec);
4447                 }
4448         }
4449
4450         return 0;
4451 }
4452
4453 int
4454 rte_eth_dev_rx_intr_ctl_q_get_fd(uint16_t port_id, uint16_t queue_id)
4455 {
4456         struct rte_intr_handle *intr_handle;
4457         struct rte_eth_dev *dev;
4458         unsigned int efd_idx;
4459         uint32_t vec;
4460         int fd;
4461
4462         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -1);
4463
4464         dev = &rte_eth_devices[port_id];
4465
4466         if (queue_id >= dev->data->nb_rx_queues) {
4467                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
4468                 return -1;
4469         }
4470
4471         if (!dev->intr_handle) {
4472                 RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
4473                 return -1;
4474         }
4475
4476         intr_handle = dev->intr_handle;
4477         if (!intr_handle->intr_vec) {
4478                 RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
4479                 return -1;
4480         }
4481
4482         vec = intr_handle->intr_vec[queue_id];
4483         efd_idx = (vec >= RTE_INTR_VEC_RXTX_OFFSET) ?
4484                 (vec - RTE_INTR_VEC_RXTX_OFFSET) : vec;
4485         fd = intr_handle->efds[efd_idx];
4486
4487         return fd;
4488 }
4489
4490 static inline int
4491 eth_dma_mzone_name(char *name, size_t len, uint16_t port_id, uint16_t queue_id,
4492                 const char *ring_name)
4493 {
4494         return snprintf(name, len, "eth_p%d_q%d_%s",
4495                         port_id, queue_id, ring_name);
4496 }
4497
4498 const struct rte_memzone *
4499 rte_eth_dma_zone_reserve(const struct rte_eth_dev *dev, const char *ring_name,
4500                          uint16_t queue_id, size_t size, unsigned align,
4501                          int socket_id)
4502 {
4503         char z_name[RTE_MEMZONE_NAMESIZE];
4504         const struct rte_memzone *mz;
4505         int rc;
4506
4507         rc = eth_dma_mzone_name(z_name, sizeof(z_name), dev->data->port_id,
4508                         queue_id, ring_name);
4509         if (rc >= RTE_MEMZONE_NAMESIZE) {
4510                 RTE_ETHDEV_LOG(ERR, "ring name too long\n");
4511                 rte_errno = ENAMETOOLONG;
4512                 return NULL;
4513         }
4514
4515         mz = rte_memzone_lookup(z_name);
4516         if (mz) {
4517                 if ((socket_id != SOCKET_ID_ANY && socket_id != mz->socket_id) ||
4518                                 size > mz->len ||
4519                                 ((uintptr_t)mz->addr & (align - 1)) != 0) {
4520                         RTE_ETHDEV_LOG(ERR,
4521                                 "memzone %s does not justify the requested attributes\n",
4522                                 mz->name);
4523                         return NULL;
4524                 }
4525
4526                 return mz;
4527         }
4528
4529         return rte_memzone_reserve_aligned(z_name, size, socket_id,
4530                         RTE_MEMZONE_IOVA_CONTIG, align);
4531 }
4532
4533 int
4534 rte_eth_dma_zone_free(const struct rte_eth_dev *dev, const char *ring_name,
4535                 uint16_t queue_id)
4536 {
4537         char z_name[RTE_MEMZONE_NAMESIZE];
4538         const struct rte_memzone *mz;
4539         int rc = 0;
4540
4541         rc = eth_dma_mzone_name(z_name, sizeof(z_name), dev->data->port_id,
4542                         queue_id, ring_name);
4543         if (rc >= RTE_MEMZONE_NAMESIZE) {
4544                 RTE_ETHDEV_LOG(ERR, "ring name too long\n");
4545                 return -ENAMETOOLONG;
4546         }
4547
4548         mz = rte_memzone_lookup(z_name);
4549         if (mz)
4550                 rc = rte_memzone_free(mz);
4551         else
4552                 rc = -ENOENT;
4553
4554         return rc;
4555 }
4556
4557 int
4558 rte_eth_dev_create(struct rte_device *device, const char *name,
4559         size_t priv_data_size,
4560         ethdev_bus_specific_init ethdev_bus_specific_init,
4561         void *bus_init_params,
4562         ethdev_init_t ethdev_init, void *init_params)
4563 {
4564         struct rte_eth_dev *ethdev;
4565         int retval;
4566
4567         RTE_FUNC_PTR_OR_ERR_RET(*ethdev_init, -EINVAL);
4568
4569         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
4570                 ethdev = rte_eth_dev_allocate(name);
4571                 if (!ethdev)
4572                         return -ENODEV;
4573
4574                 if (priv_data_size) {
4575                         ethdev->data->dev_private = rte_zmalloc_socket(
4576                                 name, priv_data_size, RTE_CACHE_LINE_SIZE,
4577                                 device->numa_node);
4578
4579                         if (!ethdev->data->dev_private) {
4580                                 RTE_ETHDEV_LOG(ERR,
4581                                         "failed to allocate private data\n");
4582                                 retval = -ENOMEM;
4583                                 goto probe_failed;
4584                         }
4585                 }
4586         } else {
4587                 ethdev = rte_eth_dev_attach_secondary(name);
4588                 if (!ethdev) {
4589                         RTE_ETHDEV_LOG(ERR,
4590                                 "secondary process attach failed, ethdev doesn't exist\n");
4591                         return  -ENODEV;
4592                 }
4593         }
4594
4595         ethdev->device = device;
4596
4597         if (ethdev_bus_specific_init) {
4598                 retval = ethdev_bus_specific_init(ethdev, bus_init_params);
4599                 if (retval) {
4600                         RTE_ETHDEV_LOG(ERR,
4601                                 "ethdev bus specific initialisation failed\n");
4602                         goto probe_failed;
4603                 }
4604         }
4605
4606         retval = ethdev_init(ethdev, init_params);
4607         if (retval) {
4608                 RTE_ETHDEV_LOG(ERR, "ethdev initialisation failed\n");
4609                 goto probe_failed;
4610         }
4611
4612         rte_eth_dev_probing_finish(ethdev);
4613
4614         return retval;
4615
4616 probe_failed:
4617         rte_eth_dev_release_port(ethdev);
4618         return retval;
4619 }
4620
4621 int
4622 rte_eth_dev_destroy(struct rte_eth_dev *ethdev,
4623         ethdev_uninit_t ethdev_uninit)
4624 {
4625         int ret;
4626
4627         ethdev = rte_eth_dev_allocated(ethdev->data->name);
4628         if (!ethdev)
4629                 return -ENODEV;
4630
4631         RTE_FUNC_PTR_OR_ERR_RET(*ethdev_uninit, -EINVAL);
4632
4633         ret = ethdev_uninit(ethdev);
4634         if (ret)
4635                 return ret;
4636
4637         return rte_eth_dev_release_port(ethdev);
4638 }
4639
4640 int
4641 rte_eth_dev_rx_intr_ctl_q(uint16_t port_id, uint16_t queue_id,
4642                           int epfd, int op, void *data)
4643 {
4644         uint32_t vec;
4645         struct rte_eth_dev *dev;
4646         struct rte_intr_handle *intr_handle;
4647         int rc;
4648
4649         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4650
4651         dev = &rte_eth_devices[port_id];
4652         if (queue_id >= dev->data->nb_rx_queues) {
4653                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
4654                 return -EINVAL;
4655         }
4656
4657         if (!dev->intr_handle) {
4658                 RTE_ETHDEV_LOG(ERR, "RX Intr handle unset\n");
4659                 return -ENOTSUP;
4660         }
4661
4662         intr_handle = dev->intr_handle;
4663         if (!intr_handle->intr_vec) {
4664                 RTE_ETHDEV_LOG(ERR, "RX Intr vector unset\n");
4665                 return -EPERM;
4666         }
4667
4668         vec = intr_handle->intr_vec[queue_id];
4669         rc = rte_intr_rx_ctl(intr_handle, epfd, op, vec, data);
4670         if (rc && rc != -EEXIST) {
4671                 RTE_ETHDEV_LOG(ERR,
4672                         "p %u q %u rx ctl error op %d epfd %d vec %u\n",
4673                         port_id, queue_id, op, epfd, vec);
4674                 return rc;
4675         }
4676
4677         return 0;
4678 }
4679
4680 int
4681 rte_eth_dev_rx_intr_enable(uint16_t port_id,
4682                            uint16_t queue_id)
4683 {
4684         struct rte_eth_dev *dev;
4685         int ret;
4686
4687         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4688
4689         dev = &rte_eth_devices[port_id];
4690
4691         ret = eth_dev_validate_rx_queue(dev, queue_id);
4692         if (ret != 0)
4693                 return ret;
4694
4695         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_intr_enable, -ENOTSUP);
4696         return eth_err(port_id, (*dev->dev_ops->rx_queue_intr_enable)(dev,
4697                                                                 queue_id));
4698 }
4699
4700 int
4701 rte_eth_dev_rx_intr_disable(uint16_t port_id,
4702                             uint16_t queue_id)
4703 {
4704         struct rte_eth_dev *dev;
4705         int ret;
4706
4707         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4708
4709         dev = &rte_eth_devices[port_id];
4710
4711         ret = eth_dev_validate_rx_queue(dev, queue_id);
4712         if (ret != 0)
4713                 return ret;
4714
4715         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_queue_intr_disable, -ENOTSUP);
4716         return eth_err(port_id, (*dev->dev_ops->rx_queue_intr_disable)(dev,
4717                                                                 queue_id));
4718 }
4719
4720
4721 int
4722 rte_eth_dev_filter_supported(uint16_t port_id,
4723                              enum rte_filter_type filter_type)
4724 {
4725         struct rte_eth_dev *dev;
4726
4727         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4728
4729         dev = &rte_eth_devices[port_id];
4730         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->filter_ctrl, -ENOTSUP);
4731         return (*dev->dev_ops->filter_ctrl)(dev, filter_type,
4732                                 RTE_ETH_FILTER_NOP, NULL);
4733 }
4734
4735 int
4736 rte_eth_dev_filter_ctrl(uint16_t port_id, enum rte_filter_type filter_type,
4737                         enum rte_filter_op filter_op, void *arg)
4738 {
4739         struct rte_eth_dev *dev;
4740
4741         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4742
4743         dev = &rte_eth_devices[port_id];
4744         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->filter_ctrl, -ENOTSUP);
4745         return eth_err(port_id, (*dev->dev_ops->filter_ctrl)(dev, filter_type,
4746                                                              filter_op, arg));
4747 }
4748
4749 const struct rte_eth_rxtx_callback *
4750 rte_eth_add_rx_callback(uint16_t port_id, uint16_t queue_id,
4751                 rte_rx_callback_fn fn, void *user_param)
4752 {
4753 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
4754         rte_errno = ENOTSUP;
4755         return NULL;
4756 #endif
4757         struct rte_eth_dev *dev;
4758
4759         /* check input parameters */
4760         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
4761                     queue_id >= rte_eth_devices[port_id].data->nb_rx_queues) {
4762                 rte_errno = EINVAL;
4763                 return NULL;
4764         }
4765         dev = &rte_eth_devices[port_id];
4766         if (rte_eth_dev_is_rx_hairpin_queue(dev, queue_id)) {
4767                 rte_errno = EINVAL;
4768                 return NULL;
4769         }
4770         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
4771
4772         if (cb == NULL) {
4773                 rte_errno = ENOMEM;
4774                 return NULL;
4775         }
4776
4777         cb->fn.rx = fn;
4778         cb->param = user_param;
4779
4780         rte_spinlock_lock(&rte_eth_rx_cb_lock);
4781         /* Add the callbacks in fifo order. */
4782         struct rte_eth_rxtx_callback *tail =
4783                 rte_eth_devices[port_id].post_rx_burst_cbs[queue_id];
4784
4785         if (!tail) {
4786                 /* Stores to cb->fn and cb->param should complete before
4787                  * cb is visible to data plane.
4788                  */
4789                 __atomic_store_n(
4790                         &rte_eth_devices[port_id].post_rx_burst_cbs[queue_id],
4791                         cb, __ATOMIC_RELEASE);
4792
4793         } else {
4794                 while (tail->next)
4795                         tail = tail->next;
4796                 /* Stores to cb->fn and cb->param should complete before
4797                  * cb is visible to data plane.
4798                  */
4799                 __atomic_store_n(&tail->next, cb, __ATOMIC_RELEASE);
4800         }
4801         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
4802
4803         return cb;
4804 }
4805
4806 const struct rte_eth_rxtx_callback *
4807 rte_eth_add_first_rx_callback(uint16_t port_id, uint16_t queue_id,
4808                 rte_rx_callback_fn fn, void *user_param)
4809 {
4810 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
4811         rte_errno = ENOTSUP;
4812         return NULL;
4813 #endif
4814         /* check input parameters */
4815         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
4816                 queue_id >= rte_eth_devices[port_id].data->nb_rx_queues) {
4817                 rte_errno = EINVAL;
4818                 return NULL;
4819         }
4820
4821         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
4822
4823         if (cb == NULL) {
4824                 rte_errno = ENOMEM;
4825                 return NULL;
4826         }
4827
4828         cb->fn.rx = fn;
4829         cb->param = user_param;
4830
4831         rte_spinlock_lock(&rte_eth_rx_cb_lock);
4832         /* Add the callbacks at first position */
4833         cb->next = rte_eth_devices[port_id].post_rx_burst_cbs[queue_id];
4834         /* Stores to cb->fn, cb->param and cb->next should complete before
4835          * cb is visible to data plane threads.
4836          */
4837         __atomic_store_n(
4838                 &rte_eth_devices[port_id].post_rx_burst_cbs[queue_id],
4839                 cb, __ATOMIC_RELEASE);
4840         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
4841
4842         return cb;
4843 }
4844
4845 const struct rte_eth_rxtx_callback *
4846 rte_eth_add_tx_callback(uint16_t port_id, uint16_t queue_id,
4847                 rte_tx_callback_fn fn, void *user_param)
4848 {
4849 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
4850         rte_errno = ENOTSUP;
4851         return NULL;
4852 #endif
4853         struct rte_eth_dev *dev;
4854
4855         /* check input parameters */
4856         if (!rte_eth_dev_is_valid_port(port_id) || fn == NULL ||
4857                     queue_id >= rte_eth_devices[port_id].data->nb_tx_queues) {
4858                 rte_errno = EINVAL;
4859                 return NULL;
4860         }
4861
4862         dev = &rte_eth_devices[port_id];
4863         if (rte_eth_dev_is_tx_hairpin_queue(dev, queue_id)) {
4864                 rte_errno = EINVAL;
4865                 return NULL;
4866         }
4867
4868         struct rte_eth_rxtx_callback *cb = rte_zmalloc(NULL, sizeof(*cb), 0);
4869
4870         if (cb == NULL) {
4871                 rte_errno = ENOMEM;
4872                 return NULL;
4873         }
4874
4875         cb->fn.tx = fn;
4876         cb->param = user_param;
4877
4878         rte_spinlock_lock(&rte_eth_tx_cb_lock);
4879         /* Add the callbacks in fifo order. */
4880         struct rte_eth_rxtx_callback *tail =
4881                 rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id];
4882
4883         if (!tail) {
4884                 /* Stores to cb->fn and cb->param should complete before
4885                  * cb is visible to data plane.
4886                  */
4887                 __atomic_store_n(
4888                         &rte_eth_devices[port_id].pre_tx_burst_cbs[queue_id],
4889                         cb, __ATOMIC_RELEASE);
4890
4891         } else {
4892                 while (tail->next)
4893                         tail = tail->next;
4894                 /* Stores to cb->fn and cb->param should complete before
4895                  * cb is visible to data plane.
4896                  */
4897                 __atomic_store_n(&tail->next, cb, __ATOMIC_RELEASE);
4898         }
4899         rte_spinlock_unlock(&rte_eth_tx_cb_lock);
4900
4901         return cb;
4902 }
4903
4904 int
4905 rte_eth_remove_rx_callback(uint16_t port_id, uint16_t queue_id,
4906                 const struct rte_eth_rxtx_callback *user_cb)
4907 {
4908 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
4909         return -ENOTSUP;
4910 #endif
4911         /* Check input parameters. */
4912         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
4913         if (user_cb == NULL ||
4914                         queue_id >= rte_eth_devices[port_id].data->nb_rx_queues)
4915                 return -EINVAL;
4916
4917         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
4918         struct rte_eth_rxtx_callback *cb;
4919         struct rte_eth_rxtx_callback **prev_cb;
4920         int ret = -EINVAL;
4921
4922         rte_spinlock_lock(&rte_eth_rx_cb_lock);
4923         prev_cb = &dev->post_rx_burst_cbs[queue_id];
4924         for (; *prev_cb != NULL; prev_cb = &cb->next) {
4925                 cb = *prev_cb;
4926                 if (cb == user_cb) {
4927                         /* Remove the user cb from the callback list. */
4928                         __atomic_store_n(prev_cb, cb->next, __ATOMIC_RELAXED);
4929                         ret = 0;
4930                         break;
4931                 }
4932         }
4933         rte_spinlock_unlock(&rte_eth_rx_cb_lock);
4934
4935         return ret;
4936 }
4937
4938 int
4939 rte_eth_remove_tx_callback(uint16_t port_id, uint16_t queue_id,
4940                 const struct rte_eth_rxtx_callback *user_cb)
4941 {
4942 #ifndef RTE_ETHDEV_RXTX_CALLBACKS
4943         return -ENOTSUP;
4944 #endif
4945         /* Check input parameters. */
4946         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
4947         if (user_cb == NULL ||
4948                         queue_id >= rte_eth_devices[port_id].data->nb_tx_queues)
4949                 return -EINVAL;
4950
4951         struct rte_eth_dev *dev = &rte_eth_devices[port_id];
4952         int ret = -EINVAL;
4953         struct rte_eth_rxtx_callback *cb;
4954         struct rte_eth_rxtx_callback **prev_cb;
4955
4956         rte_spinlock_lock(&rte_eth_tx_cb_lock);
4957         prev_cb = &dev->pre_tx_burst_cbs[queue_id];
4958         for (; *prev_cb != NULL; prev_cb = &cb->next) {
4959                 cb = *prev_cb;
4960                 if (cb == user_cb) {
4961                         /* Remove the user cb from the callback list. */
4962                         __atomic_store_n(prev_cb, cb->next, __ATOMIC_RELAXED);
4963                         ret = 0;
4964                         break;
4965                 }
4966         }
4967         rte_spinlock_unlock(&rte_eth_tx_cb_lock);
4968
4969         return ret;
4970 }
4971
4972 int
4973 rte_eth_rx_queue_info_get(uint16_t port_id, uint16_t queue_id,
4974         struct rte_eth_rxq_info *qinfo)
4975 {
4976         struct rte_eth_dev *dev;
4977
4978         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
4979
4980         if (qinfo == NULL)
4981                 return -EINVAL;
4982
4983         dev = &rte_eth_devices[port_id];
4984         if (queue_id >= dev->data->nb_rx_queues) {
4985                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
4986                 return -EINVAL;
4987         }
4988
4989         if (dev->data->rx_queues == NULL ||
4990                         dev->data->rx_queues[queue_id] == NULL) {
4991                 RTE_ETHDEV_LOG(ERR,
4992                                "Rx queue %"PRIu16" of device with port_id=%"
4993                                PRIu16" has not been setup\n",
4994                                queue_id, port_id);
4995                 return -EINVAL;
4996         }
4997
4998         if (rte_eth_dev_is_rx_hairpin_queue(dev, queue_id)) {
4999                 RTE_ETHDEV_LOG(INFO,
5000                         "Can't get hairpin Rx queue %"PRIu16" info of device with port_id=%"PRIu16"\n",
5001                         queue_id, port_id);
5002                 return -EINVAL;
5003         }
5004
5005         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rxq_info_get, -ENOTSUP);
5006
5007         memset(qinfo, 0, sizeof(*qinfo));
5008         dev->dev_ops->rxq_info_get(dev, queue_id, qinfo);
5009         return 0;
5010 }
5011
5012 int
5013 rte_eth_tx_queue_info_get(uint16_t port_id, uint16_t queue_id,
5014         struct rte_eth_txq_info *qinfo)
5015 {
5016         struct rte_eth_dev *dev;
5017
5018         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5019
5020         if (qinfo == NULL)
5021                 return -EINVAL;
5022
5023         dev = &rte_eth_devices[port_id];
5024         if (queue_id >= dev->data->nb_tx_queues) {
5025                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", queue_id);
5026                 return -EINVAL;
5027         }
5028
5029         if (dev->data->tx_queues == NULL ||
5030                         dev->data->tx_queues[queue_id] == NULL) {
5031                 RTE_ETHDEV_LOG(ERR,
5032                                "Tx queue %"PRIu16" of device with port_id=%"
5033                                PRIu16" has not been setup\n",
5034                                queue_id, port_id);
5035                 return -EINVAL;
5036         }
5037
5038         if (rte_eth_dev_is_tx_hairpin_queue(dev, queue_id)) {
5039                 RTE_ETHDEV_LOG(INFO,
5040                         "Can't get hairpin Tx queue %"PRIu16" info of device with port_id=%"PRIu16"\n",
5041                         queue_id, port_id);
5042                 return -EINVAL;
5043         }
5044
5045         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->txq_info_get, -ENOTSUP);
5046
5047         memset(qinfo, 0, sizeof(*qinfo));
5048         dev->dev_ops->txq_info_get(dev, queue_id, qinfo);
5049
5050         return 0;
5051 }
5052
5053 int
5054 rte_eth_rx_burst_mode_get(uint16_t port_id, uint16_t queue_id,
5055                           struct rte_eth_burst_mode *mode)
5056 {
5057         struct rte_eth_dev *dev;
5058
5059         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5060
5061         if (mode == NULL)
5062                 return -EINVAL;
5063
5064         dev = &rte_eth_devices[port_id];
5065
5066         if (queue_id >= dev->data->nb_rx_queues) {
5067                 RTE_ETHDEV_LOG(ERR, "Invalid RX queue_id=%u\n", queue_id);
5068                 return -EINVAL;
5069         }
5070
5071         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->rx_burst_mode_get, -ENOTSUP);
5072         memset(mode, 0, sizeof(*mode));
5073         return eth_err(port_id,
5074                        dev->dev_ops->rx_burst_mode_get(dev, queue_id, mode));
5075 }
5076
5077 int
5078 rte_eth_tx_burst_mode_get(uint16_t port_id, uint16_t queue_id,
5079                           struct rte_eth_burst_mode *mode)
5080 {
5081         struct rte_eth_dev *dev;
5082
5083         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5084
5085         if (mode == NULL)
5086                 return -EINVAL;
5087
5088         dev = &rte_eth_devices[port_id];
5089
5090         if (queue_id >= dev->data->nb_tx_queues) {
5091                 RTE_ETHDEV_LOG(ERR, "Invalid TX queue_id=%u\n", queue_id);
5092                 return -EINVAL;
5093         }
5094
5095         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->tx_burst_mode_get, -ENOTSUP);
5096         memset(mode, 0, sizeof(*mode));
5097         return eth_err(port_id,
5098                        dev->dev_ops->tx_burst_mode_get(dev, queue_id, mode));
5099 }
5100
5101 int
5102 rte_eth_dev_set_mc_addr_list(uint16_t port_id,
5103                              struct rte_ether_addr *mc_addr_set,
5104                              uint32_t nb_mc_addr)
5105 {
5106         struct rte_eth_dev *dev;
5107
5108         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5109
5110         dev = &rte_eth_devices[port_id];
5111         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_mc_addr_list, -ENOTSUP);
5112         return eth_err(port_id, dev->dev_ops->set_mc_addr_list(dev,
5113                                                 mc_addr_set, nb_mc_addr));
5114 }
5115
5116 int
5117 rte_eth_timesync_enable(uint16_t port_id)
5118 {
5119         struct rte_eth_dev *dev;
5120
5121         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5122         dev = &rte_eth_devices[port_id];
5123
5124         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_enable, -ENOTSUP);
5125         return eth_err(port_id, (*dev->dev_ops->timesync_enable)(dev));
5126 }
5127
5128 int
5129 rte_eth_timesync_disable(uint16_t port_id)
5130 {
5131         struct rte_eth_dev *dev;
5132
5133         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5134         dev = &rte_eth_devices[port_id];
5135
5136         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_disable, -ENOTSUP);
5137         return eth_err(port_id, (*dev->dev_ops->timesync_disable)(dev));
5138 }
5139
5140 int
5141 rte_eth_timesync_read_rx_timestamp(uint16_t port_id, struct timespec *timestamp,
5142                                    uint32_t flags)
5143 {
5144         struct rte_eth_dev *dev;
5145
5146         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5147         dev = &rte_eth_devices[port_id];
5148
5149         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_rx_timestamp, -ENOTSUP);
5150         return eth_err(port_id, (*dev->dev_ops->timesync_read_rx_timestamp)
5151                                 (dev, timestamp, flags));
5152 }
5153
5154 int
5155 rte_eth_timesync_read_tx_timestamp(uint16_t port_id,
5156                                    struct timespec *timestamp)
5157 {
5158         struct rte_eth_dev *dev;
5159
5160         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5161         dev = &rte_eth_devices[port_id];
5162
5163         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_tx_timestamp, -ENOTSUP);
5164         return eth_err(port_id, (*dev->dev_ops->timesync_read_tx_timestamp)
5165                                 (dev, timestamp));
5166 }
5167
5168 int
5169 rte_eth_timesync_adjust_time(uint16_t port_id, int64_t delta)
5170 {
5171         struct rte_eth_dev *dev;
5172
5173         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5174         dev = &rte_eth_devices[port_id];
5175
5176         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_adjust_time, -ENOTSUP);
5177         return eth_err(port_id, (*dev->dev_ops->timesync_adjust_time)(dev,
5178                                                                       delta));
5179 }
5180
5181 int
5182 rte_eth_timesync_read_time(uint16_t port_id, struct timespec *timestamp)
5183 {
5184         struct rte_eth_dev *dev;
5185
5186         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5187         dev = &rte_eth_devices[port_id];
5188
5189         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_read_time, -ENOTSUP);
5190         return eth_err(port_id, (*dev->dev_ops->timesync_read_time)(dev,
5191                                                                 timestamp));
5192 }
5193
5194 int
5195 rte_eth_timesync_write_time(uint16_t port_id, const struct timespec *timestamp)
5196 {
5197         struct rte_eth_dev *dev;
5198
5199         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5200         dev = &rte_eth_devices[port_id];
5201
5202         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->timesync_write_time, -ENOTSUP);
5203         return eth_err(port_id, (*dev->dev_ops->timesync_write_time)(dev,
5204                                                                 timestamp));
5205 }
5206
5207 int
5208 rte_eth_read_clock(uint16_t port_id, uint64_t *clock)
5209 {
5210         struct rte_eth_dev *dev;
5211
5212         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5213         dev = &rte_eth_devices[port_id];
5214
5215         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->read_clock, -ENOTSUP);
5216         return eth_err(port_id, (*dev->dev_ops->read_clock)(dev, clock));
5217 }
5218
5219 int
5220 rte_eth_dev_get_reg_info(uint16_t port_id, struct rte_dev_reg_info *info)
5221 {
5222         struct rte_eth_dev *dev;
5223
5224         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5225
5226         dev = &rte_eth_devices[port_id];
5227         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_reg, -ENOTSUP);
5228         return eth_err(port_id, (*dev->dev_ops->get_reg)(dev, info));
5229 }
5230
5231 int
5232 rte_eth_dev_get_eeprom_length(uint16_t port_id)
5233 {
5234         struct rte_eth_dev *dev;
5235
5236         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5237
5238         dev = &rte_eth_devices[port_id];
5239         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom_length, -ENOTSUP);
5240         return eth_err(port_id, (*dev->dev_ops->get_eeprom_length)(dev));
5241 }
5242
5243 int
5244 rte_eth_dev_get_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info)
5245 {
5246         struct rte_eth_dev *dev;
5247
5248         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5249
5250         dev = &rte_eth_devices[port_id];
5251         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_eeprom, -ENOTSUP);
5252         return eth_err(port_id, (*dev->dev_ops->get_eeprom)(dev, info));
5253 }
5254
5255 int
5256 rte_eth_dev_set_eeprom(uint16_t port_id, struct rte_dev_eeprom_info *info)
5257 {
5258         struct rte_eth_dev *dev;
5259
5260         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5261
5262         dev = &rte_eth_devices[port_id];
5263         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->set_eeprom, -ENOTSUP);
5264         return eth_err(port_id, (*dev->dev_ops->set_eeprom)(dev, info));
5265 }
5266
5267 int
5268 rte_eth_dev_get_module_info(uint16_t port_id,
5269                             struct rte_eth_dev_module_info *modinfo)
5270 {
5271         struct rte_eth_dev *dev;
5272
5273         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5274
5275         dev = &rte_eth_devices[port_id];
5276         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_module_info, -ENOTSUP);
5277         return (*dev->dev_ops->get_module_info)(dev, modinfo);
5278 }
5279
5280 int
5281 rte_eth_dev_get_module_eeprom(uint16_t port_id,
5282                               struct rte_dev_eeprom_info *info)
5283 {
5284         struct rte_eth_dev *dev;
5285
5286         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5287
5288         dev = &rte_eth_devices[port_id];
5289         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_module_eeprom, -ENOTSUP);
5290         return (*dev->dev_ops->get_module_eeprom)(dev, info);
5291 }
5292
5293 int
5294 rte_eth_dev_get_dcb_info(uint16_t port_id,
5295                              struct rte_eth_dcb_info *dcb_info)
5296 {
5297         struct rte_eth_dev *dev;
5298
5299         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5300
5301         dev = &rte_eth_devices[port_id];
5302         memset(dcb_info, 0, sizeof(struct rte_eth_dcb_info));
5303
5304         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->get_dcb_info, -ENOTSUP);
5305         return eth_err(port_id, (*dev->dev_ops->get_dcb_info)(dev, dcb_info));
5306 }
5307
5308 int
5309 rte_eth_dev_l2_tunnel_eth_type_conf(uint16_t port_id,
5310                                     struct rte_eth_l2_tunnel_conf *l2_tunnel)
5311 {
5312         struct rte_eth_dev *dev;
5313
5314         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5315         if (l2_tunnel == NULL) {
5316                 RTE_ETHDEV_LOG(ERR, "Invalid l2_tunnel parameter\n");
5317                 return -EINVAL;
5318         }
5319
5320         if (l2_tunnel->l2_tunnel_type >= RTE_TUNNEL_TYPE_MAX) {
5321                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
5322                 return -EINVAL;
5323         }
5324
5325         dev = &rte_eth_devices[port_id];
5326         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->l2_tunnel_eth_type_conf,
5327                                 -ENOTSUP);
5328         return eth_err(port_id, (*dev->dev_ops->l2_tunnel_eth_type_conf)(dev,
5329                                                                 l2_tunnel));
5330 }
5331
5332 int
5333 rte_eth_dev_l2_tunnel_offload_set(uint16_t port_id,
5334                                   struct rte_eth_l2_tunnel_conf *l2_tunnel,
5335                                   uint32_t mask,
5336                                   uint8_t en)
5337 {
5338         struct rte_eth_dev *dev;
5339
5340         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5341
5342         if (l2_tunnel == NULL) {
5343                 RTE_ETHDEV_LOG(ERR, "Invalid l2_tunnel parameter\n");
5344                 return -EINVAL;
5345         }
5346
5347         if (l2_tunnel->l2_tunnel_type >= RTE_TUNNEL_TYPE_MAX) {
5348                 RTE_ETHDEV_LOG(ERR, "Invalid tunnel type\n");
5349                 return -EINVAL;
5350         }
5351
5352         if (mask == 0) {
5353                 RTE_ETHDEV_LOG(ERR, "Mask should have a value\n");
5354                 return -EINVAL;
5355         }
5356
5357         dev = &rte_eth_devices[port_id];
5358         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->l2_tunnel_offload_set,
5359                                 -ENOTSUP);
5360         return eth_err(port_id, (*dev->dev_ops->l2_tunnel_offload_set)(dev,
5361                                                         l2_tunnel, mask, en));
5362 }
5363
5364 static void
5365 rte_eth_dev_adjust_nb_desc(uint16_t *nb_desc,
5366                            const struct rte_eth_desc_lim *desc_lim)
5367 {
5368         if (desc_lim->nb_align != 0)
5369                 *nb_desc = RTE_ALIGN_CEIL(*nb_desc, desc_lim->nb_align);
5370
5371         if (desc_lim->nb_max != 0)
5372                 *nb_desc = RTE_MIN(*nb_desc, desc_lim->nb_max);
5373
5374         *nb_desc = RTE_MAX(*nb_desc, desc_lim->nb_min);
5375 }
5376
5377 int
5378 rte_eth_dev_adjust_nb_rx_tx_desc(uint16_t port_id,
5379                                  uint16_t *nb_rx_desc,
5380                                  uint16_t *nb_tx_desc)
5381 {
5382         struct rte_eth_dev_info dev_info;
5383         int ret;
5384
5385         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5386
5387         ret = rte_eth_dev_info_get(port_id, &dev_info);
5388         if (ret != 0)
5389                 return ret;
5390
5391         if (nb_rx_desc != NULL)
5392                 rte_eth_dev_adjust_nb_desc(nb_rx_desc, &dev_info.rx_desc_lim);
5393
5394         if (nb_tx_desc != NULL)
5395                 rte_eth_dev_adjust_nb_desc(nb_tx_desc, &dev_info.tx_desc_lim);
5396
5397         return 0;
5398 }
5399
5400 int
5401 rte_eth_dev_hairpin_capability_get(uint16_t port_id,
5402                                    struct rte_eth_hairpin_cap *cap)
5403 {
5404         struct rte_eth_dev *dev;
5405
5406         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);
5407
5408         dev = &rte_eth_devices[port_id];
5409         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->hairpin_cap_get, -ENOTSUP);
5410         memset(cap, 0, sizeof(*cap));
5411         return eth_err(port_id, (*dev->dev_ops->hairpin_cap_get)(dev, cap));
5412 }
5413
5414 int
5415 rte_eth_dev_is_rx_hairpin_queue(struct rte_eth_dev *dev, uint16_t queue_id)
5416 {
5417         if (dev->data->rx_queue_state[queue_id] ==
5418             RTE_ETH_QUEUE_STATE_HAIRPIN)
5419                 return 1;
5420         return 0;
5421 }
5422
5423 int
5424 rte_eth_dev_is_tx_hairpin_queue(struct rte_eth_dev *dev, uint16_t queue_id)
5425 {
5426         if (dev->data->tx_queue_state[queue_id] ==
5427             RTE_ETH_QUEUE_STATE_HAIRPIN)
5428                 return 1;
5429         return 0;
5430 }
5431
5432 int
5433 rte_eth_dev_pool_ops_supported(uint16_t port_id, const char *pool)
5434 {
5435         struct rte_eth_dev *dev;
5436
5437         RTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -ENODEV);
5438
5439         if (pool == NULL)
5440                 return -EINVAL;
5441
5442         dev = &rte_eth_devices[port_id];
5443
5444         if (*dev->dev_ops->pool_ops_supported == NULL)
5445                 return 1; /* all pools are supported */
5446
5447         return (*dev->dev_ops->pool_ops_supported)(dev, pool);
5448 }
5449
5450 /**
5451  * A set of values to describe the possible states of a switch domain.
5452  */
5453 enum rte_eth_switch_domain_state {
5454         RTE_ETH_SWITCH_DOMAIN_UNUSED = 0,
5455         RTE_ETH_SWITCH_DOMAIN_ALLOCATED
5456 };
5457
5458 /**
5459  * Array of switch domains available for allocation. Array is sized to
5460  * RTE_MAX_ETHPORTS elements as there cannot be more active switch domains than
5461  * ethdev ports in a single process.
5462  */
5463 static struct rte_eth_dev_switch {
5464         enum rte_eth_switch_domain_state state;
5465 } rte_eth_switch_domains[RTE_MAX_ETHPORTS];
5466
5467 int
5468 rte_eth_switch_domain_alloc(uint16_t *domain_id)
5469 {
5470         unsigned int i;
5471
5472         *domain_id = RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID;
5473
5474         for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
5475                 if (rte_eth_switch_domains[i].state ==
5476                         RTE_ETH_SWITCH_DOMAIN_UNUSED) {
5477                         rte_eth_switch_domains[i].state =
5478                                 RTE_ETH_SWITCH_DOMAIN_ALLOCATED;
5479                         *domain_id = i;
5480                         return 0;
5481                 }
5482         }
5483
5484         return -ENOSPC;
5485 }
5486
5487 int
5488 rte_eth_switch_domain_free(uint16_t domain_id)
5489 {
5490         if (domain_id == RTE_ETH_DEV_SWITCH_DOMAIN_ID_INVALID ||
5491                 domain_id >= RTE_MAX_ETHPORTS)
5492                 return -EINVAL;
5493
5494         if (rte_eth_switch_domains[domain_id].state !=
5495                 RTE_ETH_SWITCH_DOMAIN_ALLOCATED)
5496                 return -EINVAL;
5497
5498         rte_eth_switch_domains[domain_id].state = RTE_ETH_SWITCH_DOMAIN_UNUSED;
5499
5500         return 0;
5501 }
5502
5503 static int
5504 rte_eth_devargs_tokenise(struct rte_kvargs *arglist, const char *str_in)
5505 {
5506         int state;
5507         struct rte_kvargs_pair *pair;
5508         char *letter;
5509
5510         arglist->str = strdup(str_in);
5511         if (arglist->str == NULL)
5512                 return -ENOMEM;
5513
5514         letter = arglist->str;
5515         state = 0;
5516         arglist->count = 0;
5517         pair = &arglist->pairs[0];
5518         while (1) {
5519                 switch (state) {
5520                 case 0: /* Initial */
5521                         if (*letter == '=')
5522                                 return -EINVAL;
5523                         else if (*letter == '\0')
5524                                 return 0;
5525
5526                         state = 1;
5527                         pair->key = letter;
5528                         /* fall-thru */
5529
5530                 case 1: /* Parsing key */
5531                         if (*letter == '=') {
5532                                 *letter = '\0';
5533                                 pair->value = letter + 1;
5534                                 state = 2;
5535                         } else if (*letter == ',' || *letter == '\0')
5536                                 return -EINVAL;
5537                         break;
5538
5539
5540                 case 2: /* Parsing value */
5541                         if (*letter == '[')
5542                                 state = 3;
5543                         else if (*letter == ',') {
5544                                 *letter = '\0';
5545                                 arglist->count++;
5546                                 pair = &arglist->pairs[arglist->count];
5547                                 state = 0;
5548                         } else if (*letter == '\0') {
5549                                 letter--;
5550                                 arglist->count++;
5551                                 pair = &arglist->pairs[arglist->count];
5552                                 state = 0;
5553                         }
5554                         break;
5555
5556                 case 3: /* Parsing list */
5557                         if (*letter == ']')
5558                                 state = 2;
5559                         else if (*letter == '\0')
5560                                 return -EINVAL;
5561                         break;
5562                 }
5563                 letter++;
5564         }
5565 }
5566
5567 int
5568 rte_eth_devargs_parse(const char *dargs, struct rte_eth_devargs *eth_da)
5569 {
5570         struct rte_kvargs args;
5571         struct rte_kvargs_pair *pair;
5572         unsigned int i;
5573         int result = 0;
5574
5575         memset(eth_da, 0, sizeof(*eth_da));
5576
5577         result = rte_eth_devargs_tokenise(&args, dargs);
5578         if (result < 0)
5579                 goto parse_cleanup;
5580
5581         for (i = 0; i < args.count; i++) {
5582                 pair = &args.pairs[i];
5583                 if (strcmp("representor", pair->key) == 0) {
5584                         result = rte_eth_devargs_parse_list(pair->value,
5585                                 rte_eth_devargs_parse_representor_ports,
5586                                 eth_da);
5587                         if (result < 0)
5588                                 goto parse_cleanup;
5589                 }
5590         }
5591
5592 parse_cleanup:
5593         if (args.str)
5594                 free(args.str);
5595
5596         return result;
5597 }
5598
5599 static int
5600 handle_port_list(const char *cmd __rte_unused,
5601                 const char *params __rte_unused,
5602                 struct rte_tel_data *d)
5603 {
5604         int port_id;
5605
5606         rte_tel_data_start_array(d, RTE_TEL_INT_VAL);
5607         RTE_ETH_FOREACH_DEV(port_id)
5608                 rte_tel_data_add_array_int(d, port_id);
5609         return 0;
5610 }
5611
5612 static void
5613 add_port_queue_stats(struct rte_tel_data *d, uint64_t *q_stats,
5614                 const char *stat_name)
5615 {
5616         int q;
5617         struct rte_tel_data *q_data = rte_tel_data_alloc();
5618         rte_tel_data_start_array(q_data, RTE_TEL_U64_VAL);
5619         for (q = 0; q < RTE_ETHDEV_QUEUE_STAT_CNTRS; q++)
5620                 rte_tel_data_add_array_u64(q_data, q_stats[q]);
5621         rte_tel_data_add_dict_container(d, stat_name, q_data, 0);
5622 }
5623
5624 #define ADD_DICT_STAT(stats, s) rte_tel_data_add_dict_u64(d, #s, stats.s)
5625
5626 static int
5627 handle_port_stats(const char *cmd __rte_unused,
5628                 const char *params,
5629                 struct rte_tel_data *d)
5630 {
5631         struct rte_eth_stats stats;
5632         int port_id, ret;
5633
5634         if (params == NULL || strlen(params) == 0 || !isdigit(*params))
5635                 return -1;
5636
5637         port_id = atoi(params);
5638         if (!rte_eth_dev_is_valid_port(port_id))
5639                 return -1;
5640
5641         ret = rte_eth_stats_get(port_id, &stats);
5642         if (ret < 0)
5643                 return -1;
5644
5645         rte_tel_data_start_dict(d);
5646         ADD_DICT_STAT(stats, ipackets);
5647         ADD_DICT_STAT(stats, opackets);
5648         ADD_DICT_STAT(stats, ibytes);
5649         ADD_DICT_STAT(stats, obytes);
5650         ADD_DICT_STAT(stats, imissed);
5651         ADD_DICT_STAT(stats, ierrors);
5652         ADD_DICT_STAT(stats, oerrors);
5653         ADD_DICT_STAT(stats, rx_nombuf);
5654         add_port_queue_stats(d, stats.q_ipackets, "q_ipackets");
5655         add_port_queue_stats(d, stats.q_opackets, "q_opackets");
5656         add_port_queue_stats(d, stats.q_ibytes, "q_ibytes");
5657         add_port_queue_stats(d, stats.q_obytes, "q_obytes");
5658         add_port_queue_stats(d, stats.q_errors, "q_errors");
5659
5660         return 0;
5661 }
5662
5663 static int
5664 handle_port_xstats(const char *cmd __rte_unused,
5665                 const char *params,
5666                 struct rte_tel_data *d)
5667 {
5668         struct rte_eth_xstat *eth_xstats;
5669         struct rte_eth_xstat_name *xstat_names;
5670         int port_id, num_xstats;
5671         int i, ret;
5672         char *end_param;
5673
5674         if (params == NULL || strlen(params) == 0 || !isdigit(*params))
5675                 return -1;
5676
5677         port_id = strtoul(params, &end_param, 0);
5678         if (*end_param != '\0')
5679                 RTE_ETHDEV_LOG(NOTICE,
5680                         "Extra parameters passed to ethdev telemetry command, ignoring");
5681         if (!rte_eth_dev_is_valid_port(port_id))
5682                 return -1;
5683
5684         num_xstats = rte_eth_xstats_get(port_id, NULL, 0);
5685         if (num_xstats < 0)
5686                 return -1;
5687
5688         /* use one malloc for both names and stats */
5689         eth_xstats = malloc((sizeof(struct rte_eth_xstat) +
5690                         sizeof(struct rte_eth_xstat_name)) * num_xstats);
5691         if (eth_xstats == NULL)
5692                 return -1;
5693         xstat_names = (void *)&eth_xstats[num_xstats];
5694
5695         ret = rte_eth_xstats_get_names(port_id, xstat_names, num_xstats);
5696         if (ret < 0 || ret > num_xstats) {
5697                 free(eth_xstats);
5698                 return -1;
5699         }
5700
5701         ret = rte_eth_xstats_get(port_id, eth_xstats, num_xstats);
5702         if (ret < 0 || ret > num_xstats) {
5703                 free(eth_xstats);
5704                 return -1;
5705         }
5706
5707         rte_tel_data_start_dict(d);
5708         for (i = 0; i < num_xstats; i++)
5709                 rte_tel_data_add_dict_u64(d, xstat_names[i].name,
5710                                 eth_xstats[i].value);
5711         return 0;
5712 }
5713
5714 static int
5715 handle_port_link_status(const char *cmd __rte_unused,
5716                 const char *params,
5717                 struct rte_tel_data *d)
5718 {
5719         static const char *status_str = "status";
5720         int ret, port_id;
5721         struct rte_eth_link link;
5722         char *end_param;
5723
5724         if (params == NULL || strlen(params) == 0 || !isdigit(*params))
5725                 return -1;
5726
5727         port_id = strtoul(params, &end_param, 0);
5728         if (*end_param != '\0')
5729                 RTE_ETHDEV_LOG(NOTICE,
5730                         "Extra parameters passed to ethdev telemetry command, ignoring");
5731         if (!rte_eth_dev_is_valid_port(port_id))
5732                 return -1;
5733
5734         ret = rte_eth_link_get(port_id, &link);
5735         if (ret < 0)
5736                 return -1;
5737
5738         rte_tel_data_start_dict(d);
5739         if (!link.link_status) {
5740                 rte_tel_data_add_dict_string(d, status_str, "DOWN");
5741                 return 0;
5742         }
5743         rte_tel_data_add_dict_string(d, status_str, "UP");
5744         rte_tel_data_add_dict_u64(d, "speed", link.link_speed);
5745         rte_tel_data_add_dict_string(d, "duplex",
5746                         (link.link_duplex == ETH_LINK_FULL_DUPLEX) ?
5747                                 "full-duplex" : "half-duplex");
5748         return 0;
5749 }
5750
5751 int
5752 rte_eth_hairpin_queue_peer_update(uint16_t peer_port, uint16_t peer_queue,
5753                                   struct rte_hairpin_peer_info *cur_info,
5754                                   struct rte_hairpin_peer_info *peer_info,
5755                                   uint32_t direction)
5756 {
5757         struct rte_eth_dev *dev;
5758
5759         /* Current queue information is not mandatory. */
5760         if (peer_info == NULL)
5761                 return -EINVAL;
5762
5763         /* No need to check the validity again. */
5764         dev = &rte_eth_devices[peer_port];
5765         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->hairpin_queue_peer_update,
5766                                 -ENOTSUP);
5767
5768         return (*dev->dev_ops->hairpin_queue_peer_update)(dev, peer_queue,
5769                                         cur_info, peer_info, direction);
5770 }
5771
5772 int
5773 rte_eth_hairpin_queue_peer_bind(uint16_t cur_port, uint16_t cur_queue,
5774                                 struct rte_hairpin_peer_info *peer_info,
5775                                 uint32_t direction)
5776 {
5777         struct rte_eth_dev *dev;
5778
5779         if (peer_info == NULL)
5780                 return -EINVAL;
5781
5782         /* No need to check the validity again. */
5783         dev = &rte_eth_devices[cur_port];
5784         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->hairpin_queue_peer_bind,
5785                                 -ENOTSUP);
5786
5787         return (*dev->dev_ops->hairpin_queue_peer_bind)(dev, cur_queue,
5788                                                         peer_info, direction);
5789 }
5790
5791 int
5792 rte_eth_hairpin_queue_peer_unbind(uint16_t cur_port, uint16_t cur_queue,
5793                                   uint32_t direction)
5794 {
5795         struct rte_eth_dev *dev;
5796
5797         /* No need to check the validity again. */
5798         dev = &rte_eth_devices[cur_port];
5799         RTE_FUNC_PTR_OR_ERR_RET(*dev->dev_ops->hairpin_queue_peer_unbind,
5800                                 -ENOTSUP);
5801
5802         return (*dev->dev_ops->hairpin_queue_peer_unbind)(dev, cur_queue,
5803                                                           direction);
5804 }
5805
5806 RTE_LOG_REGISTER(rte_eth_dev_logtype, lib.ethdev, INFO);
5807
5808 RTE_INIT(ethdev_init_telemetry)
5809 {
5810         rte_telemetry_register_cmd("/ethdev/list", handle_port_list,
5811                         "Returns list of available ethdev ports. Takes no parameters");
5812         rte_telemetry_register_cmd("/ethdev/stats", handle_port_stats,
5813                         "Returns the common stats for a port. Parameters: int port_id");
5814         rte_telemetry_register_cmd("/ethdev/xstats", handle_port_xstats,
5815                         "Returns the extended stats for a port. Parameters: int port_id");
5816         rte_telemetry_register_cmd("/ethdev/link_status",
5817                         handle_port_link_status,
5818                         "Returns the link status for a port. Parameters: int port_id");
5819 }