drivers/net/mrvl/mrvl_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Semihalf. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Semihalf nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <rte_ethdev.h>
  34 #include <rte_kvargs.h>
  35 #include <rte_log.h>
  36 #include <rte_malloc.h>
  37 #include <rte_vdev.h>
  38
  39 /* Unluckily, container_of is defined by both DPDK and MUSDK,
  40  * we'll declare only one version.
  41  *
  42  * Note that it is not used in this PMD anyway.
  43  */
  44 #ifdef container_of
  45 #undef container_of
  46 #endif
  47
  48 #include <drivers/mv_pp2.h>
  49 #include <drivers/mv_pp2_bpool.h>
  50 #include <drivers/mv_pp2_hif.h>
  51
  52 #include <fcntl.h>
  53 #include <linux/ethtool.h>
  54 #include <linux/sockios.h>
  55 #include <net/if.h>
  56 #include <net/if_arp.h>
  57 #include <sys/ioctl.h>
  58 #include <sys/socket.h>
  59 #include <sys/stat.h>
  60 #include <sys/types.h>
  61
  62 #include "mrvl_ethdev.h"
  63 #include "mrvl_qos.h"
  64
  65 /* bitmask with reserved hifs */
  66 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
  67 /* bitmask with reserved bpools */
  68 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
  69 /* bitmask with reserved kernel RSS tables */
  70 #define MRVL_MUSDK_RSS_RESERVED 0x01
  71 /* maximum number of available hifs */
  72 #define MRVL_MUSDK_HIFS_MAX 9
  73
  74 /* prefetch shift */
  75 #define MRVL_MUSDK_PREFETCH_SHIFT 2
  76
  77 /* TCAM has 25 entries reserved for uc/mc filter entries */
  78 #define MRVL_MAC_ADDRS_MAX 25
  79 #define MRVL_MATCH_LEN 16
  80 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
  81 /* Maximum allowable packet size */
  82 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
  83
  84 #define MRVL_IFACE_NAME_ARG "iface"
  85 #define MRVL_CFG_ARG "cfg"
  86
  87 #define MRVL_BURST_SIZE 64
  88
  89 #define MRVL_ARP_LENGTH 28
  90
  91 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
  92
  93 #define MRVL_COOKIE_HIGH_ADDR_SHIFT     (sizeof(pp2_cookie_t) * 8)
  94 #define MRVL_COOKIE_HIGH_ADDR_MASK      (~0ULL << MRVL_COOKIE_HIGH_ADDR_SHIFT)
  95
  96 static const char * const valid_args[] = {
  97         MRVL_IFACE_NAME_ARG,
  98         MRVL_CFG_ARG,
  99         NULL
 100 };
 101
 102 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
 103 static struct pp2_hif *hifs[RTE_MAX_LCORE];
 104 static int used_bpools[PP2_NUM_PKT_PROC] = {
 105         MRVL_MUSDK_BPOOLS_RESERVED,
 106         MRVL_MUSDK_BPOOLS_RESERVED
 107 };
 108
 109 struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
 110 int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
 111 uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
 112
 113 /*
 114  * To use buffer harvesting based on loopback port shadow queue structure
 115  * was introduced for buffers information bookkeeping.
 116  *
 117  * Before sending the packet, related buffer information (pp2_buff_inf) is
 118  * stored in shadow queue. After packet is transmitted no longer used
 119  * packet buffer is released back to it's original hardware pool,
 120  * on condition it originated from interface.
 121  * In case it  was generated by application itself i.e: mbuf->port field is
 122  * 0xff then its released to software mempool.
 123  */
 124 struct mrvl_shadow_txq {
 125         int head;           /* write index - used when sending buffers */
 126         int tail;           /* read index - used when releasing buffers */
 127         u16 size;           /* queue occupied size */
 128         u16 num_to_release; /* number of buffers sent, that can be released */
 129         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
 130 };
 131
 132 struct mrvl_rxq {
 133         struct mrvl_priv *priv;
 134         struct rte_mempool *mp;
 135         int queue_id;
 136         int port_id;
 137         int cksum_enabled;
 138         uint64_t bytes_recv;
 139         uint64_t drop_mac;
 140 };
 141
 142 struct mrvl_txq {
 143         struct mrvl_priv *priv;
 144         int queue_id;
 145         int port_id;
 146         uint64_t bytes_sent;
 147 };
 148
 149 /*
 150  * Every tx queue should have dedicated shadow tx queue.
 151  *
 152  * Ports assigned by DPDK might not start at zero or be continuous so
 153  * as a workaround define shadow queues for each possible port so that
 154  * we eventually fit somewhere.
 155  */
 156 struct mrvl_shadow_txq shadow_txqs[RTE_MAX_ETHPORTS][RTE_MAX_LCORE];
 157
 158 /** Number of ports configured. */
 159 int mrvl_ports_nb;
 160 static int mrvl_lcore_first;
 161 static int mrvl_lcore_last;
 162
 163 static inline int
 164 mrvl_get_bpool_size(int pp2_id, int pool_id)
 165 {
 166         int i;
 167         int size = 0;
 168
 169         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
 170                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
 171
 172         return size;
 173 }
 174
 175 static inline int
 176 mrvl_reserve_bit(int *bitmap, int max)
 177 {
 178         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
 179
 180         if (n >= max)
 181                 return -1;
 182
 183         *bitmap |= 1 << n;
 184
 185         return n;
 186 }
 187
 188 /**
 189  * Configure rss based on dpdk rss configuration.
 190  *
 191  * @param priv
 192  *   Pointer to private structure.
 193  * @param rss_conf
 194  *   Pointer to RSS configuration.
 195  *
 196  * @return
 197  *   0 on success, negative error value otherwise.
 198  */
 199 static int
 200 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf)
 201 {
 202         if (rss_conf->rss_key)
 203                 RTE_LOG(WARNING, PMD, "Changing hash key is not supported\n");
 204
 205         if (rss_conf->rss_hf == 0) {
 206                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 207         } else if (rss_conf->rss_hf & ETH_RSS_IPV4) {
 208                 priv->ppio_params.inqs_params.hash_type =
 209                         PP2_PPIO_HASH_T_2_TUPLE;
 210         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
 211                 priv->ppio_params.inqs_params.hash_type =
 212                         PP2_PPIO_HASH_T_5_TUPLE;
 213                 priv->rss_hf_tcp = 1;
 214         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
 215                 priv->ppio_params.inqs_params.hash_type =
 216                         PP2_PPIO_HASH_T_5_TUPLE;
 217                 priv->rss_hf_tcp = 0;
 218         } else {
 219                 return -EINVAL;
 220         }
 221
 222         return 0;
 223 }
 224
 225 /**
 226  * Ethernet device configuration.
 227  *
 228  * Prepare the driver for a given number of TX and RX queues and
 229  * configure RSS.
 230  *
 231  * @param dev
 232  *   Pointer to Ethernet device structure.
 233  *
 234  * @return
 235  *   0 on success, negative error value otherwise.
 236  */
 237 static int
 238 mrvl_dev_configure(struct rte_eth_dev *dev)
 239 {
 240         struct mrvl_priv *priv = dev->data->dev_private;
 241         int ret;
 242
 243         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
 244             dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
 245                 RTE_LOG(INFO, PMD, "Unsupported rx multi queue mode %d\n",
 246                         dev->data->dev_conf.rxmode.mq_mode);
 247                 return -EINVAL;
 248         }
 249
 250         if (!dev->data->dev_conf.rxmode.hw_strip_crc) {
 251                 RTE_LOG(INFO, PMD,
 252                         "L2 CRC stripping is always enabled in hw\n");
 253                 dev->data->dev_conf.rxmode.hw_strip_crc = 1;
 254         }
 255
 256         if (dev->data->dev_conf.rxmode.hw_vlan_strip) {
 257                 RTE_LOG(INFO, PMD, "VLAN stripping not supported\n");
 258                 return -EINVAL;
 259         }
 260
 261         if (dev->data->dev_conf.rxmode.split_hdr_size) {
 262                 RTE_LOG(INFO, PMD, "Split headers not supported\n");
 263                 return -EINVAL;
 264         }
 265
 266         if (dev->data->dev_conf.rxmode.enable_scatter) {
 267                 RTE_LOG(INFO, PMD, "RX Scatter/Gather not supported\n");
 268                 return -EINVAL;
 269         }
 270
 271         if (dev->data->dev_conf.rxmode.enable_lro) {
 272                 RTE_LOG(INFO, PMD, "LRO not supported\n");
 273                 return -EINVAL;
 274         }
 275
 276         if (dev->data->dev_conf.rxmode.jumbo_frame)
 277                 dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
 278                                  ETHER_HDR_LEN - ETHER_CRC_LEN;
 279
 280         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
 281                                   dev->data->nb_rx_queues);
 282         if (ret < 0)
 283                 return ret;
 284
 285         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
 286         priv->ppio_params.maintain_stats = 1;
 287         priv->nb_rx_queues = dev->data->nb_rx_queues;
 288
 289         if (dev->data->nb_rx_queues == 1 &&
 290             dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
 291                 RTE_LOG(WARNING, PMD, "Disabling hash for 1 rx queue\n");
 292                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 293
 294                 return 0;
 295         }
 296
 297         return mrvl_configure_rss(priv,
 298                                   &dev->data->dev_conf.rx_adv_conf.rss_conf);
 299 }
 300
 301 /**
 302  * DPDK callback to change the MTU.
 303  *
 304  * Setting the MTU affects hardware MRU (packets larger than the MRU
 305  * will be dropped).
 306  *
 307  * @param dev
 308  *   Pointer to Ethernet device structure.
 309  * @param mtu
 310  *   New MTU.
 311  *
 312  * @return
 313  *   0 on success, negative error value otherwise.
 314  */
 315 static int
 316 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 317 {
 318         struct mrvl_priv *priv = dev->data->dev_private;
 319         /* extra MV_MH_SIZE bytes are required for Marvell tag */
 320         uint16_t mru = mtu + MV_MH_SIZE + ETHER_HDR_LEN + ETHER_CRC_LEN;
 321         int ret;
 322
 323         if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX)
 324                 return -EINVAL;
 325
 326         ret = pp2_ppio_set_mru(priv->ppio, mru);
 327         if (ret)
 328                 return ret;
 329
 330         return pp2_ppio_set_mtu(priv->ppio, mtu);
 331 }
 332
 333 /**
 334  * DPDK callback to bring the link up.
 335  *
 336  * @param dev
 337  *   Pointer to Ethernet device structure.
 338  *
 339  * @return
 340  *   0 on success, negative error value otherwise.
 341  */
 342 static int
 343 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
 344 {
 345         struct mrvl_priv *priv = dev->data->dev_private;
 346         int ret;
 347
 348         ret = pp2_ppio_enable(priv->ppio);
 349         if (ret)
 350                 return ret;
 351
 352         /*
 353          * mtu/mru can be updated if pp2_ppio_enable() was called at least once
 354          * as pp2_ppio_enable() changes port->t_mode from default 0 to
 355          * PP2_TRAFFIC_INGRESS_EGRESS.
 356          *
 357          * Set mtu to default DPDK value here.
 358          */
 359         ret = mrvl_mtu_set(dev, dev->data->mtu);
 360         if (ret)
 361                 pp2_ppio_disable(priv->ppio);
 362
 363         dev->data->dev_link.link_status = ETH_LINK_UP;
 364
 365         return ret;
 366 }
 367
 368 /**
 369  * DPDK callback to bring the link down.
 370  *
 371  * @param dev
 372  *   Pointer to Ethernet device structure.
 373  *
 374  * @return
 375  *   0 on success, negative error value otherwise.
 376  */
 377 static int
 378 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
 379 {
 380         struct mrvl_priv *priv = dev->data->dev_private;
 381         int ret;
 382
 383         ret = pp2_ppio_disable(priv->ppio);
 384         if (ret)
 385                 return ret;
 386
 387         dev->data->dev_link.link_status = ETH_LINK_DOWN;
 388
 389         return ret;
 390 }
 391
 392 /**
 393  * DPDK callback to start the device.
 394  *
 395  * @param dev
 396  *   Pointer to Ethernet device structure.
 397  *
 398  * @return
 399  *   0 on success, negative errno value on failure.
 400  */
 401 static int
 402 mrvl_dev_start(struct rte_eth_dev *dev)
 403 {
 404         struct mrvl_priv *priv = dev->data->dev_private;
 405         char match[MRVL_MATCH_LEN];
 406         int ret;
 407
 408         snprintf(match, sizeof(match), "ppio-%d:%d",
 409                  priv->pp_id, priv->ppio_id);
 410         priv->ppio_params.match = match;
 411
 412         /*
 413          * Calculate the maximum bpool size for refill feature to 1.5 of the
 414          * configured size. In case the bpool size will exceed this value,
 415          * superfluous buffers will be removed
 416          */
 417         priv->bpool_max_size = priv->bpool_init_size +
 418                               (priv->bpool_init_size >> 1);
 419         /*
 420          * Calculate the minimum bpool size for refill feature as follows:
 421          * 2 default burst sizes multiply by number of rx queues.
 422          * If the bpool size will be below this value, new buffers will
 423          * be added to the pool.
 424          */
 425         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
 426
 427         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
 428         if (ret)
 429                 return ret;
 430
 431         /*
 432          * In case there are some some stale uc/mc mac addresses flush them
 433          * here. It cannot be done during mrvl_dev_close() as port information
 434          * is already gone at that point (due to pp2_ppio_deinit() in
 435          * mrvl_dev_stop()).
 436          */
 437         if (!priv->uc_mc_flushed) {
 438                 ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1);
 439                 if (ret) {
 440                         RTE_LOG(ERR, PMD,
 441                                 "Failed to flush uc/mc filter list\n");
 442                         goto out;
 443                 }
 444                 priv->uc_mc_flushed = 1;
 445         }
 446
 447         if (!priv->vlan_flushed) {
 448                 ret = pp2_ppio_flush_vlan(priv->ppio);
 449                 if (ret) {
 450                         RTE_LOG(ERR, PMD, "Failed to flush vlan list\n");
 451                         /*
 452                          * TODO
 453                          * once pp2_ppio_flush_vlan() is supported jump to out
 454                          * goto out;
 455                          */
 456                 }
 457                 priv->vlan_flushed = 1;
 458         }
 459
 460         /* For default QoS config, don't start classifier. */
 461         if (mrvl_qos_cfg) {
 462                 ret = mrvl_start_qos_mapping(priv);
 463                 if (ret) {
 464                         pp2_ppio_deinit(priv->ppio);
 465                         return ret;
 466                 }
 467         }
 468
 469         ret = mrvl_dev_set_link_up(dev);
 470         if (ret)
 471                 goto out;
 472
 473         return 0;
 474 out:
 475         pp2_ppio_deinit(priv->ppio);
 476         return ret;
 477 }
 478
 479 /**
 480  * Flush receive queues.
 481  *
 482  * @param dev
 483  *   Pointer to Ethernet device structure.
 484  */
 485 static void
 486 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
 487 {
 488         int i;
 489
 490         RTE_LOG(INFO, PMD, "Flushing rx queues\n");
 491         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 492                 int ret, num;
 493
 494                 do {
 495                         struct mrvl_rxq *q = dev->data->rx_queues[i];
 496                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
 497
 498                         num = MRVL_PP2_RXD_MAX;
 499                         ret = pp2_ppio_recv(q->priv->ppio,
 500                                             q->priv->rxq_map[q->queue_id].tc,
 501                                             q->priv->rxq_map[q->queue_id].inq,
 502                                             descs, (uint16_t *)&num);
 503                 } while (ret == 0 && num);
 504         }
 505 }
 506
 507 /**
 508  * Flush transmit shadow queues.
 509  *
 510  * @param dev
 511  *   Pointer to Ethernet device structure.
 512  */
 513 static void
 514 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
 515 {
 516         int i;
 517
 518         RTE_LOG(INFO, PMD, "Flushing tx shadow queues\n");
 519         for (i = 0; i < RTE_MAX_LCORE; i++) {
 520                 struct mrvl_shadow_txq *sq =
 521                         &shadow_txqs[dev->data->port_id][i];
 522
 523                 while (sq->tail != sq->head) {
 524                         uint64_t addr = cookie_addr_high |
 525                                         sq->ent[sq->tail].buff.cookie;
 526                         rte_pktmbuf_free((struct rte_mbuf *)addr);
 527                         sq->tail = (sq->tail + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
 528                 }
 529
 530                 memset(sq, 0, sizeof(*sq));
 531         }
 532 }
 533
 534 /**
 535  * Flush hardware bpool (buffer-pool).
 536  *
 537  * @param dev
 538  *   Pointer to Ethernet device structure.
 539  */
 540 static void
 541 mrvl_flush_bpool(struct rte_eth_dev *dev)
 542 {
 543         struct mrvl_priv *priv = dev->data->dev_private;
 544         uint32_t num;
 545         int ret;
 546
 547         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
 548         if (ret) {
 549                 RTE_LOG(ERR, PMD, "Failed to get bpool buffers number\n");
 550                 return;
 551         }
 552
 553         while (num--) {
 554                 struct pp2_buff_inf inf;
 555                 uint64_t addr;
 556
 557                 ret = pp2_bpool_get_buff(hifs[rte_lcore_id()], priv->bpool,
 558                                          &inf);
 559                 if (ret)
 560                         break;
 561
 562                 addr = cookie_addr_high | inf.cookie;
 563                 rte_pktmbuf_free((struct rte_mbuf *)addr);
 564         }
 565 }
 566
 567 /**
 568  * DPDK callback to stop the device.
 569  *
 570  * @param dev
 571  *   Pointer to Ethernet device structure.
 572  */
 573 static void
 574 mrvl_dev_stop(struct rte_eth_dev *dev)
 575 {
 576         struct mrvl_priv *priv = dev->data->dev_private;
 577
 578         mrvl_dev_set_link_down(dev);
 579         mrvl_flush_rx_queues(dev);
 580         mrvl_flush_tx_shadow_queues(dev);
 581         if (priv->qos_tbl)
 582                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
 583         pp2_ppio_deinit(priv->ppio);
 584         priv->ppio = NULL;
 585 }
 586
 587 /**
 588  * DPDK callback to close the device.
 589  *
 590  * @param dev
 591  *   Pointer to Ethernet device structure.
 592  */
 593 static void
 594 mrvl_dev_close(struct rte_eth_dev *dev)
 595 {
 596         struct mrvl_priv *priv = dev->data->dev_private;
 597         size_t i;
 598
 599         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
 600                 struct pp2_ppio_tc_params *tc_params =
 601                         &priv->ppio_params.inqs_params.tcs_params[i];
 602
 603                 if (tc_params->inqs_params) {
 604                         rte_free(tc_params->inqs_params);
 605                         tc_params->inqs_params = NULL;
 606                 }
 607         }
 608
 609         mrvl_flush_bpool(dev);
 610 }
 611
 612 /**
 613  * DPDK callback to retrieve physical link information.
 614  *
 615  * @param dev
 616  *   Pointer to Ethernet device structure.
 617  * @param wait_to_complete
 618  *   Wait for request completion (ignored).
 619  *
 620  * @return
 621  *   0 on success, negative error value otherwise.
 622  */
 623 static int
 624 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
 625 {
 626         /*
 627          * TODO
 628          * once MUSDK provides necessary API use it here
 629          */
 630         struct ethtool_cmd edata;
 631         struct ifreq req;
 632         int ret, fd;
 633
 634         edata.cmd = ETHTOOL_GSET;
 635
 636         strcpy(req.ifr_name, dev->data->name);
 637         req.ifr_data = (void *)&edata;
 638
 639         fd = socket(AF_INET, SOCK_DGRAM, 0);
 640         if (fd == -1)
 641                 return -EFAULT;
 642
 643         ret = ioctl(fd, SIOCETHTOOL, &req);
 644         if (ret == -1) {
 645                 close(fd);
 646                 return -EFAULT;
 647         }
 648
 649         close(fd);
 650
 651         switch (ethtool_cmd_speed(&edata)) {
 652         case SPEED_10:
 653                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
 654                 break;
 655         case SPEED_100:
 656                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
 657                 break;
 658         case SPEED_1000:
 659                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
 660                 break;
 661         case SPEED_10000:
 662                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
 663                 break;
 664         default:
 665                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
 666         }
 667
 668         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
 669                                                          ETH_LINK_HALF_DUPLEX;
 670         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
 671                                                            ETH_LINK_FIXED;
 672
 673         return 0;
 674 }
 675
 676 /**
 677  * DPDK callback to enable promiscuous mode.
 678  *
 679  * @param dev
 680  *   Pointer to Ethernet device structure.
 681  */
 682 static void
 683 mrvl_promiscuous_enable(struct rte_eth_dev *dev)
 684 {
 685         struct mrvl_priv *priv = dev->data->dev_private;
 686         int ret;
 687
 688         ret = pp2_ppio_set_uc_promisc(priv->ppio, 1);
 689         if (ret)
 690                 RTE_LOG(ERR, PMD, "Failed to enable promiscuous mode\n");
 691 }
 692
 693 /**
 694  * DPDK callback to enable allmulti mode.
 695  *
 696  * @param dev
 697  *   Pointer to Ethernet device structure.
 698  */
 699 static void
 700 mrvl_allmulticast_enable(struct rte_eth_dev *dev)
 701 {
 702         struct mrvl_priv *priv = dev->data->dev_private;
 703         int ret;
 704
 705         ret = pp2_ppio_set_mc_promisc(priv->ppio, 1);
 706         if (ret)
 707                 RTE_LOG(ERR, PMD, "Failed enable all-multicast mode\n");
 708 }
 709
 710 /**
 711  * DPDK callback to disable promiscuous mode.
 712  *
 713  * @param dev
 714  *   Pointer to Ethernet device structure.
 715  */
 716 static void
 717 mrvl_promiscuous_disable(struct rte_eth_dev *dev)
 718 {
 719         struct mrvl_priv *priv = dev->data->dev_private;
 720         int ret;
 721
 722         ret = pp2_ppio_set_uc_promisc(priv->ppio, 0);
 723         if (ret)
 724                 RTE_LOG(ERR, PMD, "Failed to disable promiscuous mode\n");
 725 }
 726
 727 /**
 728  * DPDK callback to disable allmulticast mode.
 729  *
 730  * @param dev
 731  *   Pointer to Ethernet device structure.
 732  */
 733 static void
 734 mrvl_allmulticast_disable(struct rte_eth_dev *dev)
 735 {
 736         struct mrvl_priv *priv = dev->data->dev_private;
 737         int ret;
 738
 739         ret = pp2_ppio_set_mc_promisc(priv->ppio, 0);
 740         if (ret)
 741                 RTE_LOG(ERR, PMD, "Failed to disable all-multicast mode\n");
 742 }
 743
 744 /**
 745  * DPDK callback to remove a MAC address.
 746  *
 747  * @param dev
 748  *   Pointer to Ethernet device structure.
 749  * @param index
 750  *   MAC address index.
 751  */
 752 static void
 753 mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
 754 {
 755         struct mrvl_priv *priv = dev->data->dev_private;
 756         char buf[ETHER_ADDR_FMT_SIZE];
 757         int ret;
 758
 759         ret = pp2_ppio_remove_mac_addr(priv->ppio,
 760                                        dev->data->mac_addrs[index].addr_bytes);
 761         if (ret) {
 762                 ether_format_addr(buf, sizeof(buf),
 763                                   &dev->data->mac_addrs[index]);
 764                 RTE_LOG(ERR, PMD, "Failed to remove mac %s\n", buf);
 765         }
 766 }
 767
 768 /**
 769  * DPDK callback to add a MAC address.
 770  *
 771  * @param dev
 772  *   Pointer to Ethernet device structure.
 773  * @param mac_addr
 774  *   MAC address to register.
 775  * @param index
 776  *   MAC address index.
 777  * @param vmdq
 778  *   VMDq pool index to associate address with (unused).
 779  *
 780  * @return
 781  *   0 on success, negative error value otherwise.
 782  */
 783 static int
 784 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 785                   uint32_t index, uint32_t vmdq __rte_unused)
 786 {
 787         struct mrvl_priv *priv = dev->data->dev_private;
 788         char buf[ETHER_ADDR_FMT_SIZE];
 789         int ret;
 790
 791         if (index == 0)
 792                 /* For setting index 0, mrvl_mac_addr_set() should be used.*/
 793                 return -1;
 794
 795         /*
 796          * Maximum number of uc addresses can be tuned via kernel module mvpp2x
 797          * parameter uc_filter_max. Maximum number of mc addresses is then
 798          * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and
 799          * 21 respectively.
 800          *
 801          * If more than uc_filter_max uc addresses were added to filter list
 802          * then NIC will switch to promiscuous mode automatically.
 803          *
 804          * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses
 805          * were added to filter list then NIC will switch to all-multicast mode
 806          * automatically.
 807          */
 808         ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
 809         if (ret) {
 810                 ether_format_addr(buf, sizeof(buf), mac_addr);
 811                 RTE_LOG(ERR, PMD, "Failed to add mac %s\n", buf);
 812                 return -1;
 813         }
 814
 815         return 0;
 816 }
 817
 818 /**
 819  * DPDK callback to set the primary MAC address.
 820  *
 821  * @param dev
 822  *   Pointer to Ethernet device structure.
 823  * @param mac_addr
 824  *   MAC address to register.
 825  */
 826 static void
 827 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
 828 {
 829         struct mrvl_priv *priv = dev->data->dev_private;
 830
 831         pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
 832         /*
 833          * TODO
 834          * Port stops sending packets if pp2_ppio_set_mac_addr()
 835          * was called after pp2_ppio_enable(). As a quick fix issue
 836          * enable port once again.
 837          */
 838         pp2_ppio_enable(priv->ppio);
 839 }
 840
 841 /**
 842  * DPDK callback to get device statistics.
 843  *
 844  * @param dev
 845  *   Pointer to Ethernet device structure.
 846  * @param stats
 847  *   Stats structure output buffer.
 848  */
 849 static void
 850 mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 851 {
 852         struct mrvl_priv *priv = dev->data->dev_private;
 853         struct pp2_ppio_statistics ppio_stats;
 854         uint64_t drop_mac = 0;
 855         unsigned int i, idx, ret;
 856
 857         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 858                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
 859                 struct pp2_ppio_inq_statistics rx_stats;
 860
 861                 if (!rxq)
 862                         continue;
 863
 864                 idx = rxq->queue_id;
 865                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
 866                         RTE_LOG(ERR, PMD,
 867                                 "rx queue %d stats out of range (0 - %d)\n",
 868                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
 869                         continue;
 870                 }
 871
 872                 ret = pp2_ppio_inq_get_statistics(priv->ppio,
 873                                                   priv->rxq_map[idx].tc,
 874                                                   priv->rxq_map[idx].inq,
 875                                                   &rx_stats, 0);
 876                 if (unlikely(ret)) {
 877                         RTE_LOG(ERR, PMD,
 878                                 "Failed to update rx queue %d stats\n", idx);
 879                         break;
 880                 }
 881
 882                 stats->q_ibytes[idx] = rxq->bytes_recv;
 883                 stats->q_ipackets[idx] = rx_stats.enq_desc - rxq->drop_mac;
 884                 stats->q_errors[idx] = rx_stats.drop_early +
 885                                        rx_stats.drop_fullq +
 886                                        rx_stats.drop_bm +
 887                                        rxq->drop_mac;
 888                 stats->ibytes += rxq->bytes_recv;
 889                 drop_mac += rxq->drop_mac;
 890         }
 891
 892         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 893                 struct mrvl_txq *txq = dev->data->tx_queues[i];
 894                 struct pp2_ppio_outq_statistics tx_stats;
 895
 896                 if (!txq)
 897                         continue;
 898
 899                 idx = txq->queue_id;
 900                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
 901                         RTE_LOG(ERR, PMD,
 902                                 "tx queue %d stats out of range (0 - %d)\n",
 903                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
 904                 }
 905
 906                 ret = pp2_ppio_outq_get_statistics(priv->ppio, idx,
 907                                                    &tx_stats, 0);
 908                 if (unlikely(ret)) {
 909                         RTE_LOG(ERR, PMD,
 910                                 "Failed to update tx queue %d stats\n", idx);
 911                         break;
 912                 }
 913
 914                 stats->q_opackets[idx] = tx_stats.deq_desc;
 915                 stats->q_obytes[idx] = txq->bytes_sent;
 916                 stats->obytes += txq->bytes_sent;
 917         }
 918
 919         ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
 920         if (unlikely(ret)) {
 921                 RTE_LOG(ERR, PMD, "Failed to update port statistics\n");
 922                 return;
 923         }
 924
 925         stats->ipackets += ppio_stats.rx_packets - drop_mac;
 926         stats->opackets += ppio_stats.tx_packets;
 927         stats->imissed += ppio_stats.rx_fullq_dropped +
 928                           ppio_stats.rx_bm_dropped +
 929                           ppio_stats.rx_early_dropped +
 930                           ppio_stats.rx_fifo_dropped +
 931                           ppio_stats.rx_cls_dropped;
 932         stats->ierrors = drop_mac;
 933 }
 934
 935 /**
 936  * DPDK callback to clear device statistics.
 937  *
 938  * @param dev
 939  *   Pointer to Ethernet device structure.
 940  */
 941 static void
 942 mrvl_stats_reset(struct rte_eth_dev *dev)
 943 {
 944         struct mrvl_priv *priv = dev->data->dev_private;
 945         int i;
 946
 947         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 948                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
 949
 950                 pp2_ppio_inq_get_statistics(priv->ppio, priv->rxq_map[i].tc,
 951                                             priv->rxq_map[i].inq, NULL, 1);
 952                 rxq->bytes_recv = 0;
 953                 rxq->drop_mac = 0;
 954         }
 955
 956         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 957                 struct mrvl_txq *txq = dev->data->tx_queues[i];
 958
 959                 pp2_ppio_outq_get_statistics(priv->ppio, i, NULL, 1);
 960                 txq->bytes_sent = 0;
 961         }
 962
 963         pp2_ppio_get_statistics(priv->ppio, NULL, 1);
 964 }
 965
 966 /**
 967  * DPDK callback to get information about the device.
 968  *
 969  * @param dev
 970  *   Pointer to Ethernet device structure (unused).
 971  * @param info
 972  *   Info structure output buffer.
 973  */
 974 static void
 975 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
 976                    struct rte_eth_dev_info *info)
 977 {
 978         info->speed_capa = ETH_LINK_SPEED_10M |
 979                            ETH_LINK_SPEED_100M |
 980                            ETH_LINK_SPEED_1G |
 981                            ETH_LINK_SPEED_10G;
 982
 983         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
 984         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
 985         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
 986
 987         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
 988         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
 989         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
 990
 991         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
 992         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
 993         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
 994
 995         info->rx_offload_capa = DEV_RX_OFFLOAD_JUMBO_FRAME |
 996                                 DEV_RX_OFFLOAD_VLAN_FILTER |
 997                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 998                                 DEV_RX_OFFLOAD_UDP_CKSUM |
 999                                 DEV_RX_OFFLOAD_TCP_CKSUM;
1000
1001         info->tx_offload_capa = DEV_TX_OFFLOAD_IPV4_CKSUM |
1002                                 DEV_TX_OFFLOAD_UDP_CKSUM |
1003                                 DEV_TX_OFFLOAD_TCP_CKSUM;
1004
1005         info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1006                                        ETH_RSS_NONFRAG_IPV4_TCP |
1007                                        ETH_RSS_NONFRAG_IPV4_UDP;
1008
1009         /* By default packets are dropped if no descriptors are available */
1010         info->default_rxconf.rx_drop_en = 1;
1011
1012         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
1013 }
1014
1015 /**
1016  * Return supported packet types.
1017  *
1018  * @param dev
1019  *   Pointer to Ethernet device structure (unused).
1020  *
1021  * @return
1022  *   Const pointer to the table with supported packet types.
1023  */
1024 static const uint32_t *
1025 mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1026 {
1027         static const uint32_t ptypes[] = {
1028                 RTE_PTYPE_L2_ETHER,
1029                 RTE_PTYPE_L3_IPV4,
1030                 RTE_PTYPE_L3_IPV4_EXT,
1031                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1032                 RTE_PTYPE_L3_IPV6,
1033                 RTE_PTYPE_L3_IPV6_EXT,
1034                 RTE_PTYPE_L2_ETHER_ARP,
1035                 RTE_PTYPE_L4_TCP,
1036                 RTE_PTYPE_L4_UDP
1037         };
1038
1039         return ptypes;
1040 }
1041
1042 /**
1043  * DPDK callback to get information about specific receive queue.
1044  *
1045  * @param dev
1046  *   Pointer to Ethernet device structure.
1047  * @param rx_queue_id
1048  *   Receive queue index.
1049  * @param qinfo
1050  *   Receive queue information structure.
1051  */
1052 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1053                               struct rte_eth_rxq_info *qinfo)
1054 {
1055         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
1056         struct mrvl_priv *priv = dev->data->dev_private;
1057         int inq = priv->rxq_map[rx_queue_id].inq;
1058         int tc = priv->rxq_map[rx_queue_id].tc;
1059         struct pp2_ppio_tc_params *tc_params =
1060                 &priv->ppio_params.inqs_params.tcs_params[tc];
1061
1062         qinfo->mp = q->mp;
1063         qinfo->nb_desc = tc_params->inqs_params[inq].size;
1064 }
1065
1066 /**
1067  * DPDK callback to get information about specific transmit queue.
1068  *
1069  * @param dev
1070  *   Pointer to Ethernet device structure.
1071  * @param tx_queue_id
1072  *   Transmit queue index.
1073  * @param qinfo
1074  *   Transmit queue information structure.
1075  */
1076 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1077                               struct rte_eth_txq_info *qinfo)
1078 {
1079         struct mrvl_priv *priv = dev->data->dev_private;
1080
1081         qinfo->nb_desc =
1082                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
1083 }
1084
1085 /**
1086  * DPDK callback to Configure a VLAN filter.
1087  *
1088  * @param dev
1089  *   Pointer to Ethernet device structure.
1090  * @param vlan_id
1091  *   VLAN ID to filter.
1092  * @param on
1093  *   Toggle filter.
1094  *
1095  * @return
1096  *   0 on success, negative error value otherwise.
1097  */
1098 static int
1099 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1100 {
1101         struct mrvl_priv *priv = dev->data->dev_private;
1102
1103         return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) :
1104                     pp2_ppio_remove_vlan(priv->ppio, vlan_id);
1105 }
1106
1107 /**
1108  * Release buffers to hardware bpool (buffer-pool)
1109  *
1110  * @param rxq
1111  *   Receive queue pointer.
1112  * @param num
1113  *   Number of buffers to release to bpool.
1114  *
1115  * @return
1116  *   0 on success, negative error value otherwise.
1117  */
1118 static int
1119 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
1120 {
1121         struct buff_release_entry entries[MRVL_PP2_TXD_MAX];
1122         struct rte_mbuf *mbufs[MRVL_PP2_TXD_MAX];
1123         int i, ret;
1124         unsigned int core_id = rte_lcore_id();
1125         struct pp2_hif *hif = hifs[core_id];
1126         struct pp2_bpool *bpool = rxq->priv->bpool;
1127
1128         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
1129         if (ret)
1130                 return ret;
1131
1132         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
1133                 cookie_addr_high =
1134                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
1135
1136         for (i = 0; i < num; i++) {
1137                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
1138                         != cookie_addr_high) {
1139                         RTE_LOG(ERR, PMD,
1140                                 "mbuf virtual addr high 0x%lx out of range\n",
1141                                 (uint64_t)mbufs[i] >> 32);
1142                         goto out;
1143                 }
1144
1145                 entries[i].buff.addr =
1146                         rte_mbuf_data_dma_addr_default(mbufs[i]);
1147                 entries[i].buff.cookie = (pp2_cookie_t)(uint64_t)mbufs[i];
1148                 entries[i].bpool = bpool;
1149         }
1150
1151         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
1152         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
1153
1154         if (i != num)
1155                 goto out;
1156
1157         return 0;
1158 out:
1159         for (; i < num; i++)
1160                 rte_pktmbuf_free(mbufs[i]);
1161
1162         return -1;
1163 }
1164
1165 /**
1166  * DPDK callback to configure the receive queue.
1167  *
1168  * @param dev
1169  *   Pointer to Ethernet device structure.
1170  * @param idx
1171  *   RX queue index.
1172  * @param desc
1173  *   Number of descriptors to configure in queue.
1174  * @param socket
1175  *   NUMA socket on which memory must be allocated.
1176  * @param conf
1177  *   Thresholds parameters (unused_).
1178  * @param mp
1179  *   Memory pool for buffer allocations.
1180  *
1181  * @return
1182  *   0 on success, negative error value otherwise.
1183  */
1184 static int
1185 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1186                     unsigned int socket,
1187                     const struct rte_eth_rxconf *conf __rte_unused,
1188                     struct rte_mempool *mp)
1189 {
1190         struct mrvl_priv *priv = dev->data->dev_private;
1191         struct mrvl_rxq *rxq;
1192         uint32_t min_size,
1193                  max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1194         int ret, tc, inq;
1195
1196         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
1197                 /*
1198                  * Unknown TC mapping, mapping will not have a correct queue.
1199                  */
1200                 RTE_LOG(ERR, PMD, "Unknown TC mapping for queue %hu eth%hhu\n",
1201                         idx, priv->ppio_id);
1202                 return -EFAULT;
1203         }
1204
1205         min_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM -
1206                    MRVL_PKT_EFFEC_OFFS;
1207         if (min_size < max_rx_pkt_len) {
1208                 RTE_LOG(ERR, PMD,
1209                         "Mbuf size must be increased to %u bytes to hold up to %u bytes of data.\n",
1210                         max_rx_pkt_len + RTE_PKTMBUF_HEADROOM +
1211                         MRVL_PKT_EFFEC_OFFS,
1212                         max_rx_pkt_len);
1213                 return -EINVAL;
1214         }
1215
1216         if (dev->data->rx_queues[idx]) {
1217                 rte_free(dev->data->rx_queues[idx]);
1218                 dev->data->rx_queues[idx] = NULL;
1219         }
1220
1221         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
1222         if (!rxq)
1223                 return -ENOMEM;
1224
1225         rxq->priv = priv;
1226         rxq->mp = mp;
1227         rxq->cksum_enabled = dev->data->dev_conf.rxmode.hw_ip_checksum;
1228         rxq->queue_id = idx;
1229         rxq->port_id = dev->data->port_id;
1230         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
1231
1232         tc = priv->rxq_map[rxq->queue_id].tc,
1233         inq = priv->rxq_map[rxq->queue_id].inq;
1234         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
1235                 desc;
1236
1237         ret = mrvl_fill_bpool(rxq, desc);
1238         if (ret) {
1239                 rte_free(rxq);
1240                 return ret;
1241         }
1242
1243         priv->bpool_init_size += desc;
1244
1245         dev->data->rx_queues[idx] = rxq;
1246
1247         return 0;
1248 }
1249
1250 /**
1251  * DPDK callback to release the receive queue.
1252  *
1253  * @param rxq
1254  *   Generic receive queue pointer.
1255  */
1256 static void
1257 mrvl_rx_queue_release(void *rxq)
1258 {
1259         struct mrvl_rxq *q = rxq;
1260         struct pp2_ppio_tc_params *tc_params;
1261         int i, num, tc, inq;
1262
1263         if (!q)
1264                 return;
1265
1266         tc = q->priv->rxq_map[q->queue_id].tc;
1267         inq = q->priv->rxq_map[q->queue_id].inq;
1268         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
1269         num = tc_params->inqs_params[inq].size;
1270         for (i = 0; i < num; i++) {
1271                 struct pp2_buff_inf inf;
1272                 uint64_t addr;
1273
1274                 pp2_bpool_get_buff(hifs[rte_lcore_id()], q->priv->bpool, &inf);
1275                 addr = cookie_addr_high | inf.cookie;
1276                 rte_pktmbuf_free((struct rte_mbuf *)addr);
1277         }
1278
1279         rte_free(q);
1280 }
1281
1282 /**
1283  * DPDK callback to configure the transmit queue.
1284  *
1285  * @param dev
1286  *   Pointer to Ethernet device structure.
1287  * @param idx
1288  *   Transmit queue index.
1289  * @param desc
1290  *   Number of descriptors to configure in the queue.
1291  * @param socket
1292  *   NUMA socket on which memory must be allocated.
1293  * @param conf
1294  *   Thresholds parameters (unused).
1295  *
1296  * @return
1297  *   0 on success, negative error value otherwise.
1298  */
1299 static int
1300 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1301                     unsigned int socket,
1302                     const struct rte_eth_txconf *conf __rte_unused)
1303 {
1304         struct mrvl_priv *priv = dev->data->dev_private;
1305         struct mrvl_txq *txq;
1306
1307         if (dev->data->tx_queues[idx]) {
1308                 rte_free(dev->data->tx_queues[idx]);
1309                 dev->data->tx_queues[idx] = NULL;
1310         }
1311
1312         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
1313         if (!txq)
1314                 return -ENOMEM;
1315
1316         txq->priv = priv;
1317         txq->queue_id = idx;
1318         txq->port_id = dev->data->port_id;
1319         dev->data->tx_queues[idx] = txq;
1320
1321         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
1322         priv->ppio_params.outqs_params.outqs_params[idx].weight = 1;
1323
1324         return 0;
1325 }
1326
1327 /**
1328  * DPDK callback to release the transmit queue.
1329  *
1330  * @param txq
1331  *   Generic transmit queue pointer.
1332  */
1333 static void
1334 mrvl_tx_queue_release(void *txq)
1335 {
1336         struct mrvl_txq *q = txq;
1337
1338         if (!q)
1339                 return;
1340
1341         rte_free(q);
1342 }
1343
1344 /**
1345  * Update RSS hash configuration
1346  *
1347  * @param dev
1348  *   Pointer to Ethernet device structure.
1349  * @param rss_conf
1350  *   Pointer to RSS configuration.
1351  *
1352  * @return
1353  *   0 on success, negative error value otherwise.
1354  */
1355 static int
1356 mrvl_rss_hash_update(struct rte_eth_dev *dev,
1357                      struct rte_eth_rss_conf *rss_conf)
1358 {
1359         struct mrvl_priv *priv = dev->data->dev_private;
1360
1361         return mrvl_configure_rss(priv, rss_conf);
1362 }
1363
1364 /**
1365  * DPDK callback to get RSS hash configuration.
1366  *
1367  * @param dev
1368  *   Pointer to Ethernet device structure.
1369  * @rss_conf
1370  *   Pointer to RSS configuration.
1371  *
1372  * @return
1373  *   Always 0.
1374  */
1375 static int
1376 mrvl_rss_hash_conf_get(struct rte_eth_dev *dev,
1377                        struct rte_eth_rss_conf *rss_conf)
1378 {
1379         struct mrvl_priv *priv = dev->data->dev_private;
1380         enum pp2_ppio_hash_type hash_type =
1381                 priv->ppio_params.inqs_params.hash_type;
1382
1383         rss_conf->rss_key = NULL;
1384
1385         if (hash_type == PP2_PPIO_HASH_T_NONE)
1386                 rss_conf->rss_hf = 0;
1387         else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE)
1388                 rss_conf->rss_hf = ETH_RSS_IPV4;
1389         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp)
1390                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP;
1391         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp)
1392                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP;
1393
1394         return 0;
1395 }
1396
1397 static const struct eth_dev_ops mrvl_ops = {
1398         .dev_configure = mrvl_dev_configure,
1399         .dev_start = mrvl_dev_start,
1400         .dev_stop = mrvl_dev_stop,
1401         .dev_set_link_up = mrvl_dev_set_link_up,
1402         .dev_set_link_down = mrvl_dev_set_link_down,
1403         .dev_close = mrvl_dev_close,
1404         .link_update = mrvl_link_update,
1405         .promiscuous_enable = mrvl_promiscuous_enable,
1406         .allmulticast_enable = mrvl_allmulticast_enable,
1407         .promiscuous_disable = mrvl_promiscuous_disable,
1408         .allmulticast_disable = mrvl_allmulticast_disable,
1409         .mac_addr_remove = mrvl_mac_addr_remove,
1410         .mac_addr_add = mrvl_mac_addr_add,
1411         .mac_addr_set = mrvl_mac_addr_set,
1412         .mtu_set = mrvl_mtu_set,
1413         .stats_get = mrvl_stats_get,
1414         .stats_reset = mrvl_stats_reset,
1415         .dev_infos_get = mrvl_dev_infos_get,
1416         .dev_supported_ptypes_get = mrvl_dev_supported_ptypes_get,
1417         .rxq_info_get = mrvl_rxq_info_get,
1418         .txq_info_get = mrvl_txq_info_get,
1419         .vlan_filter_set = mrvl_vlan_filter_set,
1420         .rx_queue_setup = mrvl_rx_queue_setup,
1421         .rx_queue_release = mrvl_rx_queue_release,
1422         .tx_queue_setup = mrvl_tx_queue_setup,
1423         .tx_queue_release = mrvl_tx_queue_release,
1424         .rss_hash_update = mrvl_rss_hash_update,
1425         .rss_hash_conf_get = mrvl_rss_hash_conf_get,
1426 };
1427
1428 /**
1429  * Return packet type information and l3/l4 offsets.
1430  *
1431  * @param desc
1432  *   Pointer to the received packet descriptor.
1433  * @param l3_offset
1434  *   l3 packet offset.
1435  * @param l4_offset
1436  *   l4 packet offset.
1437  *
1438  * @return
1439  *   Packet type information.
1440  */
1441 static inline uint64_t
1442 mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
1443                                     uint8_t *l3_offset, uint8_t *l4_offset)
1444 {
1445         enum pp2_inq_l3_type l3_type;
1446         enum pp2_inq_l4_type l4_type;
1447         uint64_t packet_type;
1448
1449         pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
1450         pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
1451
1452         packet_type = RTE_PTYPE_L2_ETHER;
1453
1454         switch (l3_type) {
1455         case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
1456                 packet_type |= RTE_PTYPE_L3_IPV4;
1457                 break;
1458         case PP2_INQ_L3_TYPE_IPV4_OK:
1459                 packet_type |= RTE_PTYPE_L3_IPV4_EXT;
1460                 break;
1461         case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO:
1462                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
1463                 break;
1464         case PP2_INQ_L3_TYPE_IPV6_NO_EXT:
1465                 packet_type |= RTE_PTYPE_L3_IPV6;
1466                 break;
1467         case PP2_INQ_L3_TYPE_IPV6_EXT:
1468                 packet_type |= RTE_PTYPE_L3_IPV6_EXT;
1469                 break;
1470         case PP2_INQ_L3_TYPE_ARP:
1471                 packet_type |= RTE_PTYPE_L2_ETHER_ARP;
1472                 /*
1473                  * In case of ARP l4_offset is set to wrong value.
1474                  * Set it to proper one so that later on mbuf->l3_len can be
1475                  * calculated subtracting l4_offset and l3_offset.
1476                  */
1477                 *l4_offset = *l3_offset + MRVL_ARP_LENGTH;
1478                 break;
1479         default:
1480                 RTE_LOG(DEBUG, PMD, "Failed to recognise l3 packet type\n");
1481                 break;
1482         }
1483
1484         switch (l4_type) {
1485         case PP2_INQ_L4_TYPE_TCP:
1486                 packet_type |= RTE_PTYPE_L4_TCP;
1487                 break;
1488         case PP2_INQ_L4_TYPE_UDP:
1489                 packet_type |= RTE_PTYPE_L4_UDP;
1490                 break;
1491         default:
1492                 RTE_LOG(DEBUG, PMD, "Failed to recognise l4 packet type\n");
1493                 break;
1494         }
1495
1496         return packet_type;
1497 }
1498
1499 /**
1500  * Get offload information from the received packet descriptor.
1501  *
1502  * @param desc
1503  *   Pointer to the received packet descriptor.
1504  *
1505  * @return
1506  *   Mbuf offload flags.
1507  */
1508 static inline uint64_t
1509 mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc)
1510 {
1511         uint64_t flags;
1512         enum pp2_inq_desc_status status;
1513
1514         status = pp2_ppio_inq_desc_get_l3_pkt_error(desc);
1515         if (unlikely(status != PP2_DESC_ERR_OK))
1516                 flags = PKT_RX_IP_CKSUM_BAD;
1517         else
1518                 flags = PKT_RX_IP_CKSUM_GOOD;
1519
1520         status = pp2_ppio_inq_desc_get_l4_pkt_error(desc);
1521         if (unlikely(status != PP2_DESC_ERR_OK))
1522                 flags |= PKT_RX_L4_CKSUM_BAD;
1523         else
1524                 flags |= PKT_RX_L4_CKSUM_GOOD;
1525
1526         return flags;
1527 }
1528
1529 /**
1530  * DPDK callback for receive.
1531  *
1532  * @param rxq
1533  *   Generic pointer to the receive queue.
1534  * @param rx_pkts
1535  *   Array to store received packets.
1536  * @param nb_pkts
1537  *   Maximum number of packets in array.
1538  *
1539  * @return
1540  *   Number of packets successfully received.
1541  */
1542 static uint16_t
1543 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
1544 {
1545         struct mrvl_rxq *q = rxq;
1546         struct pp2_ppio_desc descs[nb_pkts];
1547         struct pp2_bpool *bpool;
1548         int i, ret, rx_done = 0;
1549         int num;
1550         unsigned int core_id = rte_lcore_id();
1551
1552         if (unlikely(!q->priv->ppio))
1553                 return 0;
1554
1555         bpool = q->priv->bpool;
1556
1557         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
1558                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
1559         if (unlikely(ret < 0)) {
1560                 RTE_LOG(ERR, PMD, "Failed to receive packets\n");
1561                 return 0;
1562         }
1563         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
1564
1565         for (i = 0; i < nb_pkts; i++) {
1566                 struct rte_mbuf *mbuf;
1567                 uint8_t l3_offset, l4_offset;
1568                 enum pp2_inq_desc_status status;
1569                 uint64_t addr;
1570
1571                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
1572                         struct pp2_ppio_desc *pref_desc;
1573                         u64 pref_addr;
1574
1575                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
1576                         pref_addr = cookie_addr_high |
1577                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
1578                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
1579                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
1580                 }
1581
1582                 addr = cookie_addr_high |
1583                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
1584                 mbuf = (struct rte_mbuf *)addr;
1585                 rte_pktmbuf_reset(mbuf);
1586
1587                 /* drop packet in case of mac, overrun or resource error */
1588                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
1589                 if (unlikely(status != PP2_DESC_ERR_OK)) {
1590                         struct pp2_buff_inf binf = {
1591                                 .addr = rte_mbuf_data_dma_addr_default(mbuf),
1592                                 .cookie = (pp2_cookie_t)(uint64_t)mbuf,
1593                         };
1594
1595                         pp2_bpool_put_buff(hifs[core_id], bpool, &binf);
1596                         mrvl_port_bpool_size
1597                                 [bpool->pp2_id][bpool->id][core_id]++;
1598                         q->drop_mac++;
1599                         continue;
1600                 }
1601
1602                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
1603                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
1604                 mbuf->data_len = mbuf->pkt_len;
1605                 mbuf->port = q->port_id;
1606                 mbuf->packet_type =
1607                         mrvl_desc_to_packet_type_and_offset(&descs[i],
1608                                                             &l3_offset,
1609                                                             &l4_offset);
1610                 mbuf->l2_len = l3_offset;
1611                 mbuf->l3_len = l4_offset - l3_offset;
1612
1613                 if (likely(q->cksum_enabled))
1614                         mbuf->ol_flags = mrvl_desc_to_ol_flags(&descs[i]);
1615
1616                 rx_pkts[rx_done++] = mbuf;
1617                 q->bytes_recv += mbuf->pkt_len;
1618         }
1619
1620         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
1621                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
1622
1623                 if (unlikely(num <= q->priv->bpool_min_size ||
1624                              (!rx_done && num < q->priv->bpool_init_size))) {
1625                         ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE);
1626                         if (ret)
1627                                 RTE_LOG(ERR, PMD, "Failed to fill bpool\n");
1628                 } else if (unlikely(num > q->priv->bpool_max_size)) {
1629                         int i;
1630                         int pkt_to_remove = num - q->priv->bpool_init_size;
1631                         struct rte_mbuf *mbuf;
1632                         struct pp2_buff_inf buff;
1633
1634                         RTE_LOG(DEBUG, PMD,
1635                                 "\nport-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)\n",
1636                                 bpool->pp2_id, q->priv->ppio->port_id,
1637                                 bpool->id, pkt_to_remove, num,
1638                                 q->priv->bpool_init_size);
1639
1640                         for (i = 0; i < pkt_to_remove; i++) {
1641                                 pp2_bpool_get_buff(hifs[core_id], bpool, &buff);
1642                                 mbuf = (struct rte_mbuf *)
1643                                         (cookie_addr_high | buff.cookie);
1644                                 rte_pktmbuf_free(mbuf);
1645                         }
1646                         mrvl_port_bpool_size
1647                                 [bpool->pp2_id][bpool->id][core_id] -=
1648                                                                 pkt_to_remove;
1649                 }
1650                 rte_spinlock_unlock(&q->priv->lock);
1651         }
1652
1653         return rx_done;
1654 }
1655
1656 /**
1657  * Prepare offload information.
1658  *
1659  * @param ol_flags
1660  *   Offload flags.
1661  * @param packet_type
1662  *   Packet type bitfield.
1663  * @param l3_type
1664  *   Pointer to the pp2_ouq_l3_type structure.
1665  * @param l4_type
1666  *   Pointer to the pp2_outq_l4_type structure.
1667  * @param gen_l3_cksum
1668  *   Will be set to 1 in case l3 checksum is computed.
1669  * @param l4_cksum
1670  *   Will be set to 1 in case l4 checksum is computed.
1671  *
1672  * @return
1673  *   0 on success, negative error value otherwise.
1674  */
1675 static inline int
1676 mrvl_prepare_proto_info(uint64_t ol_flags, uint32_t packet_type,
1677                         enum pp2_outq_l3_type *l3_type,
1678                         enum pp2_outq_l4_type *l4_type,
1679                         int *gen_l3_cksum,
1680                         int *gen_l4_cksum)
1681 {
1682         /*
1683          * Based on ol_flags prepare information
1684          * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor
1685          * for offloading.
1686          */
1687         if (ol_flags & PKT_TX_IPV4) {
1688                 *l3_type = PP2_OUTQ_L3_TYPE_IPV4;
1689                 *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0;
1690         } else if (ol_flags & PKT_TX_IPV6) {
1691                 *l3_type = PP2_OUTQ_L3_TYPE_IPV6;
1692                 /* no checksum for ipv6 header */
1693                 *gen_l3_cksum = 0;
1694         } else {
1695                 /* if something different then stop processing */
1696                 return -1;
1697         }
1698
1699         ol_flags &= PKT_TX_L4_MASK;
1700         if ((packet_type & RTE_PTYPE_L4_TCP) &&
1701             ol_flags == PKT_TX_TCP_CKSUM) {
1702                 *l4_type = PP2_OUTQ_L4_TYPE_TCP;
1703                 *gen_l4_cksum = 1;
1704         } else if ((packet_type & RTE_PTYPE_L4_UDP) &&
1705                    ol_flags == PKT_TX_UDP_CKSUM) {
1706                 *l4_type = PP2_OUTQ_L4_TYPE_UDP;
1707                 *gen_l4_cksum = 1;
1708         } else {
1709                 *l4_type = PP2_OUTQ_L4_TYPE_OTHER;
1710                 /* no checksum for other type */
1711                 *gen_l4_cksum = 0;
1712         }
1713
1714         return 0;
1715 }
1716
1717 /**
1718  * Release already sent buffers to bpool (buffer-pool).
1719  *
1720  * @param ppio
1721  *   Pointer to the port structure.
1722  * @param hif
1723  *   Pointer to the MUSDK hardware interface.
1724  * @param sq
1725  *   Pointer to the shadow queue.
1726  * @param qid
1727  *   Queue id number.
1728  * @param force
1729  *   Force releasing packets.
1730  */
1731 static inline void
1732 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
1733                        struct mrvl_shadow_txq *sq, int qid, int force)
1734 {
1735         struct buff_release_entry *entry;
1736         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
1737         int i, core_id = rte_lcore_id();
1738
1739         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
1740
1741         sq->num_to_release += nb_done;
1742
1743         if (likely(!force &&
1744                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
1745                 return;
1746
1747         nb_done = sq->num_to_release;
1748         sq->num_to_release = 0;
1749
1750         for (i = 0; i < nb_done; i++) {
1751                 entry = &sq->ent[sq->tail + num];
1752                 if (unlikely(!entry->buff.addr)) {
1753                         RTE_LOG(ERR, PMD,
1754                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!\n",
1755                                 sq->tail, (u64)entry->buff.cookie,
1756                                 (u64)entry->buff.addr);
1757                         skip_bufs = 1;
1758                         goto skip;
1759                 }
1760
1761                 if (unlikely(!entry->bpool)) {
1762                         struct rte_mbuf *mbuf;
1763
1764                         mbuf = (struct rte_mbuf *)
1765                                (cookie_addr_high | entry->buff.cookie);
1766                         rte_pktmbuf_free(mbuf);
1767                         skip_bufs = 1;
1768                         goto skip;
1769                 }
1770
1771                 mrvl_port_bpool_size
1772                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
1773                 num++;
1774                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
1775                         goto skip;
1776                 continue;
1777 skip:
1778                 if (likely(num))
1779                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
1780                 num += skip_bufs;
1781                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
1782                 sq->size -= num;
1783                 num = 0;
1784         }
1785
1786         if (likely(num)) {
1787                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
1788                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
1789                 sq->size -= num;
1790         }
1791 }
1792
1793 /**
1794  * DPDK callback for transmit.
1795  *
1796  * @param txq
1797  *   Generic pointer transmit queue.
1798  * @param tx_pkts
1799  *   Packets to transmit.
1800  * @param nb_pkts
1801  *   Number of packets in array.
1802  *
1803  * @return
1804  *   Number of packets successfully transmitted.
1805  */
1806 static uint16_t
1807 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1808 {
1809         struct mrvl_txq *q = txq;
1810         struct mrvl_shadow_txq *sq = &shadow_txqs[q->port_id][rte_lcore_id()];
1811         struct pp2_hif *hif = hifs[rte_lcore_id()];
1812         struct pp2_ppio_desc descs[nb_pkts];
1813         int i, ret, bytes_sent = 0;
1814         uint16_t num, sq_free_size;
1815         uint64_t addr;
1816
1817         if (unlikely(!q->priv->ppio))
1818                 return 0;
1819
1820         if (sq->size)
1821                 mrvl_free_sent_buffers(q->priv->ppio, hif, sq, q->queue_id, 0);
1822
1823         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
1824         if (unlikely(nb_pkts > sq_free_size)) {
1825                 RTE_LOG(DEBUG, PMD,
1826                         "No room in shadow queue for %d packets! %d packets will be sent.\n",
1827                         nb_pkts, sq_free_size);
1828                 nb_pkts = sq_free_size;
1829         }
1830
1831         for (i = 0; i < nb_pkts; i++) {
1832                 struct rte_mbuf *mbuf = tx_pkts[i];
1833                 int gen_l3_cksum, gen_l4_cksum;
1834                 enum pp2_outq_l3_type l3_type;
1835                 enum pp2_outq_l4_type l4_type;
1836
1837                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
1838                         struct rte_mbuf *pref_pkt_hdr;
1839
1840                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
1841                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
1842                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
1843                 }
1844
1845                 sq->ent[sq->head].buff.cookie = (pp2_cookie_t)(uint64_t)mbuf;
1846                 sq->ent[sq->head].buff.addr =
1847                         rte_mbuf_data_dma_addr_default(mbuf);
1848                 sq->ent[sq->head].bpool =
1849                         (unlikely(mbuf->port == 0xff || mbuf->refcnt > 1)) ?
1850                          NULL : mrvl_port_to_bpool_lookup[mbuf->port];
1851                 sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
1852                 sq->size++;
1853
1854                 pp2_ppio_outq_desc_reset(&descs[i]);
1855                 pp2_ppio_outq_desc_set_phys_addr(&descs[i],
1856                                                  rte_pktmbuf_mtophys(mbuf));
1857                 pp2_ppio_outq_desc_set_pkt_offset(&descs[i], 0);
1858                 pp2_ppio_outq_desc_set_pkt_len(&descs[i],
1859                                                rte_pktmbuf_pkt_len(mbuf));
1860
1861                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
1862                 /*
1863                  * in case unsupported ol_flags were passed
1864                  * do not update descriptor offload information
1865                  */
1866                 ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type,
1867                                               &l3_type, &l4_type, &gen_l3_cksum,
1868                                               &gen_l4_cksum);
1869                 if (unlikely(ret))
1870                         continue;
1871
1872                 pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type,
1873                                                   mbuf->l2_len,
1874                                                   mbuf->l2_len + mbuf->l3_len,
1875                                                   gen_l3_cksum, gen_l4_cksum);
1876         }
1877
1878         num = nb_pkts;
1879         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
1880         /* number of packets that were not sent */
1881         if (unlikely(num > nb_pkts)) {
1882                 for (i = nb_pkts; i < num; i++) {
1883                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
1884                                 MRVL_PP2_TX_SHADOWQ_MASK;
1885                         addr = cookie_addr_high | sq->ent[sq->head].buff.cookie;
1886                         bytes_sent -=
1887                                 rte_pktmbuf_pkt_len((struct rte_mbuf *)addr);
1888                 }
1889                 sq->size -= num - nb_pkts;
1890         }
1891
1892         q->bytes_sent += bytes_sent;
1893
1894         return nb_pkts;
1895 }
1896
1897 /**
1898  * Initialize packet processor.
1899  *
1900  * @return
1901  *   0 on success, negative error value otherwise.
1902  */
1903 static int
1904 mrvl_init_pp2(void)
1905 {
1906         struct pp2_init_params init_params;
1907
1908         memset(&init_params, 0, sizeof(init_params));
1909         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
1910         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
1911         init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED;
1912
1913         return pp2_init(&init_params);
1914 }
1915
1916 /**
1917  * Deinitialize packet processor.
1918  *
1919  * @return
1920  *   0 on success, negative error value otherwise.
1921  */
1922 static void
1923 mrvl_deinit_pp2(void)
1924 {
1925         pp2_deinit();
1926 }
1927
1928 /**
1929  * Create private device structure.
1930  *
1931  * @param dev_name
1932  *   Pointer to the port name passed in the initialization parameters.
1933  *
1934  * @return
1935  *   Pointer to the newly allocated private device structure.
1936  */
1937 static struct mrvl_priv *
1938 mrvl_priv_create(const char *dev_name)
1939 {
1940         struct pp2_bpool_params bpool_params;
1941         char match[MRVL_MATCH_LEN];
1942         struct mrvl_priv *priv;
1943         int ret, bpool_bit;
1944
1945         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
1946         if (!priv)
1947                 return NULL;
1948
1949         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
1950                                        &priv->pp_id, &priv->ppio_id);
1951         if (ret)
1952                 goto out_free_priv;
1953
1954         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
1955                                      PP2_BPOOL_NUM_POOLS);
1956         if (bpool_bit < 0)
1957                 goto out_free_priv;
1958         priv->bpool_bit = bpool_bit;
1959
1960         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
1961                  priv->bpool_bit);
1962         memset(&bpool_params, 0, sizeof(bpool_params));
1963         bpool_params.match = match;
1964         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
1965         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
1966         if (ret)
1967                 goto out_clear_bpool_bit;
1968
1969         priv->ppio_params.type = PP2_PPIO_T_NIC;
1970         rte_spinlock_init(&priv->lock);
1971
1972         return priv;
1973 out_clear_bpool_bit:
1974         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
1975 out_free_priv:
1976         rte_free(priv);
1977         return NULL;
1978 }
1979
1980 /**
1981  * Create device representing Ethernet port.
1982  *
1983  * @param name
1984  *   Pointer to the port's name.
1985  *
1986  * @return
1987  *   0 on success, negative error value otherwise.
1988  */
1989 static int
1990 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
1991 {
1992         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
1993         struct rte_eth_dev *eth_dev;
1994         struct mrvl_priv *priv;
1995         struct ifreq req;
1996
1997         eth_dev = rte_eth_dev_allocate(name);
1998         if (!eth_dev)
1999                 return -ENOMEM;
2000
2001         priv = mrvl_priv_create(name);
2002         if (!priv) {
2003                 ret = -ENOMEM;
2004                 goto out_free_dev;
2005         }
2006
2007         eth_dev->data->mac_addrs =
2008                 rte_zmalloc("mac_addrs",
2009                             ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
2010         if (!eth_dev->data->mac_addrs) {
2011                 RTE_LOG(ERR, PMD, "Failed to allocate space for eth addrs\n");
2012                 ret = -ENOMEM;
2013                 goto out_free_priv;
2014         }
2015
2016         memset(&req, 0, sizeof(req));
2017         strcpy(req.ifr_name, name);
2018         ret = ioctl(fd, SIOCGIFHWADDR, &req);
2019         if (ret)
2020                 goto out_free_mac;
2021
2022         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
2023                req.ifr_addr.sa_data, ETHER_ADDR_LEN);
2024
2025         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
2026         eth_dev->tx_pkt_burst = mrvl_tx_pkt_burst;
2027         eth_dev->data->dev_private = priv;
2028         eth_dev->device = &vdev->device;
2029         eth_dev->dev_ops = &mrvl_ops;
2030
2031         return 0;
2032 out_free_mac:
2033         rte_free(eth_dev->data->mac_addrs);
2034 out_free_dev:
2035         rte_eth_dev_release_port(eth_dev);
2036 out_free_priv:
2037         rte_free(priv);
2038
2039         return ret;
2040 }
2041
2042 /**
2043  * Cleanup previously created device representing Ethernet port.
2044  *
2045  * @param name
2046  *   Pointer to the port name.
2047  */
2048 static void
2049 mrvl_eth_dev_destroy(const char *name)
2050 {
2051         struct rte_eth_dev *eth_dev;
2052         struct mrvl_priv *priv;
2053
2054         eth_dev = rte_eth_dev_allocated(name);
2055         if (!eth_dev)
2056                 return;
2057
2058         priv = eth_dev->data->dev_private;
2059         pp2_bpool_deinit(priv->bpool);
2060         rte_free(priv);
2061         rte_free(eth_dev->data->mac_addrs);
2062         rte_eth_dev_release_port(eth_dev);
2063 }
2064
2065 /**
2066  * Callback used by rte_kvargs_process() during argument parsing.
2067  *
2068  * @param key
2069  *   Pointer to the parsed key (unused).
2070  * @param value
2071  *   Pointer to the parsed value.
2072  * @param extra_args
2073  *   Pointer to the extra arguments which contains address of the
2074  *   table of pointers to parsed interface names.
2075  *
2076  * @return
2077  *   Always 0.
2078  */
2079 static int
2080 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
2081                  void *extra_args)
2082 {
2083         const char **ifnames = extra_args;
2084
2085         ifnames[mrvl_ports_nb++] = value;
2086
2087         return 0;
2088 }
2089
2090 /**
2091  * Initialize per-lcore MUSDK hardware interfaces (hifs).
2092  *
2093  * @return
2094  *   0 on success, negative error value otherwise.
2095  */
2096 static int
2097 mrvl_init_hifs(void)
2098 {
2099         struct pp2_hif_params params;
2100         char match[MRVL_MATCH_LEN];
2101         int i, ret;
2102
2103         RTE_LCORE_FOREACH(i) {
2104                 ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
2105                 if (ret < 0)
2106                         return ret;
2107
2108                 snprintf(match, sizeof(match), "hif-%d", ret);
2109                 memset(&params, 0, sizeof(params));
2110                 params.match = match;
2111                 params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
2112                 ret = pp2_hif_init(&params, &hifs[i]);
2113                 if (ret) {
2114                         RTE_LOG(ERR, PMD, "Failed to initialize hif %d\n", i);
2115                         return ret;
2116                 }
2117         }
2118
2119         return 0;
2120 }
2121
2122 /**
2123  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
2124  */
2125 static void
2126 mrvl_deinit_hifs(void)
2127 {
2128         int i;
2129
2130         RTE_LCORE_FOREACH(i) {
2131                 if (hifs[i])
2132                         pp2_hif_deinit(hifs[i]);
2133         }
2134 }
2135
2136 static void mrvl_set_first_last_cores(int core_id)
2137 {
2138         if (core_id < mrvl_lcore_first)
2139                 mrvl_lcore_first = core_id;
2140
2141         if (core_id > mrvl_lcore_last)
2142                 mrvl_lcore_last = core_id;
2143 }
2144
2145 /**
2146  * DPDK callback to register the virtual device.
2147  *
2148  * @param vdev
2149  *   Pointer to the virtual device.
2150  *
2151  * @return
2152  *   0 on success, negative error value otherwise.
2153  */
2154 static int
2155 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
2156 {
2157         struct rte_kvargs *kvlist;
2158         const char *ifnames[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
2159         int ret = -EINVAL;
2160         uint32_t i, ifnum, cfgnum, core_id;
2161         const char *params;
2162
2163         params = rte_vdev_device_args(vdev);
2164         if (!params)
2165                 return -EINVAL;
2166
2167         kvlist = rte_kvargs_parse(params, valid_args);
2168         if (!kvlist)
2169                 return -EINVAL;
2170
2171         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
2172         if (ifnum > RTE_DIM(ifnames))
2173                 goto out_free_kvlist;
2174
2175         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
2176                            mrvl_get_ifnames, &ifnames);
2177
2178         cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
2179         if (cfgnum > 1) {
2180                 RTE_LOG(ERR, PMD, "Cannot handle more than one config file!\n");
2181                 goto out_free_kvlist;
2182         } else if (cfgnum == 1) {
2183                 rte_kvargs_process(kvlist, MRVL_CFG_ARG,
2184                                    mrvl_get_qoscfg, &mrvl_qos_cfg);
2185         }
2186
2187         /*
2188          * ret == -EEXIST is correct, it means DMA
2189          * has been already initialized (by another PMD).
2190          */
2191         ret = mv_sys_dma_mem_init(RTE_MRVL_MUSDK_DMA_MEMSIZE);
2192         if (ret < 0 && ret != -EEXIST)
2193                 goto out_free_kvlist;
2194
2195         ret = mrvl_init_pp2();
2196         if (ret) {
2197                 RTE_LOG(ERR, PMD, "Failed to init PP!\n");
2198                 goto out_deinit_dma;
2199         }
2200
2201         ret = mrvl_init_hifs();
2202         if (ret)
2203                 goto out_deinit_hifs;
2204
2205         for (i = 0; i < ifnum; i++) {
2206                 RTE_LOG(INFO, PMD, "Creating %s\n", ifnames[i]);
2207                 ret = mrvl_eth_dev_create(vdev, ifnames[i]);
2208                 if (ret)
2209                         goto out_cleanup;
2210         }
2211
2212         rte_kvargs_free(kvlist);
2213
2214         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
2215
2216         mrvl_lcore_first = RTE_MAX_LCORE;
2217         mrvl_lcore_last = 0;
2218
2219         RTE_LCORE_FOREACH(core_id) {
2220                 mrvl_set_first_last_cores(core_id);
2221         }
2222
2223         return 0;
2224 out_cleanup:
2225         for (; i > 0; i--)
2226                 mrvl_eth_dev_destroy(ifnames[i]);
2227 out_deinit_hifs:
2228         mrvl_deinit_hifs();
2229         mrvl_deinit_pp2();
2230 out_deinit_dma:
2231         mv_sys_dma_mem_destroy();
2232 out_free_kvlist:
2233         rte_kvargs_free(kvlist);
2234
2235         return ret;
2236 }
2237
2238 /**
2239  * DPDK callback to remove virtual device.
2240  *
2241  * @param vdev
2242  *   Pointer to the removed virtual device.
2243  *
2244  * @return
2245  *   0 on success, negative error value otherwise.
2246  */
2247 static int
2248 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
2249 {
2250         int i;
2251         const char *name;
2252
2253         name = rte_vdev_device_name(vdev);
2254         if (!name)
2255                 return -EINVAL;
2256
2257         RTE_LOG(INFO, PMD, "Removing %s\n", name);
2258
2259         for (i = 0; i < rte_eth_dev_count(); i++) {
2260                 char ifname[RTE_ETH_NAME_MAX_LEN];
2261
2262                 rte_eth_dev_get_name_by_port(i, ifname);
2263                 mrvl_eth_dev_destroy(ifname);
2264         }
2265
2266         mrvl_deinit_hifs();
2267         mrvl_deinit_pp2();
2268         mv_sys_dma_mem_destroy();
2269
2270         return 0;
2271 }
2272
2273 static struct rte_vdev_driver pmd_mrvl_drv = {
2274         .probe = rte_pmd_mrvl_probe,
2275         .remove = rte_pmd_mrvl_remove,
2276 };
2277
2278 RTE_PMD_REGISTER_VDEV(net_mrvl, pmd_mrvl_drv);
2279 RTE_PMD_REGISTER_ALIAS(net_mrvl, eth_mrvl);