drivers/net/mrvl/mrvl_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Semihalf. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Semihalf nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <rte_ethdev.h>
  34 #include <rte_kvargs.h>
  35 #include <rte_log.h>
  36 #include <rte_malloc.h>
  37 #include <rte_vdev.h>
  38
  39 /* Unluckily, container_of is defined by both DPDK and MUSDK,
  40  * we'll declare only one version.
  41  *
  42  * Note that it is not used in this PMD anyway.
  43  */
  44 #ifdef container_of
  45 #undef container_of
  46 #endif
  47
  48 #include <drivers/mv_pp2.h>
  49 #include <drivers/mv_pp2_bpool.h>
  50 #include <drivers/mv_pp2_hif.h>
  51
  52 #include <fcntl.h>
  53 #include <linux/ethtool.h>
  54 #include <linux/sockios.h>
  55 #include <net/if.h>
  56 #include <net/if_arp.h>
  57 #include <sys/ioctl.h>
  58 #include <sys/socket.h>
  59 #include <sys/stat.h>
  60 #include <sys/types.h>
  61
  62 #include "mrvl_ethdev.h"
  63 #include "mrvl_qos.h"
  64
  65 /* bitmask with reserved hifs */
  66 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
  67 /* bitmask with reserved bpools */
  68 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
  69 /* bitmask with reserved kernel RSS tables */
  70 #define MRVL_MUSDK_RSS_RESERVED 0x01
  71 /* maximum number of available hifs */
  72 #define MRVL_MUSDK_HIFS_MAX 9
  73
  74 /* prefetch shift */
  75 #define MRVL_MUSDK_PREFETCH_SHIFT 2
  76
  77 /* TCAM has 25 entries reserved for uc/mc filter entries */
  78 #define MRVL_MAC_ADDRS_MAX 25
  79 #define MRVL_MATCH_LEN 16
  80 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
  81 /* Maximum allowable packet size */
  82 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
  83
  84 #define MRVL_IFACE_NAME_ARG "iface"
  85 #define MRVL_CFG_ARG "cfg"
  86
  87 #define MRVL_BURST_SIZE 64
  88
  89 #define MRVL_ARP_LENGTH 28
  90
  91 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
  92
  93 #define MRVL_COOKIE_HIGH_ADDR_SHIFT     (sizeof(pp2_cookie_t) * 8)
  94 #define MRVL_COOKIE_HIGH_ADDR_MASK      (~0ULL << MRVL_COOKIE_HIGH_ADDR_SHIFT)
  95
  96 static const char * const valid_args[] = {
  97         MRVL_IFACE_NAME_ARG,
  98         MRVL_CFG_ARG,
  99         NULL
 100 };
 101
 102 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
 103 static struct pp2_hif *hifs[RTE_MAX_LCORE];
 104 static int used_bpools[PP2_NUM_PKT_PROC] = {
 105         MRVL_MUSDK_BPOOLS_RESERVED,
 106         MRVL_MUSDK_BPOOLS_RESERVED
 107 };
 108
 109 struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
 110 int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
 111 uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
 112
 113 /*
 114  * To use buffer harvesting based on loopback port shadow queue structure
 115  * was introduced for buffers information bookkeeping.
 116  *
 117  * Before sending the packet, related buffer information (pp2_buff_inf) is
 118  * stored in shadow queue. After packet is transmitted no longer used
 119  * packet buffer is released back to it's original hardware pool,
 120  * on condition it originated from interface.
 121  * In case it  was generated by application itself i.e: mbuf->port field is
 122  * 0xff then its released to software mempool.
 123  */
 124 struct mrvl_shadow_txq {
 125         int head;           /* write index - used when sending buffers */
 126         int tail;           /* read index - used when releasing buffers */
 127         u16 size;           /* queue occupied size */
 128         u16 num_to_release; /* number of buffers sent, that can be released */
 129         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
 130 };
 131
 132 struct mrvl_rxq {
 133         struct mrvl_priv *priv;
 134         struct rte_mempool *mp;
 135         int queue_id;
 136         int port_id;
 137         int cksum_enabled;
 138 };
 139
 140 struct mrvl_txq {
 141         struct mrvl_priv *priv;
 142         int queue_id;
 143         int port_id;
 144 };
 145
 146 /*
 147  * Every tx queue should have dedicated shadow tx queue.
 148  *
 149  * Ports assigned by DPDK might not start at zero or be continuous so
 150  * as a workaround define shadow queues for each possible port so that
 151  * we eventually fit somewhere.
 152  */
 153 struct mrvl_shadow_txq shadow_txqs[RTE_MAX_ETHPORTS][RTE_MAX_LCORE];
 154
 155 /** Number of ports configured. */
 156 int mrvl_ports_nb;
 157 static int mrvl_lcore_first;
 158 static int mrvl_lcore_last;
 159
 160 static inline int
 161 mrvl_get_bpool_size(int pp2_id, int pool_id)
 162 {
 163         int i;
 164         int size = 0;
 165
 166         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
 167                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
 168
 169         return size;
 170 }
 171
 172 static inline int
 173 mrvl_reserve_bit(int *bitmap, int max)
 174 {
 175         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
 176
 177         if (n >= max)
 178                 return -1;
 179
 180         *bitmap |= 1 << n;
 181
 182         return n;
 183 }
 184
 185 /**
 186  * Configure rss based on dpdk rss configuration.
 187  *
 188  * @param priv
 189  *   Pointer to private structure.
 190  * @param rss_conf
 191  *   Pointer to RSS configuration.
 192  *
 193  * @return
 194  *   0 on success, negative error value otherwise.
 195  */
 196 static int
 197 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf)
 198 {
 199         if (rss_conf->rss_key)
 200                 RTE_LOG(WARNING, PMD, "Changing hash key is not supported\n");
 201
 202         if (rss_conf->rss_hf == 0) {
 203                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 204         } else if (rss_conf->rss_hf & ETH_RSS_IPV4) {
 205                 priv->ppio_params.inqs_params.hash_type =
 206                         PP2_PPIO_HASH_T_2_TUPLE;
 207         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
 208                 priv->ppio_params.inqs_params.hash_type =
 209                         PP2_PPIO_HASH_T_5_TUPLE;
 210                 priv->rss_hf_tcp = 1;
 211         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
 212                 priv->ppio_params.inqs_params.hash_type =
 213                         PP2_PPIO_HASH_T_5_TUPLE;
 214                 priv->rss_hf_tcp = 0;
 215         } else {
 216                 return -EINVAL;
 217         }
 218
 219         return 0;
 220 }
 221
 222 /**
 223  * Ethernet device configuration.
 224  *
 225  * Prepare the driver for a given number of TX and RX queues and
 226  * configure RSS.
 227  *
 228  * @param dev
 229  *   Pointer to Ethernet device structure.
 230  *
 231  * @return
 232  *   0 on success, negative error value otherwise.
 233  */
 234 static int
 235 mrvl_dev_configure(struct rte_eth_dev *dev)
 236 {
 237         struct mrvl_priv *priv = dev->data->dev_private;
 238         int ret;
 239
 240         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
 241             dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
 242                 RTE_LOG(INFO, PMD, "Unsupported rx multi queue mode %d\n",
 243                         dev->data->dev_conf.rxmode.mq_mode);
 244                 return -EINVAL;
 245         }
 246
 247         if (!dev->data->dev_conf.rxmode.hw_strip_crc) {
 248                 RTE_LOG(INFO, PMD,
 249                         "L2 CRC stripping is always enabled in hw\n");
 250                 dev->data->dev_conf.rxmode.hw_strip_crc = 1;
 251         }
 252
 253         if (dev->data->dev_conf.rxmode.hw_vlan_strip) {
 254                 RTE_LOG(INFO, PMD, "VLAN stripping not supported\n");
 255                 return -EINVAL;
 256         }
 257
 258         if (dev->data->dev_conf.rxmode.split_hdr_size) {
 259                 RTE_LOG(INFO, PMD, "Split headers not supported\n");
 260                 return -EINVAL;
 261         }
 262
 263         if (dev->data->dev_conf.rxmode.enable_scatter) {
 264                 RTE_LOG(INFO, PMD, "RX Scatter/Gather not supported\n");
 265                 return -EINVAL;
 266         }
 267
 268         if (dev->data->dev_conf.rxmode.enable_lro) {
 269                 RTE_LOG(INFO, PMD, "LRO not supported\n");
 270                 return -EINVAL;
 271         }
 272
 273         if (dev->data->dev_conf.rxmode.jumbo_frame)
 274                 dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
 275                                  ETHER_HDR_LEN - ETHER_CRC_LEN;
 276
 277         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
 278                                   dev->data->nb_rx_queues);
 279         if (ret < 0)
 280                 return ret;
 281
 282         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
 283         priv->nb_rx_queues = dev->data->nb_rx_queues;
 284
 285         if (dev->data->nb_rx_queues == 1 &&
 286             dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
 287                 RTE_LOG(WARNING, PMD, "Disabling hash for 1 rx queue\n");
 288                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 289
 290                 return 0;
 291         }
 292
 293         return mrvl_configure_rss(priv,
 294                                   &dev->data->dev_conf.rx_adv_conf.rss_conf);
 295 }
 296
 297 /**
 298  * DPDK callback to change the MTU.
 299  *
 300  * Setting the MTU affects hardware MRU (packets larger than the MRU
 301  * will be dropped).
 302  *
 303  * @param dev
 304  *   Pointer to Ethernet device structure.
 305  * @param mtu
 306  *   New MTU.
 307  *
 308  * @return
 309  *   0 on success, negative error value otherwise.
 310  */
 311 static int
 312 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 313 {
 314         struct mrvl_priv *priv = dev->data->dev_private;
 315         /* extra MV_MH_SIZE bytes are required for Marvell tag */
 316         uint16_t mru = mtu + MV_MH_SIZE + ETHER_HDR_LEN + ETHER_CRC_LEN;
 317         int ret;
 318
 319         if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX)
 320                 return -EINVAL;
 321
 322         ret = pp2_ppio_set_mru(priv->ppio, mru);
 323         if (ret)
 324                 return ret;
 325
 326         return pp2_ppio_set_mtu(priv->ppio, mtu);
 327 }
 328
 329 /**
 330  * DPDK callback to bring the link up.
 331  *
 332  * @param dev
 333  *   Pointer to Ethernet device structure.
 334  *
 335  * @return
 336  *   0 on success, negative error value otherwise.
 337  */
 338 static int
 339 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
 340 {
 341         struct mrvl_priv *priv = dev->data->dev_private;
 342         int ret;
 343
 344         ret = pp2_ppio_enable(priv->ppio);
 345         if (ret)
 346                 return ret;
 347
 348         /*
 349          * mtu/mru can be updated if pp2_ppio_enable() was called at least once
 350          * as pp2_ppio_enable() changes port->t_mode from default 0 to
 351          * PP2_TRAFFIC_INGRESS_EGRESS.
 352          *
 353          * Set mtu to default DPDK value here.
 354          */
 355         ret = mrvl_mtu_set(dev, dev->data->mtu);
 356         if (ret)
 357                 pp2_ppio_disable(priv->ppio);
 358
 359         dev->data->dev_link.link_status = ETH_LINK_UP;
 360
 361         return ret;
 362 }
 363
 364 /**
 365  * DPDK callback to bring the link down.
 366  *
 367  * @param dev
 368  *   Pointer to Ethernet device structure.
 369  *
 370  * @return
 371  *   0 on success, negative error value otherwise.
 372  */
 373 static int
 374 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
 375 {
 376         struct mrvl_priv *priv = dev->data->dev_private;
 377         int ret;
 378
 379         ret = pp2_ppio_disable(priv->ppio);
 380         if (ret)
 381                 return ret;
 382
 383         dev->data->dev_link.link_status = ETH_LINK_DOWN;
 384
 385         return ret;
 386 }
 387
 388 /**
 389  * DPDK callback to start the device.
 390  *
 391  * @param dev
 392  *   Pointer to Ethernet device structure.
 393  *
 394  * @return
 395  *   0 on success, negative errno value on failure.
 396  */
 397 static int
 398 mrvl_dev_start(struct rte_eth_dev *dev)
 399 {
 400         struct mrvl_priv *priv = dev->data->dev_private;
 401         char match[MRVL_MATCH_LEN];
 402         int ret;
 403
 404         snprintf(match, sizeof(match), "ppio-%d:%d",
 405                  priv->pp_id, priv->ppio_id);
 406         priv->ppio_params.match = match;
 407
 408         /*
 409          * Calculate the maximum bpool size for refill feature to 1.5 of the
 410          * configured size. In case the bpool size will exceed this value,
 411          * superfluous buffers will be removed
 412          */
 413         priv->bpool_max_size = priv->bpool_init_size +
 414                               (priv->bpool_init_size >> 1);
 415         /*
 416          * Calculate the minimum bpool size for refill feature as follows:
 417          * 2 default burst sizes multiply by number of rx queues.
 418          * If the bpool size will be below this value, new buffers will
 419          * be added to the pool.
 420          */
 421         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
 422
 423         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
 424         if (ret)
 425                 return ret;
 426
 427         /*
 428          * In case there are some some stale uc/mc mac addresses flush them
 429          * here. It cannot be done during mrvl_dev_close() as port information
 430          * is already gone at that point (due to pp2_ppio_deinit() in
 431          * mrvl_dev_stop()).
 432          */
 433         if (!priv->uc_mc_flushed) {
 434                 ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1);
 435                 if (ret) {
 436                         RTE_LOG(ERR, PMD,
 437                                 "Failed to flush uc/mc filter list\n");
 438                         goto out;
 439                 }
 440                 priv->uc_mc_flushed = 1;
 441         }
 442
 443         if (!priv->vlan_flushed) {
 444                 ret = pp2_ppio_flush_vlan(priv->ppio);
 445                 if (ret) {
 446                         RTE_LOG(ERR, PMD, "Failed to flush vlan list\n");
 447                         /*
 448                          * TODO
 449                          * once pp2_ppio_flush_vlan() is supported jump to out
 450                          * goto out;
 451                          */
 452                 }
 453                 priv->vlan_flushed = 1;
 454         }
 455
 456         /* For default QoS config, don't start classifier. */
 457         if (mrvl_qos_cfg) {
 458                 ret = mrvl_start_qos_mapping(priv);
 459                 if (ret) {
 460                         pp2_ppio_deinit(priv->ppio);
 461                         return ret;
 462                 }
 463         }
 464
 465         ret = mrvl_dev_set_link_up(dev);
 466         if (ret)
 467                 goto out;
 468
 469         return 0;
 470 out:
 471         pp2_ppio_deinit(priv->ppio);
 472         return ret;
 473 }
 474
 475 /**
 476  * Flush receive queues.
 477  *
 478  * @param dev
 479  *   Pointer to Ethernet device structure.
 480  */
 481 static void
 482 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
 483 {
 484         int i;
 485
 486         RTE_LOG(INFO, PMD, "Flushing rx queues\n");
 487         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 488                 int ret, num;
 489
 490                 do {
 491                         struct mrvl_rxq *q = dev->data->rx_queues[i];
 492                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
 493
 494                         num = MRVL_PP2_RXD_MAX;
 495                         ret = pp2_ppio_recv(q->priv->ppio,
 496                                             q->priv->rxq_map[q->queue_id].tc,
 497                                             q->priv->rxq_map[q->queue_id].inq,
 498                                             descs, (uint16_t *)&num);
 499                 } while (ret == 0 && num);
 500         }
 501 }
 502
 503 /**
 504  * Flush transmit shadow queues.
 505  *
 506  * @param dev
 507  *   Pointer to Ethernet device structure.
 508  */
 509 static void
 510 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
 511 {
 512         int i;
 513
 514         RTE_LOG(INFO, PMD, "Flushing tx shadow queues\n");
 515         for (i = 0; i < RTE_MAX_LCORE; i++) {
 516                 struct mrvl_shadow_txq *sq =
 517                         &shadow_txqs[dev->data->port_id][i];
 518
 519                 while (sq->tail != sq->head) {
 520                         uint64_t addr = cookie_addr_high |
 521                                         sq->ent[sq->tail].buff.cookie;
 522                         rte_pktmbuf_free((struct rte_mbuf *)addr);
 523                         sq->tail = (sq->tail + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
 524                 }
 525
 526                 memset(sq, 0, sizeof(*sq));
 527         }
 528 }
 529
 530 /**
 531  * Flush hardware bpool (buffer-pool).
 532  *
 533  * @param dev
 534  *   Pointer to Ethernet device structure.
 535  */
 536 static void
 537 mrvl_flush_bpool(struct rte_eth_dev *dev)
 538 {
 539         struct mrvl_priv *priv = dev->data->dev_private;
 540         uint32_t num;
 541         int ret;
 542
 543         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
 544         if (ret) {
 545                 RTE_LOG(ERR, PMD, "Failed to get bpool buffers number\n");
 546                 return;
 547         }
 548
 549         while (num--) {
 550                 struct pp2_buff_inf inf;
 551                 uint64_t addr;
 552
 553                 ret = pp2_bpool_get_buff(hifs[rte_lcore_id()], priv->bpool,
 554                                          &inf);
 555                 if (ret)
 556                         break;
 557
 558                 addr = cookie_addr_high | inf.cookie;
 559                 rte_pktmbuf_free((struct rte_mbuf *)addr);
 560         }
 561 }
 562
 563 /**
 564  * DPDK callback to stop the device.
 565  *
 566  * @param dev
 567  *   Pointer to Ethernet device structure.
 568  */
 569 static void
 570 mrvl_dev_stop(struct rte_eth_dev *dev)
 571 {
 572         struct mrvl_priv *priv = dev->data->dev_private;
 573
 574         mrvl_dev_set_link_down(dev);
 575         mrvl_flush_rx_queues(dev);
 576         mrvl_flush_tx_shadow_queues(dev);
 577         if (priv->qos_tbl)
 578                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
 579         pp2_ppio_deinit(priv->ppio);
 580         priv->ppio = NULL;
 581 }
 582
 583 /**
 584  * DPDK callback to close the device.
 585  *
 586  * @param dev
 587  *   Pointer to Ethernet device structure.
 588  */
 589 static void
 590 mrvl_dev_close(struct rte_eth_dev *dev)
 591 {
 592         struct mrvl_priv *priv = dev->data->dev_private;
 593         size_t i;
 594
 595         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
 596                 struct pp2_ppio_tc_params *tc_params =
 597                         &priv->ppio_params.inqs_params.tcs_params[i];
 598
 599                 if (tc_params->inqs_params) {
 600                         rte_free(tc_params->inqs_params);
 601                         tc_params->inqs_params = NULL;
 602                 }
 603         }
 604
 605         mrvl_flush_bpool(dev);
 606 }
 607
 608 /**
 609  * DPDK callback to retrieve physical link information.
 610  *
 611  * @param dev
 612  *   Pointer to Ethernet device structure.
 613  * @param wait_to_complete
 614  *   Wait for request completion (ignored).
 615  *
 616  * @return
 617  *   0 on success, negative error value otherwise.
 618  */
 619 static int
 620 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
 621 {
 622         /*
 623          * TODO
 624          * once MUSDK provides necessary API use it here
 625          */
 626         struct ethtool_cmd edata;
 627         struct ifreq req;
 628         int ret, fd;
 629
 630         edata.cmd = ETHTOOL_GSET;
 631
 632         strcpy(req.ifr_name, dev->data->name);
 633         req.ifr_data = (void *)&edata;
 634
 635         fd = socket(AF_INET, SOCK_DGRAM, 0);
 636         if (fd == -1)
 637                 return -EFAULT;
 638
 639         ret = ioctl(fd, SIOCETHTOOL, &req);
 640         if (ret == -1) {
 641                 close(fd);
 642                 return -EFAULT;
 643         }
 644
 645         close(fd);
 646
 647         switch (ethtool_cmd_speed(&edata)) {
 648         case SPEED_10:
 649                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
 650                 break;
 651         case SPEED_100:
 652                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
 653                 break;
 654         case SPEED_1000:
 655                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
 656                 break;
 657         case SPEED_10000:
 658                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
 659                 break;
 660         default:
 661                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
 662         }
 663
 664         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
 665                                                          ETH_LINK_HALF_DUPLEX;
 666         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
 667                                                            ETH_LINK_FIXED;
 668
 669         return 0;
 670 }
 671
 672 /**
 673  * DPDK callback to enable promiscuous mode.
 674  *
 675  * @param dev
 676  *   Pointer to Ethernet device structure.
 677  */
 678 static void
 679 mrvl_promiscuous_enable(struct rte_eth_dev *dev)
 680 {
 681         struct mrvl_priv *priv = dev->data->dev_private;
 682         int ret;
 683
 684         ret = pp2_ppio_set_uc_promisc(priv->ppio, 1);
 685         if (ret)
 686                 RTE_LOG(ERR, PMD, "Failed to enable promiscuous mode\n");
 687 }
 688
 689 /**
 690  * DPDK callback to enable allmulti mode.
 691  *
 692  * @param dev
 693  *   Pointer to Ethernet device structure.
 694  */
 695 static void
 696 mrvl_allmulticast_enable(struct rte_eth_dev *dev)
 697 {
 698         struct mrvl_priv *priv = dev->data->dev_private;
 699         int ret;
 700
 701         ret = pp2_ppio_set_mc_promisc(priv->ppio, 1);
 702         if (ret)
 703                 RTE_LOG(ERR, PMD, "Failed enable all-multicast mode\n");
 704 }
 705
 706 /**
 707  * DPDK callback to disable promiscuous mode.
 708  *
 709  * @param dev
 710  *   Pointer to Ethernet device structure.
 711  */
 712 static void
 713 mrvl_promiscuous_disable(struct rte_eth_dev *dev)
 714 {
 715         struct mrvl_priv *priv = dev->data->dev_private;
 716         int ret;
 717
 718         ret = pp2_ppio_set_uc_promisc(priv->ppio, 0);
 719         if (ret)
 720                 RTE_LOG(ERR, PMD, "Failed to disable promiscuous mode\n");
 721 }
 722
 723 /**
 724  * DPDK callback to disable allmulticast mode.
 725  *
 726  * @param dev
 727  *   Pointer to Ethernet device structure.
 728  */
 729 static void
 730 mrvl_allmulticast_disable(struct rte_eth_dev *dev)
 731 {
 732         struct mrvl_priv *priv = dev->data->dev_private;
 733         int ret;
 734
 735         ret = pp2_ppio_set_mc_promisc(priv->ppio, 0);
 736         if (ret)
 737                 RTE_LOG(ERR, PMD, "Failed to disable all-multicast mode\n");
 738 }
 739
 740 /**
 741  * DPDK callback to remove a MAC address.
 742  *
 743  * @param dev
 744  *   Pointer to Ethernet device structure.
 745  * @param index
 746  *   MAC address index.
 747  */
 748 static void
 749 mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
 750 {
 751         struct mrvl_priv *priv = dev->data->dev_private;
 752         char buf[ETHER_ADDR_FMT_SIZE];
 753         int ret;
 754
 755         ret = pp2_ppio_remove_mac_addr(priv->ppio,
 756                                        dev->data->mac_addrs[index].addr_bytes);
 757         if (ret) {
 758                 ether_format_addr(buf, sizeof(buf),
 759                                   &dev->data->mac_addrs[index]);
 760                 RTE_LOG(ERR, PMD, "Failed to remove mac %s\n", buf);
 761         }
 762 }
 763
 764 /**
 765  * DPDK callback to add a MAC address.
 766  *
 767  * @param dev
 768  *   Pointer to Ethernet device structure.
 769  * @param mac_addr
 770  *   MAC address to register.
 771  * @param index
 772  *   MAC address index.
 773  * @param vmdq
 774  *   VMDq pool index to associate address with (unused).
 775  *
 776  * @return
 777  *   0 on success, negative error value otherwise.
 778  */
 779 static int
 780 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 781                   uint32_t index, uint32_t vmdq __rte_unused)
 782 {
 783         struct mrvl_priv *priv = dev->data->dev_private;
 784         char buf[ETHER_ADDR_FMT_SIZE];
 785         int ret;
 786
 787         if (index == 0)
 788                 /* For setting index 0, mrvl_mac_addr_set() should be used.*/
 789                 return -1;
 790
 791         /*
 792          * Maximum number of uc addresses can be tuned via kernel module mvpp2x
 793          * parameter uc_filter_max. Maximum number of mc addresses is then
 794          * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and
 795          * 21 respectively.
 796          *
 797          * If more than uc_filter_max uc addresses were added to filter list
 798          * then NIC will switch to promiscuous mode automatically.
 799          *
 800          * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses
 801          * were added to filter list then NIC will switch to all-multicast mode
 802          * automatically.
 803          */
 804         ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
 805         if (ret) {
 806                 ether_format_addr(buf, sizeof(buf), mac_addr);
 807                 RTE_LOG(ERR, PMD, "Failed to add mac %s\n", buf);
 808                 return -1;
 809         }
 810
 811         return 0;
 812 }
 813
 814 /**
 815  * DPDK callback to set the primary MAC address.
 816  *
 817  * @param dev
 818  *   Pointer to Ethernet device structure.
 819  * @param mac_addr
 820  *   MAC address to register.
 821  */
 822 static void
 823 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
 824 {
 825         struct mrvl_priv *priv = dev->data->dev_private;
 826
 827         pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
 828         /*
 829          * TODO
 830          * Port stops sending packets if pp2_ppio_set_mac_addr()
 831          * was called after pp2_ppio_enable(). As a quick fix issue
 832          * enable port once again.
 833          */
 834         pp2_ppio_enable(priv->ppio);
 835 }
 836
 837 /**
 838  * DPDK callback to get information about the device.
 839  *
 840  * @param dev
 841  *   Pointer to Ethernet device structure (unused).
 842  * @param info
 843  *   Info structure output buffer.
 844  */
 845 static void
 846 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
 847                    struct rte_eth_dev_info *info)
 848 {
 849         info->speed_capa = ETH_LINK_SPEED_10M |
 850                            ETH_LINK_SPEED_100M |
 851                            ETH_LINK_SPEED_1G |
 852                            ETH_LINK_SPEED_10G;
 853
 854         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
 855         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
 856         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
 857
 858         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
 859         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
 860         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
 861
 862         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
 863         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
 864         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
 865
 866         info->rx_offload_capa = DEV_RX_OFFLOAD_JUMBO_FRAME |
 867                                 DEV_RX_OFFLOAD_VLAN_FILTER |
 868                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 869                                 DEV_RX_OFFLOAD_UDP_CKSUM |
 870                                 DEV_RX_OFFLOAD_TCP_CKSUM;
 871
 872         info->tx_offload_capa = DEV_TX_OFFLOAD_IPV4_CKSUM |
 873                                 DEV_TX_OFFLOAD_UDP_CKSUM |
 874                                 DEV_TX_OFFLOAD_TCP_CKSUM;
 875
 876         info->flow_type_rss_offloads = ETH_RSS_IPV4 |
 877                                        ETH_RSS_NONFRAG_IPV4_TCP |
 878                                        ETH_RSS_NONFRAG_IPV4_UDP;
 879
 880         /* By default packets are dropped if no descriptors are available */
 881         info->default_rxconf.rx_drop_en = 1;
 882
 883         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
 884 }
 885
 886 /**
 887  * DPDK callback to get information about specific receive queue.
 888  *
 889  * @param dev
 890  *   Pointer to Ethernet device structure.
 891  * @param rx_queue_id
 892  *   Receive queue index.
 893  * @param qinfo
 894  *   Receive queue information structure.
 895  */
 896 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
 897                               struct rte_eth_rxq_info *qinfo)
 898 {
 899         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
 900         struct mrvl_priv *priv = dev->data->dev_private;
 901         int inq = priv->rxq_map[rx_queue_id].inq;
 902         int tc = priv->rxq_map[rx_queue_id].tc;
 903         struct pp2_ppio_tc_params *tc_params =
 904                 &priv->ppio_params.inqs_params.tcs_params[tc];
 905
 906         qinfo->mp = q->mp;
 907         qinfo->nb_desc = tc_params->inqs_params[inq].size;
 908 }
 909
 910 /**
 911  * DPDK callback to get information about specific transmit queue.
 912  *
 913  * @param dev
 914  *   Pointer to Ethernet device structure.
 915  * @param tx_queue_id
 916  *   Transmit queue index.
 917  * @param qinfo
 918  *   Transmit queue information structure.
 919  */
 920 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
 921                               struct rte_eth_txq_info *qinfo)
 922 {
 923         struct mrvl_priv *priv = dev->data->dev_private;
 924
 925         qinfo->nb_desc =
 926                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
 927 }
 928
 929 /**
 930  * DPDK callback to Configure a VLAN filter.
 931  *
 932  * @param dev
 933  *   Pointer to Ethernet device structure.
 934  * @param vlan_id
 935  *   VLAN ID to filter.
 936  * @param on
 937  *   Toggle filter.
 938  *
 939  * @return
 940  *   0 on success, negative error value otherwise.
 941  */
 942 static int
 943 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 944 {
 945         struct mrvl_priv *priv = dev->data->dev_private;
 946
 947         return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) :
 948                     pp2_ppio_remove_vlan(priv->ppio, vlan_id);
 949 }
 950
 951 /**
 952  * Release buffers to hardware bpool (buffer-pool)
 953  *
 954  * @param rxq
 955  *   Receive queue pointer.
 956  * @param num
 957  *   Number of buffers to release to bpool.
 958  *
 959  * @return
 960  *   0 on success, negative error value otherwise.
 961  */
 962 static int
 963 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
 964 {
 965         struct buff_release_entry entries[MRVL_PP2_TXD_MAX];
 966         struct rte_mbuf *mbufs[MRVL_PP2_TXD_MAX];
 967         int i, ret;
 968         unsigned int core_id = rte_lcore_id();
 969         struct pp2_hif *hif = hifs[core_id];
 970         struct pp2_bpool *bpool = rxq->priv->bpool;
 971
 972         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
 973         if (ret)
 974                 return ret;
 975
 976         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
 977                 cookie_addr_high =
 978                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
 979
 980         for (i = 0; i < num; i++) {
 981                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
 982                         != cookie_addr_high) {
 983                         RTE_LOG(ERR, PMD,
 984                                 "mbuf virtual addr high 0x%lx out of range\n",
 985                                 (uint64_t)mbufs[i] >> 32);
 986                         goto out;
 987                 }
 988
 989                 entries[i].buff.addr =
 990                         rte_mbuf_data_dma_addr_default(mbufs[i]);
 991                 entries[i].buff.cookie = (pp2_cookie_t)(uint64_t)mbufs[i];
 992                 entries[i].bpool = bpool;
 993         }
 994
 995         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
 996         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
 997
 998         if (i != num)
 999                 goto out;
1000
1001         return 0;
1002 out:
1003         for (; i < num; i++)
1004                 rte_pktmbuf_free(mbufs[i]);
1005
1006         return -1;
1007 }
1008
1009 /**
1010  * DPDK callback to configure the receive queue.
1011  *
1012  * @param dev
1013  *   Pointer to Ethernet device structure.
1014  * @param idx
1015  *   RX queue index.
1016  * @param desc
1017  *   Number of descriptors to configure in queue.
1018  * @param socket
1019  *   NUMA socket on which memory must be allocated.
1020  * @param conf
1021  *   Thresholds parameters (unused_).
1022  * @param mp
1023  *   Memory pool for buffer allocations.
1024  *
1025  * @return
1026  *   0 on success, negative error value otherwise.
1027  */
1028 static int
1029 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1030                     unsigned int socket,
1031                     const struct rte_eth_rxconf *conf __rte_unused,
1032                     struct rte_mempool *mp)
1033 {
1034         struct mrvl_priv *priv = dev->data->dev_private;
1035         struct mrvl_rxq *rxq;
1036         uint32_t min_size,
1037                  max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1038         int ret, tc, inq;
1039
1040         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
1041                 /*
1042                  * Unknown TC mapping, mapping will not have a correct queue.
1043                  */
1044                 RTE_LOG(ERR, PMD, "Unknown TC mapping for queue %hu eth%hhu\n",
1045                         idx, priv->ppio_id);
1046                 return -EFAULT;
1047         }
1048
1049         min_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM -
1050                    MRVL_PKT_EFFEC_OFFS;
1051         if (min_size < max_rx_pkt_len) {
1052                 RTE_LOG(ERR, PMD,
1053                         "Mbuf size must be increased to %u bytes to hold up to %u bytes of data.\n",
1054                         max_rx_pkt_len + RTE_PKTMBUF_HEADROOM +
1055                         MRVL_PKT_EFFEC_OFFS,
1056                         max_rx_pkt_len);
1057                 return -EINVAL;
1058         }
1059
1060         if (dev->data->rx_queues[idx]) {
1061                 rte_free(dev->data->rx_queues[idx]);
1062                 dev->data->rx_queues[idx] = NULL;
1063         }
1064
1065         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
1066         if (!rxq)
1067                 return -ENOMEM;
1068
1069         rxq->priv = priv;
1070         rxq->mp = mp;
1071         rxq->cksum_enabled = dev->data->dev_conf.rxmode.hw_ip_checksum;
1072         rxq->queue_id = idx;
1073         rxq->port_id = dev->data->port_id;
1074         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
1075
1076         tc = priv->rxq_map[rxq->queue_id].tc,
1077         inq = priv->rxq_map[rxq->queue_id].inq;
1078         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
1079                 desc;
1080
1081         ret = mrvl_fill_bpool(rxq, desc);
1082         if (ret) {
1083                 rte_free(rxq);
1084                 return ret;
1085         }
1086
1087         priv->bpool_init_size += desc;
1088
1089         dev->data->rx_queues[idx] = rxq;
1090
1091         return 0;
1092 }
1093
1094 /**
1095  * DPDK callback to release the receive queue.
1096  *
1097  * @param rxq
1098  *   Generic receive queue pointer.
1099  */
1100 static void
1101 mrvl_rx_queue_release(void *rxq)
1102 {
1103         struct mrvl_rxq *q = rxq;
1104         struct pp2_ppio_tc_params *tc_params;
1105         int i, num, tc, inq;
1106
1107         if (!q)
1108                 return;
1109
1110         tc = q->priv->rxq_map[q->queue_id].tc;
1111         inq = q->priv->rxq_map[q->queue_id].inq;
1112         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
1113         num = tc_params->inqs_params[inq].size;
1114         for (i = 0; i < num; i++) {
1115                 struct pp2_buff_inf inf;
1116                 uint64_t addr;
1117
1118                 pp2_bpool_get_buff(hifs[rte_lcore_id()], q->priv->bpool, &inf);
1119                 addr = cookie_addr_high | inf.cookie;
1120                 rte_pktmbuf_free((struct rte_mbuf *)addr);
1121         }
1122
1123         rte_free(q);
1124 }
1125
1126 /**
1127  * DPDK callback to configure the transmit queue.
1128  *
1129  * @param dev
1130  *   Pointer to Ethernet device structure.
1131  * @param idx
1132  *   Transmit queue index.
1133  * @param desc
1134  *   Number of descriptors to configure in the queue.
1135  * @param socket
1136  *   NUMA socket on which memory must be allocated.
1137  * @param conf
1138  *   Thresholds parameters (unused).
1139  *
1140  * @return
1141  *   0 on success, negative error value otherwise.
1142  */
1143 static int
1144 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1145                     unsigned int socket,
1146                     const struct rte_eth_txconf *conf __rte_unused)
1147 {
1148         struct mrvl_priv *priv = dev->data->dev_private;
1149         struct mrvl_txq *txq;
1150
1151         if (dev->data->tx_queues[idx]) {
1152                 rte_free(dev->data->tx_queues[idx]);
1153                 dev->data->tx_queues[idx] = NULL;
1154         }
1155
1156         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
1157         if (!txq)
1158                 return -ENOMEM;
1159
1160         txq->priv = priv;
1161         txq->queue_id = idx;
1162         txq->port_id = dev->data->port_id;
1163         dev->data->tx_queues[idx] = txq;
1164
1165         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
1166         priv->ppio_params.outqs_params.outqs_params[idx].weight = 1;
1167
1168         return 0;
1169 }
1170
1171 /**
1172  * DPDK callback to release the transmit queue.
1173  *
1174  * @param txq
1175  *   Generic transmit queue pointer.
1176  */
1177 static void
1178 mrvl_tx_queue_release(void *txq)
1179 {
1180         struct mrvl_txq *q = txq;
1181
1182         if (!q)
1183                 return;
1184
1185         rte_free(q);
1186 }
1187
1188 /**
1189  * Update RSS hash configuration
1190  *
1191  * @param dev
1192  *   Pointer to Ethernet device structure.
1193  * @param rss_conf
1194  *   Pointer to RSS configuration.
1195  *
1196  * @return
1197  *   0 on success, negative error value otherwise.
1198  */
1199 static int
1200 mrvl_rss_hash_update(struct rte_eth_dev *dev,
1201                      struct rte_eth_rss_conf *rss_conf)
1202 {
1203         struct mrvl_priv *priv = dev->data->dev_private;
1204
1205         return mrvl_configure_rss(priv, rss_conf);
1206 }
1207
1208 /**
1209  * DPDK callback to get RSS hash configuration.
1210  *
1211  * @param dev
1212  *   Pointer to Ethernet device structure.
1213  * @rss_conf
1214  *   Pointer to RSS configuration.
1215  *
1216  * @return
1217  *   Always 0.
1218  */
1219 static int
1220 mrvl_rss_hash_conf_get(struct rte_eth_dev *dev,
1221                        struct rte_eth_rss_conf *rss_conf)
1222 {
1223         struct mrvl_priv *priv = dev->data->dev_private;
1224         enum pp2_ppio_hash_type hash_type =
1225                 priv->ppio_params.inqs_params.hash_type;
1226
1227         rss_conf->rss_key = NULL;
1228
1229         if (hash_type == PP2_PPIO_HASH_T_NONE)
1230                 rss_conf->rss_hf = 0;
1231         else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE)
1232                 rss_conf->rss_hf = ETH_RSS_IPV4;
1233         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp)
1234                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP;
1235         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp)
1236                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP;
1237
1238         return 0;
1239 }
1240
1241 static const struct eth_dev_ops mrvl_ops = {
1242         .dev_configure = mrvl_dev_configure,
1243         .dev_start = mrvl_dev_start,
1244         .dev_stop = mrvl_dev_stop,
1245         .dev_set_link_up = mrvl_dev_set_link_up,
1246         .dev_set_link_down = mrvl_dev_set_link_down,
1247         .dev_close = mrvl_dev_close,
1248         .link_update = mrvl_link_update,
1249         .promiscuous_enable = mrvl_promiscuous_enable,
1250         .allmulticast_enable = mrvl_allmulticast_enable,
1251         .promiscuous_disable = mrvl_promiscuous_disable,
1252         .allmulticast_disable = mrvl_allmulticast_disable,
1253         .mac_addr_remove = mrvl_mac_addr_remove,
1254         .mac_addr_add = mrvl_mac_addr_add,
1255         .mac_addr_set = mrvl_mac_addr_set,
1256         .mtu_set = mrvl_mtu_set,
1257         .dev_infos_get = mrvl_dev_infos_get,
1258         .rxq_info_get = mrvl_rxq_info_get,
1259         .txq_info_get = mrvl_txq_info_get,
1260         .vlan_filter_set = mrvl_vlan_filter_set,
1261         .rx_queue_setup = mrvl_rx_queue_setup,
1262         .rx_queue_release = mrvl_rx_queue_release,
1263         .tx_queue_setup = mrvl_tx_queue_setup,
1264         .tx_queue_release = mrvl_tx_queue_release,
1265         .rss_hash_update = mrvl_rss_hash_update,
1266         .rss_hash_conf_get = mrvl_rss_hash_conf_get,
1267 };
1268
1269 /**
1270  * Return packet type information and l3/l4 offsets.
1271  *
1272  * @param desc
1273  *   Pointer to the received packet descriptor.
1274  * @param l3_offset
1275  *   l3 packet offset.
1276  * @param l4_offset
1277  *   l4 packet offset.
1278  *
1279  * @return
1280  *   Packet type information.
1281  */
1282 static inline uint64_t
1283 mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
1284                                     uint8_t *l3_offset, uint8_t *l4_offset)
1285 {
1286         enum pp2_inq_l3_type l3_type;
1287         enum pp2_inq_l4_type l4_type;
1288         uint64_t packet_type;
1289
1290         pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
1291         pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
1292
1293         packet_type = RTE_PTYPE_L2_ETHER;
1294
1295         switch (l3_type) {
1296         case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
1297                 packet_type |= RTE_PTYPE_L3_IPV4;
1298                 break;
1299         case PP2_INQ_L3_TYPE_IPV4_OK:
1300                 packet_type |= RTE_PTYPE_L3_IPV4_EXT;
1301                 break;
1302         case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO:
1303                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
1304                 break;
1305         case PP2_INQ_L3_TYPE_IPV6_NO_EXT:
1306                 packet_type |= RTE_PTYPE_L3_IPV6;
1307                 break;
1308         case PP2_INQ_L3_TYPE_IPV6_EXT:
1309                 packet_type |= RTE_PTYPE_L3_IPV6_EXT;
1310                 break;
1311         case PP2_INQ_L3_TYPE_ARP:
1312                 packet_type |= RTE_PTYPE_L2_ETHER_ARP;
1313                 /*
1314                  * In case of ARP l4_offset is set to wrong value.
1315                  * Set it to proper one so that later on mbuf->l3_len can be
1316                  * calculated subtracting l4_offset and l3_offset.
1317                  */
1318                 *l4_offset = *l3_offset + MRVL_ARP_LENGTH;
1319                 break;
1320         default:
1321                 RTE_LOG(DEBUG, PMD, "Failed to recognise l3 packet type\n");
1322                 break;
1323         }
1324
1325         switch (l4_type) {
1326         case PP2_INQ_L4_TYPE_TCP:
1327                 packet_type |= RTE_PTYPE_L4_TCP;
1328                 break;
1329         case PP2_INQ_L4_TYPE_UDP:
1330                 packet_type |= RTE_PTYPE_L4_UDP;
1331                 break;
1332         default:
1333                 RTE_LOG(DEBUG, PMD, "Failed to recognise l4 packet type\n");
1334                 break;
1335         }
1336
1337         return packet_type;
1338 }
1339
1340 /**
1341  * Get offload information from the received packet descriptor.
1342  *
1343  * @param desc
1344  *   Pointer to the received packet descriptor.
1345  *
1346  * @return
1347  *   Mbuf offload flags.
1348  */
1349 static inline uint64_t
1350 mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc)
1351 {
1352         uint64_t flags;
1353         enum pp2_inq_desc_status status;
1354
1355         status = pp2_ppio_inq_desc_get_l3_pkt_error(desc);
1356         if (unlikely(status != PP2_DESC_ERR_OK))
1357                 flags = PKT_RX_IP_CKSUM_BAD;
1358         else
1359                 flags = PKT_RX_IP_CKSUM_GOOD;
1360
1361         status = pp2_ppio_inq_desc_get_l4_pkt_error(desc);
1362         if (unlikely(status != PP2_DESC_ERR_OK))
1363                 flags |= PKT_RX_L4_CKSUM_BAD;
1364         else
1365                 flags |= PKT_RX_L4_CKSUM_GOOD;
1366
1367         return flags;
1368 }
1369
1370 /**
1371  * DPDK callback for receive.
1372  *
1373  * @param rxq
1374  *   Generic pointer to the receive queue.
1375  * @param rx_pkts
1376  *   Array to store received packets.
1377  * @param nb_pkts
1378  *   Maximum number of packets in array.
1379  *
1380  * @return
1381  *   Number of packets successfully received.
1382  */
1383 static uint16_t
1384 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
1385 {
1386         struct mrvl_rxq *q = rxq;
1387         struct pp2_ppio_desc descs[nb_pkts];
1388         struct pp2_bpool *bpool;
1389         int i, ret, rx_done = 0;
1390         int num;
1391         unsigned int core_id = rte_lcore_id();
1392
1393         if (unlikely(!q->priv->ppio))
1394                 return 0;
1395
1396         bpool = q->priv->bpool;
1397
1398         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
1399                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
1400         if (unlikely(ret < 0)) {
1401                 RTE_LOG(ERR, PMD, "Failed to receive packets\n");
1402                 return 0;
1403         }
1404         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
1405
1406         for (i = 0; i < nb_pkts; i++) {
1407                 struct rte_mbuf *mbuf;
1408                 uint8_t l3_offset, l4_offset;
1409                 enum pp2_inq_desc_status status;
1410                 uint64_t addr;
1411
1412                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
1413                         struct pp2_ppio_desc *pref_desc;
1414                         u64 pref_addr;
1415
1416                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
1417                         pref_addr = cookie_addr_high |
1418                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
1419                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
1420                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
1421                 }
1422
1423                 addr = cookie_addr_high |
1424                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
1425                 mbuf = (struct rte_mbuf *)addr;
1426                 rte_pktmbuf_reset(mbuf);
1427
1428                 /* drop packet in case of mac, overrun or resource error */
1429                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
1430                 if (unlikely(status != PP2_DESC_ERR_OK)) {
1431                         struct pp2_buff_inf binf = {
1432                                 .addr = rte_mbuf_data_dma_addr_default(mbuf),
1433                                 .cookie = (pp2_cookie_t)(uint64_t)mbuf,
1434                         };
1435
1436                         pp2_bpool_put_buff(hifs[core_id], bpool, &binf);
1437                         mrvl_port_bpool_size
1438                                 [bpool->pp2_id][bpool->id][core_id]++;
1439                         continue;
1440                 }
1441
1442                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
1443                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
1444                 mbuf->data_len = mbuf->pkt_len;
1445                 mbuf->port = q->port_id;
1446                 mbuf->packet_type =
1447                         mrvl_desc_to_packet_type_and_offset(&descs[i],
1448                                                             &l3_offset,
1449                                                             &l4_offset);
1450                 mbuf->l2_len = l3_offset;
1451                 mbuf->l3_len = l4_offset - l3_offset;
1452
1453                 if (likely(q->cksum_enabled))
1454                         mbuf->ol_flags = mrvl_desc_to_ol_flags(&descs[i]);
1455
1456                 rx_pkts[rx_done++] = mbuf;
1457         }
1458
1459         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
1460                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
1461
1462                 if (unlikely(num <= q->priv->bpool_min_size ||
1463                              (!rx_done && num < q->priv->bpool_init_size))) {
1464                         ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE);
1465                         if (ret)
1466                                 RTE_LOG(ERR, PMD, "Failed to fill bpool\n");
1467                 } else if (unlikely(num > q->priv->bpool_max_size)) {
1468                         int i;
1469                         int pkt_to_remove = num - q->priv->bpool_init_size;
1470                         struct rte_mbuf *mbuf;
1471                         struct pp2_buff_inf buff;
1472
1473                         RTE_LOG(DEBUG, PMD,
1474                                 "\nport-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)\n",
1475                                 bpool->pp2_id, q->priv->ppio->port_id,
1476                                 bpool->id, pkt_to_remove, num,
1477                                 q->priv->bpool_init_size);
1478
1479                         for (i = 0; i < pkt_to_remove; i++) {
1480                                 pp2_bpool_get_buff(hifs[core_id], bpool, &buff);
1481                                 mbuf = (struct rte_mbuf *)
1482                                         (cookie_addr_high | buff.cookie);
1483                                 rte_pktmbuf_free(mbuf);
1484                         }
1485                         mrvl_port_bpool_size
1486                                 [bpool->pp2_id][bpool->id][core_id] -=
1487                                                                 pkt_to_remove;
1488                 }
1489                 rte_spinlock_unlock(&q->priv->lock);
1490         }
1491
1492         return rx_done;
1493 }
1494
1495 /**
1496  * Prepare offload information.
1497  *
1498  * @param ol_flags
1499  *   Offload flags.
1500  * @param packet_type
1501  *   Packet type bitfield.
1502  * @param l3_type
1503  *   Pointer to the pp2_ouq_l3_type structure.
1504  * @param l4_type
1505  *   Pointer to the pp2_outq_l4_type structure.
1506  * @param gen_l3_cksum
1507  *   Will be set to 1 in case l3 checksum is computed.
1508  * @param l4_cksum
1509  *   Will be set to 1 in case l4 checksum is computed.
1510  *
1511  * @return
1512  *   0 on success, negative error value otherwise.
1513  */
1514 static inline int
1515 mrvl_prepare_proto_info(uint64_t ol_flags, uint32_t packet_type,
1516                         enum pp2_outq_l3_type *l3_type,
1517                         enum pp2_outq_l4_type *l4_type,
1518                         int *gen_l3_cksum,
1519                         int *gen_l4_cksum)
1520 {
1521         /*
1522          * Based on ol_flags prepare information
1523          * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor
1524          * for offloading.
1525          */
1526         if (ol_flags & PKT_TX_IPV4) {
1527                 *l3_type = PP2_OUTQ_L3_TYPE_IPV4;
1528                 *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0;
1529         } else if (ol_flags & PKT_TX_IPV6) {
1530                 *l3_type = PP2_OUTQ_L3_TYPE_IPV6;
1531                 /* no checksum for ipv6 header */
1532                 *gen_l3_cksum = 0;
1533         } else {
1534                 /* if something different then stop processing */
1535                 return -1;
1536         }
1537
1538         ol_flags &= PKT_TX_L4_MASK;
1539         if ((packet_type & RTE_PTYPE_L4_TCP) &&
1540             ol_flags == PKT_TX_TCP_CKSUM) {
1541                 *l4_type = PP2_OUTQ_L4_TYPE_TCP;
1542                 *gen_l4_cksum = 1;
1543         } else if ((packet_type & RTE_PTYPE_L4_UDP) &&
1544                    ol_flags == PKT_TX_UDP_CKSUM) {
1545                 *l4_type = PP2_OUTQ_L4_TYPE_UDP;
1546                 *gen_l4_cksum = 1;
1547         } else {
1548                 *l4_type = PP2_OUTQ_L4_TYPE_OTHER;
1549                 /* no checksum for other type */
1550                 *gen_l4_cksum = 0;
1551         }
1552
1553         return 0;
1554 }
1555
1556 /**
1557  * Release already sent buffers to bpool (buffer-pool).
1558  *
1559  * @param ppio
1560  *   Pointer to the port structure.
1561  * @param hif
1562  *   Pointer to the MUSDK hardware interface.
1563  * @param sq
1564  *   Pointer to the shadow queue.
1565  * @param qid
1566  *   Queue id number.
1567  * @param force
1568  *   Force releasing packets.
1569  */
1570 static inline void
1571 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
1572                        struct mrvl_shadow_txq *sq, int qid, int force)
1573 {
1574         struct buff_release_entry *entry;
1575         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
1576         int i, core_id = rte_lcore_id();
1577
1578         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
1579
1580         sq->num_to_release += nb_done;
1581
1582         if (likely(!force &&
1583                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
1584                 return;
1585
1586         nb_done = sq->num_to_release;
1587         sq->num_to_release = 0;
1588
1589         for (i = 0; i < nb_done; i++) {
1590                 entry = &sq->ent[sq->tail + num];
1591                 if (unlikely(!entry->buff.addr)) {
1592                         RTE_LOG(ERR, PMD,
1593                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!\n",
1594                                 sq->tail, (u64)entry->buff.cookie,
1595                                 (u64)entry->buff.addr);
1596                         skip_bufs = 1;
1597                         goto skip;
1598                 }
1599
1600                 if (unlikely(!entry->bpool)) {
1601                         struct rte_mbuf *mbuf;
1602
1603                         mbuf = (struct rte_mbuf *)
1604                                (cookie_addr_high | entry->buff.cookie);
1605                         rte_pktmbuf_free(mbuf);
1606                         skip_bufs = 1;
1607                         goto skip;
1608                 }
1609
1610                 mrvl_port_bpool_size
1611                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
1612                 num++;
1613                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
1614                         goto skip;
1615                 continue;
1616 skip:
1617                 if (likely(num))
1618                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
1619                 num += skip_bufs;
1620                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
1621                 sq->size -= num;
1622                 num = 0;
1623         }
1624
1625         if (likely(num)) {
1626                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
1627                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
1628                 sq->size -= num;
1629         }
1630 }
1631
1632 /**
1633  * DPDK callback for transmit.
1634  *
1635  * @param txq
1636  *   Generic pointer transmit queue.
1637  * @param tx_pkts
1638  *   Packets to transmit.
1639  * @param nb_pkts
1640  *   Number of packets in array.
1641  *
1642  * @return
1643  *   Number of packets successfully transmitted.
1644  */
1645 static uint16_t
1646 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1647 {
1648         struct mrvl_txq *q = txq;
1649         struct mrvl_shadow_txq *sq = &shadow_txqs[q->port_id][rte_lcore_id()];
1650         struct pp2_hif *hif = hifs[rte_lcore_id()];
1651         struct pp2_ppio_desc descs[nb_pkts];
1652         int i, ret;
1653         uint16_t num, sq_free_size;
1654
1655         if (unlikely(!q->priv->ppio))
1656                 return 0;
1657
1658         if (sq->size)
1659                 mrvl_free_sent_buffers(q->priv->ppio, hif, sq, q->queue_id, 0);
1660
1661         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
1662         if (unlikely(nb_pkts > sq_free_size)) {
1663                 RTE_LOG(DEBUG, PMD,
1664                         "No room in shadow queue for %d packets! %d packets will be sent.\n",
1665                         nb_pkts, sq_free_size);
1666                 nb_pkts = sq_free_size;
1667         }
1668
1669         for (i = 0; i < nb_pkts; i++) {
1670                 struct rte_mbuf *mbuf = tx_pkts[i];
1671                 int gen_l3_cksum, gen_l4_cksum;
1672                 enum pp2_outq_l3_type l3_type;
1673                 enum pp2_outq_l4_type l4_type;
1674
1675                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
1676                         struct rte_mbuf *pref_pkt_hdr;
1677
1678                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
1679                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
1680                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
1681                 }
1682
1683                 sq->ent[sq->head].buff.cookie = (pp2_cookie_t)(uint64_t)mbuf;
1684                 sq->ent[sq->head].buff.addr =
1685                         rte_mbuf_data_dma_addr_default(mbuf);
1686                 sq->ent[sq->head].bpool =
1687                         (unlikely(mbuf->port == 0xff || mbuf->refcnt > 1)) ?
1688                          NULL : mrvl_port_to_bpool_lookup[mbuf->port];
1689                 sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
1690                 sq->size++;
1691
1692                 pp2_ppio_outq_desc_reset(&descs[i]);
1693                 pp2_ppio_outq_desc_set_phys_addr(&descs[i],
1694                                                  rte_pktmbuf_mtophys(mbuf));
1695                 pp2_ppio_outq_desc_set_pkt_offset(&descs[i], 0);
1696                 pp2_ppio_outq_desc_set_pkt_len(&descs[i],
1697                                                rte_pktmbuf_pkt_len(mbuf));
1698
1699                 /*
1700                  * in case unsupported ol_flags were passed
1701                  * do not update descriptor offload information
1702                  */
1703                 ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type,
1704                                               &l3_type, &l4_type, &gen_l3_cksum,
1705                                               &gen_l4_cksum);
1706                 if (unlikely(ret))
1707                         continue;
1708
1709                 pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type,
1710                                                   mbuf->l2_len,
1711                                                   mbuf->l2_len + mbuf->l3_len,
1712                                                   gen_l3_cksum, gen_l4_cksum);
1713         }
1714
1715         num = nb_pkts;
1716         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
1717         /* number of packets that were not sent */
1718         if (unlikely(num > nb_pkts)) {
1719                 for (i = nb_pkts; i < num; i++) {
1720                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
1721                                 MRVL_PP2_TX_SHADOWQ_MASK;
1722                 }
1723                 sq->size -= num - nb_pkts;
1724         }
1725
1726         return nb_pkts;
1727 }
1728
1729 /**
1730  * Initialize packet processor.
1731  *
1732  * @return
1733  *   0 on success, negative error value otherwise.
1734  */
1735 static int
1736 mrvl_init_pp2(void)
1737 {
1738         struct pp2_init_params init_params;
1739
1740         memset(&init_params, 0, sizeof(init_params));
1741         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
1742         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
1743         init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED;
1744
1745         return pp2_init(&init_params);
1746 }
1747
1748 /**
1749  * Deinitialize packet processor.
1750  *
1751  * @return
1752  *   0 on success, negative error value otherwise.
1753  */
1754 static void
1755 mrvl_deinit_pp2(void)
1756 {
1757         pp2_deinit();
1758 }
1759
1760 /**
1761  * Create private device structure.
1762  *
1763  * @param dev_name
1764  *   Pointer to the port name passed in the initialization parameters.
1765  *
1766  * @return
1767  *   Pointer to the newly allocated private device structure.
1768  */
1769 static struct mrvl_priv *
1770 mrvl_priv_create(const char *dev_name)
1771 {
1772         struct pp2_bpool_params bpool_params;
1773         char match[MRVL_MATCH_LEN];
1774         struct mrvl_priv *priv;
1775         int ret, bpool_bit;
1776
1777         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
1778         if (!priv)
1779                 return NULL;
1780
1781         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
1782                                        &priv->pp_id, &priv->ppio_id);
1783         if (ret)
1784                 goto out_free_priv;
1785
1786         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
1787                                      PP2_BPOOL_NUM_POOLS);
1788         if (bpool_bit < 0)
1789                 goto out_free_priv;
1790         priv->bpool_bit = bpool_bit;
1791
1792         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
1793                  priv->bpool_bit);
1794         memset(&bpool_params, 0, sizeof(bpool_params));
1795         bpool_params.match = match;
1796         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
1797         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
1798         if (ret)
1799                 goto out_clear_bpool_bit;
1800
1801         priv->ppio_params.type = PP2_PPIO_T_NIC;
1802         rte_spinlock_init(&priv->lock);
1803
1804         return priv;
1805 out_clear_bpool_bit:
1806         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
1807 out_free_priv:
1808         rte_free(priv);
1809         return NULL;
1810 }
1811
1812 /**
1813  * Create device representing Ethernet port.
1814  *
1815  * @param name
1816  *   Pointer to the port's name.
1817  *
1818  * @return
1819  *   0 on success, negative error value otherwise.
1820  */
1821 static int
1822 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
1823 {
1824         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
1825         struct rte_eth_dev *eth_dev;
1826         struct mrvl_priv *priv;
1827         struct ifreq req;
1828
1829         eth_dev = rte_eth_dev_allocate(name);
1830         if (!eth_dev)
1831                 return -ENOMEM;
1832
1833         priv = mrvl_priv_create(name);
1834         if (!priv) {
1835                 ret = -ENOMEM;
1836                 goto out_free_dev;
1837         }
1838
1839         eth_dev->data->mac_addrs =
1840                 rte_zmalloc("mac_addrs",
1841                             ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
1842         if (!eth_dev->data->mac_addrs) {
1843                 RTE_LOG(ERR, PMD, "Failed to allocate space for eth addrs\n");
1844                 ret = -ENOMEM;
1845                 goto out_free_priv;
1846         }
1847
1848         memset(&req, 0, sizeof(req));
1849         strcpy(req.ifr_name, name);
1850         ret = ioctl(fd, SIOCGIFHWADDR, &req);
1851         if (ret)
1852                 goto out_free_mac;
1853
1854         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
1855                req.ifr_addr.sa_data, ETHER_ADDR_LEN);
1856
1857         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
1858         eth_dev->tx_pkt_burst = mrvl_tx_pkt_burst;
1859         eth_dev->data->dev_private = priv;
1860         eth_dev->device = &vdev->device;
1861         eth_dev->dev_ops = &mrvl_ops;
1862
1863         return 0;
1864 out_free_mac:
1865         rte_free(eth_dev->data->mac_addrs);
1866 out_free_dev:
1867         rte_eth_dev_release_port(eth_dev);
1868 out_free_priv:
1869         rte_free(priv);
1870
1871         return ret;
1872 }
1873
1874 /**
1875  * Cleanup previously created device representing Ethernet port.
1876  *
1877  * @param name
1878  *   Pointer to the port name.
1879  */
1880 static void
1881 mrvl_eth_dev_destroy(const char *name)
1882 {
1883         struct rte_eth_dev *eth_dev;
1884         struct mrvl_priv *priv;
1885
1886         eth_dev = rte_eth_dev_allocated(name);
1887         if (!eth_dev)
1888                 return;
1889
1890         priv = eth_dev->data->dev_private;
1891         pp2_bpool_deinit(priv->bpool);
1892         rte_free(priv);
1893         rte_free(eth_dev->data->mac_addrs);
1894         rte_eth_dev_release_port(eth_dev);
1895 }
1896
1897 /**
1898  * Callback used by rte_kvargs_process() during argument parsing.
1899  *
1900  * @param key
1901  *   Pointer to the parsed key (unused).
1902  * @param value
1903  *   Pointer to the parsed value.
1904  * @param extra_args
1905  *   Pointer to the extra arguments which contains address of the
1906  *   table of pointers to parsed interface names.
1907  *
1908  * @return
1909  *   Always 0.
1910  */
1911 static int
1912 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
1913                  void *extra_args)
1914 {
1915         const char **ifnames = extra_args;
1916
1917         ifnames[mrvl_ports_nb++] = value;
1918
1919         return 0;
1920 }
1921
1922 /**
1923  * Initialize per-lcore MUSDK hardware interfaces (hifs).
1924  *
1925  * @return
1926  *   0 on success, negative error value otherwise.
1927  */
1928 static int
1929 mrvl_init_hifs(void)
1930 {
1931         struct pp2_hif_params params;
1932         char match[MRVL_MATCH_LEN];
1933         int i, ret;
1934
1935         RTE_LCORE_FOREACH(i) {
1936                 ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
1937                 if (ret < 0)
1938                         return ret;
1939
1940                 snprintf(match, sizeof(match), "hif-%d", ret);
1941                 memset(&params, 0, sizeof(params));
1942                 params.match = match;
1943                 params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
1944                 ret = pp2_hif_init(&params, &hifs[i]);
1945                 if (ret) {
1946                         RTE_LOG(ERR, PMD, "Failed to initialize hif %d\n", i);
1947                         return ret;
1948                 }
1949         }
1950
1951         return 0;
1952 }
1953
1954 /**
1955  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
1956  */
1957 static void
1958 mrvl_deinit_hifs(void)
1959 {
1960         int i;
1961
1962         RTE_LCORE_FOREACH(i) {
1963                 if (hifs[i])
1964                         pp2_hif_deinit(hifs[i]);
1965         }
1966 }
1967
1968 static void mrvl_set_first_last_cores(int core_id)
1969 {
1970         if (core_id < mrvl_lcore_first)
1971                 mrvl_lcore_first = core_id;
1972
1973         if (core_id > mrvl_lcore_last)
1974                 mrvl_lcore_last = core_id;
1975 }
1976
1977 /**
1978  * DPDK callback to register the virtual device.
1979  *
1980  * @param vdev
1981  *   Pointer to the virtual device.
1982  *
1983  * @return
1984  *   0 on success, negative error value otherwise.
1985  */
1986 static int
1987 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
1988 {
1989         struct rte_kvargs *kvlist;
1990         const char *ifnames[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
1991         int ret = -EINVAL;
1992         uint32_t i, ifnum, cfgnum, core_id;
1993         const char *params;
1994
1995         params = rte_vdev_device_args(vdev);
1996         if (!params)
1997                 return -EINVAL;
1998
1999         kvlist = rte_kvargs_parse(params, valid_args);
2000         if (!kvlist)
2001                 return -EINVAL;
2002
2003         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
2004         if (ifnum > RTE_DIM(ifnames))
2005                 goto out_free_kvlist;
2006
2007         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
2008                            mrvl_get_ifnames, &ifnames);
2009
2010         cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
2011         if (cfgnum > 1) {
2012                 RTE_LOG(ERR, PMD, "Cannot handle more than one config file!\n");
2013                 goto out_free_kvlist;
2014         } else if (cfgnum == 1) {
2015                 rte_kvargs_process(kvlist, MRVL_CFG_ARG,
2016                                    mrvl_get_qoscfg, &mrvl_qos_cfg);
2017         }
2018
2019         /*
2020          * ret == -EEXIST is correct, it means DMA
2021          * has been already initialized (by another PMD).
2022          */
2023         ret = mv_sys_dma_mem_init(RTE_MRVL_MUSDK_DMA_MEMSIZE);
2024         if (ret < 0 && ret != -EEXIST)
2025                 goto out_free_kvlist;
2026
2027         ret = mrvl_init_pp2();
2028         if (ret) {
2029                 RTE_LOG(ERR, PMD, "Failed to init PP!\n");
2030                 goto out_deinit_dma;
2031         }
2032
2033         ret = mrvl_init_hifs();
2034         if (ret)
2035                 goto out_deinit_hifs;
2036
2037         for (i = 0; i < ifnum; i++) {
2038                 RTE_LOG(INFO, PMD, "Creating %s\n", ifnames[i]);
2039                 ret = mrvl_eth_dev_create(vdev, ifnames[i]);
2040                 if (ret)
2041                         goto out_cleanup;
2042         }
2043
2044         rte_kvargs_free(kvlist);
2045
2046         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
2047
2048         mrvl_lcore_first = RTE_MAX_LCORE;
2049         mrvl_lcore_last = 0;
2050
2051         RTE_LCORE_FOREACH(core_id) {
2052                 mrvl_set_first_last_cores(core_id);
2053         }
2054
2055         return 0;
2056 out_cleanup:
2057         for (; i > 0; i--)
2058                 mrvl_eth_dev_destroy(ifnames[i]);
2059 out_deinit_hifs:
2060         mrvl_deinit_hifs();
2061         mrvl_deinit_pp2();
2062 out_deinit_dma:
2063         mv_sys_dma_mem_destroy();
2064 out_free_kvlist:
2065         rte_kvargs_free(kvlist);
2066
2067         return ret;
2068 }
2069
2070 /**
2071  * DPDK callback to remove virtual device.
2072  *
2073  * @param vdev
2074  *   Pointer to the removed virtual device.
2075  *
2076  * @return
2077  *   0 on success, negative error value otherwise.
2078  */
2079 static int
2080 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
2081 {
2082         int i;
2083         const char *name;
2084
2085         name = rte_vdev_device_name(vdev);
2086         if (!name)
2087                 return -EINVAL;
2088
2089         RTE_LOG(INFO, PMD, "Removing %s\n", name);
2090
2091         for (i = 0; i < rte_eth_dev_count(); i++) {
2092                 char ifname[RTE_ETH_NAME_MAX_LEN];
2093
2094                 rte_eth_dev_get_name_by_port(i, ifname);
2095                 mrvl_eth_dev_destroy(ifname);
2096         }
2097
2098         mrvl_deinit_hifs();
2099         mrvl_deinit_pp2();
2100         mv_sys_dma_mem_destroy();
2101
2102         return 0;
2103 }
2104
2105 static struct rte_vdev_driver pmd_mrvl_drv = {
2106         .probe = rte_pmd_mrvl_probe,
2107         .remove = rte_pmd_mrvl_remove,
2108 };
2109
2110 RTE_PMD_REGISTER_VDEV(net_mrvl, pmd_mrvl_drv);
2111 RTE_PMD_REGISTER_ALIAS(net_mrvl, eth_mrvl);