drivers/net/mrvl/mrvl_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Marvell International Ltd.
   5  *   Copyright(c) 2017 Semihalf.
   6  *   All rights reserved.
   7  *
   8  *   Redistribution and use in source and binary forms, with or without
   9  *   modification, are permitted provided that the following conditions
  10  *   are met:
  11  *
  12  *     * Redistributions of source code must retain the above copyright
  13  *       notice, this list of conditions and the following disclaimer.
  14  *     * Redistributions in binary form must reproduce the above copyright
  15  *       notice, this list of conditions and the following disclaimer in
  16  *       the documentation and/or other materials provided with the
  17  *       distribution.
  18  *     * Neither the name of Semihalf nor the names of its
  19  *       contributors may be used to endorse or promote products derived
  20  *       from this software without specific prior written permission.
  21  *
  22  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  23  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  24  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  25  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  26  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  27  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  28  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  29  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  30  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  31  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  32  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  33  */
  34
  35 #include <rte_ethdev.h>
  36 #include <rte_kvargs.h>
  37 #include <rte_log.h>
  38 #include <rte_malloc.h>
  39 #include <rte_bus_vdev.h>
  40
  41 /* Unluckily, container_of is defined by both DPDK and MUSDK,
  42  * we'll declare only one version.
  43  *
  44  * Note that it is not used in this PMD anyway.
  45  */
  46 #ifdef container_of
  47 #undef container_of
  48 #endif
  49
  50 #include <fcntl.h>
  51 #include <linux/ethtool.h>
  52 #include <linux/sockios.h>
  53 #include <net/if.h>
  54 #include <net/if_arp.h>
  55 #include <sys/ioctl.h>
  56 #include <sys/socket.h>
  57 #include <sys/stat.h>
  58 #include <sys/types.h>
  59
  60 #include "mrvl_ethdev.h"
  61 #include "mrvl_qos.h"
  62
  63 /* bitmask with reserved hifs */
  64 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
  65 /* bitmask with reserved bpools */
  66 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
  67 /* bitmask with reserved kernel RSS tables */
  68 #define MRVL_MUSDK_RSS_RESERVED 0x01
  69 /* maximum number of available hifs */
  70 #define MRVL_MUSDK_HIFS_MAX 9
  71
  72 /* prefetch shift */
  73 #define MRVL_MUSDK_PREFETCH_SHIFT 2
  74
  75 /* TCAM has 25 entries reserved for uc/mc filter entries */
  76 #define MRVL_MAC_ADDRS_MAX 25
  77 #define MRVL_MATCH_LEN 16
  78 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
  79 /* Maximum allowable packet size */
  80 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
  81
  82 #define MRVL_IFACE_NAME_ARG "iface"
  83 #define MRVL_CFG_ARG "cfg"
  84
  85 #define MRVL_BURST_SIZE 64
  86
  87 #define MRVL_ARP_LENGTH 28
  88
  89 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
  90
  91 #define MRVL_COOKIE_HIGH_ADDR_SHIFT     (sizeof(pp2_cookie_t) * 8)
  92 #define MRVL_COOKIE_HIGH_ADDR_MASK      (~0ULL << MRVL_COOKIE_HIGH_ADDR_SHIFT)
  93
  94 /* Memory size (in bytes) for MUSDK dma buffers */
  95 #define MRVL_MUSDK_DMA_MEMSIZE 41943040
  96
  97 static const char * const valid_args[] = {
  98         MRVL_IFACE_NAME_ARG,
  99         MRVL_CFG_ARG,
 100         NULL
 101 };
 102
 103 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
 104 static struct pp2_hif *hifs[RTE_MAX_LCORE];
 105 static int used_bpools[PP2_NUM_PKT_PROC] = {
 106         MRVL_MUSDK_BPOOLS_RESERVED,
 107         MRVL_MUSDK_BPOOLS_RESERVED
 108 };
 109
 110 struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
 111 int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
 112 uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
 113
 114 /*
 115  * To use buffer harvesting based on loopback port shadow queue structure
 116  * was introduced for buffers information bookkeeping.
 117  *
 118  * Before sending the packet, related buffer information (pp2_buff_inf) is
 119  * stored in shadow queue. After packet is transmitted no longer used
 120  * packet buffer is released back to it's original hardware pool,
 121  * on condition it originated from interface.
 122  * In case it  was generated by application itself i.e: mbuf->port field is
 123  * 0xff then its released to software mempool.
 124  */
 125 struct mrvl_shadow_txq {
 126         int head;           /* write index - used when sending buffers */
 127         int tail;           /* read index - used when releasing buffers */
 128         u16 size;           /* queue occupied size */
 129         u16 num_to_release; /* number of buffers sent, that can be released */
 130         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
 131 };
 132
 133 struct mrvl_rxq {
 134         struct mrvl_priv *priv;
 135         struct rte_mempool *mp;
 136         int queue_id;
 137         int port_id;
 138         int cksum_enabled;
 139         uint64_t bytes_recv;
 140         uint64_t drop_mac;
 141 };
 142
 143 struct mrvl_txq {
 144         struct mrvl_priv *priv;
 145         int queue_id;
 146         int port_id;
 147         uint64_t bytes_sent;
 148 };
 149
 150 /*
 151  * Every tx queue should have dedicated shadow tx queue.
 152  *
 153  * Ports assigned by DPDK might not start at zero or be continuous so
 154  * as a workaround define shadow queues for each possible port so that
 155  * we eventually fit somewhere.
 156  */
 157 struct mrvl_shadow_txq shadow_txqs[RTE_MAX_ETHPORTS][RTE_MAX_LCORE];
 158
 159 /** Number of ports configured. */
 160 int mrvl_ports_nb;
 161 static int mrvl_lcore_first;
 162 static int mrvl_lcore_last;
 163
 164 static inline int
 165 mrvl_get_bpool_size(int pp2_id, int pool_id)
 166 {
 167         int i;
 168         int size = 0;
 169
 170         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
 171                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
 172
 173         return size;
 174 }
 175
 176 static inline int
 177 mrvl_reserve_bit(int *bitmap, int max)
 178 {
 179         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
 180
 181         if (n >= max)
 182                 return -1;
 183
 184         *bitmap |= 1 << n;
 185
 186         return n;
 187 }
 188
 189 /**
 190  * Configure rss based on dpdk rss configuration.
 191  *
 192  * @param priv
 193  *   Pointer to private structure.
 194  * @param rss_conf
 195  *   Pointer to RSS configuration.
 196  *
 197  * @return
 198  *   0 on success, negative error value otherwise.
 199  */
 200 static int
 201 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf)
 202 {
 203         if (rss_conf->rss_key)
 204                 RTE_LOG(WARNING, PMD, "Changing hash key is not supported\n");
 205
 206         if (rss_conf->rss_hf == 0) {
 207                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 208         } else if (rss_conf->rss_hf & ETH_RSS_IPV4) {
 209                 priv->ppio_params.inqs_params.hash_type =
 210                         PP2_PPIO_HASH_T_2_TUPLE;
 211         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
 212                 priv->ppio_params.inqs_params.hash_type =
 213                         PP2_PPIO_HASH_T_5_TUPLE;
 214                 priv->rss_hf_tcp = 1;
 215         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
 216                 priv->ppio_params.inqs_params.hash_type =
 217                         PP2_PPIO_HASH_T_5_TUPLE;
 218                 priv->rss_hf_tcp = 0;
 219         } else {
 220                 return -EINVAL;
 221         }
 222
 223         return 0;
 224 }
 225
 226 /**
 227  * Ethernet device configuration.
 228  *
 229  * Prepare the driver for a given number of TX and RX queues and
 230  * configure RSS.
 231  *
 232  * @param dev
 233  *   Pointer to Ethernet device structure.
 234  *
 235  * @return
 236  *   0 on success, negative error value otherwise.
 237  */
 238 static int
 239 mrvl_dev_configure(struct rte_eth_dev *dev)
 240 {
 241         struct mrvl_priv *priv = dev->data->dev_private;
 242         int ret;
 243
 244         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
 245             dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
 246                 RTE_LOG(INFO, PMD, "Unsupported rx multi queue mode %d\n",
 247                         dev->data->dev_conf.rxmode.mq_mode);
 248                 return -EINVAL;
 249         }
 250
 251         if (!dev->data->dev_conf.rxmode.hw_strip_crc) {
 252                 RTE_LOG(INFO, PMD,
 253                         "L2 CRC stripping is always enabled in hw\n");
 254                 dev->data->dev_conf.rxmode.hw_strip_crc = 1;
 255         }
 256
 257         if (dev->data->dev_conf.rxmode.hw_vlan_strip) {
 258                 RTE_LOG(INFO, PMD, "VLAN stripping not supported\n");
 259                 return -EINVAL;
 260         }
 261
 262         if (dev->data->dev_conf.rxmode.split_hdr_size) {
 263                 RTE_LOG(INFO, PMD, "Split headers not supported\n");
 264                 return -EINVAL;
 265         }
 266
 267         if (dev->data->dev_conf.rxmode.enable_scatter) {
 268                 RTE_LOG(INFO, PMD, "RX Scatter/Gather not supported\n");
 269                 return -EINVAL;
 270         }
 271
 272         if (dev->data->dev_conf.rxmode.enable_lro) {
 273                 RTE_LOG(INFO, PMD, "LRO not supported\n");
 274                 return -EINVAL;
 275         }
 276
 277         if (dev->data->dev_conf.rxmode.jumbo_frame)
 278                 dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
 279                                  ETHER_HDR_LEN - ETHER_CRC_LEN;
 280
 281         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
 282                                   dev->data->nb_rx_queues);
 283         if (ret < 0)
 284                 return ret;
 285
 286         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
 287         priv->ppio_params.maintain_stats = 1;
 288         priv->nb_rx_queues = dev->data->nb_rx_queues;
 289
 290         if (dev->data->nb_rx_queues == 1 &&
 291             dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
 292                 RTE_LOG(WARNING, PMD, "Disabling hash for 1 rx queue\n");
 293                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 294
 295                 return 0;
 296         }
 297
 298         return mrvl_configure_rss(priv,
 299                                   &dev->data->dev_conf.rx_adv_conf.rss_conf);
 300 }
 301
 302 /**
 303  * DPDK callback to change the MTU.
 304  *
 305  * Setting the MTU affects hardware MRU (packets larger than the MRU
 306  * will be dropped).
 307  *
 308  * @param dev
 309  *   Pointer to Ethernet device structure.
 310  * @param mtu
 311  *   New MTU.
 312  *
 313  * @return
 314  *   0 on success, negative error value otherwise.
 315  */
 316 static int
 317 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 318 {
 319         struct mrvl_priv *priv = dev->data->dev_private;
 320         /* extra MV_MH_SIZE bytes are required for Marvell tag */
 321         uint16_t mru = mtu + MV_MH_SIZE + ETHER_HDR_LEN + ETHER_CRC_LEN;
 322         int ret;
 323
 324         if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX)
 325                 return -EINVAL;
 326
 327         ret = pp2_ppio_set_mru(priv->ppio, mru);
 328         if (ret)
 329                 return ret;
 330
 331         return pp2_ppio_set_mtu(priv->ppio, mtu);
 332 }
 333
 334 /**
 335  * DPDK callback to bring the link up.
 336  *
 337  * @param dev
 338  *   Pointer to Ethernet device structure.
 339  *
 340  * @return
 341  *   0 on success, negative error value otherwise.
 342  */
 343 static int
 344 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
 345 {
 346         struct mrvl_priv *priv = dev->data->dev_private;
 347         int ret;
 348
 349         ret = pp2_ppio_enable(priv->ppio);
 350         if (ret)
 351                 return ret;
 352
 353         /*
 354          * mtu/mru can be updated if pp2_ppio_enable() was called at least once
 355          * as pp2_ppio_enable() changes port->t_mode from default 0 to
 356          * PP2_TRAFFIC_INGRESS_EGRESS.
 357          *
 358          * Set mtu to default DPDK value here.
 359          */
 360         ret = mrvl_mtu_set(dev, dev->data->mtu);
 361         if (ret)
 362                 pp2_ppio_disable(priv->ppio);
 363
 364         return ret;
 365 }
 366
 367 /**
 368  * DPDK callback to bring the link down.
 369  *
 370  * @param dev
 371  *   Pointer to Ethernet device structure.
 372  *
 373  * @return
 374  *   0 on success, negative error value otherwise.
 375  */
 376 static int
 377 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
 378 {
 379         struct mrvl_priv *priv = dev->data->dev_private;
 380
 381         return pp2_ppio_disable(priv->ppio);
 382 }
 383
 384 /**
 385  * DPDK callback to start the device.
 386  *
 387  * @param dev
 388  *   Pointer to Ethernet device structure.
 389  *
 390  * @return
 391  *   0 on success, negative errno value on failure.
 392  */
 393 static int
 394 mrvl_dev_start(struct rte_eth_dev *dev)
 395 {
 396         struct mrvl_priv *priv = dev->data->dev_private;
 397         char match[MRVL_MATCH_LEN];
 398         int ret;
 399
 400         snprintf(match, sizeof(match), "ppio-%d:%d",
 401                  priv->pp_id, priv->ppio_id);
 402         priv->ppio_params.match = match;
 403
 404         /*
 405          * Calculate the maximum bpool size for refill feature to 1.5 of the
 406          * configured size. In case the bpool size will exceed this value,
 407          * superfluous buffers will be removed
 408          */
 409         priv->bpool_max_size = priv->bpool_init_size +
 410                               (priv->bpool_init_size >> 1);
 411         /*
 412          * Calculate the minimum bpool size for refill feature as follows:
 413          * 2 default burst sizes multiply by number of rx queues.
 414          * If the bpool size will be below this value, new buffers will
 415          * be added to the pool.
 416          */
 417         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
 418
 419         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
 420         if (ret)
 421                 return ret;
 422
 423         /*
 424          * In case there are some some stale uc/mc mac addresses flush them
 425          * here. It cannot be done during mrvl_dev_close() as port information
 426          * is already gone at that point (due to pp2_ppio_deinit() in
 427          * mrvl_dev_stop()).
 428          */
 429         if (!priv->uc_mc_flushed) {
 430                 ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1);
 431                 if (ret) {
 432                         RTE_LOG(ERR, PMD,
 433                                 "Failed to flush uc/mc filter list\n");
 434                         goto out;
 435                 }
 436                 priv->uc_mc_flushed = 1;
 437         }
 438
 439         if (!priv->vlan_flushed) {
 440                 ret = pp2_ppio_flush_vlan(priv->ppio);
 441                 if (ret) {
 442                         RTE_LOG(ERR, PMD, "Failed to flush vlan list\n");
 443                         /*
 444                          * TODO
 445                          * once pp2_ppio_flush_vlan() is supported jump to out
 446                          * goto out;
 447                          */
 448                 }
 449                 priv->vlan_flushed = 1;
 450         }
 451
 452         /* For default QoS config, don't start classifier. */
 453         if (mrvl_qos_cfg) {
 454                 ret = mrvl_start_qos_mapping(priv);
 455                 if (ret) {
 456                         pp2_ppio_deinit(priv->ppio);
 457                         return ret;
 458                 }
 459         }
 460
 461         ret = mrvl_dev_set_link_up(dev);
 462         if (ret)
 463                 goto out;
 464
 465         return 0;
 466 out:
 467         pp2_ppio_deinit(priv->ppio);
 468         return ret;
 469 }
 470
 471 /**
 472  * Flush receive queues.
 473  *
 474  * @param dev
 475  *   Pointer to Ethernet device structure.
 476  */
 477 static void
 478 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
 479 {
 480         int i;
 481
 482         RTE_LOG(INFO, PMD, "Flushing rx queues\n");
 483         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 484                 int ret, num;
 485
 486                 do {
 487                         struct mrvl_rxq *q = dev->data->rx_queues[i];
 488                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
 489
 490                         num = MRVL_PP2_RXD_MAX;
 491                         ret = pp2_ppio_recv(q->priv->ppio,
 492                                             q->priv->rxq_map[q->queue_id].tc,
 493                                             q->priv->rxq_map[q->queue_id].inq,
 494                                             descs, (uint16_t *)&num);
 495                 } while (ret == 0 && num);
 496         }
 497 }
 498
 499 /**
 500  * Flush transmit shadow queues.
 501  *
 502  * @param dev
 503  *   Pointer to Ethernet device structure.
 504  */
 505 static void
 506 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
 507 {
 508         int i;
 509
 510         RTE_LOG(INFO, PMD, "Flushing tx shadow queues\n");
 511         for (i = 0; i < RTE_MAX_LCORE; i++) {
 512                 struct mrvl_shadow_txq *sq =
 513                         &shadow_txqs[dev->data->port_id][i];
 514
 515                 while (sq->tail != sq->head) {
 516                         uint64_t addr = cookie_addr_high |
 517                                         sq->ent[sq->tail].buff.cookie;
 518                         rte_pktmbuf_free((struct rte_mbuf *)addr);
 519                         sq->tail = (sq->tail + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
 520                 }
 521
 522                 memset(sq, 0, sizeof(*sq));
 523         }
 524 }
 525
 526 /**
 527  * Flush hardware bpool (buffer-pool).
 528  *
 529  * @param dev
 530  *   Pointer to Ethernet device structure.
 531  */
 532 static void
 533 mrvl_flush_bpool(struct rte_eth_dev *dev)
 534 {
 535         struct mrvl_priv *priv = dev->data->dev_private;
 536         uint32_t num;
 537         int ret;
 538
 539         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
 540         if (ret) {
 541                 RTE_LOG(ERR, PMD, "Failed to get bpool buffers number\n");
 542                 return;
 543         }
 544
 545         while (num--) {
 546                 struct pp2_buff_inf inf;
 547                 uint64_t addr;
 548
 549                 ret = pp2_bpool_get_buff(hifs[rte_lcore_id()], priv->bpool,
 550                                          &inf);
 551                 if (ret)
 552                         break;
 553
 554                 addr = cookie_addr_high | inf.cookie;
 555                 rte_pktmbuf_free((struct rte_mbuf *)addr);
 556         }
 557 }
 558
 559 /**
 560  * DPDK callback to stop the device.
 561  *
 562  * @param dev
 563  *   Pointer to Ethernet device structure.
 564  */
 565 static void
 566 mrvl_dev_stop(struct rte_eth_dev *dev)
 567 {
 568         struct mrvl_priv *priv = dev->data->dev_private;
 569
 570         mrvl_dev_set_link_down(dev);
 571         mrvl_flush_rx_queues(dev);
 572         mrvl_flush_tx_shadow_queues(dev);
 573         if (priv->qos_tbl)
 574                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
 575         pp2_ppio_deinit(priv->ppio);
 576         priv->ppio = NULL;
 577 }
 578
 579 /**
 580  * DPDK callback to close the device.
 581  *
 582  * @param dev
 583  *   Pointer to Ethernet device structure.
 584  */
 585 static void
 586 mrvl_dev_close(struct rte_eth_dev *dev)
 587 {
 588         struct mrvl_priv *priv = dev->data->dev_private;
 589         size_t i;
 590
 591         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
 592                 struct pp2_ppio_tc_params *tc_params =
 593                         &priv->ppio_params.inqs_params.tcs_params[i];
 594
 595                 if (tc_params->inqs_params) {
 596                         rte_free(tc_params->inqs_params);
 597                         tc_params->inqs_params = NULL;
 598                 }
 599         }
 600
 601         mrvl_flush_bpool(dev);
 602 }
 603
 604 /**
 605  * DPDK callback to retrieve physical link information.
 606  *
 607  * @param dev
 608  *   Pointer to Ethernet device structure.
 609  * @param wait_to_complete
 610  *   Wait for request completion (ignored).
 611  *
 612  * @return
 613  *   0 on success, negative error value otherwise.
 614  */
 615 static int
 616 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
 617 {
 618         /*
 619          * TODO
 620          * once MUSDK provides necessary API use it here
 621          */
 622         struct mrvl_priv *priv = dev->data->dev_private;
 623         struct ethtool_cmd edata;
 624         struct ifreq req;
 625         int ret, fd, link_up;
 626
 627         edata.cmd = ETHTOOL_GSET;
 628
 629         strcpy(req.ifr_name, dev->data->name);
 630         req.ifr_data = (void *)&edata;
 631
 632         fd = socket(AF_INET, SOCK_DGRAM, 0);
 633         if (fd == -1)
 634                 return -EFAULT;
 635
 636         ret = ioctl(fd, SIOCETHTOOL, &req);
 637         if (ret == -1) {
 638                 close(fd);
 639                 return -EFAULT;
 640         }
 641
 642         close(fd);
 643
 644         switch (ethtool_cmd_speed(&edata)) {
 645         case SPEED_10:
 646                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
 647                 break;
 648         case SPEED_100:
 649                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
 650                 break;
 651         case SPEED_1000:
 652                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
 653                 break;
 654         case SPEED_10000:
 655                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
 656                 break;
 657         default:
 658                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
 659         }
 660
 661         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
 662                                                          ETH_LINK_HALF_DUPLEX;
 663         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
 664                                                            ETH_LINK_FIXED;
 665         pp2_ppio_get_link_state(priv->ppio, &link_up);
 666         dev->data->dev_link.link_status = link_up ? ETH_LINK_UP : ETH_LINK_DOWN;
 667
 668         return 0;
 669 }
 670
 671 /**
 672  * DPDK callback to enable promiscuous mode.
 673  *
 674  * @param dev
 675  *   Pointer to Ethernet device structure.
 676  */
 677 static void
 678 mrvl_promiscuous_enable(struct rte_eth_dev *dev)
 679 {
 680         struct mrvl_priv *priv = dev->data->dev_private;
 681         int ret;
 682
 683         ret = pp2_ppio_set_promisc(priv->ppio, 1);
 684         if (ret)
 685                 RTE_LOG(ERR, PMD, "Failed to enable promiscuous mode\n");
 686 }
 687
 688 /**
 689  * DPDK callback to enable allmulti mode.
 690  *
 691  * @param dev
 692  *   Pointer to Ethernet device structure.
 693  */
 694 static void
 695 mrvl_allmulticast_enable(struct rte_eth_dev *dev)
 696 {
 697         struct mrvl_priv *priv = dev->data->dev_private;
 698         int ret;
 699
 700         ret = pp2_ppio_set_mc_promisc(priv->ppio, 1);
 701         if (ret)
 702                 RTE_LOG(ERR, PMD, "Failed enable all-multicast mode\n");
 703 }
 704
 705 /**
 706  * DPDK callback to disable promiscuous mode.
 707  *
 708  * @param dev
 709  *   Pointer to Ethernet device structure.
 710  */
 711 static void
 712 mrvl_promiscuous_disable(struct rte_eth_dev *dev)
 713 {
 714         struct mrvl_priv *priv = dev->data->dev_private;
 715         int ret;
 716
 717         ret = pp2_ppio_set_promisc(priv->ppio, 0);
 718         if (ret)
 719                 RTE_LOG(ERR, PMD, "Failed to disable promiscuous mode\n");
 720 }
 721
 722 /**
 723  * DPDK callback to disable allmulticast mode.
 724  *
 725  * @param dev
 726  *   Pointer to Ethernet device structure.
 727  */
 728 static void
 729 mrvl_allmulticast_disable(struct rte_eth_dev *dev)
 730 {
 731         struct mrvl_priv *priv = dev->data->dev_private;
 732         int ret;
 733
 734         ret = pp2_ppio_set_mc_promisc(priv->ppio, 0);
 735         if (ret)
 736                 RTE_LOG(ERR, PMD, "Failed to disable all-multicast mode\n");
 737 }
 738
 739 /**
 740  * DPDK callback to remove a MAC address.
 741  *
 742  * @param dev
 743  *   Pointer to Ethernet device structure.
 744  * @param index
 745  *   MAC address index.
 746  */
 747 static void
 748 mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
 749 {
 750         struct mrvl_priv *priv = dev->data->dev_private;
 751         char buf[ETHER_ADDR_FMT_SIZE];
 752         int ret;
 753
 754         ret = pp2_ppio_remove_mac_addr(priv->ppio,
 755                                        dev->data->mac_addrs[index].addr_bytes);
 756         if (ret) {
 757                 ether_format_addr(buf, sizeof(buf),
 758                                   &dev->data->mac_addrs[index]);
 759                 RTE_LOG(ERR, PMD, "Failed to remove mac %s\n", buf);
 760         }
 761 }
 762
 763 /**
 764  * DPDK callback to add a MAC address.
 765  *
 766  * @param dev
 767  *   Pointer to Ethernet device structure.
 768  * @param mac_addr
 769  *   MAC address to register.
 770  * @param index
 771  *   MAC address index.
 772  * @param vmdq
 773  *   VMDq pool index to associate address with (unused).
 774  *
 775  * @return
 776  *   0 on success, negative error value otherwise.
 777  */
 778 static int
 779 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 780                   uint32_t index, uint32_t vmdq __rte_unused)
 781 {
 782         struct mrvl_priv *priv = dev->data->dev_private;
 783         char buf[ETHER_ADDR_FMT_SIZE];
 784         int ret;
 785
 786         if (index == 0)
 787                 /* For setting index 0, mrvl_mac_addr_set() should be used.*/
 788                 return -1;
 789
 790         /*
 791          * Maximum number of uc addresses can be tuned via kernel module mvpp2x
 792          * parameter uc_filter_max. Maximum number of mc addresses is then
 793          * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and
 794          * 21 respectively.
 795          *
 796          * If more than uc_filter_max uc addresses were added to filter list
 797          * then NIC will switch to promiscuous mode automatically.
 798          *
 799          * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses
 800          * were added to filter list then NIC will switch to all-multicast mode
 801          * automatically.
 802          */
 803         ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
 804         if (ret) {
 805                 ether_format_addr(buf, sizeof(buf), mac_addr);
 806                 RTE_LOG(ERR, PMD, "Failed to add mac %s\n", buf);
 807                 return -1;
 808         }
 809
 810         return 0;
 811 }
 812
 813 /**
 814  * DPDK callback to set the primary MAC address.
 815  *
 816  * @param dev
 817  *   Pointer to Ethernet device structure.
 818  * @param mac_addr
 819  *   MAC address to register.
 820  */
 821 static void
 822 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
 823 {
 824         struct mrvl_priv *priv = dev->data->dev_private;
 825         int ret;
 826
 827         ret = pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
 828         if (ret) {
 829                 char buf[ETHER_ADDR_FMT_SIZE];
 830                 ether_format_addr(buf, sizeof(buf), mac_addr);
 831                 RTE_LOG(ERR, PMD, "Failed to set mac to %s\n", buf);
 832         }
 833 }
 834
 835 /**
 836  * DPDK callback to get device statistics.
 837  *
 838  * @param dev
 839  *   Pointer to Ethernet device structure.
 840  * @param stats
 841  *   Stats structure output buffer.
 842  *
 843  * @return
 844  *   0 on success, negative error value otherwise.
 845  */
 846 static int
 847 mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 848 {
 849         struct mrvl_priv *priv = dev->data->dev_private;
 850         struct pp2_ppio_statistics ppio_stats;
 851         uint64_t drop_mac = 0;
 852         unsigned int i, idx, ret;
 853
 854         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 855                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
 856                 struct pp2_ppio_inq_statistics rx_stats;
 857
 858                 if (!rxq)
 859                         continue;
 860
 861                 idx = rxq->queue_id;
 862                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
 863                         RTE_LOG(ERR, PMD,
 864                                 "rx queue %d stats out of range (0 - %d)\n",
 865                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
 866                         continue;
 867                 }
 868
 869                 ret = pp2_ppio_inq_get_statistics(priv->ppio,
 870                                                   priv->rxq_map[idx].tc,
 871                                                   priv->rxq_map[idx].inq,
 872                                                   &rx_stats, 0);
 873                 if (unlikely(ret)) {
 874                         RTE_LOG(ERR, PMD,
 875                                 "Failed to update rx queue %d stats\n", idx);
 876                         break;
 877                 }
 878
 879                 stats->q_ibytes[idx] = rxq->bytes_recv;
 880                 stats->q_ipackets[idx] = rx_stats.enq_desc - rxq->drop_mac;
 881                 stats->q_errors[idx] = rx_stats.drop_early +
 882                                        rx_stats.drop_fullq +
 883                                        rx_stats.drop_bm +
 884                                        rxq->drop_mac;
 885                 stats->ibytes += rxq->bytes_recv;
 886                 drop_mac += rxq->drop_mac;
 887         }
 888
 889         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 890                 struct mrvl_txq *txq = dev->data->tx_queues[i];
 891                 struct pp2_ppio_outq_statistics tx_stats;
 892
 893                 if (!txq)
 894                         continue;
 895
 896                 idx = txq->queue_id;
 897                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
 898                         RTE_LOG(ERR, PMD,
 899                                 "tx queue %d stats out of range (0 - %d)\n",
 900                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
 901                 }
 902
 903                 ret = pp2_ppio_outq_get_statistics(priv->ppio, idx,
 904                                                    &tx_stats, 0);
 905                 if (unlikely(ret)) {
 906                         RTE_LOG(ERR, PMD,
 907                                 "Failed to update tx queue %d stats\n", idx);
 908                         break;
 909                 }
 910
 911                 stats->q_opackets[idx] = tx_stats.deq_desc;
 912                 stats->q_obytes[idx] = txq->bytes_sent;
 913                 stats->obytes += txq->bytes_sent;
 914         }
 915
 916         ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
 917         if (unlikely(ret)) {
 918                 RTE_LOG(ERR, PMD, "Failed to update port statistics\n");
 919                 return ret;
 920         }
 921
 922         stats->ipackets += ppio_stats.rx_packets - drop_mac;
 923         stats->opackets += ppio_stats.tx_packets;
 924         stats->imissed += ppio_stats.rx_fullq_dropped +
 925                           ppio_stats.rx_bm_dropped +
 926                           ppio_stats.rx_early_dropped +
 927                           ppio_stats.rx_fifo_dropped +
 928                           ppio_stats.rx_cls_dropped;
 929         stats->ierrors = drop_mac;
 930
 931         return 0;
 932 }
 933
 934 /**
 935  * DPDK callback to clear device statistics.
 936  *
 937  * @param dev
 938  *   Pointer to Ethernet device structure.
 939  */
 940 static void
 941 mrvl_stats_reset(struct rte_eth_dev *dev)
 942 {
 943         struct mrvl_priv *priv = dev->data->dev_private;
 944         int i;
 945
 946         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 947                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
 948
 949                 pp2_ppio_inq_get_statistics(priv->ppio, priv->rxq_map[i].tc,
 950                                             priv->rxq_map[i].inq, NULL, 1);
 951                 rxq->bytes_recv = 0;
 952                 rxq->drop_mac = 0;
 953         }
 954
 955         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 956                 struct mrvl_txq *txq = dev->data->tx_queues[i];
 957
 958                 pp2_ppio_outq_get_statistics(priv->ppio, i, NULL, 1);
 959                 txq->bytes_sent = 0;
 960         }
 961
 962         pp2_ppio_get_statistics(priv->ppio, NULL, 1);
 963 }
 964
 965 /**
 966  * DPDK callback to get information about the device.
 967  *
 968  * @param dev
 969  *   Pointer to Ethernet device structure (unused).
 970  * @param info
 971  *   Info structure output buffer.
 972  */
 973 static void
 974 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
 975                    struct rte_eth_dev_info *info)
 976 {
 977         info->speed_capa = ETH_LINK_SPEED_10M |
 978                            ETH_LINK_SPEED_100M |
 979                            ETH_LINK_SPEED_1G |
 980                            ETH_LINK_SPEED_10G;
 981
 982         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
 983         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
 984         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
 985
 986         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
 987         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
 988         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
 989
 990         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
 991         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
 992         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
 993
 994         info->rx_offload_capa = DEV_RX_OFFLOAD_JUMBO_FRAME |
 995                                 DEV_RX_OFFLOAD_VLAN_FILTER |
 996                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
 997                                 DEV_RX_OFFLOAD_UDP_CKSUM |
 998                                 DEV_RX_OFFLOAD_TCP_CKSUM;
 999
1000         info->tx_offload_capa = DEV_TX_OFFLOAD_IPV4_CKSUM |
1001                                 DEV_TX_OFFLOAD_UDP_CKSUM |
1002                                 DEV_TX_OFFLOAD_TCP_CKSUM;
1003
1004         info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1005                                        ETH_RSS_NONFRAG_IPV4_TCP |
1006                                        ETH_RSS_NONFRAG_IPV4_UDP;
1007
1008         /* By default packets are dropped if no descriptors are available */
1009         info->default_rxconf.rx_drop_en = 1;
1010
1011         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
1012 }
1013
1014 /**
1015  * Return supported packet types.
1016  *
1017  * @param dev
1018  *   Pointer to Ethernet device structure (unused).
1019  *
1020  * @return
1021  *   Const pointer to the table with supported packet types.
1022  */
1023 static const uint32_t *
1024 mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1025 {
1026         static const uint32_t ptypes[] = {
1027                 RTE_PTYPE_L2_ETHER,
1028                 RTE_PTYPE_L3_IPV4,
1029                 RTE_PTYPE_L3_IPV4_EXT,
1030                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1031                 RTE_PTYPE_L3_IPV6,
1032                 RTE_PTYPE_L3_IPV6_EXT,
1033                 RTE_PTYPE_L2_ETHER_ARP,
1034                 RTE_PTYPE_L4_TCP,
1035                 RTE_PTYPE_L4_UDP
1036         };
1037
1038         return ptypes;
1039 }
1040
1041 /**
1042  * DPDK callback to get information about specific receive queue.
1043  *
1044  * @param dev
1045  *   Pointer to Ethernet device structure.
1046  * @param rx_queue_id
1047  *   Receive queue index.
1048  * @param qinfo
1049  *   Receive queue information structure.
1050  */
1051 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1052                               struct rte_eth_rxq_info *qinfo)
1053 {
1054         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
1055         struct mrvl_priv *priv = dev->data->dev_private;
1056         int inq = priv->rxq_map[rx_queue_id].inq;
1057         int tc = priv->rxq_map[rx_queue_id].tc;
1058         struct pp2_ppio_tc_params *tc_params =
1059                 &priv->ppio_params.inqs_params.tcs_params[tc];
1060
1061         qinfo->mp = q->mp;
1062         qinfo->nb_desc = tc_params->inqs_params[inq].size;
1063 }
1064
1065 /**
1066  * DPDK callback to get information about specific transmit queue.
1067  *
1068  * @param dev
1069  *   Pointer to Ethernet device structure.
1070  * @param tx_queue_id
1071  *   Transmit queue index.
1072  * @param qinfo
1073  *   Transmit queue information structure.
1074  */
1075 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1076                               struct rte_eth_txq_info *qinfo)
1077 {
1078         struct mrvl_priv *priv = dev->data->dev_private;
1079
1080         qinfo->nb_desc =
1081                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
1082 }
1083
1084 /**
1085  * DPDK callback to Configure a VLAN filter.
1086  *
1087  * @param dev
1088  *   Pointer to Ethernet device structure.
1089  * @param vlan_id
1090  *   VLAN ID to filter.
1091  * @param on
1092  *   Toggle filter.
1093  *
1094  * @return
1095  *   0 on success, negative error value otherwise.
1096  */
1097 static int
1098 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1099 {
1100         struct mrvl_priv *priv = dev->data->dev_private;
1101
1102         return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) :
1103                     pp2_ppio_remove_vlan(priv->ppio, vlan_id);
1104 }
1105
1106 /**
1107  * Release buffers to hardware bpool (buffer-pool)
1108  *
1109  * @param rxq
1110  *   Receive queue pointer.
1111  * @param num
1112  *   Number of buffers to release to bpool.
1113  *
1114  * @return
1115  *   0 on success, negative error value otherwise.
1116  */
1117 static int
1118 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
1119 {
1120         struct buff_release_entry entries[MRVL_PP2_TXD_MAX];
1121         struct rte_mbuf *mbufs[MRVL_PP2_TXD_MAX];
1122         int i, ret;
1123         unsigned int core_id = rte_lcore_id();
1124         struct pp2_hif *hif = hifs[core_id];
1125         struct pp2_bpool *bpool = rxq->priv->bpool;
1126
1127         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
1128         if (ret)
1129                 return ret;
1130
1131         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
1132                 cookie_addr_high =
1133                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
1134
1135         for (i = 0; i < num; i++) {
1136                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
1137                         != cookie_addr_high) {
1138                         RTE_LOG(ERR, PMD,
1139                                 "mbuf virtual addr high 0x%lx out of range\n",
1140                                 (uint64_t)mbufs[i] >> 32);
1141                         goto out;
1142                 }
1143
1144                 entries[i].buff.addr =
1145                         rte_mbuf_data_iova_default(mbufs[i]);
1146                 entries[i].buff.cookie = (pp2_cookie_t)(uint64_t)mbufs[i];
1147                 entries[i].bpool = bpool;
1148         }
1149
1150         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
1151         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
1152
1153         if (i != num)
1154                 goto out;
1155
1156         return 0;
1157 out:
1158         for (; i < num; i++)
1159                 rte_pktmbuf_free(mbufs[i]);
1160
1161         return -1;
1162 }
1163
1164 /**
1165  * DPDK callback to configure the receive queue.
1166  *
1167  * @param dev
1168  *   Pointer to Ethernet device structure.
1169  * @param idx
1170  *   RX queue index.
1171  * @param desc
1172  *   Number of descriptors to configure in queue.
1173  * @param socket
1174  *   NUMA socket on which memory must be allocated.
1175  * @param conf
1176  *   Thresholds parameters (unused_).
1177  * @param mp
1178  *   Memory pool for buffer allocations.
1179  *
1180  * @return
1181  *   0 on success, negative error value otherwise.
1182  */
1183 static int
1184 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1185                     unsigned int socket,
1186                     const struct rte_eth_rxconf *conf __rte_unused,
1187                     struct rte_mempool *mp)
1188 {
1189         struct mrvl_priv *priv = dev->data->dev_private;
1190         struct mrvl_rxq *rxq;
1191         uint32_t min_size,
1192                  max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1193         int ret, tc, inq;
1194
1195         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
1196                 /*
1197                  * Unknown TC mapping, mapping will not have a correct queue.
1198                  */
1199                 RTE_LOG(ERR, PMD, "Unknown TC mapping for queue %hu eth%hhu\n",
1200                         idx, priv->ppio_id);
1201                 return -EFAULT;
1202         }
1203
1204         min_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM -
1205                    MRVL_PKT_EFFEC_OFFS;
1206         if (min_size < max_rx_pkt_len) {
1207                 RTE_LOG(ERR, PMD,
1208                         "Mbuf size must be increased to %u bytes to hold up to %u bytes of data.\n",
1209                         max_rx_pkt_len + RTE_PKTMBUF_HEADROOM +
1210                         MRVL_PKT_EFFEC_OFFS,
1211                         max_rx_pkt_len);
1212                 return -EINVAL;
1213         }
1214
1215         if (dev->data->rx_queues[idx]) {
1216                 rte_free(dev->data->rx_queues[idx]);
1217                 dev->data->rx_queues[idx] = NULL;
1218         }
1219
1220         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
1221         if (!rxq)
1222                 return -ENOMEM;
1223
1224         rxq->priv = priv;
1225         rxq->mp = mp;
1226         rxq->cksum_enabled = dev->data->dev_conf.rxmode.hw_ip_checksum;
1227         rxq->queue_id = idx;
1228         rxq->port_id = dev->data->port_id;
1229         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
1230
1231         tc = priv->rxq_map[rxq->queue_id].tc,
1232         inq = priv->rxq_map[rxq->queue_id].inq;
1233         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
1234                 desc;
1235
1236         ret = mrvl_fill_bpool(rxq, desc);
1237         if (ret) {
1238                 rte_free(rxq);
1239                 return ret;
1240         }
1241
1242         priv->bpool_init_size += desc;
1243
1244         dev->data->rx_queues[idx] = rxq;
1245
1246         return 0;
1247 }
1248
1249 /**
1250  * DPDK callback to release the receive queue.
1251  *
1252  * @param rxq
1253  *   Generic receive queue pointer.
1254  */
1255 static void
1256 mrvl_rx_queue_release(void *rxq)
1257 {
1258         struct mrvl_rxq *q = rxq;
1259         struct pp2_ppio_tc_params *tc_params;
1260         int i, num, tc, inq;
1261
1262         if (!q)
1263                 return;
1264
1265         tc = q->priv->rxq_map[q->queue_id].tc;
1266         inq = q->priv->rxq_map[q->queue_id].inq;
1267         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
1268         num = tc_params->inqs_params[inq].size;
1269         for (i = 0; i < num; i++) {
1270                 struct pp2_buff_inf inf;
1271                 uint64_t addr;
1272
1273                 pp2_bpool_get_buff(hifs[rte_lcore_id()], q->priv->bpool, &inf);
1274                 addr = cookie_addr_high | inf.cookie;
1275                 rte_pktmbuf_free((struct rte_mbuf *)addr);
1276         }
1277
1278         rte_free(q);
1279 }
1280
1281 /**
1282  * DPDK callback to configure the transmit queue.
1283  *
1284  * @param dev
1285  *   Pointer to Ethernet device structure.
1286  * @param idx
1287  *   Transmit queue index.
1288  * @param desc
1289  *   Number of descriptors to configure in the queue.
1290  * @param socket
1291  *   NUMA socket on which memory must be allocated.
1292  * @param conf
1293  *   Thresholds parameters (unused).
1294  *
1295  * @return
1296  *   0 on success, negative error value otherwise.
1297  */
1298 static int
1299 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1300                     unsigned int socket,
1301                     const struct rte_eth_txconf *conf __rte_unused)
1302 {
1303         struct mrvl_priv *priv = dev->data->dev_private;
1304         struct mrvl_txq *txq;
1305
1306         if (dev->data->tx_queues[idx]) {
1307                 rte_free(dev->data->tx_queues[idx]);
1308                 dev->data->tx_queues[idx] = NULL;
1309         }
1310
1311         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
1312         if (!txq)
1313                 return -ENOMEM;
1314
1315         txq->priv = priv;
1316         txq->queue_id = idx;
1317         txq->port_id = dev->data->port_id;
1318         dev->data->tx_queues[idx] = txq;
1319
1320         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
1321         priv->ppio_params.outqs_params.outqs_params[idx].weight = 1;
1322
1323         return 0;
1324 }
1325
1326 /**
1327  * DPDK callback to release the transmit queue.
1328  *
1329  * @param txq
1330  *   Generic transmit queue pointer.
1331  */
1332 static void
1333 mrvl_tx_queue_release(void *txq)
1334 {
1335         struct mrvl_txq *q = txq;
1336
1337         if (!q)
1338                 return;
1339
1340         rte_free(q);
1341 }
1342
1343 /**
1344  * Update RSS hash configuration
1345  *
1346  * @param dev
1347  *   Pointer to Ethernet device structure.
1348  * @param rss_conf
1349  *   Pointer to RSS configuration.
1350  *
1351  * @return
1352  *   0 on success, negative error value otherwise.
1353  */
1354 static int
1355 mrvl_rss_hash_update(struct rte_eth_dev *dev,
1356                      struct rte_eth_rss_conf *rss_conf)
1357 {
1358         struct mrvl_priv *priv = dev->data->dev_private;
1359
1360         return mrvl_configure_rss(priv, rss_conf);
1361 }
1362
1363 /**
1364  * DPDK callback to get RSS hash configuration.
1365  *
1366  * @param dev
1367  *   Pointer to Ethernet device structure.
1368  * @rss_conf
1369  *   Pointer to RSS configuration.
1370  *
1371  * @return
1372  *   Always 0.
1373  */
1374 static int
1375 mrvl_rss_hash_conf_get(struct rte_eth_dev *dev,
1376                        struct rte_eth_rss_conf *rss_conf)
1377 {
1378         struct mrvl_priv *priv = dev->data->dev_private;
1379         enum pp2_ppio_hash_type hash_type =
1380                 priv->ppio_params.inqs_params.hash_type;
1381
1382         rss_conf->rss_key = NULL;
1383
1384         if (hash_type == PP2_PPIO_HASH_T_NONE)
1385                 rss_conf->rss_hf = 0;
1386         else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE)
1387                 rss_conf->rss_hf = ETH_RSS_IPV4;
1388         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp)
1389                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP;
1390         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp)
1391                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP;
1392
1393         return 0;
1394 }
1395
1396 static const struct eth_dev_ops mrvl_ops = {
1397         .dev_configure = mrvl_dev_configure,
1398         .dev_start = mrvl_dev_start,
1399         .dev_stop = mrvl_dev_stop,
1400         .dev_set_link_up = mrvl_dev_set_link_up,
1401         .dev_set_link_down = mrvl_dev_set_link_down,
1402         .dev_close = mrvl_dev_close,
1403         .link_update = mrvl_link_update,
1404         .promiscuous_enable = mrvl_promiscuous_enable,
1405         .allmulticast_enable = mrvl_allmulticast_enable,
1406         .promiscuous_disable = mrvl_promiscuous_disable,
1407         .allmulticast_disable = mrvl_allmulticast_disable,
1408         .mac_addr_remove = mrvl_mac_addr_remove,
1409         .mac_addr_add = mrvl_mac_addr_add,
1410         .mac_addr_set = mrvl_mac_addr_set,
1411         .mtu_set = mrvl_mtu_set,
1412         .stats_get = mrvl_stats_get,
1413         .stats_reset = mrvl_stats_reset,
1414         .dev_infos_get = mrvl_dev_infos_get,
1415         .dev_supported_ptypes_get = mrvl_dev_supported_ptypes_get,
1416         .rxq_info_get = mrvl_rxq_info_get,
1417         .txq_info_get = mrvl_txq_info_get,
1418         .vlan_filter_set = mrvl_vlan_filter_set,
1419         .rx_queue_setup = mrvl_rx_queue_setup,
1420         .rx_queue_release = mrvl_rx_queue_release,
1421         .tx_queue_setup = mrvl_tx_queue_setup,
1422         .tx_queue_release = mrvl_tx_queue_release,
1423         .rss_hash_update = mrvl_rss_hash_update,
1424         .rss_hash_conf_get = mrvl_rss_hash_conf_get,
1425 };
1426
1427 /**
1428  * Return packet type information and l3/l4 offsets.
1429  *
1430  * @param desc
1431  *   Pointer to the received packet descriptor.
1432  * @param l3_offset
1433  *   l3 packet offset.
1434  * @param l4_offset
1435  *   l4 packet offset.
1436  *
1437  * @return
1438  *   Packet type information.
1439  */
1440 static inline uint64_t
1441 mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
1442                                     uint8_t *l3_offset, uint8_t *l4_offset)
1443 {
1444         enum pp2_inq_l3_type l3_type;
1445         enum pp2_inq_l4_type l4_type;
1446         uint64_t packet_type;
1447
1448         pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
1449         pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
1450
1451         packet_type = RTE_PTYPE_L2_ETHER;
1452
1453         switch (l3_type) {
1454         case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
1455                 packet_type |= RTE_PTYPE_L3_IPV4;
1456                 break;
1457         case PP2_INQ_L3_TYPE_IPV4_OK:
1458                 packet_type |= RTE_PTYPE_L3_IPV4_EXT;
1459                 break;
1460         case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO:
1461                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
1462                 break;
1463         case PP2_INQ_L3_TYPE_IPV6_NO_EXT:
1464                 packet_type |= RTE_PTYPE_L3_IPV6;
1465                 break;
1466         case PP2_INQ_L3_TYPE_IPV6_EXT:
1467                 packet_type |= RTE_PTYPE_L3_IPV6_EXT;
1468                 break;
1469         case PP2_INQ_L3_TYPE_ARP:
1470                 packet_type |= RTE_PTYPE_L2_ETHER_ARP;
1471                 /*
1472                  * In case of ARP l4_offset is set to wrong value.
1473                  * Set it to proper one so that later on mbuf->l3_len can be
1474                  * calculated subtracting l4_offset and l3_offset.
1475                  */
1476                 *l4_offset = *l3_offset + MRVL_ARP_LENGTH;
1477                 break;
1478         default:
1479                 RTE_LOG(DEBUG, PMD, "Failed to recognise l3 packet type\n");
1480                 break;
1481         }
1482
1483         switch (l4_type) {
1484         case PP2_INQ_L4_TYPE_TCP:
1485                 packet_type |= RTE_PTYPE_L4_TCP;
1486                 break;
1487         case PP2_INQ_L4_TYPE_UDP:
1488                 packet_type |= RTE_PTYPE_L4_UDP;
1489                 break;
1490         default:
1491                 RTE_LOG(DEBUG, PMD, "Failed to recognise l4 packet type\n");
1492                 break;
1493         }
1494
1495         return packet_type;
1496 }
1497
1498 /**
1499  * Get offload information from the received packet descriptor.
1500  *
1501  * @param desc
1502  *   Pointer to the received packet descriptor.
1503  *
1504  * @return
1505  *   Mbuf offload flags.
1506  */
1507 static inline uint64_t
1508 mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc)
1509 {
1510         uint64_t flags;
1511         enum pp2_inq_desc_status status;
1512
1513         status = pp2_ppio_inq_desc_get_l3_pkt_error(desc);
1514         if (unlikely(status != PP2_DESC_ERR_OK))
1515                 flags = PKT_RX_IP_CKSUM_BAD;
1516         else
1517                 flags = PKT_RX_IP_CKSUM_GOOD;
1518
1519         status = pp2_ppio_inq_desc_get_l4_pkt_error(desc);
1520         if (unlikely(status != PP2_DESC_ERR_OK))
1521                 flags |= PKT_RX_L4_CKSUM_BAD;
1522         else
1523                 flags |= PKT_RX_L4_CKSUM_GOOD;
1524
1525         return flags;
1526 }
1527
1528 /**
1529  * DPDK callback for receive.
1530  *
1531  * @param rxq
1532  *   Generic pointer to the receive queue.
1533  * @param rx_pkts
1534  *   Array to store received packets.
1535  * @param nb_pkts
1536  *   Maximum number of packets in array.
1537  *
1538  * @return
1539  *   Number of packets successfully received.
1540  */
1541 static uint16_t
1542 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
1543 {
1544         struct mrvl_rxq *q = rxq;
1545         struct pp2_ppio_desc descs[nb_pkts];
1546         struct pp2_bpool *bpool;
1547         int i, ret, rx_done = 0;
1548         int num;
1549         unsigned int core_id = rte_lcore_id();
1550
1551         if (unlikely(!q->priv->ppio))
1552                 return 0;
1553
1554         bpool = q->priv->bpool;
1555
1556         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
1557                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
1558         if (unlikely(ret < 0)) {
1559                 RTE_LOG(ERR, PMD, "Failed to receive packets\n");
1560                 return 0;
1561         }
1562         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
1563
1564         for (i = 0; i < nb_pkts; i++) {
1565                 struct rte_mbuf *mbuf;
1566                 uint8_t l3_offset, l4_offset;
1567                 enum pp2_inq_desc_status status;
1568                 uint64_t addr;
1569
1570                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
1571                         struct pp2_ppio_desc *pref_desc;
1572                         u64 pref_addr;
1573
1574                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
1575                         pref_addr = cookie_addr_high |
1576                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
1577                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
1578                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
1579                 }
1580
1581                 addr = cookie_addr_high |
1582                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
1583                 mbuf = (struct rte_mbuf *)addr;
1584                 rte_pktmbuf_reset(mbuf);
1585
1586                 /* drop packet in case of mac, overrun or resource error */
1587                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
1588                 if (unlikely(status != PP2_DESC_ERR_OK)) {
1589                         struct pp2_buff_inf binf = {
1590                                 .addr = rte_mbuf_data_iova_default(mbuf),
1591                                 .cookie = (pp2_cookie_t)(uint64_t)mbuf,
1592                         };
1593
1594                         pp2_bpool_put_buff(hifs[core_id], bpool, &binf);
1595                         mrvl_port_bpool_size
1596                                 [bpool->pp2_id][bpool->id][core_id]++;
1597                         q->drop_mac++;
1598                         continue;
1599                 }
1600
1601                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
1602                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
1603                 mbuf->data_len = mbuf->pkt_len;
1604                 mbuf->port = q->port_id;
1605                 mbuf->packet_type =
1606                         mrvl_desc_to_packet_type_and_offset(&descs[i],
1607                                                             &l3_offset,
1608                                                             &l4_offset);
1609                 mbuf->l2_len = l3_offset;
1610                 mbuf->l3_len = l4_offset - l3_offset;
1611
1612                 if (likely(q->cksum_enabled))
1613                         mbuf->ol_flags = mrvl_desc_to_ol_flags(&descs[i]);
1614
1615                 rx_pkts[rx_done++] = mbuf;
1616                 q->bytes_recv += mbuf->pkt_len;
1617         }
1618
1619         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
1620                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
1621
1622                 if (unlikely(num <= q->priv->bpool_min_size ||
1623                              (!rx_done && num < q->priv->bpool_init_size))) {
1624                         ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE);
1625                         if (ret)
1626                                 RTE_LOG(ERR, PMD, "Failed to fill bpool\n");
1627                 } else if (unlikely(num > q->priv->bpool_max_size)) {
1628                         int i;
1629                         int pkt_to_remove = num - q->priv->bpool_init_size;
1630                         struct rte_mbuf *mbuf;
1631                         struct pp2_buff_inf buff;
1632
1633                         RTE_LOG(DEBUG, PMD,
1634                                 "\nport-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)\n",
1635                                 bpool->pp2_id, q->priv->ppio->port_id,
1636                                 bpool->id, pkt_to_remove, num,
1637                                 q->priv->bpool_init_size);
1638
1639                         for (i = 0; i < pkt_to_remove; i++) {
1640                                 pp2_bpool_get_buff(hifs[core_id], bpool, &buff);
1641                                 mbuf = (struct rte_mbuf *)
1642                                         (cookie_addr_high | buff.cookie);
1643                                 rte_pktmbuf_free(mbuf);
1644                         }
1645                         mrvl_port_bpool_size
1646                                 [bpool->pp2_id][bpool->id][core_id] -=
1647                                                                 pkt_to_remove;
1648                 }
1649                 rte_spinlock_unlock(&q->priv->lock);
1650         }
1651
1652         return rx_done;
1653 }
1654
1655 /**
1656  * Prepare offload information.
1657  *
1658  * @param ol_flags
1659  *   Offload flags.
1660  * @param packet_type
1661  *   Packet type bitfield.
1662  * @param l3_type
1663  *   Pointer to the pp2_ouq_l3_type structure.
1664  * @param l4_type
1665  *   Pointer to the pp2_outq_l4_type structure.
1666  * @param gen_l3_cksum
1667  *   Will be set to 1 in case l3 checksum is computed.
1668  * @param l4_cksum
1669  *   Will be set to 1 in case l4 checksum is computed.
1670  *
1671  * @return
1672  *   0 on success, negative error value otherwise.
1673  */
1674 static inline int
1675 mrvl_prepare_proto_info(uint64_t ol_flags, uint32_t packet_type,
1676                         enum pp2_outq_l3_type *l3_type,
1677                         enum pp2_outq_l4_type *l4_type,
1678                         int *gen_l3_cksum,
1679                         int *gen_l4_cksum)
1680 {
1681         /*
1682          * Based on ol_flags prepare information
1683          * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor
1684          * for offloading.
1685          */
1686         if (ol_flags & PKT_TX_IPV4) {
1687                 *l3_type = PP2_OUTQ_L3_TYPE_IPV4;
1688                 *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0;
1689         } else if (ol_flags & PKT_TX_IPV6) {
1690                 *l3_type = PP2_OUTQ_L3_TYPE_IPV6;
1691                 /* no checksum for ipv6 header */
1692                 *gen_l3_cksum = 0;
1693         } else {
1694                 /* if something different then stop processing */
1695                 return -1;
1696         }
1697
1698         ol_flags &= PKT_TX_L4_MASK;
1699         if ((packet_type & RTE_PTYPE_L4_TCP) &&
1700             ol_flags == PKT_TX_TCP_CKSUM) {
1701                 *l4_type = PP2_OUTQ_L4_TYPE_TCP;
1702                 *gen_l4_cksum = 1;
1703         } else if ((packet_type & RTE_PTYPE_L4_UDP) &&
1704                    ol_flags == PKT_TX_UDP_CKSUM) {
1705                 *l4_type = PP2_OUTQ_L4_TYPE_UDP;
1706                 *gen_l4_cksum = 1;
1707         } else {
1708                 *l4_type = PP2_OUTQ_L4_TYPE_OTHER;
1709                 /* no checksum for other type */
1710                 *gen_l4_cksum = 0;
1711         }
1712
1713         return 0;
1714 }
1715
1716 /**
1717  * Release already sent buffers to bpool (buffer-pool).
1718  *
1719  * @param ppio
1720  *   Pointer to the port structure.
1721  * @param hif
1722  *   Pointer to the MUSDK hardware interface.
1723  * @param sq
1724  *   Pointer to the shadow queue.
1725  * @param qid
1726  *   Queue id number.
1727  * @param force
1728  *   Force releasing packets.
1729  */
1730 static inline void
1731 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
1732                        struct mrvl_shadow_txq *sq, int qid, int force)
1733 {
1734         struct buff_release_entry *entry;
1735         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
1736         int i, core_id = rte_lcore_id();
1737
1738         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
1739
1740         sq->num_to_release += nb_done;
1741
1742         if (likely(!force &&
1743                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
1744                 return;
1745
1746         nb_done = sq->num_to_release;
1747         sq->num_to_release = 0;
1748
1749         for (i = 0; i < nb_done; i++) {
1750                 entry = &sq->ent[sq->tail + num];
1751                 if (unlikely(!entry->buff.addr)) {
1752                         RTE_LOG(ERR, PMD,
1753                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!\n",
1754                                 sq->tail, (u64)entry->buff.cookie,
1755                                 (u64)entry->buff.addr);
1756                         skip_bufs = 1;
1757                         goto skip;
1758                 }
1759
1760                 if (unlikely(!entry->bpool)) {
1761                         struct rte_mbuf *mbuf;
1762
1763                         mbuf = (struct rte_mbuf *)
1764                                (cookie_addr_high | entry->buff.cookie);
1765                         rte_pktmbuf_free(mbuf);
1766                         skip_bufs = 1;
1767                         goto skip;
1768                 }
1769
1770                 mrvl_port_bpool_size
1771                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
1772                 num++;
1773                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
1774                         goto skip;
1775                 continue;
1776 skip:
1777                 if (likely(num))
1778                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
1779                 num += skip_bufs;
1780                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
1781                 sq->size -= num;
1782                 num = 0;
1783         }
1784
1785         if (likely(num)) {
1786                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
1787                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
1788                 sq->size -= num;
1789         }
1790 }
1791
1792 /**
1793  * DPDK callback for transmit.
1794  *
1795  * @param txq
1796  *   Generic pointer transmit queue.
1797  * @param tx_pkts
1798  *   Packets to transmit.
1799  * @param nb_pkts
1800  *   Number of packets in array.
1801  *
1802  * @return
1803  *   Number of packets successfully transmitted.
1804  */
1805 static uint16_t
1806 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1807 {
1808         struct mrvl_txq *q = txq;
1809         struct mrvl_shadow_txq *sq = &shadow_txqs[q->port_id][rte_lcore_id()];
1810         struct pp2_hif *hif = hifs[rte_lcore_id()];
1811         struct pp2_ppio_desc descs[nb_pkts];
1812         int i, ret, bytes_sent = 0;
1813         uint16_t num, sq_free_size;
1814         uint64_t addr;
1815
1816         if (unlikely(!q->priv->ppio))
1817                 return 0;
1818
1819         if (sq->size)
1820                 mrvl_free_sent_buffers(q->priv->ppio, hif, sq, q->queue_id, 0);
1821
1822         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
1823         if (unlikely(nb_pkts > sq_free_size)) {
1824                 RTE_LOG(DEBUG, PMD,
1825                         "No room in shadow queue for %d packets! %d packets will be sent.\n",
1826                         nb_pkts, sq_free_size);
1827                 nb_pkts = sq_free_size;
1828         }
1829
1830         for (i = 0; i < nb_pkts; i++) {
1831                 struct rte_mbuf *mbuf = tx_pkts[i];
1832                 int gen_l3_cksum, gen_l4_cksum;
1833                 enum pp2_outq_l3_type l3_type;
1834                 enum pp2_outq_l4_type l4_type;
1835
1836                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
1837                         struct rte_mbuf *pref_pkt_hdr;
1838
1839                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
1840                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
1841                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
1842                 }
1843
1844                 sq->ent[sq->head].buff.cookie = (pp2_cookie_t)(uint64_t)mbuf;
1845                 sq->ent[sq->head].buff.addr =
1846                         rte_mbuf_data_iova_default(mbuf);
1847                 sq->ent[sq->head].bpool =
1848                         (unlikely(mbuf->port == 0xff || mbuf->refcnt > 1)) ?
1849                          NULL : mrvl_port_to_bpool_lookup[mbuf->port];
1850                 sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
1851                 sq->size++;
1852
1853                 pp2_ppio_outq_desc_reset(&descs[i]);
1854                 pp2_ppio_outq_desc_set_phys_addr(&descs[i],
1855                                                  rte_pktmbuf_iova(mbuf));
1856                 pp2_ppio_outq_desc_set_pkt_offset(&descs[i], 0);
1857                 pp2_ppio_outq_desc_set_pkt_len(&descs[i],
1858                                                rte_pktmbuf_pkt_len(mbuf));
1859
1860                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
1861                 /*
1862                  * in case unsupported ol_flags were passed
1863                  * do not update descriptor offload information
1864                  */
1865                 ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type,
1866                                               &l3_type, &l4_type, &gen_l3_cksum,
1867                                               &gen_l4_cksum);
1868                 if (unlikely(ret))
1869                         continue;
1870
1871                 pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type,
1872                                                   mbuf->l2_len,
1873                                                   mbuf->l2_len + mbuf->l3_len,
1874                                                   gen_l3_cksum, gen_l4_cksum);
1875         }
1876
1877         num = nb_pkts;
1878         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
1879         /* number of packets that were not sent */
1880         if (unlikely(num > nb_pkts)) {
1881                 for (i = nb_pkts; i < num; i++) {
1882                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
1883                                 MRVL_PP2_TX_SHADOWQ_MASK;
1884                         addr = cookie_addr_high | sq->ent[sq->head].buff.cookie;
1885                         bytes_sent -=
1886                                 rte_pktmbuf_pkt_len((struct rte_mbuf *)addr);
1887                 }
1888                 sq->size -= num - nb_pkts;
1889         }
1890
1891         q->bytes_sent += bytes_sent;
1892
1893         return nb_pkts;
1894 }
1895
1896 /**
1897  * Initialize packet processor.
1898  *
1899  * @return
1900  *   0 on success, negative error value otherwise.
1901  */
1902 static int
1903 mrvl_init_pp2(void)
1904 {
1905         struct pp2_init_params init_params;
1906
1907         memset(&init_params, 0, sizeof(init_params));
1908         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
1909         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
1910         init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED;
1911
1912         return pp2_init(&init_params);
1913 }
1914
1915 /**
1916  * Deinitialize packet processor.
1917  *
1918  * @return
1919  *   0 on success, negative error value otherwise.
1920  */
1921 static void
1922 mrvl_deinit_pp2(void)
1923 {
1924         pp2_deinit();
1925 }
1926
1927 /**
1928  * Create private device structure.
1929  *
1930  * @param dev_name
1931  *   Pointer to the port name passed in the initialization parameters.
1932  *
1933  * @return
1934  *   Pointer to the newly allocated private device structure.
1935  */
1936 static struct mrvl_priv *
1937 mrvl_priv_create(const char *dev_name)
1938 {
1939         struct pp2_bpool_params bpool_params;
1940         char match[MRVL_MATCH_LEN];
1941         struct mrvl_priv *priv;
1942         int ret, bpool_bit;
1943
1944         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
1945         if (!priv)
1946                 return NULL;
1947
1948         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
1949                                        &priv->pp_id, &priv->ppio_id);
1950         if (ret)
1951                 goto out_free_priv;
1952
1953         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
1954                                      PP2_BPOOL_NUM_POOLS);
1955         if (bpool_bit < 0)
1956                 goto out_free_priv;
1957         priv->bpool_bit = bpool_bit;
1958
1959         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
1960                  priv->bpool_bit);
1961         memset(&bpool_params, 0, sizeof(bpool_params));
1962         bpool_params.match = match;
1963         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
1964         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
1965         if (ret)
1966                 goto out_clear_bpool_bit;
1967
1968         priv->ppio_params.type = PP2_PPIO_T_NIC;
1969         rte_spinlock_init(&priv->lock);
1970
1971         return priv;
1972 out_clear_bpool_bit:
1973         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
1974 out_free_priv:
1975         rte_free(priv);
1976         return NULL;
1977 }
1978
1979 /**
1980  * Create device representing Ethernet port.
1981  *
1982  * @param name
1983  *   Pointer to the port's name.
1984  *
1985  * @return
1986  *   0 on success, negative error value otherwise.
1987  */
1988 static int
1989 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
1990 {
1991         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
1992         struct rte_eth_dev *eth_dev;
1993         struct mrvl_priv *priv;
1994         struct ifreq req;
1995
1996         eth_dev = rte_eth_dev_allocate(name);
1997         if (!eth_dev)
1998                 return -ENOMEM;
1999
2000         priv = mrvl_priv_create(name);
2001         if (!priv) {
2002                 ret = -ENOMEM;
2003                 goto out_free_dev;
2004         }
2005
2006         eth_dev->data->mac_addrs =
2007                 rte_zmalloc("mac_addrs",
2008                             ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
2009         if (!eth_dev->data->mac_addrs) {
2010                 RTE_LOG(ERR, PMD, "Failed to allocate space for eth addrs\n");
2011                 ret = -ENOMEM;
2012                 goto out_free_priv;
2013         }
2014
2015         memset(&req, 0, sizeof(req));
2016         strcpy(req.ifr_name, name);
2017         ret = ioctl(fd, SIOCGIFHWADDR, &req);
2018         if (ret)
2019                 goto out_free_mac;
2020
2021         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
2022                req.ifr_addr.sa_data, ETHER_ADDR_LEN);
2023
2024         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
2025         eth_dev->tx_pkt_burst = mrvl_tx_pkt_burst;
2026         eth_dev->data->dev_private = priv;
2027         eth_dev->device = &vdev->device;
2028         eth_dev->dev_ops = &mrvl_ops;
2029
2030         return 0;
2031 out_free_mac:
2032         rte_free(eth_dev->data->mac_addrs);
2033 out_free_dev:
2034         rte_eth_dev_release_port(eth_dev);
2035 out_free_priv:
2036         rte_free(priv);
2037
2038         return ret;
2039 }
2040
2041 /**
2042  * Cleanup previously created device representing Ethernet port.
2043  *
2044  * @param name
2045  *   Pointer to the port name.
2046  */
2047 static void
2048 mrvl_eth_dev_destroy(const char *name)
2049 {
2050         struct rte_eth_dev *eth_dev;
2051         struct mrvl_priv *priv;
2052
2053         eth_dev = rte_eth_dev_allocated(name);
2054         if (!eth_dev)
2055                 return;
2056
2057         priv = eth_dev->data->dev_private;
2058         pp2_bpool_deinit(priv->bpool);
2059         rte_free(priv);
2060         rte_free(eth_dev->data->mac_addrs);
2061         rte_eth_dev_release_port(eth_dev);
2062 }
2063
2064 /**
2065  * Callback used by rte_kvargs_process() during argument parsing.
2066  *
2067  * @param key
2068  *   Pointer to the parsed key (unused).
2069  * @param value
2070  *   Pointer to the parsed value.
2071  * @param extra_args
2072  *   Pointer to the extra arguments which contains address of the
2073  *   table of pointers to parsed interface names.
2074  *
2075  * @return
2076  *   Always 0.
2077  */
2078 static int
2079 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
2080                  void *extra_args)
2081 {
2082         const char **ifnames = extra_args;
2083
2084         ifnames[mrvl_ports_nb++] = value;
2085
2086         return 0;
2087 }
2088
2089 /**
2090  * Initialize per-lcore MUSDK hardware interfaces (hifs).
2091  *
2092  * @return
2093  *   0 on success, negative error value otherwise.
2094  */
2095 static int
2096 mrvl_init_hifs(void)
2097 {
2098         struct pp2_hif_params params;
2099         char match[MRVL_MATCH_LEN];
2100         int i, ret;
2101
2102         RTE_LCORE_FOREACH(i) {
2103                 ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
2104                 if (ret < 0)
2105                         return ret;
2106
2107                 snprintf(match, sizeof(match), "hif-%d", ret);
2108                 memset(&params, 0, sizeof(params));
2109                 params.match = match;
2110                 params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
2111                 ret = pp2_hif_init(&params, &hifs[i]);
2112                 if (ret) {
2113                         RTE_LOG(ERR, PMD, "Failed to initialize hif %d\n", i);
2114                         return ret;
2115                 }
2116         }
2117
2118         return 0;
2119 }
2120
2121 /**
2122  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
2123  */
2124 static void
2125 mrvl_deinit_hifs(void)
2126 {
2127         int i;
2128
2129         RTE_LCORE_FOREACH(i) {
2130                 if (hifs[i])
2131                         pp2_hif_deinit(hifs[i]);
2132         }
2133 }
2134
2135 static void mrvl_set_first_last_cores(int core_id)
2136 {
2137         if (core_id < mrvl_lcore_first)
2138                 mrvl_lcore_first = core_id;
2139
2140         if (core_id > mrvl_lcore_last)
2141                 mrvl_lcore_last = core_id;
2142 }
2143
2144 /**
2145  * DPDK callback to register the virtual device.
2146  *
2147  * @param vdev
2148  *   Pointer to the virtual device.
2149  *
2150  * @return
2151  *   0 on success, negative error value otherwise.
2152  */
2153 static int
2154 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
2155 {
2156         struct rte_kvargs *kvlist;
2157         const char *ifnames[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
2158         int ret = -EINVAL;
2159         uint32_t i, ifnum, cfgnum, core_id;
2160         const char *params;
2161
2162         params = rte_vdev_device_args(vdev);
2163         if (!params)
2164                 return -EINVAL;
2165
2166         kvlist = rte_kvargs_parse(params, valid_args);
2167         if (!kvlist)
2168                 return -EINVAL;
2169
2170         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
2171         if (ifnum > RTE_DIM(ifnames))
2172                 goto out_free_kvlist;
2173
2174         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
2175                            mrvl_get_ifnames, &ifnames);
2176
2177         cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
2178         if (cfgnum > 1) {
2179                 RTE_LOG(ERR, PMD, "Cannot handle more than one config file!\n");
2180                 goto out_free_kvlist;
2181         } else if (cfgnum == 1) {
2182                 rte_kvargs_process(kvlist, MRVL_CFG_ARG,
2183                                    mrvl_get_qoscfg, &mrvl_qos_cfg);
2184         }
2185
2186         /*
2187          * ret == -EEXIST is correct, it means DMA
2188          * has been already initialized (by another PMD).
2189          */
2190         ret = mv_sys_dma_mem_init(MRVL_MUSDK_DMA_MEMSIZE);
2191         if (ret < 0) {
2192                 if (ret != -EEXIST)
2193                         goto out_free_kvlist;
2194                 else
2195                         RTE_LOG(INFO, PMD,
2196                                 "DMA memory has been already initialized by a different driver.\n");
2197         }
2198
2199         ret = mrvl_init_pp2();
2200         if (ret) {
2201                 RTE_LOG(ERR, PMD, "Failed to init PP!\n");
2202                 goto out_deinit_dma;
2203         }
2204
2205         ret = mrvl_init_hifs();
2206         if (ret)
2207                 goto out_deinit_hifs;
2208
2209         for (i = 0; i < ifnum; i++) {
2210                 RTE_LOG(INFO, PMD, "Creating %s\n", ifnames[i]);
2211                 ret = mrvl_eth_dev_create(vdev, ifnames[i]);
2212                 if (ret)
2213                         goto out_cleanup;
2214         }
2215
2216         rte_kvargs_free(kvlist);
2217
2218         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
2219
2220         mrvl_lcore_first = RTE_MAX_LCORE;
2221         mrvl_lcore_last = 0;
2222
2223         RTE_LCORE_FOREACH(core_id) {
2224                 mrvl_set_first_last_cores(core_id);
2225         }
2226
2227         return 0;
2228 out_cleanup:
2229         for (; i > 0; i--)
2230                 mrvl_eth_dev_destroy(ifnames[i]);
2231 out_deinit_hifs:
2232         mrvl_deinit_hifs();
2233         mrvl_deinit_pp2();
2234 out_deinit_dma:
2235         mv_sys_dma_mem_destroy();
2236 out_free_kvlist:
2237         rte_kvargs_free(kvlist);
2238
2239         return ret;
2240 }
2241
2242 /**
2243  * DPDK callback to remove virtual device.
2244  *
2245  * @param vdev
2246  *   Pointer to the removed virtual device.
2247  *
2248  * @return
2249  *   0 on success, negative error value otherwise.
2250  */
2251 static int
2252 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
2253 {
2254         int i;
2255         const char *name;
2256
2257         name = rte_vdev_device_name(vdev);
2258         if (!name)
2259                 return -EINVAL;
2260
2261         RTE_LOG(INFO, PMD, "Removing %s\n", name);
2262
2263         for (i = 0; i < rte_eth_dev_count(); i++) {
2264                 char ifname[RTE_ETH_NAME_MAX_LEN];
2265
2266                 rte_eth_dev_get_name_by_port(i, ifname);
2267                 mrvl_eth_dev_destroy(ifname);
2268         }
2269
2270         mrvl_deinit_hifs();
2271         mrvl_deinit_pp2();
2272         mv_sys_dma_mem_destroy();
2273
2274         return 0;
2275 }
2276
2277 static struct rte_vdev_driver pmd_mrvl_drv = {
2278         .probe = rte_pmd_mrvl_probe,
2279         .remove = rte_pmd_mrvl_remove,
2280 };
2281
2282 RTE_PMD_REGISTER_VDEV(net_mrvl, pmd_mrvl_drv);
2283 RTE_PMD_REGISTER_ALIAS(net_mrvl, eth_mrvl);