drivers/net/mrvl/mrvl_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Marvell International Ltd.
   5  *   Copyright(c) 2017 Semihalf.
   6  *   All rights reserved.
   7  *
   8  *   Redistribution and use in source and binary forms, with or without
   9  *   modification, are permitted provided that the following conditions
  10  *   are met:
  11  *
  12  *     * Redistributions of source code must retain the above copyright
  13  *       notice, this list of conditions and the following disclaimer.
  14  *     * Redistributions in binary form must reproduce the above copyright
  15  *       notice, this list of conditions and the following disclaimer in
  16  *       the documentation and/or other materials provided with the
  17  *       distribution.
  18  *     * Neither the name of Semihalf nor the names of its
  19  *       contributors may be used to endorse or promote products derived
  20  *       from this software without specific prior written permission.
  21  *
  22  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  23  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  24  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  25  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  26  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  27  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  28  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  29  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  30  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  31  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  32  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  33  */
  34
  35 #include <rte_ethdev.h>
  36 #include <rte_kvargs.h>
  37 #include <rte_log.h>
  38 #include <rte_malloc.h>
  39 #include <rte_bus_vdev.h>
  40
  41 /* Unluckily, container_of is defined by both DPDK and MUSDK,
  42  * we'll declare only one version.
  43  *
  44  * Note that it is not used in this PMD anyway.
  45  */
  46 #ifdef container_of
  47 #undef container_of
  48 #endif
  49
  50 #include <fcntl.h>
  51 #include <linux/ethtool.h>
  52 #include <linux/sockios.h>
  53 #include <net/if.h>
  54 #include <net/if_arp.h>
  55 #include <sys/ioctl.h>
  56 #include <sys/socket.h>
  57 #include <sys/stat.h>
  58 #include <sys/types.h>
  59
  60 #include "mrvl_ethdev.h"
  61 #include "mrvl_qos.h"
  62
  63 /* bitmask with reserved hifs */
  64 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
  65 /* bitmask with reserved bpools */
  66 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
  67 /* bitmask with reserved kernel RSS tables */
  68 #define MRVL_MUSDK_RSS_RESERVED 0x01
  69 /* maximum number of available hifs */
  70 #define MRVL_MUSDK_HIFS_MAX 9
  71
  72 /* prefetch shift */
  73 #define MRVL_MUSDK_PREFETCH_SHIFT 2
  74
  75 /* TCAM has 25 entries reserved for uc/mc filter entries */
  76 #define MRVL_MAC_ADDRS_MAX 25
  77 #define MRVL_MATCH_LEN 16
  78 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
  79 /* Maximum allowable packet size */
  80 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
  81
  82 #define MRVL_IFACE_NAME_ARG "iface"
  83 #define MRVL_CFG_ARG "cfg"
  84
  85 #define MRVL_BURST_SIZE 64
  86
  87 #define MRVL_ARP_LENGTH 28
  88
  89 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
  90
  91 #define MRVL_COOKIE_HIGH_ADDR_SHIFT     (sizeof(pp2_cookie_t) * 8)
  92 #define MRVL_COOKIE_HIGH_ADDR_MASK      (~0ULL << MRVL_COOKIE_HIGH_ADDR_SHIFT)
  93
  94 /* Memory size (in bytes) for MUSDK dma buffers */
  95 #define MRVL_MUSDK_DMA_MEMSIZE 41943040
  96
  97 static const char * const valid_args[] = {
  98         MRVL_IFACE_NAME_ARG,
  99         MRVL_CFG_ARG,
 100         NULL
 101 };
 102
 103 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
 104 static struct pp2_hif *hifs[RTE_MAX_LCORE];
 105 static int used_bpools[PP2_NUM_PKT_PROC] = {
 106         MRVL_MUSDK_BPOOLS_RESERVED,
 107         MRVL_MUSDK_BPOOLS_RESERVED
 108 };
 109
 110 struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
 111 int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
 112 uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
 113
 114 struct mrvl_ifnames {
 115         const char *names[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
 116         int idx;
 117 };
 118
 119 /*
 120  * To use buffer harvesting based on loopback port shadow queue structure
 121  * was introduced for buffers information bookkeeping.
 122  *
 123  * Before sending the packet, related buffer information (pp2_buff_inf) is
 124  * stored in shadow queue. After packet is transmitted no longer used
 125  * packet buffer is released back to it's original hardware pool,
 126  * on condition it originated from interface.
 127  * In case it  was generated by application itself i.e: mbuf->port field is
 128  * 0xff then its released to software mempool.
 129  */
 130 struct mrvl_shadow_txq {
 131         int head;           /* write index - used when sending buffers */
 132         int tail;           /* read index - used when releasing buffers */
 133         u16 size;           /* queue occupied size */
 134         u16 num_to_release; /* number of buffers sent, that can be released */
 135         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
 136 };
 137
 138 struct mrvl_rxq {
 139         struct mrvl_priv *priv;
 140         struct rte_mempool *mp;
 141         int queue_id;
 142         int port_id;
 143         int cksum_enabled;
 144         uint64_t bytes_recv;
 145         uint64_t drop_mac;
 146 };
 147
 148 struct mrvl_txq {
 149         struct mrvl_priv *priv;
 150         int queue_id;
 151         int port_id;
 152         uint64_t bytes_sent;
 153 };
 154
 155 /*
 156  * Every tx queue should have dedicated shadow tx queue.
 157  *
 158  * Ports assigned by DPDK might not start at zero or be continuous so
 159  * as a workaround define shadow queues for each possible port so that
 160  * we eventually fit somewhere.
 161  */
 162 struct mrvl_shadow_txq shadow_txqs[RTE_MAX_ETHPORTS][RTE_MAX_LCORE];
 163
 164 static int mrvl_lcore_first;
 165 static int mrvl_lcore_last;
 166 static int mrvl_dev_num;
 167
 168 static int mrvl_fill_bpool(struct mrvl_rxq *rxq, int num);
 169
 170 static inline int
 171 mrvl_get_bpool_size(int pp2_id, int pool_id)
 172 {
 173         int i;
 174         int size = 0;
 175
 176         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
 177                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
 178
 179         return size;
 180 }
 181
 182 static inline int
 183 mrvl_reserve_bit(int *bitmap, int max)
 184 {
 185         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
 186
 187         if (n >= max)
 188                 return -1;
 189
 190         *bitmap |= 1 << n;
 191
 192         return n;
 193 }
 194
 195 static int
 196 mrvl_init_hif(int core_id)
 197 {
 198         struct pp2_hif_params params;
 199         char match[MRVL_MATCH_LEN];
 200         int ret;
 201
 202         ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
 203         if (ret < 0) {
 204                 RTE_LOG(ERR, PMD, "Failed to allocate hif %d\n", core_id);
 205                 return ret;
 206         }
 207
 208         snprintf(match, sizeof(match), "hif-%d", ret);
 209         memset(&params, 0, sizeof(params));
 210         params.match = match;
 211         params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
 212         ret = pp2_hif_init(&params, &hifs[core_id]);
 213         if (ret) {
 214                 RTE_LOG(ERR, PMD, "Failed to initialize hif %d\n", core_id);
 215                 return ret;
 216         }
 217
 218         return 0;
 219 }
 220
 221 static inline struct pp2_hif*
 222 mrvl_get_hif(struct mrvl_priv *priv, int core_id)
 223 {
 224         int ret;
 225
 226         if (likely(hifs[core_id] != NULL))
 227                 return hifs[core_id];
 228
 229         rte_spinlock_lock(&priv->lock);
 230
 231         ret = mrvl_init_hif(core_id);
 232         if (ret < 0) {
 233                 RTE_LOG(ERR, PMD, "Failed to allocate hif %d\n", core_id);
 234                 goto out;
 235         }
 236
 237         if (core_id < mrvl_lcore_first)
 238                 mrvl_lcore_first = core_id;
 239
 240         if (core_id > mrvl_lcore_last)
 241                 mrvl_lcore_last = core_id;
 242 out:
 243         rte_spinlock_unlock(&priv->lock);
 244
 245         return hifs[core_id];
 246 }
 247
 248 /**
 249  * Configure rss based on dpdk rss configuration.
 250  *
 251  * @param priv
 252  *   Pointer to private structure.
 253  * @param rss_conf
 254  *   Pointer to RSS configuration.
 255  *
 256  * @return
 257  *   0 on success, negative error value otherwise.
 258  */
 259 static int
 260 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf)
 261 {
 262         if (rss_conf->rss_key)
 263                 RTE_LOG(WARNING, PMD, "Changing hash key is not supported\n");
 264
 265         if (rss_conf->rss_hf == 0) {
 266                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 267         } else if (rss_conf->rss_hf & ETH_RSS_IPV4) {
 268                 priv->ppio_params.inqs_params.hash_type =
 269                         PP2_PPIO_HASH_T_2_TUPLE;
 270         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
 271                 priv->ppio_params.inqs_params.hash_type =
 272                         PP2_PPIO_HASH_T_5_TUPLE;
 273                 priv->rss_hf_tcp = 1;
 274         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
 275                 priv->ppio_params.inqs_params.hash_type =
 276                         PP2_PPIO_HASH_T_5_TUPLE;
 277                 priv->rss_hf_tcp = 0;
 278         } else {
 279                 return -EINVAL;
 280         }
 281
 282         return 0;
 283 }
 284
 285 /**
 286  * Ethernet device configuration.
 287  *
 288  * Prepare the driver for a given number of TX and RX queues and
 289  * configure RSS.
 290  *
 291  * @param dev
 292  *   Pointer to Ethernet device structure.
 293  *
 294  * @return
 295  *   0 on success, negative error value otherwise.
 296  */
 297 static int
 298 mrvl_dev_configure(struct rte_eth_dev *dev)
 299 {
 300         struct mrvl_priv *priv = dev->data->dev_private;
 301         int ret;
 302
 303         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
 304             dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
 305                 RTE_LOG(INFO, PMD, "Unsupported rx multi queue mode %d\n",
 306                         dev->data->dev_conf.rxmode.mq_mode);
 307                 return -EINVAL;
 308         }
 309
 310         if (!dev->data->dev_conf.rxmode.hw_strip_crc) {
 311                 RTE_LOG(INFO, PMD,
 312                         "L2 CRC stripping is always enabled in hw\n");
 313                 dev->data->dev_conf.rxmode.hw_strip_crc = 1;
 314         }
 315
 316         if (dev->data->dev_conf.rxmode.hw_vlan_strip) {
 317                 RTE_LOG(INFO, PMD, "VLAN stripping not supported\n");
 318                 return -EINVAL;
 319         }
 320
 321         if (dev->data->dev_conf.rxmode.split_hdr_size) {
 322                 RTE_LOG(INFO, PMD, "Split headers not supported\n");
 323                 return -EINVAL;
 324         }
 325
 326         if (dev->data->dev_conf.rxmode.enable_scatter) {
 327                 RTE_LOG(INFO, PMD, "RX Scatter/Gather not supported\n");
 328                 return -EINVAL;
 329         }
 330
 331         if (dev->data->dev_conf.rxmode.enable_lro) {
 332                 RTE_LOG(INFO, PMD, "LRO not supported\n");
 333                 return -EINVAL;
 334         }
 335
 336         if (dev->data->dev_conf.rxmode.jumbo_frame)
 337                 dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
 338                                  ETHER_HDR_LEN - ETHER_CRC_LEN;
 339
 340         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
 341                                   dev->data->nb_rx_queues);
 342         if (ret < 0)
 343                 return ret;
 344
 345         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
 346         priv->ppio_params.maintain_stats = 1;
 347         priv->nb_rx_queues = dev->data->nb_rx_queues;
 348
 349         if (dev->data->nb_rx_queues == 1 &&
 350             dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
 351                 RTE_LOG(WARNING, PMD, "Disabling hash for 1 rx queue\n");
 352                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 353
 354                 return 0;
 355         }
 356
 357         return mrvl_configure_rss(priv,
 358                                   &dev->data->dev_conf.rx_adv_conf.rss_conf);
 359 }
 360
 361 /**
 362  * DPDK callback to change the MTU.
 363  *
 364  * Setting the MTU affects hardware MRU (packets larger than the MRU
 365  * will be dropped).
 366  *
 367  * @param dev
 368  *   Pointer to Ethernet device structure.
 369  * @param mtu
 370  *   New MTU.
 371  *
 372  * @return
 373  *   0 on success, negative error value otherwise.
 374  */
 375 static int
 376 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 377 {
 378         struct mrvl_priv *priv = dev->data->dev_private;
 379         /* extra MV_MH_SIZE bytes are required for Marvell tag */
 380         uint16_t mru = mtu + MV_MH_SIZE + ETHER_HDR_LEN + ETHER_CRC_LEN;
 381         int ret;
 382
 383         if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX)
 384                 return -EINVAL;
 385
 386         if (!priv->ppio)
 387                 return -EPERM;
 388
 389         ret = pp2_ppio_set_mru(priv->ppio, mru);
 390         if (ret)
 391                 return ret;
 392
 393         return pp2_ppio_set_mtu(priv->ppio, mtu);
 394 }
 395
 396 /**
 397  * DPDK callback to bring the link up.
 398  *
 399  * @param dev
 400  *   Pointer to Ethernet device structure.
 401  *
 402  * @return
 403  *   0 on success, negative error value otherwise.
 404  */
 405 static int
 406 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
 407 {
 408         struct mrvl_priv *priv = dev->data->dev_private;
 409         int ret;
 410
 411         if (!priv->ppio)
 412                 return -EPERM;
 413
 414         ret = pp2_ppio_enable(priv->ppio);
 415         if (ret)
 416                 return ret;
 417
 418         /*
 419          * mtu/mru can be updated if pp2_ppio_enable() was called at least once
 420          * as pp2_ppio_enable() changes port->t_mode from default 0 to
 421          * PP2_TRAFFIC_INGRESS_EGRESS.
 422          *
 423          * Set mtu to default DPDK value here.
 424          */
 425         ret = mrvl_mtu_set(dev, dev->data->mtu);
 426         if (ret)
 427                 pp2_ppio_disable(priv->ppio);
 428
 429         return ret;
 430 }
 431
 432 /**
 433  * DPDK callback to bring the link down.
 434  *
 435  * @param dev
 436  *   Pointer to Ethernet device structure.
 437  *
 438  * @return
 439  *   0 on success, negative error value otherwise.
 440  */
 441 static int
 442 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
 443 {
 444         struct mrvl_priv *priv = dev->data->dev_private;
 445
 446         if (!priv->ppio)
 447                 return -EPERM;
 448
 449         return pp2_ppio_disable(priv->ppio);
 450 }
 451
 452 /**
 453  * DPDK callback to start the device.
 454  *
 455  * @param dev
 456  *   Pointer to Ethernet device structure.
 457  *
 458  * @return
 459  *   0 on success, negative errno value on failure.
 460  */
 461 static int
 462 mrvl_dev_start(struct rte_eth_dev *dev)
 463 {
 464         struct mrvl_priv *priv = dev->data->dev_private;
 465         char match[MRVL_MATCH_LEN];
 466         int ret = 0, def_init_size;
 467
 468         snprintf(match, sizeof(match), "ppio-%d:%d",
 469                  priv->pp_id, priv->ppio_id);
 470         priv->ppio_params.match = match;
 471
 472         /*
 473          * Calculate the minimum bpool size for refill feature as follows:
 474          * 2 default burst sizes multiply by number of rx queues.
 475          * If the bpool size will be below this value, new buffers will
 476          * be added to the pool.
 477          */
 478         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
 479
 480         /* In case initial bpool size configured in queues setup is
 481          * smaller than minimum size add more buffers
 482          */
 483         def_init_size = priv->bpool_min_size + MRVL_BURST_SIZE * 2;
 484         if (priv->bpool_init_size < def_init_size) {
 485                 int buffs_to_add = def_init_size - priv->bpool_init_size;
 486
 487                 priv->bpool_init_size += buffs_to_add;
 488                 ret = mrvl_fill_bpool(dev->data->rx_queues[0], buffs_to_add);
 489                 if (ret)
 490                         RTE_LOG(ERR, PMD, "Failed to add buffers to bpool\n");
 491         }
 492
 493         /*
 494          * Calculate the maximum bpool size for refill feature as follows:
 495          * maximum number of descriptors in rx queue multiply by number
 496          * of rx queues plus minimum bpool size.
 497          * In case the bpool size will exceed this value, superfluous buffers
 498          * will be removed
 499          */
 500         priv->bpool_max_size = (priv->nb_rx_queues * MRVL_PP2_RXD_MAX) +
 501                                 priv->bpool_min_size;
 502
 503         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
 504         if (ret) {
 505                 RTE_LOG(ERR, PMD, "Failed to init ppio\n");
 506                 return ret;
 507         }
 508
 509         /*
 510          * In case there are some some stale uc/mc mac addresses flush them
 511          * here. It cannot be done during mrvl_dev_close() as port information
 512          * is already gone at that point (due to pp2_ppio_deinit() in
 513          * mrvl_dev_stop()).
 514          */
 515         if (!priv->uc_mc_flushed) {
 516                 ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1);
 517                 if (ret) {
 518                         RTE_LOG(ERR, PMD,
 519                                 "Failed to flush uc/mc filter list\n");
 520                         goto out;
 521                 }
 522                 priv->uc_mc_flushed = 1;
 523         }
 524
 525         if (!priv->vlan_flushed) {
 526                 ret = pp2_ppio_flush_vlan(priv->ppio);
 527                 if (ret) {
 528                         RTE_LOG(ERR, PMD, "Failed to flush vlan list\n");
 529                         /*
 530                          * TODO
 531                          * once pp2_ppio_flush_vlan() is supported jump to out
 532                          * goto out;
 533                          */
 534                 }
 535                 priv->vlan_flushed = 1;
 536         }
 537
 538         /* For default QoS config, don't start classifier. */
 539         if (mrvl_qos_cfg) {
 540                 ret = mrvl_start_qos_mapping(priv);
 541                 if (ret) {
 542                         RTE_LOG(ERR, PMD, "Failed to setup QoS mapping\n");
 543                         goto out;
 544                 }
 545         }
 546
 547         ret = mrvl_dev_set_link_up(dev);
 548         if (ret) {
 549                 RTE_LOG(ERR, PMD, "Failed to set link up\n");
 550                 goto out;
 551         }
 552
 553         return 0;
 554 out:
 555         RTE_LOG(ERR, PMD, "Failed to start device\n");
 556         pp2_ppio_deinit(priv->ppio);
 557         return ret;
 558 }
 559
 560 /**
 561  * Flush receive queues.
 562  *
 563  * @param dev
 564  *   Pointer to Ethernet device structure.
 565  */
 566 static void
 567 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
 568 {
 569         int i;
 570
 571         RTE_LOG(INFO, PMD, "Flushing rx queues\n");
 572         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 573                 int ret, num;
 574
 575                 do {
 576                         struct mrvl_rxq *q = dev->data->rx_queues[i];
 577                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
 578
 579                         num = MRVL_PP2_RXD_MAX;
 580                         ret = pp2_ppio_recv(q->priv->ppio,
 581                                             q->priv->rxq_map[q->queue_id].tc,
 582                                             q->priv->rxq_map[q->queue_id].inq,
 583                                             descs, (uint16_t *)&num);
 584                 } while (ret == 0 && num);
 585         }
 586 }
 587
 588 /**
 589  * Flush transmit shadow queues.
 590  *
 591  * @param dev
 592  *   Pointer to Ethernet device structure.
 593  */
 594 static void
 595 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
 596 {
 597         int i;
 598
 599         RTE_LOG(INFO, PMD, "Flushing tx shadow queues\n");
 600         for (i = 0; i < RTE_MAX_LCORE; i++) {
 601                 struct mrvl_shadow_txq *sq =
 602                         &shadow_txqs[dev->data->port_id][i];
 603
 604                 while (sq->tail != sq->head) {
 605                         uint64_t addr = cookie_addr_high |
 606                                         sq->ent[sq->tail].buff.cookie;
 607                         rte_pktmbuf_free((struct rte_mbuf *)addr);
 608                         sq->tail = (sq->tail + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
 609                 }
 610
 611                 memset(sq, 0, sizeof(*sq));
 612         }
 613 }
 614
 615 /**
 616  * Flush hardware bpool (buffer-pool).
 617  *
 618  * @param dev
 619  *   Pointer to Ethernet device structure.
 620  */
 621 static void
 622 mrvl_flush_bpool(struct rte_eth_dev *dev)
 623 {
 624         struct mrvl_priv *priv = dev->data->dev_private;
 625         struct pp2_hif *hif;
 626         uint32_t num;
 627         int ret;
 628         unsigned int core_id = rte_lcore_id();
 629
 630         if (core_id == LCORE_ID_ANY)
 631                 core_id = 0;
 632
 633         hif = mrvl_get_hif(priv, core_id);
 634
 635         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
 636         if (ret) {
 637                 RTE_LOG(ERR, PMD, "Failed to get bpool buffers number\n");
 638                 return;
 639         }
 640
 641         while (num--) {
 642                 struct pp2_buff_inf inf;
 643                 uint64_t addr;
 644
 645                 ret = pp2_bpool_get_buff(hif, priv->bpool, &inf);
 646                 if (ret)
 647                         break;
 648
 649                 addr = cookie_addr_high | inf.cookie;
 650                 rte_pktmbuf_free((struct rte_mbuf *)addr);
 651         }
 652 }
 653
 654 /**
 655  * DPDK callback to stop the device.
 656  *
 657  * @param dev
 658  *   Pointer to Ethernet device structure.
 659  */
 660 static void
 661 mrvl_dev_stop(struct rte_eth_dev *dev)
 662 {
 663         struct mrvl_priv *priv = dev->data->dev_private;
 664
 665         mrvl_dev_set_link_down(dev);
 666         mrvl_flush_rx_queues(dev);
 667         mrvl_flush_tx_shadow_queues(dev);
 668         if (priv->qos_tbl) {
 669                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
 670                 priv->qos_tbl = NULL;
 671         }
 672         pp2_ppio_deinit(priv->ppio);
 673         priv->ppio = NULL;
 674 }
 675
 676 /**
 677  * DPDK callback to close the device.
 678  *
 679  * @param dev
 680  *   Pointer to Ethernet device structure.
 681  */
 682 static void
 683 mrvl_dev_close(struct rte_eth_dev *dev)
 684 {
 685         struct mrvl_priv *priv = dev->data->dev_private;
 686         size_t i;
 687
 688         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
 689                 struct pp2_ppio_tc_params *tc_params =
 690                         &priv->ppio_params.inqs_params.tcs_params[i];
 691
 692                 if (tc_params->inqs_params) {
 693                         rte_free(tc_params->inqs_params);
 694                         tc_params->inqs_params = NULL;
 695                 }
 696         }
 697
 698         mrvl_flush_bpool(dev);
 699 }
 700
 701 /**
 702  * DPDK callback to retrieve physical link information.
 703  *
 704  * @param dev
 705  *   Pointer to Ethernet device structure.
 706  * @param wait_to_complete
 707  *   Wait for request completion (ignored).
 708  *
 709  * @return
 710  *   0 on success, negative error value otherwise.
 711  */
 712 static int
 713 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
 714 {
 715         /*
 716          * TODO
 717          * once MUSDK provides necessary API use it here
 718          */
 719         struct mrvl_priv *priv = dev->data->dev_private;
 720         struct ethtool_cmd edata;
 721         struct ifreq req;
 722         int ret, fd, link_up;
 723
 724         if (!priv->ppio)
 725                 return -EPERM;
 726
 727         edata.cmd = ETHTOOL_GSET;
 728
 729         strcpy(req.ifr_name, dev->data->name);
 730         req.ifr_data = (void *)&edata;
 731
 732         fd = socket(AF_INET, SOCK_DGRAM, 0);
 733         if (fd == -1)
 734                 return -EFAULT;
 735
 736         ret = ioctl(fd, SIOCETHTOOL, &req);
 737         if (ret == -1) {
 738                 close(fd);
 739                 return -EFAULT;
 740         }
 741
 742         close(fd);
 743
 744         switch (ethtool_cmd_speed(&edata)) {
 745         case SPEED_10:
 746                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
 747                 break;
 748         case SPEED_100:
 749                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
 750                 break;
 751         case SPEED_1000:
 752                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
 753                 break;
 754         case SPEED_10000:
 755                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
 756                 break;
 757         default:
 758                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
 759         }
 760
 761         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
 762                                                          ETH_LINK_HALF_DUPLEX;
 763         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
 764                                                            ETH_LINK_FIXED;
 765         pp2_ppio_get_link_state(priv->ppio, &link_up);
 766         dev->data->dev_link.link_status = link_up ? ETH_LINK_UP : ETH_LINK_DOWN;
 767
 768         return 0;
 769 }
 770
 771 /**
 772  * DPDK callback to enable promiscuous mode.
 773  *
 774  * @param dev
 775  *   Pointer to Ethernet device structure.
 776  */
 777 static void
 778 mrvl_promiscuous_enable(struct rte_eth_dev *dev)
 779 {
 780         struct mrvl_priv *priv = dev->data->dev_private;
 781         int ret;
 782
 783         if (!priv->ppio)
 784                 return;
 785
 786         ret = pp2_ppio_set_promisc(priv->ppio, 1);
 787         if (ret)
 788                 RTE_LOG(ERR, PMD, "Failed to enable promiscuous mode\n");
 789 }
 790
 791 /**
 792  * DPDK callback to enable allmulti mode.
 793  *
 794  * @param dev
 795  *   Pointer to Ethernet device structure.
 796  */
 797 static void
 798 mrvl_allmulticast_enable(struct rte_eth_dev *dev)
 799 {
 800         struct mrvl_priv *priv = dev->data->dev_private;
 801         int ret;
 802
 803         if (!priv->ppio)
 804                 return;
 805
 806         ret = pp2_ppio_set_mc_promisc(priv->ppio, 1);
 807         if (ret)
 808                 RTE_LOG(ERR, PMD, "Failed enable all-multicast mode\n");
 809 }
 810
 811 /**
 812  * DPDK callback to disable promiscuous mode.
 813  *
 814  * @param dev
 815  *   Pointer to Ethernet device structure.
 816  */
 817 static void
 818 mrvl_promiscuous_disable(struct rte_eth_dev *dev)
 819 {
 820         struct mrvl_priv *priv = dev->data->dev_private;
 821         int ret;
 822
 823         if (!priv->ppio)
 824                 return;
 825
 826         ret = pp2_ppio_set_promisc(priv->ppio, 0);
 827         if (ret)
 828                 RTE_LOG(ERR, PMD, "Failed to disable promiscuous mode\n");
 829 }
 830
 831 /**
 832  * DPDK callback to disable allmulticast mode.
 833  *
 834  * @param dev
 835  *   Pointer to Ethernet device structure.
 836  */
 837 static void
 838 mrvl_allmulticast_disable(struct rte_eth_dev *dev)
 839 {
 840         struct mrvl_priv *priv = dev->data->dev_private;
 841         int ret;
 842
 843         if (!priv->ppio)
 844                 return;
 845
 846         ret = pp2_ppio_set_mc_promisc(priv->ppio, 0);
 847         if (ret)
 848                 RTE_LOG(ERR, PMD, "Failed to disable all-multicast mode\n");
 849 }
 850
 851 /**
 852  * DPDK callback to remove a MAC address.
 853  *
 854  * @param dev
 855  *   Pointer to Ethernet device structure.
 856  * @param index
 857  *   MAC address index.
 858  */
 859 static void
 860 mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
 861 {
 862         struct mrvl_priv *priv = dev->data->dev_private;
 863         char buf[ETHER_ADDR_FMT_SIZE];
 864         int ret;
 865
 866         if (!priv->ppio)
 867                 return;
 868
 869         ret = pp2_ppio_remove_mac_addr(priv->ppio,
 870                                        dev->data->mac_addrs[index].addr_bytes);
 871         if (ret) {
 872                 ether_format_addr(buf, sizeof(buf),
 873                                   &dev->data->mac_addrs[index]);
 874                 RTE_LOG(ERR, PMD, "Failed to remove mac %s\n", buf);
 875         }
 876 }
 877
 878 /**
 879  * DPDK callback to add a MAC address.
 880  *
 881  * @param dev
 882  *   Pointer to Ethernet device structure.
 883  * @param mac_addr
 884  *   MAC address to register.
 885  * @param index
 886  *   MAC address index.
 887  * @param vmdq
 888  *   VMDq pool index to associate address with (unused).
 889  *
 890  * @return
 891  *   0 on success, negative error value otherwise.
 892  */
 893 static int
 894 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct ether_addr *mac_addr,
 895                   uint32_t index, uint32_t vmdq __rte_unused)
 896 {
 897         struct mrvl_priv *priv = dev->data->dev_private;
 898         char buf[ETHER_ADDR_FMT_SIZE];
 899         int ret;
 900
 901         if (index == 0)
 902                 /* For setting index 0, mrvl_mac_addr_set() should be used.*/
 903                 return -1;
 904
 905         if (!priv->ppio)
 906                 return -EPERM;
 907
 908         /*
 909          * Maximum number of uc addresses can be tuned via kernel module mvpp2x
 910          * parameter uc_filter_max. Maximum number of mc addresses is then
 911          * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and
 912          * 21 respectively.
 913          *
 914          * If more than uc_filter_max uc addresses were added to filter list
 915          * then NIC will switch to promiscuous mode automatically.
 916          *
 917          * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses
 918          * were added to filter list then NIC will switch to all-multicast mode
 919          * automatically.
 920          */
 921         ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
 922         if (ret) {
 923                 ether_format_addr(buf, sizeof(buf), mac_addr);
 924                 RTE_LOG(ERR, PMD, "Failed to add mac %s\n", buf);
 925                 return -1;
 926         }
 927
 928         return 0;
 929 }
 930
 931 /**
 932  * DPDK callback to set the primary MAC address.
 933  *
 934  * @param dev
 935  *   Pointer to Ethernet device structure.
 936  * @param mac_addr
 937  *   MAC address to register.
 938  */
 939 static void
 940 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
 941 {
 942         struct mrvl_priv *priv = dev->data->dev_private;
 943         int ret;
 944
 945         if (!priv->ppio)
 946                 return;
 947
 948         ret = pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
 949         if (ret) {
 950                 char buf[ETHER_ADDR_FMT_SIZE];
 951                 ether_format_addr(buf, sizeof(buf), mac_addr);
 952                 RTE_LOG(ERR, PMD, "Failed to set mac to %s\n", buf);
 953         }
 954 }
 955
 956 /**
 957  * DPDK callback to get device statistics.
 958  *
 959  * @param dev
 960  *   Pointer to Ethernet device structure.
 961  * @param stats
 962  *   Stats structure output buffer.
 963  *
 964  * @return
 965  *   0 on success, negative error value otherwise.
 966  */
 967 static int
 968 mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 969 {
 970         struct mrvl_priv *priv = dev->data->dev_private;
 971         struct pp2_ppio_statistics ppio_stats;
 972         uint64_t drop_mac = 0;
 973         unsigned int i, idx, ret;
 974
 975         if (!priv->ppio)
 976                 return -EPERM;
 977
 978         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 979                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
 980                 struct pp2_ppio_inq_statistics rx_stats;
 981
 982                 if (!rxq)
 983                         continue;
 984
 985                 idx = rxq->queue_id;
 986                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
 987                         RTE_LOG(ERR, PMD,
 988                                 "rx queue %d stats out of range (0 - %d)\n",
 989                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
 990                         continue;
 991                 }
 992
 993                 ret = pp2_ppio_inq_get_statistics(priv->ppio,
 994                                                   priv->rxq_map[idx].tc,
 995                                                   priv->rxq_map[idx].inq,
 996                                                   &rx_stats, 0);
 997                 if (unlikely(ret)) {
 998                         RTE_LOG(ERR, PMD,
 999                                 "Failed to update rx queue %d stats\n", idx);
1000                         break;
1001                 }
1002
1003                 stats->q_ibytes[idx] = rxq->bytes_recv;
1004                 stats->q_ipackets[idx] = rx_stats.enq_desc - rxq->drop_mac;
1005                 stats->q_errors[idx] = rx_stats.drop_early +
1006                                        rx_stats.drop_fullq +
1007                                        rx_stats.drop_bm +
1008                                        rxq->drop_mac;
1009                 stats->ibytes += rxq->bytes_recv;
1010                 drop_mac += rxq->drop_mac;
1011         }
1012
1013         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1014                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1015                 struct pp2_ppio_outq_statistics tx_stats;
1016
1017                 if (!txq)
1018                         continue;
1019
1020                 idx = txq->queue_id;
1021                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1022                         RTE_LOG(ERR, PMD,
1023                                 "tx queue %d stats out of range (0 - %d)\n",
1024                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1025                 }
1026
1027                 ret = pp2_ppio_outq_get_statistics(priv->ppio, idx,
1028                                                    &tx_stats, 0);
1029                 if (unlikely(ret)) {
1030                         RTE_LOG(ERR, PMD,
1031                                 "Failed to update tx queue %d stats\n", idx);
1032                         break;
1033                 }
1034
1035                 stats->q_opackets[idx] = tx_stats.deq_desc;
1036                 stats->q_obytes[idx] = txq->bytes_sent;
1037                 stats->obytes += txq->bytes_sent;
1038         }
1039
1040         ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1041         if (unlikely(ret)) {
1042                 RTE_LOG(ERR, PMD, "Failed to update port statistics\n");
1043                 return ret;
1044         }
1045
1046         stats->ipackets += ppio_stats.rx_packets - drop_mac;
1047         stats->opackets += ppio_stats.tx_packets;
1048         stats->imissed += ppio_stats.rx_fullq_dropped +
1049                           ppio_stats.rx_bm_dropped +
1050                           ppio_stats.rx_early_dropped +
1051                           ppio_stats.rx_fifo_dropped +
1052                           ppio_stats.rx_cls_dropped;
1053         stats->ierrors = drop_mac;
1054
1055         return 0;
1056 }
1057
1058 /**
1059  * DPDK callback to clear device statistics.
1060  *
1061  * @param dev
1062  *   Pointer to Ethernet device structure.
1063  */
1064 static void
1065 mrvl_stats_reset(struct rte_eth_dev *dev)
1066 {
1067         struct mrvl_priv *priv = dev->data->dev_private;
1068         int i;
1069
1070         if (!priv->ppio)
1071                 return;
1072
1073         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1074                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1075
1076                 pp2_ppio_inq_get_statistics(priv->ppio, priv->rxq_map[i].tc,
1077                                             priv->rxq_map[i].inq, NULL, 1);
1078                 rxq->bytes_recv = 0;
1079                 rxq->drop_mac = 0;
1080         }
1081
1082         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1083                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1084
1085                 pp2_ppio_outq_get_statistics(priv->ppio, i, NULL, 1);
1086                 txq->bytes_sent = 0;
1087         }
1088
1089         pp2_ppio_get_statistics(priv->ppio, NULL, 1);
1090 }
1091
1092 /**
1093  * DPDK callback to get information about the device.
1094  *
1095  * @param dev
1096  *   Pointer to Ethernet device structure (unused).
1097  * @param info
1098  *   Info structure output buffer.
1099  */
1100 static void
1101 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
1102                    struct rte_eth_dev_info *info)
1103 {
1104         info->speed_capa = ETH_LINK_SPEED_10M |
1105                            ETH_LINK_SPEED_100M |
1106                            ETH_LINK_SPEED_1G |
1107                            ETH_LINK_SPEED_10G;
1108
1109         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
1110         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
1111         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
1112
1113         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
1114         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
1115         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
1116
1117         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
1118         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
1119         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
1120
1121         info->rx_offload_capa = DEV_RX_OFFLOAD_JUMBO_FRAME |
1122                                 DEV_RX_OFFLOAD_VLAN_FILTER |
1123                                 DEV_RX_OFFLOAD_IPV4_CKSUM |
1124                                 DEV_RX_OFFLOAD_UDP_CKSUM |
1125                                 DEV_RX_OFFLOAD_TCP_CKSUM;
1126
1127         info->tx_offload_capa = DEV_TX_OFFLOAD_IPV4_CKSUM |
1128                                 DEV_TX_OFFLOAD_UDP_CKSUM |
1129                                 DEV_TX_OFFLOAD_TCP_CKSUM;
1130
1131         info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1132                                        ETH_RSS_NONFRAG_IPV4_TCP |
1133                                        ETH_RSS_NONFRAG_IPV4_UDP;
1134
1135         /* By default packets are dropped if no descriptors are available */
1136         info->default_rxconf.rx_drop_en = 1;
1137
1138         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
1139 }
1140
1141 /**
1142  * Return supported packet types.
1143  *
1144  * @param dev
1145  *   Pointer to Ethernet device structure (unused).
1146  *
1147  * @return
1148  *   Const pointer to the table with supported packet types.
1149  */
1150 static const uint32_t *
1151 mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1152 {
1153         static const uint32_t ptypes[] = {
1154                 RTE_PTYPE_L2_ETHER,
1155                 RTE_PTYPE_L3_IPV4,
1156                 RTE_PTYPE_L3_IPV4_EXT,
1157                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1158                 RTE_PTYPE_L3_IPV6,
1159                 RTE_PTYPE_L3_IPV6_EXT,
1160                 RTE_PTYPE_L2_ETHER_ARP,
1161                 RTE_PTYPE_L4_TCP,
1162                 RTE_PTYPE_L4_UDP
1163         };
1164
1165         return ptypes;
1166 }
1167
1168 /**
1169  * DPDK callback to get information about specific receive queue.
1170  *
1171  * @param dev
1172  *   Pointer to Ethernet device structure.
1173  * @param rx_queue_id
1174  *   Receive queue index.
1175  * @param qinfo
1176  *   Receive queue information structure.
1177  */
1178 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1179                               struct rte_eth_rxq_info *qinfo)
1180 {
1181         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
1182         struct mrvl_priv *priv = dev->data->dev_private;
1183         int inq = priv->rxq_map[rx_queue_id].inq;
1184         int tc = priv->rxq_map[rx_queue_id].tc;
1185         struct pp2_ppio_tc_params *tc_params =
1186                 &priv->ppio_params.inqs_params.tcs_params[tc];
1187
1188         qinfo->mp = q->mp;
1189         qinfo->nb_desc = tc_params->inqs_params[inq].size;
1190 }
1191
1192 /**
1193  * DPDK callback to get information about specific transmit queue.
1194  *
1195  * @param dev
1196  *   Pointer to Ethernet device structure.
1197  * @param tx_queue_id
1198  *   Transmit queue index.
1199  * @param qinfo
1200  *   Transmit queue information structure.
1201  */
1202 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1203                               struct rte_eth_txq_info *qinfo)
1204 {
1205         struct mrvl_priv *priv = dev->data->dev_private;
1206
1207         qinfo->nb_desc =
1208                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
1209 }
1210
1211 /**
1212  * DPDK callback to Configure a VLAN filter.
1213  *
1214  * @param dev
1215  *   Pointer to Ethernet device structure.
1216  * @param vlan_id
1217  *   VLAN ID to filter.
1218  * @param on
1219  *   Toggle filter.
1220  *
1221  * @return
1222  *   0 on success, negative error value otherwise.
1223  */
1224 static int
1225 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1226 {
1227         struct mrvl_priv *priv = dev->data->dev_private;
1228
1229         if (!priv->ppio)
1230                 return -EPERM;
1231
1232         return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) :
1233                     pp2_ppio_remove_vlan(priv->ppio, vlan_id);
1234 }
1235
1236 /**
1237  * Release buffers to hardware bpool (buffer-pool)
1238  *
1239  * @param rxq
1240  *   Receive queue pointer.
1241  * @param num
1242  *   Number of buffers to release to bpool.
1243  *
1244  * @return
1245  *   0 on success, negative error value otherwise.
1246  */
1247 static int
1248 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
1249 {
1250         struct buff_release_entry entries[MRVL_PP2_TXD_MAX];
1251         struct rte_mbuf *mbufs[MRVL_PP2_TXD_MAX];
1252         int i, ret;
1253         unsigned int core_id;
1254         struct pp2_hif *hif;
1255         struct pp2_bpool *bpool;
1256
1257         core_id = rte_lcore_id();
1258         if (core_id == LCORE_ID_ANY)
1259                 core_id = 0;
1260
1261         hif = mrvl_get_hif(rxq->priv, core_id);
1262         if (!hif)
1263                 return -1;
1264
1265         bpool = rxq->priv->bpool;
1266
1267         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
1268         if (ret)
1269                 return ret;
1270
1271         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
1272                 cookie_addr_high =
1273                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
1274
1275         for (i = 0; i < num; i++) {
1276                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
1277                         != cookie_addr_high) {
1278                         RTE_LOG(ERR, PMD,
1279                                 "mbuf virtual addr high 0x%lx out of range\n",
1280                                 (uint64_t)mbufs[i] >> 32);
1281                         goto out;
1282                 }
1283
1284                 entries[i].buff.addr =
1285                         rte_mbuf_data_iova_default(mbufs[i]);
1286                 entries[i].buff.cookie = (pp2_cookie_t)(uint64_t)mbufs[i];
1287                 entries[i].bpool = bpool;
1288         }
1289
1290         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
1291         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
1292
1293         if (i != num)
1294                 goto out;
1295
1296         return 0;
1297 out:
1298         for (; i < num; i++)
1299                 rte_pktmbuf_free(mbufs[i]);
1300
1301         return -1;
1302 }
1303
1304 /**
1305  * DPDK callback to configure the receive queue.
1306  *
1307  * @param dev
1308  *   Pointer to Ethernet device structure.
1309  * @param idx
1310  *   RX queue index.
1311  * @param desc
1312  *   Number of descriptors to configure in queue.
1313  * @param socket
1314  *   NUMA socket on which memory must be allocated.
1315  * @param conf
1316  *   Thresholds parameters (unused_).
1317  * @param mp
1318  *   Memory pool for buffer allocations.
1319  *
1320  * @return
1321  *   0 on success, negative error value otherwise.
1322  */
1323 static int
1324 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1325                     unsigned int socket,
1326                     const struct rte_eth_rxconf *conf __rte_unused,
1327                     struct rte_mempool *mp)
1328 {
1329         struct mrvl_priv *priv = dev->data->dev_private;
1330         struct mrvl_rxq *rxq;
1331         uint32_t min_size,
1332                  max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1333         int ret, tc, inq;
1334
1335         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
1336                 /*
1337                  * Unknown TC mapping, mapping will not have a correct queue.
1338                  */
1339                 RTE_LOG(ERR, PMD, "Unknown TC mapping for queue %hu eth%hhu\n",
1340                         idx, priv->ppio_id);
1341                 return -EFAULT;
1342         }
1343
1344         min_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM -
1345                    MRVL_PKT_EFFEC_OFFS;
1346         if (min_size < max_rx_pkt_len) {
1347                 RTE_LOG(ERR, PMD,
1348                         "Mbuf size must be increased to %u bytes to hold up to %u bytes of data.\n",
1349                         max_rx_pkt_len + RTE_PKTMBUF_HEADROOM +
1350                         MRVL_PKT_EFFEC_OFFS,
1351                         max_rx_pkt_len);
1352                 return -EINVAL;
1353         }
1354
1355         if (dev->data->rx_queues[idx]) {
1356                 rte_free(dev->data->rx_queues[idx]);
1357                 dev->data->rx_queues[idx] = NULL;
1358         }
1359
1360         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
1361         if (!rxq)
1362                 return -ENOMEM;
1363
1364         rxq->priv = priv;
1365         rxq->mp = mp;
1366         rxq->cksum_enabled = dev->data->dev_conf.rxmode.hw_ip_checksum;
1367         rxq->queue_id = idx;
1368         rxq->port_id = dev->data->port_id;
1369         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
1370
1371         tc = priv->rxq_map[rxq->queue_id].tc,
1372         inq = priv->rxq_map[rxq->queue_id].inq;
1373         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
1374                 desc;
1375
1376         ret = mrvl_fill_bpool(rxq, desc);
1377         if (ret) {
1378                 rte_free(rxq);
1379                 return ret;
1380         }
1381
1382         priv->bpool_init_size += desc;
1383
1384         dev->data->rx_queues[idx] = rxq;
1385
1386         return 0;
1387 }
1388
1389 /**
1390  * DPDK callback to release the receive queue.
1391  *
1392  * @param rxq
1393  *   Generic receive queue pointer.
1394  */
1395 static void
1396 mrvl_rx_queue_release(void *rxq)
1397 {
1398         struct mrvl_rxq *q = rxq;
1399         struct pp2_ppio_tc_params *tc_params;
1400         int i, num, tc, inq;
1401         struct pp2_hif *hif;
1402         unsigned int core_id = rte_lcore_id();
1403
1404         if (core_id == LCORE_ID_ANY)
1405                 core_id = 0;
1406
1407         hif = mrvl_get_hif(q->priv, core_id);
1408
1409         if (!q || !hif)
1410                 return;
1411
1412         tc = q->priv->rxq_map[q->queue_id].tc;
1413         inq = q->priv->rxq_map[q->queue_id].inq;
1414         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
1415         num = tc_params->inqs_params[inq].size;
1416         for (i = 0; i < num; i++) {
1417                 struct pp2_buff_inf inf;
1418                 uint64_t addr;
1419
1420                 pp2_bpool_get_buff(hif, q->priv->bpool, &inf);
1421                 addr = cookie_addr_high | inf.cookie;
1422                 rte_pktmbuf_free((struct rte_mbuf *)addr);
1423         }
1424
1425         rte_free(q);
1426 }
1427
1428 /**
1429  * DPDK callback to configure the transmit queue.
1430  *
1431  * @param dev
1432  *   Pointer to Ethernet device structure.
1433  * @param idx
1434  *   Transmit queue index.
1435  * @param desc
1436  *   Number of descriptors to configure in the queue.
1437  * @param socket
1438  *   NUMA socket on which memory must be allocated.
1439  * @param conf
1440  *   Thresholds parameters (unused).
1441  *
1442  * @return
1443  *   0 on success, negative error value otherwise.
1444  */
1445 static int
1446 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1447                     unsigned int socket,
1448                     const struct rte_eth_txconf *conf __rte_unused)
1449 {
1450         struct mrvl_priv *priv = dev->data->dev_private;
1451         struct mrvl_txq *txq;
1452
1453         if (dev->data->tx_queues[idx]) {
1454                 rte_free(dev->data->tx_queues[idx]);
1455                 dev->data->tx_queues[idx] = NULL;
1456         }
1457
1458         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
1459         if (!txq)
1460                 return -ENOMEM;
1461
1462         txq->priv = priv;
1463         txq->queue_id = idx;
1464         txq->port_id = dev->data->port_id;
1465         dev->data->tx_queues[idx] = txq;
1466
1467         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
1468         priv->ppio_params.outqs_params.outqs_params[idx].weight = 1;
1469
1470         return 0;
1471 }
1472
1473 /**
1474  * DPDK callback to release the transmit queue.
1475  *
1476  * @param txq
1477  *   Generic transmit queue pointer.
1478  */
1479 static void
1480 mrvl_tx_queue_release(void *txq)
1481 {
1482         struct mrvl_txq *q = txq;
1483
1484         if (!q)
1485                 return;
1486
1487         rte_free(q);
1488 }
1489
1490 /**
1491  * Update RSS hash configuration
1492  *
1493  * @param dev
1494  *   Pointer to Ethernet device structure.
1495  * @param rss_conf
1496  *   Pointer to RSS configuration.
1497  *
1498  * @return
1499  *   0 on success, negative error value otherwise.
1500  */
1501 static int
1502 mrvl_rss_hash_update(struct rte_eth_dev *dev,
1503                      struct rte_eth_rss_conf *rss_conf)
1504 {
1505         struct mrvl_priv *priv = dev->data->dev_private;
1506
1507         return mrvl_configure_rss(priv, rss_conf);
1508 }
1509
1510 /**
1511  * DPDK callback to get RSS hash configuration.
1512  *
1513  * @param dev
1514  *   Pointer to Ethernet device structure.
1515  * @rss_conf
1516  *   Pointer to RSS configuration.
1517  *
1518  * @return
1519  *   Always 0.
1520  */
1521 static int
1522 mrvl_rss_hash_conf_get(struct rte_eth_dev *dev,
1523                        struct rte_eth_rss_conf *rss_conf)
1524 {
1525         struct mrvl_priv *priv = dev->data->dev_private;
1526         enum pp2_ppio_hash_type hash_type =
1527                 priv->ppio_params.inqs_params.hash_type;
1528
1529         rss_conf->rss_key = NULL;
1530
1531         if (hash_type == PP2_PPIO_HASH_T_NONE)
1532                 rss_conf->rss_hf = 0;
1533         else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE)
1534                 rss_conf->rss_hf = ETH_RSS_IPV4;
1535         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp)
1536                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP;
1537         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp)
1538                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP;
1539
1540         return 0;
1541 }
1542
1543 static const struct eth_dev_ops mrvl_ops = {
1544         .dev_configure = mrvl_dev_configure,
1545         .dev_start = mrvl_dev_start,
1546         .dev_stop = mrvl_dev_stop,
1547         .dev_set_link_up = mrvl_dev_set_link_up,
1548         .dev_set_link_down = mrvl_dev_set_link_down,
1549         .dev_close = mrvl_dev_close,
1550         .link_update = mrvl_link_update,
1551         .promiscuous_enable = mrvl_promiscuous_enable,
1552         .allmulticast_enable = mrvl_allmulticast_enable,
1553         .promiscuous_disable = mrvl_promiscuous_disable,
1554         .allmulticast_disable = mrvl_allmulticast_disable,
1555         .mac_addr_remove = mrvl_mac_addr_remove,
1556         .mac_addr_add = mrvl_mac_addr_add,
1557         .mac_addr_set = mrvl_mac_addr_set,
1558         .mtu_set = mrvl_mtu_set,
1559         .stats_get = mrvl_stats_get,
1560         .stats_reset = mrvl_stats_reset,
1561         .dev_infos_get = mrvl_dev_infos_get,
1562         .dev_supported_ptypes_get = mrvl_dev_supported_ptypes_get,
1563         .rxq_info_get = mrvl_rxq_info_get,
1564         .txq_info_get = mrvl_txq_info_get,
1565         .vlan_filter_set = mrvl_vlan_filter_set,
1566         .rx_queue_setup = mrvl_rx_queue_setup,
1567         .rx_queue_release = mrvl_rx_queue_release,
1568         .tx_queue_setup = mrvl_tx_queue_setup,
1569         .tx_queue_release = mrvl_tx_queue_release,
1570         .rss_hash_update = mrvl_rss_hash_update,
1571         .rss_hash_conf_get = mrvl_rss_hash_conf_get,
1572 };
1573
1574 /**
1575  * Return packet type information and l3/l4 offsets.
1576  *
1577  * @param desc
1578  *   Pointer to the received packet descriptor.
1579  * @param l3_offset
1580  *   l3 packet offset.
1581  * @param l4_offset
1582  *   l4 packet offset.
1583  *
1584  * @return
1585  *   Packet type information.
1586  */
1587 static inline uint64_t
1588 mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
1589                                     uint8_t *l3_offset, uint8_t *l4_offset)
1590 {
1591         enum pp2_inq_l3_type l3_type;
1592         enum pp2_inq_l4_type l4_type;
1593         uint64_t packet_type;
1594
1595         pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
1596         pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
1597
1598         packet_type = RTE_PTYPE_L2_ETHER;
1599
1600         switch (l3_type) {
1601         case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
1602                 packet_type |= RTE_PTYPE_L3_IPV4;
1603                 break;
1604         case PP2_INQ_L3_TYPE_IPV4_OK:
1605                 packet_type |= RTE_PTYPE_L3_IPV4_EXT;
1606                 break;
1607         case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO:
1608                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
1609                 break;
1610         case PP2_INQ_L3_TYPE_IPV6_NO_EXT:
1611                 packet_type |= RTE_PTYPE_L3_IPV6;
1612                 break;
1613         case PP2_INQ_L3_TYPE_IPV6_EXT:
1614                 packet_type |= RTE_PTYPE_L3_IPV6_EXT;
1615                 break;
1616         case PP2_INQ_L3_TYPE_ARP:
1617                 packet_type |= RTE_PTYPE_L2_ETHER_ARP;
1618                 /*
1619                  * In case of ARP l4_offset is set to wrong value.
1620                  * Set it to proper one so that later on mbuf->l3_len can be
1621                  * calculated subtracting l4_offset and l3_offset.
1622                  */
1623                 *l4_offset = *l3_offset + MRVL_ARP_LENGTH;
1624                 break;
1625         default:
1626                 RTE_LOG(DEBUG, PMD, "Failed to recognise l3 packet type\n");
1627                 break;
1628         }
1629
1630         switch (l4_type) {
1631         case PP2_INQ_L4_TYPE_TCP:
1632                 packet_type |= RTE_PTYPE_L4_TCP;
1633                 break;
1634         case PP2_INQ_L4_TYPE_UDP:
1635                 packet_type |= RTE_PTYPE_L4_UDP;
1636                 break;
1637         default:
1638                 RTE_LOG(DEBUG, PMD, "Failed to recognise l4 packet type\n");
1639                 break;
1640         }
1641
1642         return packet_type;
1643 }
1644
1645 /**
1646  * Get offload information from the received packet descriptor.
1647  *
1648  * @param desc
1649  *   Pointer to the received packet descriptor.
1650  *
1651  * @return
1652  *   Mbuf offload flags.
1653  */
1654 static inline uint64_t
1655 mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc)
1656 {
1657         uint64_t flags;
1658         enum pp2_inq_desc_status status;
1659
1660         status = pp2_ppio_inq_desc_get_l3_pkt_error(desc);
1661         if (unlikely(status != PP2_DESC_ERR_OK))
1662                 flags = PKT_RX_IP_CKSUM_BAD;
1663         else
1664                 flags = PKT_RX_IP_CKSUM_GOOD;
1665
1666         status = pp2_ppio_inq_desc_get_l4_pkt_error(desc);
1667         if (unlikely(status != PP2_DESC_ERR_OK))
1668                 flags |= PKT_RX_L4_CKSUM_BAD;
1669         else
1670                 flags |= PKT_RX_L4_CKSUM_GOOD;
1671
1672         return flags;
1673 }
1674
1675 /**
1676  * DPDK callback for receive.
1677  *
1678  * @param rxq
1679  *   Generic pointer to the receive queue.
1680  * @param rx_pkts
1681  *   Array to store received packets.
1682  * @param nb_pkts
1683  *   Maximum number of packets in array.
1684  *
1685  * @return
1686  *   Number of packets successfully received.
1687  */
1688 static uint16_t
1689 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
1690 {
1691         struct mrvl_rxq *q = rxq;
1692         struct pp2_ppio_desc descs[nb_pkts];
1693         struct pp2_bpool *bpool;
1694         int i, ret, rx_done = 0;
1695         int num;
1696         struct pp2_hif *hif;
1697         unsigned int core_id = rte_lcore_id();
1698
1699         hif = mrvl_get_hif(q->priv, core_id);
1700
1701         if (unlikely(!q->priv->ppio || !hif))
1702                 return 0;
1703
1704         bpool = q->priv->bpool;
1705
1706         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
1707                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
1708         if (unlikely(ret < 0)) {
1709                 RTE_LOG(ERR, PMD, "Failed to receive packets\n");
1710                 return 0;
1711         }
1712         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
1713
1714         for (i = 0; i < nb_pkts; i++) {
1715                 struct rte_mbuf *mbuf;
1716                 uint8_t l3_offset, l4_offset;
1717                 enum pp2_inq_desc_status status;
1718                 uint64_t addr;
1719
1720                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
1721                         struct pp2_ppio_desc *pref_desc;
1722                         u64 pref_addr;
1723
1724                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
1725                         pref_addr = cookie_addr_high |
1726                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
1727                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
1728                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
1729                 }
1730
1731                 addr = cookie_addr_high |
1732                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
1733                 mbuf = (struct rte_mbuf *)addr;
1734                 rte_pktmbuf_reset(mbuf);
1735
1736                 /* drop packet in case of mac, overrun or resource error */
1737                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
1738                 if (unlikely(status != PP2_DESC_ERR_OK)) {
1739                         struct pp2_buff_inf binf = {
1740                                 .addr = rte_mbuf_data_iova_default(mbuf),
1741                                 .cookie = (pp2_cookie_t)(uint64_t)mbuf,
1742                         };
1743
1744                         pp2_bpool_put_buff(hif, bpool, &binf);
1745                         mrvl_port_bpool_size
1746                                 [bpool->pp2_id][bpool->id][core_id]++;
1747                         q->drop_mac++;
1748                         continue;
1749                 }
1750
1751                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
1752                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
1753                 mbuf->data_len = mbuf->pkt_len;
1754                 mbuf->port = q->port_id;
1755                 mbuf->packet_type =
1756                         mrvl_desc_to_packet_type_and_offset(&descs[i],
1757                                                             &l3_offset,
1758                                                             &l4_offset);
1759                 mbuf->l2_len = l3_offset;
1760                 mbuf->l3_len = l4_offset - l3_offset;
1761
1762                 if (likely(q->cksum_enabled))
1763                         mbuf->ol_flags = mrvl_desc_to_ol_flags(&descs[i]);
1764
1765                 rx_pkts[rx_done++] = mbuf;
1766                 q->bytes_recv += mbuf->pkt_len;
1767         }
1768
1769         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
1770                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
1771
1772                 if (unlikely(num <= q->priv->bpool_min_size ||
1773                              (!rx_done && num < q->priv->bpool_init_size))) {
1774                         ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE);
1775                         if (ret)
1776                                 RTE_LOG(ERR, PMD, "Failed to fill bpool\n");
1777                 } else if (unlikely(num > q->priv->bpool_max_size)) {
1778                         int i;
1779                         int pkt_to_remove = num - q->priv->bpool_init_size;
1780                         struct rte_mbuf *mbuf;
1781                         struct pp2_buff_inf buff;
1782
1783                         RTE_LOG(DEBUG, PMD,
1784                                 "\nport-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)\n",
1785                                 bpool->pp2_id, q->priv->ppio->port_id,
1786                                 bpool->id, pkt_to_remove, num,
1787                                 q->priv->bpool_init_size);
1788
1789                         for (i = 0; i < pkt_to_remove; i++) {
1790                                 ret = pp2_bpool_get_buff(hif, bpool, &buff);
1791                                 if (ret)
1792                                         break;
1793                                 mbuf = (struct rte_mbuf *)
1794                                         (cookie_addr_high | buff.cookie);
1795                                 rte_pktmbuf_free(mbuf);
1796                         }
1797                         mrvl_port_bpool_size
1798                                 [bpool->pp2_id][bpool->id][core_id] -= i;
1799                 }
1800                 rte_spinlock_unlock(&q->priv->lock);
1801         }
1802
1803         return rx_done;
1804 }
1805
1806 /**
1807  * Prepare offload information.
1808  *
1809  * @param ol_flags
1810  *   Offload flags.
1811  * @param packet_type
1812  *   Packet type bitfield.
1813  * @param l3_type
1814  *   Pointer to the pp2_ouq_l3_type structure.
1815  * @param l4_type
1816  *   Pointer to the pp2_outq_l4_type structure.
1817  * @param gen_l3_cksum
1818  *   Will be set to 1 in case l3 checksum is computed.
1819  * @param l4_cksum
1820  *   Will be set to 1 in case l4 checksum is computed.
1821  *
1822  * @return
1823  *   0 on success, negative error value otherwise.
1824  */
1825 static inline int
1826 mrvl_prepare_proto_info(uint64_t ol_flags, uint32_t packet_type,
1827                         enum pp2_outq_l3_type *l3_type,
1828                         enum pp2_outq_l4_type *l4_type,
1829                         int *gen_l3_cksum,
1830                         int *gen_l4_cksum)
1831 {
1832         /*
1833          * Based on ol_flags prepare information
1834          * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor
1835          * for offloading.
1836          */
1837         if (ol_flags & PKT_TX_IPV4) {
1838                 *l3_type = PP2_OUTQ_L3_TYPE_IPV4;
1839                 *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0;
1840         } else if (ol_flags & PKT_TX_IPV6) {
1841                 *l3_type = PP2_OUTQ_L3_TYPE_IPV6;
1842                 /* no checksum for ipv6 header */
1843                 *gen_l3_cksum = 0;
1844         } else {
1845                 /* if something different then stop processing */
1846                 return -1;
1847         }
1848
1849         ol_flags &= PKT_TX_L4_MASK;
1850         if ((packet_type & RTE_PTYPE_L4_TCP) &&
1851             ol_flags == PKT_TX_TCP_CKSUM) {
1852                 *l4_type = PP2_OUTQ_L4_TYPE_TCP;
1853                 *gen_l4_cksum = 1;
1854         } else if ((packet_type & RTE_PTYPE_L4_UDP) &&
1855                    ol_flags == PKT_TX_UDP_CKSUM) {
1856                 *l4_type = PP2_OUTQ_L4_TYPE_UDP;
1857                 *gen_l4_cksum = 1;
1858         } else {
1859                 *l4_type = PP2_OUTQ_L4_TYPE_OTHER;
1860                 /* no checksum for other type */
1861                 *gen_l4_cksum = 0;
1862         }
1863
1864         return 0;
1865 }
1866
1867 /**
1868  * Release already sent buffers to bpool (buffer-pool).
1869  *
1870  * @param ppio
1871  *   Pointer to the port structure.
1872  * @param hif
1873  *   Pointer to the MUSDK hardware interface.
1874  * @param sq
1875  *   Pointer to the shadow queue.
1876  * @param qid
1877  *   Queue id number.
1878  * @param force
1879  *   Force releasing packets.
1880  */
1881 static inline void
1882 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
1883                        unsigned int core_id, struct mrvl_shadow_txq *sq,
1884                        int qid, int force)
1885 {
1886         struct buff_release_entry *entry;
1887         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
1888         int i;
1889
1890         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
1891
1892         sq->num_to_release += nb_done;
1893
1894         if (likely(!force &&
1895                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
1896                 return;
1897
1898         nb_done = sq->num_to_release;
1899         sq->num_to_release = 0;
1900
1901         for (i = 0; i < nb_done; i++) {
1902                 entry = &sq->ent[sq->tail + num];
1903                 if (unlikely(!entry->buff.addr)) {
1904                         RTE_LOG(ERR, PMD,
1905                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!\n",
1906                                 sq->tail, (u64)entry->buff.cookie,
1907                                 (u64)entry->buff.addr);
1908                         skip_bufs = 1;
1909                         goto skip;
1910                 }
1911
1912                 if (unlikely(!entry->bpool)) {
1913                         struct rte_mbuf *mbuf;
1914
1915                         mbuf = (struct rte_mbuf *)
1916                                (cookie_addr_high | entry->buff.cookie);
1917                         rte_pktmbuf_free(mbuf);
1918                         skip_bufs = 1;
1919                         goto skip;
1920                 }
1921
1922                 mrvl_port_bpool_size
1923                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
1924                 num++;
1925                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
1926                         goto skip;
1927                 continue;
1928 skip:
1929                 if (likely(num))
1930                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
1931                 num += skip_bufs;
1932                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
1933                 sq->size -= num;
1934                 num = 0;
1935         }
1936
1937         if (likely(num)) {
1938                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
1939                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
1940                 sq->size -= num;
1941         }
1942 }
1943
1944 /**
1945  * DPDK callback for transmit.
1946  *
1947  * @param txq
1948  *   Generic pointer transmit queue.
1949  * @param tx_pkts
1950  *   Packets to transmit.
1951  * @param nb_pkts
1952  *   Number of packets in array.
1953  *
1954  * @return
1955  *   Number of packets successfully transmitted.
1956  */
1957 static uint16_t
1958 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1959 {
1960         struct mrvl_txq *q = txq;
1961         struct mrvl_shadow_txq *sq = &shadow_txqs[q->port_id][rte_lcore_id()];
1962         struct pp2_hif *hif;
1963         struct pp2_ppio_desc descs[nb_pkts];
1964         unsigned int core_id = rte_lcore_id();
1965         int i, ret, bytes_sent = 0;
1966         uint16_t num, sq_free_size;
1967         uint64_t addr;
1968
1969         hif = mrvl_get_hif(q->priv, core_id);
1970
1971         if (unlikely(!q->priv->ppio || !hif))
1972                 return 0;
1973
1974         if (sq->size)
1975                 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
1976                                        sq, q->queue_id, 0);
1977
1978         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
1979         if (unlikely(nb_pkts > sq_free_size)) {
1980                 RTE_LOG(DEBUG, PMD,
1981                         "No room in shadow queue for %d packets! %d packets will be sent.\n",
1982                         nb_pkts, sq_free_size);
1983                 nb_pkts = sq_free_size;
1984         }
1985
1986         for (i = 0; i < nb_pkts; i++) {
1987                 struct rte_mbuf *mbuf = tx_pkts[i];
1988                 int gen_l3_cksum, gen_l4_cksum;
1989                 enum pp2_outq_l3_type l3_type;
1990                 enum pp2_outq_l4_type l4_type;
1991
1992                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
1993                         struct rte_mbuf *pref_pkt_hdr;
1994
1995                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
1996                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
1997                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
1998                 }
1999
2000                 sq->ent[sq->head].buff.cookie = (pp2_cookie_t)(uint64_t)mbuf;
2001                 sq->ent[sq->head].buff.addr =
2002                         rte_mbuf_data_iova_default(mbuf);
2003                 sq->ent[sq->head].bpool =
2004                         (unlikely(mbuf->port == 0xff || mbuf->refcnt > 1)) ?
2005                          NULL : mrvl_port_to_bpool_lookup[mbuf->port];
2006                 sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
2007                 sq->size++;
2008
2009                 pp2_ppio_outq_desc_reset(&descs[i]);
2010                 pp2_ppio_outq_desc_set_phys_addr(&descs[i],
2011                                                  rte_pktmbuf_iova(mbuf));
2012                 pp2_ppio_outq_desc_set_pkt_offset(&descs[i], 0);
2013                 pp2_ppio_outq_desc_set_pkt_len(&descs[i],
2014                                                rte_pktmbuf_pkt_len(mbuf));
2015
2016                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
2017                 /*
2018                  * in case unsupported ol_flags were passed
2019                  * do not update descriptor offload information
2020                  */
2021                 ret = mrvl_prepare_proto_info(mbuf->ol_flags, mbuf->packet_type,
2022                                               &l3_type, &l4_type, &gen_l3_cksum,
2023                                               &gen_l4_cksum);
2024                 if (unlikely(ret))
2025                         continue;
2026
2027                 pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type,
2028                                                   mbuf->l2_len,
2029                                                   mbuf->l2_len + mbuf->l3_len,
2030                                                   gen_l3_cksum, gen_l4_cksum);
2031         }
2032
2033         num = nb_pkts;
2034         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
2035         /* number of packets that were not sent */
2036         if (unlikely(num > nb_pkts)) {
2037                 for (i = nb_pkts; i < num; i++) {
2038                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
2039                                 MRVL_PP2_TX_SHADOWQ_MASK;
2040                         addr = cookie_addr_high | sq->ent[sq->head].buff.cookie;
2041                         bytes_sent -=
2042                                 rte_pktmbuf_pkt_len((struct rte_mbuf *)addr);
2043                 }
2044                 sq->size -= num - nb_pkts;
2045         }
2046
2047         q->bytes_sent += bytes_sent;
2048
2049         return nb_pkts;
2050 }
2051
2052 /**
2053  * Initialize packet processor.
2054  *
2055  * @return
2056  *   0 on success, negative error value otherwise.
2057  */
2058 static int
2059 mrvl_init_pp2(void)
2060 {
2061         struct pp2_init_params init_params;
2062
2063         memset(&init_params, 0, sizeof(init_params));
2064         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
2065         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
2066         init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED;
2067
2068         return pp2_init(&init_params);
2069 }
2070
2071 /**
2072  * Deinitialize packet processor.
2073  *
2074  * @return
2075  *   0 on success, negative error value otherwise.
2076  */
2077 static void
2078 mrvl_deinit_pp2(void)
2079 {
2080         pp2_deinit();
2081 }
2082
2083 /**
2084  * Create private device structure.
2085  *
2086  * @param dev_name
2087  *   Pointer to the port name passed in the initialization parameters.
2088  *
2089  * @return
2090  *   Pointer to the newly allocated private device structure.
2091  */
2092 static struct mrvl_priv *
2093 mrvl_priv_create(const char *dev_name)
2094 {
2095         struct pp2_bpool_params bpool_params;
2096         char match[MRVL_MATCH_LEN];
2097         struct mrvl_priv *priv;
2098         int ret, bpool_bit;
2099
2100         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
2101         if (!priv)
2102                 return NULL;
2103
2104         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
2105                                        &priv->pp_id, &priv->ppio_id);
2106         if (ret)
2107                 goto out_free_priv;
2108
2109         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
2110                                      PP2_BPOOL_NUM_POOLS);
2111         if (bpool_bit < 0)
2112                 goto out_free_priv;
2113         priv->bpool_bit = bpool_bit;
2114
2115         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
2116                  priv->bpool_bit);
2117         memset(&bpool_params, 0, sizeof(bpool_params));
2118         bpool_params.match = match;
2119         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
2120         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
2121         if (ret)
2122                 goto out_clear_bpool_bit;
2123
2124         priv->ppio_params.type = PP2_PPIO_T_NIC;
2125         rte_spinlock_init(&priv->lock);
2126
2127         return priv;
2128 out_clear_bpool_bit:
2129         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2130 out_free_priv:
2131         rte_free(priv);
2132         return NULL;
2133 }
2134
2135 /**
2136  * Create device representing Ethernet port.
2137  *
2138  * @param name
2139  *   Pointer to the port's name.
2140  *
2141  * @return
2142  *   0 on success, negative error value otherwise.
2143  */
2144 static int
2145 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
2146 {
2147         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
2148         struct rte_eth_dev *eth_dev;
2149         struct mrvl_priv *priv;
2150         struct ifreq req;
2151
2152         eth_dev = rte_eth_dev_allocate(name);
2153         if (!eth_dev)
2154                 return -ENOMEM;
2155
2156         priv = mrvl_priv_create(name);
2157         if (!priv) {
2158                 ret = -ENOMEM;
2159                 goto out_free_dev;
2160         }
2161
2162         eth_dev->data->mac_addrs =
2163                 rte_zmalloc("mac_addrs",
2164                             ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
2165         if (!eth_dev->data->mac_addrs) {
2166                 RTE_LOG(ERR, PMD, "Failed to allocate space for eth addrs\n");
2167                 ret = -ENOMEM;
2168                 goto out_free_priv;
2169         }
2170
2171         memset(&req, 0, sizeof(req));
2172         strcpy(req.ifr_name, name);
2173         ret = ioctl(fd, SIOCGIFHWADDR, &req);
2174         if (ret)
2175                 goto out_free_mac;
2176
2177         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
2178                req.ifr_addr.sa_data, ETHER_ADDR_LEN);
2179
2180         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
2181         eth_dev->tx_pkt_burst = mrvl_tx_pkt_burst;
2182         eth_dev->data->kdrv = RTE_KDRV_NONE;
2183         eth_dev->data->dev_private = priv;
2184         eth_dev->device = &vdev->device;
2185         eth_dev->dev_ops = &mrvl_ops;
2186
2187         return 0;
2188 out_free_mac:
2189         rte_free(eth_dev->data->mac_addrs);
2190 out_free_dev:
2191         rte_eth_dev_release_port(eth_dev);
2192 out_free_priv:
2193         rte_free(priv);
2194
2195         return ret;
2196 }
2197
2198 /**
2199  * Cleanup previously created device representing Ethernet port.
2200  *
2201  * @param name
2202  *   Pointer to the port name.
2203  */
2204 static void
2205 mrvl_eth_dev_destroy(const char *name)
2206 {
2207         struct rte_eth_dev *eth_dev;
2208         struct mrvl_priv *priv;
2209
2210         eth_dev = rte_eth_dev_allocated(name);
2211         if (!eth_dev)
2212                 return;
2213
2214         priv = eth_dev->data->dev_private;
2215         pp2_bpool_deinit(priv->bpool);
2216         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2217         rte_free(priv);
2218         rte_free(eth_dev->data->mac_addrs);
2219         rte_eth_dev_release_port(eth_dev);
2220 }
2221
2222 /**
2223  * Callback used by rte_kvargs_process() during argument parsing.
2224  *
2225  * @param key
2226  *   Pointer to the parsed key (unused).
2227  * @param value
2228  *   Pointer to the parsed value.
2229  * @param extra_args
2230  *   Pointer to the extra arguments which contains address of the
2231  *   table of pointers to parsed interface names.
2232  *
2233  * @return
2234  *   Always 0.
2235  */
2236 static int
2237 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
2238                  void *extra_args)
2239 {
2240         struct mrvl_ifnames *ifnames = extra_args;
2241
2242         ifnames->names[ifnames->idx++] = value;
2243
2244         return 0;
2245 }
2246
2247 /**
2248  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
2249  */
2250 static void
2251 mrvl_deinit_hifs(void)
2252 {
2253         int i;
2254
2255         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++) {
2256                 if (hifs[i])
2257                         pp2_hif_deinit(hifs[i]);
2258         }
2259         used_hifs = MRVL_MUSDK_HIFS_RESERVED;
2260         memset(hifs, 0, sizeof(hifs));
2261 }
2262
2263 /**
2264  * DPDK callback to register the virtual device.
2265  *
2266  * @param vdev
2267  *   Pointer to the virtual device.
2268  *
2269  * @return
2270  *   0 on success, negative error value otherwise.
2271  */
2272 static int
2273 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
2274 {
2275         struct rte_kvargs *kvlist;
2276         struct mrvl_ifnames ifnames;
2277         int ret = -EINVAL;
2278         uint32_t i, ifnum, cfgnum;
2279         const char *params;
2280
2281         params = rte_vdev_device_args(vdev);
2282         if (!params)
2283                 return -EINVAL;
2284
2285         kvlist = rte_kvargs_parse(params, valid_args);
2286         if (!kvlist)
2287                 return -EINVAL;
2288
2289         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
2290         if (ifnum > RTE_DIM(ifnames.names))
2291                 goto out_free_kvlist;
2292
2293         ifnames.idx = 0;
2294         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
2295                            mrvl_get_ifnames, &ifnames);
2296
2297
2298         /*
2299          * The below system initialization should be done only once,
2300          * on the first provided configuration file
2301          */
2302         if (!mrvl_qos_cfg) {
2303                 cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
2304                 RTE_LOG(INFO, PMD, "Parsing config file!\n");
2305                 if (cfgnum > 1) {
2306                         RTE_LOG(ERR, PMD, "Cannot handle more than one config file!\n");
2307                         goto out_free_kvlist;
2308                 } else if (cfgnum == 1) {
2309                         rte_kvargs_process(kvlist, MRVL_CFG_ARG,
2310                                            mrvl_get_qoscfg, &mrvl_qos_cfg);
2311                 }
2312         }
2313
2314         if (mrvl_dev_num)
2315                 goto init_devices;
2316
2317         RTE_LOG(INFO, PMD, "Perform MUSDK initializations\n");
2318         /*
2319          * ret == -EEXIST is correct, it means DMA
2320          * has been already initialized (by another PMD).
2321          */
2322         ret = mv_sys_dma_mem_init(MRVL_MUSDK_DMA_MEMSIZE);
2323         if (ret < 0) {
2324                 if (ret != -EEXIST)
2325                         goto out_free_kvlist;
2326                 else
2327                         RTE_LOG(INFO, PMD,
2328                                 "DMA memory has been already initialized by a different driver.\n");
2329         }
2330
2331         ret = mrvl_init_pp2();
2332         if (ret) {
2333                 RTE_LOG(ERR, PMD, "Failed to init PP!\n");
2334                 goto out_deinit_dma;
2335         }
2336
2337         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
2338
2339         mrvl_lcore_first = RTE_MAX_LCORE;
2340         mrvl_lcore_last = 0;
2341
2342 init_devices:
2343         for (i = 0; i < ifnum; i++) {
2344                 RTE_LOG(INFO, PMD, "Creating %s\n", ifnames.names[i]);
2345                 ret = mrvl_eth_dev_create(vdev, ifnames.names[i]);
2346                 if (ret)
2347                         goto out_cleanup;
2348         }
2349         mrvl_dev_num += ifnum;
2350
2351         rte_kvargs_free(kvlist);
2352
2353         return 0;
2354 out_cleanup:
2355         for (; i > 0; i--)
2356                 mrvl_eth_dev_destroy(ifnames.names[i]);
2357
2358         if (mrvl_dev_num == 0)
2359                 mrvl_deinit_pp2();
2360 out_deinit_dma:
2361         if (mrvl_dev_num == 0)
2362                 mv_sys_dma_mem_destroy();
2363 out_free_kvlist:
2364         rte_kvargs_free(kvlist);
2365
2366         return ret;
2367 }
2368
2369 /**
2370  * DPDK callback to remove virtual device.
2371  *
2372  * @param vdev
2373  *   Pointer to the removed virtual device.
2374  *
2375  * @return
2376  *   0 on success, negative error value otherwise.
2377  */
2378 static int
2379 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
2380 {
2381         int i;
2382         const char *name;
2383
2384         name = rte_vdev_device_name(vdev);
2385         if (!name)
2386                 return -EINVAL;
2387
2388         RTE_LOG(INFO, PMD, "Removing %s\n", name);
2389
2390         for (i = 0; i < rte_eth_dev_count(); i++) {
2391                 char ifname[RTE_ETH_NAME_MAX_LEN];
2392
2393                 rte_eth_dev_get_name_by_port(i, ifname);
2394                 mrvl_eth_dev_destroy(ifname);
2395                 mrvl_dev_num--;
2396         }
2397
2398         if (mrvl_dev_num == 0) {
2399                 RTE_LOG(INFO, PMD, "Perform MUSDK deinit\n");
2400                 mrvl_deinit_hifs();
2401                 mrvl_deinit_pp2();
2402                 mv_sys_dma_mem_destroy();
2403         }
2404
2405         return 0;
2406 }
2407
2408 static struct rte_vdev_driver pmd_mrvl_drv = {
2409         .probe = rte_pmd_mrvl_probe,
2410         .remove = rte_pmd_mrvl_remove,
2411 };
2412
2413 RTE_PMD_REGISTER_VDEV(net_mrvl, pmd_mrvl_drv);
2414 RTE_PMD_REGISTER_ALIAS(net_mrvl, eth_mrvl);