drivers/net/mrvl/mrvl_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Semihalf. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Semihalf nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <rte_ethdev.h>
  34 #include <rte_kvargs.h>
  35 #include <rte_log.h>
  36 #include <rte_malloc.h>
  37 #include <rte_vdev.h>
  38
  39 /* Unluckily, container_of is defined by both DPDK and MUSDK,
  40  * we'll declare only one version.
  41  *
  42  * Note that it is not used in this PMD anyway.
  43  */
  44 #ifdef container_of
  45 #undef container_of
  46 #endif
  47
  48 #include <drivers/mv_pp2.h>
  49 #include <drivers/mv_pp2_bpool.h>
  50 #include <drivers/mv_pp2_hif.h>
  51
  52 #include <fcntl.h>
  53 #include <linux/ethtool.h>
  54 #include <linux/sockios.h>
  55 #include <net/if.h>
  56 #include <net/if_arp.h>
  57 #include <sys/ioctl.h>
  58 #include <sys/socket.h>
  59 #include <sys/stat.h>
  60 #include <sys/types.h>
  61
  62 #include "mrvl_ethdev.h"
  63 #include "mrvl_qos.h"
  64
  65 /* bitmask with reserved hifs */
  66 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
  67 /* bitmask with reserved bpools */
  68 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
  69 /* maximum number of available hifs */
  70 #define MRVL_MUSDK_HIFS_MAX 9
  71
  72 #define MRVL_MAC_ADDRS_MAX 1
  73 /* prefetch shift */
  74 #define MRVL_MUSDK_PREFETCH_SHIFT 2
  75
  76 #define MRVL_MATCH_LEN 16
  77 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
  78 /* Maximum allowable packet size */
  79 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
  80
  81 #define MRVL_IFACE_NAME_ARG "iface"
  82 #define MRVL_CFG_ARG "cfg"
  83
  84 #define MRVL_BURST_SIZE 64
  85
  86 #define MRVL_ARP_LENGTH 28
  87
  88 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
  89
  90 #define MRVL_COOKIE_HIGH_ADDR_SHIFT     (sizeof(pp2_cookie_t) * 8)
  91 #define MRVL_COOKIE_HIGH_ADDR_MASK      (~0ULL << MRVL_COOKIE_HIGH_ADDR_SHIFT)
  92
  93 static const char * const valid_args[] = {
  94         MRVL_IFACE_NAME_ARG,
  95         MRVL_CFG_ARG,
  96         NULL
  97 };
  98
  99 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
 100 static struct pp2_hif *hifs[RTE_MAX_LCORE];
 101 static int used_bpools[PP2_NUM_PKT_PROC] = {
 102         MRVL_MUSDK_BPOOLS_RESERVED,
 103         MRVL_MUSDK_BPOOLS_RESERVED
 104 };
 105
 106 struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
 107 int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
 108 uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
 109
 110 /*
 111  * To use buffer harvesting based on loopback port shadow queue structure
 112  * was introduced for buffers information bookkeeping.
 113  *
 114  * Before sending the packet, related buffer information (pp2_buff_inf) is
 115  * stored in shadow queue. After packet is transmitted no longer used
 116  * packet buffer is released back to it's original hardware pool,
 117  * on condition it originated from interface.
 118  * In case it  was generated by application itself i.e: mbuf->port field is
 119  * 0xff then its released to software mempool.
 120  */
 121 struct mrvl_shadow_txq {
 122         int head;           /* write index - used when sending buffers */
 123         int tail;           /* read index - used when releasing buffers */
 124         u16 size;           /* queue occupied size */
 125         u16 num_to_release; /* number of buffers sent, that can be released */
 126         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
 127 };
 128
 129 struct mrvl_rxq {
 130         struct mrvl_priv *priv;
 131         struct rte_mempool *mp;
 132         int queue_id;
 133         int port_id;
 134 };
 135
 136 struct mrvl_txq {
 137         struct mrvl_priv *priv;
 138         int queue_id;
 139         int port_id;
 140 };
 141
 142 /*
 143  * Every tx queue should have dedicated shadow tx queue.
 144  *
 145  * Ports assigned by DPDK might not start at zero or be continuous so
 146  * as a workaround define shadow queues for each possible port so that
 147  * we eventually fit somewhere.
 148  */
 149 struct mrvl_shadow_txq shadow_txqs[RTE_MAX_ETHPORTS][RTE_MAX_LCORE];
 150
 151 /** Number of ports configured. */
 152 int mrvl_ports_nb;
 153 static int mrvl_lcore_first;
 154 static int mrvl_lcore_last;
 155
 156 static inline int
 157 mrvl_get_bpool_size(int pp2_id, int pool_id)
 158 {
 159         int i;
 160         int size = 0;
 161
 162         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
 163                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
 164
 165         return size;
 166 }
 167
 168 static inline int
 169 mrvl_reserve_bit(int *bitmap, int max)
 170 {
 171         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
 172
 173         if (n >= max)
 174                 return -1;
 175
 176         *bitmap |= 1 << n;
 177
 178         return n;
 179 }
 180
 181 /**
 182  * Ethernet device configuration.
 183  *
 184  * Prepare the driver for a given number of TX and RX queues.
 185  *
 186  * @param dev
 187  *   Pointer to Ethernet device structure.
 188  *
 189  * @return
 190  *   0 on success, negative error value otherwise.
 191  */
 192 static int
 193 mrvl_dev_configure(struct rte_eth_dev *dev)
 194 {
 195         struct mrvl_priv *priv = dev->data->dev_private;
 196         int ret;
 197
 198         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE) {
 199                 RTE_LOG(INFO, PMD, "Unsupported rx multi queue mode %d\n",
 200                         dev->data->dev_conf.rxmode.mq_mode);
 201                 return -EINVAL;
 202         }
 203
 204         if (!dev->data->dev_conf.rxmode.hw_strip_crc) {
 205                 RTE_LOG(INFO, PMD,
 206                         "L2 CRC stripping is always enabled in hw\n");
 207                 dev->data->dev_conf.rxmode.hw_strip_crc = 1;
 208         }
 209
 210         if (dev->data->dev_conf.rxmode.hw_vlan_strip) {
 211                 RTE_LOG(INFO, PMD, "VLAN stripping not supported\n");
 212                 return -EINVAL;
 213         }
 214
 215         if (dev->data->dev_conf.rxmode.split_hdr_size) {
 216                 RTE_LOG(INFO, PMD, "Split headers not supported\n");
 217                 return -EINVAL;
 218         }
 219
 220         if (dev->data->dev_conf.rxmode.enable_scatter) {
 221                 RTE_LOG(INFO, PMD, "RX Scatter/Gather not supported\n");
 222                 return -EINVAL;
 223         }
 224
 225         if (dev->data->dev_conf.rxmode.enable_lro) {
 226                 RTE_LOG(INFO, PMD, "LRO not supported\n");
 227                 return -EINVAL;
 228         }
 229
 230         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
 231                                   dev->data->nb_rx_queues);
 232         if (ret < 0)
 233                 return ret;
 234
 235         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
 236         priv->nb_rx_queues = dev->data->nb_rx_queues;
 237
 238         return 0;
 239 }
 240
 241 /**
 242  * DPDK callback to bring the link up.
 243  *
 244  * @param dev
 245  *   Pointer to Ethernet device structure.
 246  *
 247  * @return
 248  *   0 on success, negative error value otherwise.
 249  */
 250 static int
 251 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
 252 {
 253         struct mrvl_priv *priv = dev->data->dev_private;
 254         int ret;
 255
 256         ret = pp2_ppio_enable(priv->ppio);
 257         if (ret)
 258                 return ret;
 259
 260         dev->data->dev_link.link_status = ETH_LINK_UP;
 261
 262         return ret;
 263 }
 264
 265 /**
 266  * DPDK callback to bring the link down.
 267  *
 268  * @param dev
 269  *   Pointer to Ethernet device structure.
 270  *
 271  * @return
 272  *   0 on success, negative error value otherwise.
 273  */
 274 static int
 275 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
 276 {
 277         struct mrvl_priv *priv = dev->data->dev_private;
 278         int ret;
 279
 280         ret = pp2_ppio_disable(priv->ppio);
 281         if (ret)
 282                 return ret;
 283
 284         dev->data->dev_link.link_status = ETH_LINK_DOWN;
 285
 286         return ret;
 287 }
 288
 289 /**
 290  * DPDK callback to start the device.
 291  *
 292  * @param dev
 293  *   Pointer to Ethernet device structure.
 294  *
 295  * @return
 296  *   0 on success, negative errno value on failure.
 297  */
 298 static int
 299 mrvl_dev_start(struct rte_eth_dev *dev)
 300 {
 301         struct mrvl_priv *priv = dev->data->dev_private;
 302         char match[MRVL_MATCH_LEN];
 303         int ret;
 304
 305         snprintf(match, sizeof(match), "ppio-%d:%d",
 306                  priv->pp_id, priv->ppio_id);
 307         priv->ppio_params.match = match;
 308
 309         /*
 310          * Calculate the maximum bpool size for refill feature to 1.5 of the
 311          * configured size. In case the bpool size will exceed this value,
 312          * superfluous buffers will be removed
 313          */
 314         priv->bpool_max_size = priv->bpool_init_size +
 315                               (priv->bpool_init_size >> 1);
 316         /*
 317          * Calculate the minimum bpool size for refill feature as follows:
 318          * 2 default burst sizes multiply by number of rx queues.
 319          * If the bpool size will be below this value, new buffers will
 320          * be added to the pool.
 321          */
 322         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
 323
 324         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
 325         if (ret)
 326                 return ret;
 327
 328         /* For default QoS config, don't start classifier. */
 329         if (mrvl_qos_cfg) {
 330                 ret = mrvl_start_qos_mapping(priv);
 331                 if (ret) {
 332                         pp2_ppio_deinit(priv->ppio);
 333                         return ret;
 334                 }
 335         }
 336
 337         ret = mrvl_dev_set_link_up(dev);
 338         if (ret)
 339                 goto out;
 340
 341         return 0;
 342 out:
 343         pp2_ppio_deinit(priv->ppio);
 344         return ret;
 345 }
 346
 347 /**
 348  * Flush receive queues.
 349  *
 350  * @param dev
 351  *   Pointer to Ethernet device structure.
 352  */
 353 static void
 354 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
 355 {
 356         int i;
 357
 358         RTE_LOG(INFO, PMD, "Flushing rx queues\n");
 359         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 360                 int ret, num;
 361
 362                 do {
 363                         struct mrvl_rxq *q = dev->data->rx_queues[i];
 364                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
 365
 366                         num = MRVL_PP2_RXD_MAX;
 367                         ret = pp2_ppio_recv(q->priv->ppio,
 368                                             q->priv->rxq_map[q->queue_id].tc,
 369                                             q->priv->rxq_map[q->queue_id].inq,
 370                                             descs, (uint16_t *)&num);
 371                 } while (ret == 0 && num);
 372         }
 373 }
 374
 375 /**
 376  * Flush transmit shadow queues.
 377  *
 378  * @param dev
 379  *   Pointer to Ethernet device structure.
 380  */
 381 static void
 382 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
 383 {
 384         int i;
 385
 386         RTE_LOG(INFO, PMD, "Flushing tx shadow queues\n");
 387         for (i = 0; i < RTE_MAX_LCORE; i++) {
 388                 struct mrvl_shadow_txq *sq =
 389                         &shadow_txqs[dev->data->port_id][i];
 390
 391                 while (sq->tail != sq->head) {
 392                         uint64_t addr = cookie_addr_high |
 393                                         sq->ent[sq->tail].buff.cookie;
 394                         rte_pktmbuf_free((struct rte_mbuf *)addr);
 395                         sq->tail = (sq->tail + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
 396                 }
 397
 398                 memset(sq, 0, sizeof(*sq));
 399         }
 400 }
 401
 402 /**
 403  * Flush hardware bpool (buffer-pool).
 404  *
 405  * @param dev
 406  *   Pointer to Ethernet device structure.
 407  */
 408 static void
 409 mrvl_flush_bpool(struct rte_eth_dev *dev)
 410 {
 411         struct mrvl_priv *priv = dev->data->dev_private;
 412         uint32_t num;
 413         int ret;
 414
 415         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
 416         if (ret) {
 417                 RTE_LOG(ERR, PMD, "Failed to get bpool buffers number\n");
 418                 return;
 419         }
 420
 421         while (num--) {
 422                 struct pp2_buff_inf inf;
 423                 uint64_t addr;
 424
 425                 ret = pp2_bpool_get_buff(hifs[rte_lcore_id()], priv->bpool,
 426                                          &inf);
 427                 if (ret)
 428                         break;
 429
 430                 addr = cookie_addr_high | inf.cookie;
 431                 rte_pktmbuf_free((struct rte_mbuf *)addr);
 432         }
 433 }
 434
 435 /**
 436  * DPDK callback to stop the device.
 437  *
 438  * @param dev
 439  *   Pointer to Ethernet device structure.
 440  */
 441 static void
 442 mrvl_dev_stop(struct rte_eth_dev *dev)
 443 {
 444         struct mrvl_priv *priv = dev->data->dev_private;
 445
 446         mrvl_dev_set_link_down(dev);
 447         mrvl_flush_rx_queues(dev);
 448         mrvl_flush_tx_shadow_queues(dev);
 449         if (priv->qos_tbl)
 450                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
 451         pp2_ppio_deinit(priv->ppio);
 452         priv->ppio = NULL;
 453 }
 454
 455 /**
 456  * DPDK callback to close the device.
 457  *
 458  * @param dev
 459  *   Pointer to Ethernet device structure.
 460  */
 461 static void
 462 mrvl_dev_close(struct rte_eth_dev *dev)
 463 {
 464         struct mrvl_priv *priv = dev->data->dev_private;
 465         size_t i;
 466
 467         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
 468                 struct pp2_ppio_tc_params *tc_params =
 469                         &priv->ppio_params.inqs_params.tcs_params[i];
 470
 471                 if (tc_params->inqs_params) {
 472                         rte_free(tc_params->inqs_params);
 473                         tc_params->inqs_params = NULL;
 474                 }
 475         }
 476
 477         mrvl_flush_bpool(dev);
 478 }
 479
 480 /**
 481  * DPDK callback to retrieve physical link information.
 482  *
 483  * @param dev
 484  *   Pointer to Ethernet device structure.
 485  * @param wait_to_complete
 486  *   Wait for request completion (ignored).
 487  *
 488  * @return
 489  *   0 on success, negative error value otherwise.
 490  */
 491 static int
 492 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
 493 {
 494         /*
 495          * TODO
 496          * once MUSDK provides necessary API use it here
 497          */
 498         struct ethtool_cmd edata;
 499         struct ifreq req;
 500         int ret, fd;
 501
 502         edata.cmd = ETHTOOL_GSET;
 503
 504         strcpy(req.ifr_name, dev->data->name);
 505         req.ifr_data = (void *)&edata;
 506
 507         fd = socket(AF_INET, SOCK_DGRAM, 0);
 508         if (fd == -1)
 509                 return -EFAULT;
 510
 511         ret = ioctl(fd, SIOCETHTOOL, &req);
 512         if (ret == -1) {
 513                 close(fd);
 514                 return -EFAULT;
 515         }
 516
 517         close(fd);
 518
 519         switch (ethtool_cmd_speed(&edata)) {
 520         case SPEED_10:
 521                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
 522                 break;
 523         case SPEED_100:
 524                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
 525                 break;
 526         case SPEED_1000:
 527                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
 528                 break;
 529         case SPEED_10000:
 530                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
 531                 break;
 532         default:
 533                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
 534         }
 535
 536         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
 537                                                          ETH_LINK_HALF_DUPLEX;
 538         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
 539                                                            ETH_LINK_FIXED;
 540
 541         return 0;
 542 }
 543
 544 /**
 545  * DPDK callback to set the primary MAC address.
 546  *
 547  * @param dev
 548  *   Pointer to Ethernet device structure.
 549  * @param mac_addr
 550  *   MAC address to register.
 551  */
 552 static void
 553 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
 554 {
 555         struct mrvl_priv *priv = dev->data->dev_private;
 556
 557         pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
 558         /*
 559          * TODO
 560          * Port stops sending packets if pp2_ppio_set_mac_addr()
 561          * was called after pp2_ppio_enable(). As a quick fix issue
 562          * enable port once again.
 563          */
 564         pp2_ppio_enable(priv->ppio);
 565 }
 566
 567 /**
 568  * DPDK callback to get information about the device.
 569  *
 570  * @param dev
 571  *   Pointer to Ethernet device structure (unused).
 572  * @param info
 573  *   Info structure output buffer.
 574  */
 575 static void
 576 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
 577                    struct rte_eth_dev_info *info)
 578 {
 579         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
 580         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
 581         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
 582
 583         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
 584         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
 585         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
 586
 587         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
 588         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
 589         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
 590
 591         /* By default packets are dropped if no descriptors are available */
 592         info->default_rxconf.rx_drop_en = 1;
 593
 594         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
 595 }
 596
 597 /**
 598  * DPDK callback to get information about specific receive queue.
 599  *
 600  * @param dev
 601  *   Pointer to Ethernet device structure.
 602  * @param rx_queue_id
 603  *   Receive queue index.
 604  * @param qinfo
 605  *   Receive queue information structure.
 606  */
 607 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
 608                               struct rte_eth_rxq_info *qinfo)
 609 {
 610         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
 611         struct mrvl_priv *priv = dev->data->dev_private;
 612         int inq = priv->rxq_map[rx_queue_id].inq;
 613         int tc = priv->rxq_map[rx_queue_id].tc;
 614         struct pp2_ppio_tc_params *tc_params =
 615                 &priv->ppio_params.inqs_params.tcs_params[tc];
 616
 617         qinfo->mp = q->mp;
 618         qinfo->nb_desc = tc_params->inqs_params[inq].size;
 619 }
 620
 621 /**
 622  * DPDK callback to get information about specific transmit queue.
 623  *
 624  * @param dev
 625  *   Pointer to Ethernet device structure.
 626  * @param tx_queue_id
 627  *   Transmit queue index.
 628  * @param qinfo
 629  *   Transmit queue information structure.
 630  */
 631 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
 632                               struct rte_eth_txq_info *qinfo)
 633 {
 634         struct mrvl_priv *priv = dev->data->dev_private;
 635
 636         qinfo->nb_desc =
 637                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
 638 }
 639
 640 /**
 641  * Release buffers to hardware bpool (buffer-pool)
 642  *
 643  * @param rxq
 644  *   Receive queue pointer.
 645  * @param num
 646  *   Number of buffers to release to bpool.
 647  *
 648  * @return
 649  *   0 on success, negative error value otherwise.
 650  */
 651 static int
 652 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
 653 {
 654         struct buff_release_entry entries[MRVL_PP2_TXD_MAX];
 655         struct rte_mbuf *mbufs[MRVL_PP2_TXD_MAX];
 656         int i, ret;
 657         unsigned int core_id = rte_lcore_id();
 658         struct pp2_hif *hif = hifs[core_id];
 659         struct pp2_bpool *bpool = rxq->priv->bpool;
 660
 661         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
 662         if (ret)
 663                 return ret;
 664
 665         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
 666                 cookie_addr_high =
 667                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
 668
 669         for (i = 0; i < num; i++) {
 670                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
 671                         != cookie_addr_high) {
 672                         RTE_LOG(ERR, PMD,
 673                                 "mbuf virtual addr high 0x%lx out of range\n",
 674                                 (uint64_t)mbufs[i] >> 32);
 675                         goto out;
 676                 }
 677
 678                 entries[i].buff.addr =
 679                         rte_mbuf_data_dma_addr_default(mbufs[i]);
 680                 entries[i].buff.cookie = (pp2_cookie_t)(uint64_t)mbufs[i];
 681                 entries[i].bpool = bpool;
 682         }
 683
 684         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
 685         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
 686
 687         if (i != num)
 688                 goto out;
 689
 690         return 0;
 691 out:
 692         for (; i < num; i++)
 693                 rte_pktmbuf_free(mbufs[i]);
 694
 695         return -1;
 696 }
 697
 698 /**
 699  * DPDK callback to configure the receive queue.
 700  *
 701  * @param dev
 702  *   Pointer to Ethernet device structure.
 703  * @param idx
 704  *   RX queue index.
 705  * @param desc
 706  *   Number of descriptors to configure in queue.
 707  * @param socket
 708  *   NUMA socket on which memory must be allocated.
 709  * @param conf
 710  *   Thresholds parameters (unused_).
 711  * @param mp
 712  *   Memory pool for buffer allocations.
 713  *
 714  * @return
 715  *   0 on success, negative error value otherwise.
 716  */
 717 static int
 718 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 719                     unsigned int socket,
 720                     const struct rte_eth_rxconf *conf __rte_unused,
 721                     struct rte_mempool *mp)
 722 {
 723         struct mrvl_priv *priv = dev->data->dev_private;
 724         struct mrvl_rxq *rxq;
 725         uint32_t min_size,
 726                  max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
 727         int ret, tc, inq;
 728
 729         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
 730                 /*
 731                  * Unknown TC mapping, mapping will not have a correct queue.
 732                  */
 733                 RTE_LOG(ERR, PMD, "Unknown TC mapping for queue %hu eth%hhu\n",
 734                         idx, priv->ppio_id);
 735                 return -EFAULT;
 736         }
 737
 738         min_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM -
 739                    MRVL_PKT_EFFEC_OFFS;
 740         if (min_size < max_rx_pkt_len) {
 741                 RTE_LOG(ERR, PMD,
 742                         "Mbuf size must be increased to %u bytes to hold up to %u bytes of data.\n",
 743                         max_rx_pkt_len + RTE_PKTMBUF_HEADROOM +
 744                         MRVL_PKT_EFFEC_OFFS,
 745                         max_rx_pkt_len);
 746                 return -EINVAL;
 747         }
 748
 749         if (dev->data->rx_queues[idx]) {
 750                 rte_free(dev->data->rx_queues[idx]);
 751                 dev->data->rx_queues[idx] = NULL;
 752         }
 753
 754         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
 755         if (!rxq)
 756                 return -ENOMEM;
 757
 758         rxq->priv = priv;
 759         rxq->mp = mp;
 760         rxq->queue_id = idx;
 761         rxq->port_id = dev->data->port_id;
 762         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
 763
 764         tc = priv->rxq_map[rxq->queue_id].tc,
 765         inq = priv->rxq_map[rxq->queue_id].inq;
 766         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
 767                 desc;
 768
 769         ret = mrvl_fill_bpool(rxq, desc);
 770         if (ret) {
 771                 rte_free(rxq);
 772                 return ret;
 773         }
 774
 775         priv->bpool_init_size += desc;
 776
 777         dev->data->rx_queues[idx] = rxq;
 778
 779         return 0;
 780 }
 781
 782 /**
 783  * DPDK callback to release the receive queue.
 784  *
 785  * @param rxq
 786  *   Generic receive queue pointer.
 787  */
 788 static void
 789 mrvl_rx_queue_release(void *rxq)
 790 {
 791         struct mrvl_rxq *q = rxq;
 792         struct pp2_ppio_tc_params *tc_params;
 793         int i, num, tc, inq;
 794
 795         if (!q)
 796                 return;
 797
 798         tc = q->priv->rxq_map[q->queue_id].tc;
 799         inq = q->priv->rxq_map[q->queue_id].inq;
 800         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
 801         num = tc_params->inqs_params[inq].size;
 802         for (i = 0; i < num; i++) {
 803                 struct pp2_buff_inf inf;
 804                 uint64_t addr;
 805
 806                 pp2_bpool_get_buff(hifs[rte_lcore_id()], q->priv->bpool, &inf);
 807                 addr = cookie_addr_high | inf.cookie;
 808                 rte_pktmbuf_free((struct rte_mbuf *)addr);
 809         }
 810
 811         rte_free(q);
 812 }
 813
 814 /**
 815  * DPDK callback to configure the transmit queue.
 816  *
 817  * @param dev
 818  *   Pointer to Ethernet device structure.
 819  * @param idx
 820  *   Transmit queue index.
 821  * @param desc
 822  *   Number of descriptors to configure in the queue.
 823  * @param socket
 824  *   NUMA socket on which memory must be allocated.
 825  * @param conf
 826  *   Thresholds parameters (unused).
 827  *
 828  * @return
 829  *   0 on success, negative error value otherwise.
 830  */
 831 static int
 832 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 833                     unsigned int socket,
 834                     const struct rte_eth_txconf *conf __rte_unused)
 835 {
 836         struct mrvl_priv *priv = dev->data->dev_private;
 837         struct mrvl_txq *txq;
 838
 839         if (dev->data->tx_queues[idx]) {
 840                 rte_free(dev->data->tx_queues[idx]);
 841                 dev->data->tx_queues[idx] = NULL;
 842         }
 843
 844         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
 845         if (!txq)
 846                 return -ENOMEM;
 847
 848         txq->priv = priv;
 849         txq->queue_id = idx;
 850         txq->port_id = dev->data->port_id;
 851         dev->data->tx_queues[idx] = txq;
 852
 853         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
 854         priv->ppio_params.outqs_params.outqs_params[idx].weight = 1;
 855
 856         return 0;
 857 }
 858
 859 /**
 860  * DPDK callback to release the transmit queue.
 861  *
 862  * @param txq
 863  *   Generic transmit queue pointer.
 864  */
 865 static void
 866 mrvl_tx_queue_release(void *txq)
 867 {
 868         struct mrvl_txq *q = txq;
 869
 870         if (!q)
 871                 return;
 872
 873         rte_free(q);
 874 }
 875
 876 static const struct eth_dev_ops mrvl_ops = {
 877         .dev_configure = mrvl_dev_configure,
 878         .dev_start = mrvl_dev_start,
 879         .dev_stop = mrvl_dev_stop,
 880         .dev_set_link_up = mrvl_dev_set_link_up,
 881         .dev_set_link_down = mrvl_dev_set_link_down,
 882         .dev_close = mrvl_dev_close,
 883         .link_update = mrvl_link_update,
 884         .mac_addr_set = mrvl_mac_addr_set,
 885         .dev_infos_get = mrvl_dev_infos_get,
 886         .rxq_info_get = mrvl_rxq_info_get,
 887         .txq_info_get = mrvl_txq_info_get,
 888         .rx_queue_setup = mrvl_rx_queue_setup,
 889         .rx_queue_release = mrvl_rx_queue_release,
 890         .tx_queue_setup = mrvl_tx_queue_setup,
 891         .tx_queue_release = mrvl_tx_queue_release,
 892 };
 893
 894 /**
 895  * DPDK callback for receive.
 896  *
 897  * @param rxq
 898  *   Generic pointer to the receive queue.
 899  * @param rx_pkts
 900  *   Array to store received packets.
 901  * @param nb_pkts
 902  *   Maximum number of packets in array.
 903  *
 904  * @return
 905  *   Number of packets successfully received.
 906  */
 907 static uint16_t
 908 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 909 {
 910         struct mrvl_rxq *q = rxq;
 911         struct pp2_ppio_desc descs[nb_pkts];
 912         struct pp2_bpool *bpool;
 913         int i, ret, rx_done = 0;
 914         int num;
 915         unsigned int core_id = rte_lcore_id();
 916
 917         if (unlikely(!q->priv->ppio))
 918                 return 0;
 919
 920         bpool = q->priv->bpool;
 921
 922         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
 923                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
 924         if (unlikely(ret < 0)) {
 925                 RTE_LOG(ERR, PMD, "Failed to receive packets\n");
 926                 return 0;
 927         }
 928         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
 929
 930         for (i = 0; i < nb_pkts; i++) {
 931                 struct rte_mbuf *mbuf;
 932                 enum pp2_inq_desc_status status;
 933                 uint64_t addr;
 934
 935                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
 936                         struct pp2_ppio_desc *pref_desc;
 937                         u64 pref_addr;
 938
 939                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
 940                         pref_addr = cookie_addr_high |
 941                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
 942                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
 943                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
 944                 }
 945
 946                 addr = cookie_addr_high |
 947                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
 948                 mbuf = (struct rte_mbuf *)addr;
 949                 rte_pktmbuf_reset(mbuf);
 950
 951                 /* drop packet in case of mac, overrun or resource error */
 952                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
 953                 if (unlikely(status != PP2_DESC_ERR_OK)) {
 954                         struct pp2_buff_inf binf = {
 955                                 .addr = rte_mbuf_data_dma_addr_default(mbuf),
 956                                 .cookie = (pp2_cookie_t)(uint64_t)mbuf,
 957                         };
 958
 959                         pp2_bpool_put_buff(hifs[core_id], bpool, &binf);
 960                         mrvl_port_bpool_size
 961                                 [bpool->pp2_id][bpool->id][core_id]++;
 962                         continue;
 963                 }
 964
 965                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
 966                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
 967                 mbuf->data_len = mbuf->pkt_len;
 968                 mbuf->port = q->port_id;
 969
 970                 rx_pkts[rx_done++] = mbuf;
 971         }
 972
 973         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
 974                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
 975
 976                 if (unlikely(num <= q->priv->bpool_min_size ||
 977                              (!rx_done && num < q->priv->bpool_init_size))) {
 978                         ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE);
 979                         if (ret)
 980                                 RTE_LOG(ERR, PMD, "Failed to fill bpool\n");
 981                 } else if (unlikely(num > q->priv->bpool_max_size)) {
 982                         int i;
 983                         int pkt_to_remove = num - q->priv->bpool_init_size;
 984                         struct rte_mbuf *mbuf;
 985                         struct pp2_buff_inf buff;
 986
 987                         RTE_LOG(DEBUG, PMD,
 988                                 "\nport-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)\n",
 989                                 bpool->pp2_id, q->priv->ppio->port_id,
 990                                 bpool->id, pkt_to_remove, num,
 991                                 q->priv->bpool_init_size);
 992
 993                         for (i = 0; i < pkt_to_remove; i++) {
 994                                 pp2_bpool_get_buff(hifs[core_id], bpool, &buff);
 995                                 mbuf = (struct rte_mbuf *)
 996                                         (cookie_addr_high | buff.cookie);
 997                                 rte_pktmbuf_free(mbuf);
 998                         }
 999                         mrvl_port_bpool_size
1000                                 [bpool->pp2_id][bpool->id][core_id] -=
1001                                                                 pkt_to_remove;
1002                 }
1003                 rte_spinlock_unlock(&q->priv->lock);
1004         }
1005
1006         return rx_done;
1007 }
1008
1009 /**
1010  * Release already sent buffers to bpool (buffer-pool).
1011  *
1012  * @param ppio
1013  *   Pointer to the port structure.
1014  * @param hif
1015  *   Pointer to the MUSDK hardware interface.
1016  * @param sq
1017  *   Pointer to the shadow queue.
1018  * @param qid
1019  *   Queue id number.
1020  * @param force
1021  *   Force releasing packets.
1022  */
1023 static inline void
1024 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
1025                        struct mrvl_shadow_txq *sq, int qid, int force)
1026 {
1027         struct buff_release_entry *entry;
1028         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
1029         int i, core_id = rte_lcore_id();
1030
1031         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
1032
1033         sq->num_to_release += nb_done;
1034
1035         if (likely(!force &&
1036                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
1037                 return;
1038
1039         nb_done = sq->num_to_release;
1040         sq->num_to_release = 0;
1041
1042         for (i = 0; i < nb_done; i++) {
1043                 entry = &sq->ent[sq->tail + num];
1044                 if (unlikely(!entry->buff.addr)) {
1045                         RTE_LOG(ERR, PMD,
1046                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!\n",
1047                                 sq->tail, (u64)entry->buff.cookie,
1048                                 (u64)entry->buff.addr);
1049                         skip_bufs = 1;
1050                         goto skip;
1051                 }
1052
1053                 if (unlikely(!entry->bpool)) {
1054                         struct rte_mbuf *mbuf;
1055
1056                         mbuf = (struct rte_mbuf *)
1057                                (cookie_addr_high | entry->buff.cookie);
1058                         rte_pktmbuf_free(mbuf);
1059                         skip_bufs = 1;
1060                         goto skip;
1061                 }
1062
1063                 mrvl_port_bpool_size
1064                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
1065                 num++;
1066                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
1067                         goto skip;
1068                 continue;
1069 skip:
1070                 if (likely(num))
1071                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
1072                 num += skip_bufs;
1073                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
1074                 sq->size -= num;
1075                 num = 0;
1076         }
1077
1078         if (likely(num)) {
1079                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
1080                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
1081                 sq->size -= num;
1082         }
1083 }
1084
1085 /**
1086  * DPDK callback for transmit.
1087  *
1088  * @param txq
1089  *   Generic pointer transmit queue.
1090  * @param tx_pkts
1091  *   Packets to transmit.
1092  * @param nb_pkts
1093  *   Number of packets in array.
1094  *
1095  * @return
1096  *   Number of packets successfully transmitted.
1097  */
1098 static uint16_t
1099 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1100 {
1101         struct mrvl_txq *q = txq;
1102         struct mrvl_shadow_txq *sq = &shadow_txqs[q->port_id][rte_lcore_id()];
1103         struct pp2_hif *hif = hifs[rte_lcore_id()];
1104         struct pp2_ppio_desc descs[nb_pkts];
1105         int i;
1106         uint16_t num, sq_free_size;
1107
1108         if (unlikely(!q->priv->ppio))
1109                 return 0;
1110
1111         if (sq->size)
1112                 mrvl_free_sent_buffers(q->priv->ppio, hif, sq, q->queue_id, 0);
1113
1114         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
1115         if (unlikely(nb_pkts > sq_free_size)) {
1116                 RTE_LOG(DEBUG, PMD,
1117                         "No room in shadow queue for %d packets! %d packets will be sent.\n",
1118                         nb_pkts, sq_free_size);
1119                 nb_pkts = sq_free_size;
1120         }
1121
1122         for (i = 0; i < nb_pkts; i++) {
1123                 struct rte_mbuf *mbuf = tx_pkts[i];
1124
1125                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
1126                         struct rte_mbuf *pref_pkt_hdr;
1127
1128                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
1129                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
1130                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
1131                 }
1132
1133                 sq->ent[sq->head].buff.cookie = (pp2_cookie_t)(uint64_t)mbuf;
1134                 sq->ent[sq->head].buff.addr =
1135                         rte_mbuf_data_dma_addr_default(mbuf);
1136                 sq->ent[sq->head].bpool =
1137                         (unlikely(mbuf->port == 0xff || mbuf->refcnt > 1)) ?
1138                          NULL : mrvl_port_to_bpool_lookup[mbuf->port];
1139                 sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
1140                 sq->size++;
1141
1142                 pp2_ppio_outq_desc_reset(&descs[i]);
1143                 pp2_ppio_outq_desc_set_phys_addr(&descs[i],
1144                                                  rte_pktmbuf_mtophys(mbuf));
1145                 pp2_ppio_outq_desc_set_pkt_offset(&descs[i], 0);
1146                 pp2_ppio_outq_desc_set_pkt_len(&descs[i],
1147                                                rte_pktmbuf_pkt_len(mbuf));
1148         }
1149
1150         num = nb_pkts;
1151         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
1152         /* number of packets that were not sent */
1153         if (unlikely(num > nb_pkts)) {
1154                 for (i = nb_pkts; i < num; i++) {
1155                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
1156                                 MRVL_PP2_TX_SHADOWQ_MASK;
1157                 }
1158                 sq->size -= num - nb_pkts;
1159         }
1160
1161         return nb_pkts;
1162 }
1163
1164 /**
1165  * Initialize packet processor.
1166  *
1167  * @return
1168  *   0 on success, negative error value otherwise.
1169  */
1170 static int
1171 mrvl_init_pp2(void)
1172 {
1173         struct pp2_init_params init_params;
1174
1175         memset(&init_params, 0, sizeof(init_params));
1176         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
1177         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
1178
1179         return pp2_init(&init_params);
1180 }
1181
1182 /**
1183  * Deinitialize packet processor.
1184  *
1185  * @return
1186  *   0 on success, negative error value otherwise.
1187  */
1188 static void
1189 mrvl_deinit_pp2(void)
1190 {
1191         pp2_deinit();
1192 }
1193
1194 /**
1195  * Create private device structure.
1196  *
1197  * @param dev_name
1198  *   Pointer to the port name passed in the initialization parameters.
1199  *
1200  * @return
1201  *   Pointer to the newly allocated private device structure.
1202  */
1203 static struct mrvl_priv *
1204 mrvl_priv_create(const char *dev_name)
1205 {
1206         struct pp2_bpool_params bpool_params;
1207         char match[MRVL_MATCH_LEN];
1208         struct mrvl_priv *priv;
1209         int ret, bpool_bit;
1210
1211         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
1212         if (!priv)
1213                 return NULL;
1214
1215         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
1216                                        &priv->pp_id, &priv->ppio_id);
1217         if (ret)
1218                 goto out_free_priv;
1219
1220         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
1221                                      PP2_BPOOL_NUM_POOLS);
1222         if (bpool_bit < 0)
1223                 goto out_free_priv;
1224         priv->bpool_bit = bpool_bit;
1225
1226         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
1227                  priv->bpool_bit);
1228         memset(&bpool_params, 0, sizeof(bpool_params));
1229         bpool_params.match = match;
1230         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
1231         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
1232         if (ret)
1233                 goto out_clear_bpool_bit;
1234
1235         priv->ppio_params.type = PP2_PPIO_T_NIC;
1236         rte_spinlock_init(&priv->lock);
1237
1238         return priv;
1239 out_clear_bpool_bit:
1240         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
1241 out_free_priv:
1242         rte_free(priv);
1243         return NULL;
1244 }
1245
1246 /**
1247  * Create device representing Ethernet port.
1248  *
1249  * @param name
1250  *   Pointer to the port's name.
1251  *
1252  * @return
1253  *   0 on success, negative error value otherwise.
1254  */
1255 static int
1256 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
1257 {
1258         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
1259         struct rte_eth_dev *eth_dev;
1260         struct mrvl_priv *priv;
1261         struct ifreq req;
1262
1263         eth_dev = rte_eth_dev_allocate(name);
1264         if (!eth_dev)
1265                 return -ENOMEM;
1266
1267         priv = mrvl_priv_create(name);
1268         if (!priv) {
1269                 ret = -ENOMEM;
1270                 goto out_free_dev;
1271         }
1272
1273         eth_dev->data->mac_addrs =
1274                 rte_zmalloc("mac_addrs",
1275                             ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
1276         if (!eth_dev->data->mac_addrs) {
1277                 RTE_LOG(ERR, PMD, "Failed to allocate space for eth addrs\n");
1278                 ret = -ENOMEM;
1279                 goto out_free_priv;
1280         }
1281
1282         memset(&req, 0, sizeof(req));
1283         strcpy(req.ifr_name, name);
1284         ret = ioctl(fd, SIOCGIFHWADDR, &req);
1285         if (ret)
1286                 goto out_free_mac;
1287
1288         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
1289                req.ifr_addr.sa_data, ETHER_ADDR_LEN);
1290
1291         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
1292         eth_dev->tx_pkt_burst = mrvl_tx_pkt_burst;
1293         eth_dev->data->dev_private = priv;
1294         eth_dev->device = &vdev->device;
1295         eth_dev->dev_ops = &mrvl_ops;
1296
1297         return 0;
1298 out_free_mac:
1299         rte_free(eth_dev->data->mac_addrs);
1300 out_free_dev:
1301         rte_eth_dev_release_port(eth_dev);
1302 out_free_priv:
1303         rte_free(priv);
1304
1305         return ret;
1306 }
1307
1308 /**
1309  * Cleanup previously created device representing Ethernet port.
1310  *
1311  * @param name
1312  *   Pointer to the port name.
1313  */
1314 static void
1315 mrvl_eth_dev_destroy(const char *name)
1316 {
1317         struct rte_eth_dev *eth_dev;
1318         struct mrvl_priv *priv;
1319
1320         eth_dev = rte_eth_dev_allocated(name);
1321         if (!eth_dev)
1322                 return;
1323
1324         priv = eth_dev->data->dev_private;
1325         pp2_bpool_deinit(priv->bpool);
1326         rte_free(priv);
1327         rte_free(eth_dev->data->mac_addrs);
1328         rte_eth_dev_release_port(eth_dev);
1329 }
1330
1331 /**
1332  * Callback used by rte_kvargs_process() during argument parsing.
1333  *
1334  * @param key
1335  *   Pointer to the parsed key (unused).
1336  * @param value
1337  *   Pointer to the parsed value.
1338  * @param extra_args
1339  *   Pointer to the extra arguments which contains address of the
1340  *   table of pointers to parsed interface names.
1341  *
1342  * @return
1343  *   Always 0.
1344  */
1345 static int
1346 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
1347                  void *extra_args)
1348 {
1349         const char **ifnames = extra_args;
1350
1351         ifnames[mrvl_ports_nb++] = value;
1352
1353         return 0;
1354 }
1355
1356 /**
1357  * Initialize per-lcore MUSDK hardware interfaces (hifs).
1358  *
1359  * @return
1360  *   0 on success, negative error value otherwise.
1361  */
1362 static int
1363 mrvl_init_hifs(void)
1364 {
1365         struct pp2_hif_params params;
1366         char match[MRVL_MATCH_LEN];
1367         int i, ret;
1368
1369         RTE_LCORE_FOREACH(i) {
1370                 ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
1371                 if (ret < 0)
1372                         return ret;
1373
1374                 snprintf(match, sizeof(match), "hif-%d", ret);
1375                 memset(&params, 0, sizeof(params));
1376                 params.match = match;
1377                 params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
1378                 ret = pp2_hif_init(&params, &hifs[i]);
1379                 if (ret) {
1380                         RTE_LOG(ERR, PMD, "Failed to initialize hif %d\n", i);
1381                         return ret;
1382                 }
1383         }
1384
1385         return 0;
1386 }
1387
1388 /**
1389  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
1390  */
1391 static void
1392 mrvl_deinit_hifs(void)
1393 {
1394         int i;
1395
1396         RTE_LCORE_FOREACH(i) {
1397                 if (hifs[i])
1398                         pp2_hif_deinit(hifs[i]);
1399         }
1400 }
1401
1402 static void mrvl_set_first_last_cores(int core_id)
1403 {
1404         if (core_id < mrvl_lcore_first)
1405                 mrvl_lcore_first = core_id;
1406
1407         if (core_id > mrvl_lcore_last)
1408                 mrvl_lcore_last = core_id;
1409 }
1410
1411 /**
1412  * DPDK callback to register the virtual device.
1413  *
1414  * @param vdev
1415  *   Pointer to the virtual device.
1416  *
1417  * @return
1418  *   0 on success, negative error value otherwise.
1419  */
1420 static int
1421 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
1422 {
1423         struct rte_kvargs *kvlist;
1424         const char *ifnames[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
1425         int ret = -EINVAL;
1426         uint32_t i, ifnum, cfgnum, core_id;
1427         const char *params;
1428
1429         params = rte_vdev_device_args(vdev);
1430         if (!params)
1431                 return -EINVAL;
1432
1433         kvlist = rte_kvargs_parse(params, valid_args);
1434         if (!kvlist)
1435                 return -EINVAL;
1436
1437         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
1438         if (ifnum > RTE_DIM(ifnames))
1439                 goto out_free_kvlist;
1440
1441         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
1442                            mrvl_get_ifnames, &ifnames);
1443
1444         cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
1445         if (cfgnum > 1) {
1446                 RTE_LOG(ERR, PMD, "Cannot handle more than one config file!\n");
1447                 goto out_free_kvlist;
1448         } else if (cfgnum == 1) {
1449                 rte_kvargs_process(kvlist, MRVL_CFG_ARG,
1450                                    mrvl_get_qoscfg, &mrvl_qos_cfg);
1451         }
1452
1453         /*
1454          * ret == -EEXIST is correct, it means DMA
1455          * has been already initialized (by another PMD).
1456          */
1457         ret = mv_sys_dma_mem_init(RTE_MRVL_MUSDK_DMA_MEMSIZE);
1458         if (ret < 0 && ret != -EEXIST)
1459                 goto out_free_kvlist;
1460
1461         ret = mrvl_init_pp2();
1462         if (ret) {
1463                 RTE_LOG(ERR, PMD, "Failed to init PP!\n");
1464                 goto out_deinit_dma;
1465         }
1466
1467         ret = mrvl_init_hifs();
1468         if (ret)
1469                 goto out_deinit_hifs;
1470
1471         for (i = 0; i < ifnum; i++) {
1472                 RTE_LOG(INFO, PMD, "Creating %s\n", ifnames[i]);
1473                 ret = mrvl_eth_dev_create(vdev, ifnames[i]);
1474                 if (ret)
1475                         goto out_cleanup;
1476         }
1477
1478         rte_kvargs_free(kvlist);
1479
1480         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
1481
1482         mrvl_lcore_first = RTE_MAX_LCORE;
1483         mrvl_lcore_last = 0;
1484
1485         RTE_LCORE_FOREACH(core_id) {
1486                 mrvl_set_first_last_cores(core_id);
1487         }
1488
1489         return 0;
1490 out_cleanup:
1491         for (; i > 0; i--)
1492                 mrvl_eth_dev_destroy(ifnames[i]);
1493 out_deinit_hifs:
1494         mrvl_deinit_hifs();
1495         mrvl_deinit_pp2();
1496 out_deinit_dma:
1497         mv_sys_dma_mem_destroy();
1498 out_free_kvlist:
1499         rte_kvargs_free(kvlist);
1500
1501         return ret;
1502 }
1503
1504 /**
1505  * DPDK callback to remove virtual device.
1506  *
1507  * @param vdev
1508  *   Pointer to the removed virtual device.
1509  *
1510  * @return
1511  *   0 on success, negative error value otherwise.
1512  */
1513 static int
1514 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
1515 {
1516         int i;
1517         const char *name;
1518
1519         name = rte_vdev_device_name(vdev);
1520         if (!name)
1521                 return -EINVAL;
1522
1523         RTE_LOG(INFO, PMD, "Removing %s\n", name);
1524
1525         for (i = 0; i < rte_eth_dev_count(); i++) {
1526                 char ifname[RTE_ETH_NAME_MAX_LEN];
1527
1528                 rte_eth_dev_get_name_by_port(i, ifname);
1529                 mrvl_eth_dev_destroy(ifname);
1530         }
1531
1532         mrvl_deinit_hifs();
1533         mrvl_deinit_pp2();
1534         mv_sys_dma_mem_destroy();
1535
1536         return 0;
1537 }
1538
1539 static struct rte_vdev_driver pmd_mrvl_drv = {
1540         .probe = rte_pmd_mrvl_probe,
1541         .remove = rte_pmd_mrvl_remove,
1542 };
1543
1544 RTE_PMD_REGISTER_VDEV(net_mrvl, pmd_mrvl_drv);
1545 RTE_PMD_REGISTER_ALIAS(net_mrvl, eth_mrvl);