drivers/net/mrvl/mrvl_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Semihalf. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Semihalf nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <rte_ethdev.h>
  34 #include <rte_kvargs.h>
  35 #include <rte_log.h>
  36 #include <rte_malloc.h>
  37 #include <rte_vdev.h>
  38
  39 /* Unluckily, container_of is defined by both DPDK and MUSDK,
  40  * we'll declare only one version.
  41  *
  42  * Note that it is not used in this PMD anyway.
  43  */
  44 #ifdef container_of
  45 #undef container_of
  46 #endif
  47
  48 #include <drivers/mv_pp2.h>
  49 #include <drivers/mv_pp2_bpool.h>
  50 #include <drivers/mv_pp2_hif.h>
  51
  52 #include <fcntl.h>
  53 #include <linux/ethtool.h>
  54 #include <linux/sockios.h>
  55 #include <net/if.h>
  56 #include <net/if_arp.h>
  57 #include <sys/ioctl.h>
  58 #include <sys/socket.h>
  59 #include <sys/stat.h>
  60 #include <sys/types.h>
  61
  62 #include "mrvl_ethdev.h"
  63 #include "mrvl_qos.h"
  64
  65 /* bitmask with reserved hifs */
  66 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
  67 /* bitmask with reserved bpools */
  68 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
  69 /* maximum number of available hifs */
  70 #define MRVL_MUSDK_HIFS_MAX 9
  71
  72 #define MRVL_MAC_ADDRS_MAX 1
  73 /* prefetch shift */
  74 #define MRVL_MUSDK_PREFETCH_SHIFT 2
  75
  76 #define MRVL_MATCH_LEN 16
  77 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
  78 /* Maximum allowable packet size */
  79 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
  80
  81 #define MRVL_IFACE_NAME_ARG "iface"
  82 #define MRVL_CFG_ARG "cfg"
  83
  84 #define MRVL_BURST_SIZE 64
  85
  86 #define MRVL_ARP_LENGTH 28
  87
  88 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
  89
  90 #define MRVL_COOKIE_HIGH_ADDR_SHIFT     (sizeof(pp2_cookie_t) * 8)
  91 #define MRVL_COOKIE_HIGH_ADDR_MASK      (~0ULL << MRVL_COOKIE_HIGH_ADDR_SHIFT)
  92
  93 static const char * const valid_args[] = {
  94         MRVL_IFACE_NAME_ARG,
  95         MRVL_CFG_ARG,
  96         NULL
  97 };
  98
  99 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
 100 static struct pp2_hif *hifs[RTE_MAX_LCORE];
 101 static int used_bpools[PP2_NUM_PKT_PROC] = {
 102         MRVL_MUSDK_BPOOLS_RESERVED,
 103         MRVL_MUSDK_BPOOLS_RESERVED
 104 };
 105
 106 struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
 107 int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
 108 uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
 109
 110 /*
 111  * To use buffer harvesting based on loopback port shadow queue structure
 112  * was introduced for buffers information bookkeeping.
 113  *
 114  * Before sending the packet, related buffer information (pp2_buff_inf) is
 115  * stored in shadow queue. After packet is transmitted no longer used
 116  * packet buffer is released back to it's original hardware pool,
 117  * on condition it originated from interface.
 118  * In case it  was generated by application itself i.e: mbuf->port field is
 119  * 0xff then its released to software mempool.
 120  */
 121 struct mrvl_shadow_txq {
 122         int head;           /* write index - used when sending buffers */
 123         int tail;           /* read index - used when releasing buffers */
 124         u16 size;           /* queue occupied size */
 125         u16 num_to_release; /* number of buffers sent, that can be released */
 126         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
 127 };
 128
 129 struct mrvl_rxq {
 130         struct mrvl_priv *priv;
 131         struct rte_mempool *mp;
 132         int queue_id;
 133         int port_id;
 134 };
 135
 136 struct mrvl_txq {
 137         struct mrvl_priv *priv;
 138         int queue_id;
 139         int port_id;
 140 };
 141
 142 /*
 143  * Every tx queue should have dedicated shadow tx queue.
 144  *
 145  * Ports assigned by DPDK might not start at zero or be continuous so
 146  * as a workaround define shadow queues for each possible port so that
 147  * we eventually fit somewhere.
 148  */
 149 struct mrvl_shadow_txq shadow_txqs[RTE_MAX_ETHPORTS][RTE_MAX_LCORE];
 150
 151 /** Number of ports configured. */
 152 int mrvl_ports_nb;
 153 static int mrvl_lcore_first;
 154 static int mrvl_lcore_last;
 155
 156 static inline int
 157 mrvl_get_bpool_size(int pp2_id, int pool_id)
 158 {
 159         int i;
 160         int size = 0;
 161
 162         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
 163                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
 164
 165         return size;
 166 }
 167
 168 static inline int
 169 mrvl_reserve_bit(int *bitmap, int max)
 170 {
 171         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
 172
 173         if (n >= max)
 174                 return -1;
 175
 176         *bitmap |= 1 << n;
 177
 178         return n;
 179 }
 180
 181 /**
 182  * Ethernet device configuration.
 183  *
 184  * Prepare the driver for a given number of TX and RX queues.
 185  *
 186  * @param dev
 187  *   Pointer to Ethernet device structure.
 188  *
 189  * @return
 190  *   0 on success, negative error value otherwise.
 191  */
 192 static int
 193 mrvl_dev_configure(struct rte_eth_dev *dev)
 194 {
 195         struct mrvl_priv *priv = dev->data->dev_private;
 196         int ret;
 197
 198         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE) {
 199                 RTE_LOG(INFO, PMD, "Unsupported rx multi queue mode %d\n",
 200                         dev->data->dev_conf.rxmode.mq_mode);
 201                 return -EINVAL;
 202         }
 203
 204         if (!dev->data->dev_conf.rxmode.hw_strip_crc) {
 205                 RTE_LOG(INFO, PMD,
 206                         "L2 CRC stripping is always enabled in hw\n");
 207                 dev->data->dev_conf.rxmode.hw_strip_crc = 1;
 208         }
 209
 210         if (dev->data->dev_conf.rxmode.hw_vlan_strip) {
 211                 RTE_LOG(INFO, PMD, "VLAN stripping not supported\n");
 212                 return -EINVAL;
 213         }
 214
 215         if (dev->data->dev_conf.rxmode.split_hdr_size) {
 216                 RTE_LOG(INFO, PMD, "Split headers not supported\n");
 217                 return -EINVAL;
 218         }
 219
 220         if (dev->data->dev_conf.rxmode.enable_scatter) {
 221                 RTE_LOG(INFO, PMD, "RX Scatter/Gather not supported\n");
 222                 return -EINVAL;
 223         }
 224
 225         if (dev->data->dev_conf.rxmode.enable_lro) {
 226                 RTE_LOG(INFO, PMD, "LRO not supported\n");
 227                 return -EINVAL;
 228         }
 229
 230         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
 231                                   dev->data->nb_rx_queues);
 232         if (ret < 0)
 233                 return ret;
 234
 235         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
 236         priv->nb_rx_queues = dev->data->nb_rx_queues;
 237
 238         return 0;
 239 }
 240
 241 /**
 242  * DPDK callback to bring the link up.
 243  *
 244  * @param dev
 245  *   Pointer to Ethernet device structure.
 246  *
 247  * @return
 248  *   0 on success, negative error value otherwise.
 249  */
 250 static int
 251 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
 252 {
 253         struct mrvl_priv *priv = dev->data->dev_private;
 254         int ret;
 255
 256         ret = pp2_ppio_enable(priv->ppio);
 257         if (ret)
 258                 return ret;
 259
 260         dev->data->dev_link.link_status = ETH_LINK_UP;
 261
 262         return ret;
 263 }
 264
 265 /**
 266  * DPDK callback to bring the link down.
 267  *
 268  * @param dev
 269  *   Pointer to Ethernet device structure.
 270  *
 271  * @return
 272  *   0 on success, negative error value otherwise.
 273  */
 274 static int
 275 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
 276 {
 277         struct mrvl_priv *priv = dev->data->dev_private;
 278         int ret;
 279
 280         ret = pp2_ppio_disable(priv->ppio);
 281         if (ret)
 282                 return ret;
 283
 284         dev->data->dev_link.link_status = ETH_LINK_DOWN;
 285
 286         return ret;
 287 }
 288
 289 /**
 290  * DPDK callback to start the device.
 291  *
 292  * @param dev
 293  *   Pointer to Ethernet device structure.
 294  *
 295  * @return
 296  *   0 on success, negative errno value on failure.
 297  */
 298 static int
 299 mrvl_dev_start(struct rte_eth_dev *dev)
 300 {
 301         struct mrvl_priv *priv = dev->data->dev_private;
 302         char match[MRVL_MATCH_LEN];
 303         int ret;
 304
 305         snprintf(match, sizeof(match), "ppio-%d:%d",
 306                  priv->pp_id, priv->ppio_id);
 307         priv->ppio_params.match = match;
 308
 309         /*
 310          * Calculate the maximum bpool size for refill feature to 1.5 of the
 311          * configured size. In case the bpool size will exceed this value,
 312          * superfluous buffers will be removed
 313          */
 314         priv->bpool_max_size = priv->bpool_init_size +
 315                               (priv->bpool_init_size >> 1);
 316         /*
 317          * Calculate the minimum bpool size for refill feature as follows:
 318          * 2 default burst sizes multiply by number of rx queues.
 319          * If the bpool size will be below this value, new buffers will
 320          * be added to the pool.
 321          */
 322         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
 323
 324         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
 325         if (ret)
 326                 return ret;
 327
 328         /* For default QoS config, don't start classifier. */
 329         if (mrvl_qos_cfg) {
 330                 ret = mrvl_start_qos_mapping(priv);
 331                 if (ret) {
 332                         pp2_ppio_deinit(priv->ppio);
 333                         return ret;
 334                 }
 335         }
 336
 337         ret = mrvl_dev_set_link_up(dev);
 338         if (ret)
 339                 goto out;
 340
 341         return 0;
 342 out:
 343         pp2_ppio_deinit(priv->ppio);
 344         return ret;
 345 }
 346
 347 /**
 348  * Flush receive queues.
 349  *
 350  * @param dev
 351  *   Pointer to Ethernet device structure.
 352  */
 353 static void
 354 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
 355 {
 356         int i;
 357
 358         RTE_LOG(INFO, PMD, "Flushing rx queues\n");
 359         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 360                 int ret, num;
 361
 362                 do {
 363                         struct mrvl_rxq *q = dev->data->rx_queues[i];
 364                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
 365
 366                         num = MRVL_PP2_RXD_MAX;
 367                         ret = pp2_ppio_recv(q->priv->ppio,
 368                                             q->priv->rxq_map[q->queue_id].tc,
 369                                             q->priv->rxq_map[q->queue_id].inq,
 370                                             descs, (uint16_t *)&num);
 371                 } while (ret == 0 && num);
 372         }
 373 }
 374
 375 /**
 376  * Flush transmit shadow queues.
 377  *
 378  * @param dev
 379  *   Pointer to Ethernet device structure.
 380  */
 381 static void
 382 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
 383 {
 384         int i;
 385
 386         RTE_LOG(INFO, PMD, "Flushing tx shadow queues\n");
 387         for (i = 0; i < RTE_MAX_LCORE; i++) {
 388                 struct mrvl_shadow_txq *sq =
 389                         &shadow_txqs[dev->data->port_id][i];
 390
 391                 while (sq->tail != sq->head) {
 392                         uint64_t addr = cookie_addr_high |
 393                                         sq->ent[sq->tail].buff.cookie;
 394                         rte_pktmbuf_free((struct rte_mbuf *)addr);
 395                         sq->tail = (sq->tail + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
 396                 }
 397
 398                 memset(sq, 0, sizeof(*sq));
 399         }
 400 }
 401
 402 /**
 403  * Flush hardware bpool (buffer-pool).
 404  *
 405  * @param dev
 406  *   Pointer to Ethernet device structure.
 407  */
 408 static void
 409 mrvl_flush_bpool(struct rte_eth_dev *dev)
 410 {
 411         struct mrvl_priv *priv = dev->data->dev_private;
 412         uint32_t num;
 413         int ret;
 414
 415         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
 416         if (ret) {
 417                 RTE_LOG(ERR, PMD, "Failed to get bpool buffers number\n");
 418                 return;
 419         }
 420
 421         while (num--) {
 422                 struct pp2_buff_inf inf;
 423                 uint64_t addr;
 424
 425                 ret = pp2_bpool_get_buff(hifs[rte_lcore_id()], priv->bpool,
 426                                          &inf);
 427                 if (ret)
 428                         break;
 429
 430                 addr = cookie_addr_high | inf.cookie;
 431                 rte_pktmbuf_free((struct rte_mbuf *)addr);
 432         }
 433 }
 434
 435 /**
 436  * DPDK callback to stop the device.
 437  *
 438  * @param dev
 439  *   Pointer to Ethernet device structure.
 440  */
 441 static void
 442 mrvl_dev_stop(struct rte_eth_dev *dev)
 443 {
 444         struct mrvl_priv *priv = dev->data->dev_private;
 445
 446         mrvl_dev_set_link_down(dev);
 447         mrvl_flush_rx_queues(dev);
 448         mrvl_flush_tx_shadow_queues(dev);
 449         if (priv->qos_tbl)
 450                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
 451         pp2_ppio_deinit(priv->ppio);
 452         priv->ppio = NULL;
 453 }
 454
 455 /**
 456  * DPDK callback to close the device.
 457  *
 458  * @param dev
 459  *   Pointer to Ethernet device structure.
 460  */
 461 static void
 462 mrvl_dev_close(struct rte_eth_dev *dev)
 463 {
 464         struct mrvl_priv *priv = dev->data->dev_private;
 465         size_t i;
 466
 467         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
 468                 struct pp2_ppio_tc_params *tc_params =
 469                         &priv->ppio_params.inqs_params.tcs_params[i];
 470
 471                 if (tc_params->inqs_params) {
 472                         rte_free(tc_params->inqs_params);
 473                         tc_params->inqs_params = NULL;
 474                 }
 475         }
 476
 477         mrvl_flush_bpool(dev);
 478 }
 479
 480 /**
 481  * DPDK callback to retrieve physical link information.
 482  *
 483  * @param dev
 484  *   Pointer to Ethernet device structure.
 485  * @param wait_to_complete
 486  *   Wait for request completion (ignored).
 487  *
 488  * @return
 489  *   0 on success, negative error value otherwise.
 490  */
 491 static int
 492 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
 493 {
 494         /*
 495          * TODO
 496          * once MUSDK provides necessary API use it here
 497          */
 498         struct ethtool_cmd edata;
 499         struct ifreq req;
 500         int ret, fd;
 501
 502         edata.cmd = ETHTOOL_GSET;
 503
 504         strcpy(req.ifr_name, dev->data->name);
 505         req.ifr_data = (void *)&edata;
 506
 507         fd = socket(AF_INET, SOCK_DGRAM, 0);
 508         if (fd == -1)
 509                 return -EFAULT;
 510
 511         ret = ioctl(fd, SIOCETHTOOL, &req);
 512         if (ret == -1) {
 513                 close(fd);
 514                 return -EFAULT;
 515         }
 516
 517         close(fd);
 518
 519         switch (ethtool_cmd_speed(&edata)) {
 520         case SPEED_10:
 521                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
 522                 break;
 523         case SPEED_100:
 524                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
 525                 break;
 526         case SPEED_1000:
 527                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
 528                 break;
 529         case SPEED_10000:
 530                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
 531                 break;
 532         default:
 533                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
 534         }
 535
 536         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
 537                                                          ETH_LINK_HALF_DUPLEX;
 538         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
 539                                                            ETH_LINK_FIXED;
 540
 541         return 0;
 542 }
 543
 544 /**
 545  * DPDK callback to set the primary MAC address.
 546  *
 547  * @param dev
 548  *   Pointer to Ethernet device structure.
 549  * @param mac_addr
 550  *   MAC address to register.
 551  */
 552 static void
 553 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
 554 {
 555         struct mrvl_priv *priv = dev->data->dev_private;
 556
 557         pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
 558         /*
 559          * TODO
 560          * Port stops sending packets if pp2_ppio_set_mac_addr()
 561          * was called after pp2_ppio_enable(). As a quick fix issue
 562          * enable port once again.
 563          */
 564         pp2_ppio_enable(priv->ppio);
 565 }
 566
 567 /**
 568  * DPDK callback to get information about the device.
 569  *
 570  * @param dev
 571  *   Pointer to Ethernet device structure (unused).
 572  * @param info
 573  *   Info structure output buffer.
 574  */
 575 static void
 576 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
 577                    struct rte_eth_dev_info *info)
 578 {
 579         info->speed_capa = ETH_LINK_SPEED_10M |
 580                            ETH_LINK_SPEED_100M |
 581                            ETH_LINK_SPEED_1G |
 582                            ETH_LINK_SPEED_10G;
 583
 584         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
 585         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
 586         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
 587
 588         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
 589         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
 590         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
 591
 592         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
 593         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
 594         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
 595
 596         /* By default packets are dropped if no descriptors are available */
 597         info->default_rxconf.rx_drop_en = 1;
 598
 599         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
 600 }
 601
 602 /**
 603  * DPDK callback to get information about specific receive queue.
 604  *
 605  * @param dev
 606  *   Pointer to Ethernet device structure.
 607  * @param rx_queue_id
 608  *   Receive queue index.
 609  * @param qinfo
 610  *   Receive queue information structure.
 611  */
 612 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
 613                               struct rte_eth_rxq_info *qinfo)
 614 {
 615         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
 616         struct mrvl_priv *priv = dev->data->dev_private;
 617         int inq = priv->rxq_map[rx_queue_id].inq;
 618         int tc = priv->rxq_map[rx_queue_id].tc;
 619         struct pp2_ppio_tc_params *tc_params =
 620                 &priv->ppio_params.inqs_params.tcs_params[tc];
 621
 622         qinfo->mp = q->mp;
 623         qinfo->nb_desc = tc_params->inqs_params[inq].size;
 624 }
 625
 626 /**
 627  * DPDK callback to get information about specific transmit queue.
 628  *
 629  * @param dev
 630  *   Pointer to Ethernet device structure.
 631  * @param tx_queue_id
 632  *   Transmit queue index.
 633  * @param qinfo
 634  *   Transmit queue information structure.
 635  */
 636 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
 637                               struct rte_eth_txq_info *qinfo)
 638 {
 639         struct mrvl_priv *priv = dev->data->dev_private;
 640
 641         qinfo->nb_desc =
 642                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
 643 }
 644
 645 /**
 646  * Release buffers to hardware bpool (buffer-pool)
 647  *
 648  * @param rxq
 649  *   Receive queue pointer.
 650  * @param num
 651  *   Number of buffers to release to bpool.
 652  *
 653  * @return
 654  *   0 on success, negative error value otherwise.
 655  */
 656 static int
 657 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
 658 {
 659         struct buff_release_entry entries[MRVL_PP2_TXD_MAX];
 660         struct rte_mbuf *mbufs[MRVL_PP2_TXD_MAX];
 661         int i, ret;
 662         unsigned int core_id = rte_lcore_id();
 663         struct pp2_hif *hif = hifs[core_id];
 664         struct pp2_bpool *bpool = rxq->priv->bpool;
 665
 666         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
 667         if (ret)
 668                 return ret;
 669
 670         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
 671                 cookie_addr_high =
 672                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
 673
 674         for (i = 0; i < num; i++) {
 675                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
 676                         != cookie_addr_high) {
 677                         RTE_LOG(ERR, PMD,
 678                                 "mbuf virtual addr high 0x%lx out of range\n",
 679                                 (uint64_t)mbufs[i] >> 32);
 680                         goto out;
 681                 }
 682
 683                 entries[i].buff.addr =
 684                         rte_mbuf_data_dma_addr_default(mbufs[i]);
 685                 entries[i].buff.cookie = (pp2_cookie_t)(uint64_t)mbufs[i];
 686                 entries[i].bpool = bpool;
 687         }
 688
 689         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
 690         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
 691
 692         if (i != num)
 693                 goto out;
 694
 695         return 0;
 696 out:
 697         for (; i < num; i++)
 698                 rte_pktmbuf_free(mbufs[i]);
 699
 700         return -1;
 701 }
 702
 703 /**
 704  * DPDK callback to configure the receive queue.
 705  *
 706  * @param dev
 707  *   Pointer to Ethernet device structure.
 708  * @param idx
 709  *   RX queue index.
 710  * @param desc
 711  *   Number of descriptors to configure in queue.
 712  * @param socket
 713  *   NUMA socket on which memory must be allocated.
 714  * @param conf
 715  *   Thresholds parameters (unused_).
 716  * @param mp
 717  *   Memory pool for buffer allocations.
 718  *
 719  * @return
 720  *   0 on success, negative error value otherwise.
 721  */
 722 static int
 723 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 724                     unsigned int socket,
 725                     const struct rte_eth_rxconf *conf __rte_unused,
 726                     struct rte_mempool *mp)
 727 {
 728         struct mrvl_priv *priv = dev->data->dev_private;
 729         struct mrvl_rxq *rxq;
 730         uint32_t min_size,
 731                  max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
 732         int ret, tc, inq;
 733
 734         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
 735                 /*
 736                  * Unknown TC mapping, mapping will not have a correct queue.
 737                  */
 738                 RTE_LOG(ERR, PMD, "Unknown TC mapping for queue %hu eth%hhu\n",
 739                         idx, priv->ppio_id);
 740                 return -EFAULT;
 741         }
 742
 743         min_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM -
 744                    MRVL_PKT_EFFEC_OFFS;
 745         if (min_size < max_rx_pkt_len) {
 746                 RTE_LOG(ERR, PMD,
 747                         "Mbuf size must be increased to %u bytes to hold up to %u bytes of data.\n",
 748                         max_rx_pkt_len + RTE_PKTMBUF_HEADROOM +
 749                         MRVL_PKT_EFFEC_OFFS,
 750                         max_rx_pkt_len);
 751                 return -EINVAL;
 752         }
 753
 754         if (dev->data->rx_queues[idx]) {
 755                 rte_free(dev->data->rx_queues[idx]);
 756                 dev->data->rx_queues[idx] = NULL;
 757         }
 758
 759         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
 760         if (!rxq)
 761                 return -ENOMEM;
 762
 763         rxq->priv = priv;
 764         rxq->mp = mp;
 765         rxq->queue_id = idx;
 766         rxq->port_id = dev->data->port_id;
 767         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
 768
 769         tc = priv->rxq_map[rxq->queue_id].tc,
 770         inq = priv->rxq_map[rxq->queue_id].inq;
 771         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
 772                 desc;
 773
 774         ret = mrvl_fill_bpool(rxq, desc);
 775         if (ret) {
 776                 rte_free(rxq);
 777                 return ret;
 778         }
 779
 780         priv->bpool_init_size += desc;
 781
 782         dev->data->rx_queues[idx] = rxq;
 783
 784         return 0;
 785 }
 786
 787 /**
 788  * DPDK callback to release the receive queue.
 789  *
 790  * @param rxq
 791  *   Generic receive queue pointer.
 792  */
 793 static void
 794 mrvl_rx_queue_release(void *rxq)
 795 {
 796         struct mrvl_rxq *q = rxq;
 797         struct pp2_ppio_tc_params *tc_params;
 798         int i, num, tc, inq;
 799
 800         if (!q)
 801                 return;
 802
 803         tc = q->priv->rxq_map[q->queue_id].tc;
 804         inq = q->priv->rxq_map[q->queue_id].inq;
 805         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
 806         num = tc_params->inqs_params[inq].size;
 807         for (i = 0; i < num; i++) {
 808                 struct pp2_buff_inf inf;
 809                 uint64_t addr;
 810
 811                 pp2_bpool_get_buff(hifs[rte_lcore_id()], q->priv->bpool, &inf);
 812                 addr = cookie_addr_high | inf.cookie;
 813                 rte_pktmbuf_free((struct rte_mbuf *)addr);
 814         }
 815
 816         rte_free(q);
 817 }
 818
 819 /**
 820  * DPDK callback to configure the transmit queue.
 821  *
 822  * @param dev
 823  *   Pointer to Ethernet device structure.
 824  * @param idx
 825  *   Transmit queue index.
 826  * @param desc
 827  *   Number of descriptors to configure in the queue.
 828  * @param socket
 829  *   NUMA socket on which memory must be allocated.
 830  * @param conf
 831  *   Thresholds parameters (unused).
 832  *
 833  * @return
 834  *   0 on success, negative error value otherwise.
 835  */
 836 static int
 837 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 838                     unsigned int socket,
 839                     const struct rte_eth_txconf *conf __rte_unused)
 840 {
 841         struct mrvl_priv *priv = dev->data->dev_private;
 842         struct mrvl_txq *txq;
 843
 844         if (dev->data->tx_queues[idx]) {
 845                 rte_free(dev->data->tx_queues[idx]);
 846                 dev->data->tx_queues[idx] = NULL;
 847         }
 848
 849         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
 850         if (!txq)
 851                 return -ENOMEM;
 852
 853         txq->priv = priv;
 854         txq->queue_id = idx;
 855         txq->port_id = dev->data->port_id;
 856         dev->data->tx_queues[idx] = txq;
 857
 858         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
 859         priv->ppio_params.outqs_params.outqs_params[idx].weight = 1;
 860
 861         return 0;
 862 }
 863
 864 /**
 865  * DPDK callback to release the transmit queue.
 866  *
 867  * @param txq
 868  *   Generic transmit queue pointer.
 869  */
 870 static void
 871 mrvl_tx_queue_release(void *txq)
 872 {
 873         struct mrvl_txq *q = txq;
 874
 875         if (!q)
 876                 return;
 877
 878         rte_free(q);
 879 }
 880
 881 static const struct eth_dev_ops mrvl_ops = {
 882         .dev_configure = mrvl_dev_configure,
 883         .dev_start = mrvl_dev_start,
 884         .dev_stop = mrvl_dev_stop,
 885         .dev_set_link_up = mrvl_dev_set_link_up,
 886         .dev_set_link_down = mrvl_dev_set_link_down,
 887         .dev_close = mrvl_dev_close,
 888         .link_update = mrvl_link_update,
 889         .mac_addr_set = mrvl_mac_addr_set,
 890         .dev_infos_get = mrvl_dev_infos_get,
 891         .rxq_info_get = mrvl_rxq_info_get,
 892         .txq_info_get = mrvl_txq_info_get,
 893         .rx_queue_setup = mrvl_rx_queue_setup,
 894         .rx_queue_release = mrvl_rx_queue_release,
 895         .tx_queue_setup = mrvl_tx_queue_setup,
 896         .tx_queue_release = mrvl_tx_queue_release,
 897 };
 898
 899 /**
 900  * DPDK callback for receive.
 901  *
 902  * @param rxq
 903  *   Generic pointer to the receive queue.
 904  * @param rx_pkts
 905  *   Array to store received packets.
 906  * @param nb_pkts
 907  *   Maximum number of packets in array.
 908  *
 909  * @return
 910  *   Number of packets successfully received.
 911  */
 912 static uint16_t
 913 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 914 {
 915         struct mrvl_rxq *q = rxq;
 916         struct pp2_ppio_desc descs[nb_pkts];
 917         struct pp2_bpool *bpool;
 918         int i, ret, rx_done = 0;
 919         int num;
 920         unsigned int core_id = rte_lcore_id();
 921
 922         if (unlikely(!q->priv->ppio))
 923                 return 0;
 924
 925         bpool = q->priv->bpool;
 926
 927         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
 928                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
 929         if (unlikely(ret < 0)) {
 930                 RTE_LOG(ERR, PMD, "Failed to receive packets\n");
 931                 return 0;
 932         }
 933         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
 934
 935         for (i = 0; i < nb_pkts; i++) {
 936                 struct rte_mbuf *mbuf;
 937                 enum pp2_inq_desc_status status;
 938                 uint64_t addr;
 939
 940                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
 941                         struct pp2_ppio_desc *pref_desc;
 942                         u64 pref_addr;
 943
 944                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
 945                         pref_addr = cookie_addr_high |
 946                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
 947                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
 948                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
 949                 }
 950
 951                 addr = cookie_addr_high |
 952                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
 953                 mbuf = (struct rte_mbuf *)addr;
 954                 rte_pktmbuf_reset(mbuf);
 955
 956                 /* drop packet in case of mac, overrun or resource error */
 957                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
 958                 if (unlikely(status != PP2_DESC_ERR_OK)) {
 959                         struct pp2_buff_inf binf = {
 960                                 .addr = rte_mbuf_data_dma_addr_default(mbuf),
 961                                 .cookie = (pp2_cookie_t)(uint64_t)mbuf,
 962                         };
 963
 964                         pp2_bpool_put_buff(hifs[core_id], bpool, &binf);
 965                         mrvl_port_bpool_size
 966                                 [bpool->pp2_id][bpool->id][core_id]++;
 967                         continue;
 968                 }
 969
 970                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
 971                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
 972                 mbuf->data_len = mbuf->pkt_len;
 973                 mbuf->port = q->port_id;
 974
 975                 rx_pkts[rx_done++] = mbuf;
 976         }
 977
 978         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
 979                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
 980
 981                 if (unlikely(num <= q->priv->bpool_min_size ||
 982                              (!rx_done && num < q->priv->bpool_init_size))) {
 983                         ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE);
 984                         if (ret)
 985                                 RTE_LOG(ERR, PMD, "Failed to fill bpool\n");
 986                 } else if (unlikely(num > q->priv->bpool_max_size)) {
 987                         int i;
 988                         int pkt_to_remove = num - q->priv->bpool_init_size;
 989                         struct rte_mbuf *mbuf;
 990                         struct pp2_buff_inf buff;
 991
 992                         RTE_LOG(DEBUG, PMD,
 993                                 "\nport-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)\n",
 994                                 bpool->pp2_id, q->priv->ppio->port_id,
 995                                 bpool->id, pkt_to_remove, num,
 996                                 q->priv->bpool_init_size);
 997
 998                         for (i = 0; i < pkt_to_remove; i++) {
 999                                 pp2_bpool_get_buff(hifs[core_id], bpool, &buff);
1000                                 mbuf = (struct rte_mbuf *)
1001                                         (cookie_addr_high | buff.cookie);
1002                                 rte_pktmbuf_free(mbuf);
1003                         }
1004                         mrvl_port_bpool_size
1005                                 [bpool->pp2_id][bpool->id][core_id] -=
1006                                                                 pkt_to_remove;
1007                 }
1008                 rte_spinlock_unlock(&q->priv->lock);
1009         }
1010
1011         return rx_done;
1012 }
1013
1014 /**
1015  * Release already sent buffers to bpool (buffer-pool).
1016  *
1017  * @param ppio
1018  *   Pointer to the port structure.
1019  * @param hif
1020  *   Pointer to the MUSDK hardware interface.
1021  * @param sq
1022  *   Pointer to the shadow queue.
1023  * @param qid
1024  *   Queue id number.
1025  * @param force
1026  *   Force releasing packets.
1027  */
1028 static inline void
1029 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
1030                        struct mrvl_shadow_txq *sq, int qid, int force)
1031 {
1032         struct buff_release_entry *entry;
1033         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
1034         int i, core_id = rte_lcore_id();
1035
1036         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
1037
1038         sq->num_to_release += nb_done;
1039
1040         if (likely(!force &&
1041                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
1042                 return;
1043
1044         nb_done = sq->num_to_release;
1045         sq->num_to_release = 0;
1046
1047         for (i = 0; i < nb_done; i++) {
1048                 entry = &sq->ent[sq->tail + num];
1049                 if (unlikely(!entry->buff.addr)) {
1050                         RTE_LOG(ERR, PMD,
1051                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!\n",
1052                                 sq->tail, (u64)entry->buff.cookie,
1053                                 (u64)entry->buff.addr);
1054                         skip_bufs = 1;
1055                         goto skip;
1056                 }
1057
1058                 if (unlikely(!entry->bpool)) {
1059                         struct rte_mbuf *mbuf;
1060
1061                         mbuf = (struct rte_mbuf *)
1062                                (cookie_addr_high | entry->buff.cookie);
1063                         rte_pktmbuf_free(mbuf);
1064                         skip_bufs = 1;
1065                         goto skip;
1066                 }
1067
1068                 mrvl_port_bpool_size
1069                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
1070                 num++;
1071                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
1072                         goto skip;
1073                 continue;
1074 skip:
1075                 if (likely(num))
1076                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
1077                 num += skip_bufs;
1078                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
1079                 sq->size -= num;
1080                 num = 0;
1081         }
1082
1083         if (likely(num)) {
1084                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
1085                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
1086                 sq->size -= num;
1087         }
1088 }
1089
1090 /**
1091  * DPDK callback for transmit.
1092  *
1093  * @param txq
1094  *   Generic pointer transmit queue.
1095  * @param tx_pkts
1096  *   Packets to transmit.
1097  * @param nb_pkts
1098  *   Number of packets in array.
1099  *
1100  * @return
1101  *   Number of packets successfully transmitted.
1102  */
1103 static uint16_t
1104 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1105 {
1106         struct mrvl_txq *q = txq;
1107         struct mrvl_shadow_txq *sq = &shadow_txqs[q->port_id][rte_lcore_id()];
1108         struct pp2_hif *hif = hifs[rte_lcore_id()];
1109         struct pp2_ppio_desc descs[nb_pkts];
1110         int i;
1111         uint16_t num, sq_free_size;
1112
1113         if (unlikely(!q->priv->ppio))
1114                 return 0;
1115
1116         if (sq->size)
1117                 mrvl_free_sent_buffers(q->priv->ppio, hif, sq, q->queue_id, 0);
1118
1119         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
1120         if (unlikely(nb_pkts > sq_free_size)) {
1121                 RTE_LOG(DEBUG, PMD,
1122                         "No room in shadow queue for %d packets! %d packets will be sent.\n",
1123                         nb_pkts, sq_free_size);
1124                 nb_pkts = sq_free_size;
1125         }
1126
1127         for (i = 0; i < nb_pkts; i++) {
1128                 struct rte_mbuf *mbuf = tx_pkts[i];
1129
1130                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
1131                         struct rte_mbuf *pref_pkt_hdr;
1132
1133                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
1134                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
1135                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
1136                 }
1137
1138                 sq->ent[sq->head].buff.cookie = (pp2_cookie_t)(uint64_t)mbuf;
1139                 sq->ent[sq->head].buff.addr =
1140                         rte_mbuf_data_dma_addr_default(mbuf);
1141                 sq->ent[sq->head].bpool =
1142                         (unlikely(mbuf->port == 0xff || mbuf->refcnt > 1)) ?
1143                          NULL : mrvl_port_to_bpool_lookup[mbuf->port];
1144                 sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
1145                 sq->size++;
1146
1147                 pp2_ppio_outq_desc_reset(&descs[i]);
1148                 pp2_ppio_outq_desc_set_phys_addr(&descs[i],
1149                                                  rte_pktmbuf_mtophys(mbuf));
1150                 pp2_ppio_outq_desc_set_pkt_offset(&descs[i], 0);
1151                 pp2_ppio_outq_desc_set_pkt_len(&descs[i],
1152                                                rte_pktmbuf_pkt_len(mbuf));
1153         }
1154
1155         num = nb_pkts;
1156         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
1157         /* number of packets that were not sent */
1158         if (unlikely(num > nb_pkts)) {
1159                 for (i = nb_pkts; i < num; i++) {
1160                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
1161                                 MRVL_PP2_TX_SHADOWQ_MASK;
1162                 }
1163                 sq->size -= num - nb_pkts;
1164         }
1165
1166         return nb_pkts;
1167 }
1168
1169 /**
1170  * Initialize packet processor.
1171  *
1172  * @return
1173  *   0 on success, negative error value otherwise.
1174  */
1175 static int
1176 mrvl_init_pp2(void)
1177 {
1178         struct pp2_init_params init_params;
1179
1180         memset(&init_params, 0, sizeof(init_params));
1181         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
1182         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
1183
1184         return pp2_init(&init_params);
1185 }
1186
1187 /**
1188  * Deinitialize packet processor.
1189  *
1190  * @return
1191  *   0 on success, negative error value otherwise.
1192  */
1193 static void
1194 mrvl_deinit_pp2(void)
1195 {
1196         pp2_deinit();
1197 }
1198
1199 /**
1200  * Create private device structure.
1201  *
1202  * @param dev_name
1203  *   Pointer to the port name passed in the initialization parameters.
1204  *
1205  * @return
1206  *   Pointer to the newly allocated private device structure.
1207  */
1208 static struct mrvl_priv *
1209 mrvl_priv_create(const char *dev_name)
1210 {
1211         struct pp2_bpool_params bpool_params;
1212         char match[MRVL_MATCH_LEN];
1213         struct mrvl_priv *priv;
1214         int ret, bpool_bit;
1215
1216         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
1217         if (!priv)
1218                 return NULL;
1219
1220         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
1221                                        &priv->pp_id, &priv->ppio_id);
1222         if (ret)
1223                 goto out_free_priv;
1224
1225         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
1226                                      PP2_BPOOL_NUM_POOLS);
1227         if (bpool_bit < 0)
1228                 goto out_free_priv;
1229         priv->bpool_bit = bpool_bit;
1230
1231         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
1232                  priv->bpool_bit);
1233         memset(&bpool_params, 0, sizeof(bpool_params));
1234         bpool_params.match = match;
1235         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
1236         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
1237         if (ret)
1238                 goto out_clear_bpool_bit;
1239
1240         priv->ppio_params.type = PP2_PPIO_T_NIC;
1241         rte_spinlock_init(&priv->lock);
1242
1243         return priv;
1244 out_clear_bpool_bit:
1245         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
1246 out_free_priv:
1247         rte_free(priv);
1248         return NULL;
1249 }
1250
1251 /**
1252  * Create device representing Ethernet port.
1253  *
1254  * @param name
1255  *   Pointer to the port's name.
1256  *
1257  * @return
1258  *   0 on success, negative error value otherwise.
1259  */
1260 static int
1261 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
1262 {
1263         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
1264         struct rte_eth_dev *eth_dev;
1265         struct mrvl_priv *priv;
1266         struct ifreq req;
1267
1268         eth_dev = rte_eth_dev_allocate(name);
1269         if (!eth_dev)
1270                 return -ENOMEM;
1271
1272         priv = mrvl_priv_create(name);
1273         if (!priv) {
1274                 ret = -ENOMEM;
1275                 goto out_free_dev;
1276         }
1277
1278         eth_dev->data->mac_addrs =
1279                 rte_zmalloc("mac_addrs",
1280                             ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
1281         if (!eth_dev->data->mac_addrs) {
1282                 RTE_LOG(ERR, PMD, "Failed to allocate space for eth addrs\n");
1283                 ret = -ENOMEM;
1284                 goto out_free_priv;
1285         }
1286
1287         memset(&req, 0, sizeof(req));
1288         strcpy(req.ifr_name, name);
1289         ret = ioctl(fd, SIOCGIFHWADDR, &req);
1290         if (ret)
1291                 goto out_free_mac;
1292
1293         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
1294                req.ifr_addr.sa_data, ETHER_ADDR_LEN);
1295
1296         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
1297         eth_dev->tx_pkt_burst = mrvl_tx_pkt_burst;
1298         eth_dev->data->dev_private = priv;
1299         eth_dev->device = &vdev->device;
1300         eth_dev->dev_ops = &mrvl_ops;
1301
1302         return 0;
1303 out_free_mac:
1304         rte_free(eth_dev->data->mac_addrs);
1305 out_free_dev:
1306         rte_eth_dev_release_port(eth_dev);
1307 out_free_priv:
1308         rte_free(priv);
1309
1310         return ret;
1311 }
1312
1313 /**
1314  * Cleanup previously created device representing Ethernet port.
1315  *
1316  * @param name
1317  *   Pointer to the port name.
1318  */
1319 static void
1320 mrvl_eth_dev_destroy(const char *name)
1321 {
1322         struct rte_eth_dev *eth_dev;
1323         struct mrvl_priv *priv;
1324
1325         eth_dev = rte_eth_dev_allocated(name);
1326         if (!eth_dev)
1327                 return;
1328
1329         priv = eth_dev->data->dev_private;
1330         pp2_bpool_deinit(priv->bpool);
1331         rte_free(priv);
1332         rte_free(eth_dev->data->mac_addrs);
1333         rte_eth_dev_release_port(eth_dev);
1334 }
1335
1336 /**
1337  * Callback used by rte_kvargs_process() during argument parsing.
1338  *
1339  * @param key
1340  *   Pointer to the parsed key (unused).
1341  * @param value
1342  *   Pointer to the parsed value.
1343  * @param extra_args
1344  *   Pointer to the extra arguments which contains address of the
1345  *   table of pointers to parsed interface names.
1346  *
1347  * @return
1348  *   Always 0.
1349  */
1350 static int
1351 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
1352                  void *extra_args)
1353 {
1354         const char **ifnames = extra_args;
1355
1356         ifnames[mrvl_ports_nb++] = value;
1357
1358         return 0;
1359 }
1360
1361 /**
1362  * Initialize per-lcore MUSDK hardware interfaces (hifs).
1363  *
1364  * @return
1365  *   0 on success, negative error value otherwise.
1366  */
1367 static int
1368 mrvl_init_hifs(void)
1369 {
1370         struct pp2_hif_params params;
1371         char match[MRVL_MATCH_LEN];
1372         int i, ret;
1373
1374         RTE_LCORE_FOREACH(i) {
1375                 ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
1376                 if (ret < 0)
1377                         return ret;
1378
1379                 snprintf(match, sizeof(match), "hif-%d", ret);
1380                 memset(&params, 0, sizeof(params));
1381                 params.match = match;
1382                 params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
1383                 ret = pp2_hif_init(&params, &hifs[i]);
1384                 if (ret) {
1385                         RTE_LOG(ERR, PMD, "Failed to initialize hif %d\n", i);
1386                         return ret;
1387                 }
1388         }
1389
1390         return 0;
1391 }
1392
1393 /**
1394  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
1395  */
1396 static void
1397 mrvl_deinit_hifs(void)
1398 {
1399         int i;
1400
1401         RTE_LCORE_FOREACH(i) {
1402                 if (hifs[i])
1403                         pp2_hif_deinit(hifs[i]);
1404         }
1405 }
1406
1407 static void mrvl_set_first_last_cores(int core_id)
1408 {
1409         if (core_id < mrvl_lcore_first)
1410                 mrvl_lcore_first = core_id;
1411
1412         if (core_id > mrvl_lcore_last)
1413                 mrvl_lcore_last = core_id;
1414 }
1415
1416 /**
1417  * DPDK callback to register the virtual device.
1418  *
1419  * @param vdev
1420  *   Pointer to the virtual device.
1421  *
1422  * @return
1423  *   0 on success, negative error value otherwise.
1424  */
1425 static int
1426 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
1427 {
1428         struct rte_kvargs *kvlist;
1429         const char *ifnames[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
1430         int ret = -EINVAL;
1431         uint32_t i, ifnum, cfgnum, core_id;
1432         const char *params;
1433
1434         params = rte_vdev_device_args(vdev);
1435         if (!params)
1436                 return -EINVAL;
1437
1438         kvlist = rte_kvargs_parse(params, valid_args);
1439         if (!kvlist)
1440                 return -EINVAL;
1441
1442         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
1443         if (ifnum > RTE_DIM(ifnames))
1444                 goto out_free_kvlist;
1445
1446         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
1447                            mrvl_get_ifnames, &ifnames);
1448
1449         cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
1450         if (cfgnum > 1) {
1451                 RTE_LOG(ERR, PMD, "Cannot handle more than one config file!\n");
1452                 goto out_free_kvlist;
1453         } else if (cfgnum == 1) {
1454                 rte_kvargs_process(kvlist, MRVL_CFG_ARG,
1455                                    mrvl_get_qoscfg, &mrvl_qos_cfg);
1456         }
1457
1458         /*
1459          * ret == -EEXIST is correct, it means DMA
1460          * has been already initialized (by another PMD).
1461          */
1462         ret = mv_sys_dma_mem_init(RTE_MRVL_MUSDK_DMA_MEMSIZE);
1463         if (ret < 0 && ret != -EEXIST)
1464                 goto out_free_kvlist;
1465
1466         ret = mrvl_init_pp2();
1467         if (ret) {
1468                 RTE_LOG(ERR, PMD, "Failed to init PP!\n");
1469                 goto out_deinit_dma;
1470         }
1471
1472         ret = mrvl_init_hifs();
1473         if (ret)
1474                 goto out_deinit_hifs;
1475
1476         for (i = 0; i < ifnum; i++) {
1477                 RTE_LOG(INFO, PMD, "Creating %s\n", ifnames[i]);
1478                 ret = mrvl_eth_dev_create(vdev, ifnames[i]);
1479                 if (ret)
1480                         goto out_cleanup;
1481         }
1482
1483         rte_kvargs_free(kvlist);
1484
1485         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
1486
1487         mrvl_lcore_first = RTE_MAX_LCORE;
1488         mrvl_lcore_last = 0;
1489
1490         RTE_LCORE_FOREACH(core_id) {
1491                 mrvl_set_first_last_cores(core_id);
1492         }
1493
1494         return 0;
1495 out_cleanup:
1496         for (; i > 0; i--)
1497                 mrvl_eth_dev_destroy(ifnames[i]);
1498 out_deinit_hifs:
1499         mrvl_deinit_hifs();
1500         mrvl_deinit_pp2();
1501 out_deinit_dma:
1502         mv_sys_dma_mem_destroy();
1503 out_free_kvlist:
1504         rte_kvargs_free(kvlist);
1505
1506         return ret;
1507 }
1508
1509 /**
1510  * DPDK callback to remove virtual device.
1511  *
1512  * @param vdev
1513  *   Pointer to the removed virtual device.
1514  *
1515  * @return
1516  *   0 on success, negative error value otherwise.
1517  */
1518 static int
1519 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
1520 {
1521         int i;
1522         const char *name;
1523
1524         name = rte_vdev_device_name(vdev);
1525         if (!name)
1526                 return -EINVAL;
1527
1528         RTE_LOG(INFO, PMD, "Removing %s\n", name);
1529
1530         for (i = 0; i < rte_eth_dev_count(); i++) {
1531                 char ifname[RTE_ETH_NAME_MAX_LEN];
1532
1533                 rte_eth_dev_get_name_by_port(i, ifname);
1534                 mrvl_eth_dev_destroy(ifname);
1535         }
1536
1537         mrvl_deinit_hifs();
1538         mrvl_deinit_pp2();
1539         mv_sys_dma_mem_destroy();
1540
1541         return 0;
1542 }
1543
1544 static struct rte_vdev_driver pmd_mrvl_drv = {
1545         .probe = rte_pmd_mrvl_probe,
1546         .remove = rte_pmd_mrvl_remove,
1547 };
1548
1549 RTE_PMD_REGISTER_VDEV(net_mrvl, pmd_mrvl_drv);
1550 RTE_PMD_REGISTER_ALIAS(net_mrvl, eth_mrvl);