drivers/net/mrvl/mrvl_ethdev.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2017 Semihalf. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Semihalf nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <rte_ethdev.h>
  34 #include <rte_kvargs.h>
  35 #include <rte_log.h>
  36 #include <rte_malloc.h>
  37 #include <rte_vdev.h>
  38
  39 /* Unluckily, container_of is defined by both DPDK and MUSDK,
  40  * we'll declare only one version.
  41  *
  42  * Note that it is not used in this PMD anyway.
  43  */
  44 #ifdef container_of
  45 #undef container_of
  46 #endif
  47
  48 #include <drivers/mv_pp2.h>
  49 #include <drivers/mv_pp2_bpool.h>
  50 #include <drivers/mv_pp2_hif.h>
  51
  52 #include <fcntl.h>
  53 #include <linux/ethtool.h>
  54 #include <linux/sockios.h>
  55 #include <net/if.h>
  56 #include <net/if_arp.h>
  57 #include <sys/ioctl.h>
  58 #include <sys/socket.h>
  59 #include <sys/stat.h>
  60 #include <sys/types.h>
  61
  62 #include "mrvl_ethdev.h"
  63 #include "mrvl_qos.h"
  64
  65 /* bitmask with reserved hifs */
  66 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
  67 /* bitmask with reserved bpools */
  68 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
  69 /* maximum number of available hifs */
  70 #define MRVL_MUSDK_HIFS_MAX 9
  71
  72 #define MRVL_MAC_ADDRS_MAX 1
  73 /* prefetch shift */
  74 #define MRVL_MUSDK_PREFETCH_SHIFT 2
  75
  76 #define MRVL_MATCH_LEN 16
  77 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
  78 /* Maximum allowable packet size */
  79 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
  80
  81 #define MRVL_IFACE_NAME_ARG "iface"
  82 #define MRVL_CFG_ARG "cfg"
  83
  84 #define MRVL_BURST_SIZE 64
  85
  86 #define MRVL_ARP_LENGTH 28
  87
  88 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
  89
  90 #define MRVL_COOKIE_HIGH_ADDR_SHIFT     (sizeof(pp2_cookie_t) * 8)
  91 #define MRVL_COOKIE_HIGH_ADDR_MASK      (~0ULL << MRVL_COOKIE_HIGH_ADDR_SHIFT)
  92
  93 static const char * const valid_args[] = {
  94         MRVL_IFACE_NAME_ARG,
  95         MRVL_CFG_ARG,
  96         NULL
  97 };
  98
  99 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
 100 static struct pp2_hif *hifs[RTE_MAX_LCORE];
 101 static int used_bpools[PP2_NUM_PKT_PROC] = {
 102         MRVL_MUSDK_BPOOLS_RESERVED,
 103         MRVL_MUSDK_BPOOLS_RESERVED
 104 };
 105
 106 struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
 107 int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
 108 uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
 109
 110 /*
 111  * To use buffer harvesting based on loopback port shadow queue structure
 112  * was introduced for buffers information bookkeeping.
 113  *
 114  * Before sending the packet, related buffer information (pp2_buff_inf) is
 115  * stored in shadow queue. After packet is transmitted no longer used
 116  * packet buffer is released back to it's original hardware pool,
 117  * on condition it originated from interface.
 118  * In case it  was generated by application itself i.e: mbuf->port field is
 119  * 0xff then its released to software mempool.
 120  */
 121 struct mrvl_shadow_txq {
 122         int head;           /* write index - used when sending buffers */
 123         int tail;           /* read index - used when releasing buffers */
 124         u16 size;           /* queue occupied size */
 125         u16 num_to_release; /* number of buffers sent, that can be released */
 126         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
 127 };
 128
 129 struct mrvl_rxq {
 130         struct mrvl_priv *priv;
 131         struct rte_mempool *mp;
 132         int queue_id;
 133         int port_id;
 134 };
 135
 136 struct mrvl_txq {
 137         struct mrvl_priv *priv;
 138         int queue_id;
 139         int port_id;
 140 };
 141
 142 /*
 143  * Every tx queue should have dedicated shadow tx queue.
 144  *
 145  * Ports assigned by DPDK might not start at zero or be continuous so
 146  * as a workaround define shadow queues for each possible port so that
 147  * we eventually fit somewhere.
 148  */
 149 struct mrvl_shadow_txq shadow_txqs[RTE_MAX_ETHPORTS][RTE_MAX_LCORE];
 150
 151 /** Number of ports configured. */
 152 int mrvl_ports_nb;
 153 static int mrvl_lcore_first;
 154 static int mrvl_lcore_last;
 155
 156 static inline int
 157 mrvl_get_bpool_size(int pp2_id, int pool_id)
 158 {
 159         int i;
 160         int size = 0;
 161
 162         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
 163                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
 164
 165         return size;
 166 }
 167
 168 static inline int
 169 mrvl_reserve_bit(int *bitmap, int max)
 170 {
 171         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
 172
 173         if (n >= max)
 174                 return -1;
 175
 176         *bitmap |= 1 << n;
 177
 178         return n;
 179 }
 180
 181 /**
 182  * Ethernet device configuration.
 183  *
 184  * Prepare the driver for a given number of TX and RX queues.
 185  *
 186  * @param dev
 187  *   Pointer to Ethernet device structure.
 188  *
 189  * @return
 190  *   0 on success, negative error value otherwise.
 191  */
 192 static int
 193 mrvl_dev_configure(struct rte_eth_dev *dev)
 194 {
 195         struct mrvl_priv *priv = dev->data->dev_private;
 196         int ret;
 197
 198         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE) {
 199                 RTE_LOG(INFO, PMD, "Unsupported rx multi queue mode %d\n",
 200                         dev->data->dev_conf.rxmode.mq_mode);
 201                 return -EINVAL;
 202         }
 203
 204         if (!dev->data->dev_conf.rxmode.hw_strip_crc) {
 205                 RTE_LOG(INFO, PMD,
 206                         "L2 CRC stripping is always enabled in hw\n");
 207                 dev->data->dev_conf.rxmode.hw_strip_crc = 1;
 208         }
 209
 210         if (dev->data->dev_conf.rxmode.hw_vlan_strip) {
 211                 RTE_LOG(INFO, PMD, "VLAN stripping not supported\n");
 212                 return -EINVAL;
 213         }
 214
 215         if (dev->data->dev_conf.rxmode.split_hdr_size) {
 216                 RTE_LOG(INFO, PMD, "Split headers not supported\n");
 217                 return -EINVAL;
 218         }
 219
 220         if (dev->data->dev_conf.rxmode.enable_scatter) {
 221                 RTE_LOG(INFO, PMD, "RX Scatter/Gather not supported\n");
 222                 return -EINVAL;
 223         }
 224
 225         if (dev->data->dev_conf.rxmode.enable_lro) {
 226                 RTE_LOG(INFO, PMD, "LRO not supported\n");
 227                 return -EINVAL;
 228         }
 229
 230         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
 231                                   dev->data->nb_rx_queues);
 232         if (ret < 0)
 233                 return ret;
 234
 235         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
 236         priv->nb_rx_queues = dev->data->nb_rx_queues;
 237
 238         return 0;
 239 }
 240
 241 /**
 242  * DPDK callback to change the MTU.
 243  *
 244  * Setting the MTU affects hardware MRU (packets larger than the MRU
 245  * will be dropped).
 246  *
 247  * @param dev
 248  *   Pointer to Ethernet device structure.
 249  * @param mtu
 250  *   New MTU.
 251  *
 252  * @return
 253  *   0 on success, negative error value otherwise.
 254  */
 255 static int
 256 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 257 {
 258         struct mrvl_priv *priv = dev->data->dev_private;
 259         /* extra MV_MH_SIZE bytes are required for Marvell tag */
 260         uint16_t mru = mtu + MV_MH_SIZE + ETHER_HDR_LEN + ETHER_CRC_LEN;
 261         int ret;
 262
 263         if (mtu < ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX)
 264                 return -EINVAL;
 265
 266         ret = pp2_ppio_set_mru(priv->ppio, mru);
 267         if (ret)
 268                 return ret;
 269
 270         return pp2_ppio_set_mtu(priv->ppio, mtu);
 271 }
 272
 273 /**
 274  * DPDK callback to bring the link up.
 275  *
 276  * @param dev
 277  *   Pointer to Ethernet device structure.
 278  *
 279  * @return
 280  *   0 on success, negative error value otherwise.
 281  */
 282 static int
 283 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
 284 {
 285         struct mrvl_priv *priv = dev->data->dev_private;
 286         int ret;
 287
 288         ret = pp2_ppio_enable(priv->ppio);
 289         if (ret)
 290                 return ret;
 291
 292         /*
 293          * mtu/mru can be updated if pp2_ppio_enable() was called at least once
 294          * as pp2_ppio_enable() changes port->t_mode from default 0 to
 295          * PP2_TRAFFIC_INGRESS_EGRESS.
 296          *
 297          * Set mtu to default DPDK value here.
 298          */
 299         ret = mrvl_mtu_set(dev, dev->data->mtu);
 300         if (ret)
 301                 pp2_ppio_disable(priv->ppio);
 302
 303         dev->data->dev_link.link_status = ETH_LINK_UP;
 304
 305         return ret;
 306 }
 307
 308 /**
 309  * DPDK callback to bring the link down.
 310  *
 311  * @param dev
 312  *   Pointer to Ethernet device structure.
 313  *
 314  * @return
 315  *   0 on success, negative error value otherwise.
 316  */
 317 static int
 318 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
 319 {
 320         struct mrvl_priv *priv = dev->data->dev_private;
 321         int ret;
 322
 323         ret = pp2_ppio_disable(priv->ppio);
 324         if (ret)
 325                 return ret;
 326
 327         dev->data->dev_link.link_status = ETH_LINK_DOWN;
 328
 329         return ret;
 330 }
 331
 332 /**
 333  * DPDK callback to start the device.
 334  *
 335  * @param dev
 336  *   Pointer to Ethernet device structure.
 337  *
 338  * @return
 339  *   0 on success, negative errno value on failure.
 340  */
 341 static int
 342 mrvl_dev_start(struct rte_eth_dev *dev)
 343 {
 344         struct mrvl_priv *priv = dev->data->dev_private;
 345         char match[MRVL_MATCH_LEN];
 346         int ret;
 347
 348         snprintf(match, sizeof(match), "ppio-%d:%d",
 349                  priv->pp_id, priv->ppio_id);
 350         priv->ppio_params.match = match;
 351
 352         /*
 353          * Calculate the maximum bpool size for refill feature to 1.5 of the
 354          * configured size. In case the bpool size will exceed this value,
 355          * superfluous buffers will be removed
 356          */
 357         priv->bpool_max_size = priv->bpool_init_size +
 358                               (priv->bpool_init_size >> 1);
 359         /*
 360          * Calculate the minimum bpool size for refill feature as follows:
 361          * 2 default burst sizes multiply by number of rx queues.
 362          * If the bpool size will be below this value, new buffers will
 363          * be added to the pool.
 364          */
 365         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
 366
 367         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
 368         if (ret)
 369                 return ret;
 370
 371         /* For default QoS config, don't start classifier. */
 372         if (mrvl_qos_cfg) {
 373                 ret = mrvl_start_qos_mapping(priv);
 374                 if (ret) {
 375                         pp2_ppio_deinit(priv->ppio);
 376                         return ret;
 377                 }
 378         }
 379
 380         ret = mrvl_dev_set_link_up(dev);
 381         if (ret)
 382                 goto out;
 383
 384         return 0;
 385 out:
 386         pp2_ppio_deinit(priv->ppio);
 387         return ret;
 388 }
 389
 390 /**
 391  * Flush receive queues.
 392  *
 393  * @param dev
 394  *   Pointer to Ethernet device structure.
 395  */
 396 static void
 397 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
 398 {
 399         int i;
 400
 401         RTE_LOG(INFO, PMD, "Flushing rx queues\n");
 402         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 403                 int ret, num;
 404
 405                 do {
 406                         struct mrvl_rxq *q = dev->data->rx_queues[i];
 407                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
 408
 409                         num = MRVL_PP2_RXD_MAX;
 410                         ret = pp2_ppio_recv(q->priv->ppio,
 411                                             q->priv->rxq_map[q->queue_id].tc,
 412                                             q->priv->rxq_map[q->queue_id].inq,
 413                                             descs, (uint16_t *)&num);
 414                 } while (ret == 0 && num);
 415         }
 416 }
 417
 418 /**
 419  * Flush transmit shadow queues.
 420  *
 421  * @param dev
 422  *   Pointer to Ethernet device structure.
 423  */
 424 static void
 425 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
 426 {
 427         int i;
 428
 429         RTE_LOG(INFO, PMD, "Flushing tx shadow queues\n");
 430         for (i = 0; i < RTE_MAX_LCORE; i++) {
 431                 struct mrvl_shadow_txq *sq =
 432                         &shadow_txqs[dev->data->port_id][i];
 433
 434                 while (sq->tail != sq->head) {
 435                         uint64_t addr = cookie_addr_high |
 436                                         sq->ent[sq->tail].buff.cookie;
 437                         rte_pktmbuf_free((struct rte_mbuf *)addr);
 438                         sq->tail = (sq->tail + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
 439                 }
 440
 441                 memset(sq, 0, sizeof(*sq));
 442         }
 443 }
 444
 445 /**
 446  * Flush hardware bpool (buffer-pool).
 447  *
 448  * @param dev
 449  *   Pointer to Ethernet device structure.
 450  */
 451 static void
 452 mrvl_flush_bpool(struct rte_eth_dev *dev)
 453 {
 454         struct mrvl_priv *priv = dev->data->dev_private;
 455         uint32_t num;
 456         int ret;
 457
 458         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
 459         if (ret) {
 460                 RTE_LOG(ERR, PMD, "Failed to get bpool buffers number\n");
 461                 return;
 462         }
 463
 464         while (num--) {
 465                 struct pp2_buff_inf inf;
 466                 uint64_t addr;
 467
 468                 ret = pp2_bpool_get_buff(hifs[rte_lcore_id()], priv->bpool,
 469                                          &inf);
 470                 if (ret)
 471                         break;
 472
 473                 addr = cookie_addr_high | inf.cookie;
 474                 rte_pktmbuf_free((struct rte_mbuf *)addr);
 475         }
 476 }
 477
 478 /**
 479  * DPDK callback to stop the device.
 480  *
 481  * @param dev
 482  *   Pointer to Ethernet device structure.
 483  */
 484 static void
 485 mrvl_dev_stop(struct rte_eth_dev *dev)
 486 {
 487         struct mrvl_priv *priv = dev->data->dev_private;
 488
 489         mrvl_dev_set_link_down(dev);
 490         mrvl_flush_rx_queues(dev);
 491         mrvl_flush_tx_shadow_queues(dev);
 492         if (priv->qos_tbl)
 493                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
 494         pp2_ppio_deinit(priv->ppio);
 495         priv->ppio = NULL;
 496 }
 497
 498 /**
 499  * DPDK callback to close the device.
 500  *
 501  * @param dev
 502  *   Pointer to Ethernet device structure.
 503  */
 504 static void
 505 mrvl_dev_close(struct rte_eth_dev *dev)
 506 {
 507         struct mrvl_priv *priv = dev->data->dev_private;
 508         size_t i;
 509
 510         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
 511                 struct pp2_ppio_tc_params *tc_params =
 512                         &priv->ppio_params.inqs_params.tcs_params[i];
 513
 514                 if (tc_params->inqs_params) {
 515                         rte_free(tc_params->inqs_params);
 516                         tc_params->inqs_params = NULL;
 517                 }
 518         }
 519
 520         mrvl_flush_bpool(dev);
 521 }
 522
 523 /**
 524  * DPDK callback to retrieve physical link information.
 525  *
 526  * @param dev
 527  *   Pointer to Ethernet device structure.
 528  * @param wait_to_complete
 529  *   Wait for request completion (ignored).
 530  *
 531  * @return
 532  *   0 on success, negative error value otherwise.
 533  */
 534 static int
 535 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
 536 {
 537         /*
 538          * TODO
 539          * once MUSDK provides necessary API use it here
 540          */
 541         struct ethtool_cmd edata;
 542         struct ifreq req;
 543         int ret, fd;
 544
 545         edata.cmd = ETHTOOL_GSET;
 546
 547         strcpy(req.ifr_name, dev->data->name);
 548         req.ifr_data = (void *)&edata;
 549
 550         fd = socket(AF_INET, SOCK_DGRAM, 0);
 551         if (fd == -1)
 552                 return -EFAULT;
 553
 554         ret = ioctl(fd, SIOCETHTOOL, &req);
 555         if (ret == -1) {
 556                 close(fd);
 557                 return -EFAULT;
 558         }
 559
 560         close(fd);
 561
 562         switch (ethtool_cmd_speed(&edata)) {
 563         case SPEED_10:
 564                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
 565                 break;
 566         case SPEED_100:
 567                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
 568                 break;
 569         case SPEED_1000:
 570                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
 571                 break;
 572         case SPEED_10000:
 573                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
 574                 break;
 575         default:
 576                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
 577         }
 578
 579         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
 580                                                          ETH_LINK_HALF_DUPLEX;
 581         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
 582                                                            ETH_LINK_FIXED;
 583
 584         return 0;
 585 }
 586
 587 /**
 588  * DPDK callback to set the primary MAC address.
 589  *
 590  * @param dev
 591  *   Pointer to Ethernet device structure.
 592  * @param mac_addr
 593  *   MAC address to register.
 594  */
 595 static void
 596 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct ether_addr *mac_addr)
 597 {
 598         struct mrvl_priv *priv = dev->data->dev_private;
 599
 600         pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
 601         /*
 602          * TODO
 603          * Port stops sending packets if pp2_ppio_set_mac_addr()
 604          * was called after pp2_ppio_enable(). As a quick fix issue
 605          * enable port once again.
 606          */
 607         pp2_ppio_enable(priv->ppio);
 608 }
 609
 610 /**
 611  * DPDK callback to get information about the device.
 612  *
 613  * @param dev
 614  *   Pointer to Ethernet device structure (unused).
 615  * @param info
 616  *   Info structure output buffer.
 617  */
 618 static void
 619 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
 620                    struct rte_eth_dev_info *info)
 621 {
 622         info->speed_capa = ETH_LINK_SPEED_10M |
 623                            ETH_LINK_SPEED_100M |
 624                            ETH_LINK_SPEED_1G |
 625                            ETH_LINK_SPEED_10G;
 626
 627         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
 628         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
 629         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
 630
 631         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
 632         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
 633         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
 634
 635         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
 636         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
 637         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
 638
 639         /* By default packets are dropped if no descriptors are available */
 640         info->default_rxconf.rx_drop_en = 1;
 641
 642         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
 643 }
 644
 645 /**
 646  * DPDK callback to get information about specific receive queue.
 647  *
 648  * @param dev
 649  *   Pointer to Ethernet device structure.
 650  * @param rx_queue_id
 651  *   Receive queue index.
 652  * @param qinfo
 653  *   Receive queue information structure.
 654  */
 655 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
 656                               struct rte_eth_rxq_info *qinfo)
 657 {
 658         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
 659         struct mrvl_priv *priv = dev->data->dev_private;
 660         int inq = priv->rxq_map[rx_queue_id].inq;
 661         int tc = priv->rxq_map[rx_queue_id].tc;
 662         struct pp2_ppio_tc_params *tc_params =
 663                 &priv->ppio_params.inqs_params.tcs_params[tc];
 664
 665         qinfo->mp = q->mp;
 666         qinfo->nb_desc = tc_params->inqs_params[inq].size;
 667 }
 668
 669 /**
 670  * DPDK callback to get information about specific transmit queue.
 671  *
 672  * @param dev
 673  *   Pointer to Ethernet device structure.
 674  * @param tx_queue_id
 675  *   Transmit queue index.
 676  * @param qinfo
 677  *   Transmit queue information structure.
 678  */
 679 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
 680                               struct rte_eth_txq_info *qinfo)
 681 {
 682         struct mrvl_priv *priv = dev->data->dev_private;
 683
 684         qinfo->nb_desc =
 685                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
 686 }
 687
 688 /**
 689  * Release buffers to hardware bpool (buffer-pool)
 690  *
 691  * @param rxq
 692  *   Receive queue pointer.
 693  * @param num
 694  *   Number of buffers to release to bpool.
 695  *
 696  * @return
 697  *   0 on success, negative error value otherwise.
 698  */
 699 static int
 700 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
 701 {
 702         struct buff_release_entry entries[MRVL_PP2_TXD_MAX];
 703         struct rte_mbuf *mbufs[MRVL_PP2_TXD_MAX];
 704         int i, ret;
 705         unsigned int core_id = rte_lcore_id();
 706         struct pp2_hif *hif = hifs[core_id];
 707         struct pp2_bpool *bpool = rxq->priv->bpool;
 708
 709         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
 710         if (ret)
 711                 return ret;
 712
 713         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
 714                 cookie_addr_high =
 715                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
 716
 717         for (i = 0; i < num; i++) {
 718                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
 719                         != cookie_addr_high) {
 720                         RTE_LOG(ERR, PMD,
 721                                 "mbuf virtual addr high 0x%lx out of range\n",
 722                                 (uint64_t)mbufs[i] >> 32);
 723                         goto out;
 724                 }
 725
 726                 entries[i].buff.addr =
 727                         rte_mbuf_data_dma_addr_default(mbufs[i]);
 728                 entries[i].buff.cookie = (pp2_cookie_t)(uint64_t)mbufs[i];
 729                 entries[i].bpool = bpool;
 730         }
 731
 732         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
 733         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
 734
 735         if (i != num)
 736                 goto out;
 737
 738         return 0;
 739 out:
 740         for (; i < num; i++)
 741                 rte_pktmbuf_free(mbufs[i]);
 742
 743         return -1;
 744 }
 745
 746 /**
 747  * DPDK callback to configure the receive queue.
 748  *
 749  * @param dev
 750  *   Pointer to Ethernet device structure.
 751  * @param idx
 752  *   RX queue index.
 753  * @param desc
 754  *   Number of descriptors to configure in queue.
 755  * @param socket
 756  *   NUMA socket on which memory must be allocated.
 757  * @param conf
 758  *   Thresholds parameters (unused_).
 759  * @param mp
 760  *   Memory pool for buffer allocations.
 761  *
 762  * @return
 763  *   0 on success, negative error value otherwise.
 764  */
 765 static int
 766 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 767                     unsigned int socket,
 768                     const struct rte_eth_rxconf *conf __rte_unused,
 769                     struct rte_mempool *mp)
 770 {
 771         struct mrvl_priv *priv = dev->data->dev_private;
 772         struct mrvl_rxq *rxq;
 773         uint32_t min_size,
 774                  max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
 775         int ret, tc, inq;
 776
 777         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
 778                 /*
 779                  * Unknown TC mapping, mapping will not have a correct queue.
 780                  */
 781                 RTE_LOG(ERR, PMD, "Unknown TC mapping for queue %hu eth%hhu\n",
 782                         idx, priv->ppio_id);
 783                 return -EFAULT;
 784         }
 785
 786         min_size = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM -
 787                    MRVL_PKT_EFFEC_OFFS;
 788         if (min_size < max_rx_pkt_len) {
 789                 RTE_LOG(ERR, PMD,
 790                         "Mbuf size must be increased to %u bytes to hold up to %u bytes of data.\n",
 791                         max_rx_pkt_len + RTE_PKTMBUF_HEADROOM +
 792                         MRVL_PKT_EFFEC_OFFS,
 793                         max_rx_pkt_len);
 794                 return -EINVAL;
 795         }
 796
 797         if (dev->data->rx_queues[idx]) {
 798                 rte_free(dev->data->rx_queues[idx]);
 799                 dev->data->rx_queues[idx] = NULL;
 800         }
 801
 802         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
 803         if (!rxq)
 804                 return -ENOMEM;
 805
 806         rxq->priv = priv;
 807         rxq->mp = mp;
 808         rxq->queue_id = idx;
 809         rxq->port_id = dev->data->port_id;
 810         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
 811
 812         tc = priv->rxq_map[rxq->queue_id].tc,
 813         inq = priv->rxq_map[rxq->queue_id].inq;
 814         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
 815                 desc;
 816
 817         ret = mrvl_fill_bpool(rxq, desc);
 818         if (ret) {
 819                 rte_free(rxq);
 820                 return ret;
 821         }
 822
 823         priv->bpool_init_size += desc;
 824
 825         dev->data->rx_queues[idx] = rxq;
 826
 827         return 0;
 828 }
 829
 830 /**
 831  * DPDK callback to release the receive queue.
 832  *
 833  * @param rxq
 834  *   Generic receive queue pointer.
 835  */
 836 static void
 837 mrvl_rx_queue_release(void *rxq)
 838 {
 839         struct mrvl_rxq *q = rxq;
 840         struct pp2_ppio_tc_params *tc_params;
 841         int i, num, tc, inq;
 842
 843         if (!q)
 844                 return;
 845
 846         tc = q->priv->rxq_map[q->queue_id].tc;
 847         inq = q->priv->rxq_map[q->queue_id].inq;
 848         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
 849         num = tc_params->inqs_params[inq].size;
 850         for (i = 0; i < num; i++) {
 851                 struct pp2_buff_inf inf;
 852                 uint64_t addr;
 853
 854                 pp2_bpool_get_buff(hifs[rte_lcore_id()], q->priv->bpool, &inf);
 855                 addr = cookie_addr_high | inf.cookie;
 856                 rte_pktmbuf_free((struct rte_mbuf *)addr);
 857         }
 858
 859         rte_free(q);
 860 }
 861
 862 /**
 863  * DPDK callback to configure the transmit queue.
 864  *
 865  * @param dev
 866  *   Pointer to Ethernet device structure.
 867  * @param idx
 868  *   Transmit queue index.
 869  * @param desc
 870  *   Number of descriptors to configure in the queue.
 871  * @param socket
 872  *   NUMA socket on which memory must be allocated.
 873  * @param conf
 874  *   Thresholds parameters (unused).
 875  *
 876  * @return
 877  *   0 on success, negative error value otherwise.
 878  */
 879 static int
 880 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
 881                     unsigned int socket,
 882                     const struct rte_eth_txconf *conf __rte_unused)
 883 {
 884         struct mrvl_priv *priv = dev->data->dev_private;
 885         struct mrvl_txq *txq;
 886
 887         if (dev->data->tx_queues[idx]) {
 888                 rte_free(dev->data->tx_queues[idx]);
 889                 dev->data->tx_queues[idx] = NULL;
 890         }
 891
 892         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
 893         if (!txq)
 894                 return -ENOMEM;
 895
 896         txq->priv = priv;
 897         txq->queue_id = idx;
 898         txq->port_id = dev->data->port_id;
 899         dev->data->tx_queues[idx] = txq;
 900
 901         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
 902         priv->ppio_params.outqs_params.outqs_params[idx].weight = 1;
 903
 904         return 0;
 905 }
 906
 907 /**
 908  * DPDK callback to release the transmit queue.
 909  *
 910  * @param txq
 911  *   Generic transmit queue pointer.
 912  */
 913 static void
 914 mrvl_tx_queue_release(void *txq)
 915 {
 916         struct mrvl_txq *q = txq;
 917
 918         if (!q)
 919                 return;
 920
 921         rte_free(q);
 922 }
 923
 924 static const struct eth_dev_ops mrvl_ops = {
 925         .dev_configure = mrvl_dev_configure,
 926         .dev_start = mrvl_dev_start,
 927         .dev_stop = mrvl_dev_stop,
 928         .dev_set_link_up = mrvl_dev_set_link_up,
 929         .dev_set_link_down = mrvl_dev_set_link_down,
 930         .dev_close = mrvl_dev_close,
 931         .link_update = mrvl_link_update,
 932         .mac_addr_set = mrvl_mac_addr_set,
 933         .mtu_set = mrvl_mtu_set,
 934         .dev_infos_get = mrvl_dev_infos_get,
 935         .rxq_info_get = mrvl_rxq_info_get,
 936         .txq_info_get = mrvl_txq_info_get,
 937         .rx_queue_setup = mrvl_rx_queue_setup,
 938         .rx_queue_release = mrvl_rx_queue_release,
 939         .tx_queue_setup = mrvl_tx_queue_setup,
 940         .tx_queue_release = mrvl_tx_queue_release,
 941 };
 942
 943 /**
 944  * DPDK callback for receive.
 945  *
 946  * @param rxq
 947  *   Generic pointer to the receive queue.
 948  * @param rx_pkts
 949  *   Array to store received packets.
 950  * @param nb_pkts
 951  *   Maximum number of packets in array.
 952  *
 953  * @return
 954  *   Number of packets successfully received.
 955  */
 956 static uint16_t
 957 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 958 {
 959         struct mrvl_rxq *q = rxq;
 960         struct pp2_ppio_desc descs[nb_pkts];
 961         struct pp2_bpool *bpool;
 962         int i, ret, rx_done = 0;
 963         int num;
 964         unsigned int core_id = rte_lcore_id();
 965
 966         if (unlikely(!q->priv->ppio))
 967                 return 0;
 968
 969         bpool = q->priv->bpool;
 970
 971         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
 972                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
 973         if (unlikely(ret < 0)) {
 974                 RTE_LOG(ERR, PMD, "Failed to receive packets\n");
 975                 return 0;
 976         }
 977         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
 978
 979         for (i = 0; i < nb_pkts; i++) {
 980                 struct rte_mbuf *mbuf;
 981                 enum pp2_inq_desc_status status;
 982                 uint64_t addr;
 983
 984                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
 985                         struct pp2_ppio_desc *pref_desc;
 986                         u64 pref_addr;
 987
 988                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
 989                         pref_addr = cookie_addr_high |
 990                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
 991                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
 992                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
 993                 }
 994
 995                 addr = cookie_addr_high |
 996                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
 997                 mbuf = (struct rte_mbuf *)addr;
 998                 rte_pktmbuf_reset(mbuf);
 999
1000                 /* drop packet in case of mac, overrun or resource error */
1001                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
1002                 if (unlikely(status != PP2_DESC_ERR_OK)) {
1003                         struct pp2_buff_inf binf = {
1004                                 .addr = rte_mbuf_data_dma_addr_default(mbuf),
1005                                 .cookie = (pp2_cookie_t)(uint64_t)mbuf,
1006                         };
1007
1008                         pp2_bpool_put_buff(hifs[core_id], bpool, &binf);
1009                         mrvl_port_bpool_size
1010                                 [bpool->pp2_id][bpool->id][core_id]++;
1011                         continue;
1012                 }
1013
1014                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
1015                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
1016                 mbuf->data_len = mbuf->pkt_len;
1017                 mbuf->port = q->port_id;
1018
1019                 rx_pkts[rx_done++] = mbuf;
1020         }
1021
1022         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
1023                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
1024
1025                 if (unlikely(num <= q->priv->bpool_min_size ||
1026                              (!rx_done && num < q->priv->bpool_init_size))) {
1027                         ret = mrvl_fill_bpool(q, MRVL_BURST_SIZE);
1028                         if (ret)
1029                                 RTE_LOG(ERR, PMD, "Failed to fill bpool\n");
1030                 } else if (unlikely(num > q->priv->bpool_max_size)) {
1031                         int i;
1032                         int pkt_to_remove = num - q->priv->bpool_init_size;
1033                         struct rte_mbuf *mbuf;
1034                         struct pp2_buff_inf buff;
1035
1036                         RTE_LOG(DEBUG, PMD,
1037                                 "\nport-%d:%d: bpool %d oversize - remove %d buffers (pool size: %d -> %d)\n",
1038                                 bpool->pp2_id, q->priv->ppio->port_id,
1039                                 bpool->id, pkt_to_remove, num,
1040                                 q->priv->bpool_init_size);
1041
1042                         for (i = 0; i < pkt_to_remove; i++) {
1043                                 pp2_bpool_get_buff(hifs[core_id], bpool, &buff);
1044                                 mbuf = (struct rte_mbuf *)
1045                                         (cookie_addr_high | buff.cookie);
1046                                 rte_pktmbuf_free(mbuf);
1047                         }
1048                         mrvl_port_bpool_size
1049                                 [bpool->pp2_id][bpool->id][core_id] -=
1050                                                                 pkt_to_remove;
1051                 }
1052                 rte_spinlock_unlock(&q->priv->lock);
1053         }
1054
1055         return rx_done;
1056 }
1057
1058 /**
1059  * Release already sent buffers to bpool (buffer-pool).
1060  *
1061  * @param ppio
1062  *   Pointer to the port structure.
1063  * @param hif
1064  *   Pointer to the MUSDK hardware interface.
1065  * @param sq
1066  *   Pointer to the shadow queue.
1067  * @param qid
1068  *   Queue id number.
1069  * @param force
1070  *   Force releasing packets.
1071  */
1072 static inline void
1073 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
1074                        struct mrvl_shadow_txq *sq, int qid, int force)
1075 {
1076         struct buff_release_entry *entry;
1077         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
1078         int i, core_id = rte_lcore_id();
1079
1080         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
1081
1082         sq->num_to_release += nb_done;
1083
1084         if (likely(!force &&
1085                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
1086                 return;
1087
1088         nb_done = sq->num_to_release;
1089         sq->num_to_release = 0;
1090
1091         for (i = 0; i < nb_done; i++) {
1092                 entry = &sq->ent[sq->tail + num];
1093                 if (unlikely(!entry->buff.addr)) {
1094                         RTE_LOG(ERR, PMD,
1095                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!\n",
1096                                 sq->tail, (u64)entry->buff.cookie,
1097                                 (u64)entry->buff.addr);
1098                         skip_bufs = 1;
1099                         goto skip;
1100                 }
1101
1102                 if (unlikely(!entry->bpool)) {
1103                         struct rte_mbuf *mbuf;
1104
1105                         mbuf = (struct rte_mbuf *)
1106                                (cookie_addr_high | entry->buff.cookie);
1107                         rte_pktmbuf_free(mbuf);
1108                         skip_bufs = 1;
1109                         goto skip;
1110                 }
1111
1112                 mrvl_port_bpool_size
1113                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
1114                 num++;
1115                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
1116                         goto skip;
1117                 continue;
1118 skip:
1119                 if (likely(num))
1120                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
1121                 num += skip_bufs;
1122                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
1123                 sq->size -= num;
1124                 num = 0;
1125         }
1126
1127         if (likely(num)) {
1128                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
1129                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
1130                 sq->size -= num;
1131         }
1132 }
1133
1134 /**
1135  * DPDK callback for transmit.
1136  *
1137  * @param txq
1138  *   Generic pointer transmit queue.
1139  * @param tx_pkts
1140  *   Packets to transmit.
1141  * @param nb_pkts
1142  *   Number of packets in array.
1143  *
1144  * @return
1145  *   Number of packets successfully transmitted.
1146  */
1147 static uint16_t
1148 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
1149 {
1150         struct mrvl_txq *q = txq;
1151         struct mrvl_shadow_txq *sq = &shadow_txqs[q->port_id][rte_lcore_id()];
1152         struct pp2_hif *hif = hifs[rte_lcore_id()];
1153         struct pp2_ppio_desc descs[nb_pkts];
1154         int i;
1155         uint16_t num, sq_free_size;
1156
1157         if (unlikely(!q->priv->ppio))
1158                 return 0;
1159
1160         if (sq->size)
1161                 mrvl_free_sent_buffers(q->priv->ppio, hif, sq, q->queue_id, 0);
1162
1163         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
1164         if (unlikely(nb_pkts > sq_free_size)) {
1165                 RTE_LOG(DEBUG, PMD,
1166                         "No room in shadow queue for %d packets! %d packets will be sent.\n",
1167                         nb_pkts, sq_free_size);
1168                 nb_pkts = sq_free_size;
1169         }
1170
1171         for (i = 0; i < nb_pkts; i++) {
1172                 struct rte_mbuf *mbuf = tx_pkts[i];
1173
1174                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
1175                         struct rte_mbuf *pref_pkt_hdr;
1176
1177                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
1178                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
1179                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
1180                 }
1181
1182                 sq->ent[sq->head].buff.cookie = (pp2_cookie_t)(uint64_t)mbuf;
1183                 sq->ent[sq->head].buff.addr =
1184                         rte_mbuf_data_dma_addr_default(mbuf);
1185                 sq->ent[sq->head].bpool =
1186                         (unlikely(mbuf->port == 0xff || mbuf->refcnt > 1)) ?
1187                          NULL : mrvl_port_to_bpool_lookup[mbuf->port];
1188                 sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
1189                 sq->size++;
1190
1191                 pp2_ppio_outq_desc_reset(&descs[i]);
1192                 pp2_ppio_outq_desc_set_phys_addr(&descs[i],
1193                                                  rte_pktmbuf_mtophys(mbuf));
1194                 pp2_ppio_outq_desc_set_pkt_offset(&descs[i], 0);
1195                 pp2_ppio_outq_desc_set_pkt_len(&descs[i],
1196                                                rte_pktmbuf_pkt_len(mbuf));
1197         }
1198
1199         num = nb_pkts;
1200         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
1201         /* number of packets that were not sent */
1202         if (unlikely(num > nb_pkts)) {
1203                 for (i = nb_pkts; i < num; i++) {
1204                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
1205                                 MRVL_PP2_TX_SHADOWQ_MASK;
1206                 }
1207                 sq->size -= num - nb_pkts;
1208         }
1209
1210         return nb_pkts;
1211 }
1212
1213 /**
1214  * Initialize packet processor.
1215  *
1216  * @return
1217  *   0 on success, negative error value otherwise.
1218  */
1219 static int
1220 mrvl_init_pp2(void)
1221 {
1222         struct pp2_init_params init_params;
1223
1224         memset(&init_params, 0, sizeof(init_params));
1225         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
1226         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
1227
1228         return pp2_init(&init_params);
1229 }
1230
1231 /**
1232  * Deinitialize packet processor.
1233  *
1234  * @return
1235  *   0 on success, negative error value otherwise.
1236  */
1237 static void
1238 mrvl_deinit_pp2(void)
1239 {
1240         pp2_deinit();
1241 }
1242
1243 /**
1244  * Create private device structure.
1245  *
1246  * @param dev_name
1247  *   Pointer to the port name passed in the initialization parameters.
1248  *
1249  * @return
1250  *   Pointer to the newly allocated private device structure.
1251  */
1252 static struct mrvl_priv *
1253 mrvl_priv_create(const char *dev_name)
1254 {
1255         struct pp2_bpool_params bpool_params;
1256         char match[MRVL_MATCH_LEN];
1257         struct mrvl_priv *priv;
1258         int ret, bpool_bit;
1259
1260         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
1261         if (!priv)
1262                 return NULL;
1263
1264         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
1265                                        &priv->pp_id, &priv->ppio_id);
1266         if (ret)
1267                 goto out_free_priv;
1268
1269         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
1270                                      PP2_BPOOL_NUM_POOLS);
1271         if (bpool_bit < 0)
1272                 goto out_free_priv;
1273         priv->bpool_bit = bpool_bit;
1274
1275         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
1276                  priv->bpool_bit);
1277         memset(&bpool_params, 0, sizeof(bpool_params));
1278         bpool_params.match = match;
1279         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
1280         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
1281         if (ret)
1282                 goto out_clear_bpool_bit;
1283
1284         priv->ppio_params.type = PP2_PPIO_T_NIC;
1285         rte_spinlock_init(&priv->lock);
1286
1287         return priv;
1288 out_clear_bpool_bit:
1289         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
1290 out_free_priv:
1291         rte_free(priv);
1292         return NULL;
1293 }
1294
1295 /**
1296  * Create device representing Ethernet port.
1297  *
1298  * @param name
1299  *   Pointer to the port's name.
1300  *
1301  * @return
1302  *   0 on success, negative error value otherwise.
1303  */
1304 static int
1305 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
1306 {
1307         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
1308         struct rte_eth_dev *eth_dev;
1309         struct mrvl_priv *priv;
1310         struct ifreq req;
1311
1312         eth_dev = rte_eth_dev_allocate(name);
1313         if (!eth_dev)
1314                 return -ENOMEM;
1315
1316         priv = mrvl_priv_create(name);
1317         if (!priv) {
1318                 ret = -ENOMEM;
1319                 goto out_free_dev;
1320         }
1321
1322         eth_dev->data->mac_addrs =
1323                 rte_zmalloc("mac_addrs",
1324                             ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
1325         if (!eth_dev->data->mac_addrs) {
1326                 RTE_LOG(ERR, PMD, "Failed to allocate space for eth addrs\n");
1327                 ret = -ENOMEM;
1328                 goto out_free_priv;
1329         }
1330
1331         memset(&req, 0, sizeof(req));
1332         strcpy(req.ifr_name, name);
1333         ret = ioctl(fd, SIOCGIFHWADDR, &req);
1334         if (ret)
1335                 goto out_free_mac;
1336
1337         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
1338                req.ifr_addr.sa_data, ETHER_ADDR_LEN);
1339
1340         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
1341         eth_dev->tx_pkt_burst = mrvl_tx_pkt_burst;
1342         eth_dev->data->dev_private = priv;
1343         eth_dev->device = &vdev->device;
1344         eth_dev->dev_ops = &mrvl_ops;
1345
1346         return 0;
1347 out_free_mac:
1348         rte_free(eth_dev->data->mac_addrs);
1349 out_free_dev:
1350         rte_eth_dev_release_port(eth_dev);
1351 out_free_priv:
1352         rte_free(priv);
1353
1354         return ret;
1355 }
1356
1357 /**
1358  * Cleanup previously created device representing Ethernet port.
1359  *
1360  * @param name
1361  *   Pointer to the port name.
1362  */
1363 static void
1364 mrvl_eth_dev_destroy(const char *name)
1365 {
1366         struct rte_eth_dev *eth_dev;
1367         struct mrvl_priv *priv;
1368
1369         eth_dev = rte_eth_dev_allocated(name);
1370         if (!eth_dev)
1371                 return;
1372
1373         priv = eth_dev->data->dev_private;
1374         pp2_bpool_deinit(priv->bpool);
1375         rte_free(priv);
1376         rte_free(eth_dev->data->mac_addrs);
1377         rte_eth_dev_release_port(eth_dev);
1378 }
1379
1380 /**
1381  * Callback used by rte_kvargs_process() during argument parsing.
1382  *
1383  * @param key
1384  *   Pointer to the parsed key (unused).
1385  * @param value
1386  *   Pointer to the parsed value.
1387  * @param extra_args
1388  *   Pointer to the extra arguments which contains address of the
1389  *   table of pointers to parsed interface names.
1390  *
1391  * @return
1392  *   Always 0.
1393  */
1394 static int
1395 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
1396                  void *extra_args)
1397 {
1398         const char **ifnames = extra_args;
1399
1400         ifnames[mrvl_ports_nb++] = value;
1401
1402         return 0;
1403 }
1404
1405 /**
1406  * Initialize per-lcore MUSDK hardware interfaces (hifs).
1407  *
1408  * @return
1409  *   0 on success, negative error value otherwise.
1410  */
1411 static int
1412 mrvl_init_hifs(void)
1413 {
1414         struct pp2_hif_params params;
1415         char match[MRVL_MATCH_LEN];
1416         int i, ret;
1417
1418         RTE_LCORE_FOREACH(i) {
1419                 ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
1420                 if (ret < 0)
1421                         return ret;
1422
1423                 snprintf(match, sizeof(match), "hif-%d", ret);
1424                 memset(&params, 0, sizeof(params));
1425                 params.match = match;
1426                 params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
1427                 ret = pp2_hif_init(&params, &hifs[i]);
1428                 if (ret) {
1429                         RTE_LOG(ERR, PMD, "Failed to initialize hif %d\n", i);
1430                         return ret;
1431                 }
1432         }
1433
1434         return 0;
1435 }
1436
1437 /**
1438  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
1439  */
1440 static void
1441 mrvl_deinit_hifs(void)
1442 {
1443         int i;
1444
1445         RTE_LCORE_FOREACH(i) {
1446                 if (hifs[i])
1447                         pp2_hif_deinit(hifs[i]);
1448         }
1449 }
1450
1451 static void mrvl_set_first_last_cores(int core_id)
1452 {
1453         if (core_id < mrvl_lcore_first)
1454                 mrvl_lcore_first = core_id;
1455
1456         if (core_id > mrvl_lcore_last)
1457                 mrvl_lcore_last = core_id;
1458 }
1459
1460 /**
1461  * DPDK callback to register the virtual device.
1462  *
1463  * @param vdev
1464  *   Pointer to the virtual device.
1465  *
1466  * @return
1467  *   0 on success, negative error value otherwise.
1468  */
1469 static int
1470 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
1471 {
1472         struct rte_kvargs *kvlist;
1473         const char *ifnames[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
1474         int ret = -EINVAL;
1475         uint32_t i, ifnum, cfgnum, core_id;
1476         const char *params;
1477
1478         params = rte_vdev_device_args(vdev);
1479         if (!params)
1480                 return -EINVAL;
1481
1482         kvlist = rte_kvargs_parse(params, valid_args);
1483         if (!kvlist)
1484                 return -EINVAL;
1485
1486         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
1487         if (ifnum > RTE_DIM(ifnames))
1488                 goto out_free_kvlist;
1489
1490         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
1491                            mrvl_get_ifnames, &ifnames);
1492
1493         cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
1494         if (cfgnum > 1) {
1495                 RTE_LOG(ERR, PMD, "Cannot handle more than one config file!\n");
1496                 goto out_free_kvlist;
1497         } else if (cfgnum == 1) {
1498                 rte_kvargs_process(kvlist, MRVL_CFG_ARG,
1499                                    mrvl_get_qoscfg, &mrvl_qos_cfg);
1500         }
1501
1502         /*
1503          * ret == -EEXIST is correct, it means DMA
1504          * has been already initialized (by another PMD).
1505          */
1506         ret = mv_sys_dma_mem_init(RTE_MRVL_MUSDK_DMA_MEMSIZE);
1507         if (ret < 0 && ret != -EEXIST)
1508                 goto out_free_kvlist;
1509
1510         ret = mrvl_init_pp2();
1511         if (ret) {
1512                 RTE_LOG(ERR, PMD, "Failed to init PP!\n");
1513                 goto out_deinit_dma;
1514         }
1515
1516         ret = mrvl_init_hifs();
1517         if (ret)
1518                 goto out_deinit_hifs;
1519
1520         for (i = 0; i < ifnum; i++) {
1521                 RTE_LOG(INFO, PMD, "Creating %s\n", ifnames[i]);
1522                 ret = mrvl_eth_dev_create(vdev, ifnames[i]);
1523                 if (ret)
1524                         goto out_cleanup;
1525         }
1526
1527         rte_kvargs_free(kvlist);
1528
1529         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
1530
1531         mrvl_lcore_first = RTE_MAX_LCORE;
1532         mrvl_lcore_last = 0;
1533
1534         RTE_LCORE_FOREACH(core_id) {
1535                 mrvl_set_first_last_cores(core_id);
1536         }
1537
1538         return 0;
1539 out_cleanup:
1540         for (; i > 0; i--)
1541                 mrvl_eth_dev_destroy(ifnames[i]);
1542 out_deinit_hifs:
1543         mrvl_deinit_hifs();
1544         mrvl_deinit_pp2();
1545 out_deinit_dma:
1546         mv_sys_dma_mem_destroy();
1547 out_free_kvlist:
1548         rte_kvargs_free(kvlist);
1549
1550         return ret;
1551 }
1552
1553 /**
1554  * DPDK callback to remove virtual device.
1555  *
1556  * @param vdev
1557  *   Pointer to the removed virtual device.
1558  *
1559  * @return
1560  *   0 on success, negative error value otherwise.
1561  */
1562 static int
1563 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
1564 {
1565         int i;
1566         const char *name;
1567
1568         name = rte_vdev_device_name(vdev);
1569         if (!name)
1570                 return -EINVAL;
1571
1572         RTE_LOG(INFO, PMD, "Removing %s\n", name);
1573
1574         for (i = 0; i < rte_eth_dev_count(); i++) {
1575                 char ifname[RTE_ETH_NAME_MAX_LEN];
1576
1577                 rte_eth_dev_get_name_by_port(i, ifname);
1578                 mrvl_eth_dev_destroy(ifname);
1579         }
1580
1581         mrvl_deinit_hifs();
1582         mrvl_deinit_pp2();
1583         mv_sys_dma_mem_destroy();
1584
1585         return 0;
1586 }
1587
1588 static struct rte_vdev_driver pmd_mrvl_drv = {
1589         .probe = rte_pmd_mrvl_probe,
1590         .remove = rte_pmd_mrvl_remove,
1591 };
1592
1593 RTE_PMD_REGISTER_VDEV(net_mrvl, pmd_mrvl_drv);
1594 RTE_PMD_REGISTER_ALIAS(net_mrvl, eth_mrvl);