drivers/net/mvpp2/mrvl_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017-2021 Marvell International Ltd.
   3  * Copyright(c) 2017-2021 Semihalf.
   4  * All rights reserved.
   5  */
   6
   7 #include <rte_string_fns.h>
   8 #include <ethdev_driver.h>
   9 #include <rte_kvargs.h>
  10 #include <rte_log.h>
  11 #include <rte_malloc.h>
  12 #include <rte_bus_vdev.h>
  13
  14 #include <fcntl.h>
  15 #include <linux/ethtool.h>
  16 #include <linux/sockios.h>
  17 #include <net/if.h>
  18 #include <net/if_arp.h>
  19 #include <sys/ioctl.h>
  20 #include <sys/socket.h>
  21 #include <sys/stat.h>
  22 #include <sys/types.h>
  23
  24 #include <rte_mvep_common.h>
  25 #include "mrvl_ethdev.h"
  26 #include "mrvl_qos.h"
  27 #include "mrvl_flow.h"
  28 #include "mrvl_mtr.h"
  29 #include "mrvl_tm.h"
  30
  31 /* bitmask with reserved hifs */
  32 #define MRVL_MUSDK_HIFS_RESERVED 0x0F
  33 /* bitmask with reserved bpools */
  34 #define MRVL_MUSDK_BPOOLS_RESERVED 0x07
  35 /* bitmask with reserved kernel RSS tables */
  36 #define MRVL_MUSDK_RSS_RESERVED 0x0F
  37 /* maximum number of available hifs */
  38 #define MRVL_MUSDK_HIFS_MAX 9
  39
  40 /* prefetch shift */
  41 #define MRVL_MUSDK_PREFETCH_SHIFT 2
  42
  43 /* TCAM has 25 entries reserved for uc/mc filter entries
  44  * + 1 for primary mac address
  45  */
  46 #define MRVL_MAC_ADDRS_MAX (1 + 25)
  47 #define MRVL_MATCH_LEN 16
  48 #define MRVL_PKT_EFFEC_OFFS (MRVL_PKT_OFFS + MV_MH_SIZE)
  49 /* Maximum allowable packet size */
  50 #define MRVL_PKT_SIZE_MAX (10240 - MV_MH_SIZE)
  51
  52 #define MRVL_IFACE_NAME_ARG "iface"
  53 #define MRVL_CFG_ARG "cfg"
  54
  55 #define MRVL_BURST_SIZE 64
  56
  57 #define MRVL_ARP_LENGTH 28
  58
  59 #define MRVL_COOKIE_ADDR_INVALID ~0ULL
  60 #define MRVL_COOKIE_HIGH_ADDR_MASK 0xffffff0000000000
  61
  62 /** Port Rx offload capabilities */
  63 #define MRVL_RX_OFFLOADS (DEV_RX_OFFLOAD_VLAN_FILTER | \
  64                           DEV_RX_OFFLOAD_JUMBO_FRAME | \
  65                           DEV_RX_OFFLOAD_CHECKSUM)
  66
  67 /** Port Tx offloads capabilities */
  68 #define MRVL_TX_OFFLOAD_CHECKSUM (DEV_TX_OFFLOAD_IPV4_CKSUM | \
  69                                   DEV_TX_OFFLOAD_UDP_CKSUM  | \
  70                                   DEV_TX_OFFLOAD_TCP_CKSUM)
  71 #define MRVL_TX_OFFLOADS (MRVL_TX_OFFLOAD_CHECKSUM | \
  72                           DEV_TX_OFFLOAD_MULTI_SEGS)
  73
  74 #define MRVL_TX_PKT_OFFLOADS (PKT_TX_IP_CKSUM | \
  75                               PKT_TX_TCP_CKSUM | \
  76                               PKT_TX_UDP_CKSUM)
  77
  78 static const char * const valid_args[] = {
  79         MRVL_IFACE_NAME_ARG,
  80         MRVL_CFG_ARG,
  81         NULL
  82 };
  83
  84 static int used_hifs = MRVL_MUSDK_HIFS_RESERVED;
  85 static struct pp2_hif *hifs[RTE_MAX_LCORE];
  86 static int used_bpools[PP2_NUM_PKT_PROC] = {
  87         [0 ... PP2_NUM_PKT_PROC - 1] = MRVL_MUSDK_BPOOLS_RESERVED
  88 };
  89
  90 static struct pp2_bpool *mrvl_port_to_bpool_lookup[RTE_MAX_ETHPORTS];
  91 static int mrvl_port_bpool_size[PP2_NUM_PKT_PROC][PP2_BPOOL_NUM_POOLS][RTE_MAX_LCORE];
  92 static uint64_t cookie_addr_high = MRVL_COOKIE_ADDR_INVALID;
  93
  94 struct mrvl_ifnames {
  95         const char *names[PP2_NUM_ETH_PPIO * PP2_NUM_PKT_PROC];
  96         int idx;
  97 };
  98
  99 /*
 100  * To use buffer harvesting based on loopback port shadow queue structure
 101  * was introduced for buffers information bookkeeping.
 102  *
 103  * Before sending the packet, related buffer information (pp2_buff_inf) is
 104  * stored in shadow queue. After packet is transmitted no longer used
 105  * packet buffer is released back to it's original hardware pool,
 106  * on condition it originated from interface.
 107  * In case it  was generated by application itself i.e: mbuf->port field is
 108  * 0xff then its released to software mempool.
 109  */
 110 struct mrvl_shadow_txq {
 111         int head;           /* write index - used when sending buffers */
 112         int tail;           /* read index - used when releasing buffers */
 113         u16 size;           /* queue occupied size */
 114         u16 num_to_release; /* number of descriptors sent, that can be
 115                              * released
 116                              */
 117         struct buff_release_entry ent[MRVL_PP2_TX_SHADOWQ_SIZE]; /* q entries */
 118 };
 119
 120 struct mrvl_rxq {
 121         struct mrvl_priv *priv;
 122         struct rte_mempool *mp;
 123         int queue_id;
 124         int port_id;
 125         int cksum_enabled;
 126         uint64_t bytes_recv;
 127         uint64_t drop_mac;
 128 };
 129
 130 struct mrvl_txq {
 131         struct mrvl_priv *priv;
 132         int queue_id;
 133         int port_id;
 134         uint64_t bytes_sent;
 135         struct mrvl_shadow_txq shadow_txqs[RTE_MAX_LCORE];
 136         int tx_deferred_start;
 137 };
 138
 139 static int mrvl_lcore_first;
 140 static int mrvl_lcore_last;
 141 static int mrvl_dev_num;
 142
 143 static int mrvl_fill_bpool(struct mrvl_rxq *rxq, int num);
 144 static inline void mrvl_free_sent_buffers(struct pp2_ppio *ppio,
 145                         struct pp2_hif *hif, unsigned int core_id,
 146                         struct mrvl_shadow_txq *sq, int qid, int force);
 147
 148 static uint16_t mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts,
 149                                   uint16_t nb_pkts);
 150 static uint16_t mrvl_tx_sg_pkt_burst(void *txq, struct rte_mbuf **tx_pkts,
 151                                      uint16_t nb_pkts);
 152 static int rte_pmd_mrvl_remove(struct rte_vdev_device *vdev);
 153 static void mrvl_deinit_pp2(void);
 154 static void mrvl_deinit_hifs(void);
 155
 156 static int
 157 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
 158                   uint32_t index, uint32_t vmdq __rte_unused);
 159 static int
 160 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr);
 161 static int
 162 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on);
 163 static int mrvl_promiscuous_enable(struct rte_eth_dev *dev);
 164 static int mrvl_allmulticast_enable(struct rte_eth_dev *dev);
 165
 166 #define MRVL_XSTATS_TBL_ENTRY(name) { \
 167         #name, offsetof(struct pp2_ppio_statistics, name),      \
 168         sizeof(((struct pp2_ppio_statistics *)0)->name)         \
 169 }
 170
 171 /* Table with xstats data */
 172 static struct {
 173         const char *name;
 174         unsigned int offset;
 175         unsigned int size;
 176 } mrvl_xstats_tbl[] = {
 177         MRVL_XSTATS_TBL_ENTRY(rx_bytes),
 178         MRVL_XSTATS_TBL_ENTRY(rx_packets),
 179         MRVL_XSTATS_TBL_ENTRY(rx_unicast_packets),
 180         MRVL_XSTATS_TBL_ENTRY(rx_errors),
 181         MRVL_XSTATS_TBL_ENTRY(rx_fullq_dropped),
 182         MRVL_XSTATS_TBL_ENTRY(rx_bm_dropped),
 183         MRVL_XSTATS_TBL_ENTRY(rx_early_dropped),
 184         MRVL_XSTATS_TBL_ENTRY(rx_fifo_dropped),
 185         MRVL_XSTATS_TBL_ENTRY(rx_cls_dropped),
 186         MRVL_XSTATS_TBL_ENTRY(tx_bytes),
 187         MRVL_XSTATS_TBL_ENTRY(tx_packets),
 188         MRVL_XSTATS_TBL_ENTRY(tx_unicast_packets),
 189         MRVL_XSTATS_TBL_ENTRY(tx_errors)
 190 };
 191
 192 /**
 193  * Initialize packet processor.
 194  *
 195  * @return
 196  *   0 on success, negative error value otherwise.
 197  */
 198 static int
 199 mrvl_init_pp2(void)
 200 {
 201         struct pp2_init_params init_params;
 202
 203         memset(&init_params, 0, sizeof(init_params));
 204         init_params.hif_reserved_map = MRVL_MUSDK_HIFS_RESERVED;
 205         init_params.bm_pool_reserved_map = MRVL_MUSDK_BPOOLS_RESERVED;
 206         init_params.rss_tbl_reserved_map = MRVL_MUSDK_RSS_RESERVED;
 207         if (mrvl_cfg && mrvl_cfg->pp2_cfg.prs_udfs.num_udfs)
 208                 memcpy(&init_params.prs_udfs, &mrvl_cfg->pp2_cfg.prs_udfs,
 209                        sizeof(struct pp2_parse_udfs));
 210         return pp2_init(&init_params);
 211 }
 212
 213 /**
 214  * Deinitialize packet processor.
 215  *
 216  * @return
 217  *   0 on success, negative error value otherwise.
 218  */
 219 static void
 220 mrvl_deinit_pp2(void)
 221 {
 222         pp2_deinit();
 223 }
 224
 225 static inline void
 226 mrvl_fill_shadowq(struct mrvl_shadow_txq *sq, struct rte_mbuf *buf)
 227 {
 228         sq->ent[sq->head].buff.cookie = (uint64_t)buf;
 229         sq->ent[sq->head].buff.addr = buf ?
 230                 rte_mbuf_data_iova_default(buf) : 0;
 231
 232         sq->ent[sq->head].bpool =
 233                 (unlikely(!buf || buf->port >= RTE_MAX_ETHPORTS ||
 234                  buf->refcnt > 1)) ? NULL :
 235                  mrvl_port_to_bpool_lookup[buf->port];
 236
 237         sq->head = (sq->head + 1) & MRVL_PP2_TX_SHADOWQ_MASK;
 238         sq->size++;
 239 }
 240
 241 /**
 242  * Deinitialize per-lcore MUSDK hardware interfaces (hifs).
 243  */
 244 static void
 245 mrvl_deinit_hifs(void)
 246 {
 247         int i;
 248
 249         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++) {
 250                 if (hifs[i])
 251                         pp2_hif_deinit(hifs[i]);
 252         }
 253         used_hifs = MRVL_MUSDK_HIFS_RESERVED;
 254         memset(hifs, 0, sizeof(hifs));
 255 }
 256
 257 static inline void
 258 mrvl_fill_desc(struct pp2_ppio_desc *desc, struct rte_mbuf *buf)
 259 {
 260         pp2_ppio_outq_desc_reset(desc);
 261         pp2_ppio_outq_desc_set_phys_addr(desc, rte_pktmbuf_iova(buf));
 262         pp2_ppio_outq_desc_set_pkt_offset(desc, 0);
 263         pp2_ppio_outq_desc_set_pkt_len(desc, rte_pktmbuf_data_len(buf));
 264 }
 265
 266 static inline int
 267 mrvl_get_bpool_size(int pp2_id, int pool_id)
 268 {
 269         int i;
 270         int size = 0;
 271
 272         for (i = mrvl_lcore_first; i <= mrvl_lcore_last; i++)
 273                 size += mrvl_port_bpool_size[pp2_id][pool_id][i];
 274
 275         return size;
 276 }
 277
 278 static inline int
 279 mrvl_reserve_bit(int *bitmap, int max)
 280 {
 281         int n = sizeof(*bitmap) * 8 - __builtin_clz(*bitmap);
 282
 283         if (n >= max)
 284                 return -1;
 285
 286         *bitmap |= 1 << n;
 287
 288         return n;
 289 }
 290
 291 static int
 292 mrvl_init_hif(int core_id)
 293 {
 294         struct pp2_hif_params params;
 295         char match[MRVL_MATCH_LEN];
 296         int ret;
 297
 298         ret = mrvl_reserve_bit(&used_hifs, MRVL_MUSDK_HIFS_MAX);
 299         if (ret < 0) {
 300                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 301                 return ret;
 302         }
 303
 304         snprintf(match, sizeof(match), "hif-%d", ret);
 305         memset(&params, 0, sizeof(params));
 306         params.match = match;
 307         params.out_size = MRVL_PP2_AGGR_TXQD_MAX;
 308         ret = pp2_hif_init(&params, &hifs[core_id]);
 309         if (ret) {
 310                 MRVL_LOG(ERR, "Failed to initialize hif %d", core_id);
 311                 return ret;
 312         }
 313
 314         return 0;
 315 }
 316
 317 static inline struct pp2_hif*
 318 mrvl_get_hif(struct mrvl_priv *priv, int core_id)
 319 {
 320         int ret;
 321
 322         if (likely(hifs[core_id] != NULL))
 323                 return hifs[core_id];
 324
 325         rte_spinlock_lock(&priv->lock);
 326
 327         ret = mrvl_init_hif(core_id);
 328         if (ret < 0) {
 329                 MRVL_LOG(ERR, "Failed to allocate hif %d", core_id);
 330                 goto out;
 331         }
 332
 333         if (core_id < mrvl_lcore_first)
 334                 mrvl_lcore_first = core_id;
 335
 336         if (core_id > mrvl_lcore_last)
 337                 mrvl_lcore_last = core_id;
 338 out:
 339         rte_spinlock_unlock(&priv->lock);
 340
 341         return hifs[core_id];
 342 }
 343
 344 /**
 345  * Set tx burst function according to offload flag
 346  *
 347  * @param dev
 348  *   Pointer to Ethernet device structure.
 349  */
 350 static void
 351 mrvl_set_tx_function(struct rte_eth_dev *dev)
 352 {
 353         struct mrvl_priv *priv = dev->data->dev_private;
 354
 355         /* Use a simple Tx queue (no offloads, no multi segs) if possible */
 356         if (priv->multiseg) {
 357                 RTE_LOG(INFO, PMD, "Using multi-segment tx callback\n");
 358                 dev->tx_pkt_burst = mrvl_tx_sg_pkt_burst;
 359         } else {
 360                 RTE_LOG(INFO, PMD, "Using single-segment tx callback\n");
 361                 dev->tx_pkt_burst = mrvl_tx_pkt_burst;
 362         }
 363 }
 364
 365 /**
 366  * Configure rss based on dpdk rss configuration.
 367  *
 368  * @param priv
 369  *   Pointer to private structure.
 370  * @param rss_conf
 371  *   Pointer to RSS configuration.
 372  *
 373  * @return
 374  *   0 on success, negative error value otherwise.
 375  */
 376 static int
 377 mrvl_configure_rss(struct mrvl_priv *priv, struct rte_eth_rss_conf *rss_conf)
 378 {
 379         if (rss_conf->rss_key)
 380                 MRVL_LOG(WARNING, "Changing hash key is not supported");
 381
 382         if (rss_conf->rss_hf == 0) {
 383                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 384         } else if (rss_conf->rss_hf & ETH_RSS_IPV4) {
 385                 priv->ppio_params.inqs_params.hash_type =
 386                         PP2_PPIO_HASH_T_2_TUPLE;
 387         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) {
 388                 priv->ppio_params.inqs_params.hash_type =
 389                         PP2_PPIO_HASH_T_5_TUPLE;
 390                 priv->rss_hf_tcp = 1;
 391         } else if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_UDP) {
 392                 priv->ppio_params.inqs_params.hash_type =
 393                         PP2_PPIO_HASH_T_5_TUPLE;
 394                 priv->rss_hf_tcp = 0;
 395         } else {
 396                 return -EINVAL;
 397         }
 398
 399         return 0;
 400 }
 401
 402 /**
 403  * Ethernet device configuration.
 404  *
 405  * Prepare the driver for a given number of TX and RX queues and
 406  * configure RSS.
 407  *
 408  * @param dev
 409  *   Pointer to Ethernet device structure.
 410  *
 411  * @return
 412  *   0 on success, negative error value otherwise.
 413  */
 414 static int
 415 mrvl_dev_configure(struct rte_eth_dev *dev)
 416 {
 417         struct mrvl_priv *priv = dev->data->dev_private;
 418         int ret;
 419
 420         if (priv->ppio) {
 421                 MRVL_LOG(INFO, "Device reconfiguration is not supported");
 422                 return -EINVAL;
 423         }
 424
 425         if (dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_NONE &&
 426             dev->data->dev_conf.rxmode.mq_mode != ETH_MQ_RX_RSS) {
 427                 MRVL_LOG(INFO, "Unsupported rx multi queue mode %d",
 428                         dev->data->dev_conf.rxmode.mq_mode);
 429                 return -EINVAL;
 430         }
 431
 432         if (dev->data->dev_conf.rxmode.split_hdr_size) {
 433                 MRVL_LOG(INFO, "Split headers not supported");
 434                 return -EINVAL;
 435         }
 436
 437         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_JUMBO_FRAME)
 438                 dev->data->mtu = dev->data->dev_conf.rxmode.max_rx_pkt_len -
 439                                  MRVL_PP2_ETH_HDRS_LEN;
 440
 441         if (dev->data->dev_conf.txmode.offloads & DEV_TX_OFFLOAD_MULTI_SEGS)
 442                 priv->multiseg = 1;
 443
 444         ret = mrvl_configure_rxqs(priv, dev->data->port_id,
 445                                   dev->data->nb_rx_queues);
 446         if (ret < 0)
 447                 return ret;
 448
 449         ret = mrvl_configure_txqs(priv, dev->data->port_id,
 450                                   dev->data->nb_tx_queues);
 451         if (ret < 0)
 452                 return ret;
 453
 454         priv->ppio_params.outqs_params.num_outqs = dev->data->nb_tx_queues;
 455         priv->ppio_params.maintain_stats = 1;
 456         priv->nb_rx_queues = dev->data->nb_rx_queues;
 457
 458         ret = mrvl_tm_init(dev);
 459         if (ret < 0)
 460                 return ret;
 461
 462         if (dev->data->nb_rx_queues == 1 &&
 463             dev->data->dev_conf.rxmode.mq_mode == ETH_MQ_RX_RSS) {
 464                 MRVL_LOG(WARNING, "Disabling hash for 1 rx queue");
 465                 priv->ppio_params.inqs_params.hash_type = PP2_PPIO_HASH_T_NONE;
 466
 467                 return 0;
 468         }
 469
 470         return mrvl_configure_rss(priv,
 471                         &dev->data->dev_conf.rx_adv_conf.rss_conf);
 472 }
 473
 474 /**
 475  * DPDK callback to change the MTU.
 476  *
 477  * Setting the MTU affects hardware MRU (packets larger than the MRU
 478  * will be dropped).
 479  *
 480  * @param dev
 481  *   Pointer to Ethernet device structure.
 482  * @param mtu
 483  *   New MTU.
 484  *
 485  * @return
 486  *   0 on success, negative error value otherwise.
 487  */
 488 static int
 489 mrvl_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 490 {
 491         struct mrvl_priv *priv = dev->data->dev_private;
 492         uint16_t mru;
 493         uint16_t mbuf_data_size = 0; /* SW buffer size */
 494         int ret;
 495
 496         mru = MRVL_PP2_MTU_TO_MRU(mtu);
 497         /*
 498          * min_rx_buf_size is equal to mbuf data size
 499          * if pmd didn't set it differently
 500          */
 501         mbuf_data_size = dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM;
 502         /* Prevent PMD from:
 503          * - setting mru greater than the mbuf size resulting in
 504          * hw and sw buffer size mismatch
 505          * - setting mtu that requires the support of scattered packets
 506          * when this feature has not been enabled/supported so far
 507          * (TODO check scattered_rx flag here once scattered RX is supported).
 508          */
 509         if (mru - RTE_ETHER_CRC_LEN + MRVL_PKT_OFFS > mbuf_data_size) {
 510                 mru = mbuf_data_size + RTE_ETHER_CRC_LEN - MRVL_PKT_OFFS;
 511                 mtu = MRVL_PP2_MRU_TO_MTU(mru);
 512                 MRVL_LOG(WARNING, "MTU too big, max MTU possible limitted "
 513                         "by current mbuf size: %u. Set MTU to %u, MRU to %u",
 514                         mbuf_data_size, mtu, mru);
 515         }
 516
 517         if (mtu < RTE_ETHER_MIN_MTU || mru > MRVL_PKT_SIZE_MAX) {
 518                 MRVL_LOG(ERR, "Invalid MTU [%u] or MRU [%u]", mtu, mru);
 519                 return -EINVAL;
 520         }
 521
 522         dev->data->mtu = mtu;
 523         dev->data->dev_conf.rxmode.max_rx_pkt_len = mru - MV_MH_SIZE;
 524
 525         if (!priv->ppio)
 526                 return 0;
 527
 528         ret = pp2_ppio_set_mru(priv->ppio, mru);
 529         if (ret) {
 530                 MRVL_LOG(ERR, "Failed to change MRU");
 531                 return ret;
 532         }
 533
 534         ret = pp2_ppio_set_mtu(priv->ppio, mtu);
 535         if (ret) {
 536                 MRVL_LOG(ERR, "Failed to change MTU");
 537                 return ret;
 538         }
 539
 540         return 0;
 541 }
 542
 543 /**
 544  * DPDK callback to bring the link up.
 545  *
 546  * @param dev
 547  *   Pointer to Ethernet device structure.
 548  *
 549  * @return
 550  *   0 on success, negative error value otherwise.
 551  */
 552 static int
 553 mrvl_dev_set_link_up(struct rte_eth_dev *dev)
 554 {
 555         struct mrvl_priv *priv = dev->data->dev_private;
 556         int ret;
 557
 558         if (!priv->ppio) {
 559                 dev->data->dev_link.link_status = ETH_LINK_UP;
 560                 return 0;
 561         }
 562
 563         ret = pp2_ppio_enable(priv->ppio);
 564         if (ret)
 565                 return ret;
 566
 567         /*
 568          * mtu/mru can be updated if pp2_ppio_enable() was called at least once
 569          * as pp2_ppio_enable() changes port->t_mode from default 0 to
 570          * PP2_TRAFFIC_INGRESS_EGRESS.
 571          *
 572          * Set mtu to default DPDK value here.
 573          */
 574         ret = mrvl_mtu_set(dev, dev->data->mtu);
 575         if (ret) {
 576                 pp2_ppio_disable(priv->ppio);
 577                 return ret;
 578         }
 579
 580         dev->data->dev_link.link_status = ETH_LINK_UP;
 581         return 0;
 582 }
 583
 584 /**
 585  * DPDK callback to bring the link down.
 586  *
 587  * @param dev
 588  *   Pointer to Ethernet device structure.
 589  *
 590  * @return
 591  *   0 on success, negative error value otherwise.
 592  */
 593 static int
 594 mrvl_dev_set_link_down(struct rte_eth_dev *dev)
 595 {
 596         struct mrvl_priv *priv = dev->data->dev_private;
 597         int ret;
 598
 599         if (!priv->ppio) {
 600                 dev->data->dev_link.link_status = ETH_LINK_DOWN;
 601                 return 0;
 602         }
 603         ret = pp2_ppio_disable(priv->ppio);
 604         if (ret)
 605                 return ret;
 606
 607         dev->data->dev_link.link_status = ETH_LINK_DOWN;
 608         return 0;
 609 }
 610
 611 /**
 612  * DPDK callback to start tx queue.
 613  *
 614  * @param dev
 615  *   Pointer to Ethernet device structure.
 616  * @param queue_id
 617  *   Transmit queue index.
 618  *
 619  * @return
 620  *   0 on success, negative error value otherwise.
 621  */
 622 static int
 623 mrvl_tx_queue_start(struct rte_eth_dev *dev, uint16_t queue_id)
 624 {
 625         struct mrvl_priv *priv = dev->data->dev_private;
 626         int ret;
 627
 628         if (!priv)
 629                 return -EPERM;
 630
 631         /* passing 1 enables given tx queue */
 632         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 1);
 633         if (ret) {
 634                 MRVL_LOG(ERR, "Failed to start txq %d", queue_id);
 635                 return ret;
 636         }
 637
 638         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STARTED;
 639
 640         return 0;
 641 }
 642
 643 /**
 644  * DPDK callback to stop tx queue.
 645  *
 646  * @param dev
 647  *   Pointer to Ethernet device structure.
 648  * @param queue_id
 649  *   Transmit queue index.
 650  *
 651  * @return
 652  *   0 on success, negative error value otherwise.
 653  */
 654 static int
 655 mrvl_tx_queue_stop(struct rte_eth_dev *dev, uint16_t queue_id)
 656 {
 657         struct mrvl_priv *priv = dev->data->dev_private;
 658         int ret;
 659
 660         if (!priv->ppio)
 661                 return -EPERM;
 662
 663         /* passing 0 disables given tx queue */
 664         ret = pp2_ppio_set_outq_state(priv->ppio, queue_id, 0);
 665         if (ret) {
 666                 MRVL_LOG(ERR, "Failed to stop txq %d", queue_id);
 667                 return ret;
 668         }
 669
 670         dev->data->tx_queue_state[queue_id] = RTE_ETH_QUEUE_STATE_STOPPED;
 671
 672         return 0;
 673 }
 674
 675 /**
 676  * Populate VLAN Filter configuration.
 677  *
 678  * @param dev
 679  *   Pointer to Ethernet device structure.
 680  * @param on
 681  *   Toggle filter.
 682  *
 683  * @return
 684  *   0 on success, negative error value otherwise.
 685  */
 686 static int mrvl_populate_vlan_table(struct rte_eth_dev *dev, int on)
 687 {
 688         uint32_t j;
 689         int ret;
 690         struct rte_vlan_filter_conf *vfc;
 691
 692         vfc = &dev->data->vlan_filter_conf;
 693         for (j = 0; j < RTE_DIM(vfc->ids); j++) {
 694                 uint64_t vlan;
 695                 uint64_t vbit;
 696                 uint64_t ids = vfc->ids[j];
 697
 698                 if (ids == 0)
 699                         continue;
 700
 701                 while (ids) {
 702                         vlan = 64 * j;
 703                         /* count trailing zeroes */
 704                         vbit = ~ids & (ids - 1);
 705                         /* clear least significant bit set */
 706                         ids ^= (ids ^ (ids - 1)) ^ vbit;
 707                         for (; vbit; vlan++)
 708                                 vbit >>= 1;
 709                         ret = mrvl_vlan_filter_set(dev, vlan, on);
 710                         if (ret) {
 711                                 MRVL_LOG(ERR, "Failed to setup VLAN filter\n");
 712                                 return ret;
 713                         }
 714                 }
 715         }
 716
 717         return 0;
 718 }
 719
 720 /**
 721  * DPDK callback to start the device.
 722  *
 723  * @param dev
 724  *   Pointer to Ethernet device structure.
 725  *
 726  * @return
 727  *   0 on success, negative errno value on failure.
 728  */
 729 static int
 730 mrvl_dev_start(struct rte_eth_dev *dev)
 731 {
 732         struct mrvl_priv *priv = dev->data->dev_private;
 733         char match[MRVL_MATCH_LEN];
 734         int ret = 0, i, def_init_size;
 735         struct rte_ether_addr *mac_addr;
 736
 737         if (priv->ppio)
 738                 return mrvl_dev_set_link_up(dev);
 739
 740         snprintf(match, sizeof(match), "ppio-%d:%d",
 741                  priv->pp_id, priv->ppio_id);
 742         priv->ppio_params.match = match;
 743         priv->ppio_params.eth_start_hdr = PP2_PPIO_HDR_ETH;
 744         if (mrvl_cfg)
 745                 priv->ppio_params.eth_start_hdr =
 746                         mrvl_cfg->port[dev->data->port_id].eth_start_hdr;
 747
 748         /*
 749          * Calculate the minimum bpool size for refill feature as follows:
 750          * 2 default burst sizes multiply by number of rx queues.
 751          * If the bpool size will be below this value, new buffers will
 752          * be added to the pool.
 753          */
 754         priv->bpool_min_size = priv->nb_rx_queues * MRVL_BURST_SIZE * 2;
 755
 756         /* In case initial bpool size configured in queues setup is
 757          * smaller than minimum size add more buffers
 758          */
 759         def_init_size = priv->bpool_min_size + MRVL_BURST_SIZE * 2;
 760         if (priv->bpool_init_size < def_init_size) {
 761                 int buffs_to_add = def_init_size - priv->bpool_init_size;
 762
 763                 priv->bpool_init_size += buffs_to_add;
 764                 ret = mrvl_fill_bpool(dev->data->rx_queues[0], buffs_to_add);
 765                 if (ret)
 766                         MRVL_LOG(ERR, "Failed to add buffers to bpool");
 767         }
 768
 769         /*
 770          * Calculate the maximum bpool size for refill feature as follows:
 771          * maximum number of descriptors in rx queue multiply by number
 772          * of rx queues plus minimum bpool size.
 773          * In case the bpool size will exceed this value, superfluous buffers
 774          * will be removed
 775          */
 776         priv->bpool_max_size = (priv->nb_rx_queues * MRVL_PP2_RXD_MAX) +
 777                                 priv->bpool_min_size;
 778
 779         ret = pp2_ppio_init(&priv->ppio_params, &priv->ppio);
 780         if (ret) {
 781                 MRVL_LOG(ERR, "Failed to init ppio");
 782                 return ret;
 783         }
 784
 785         /*
 786          * In case there are some some stale uc/mc mac addresses flush them
 787          * here. It cannot be done during mrvl_dev_close() as port information
 788          * is already gone at that point (due to pp2_ppio_deinit() in
 789          * mrvl_dev_stop()).
 790          */
 791         if (!priv->uc_mc_flushed) {
 792                 ret = pp2_ppio_flush_mac_addrs(priv->ppio, 1, 1);
 793                 if (ret) {
 794                         MRVL_LOG(ERR,
 795                                 "Failed to flush uc/mc filter list");
 796                         goto out;
 797                 }
 798                 priv->uc_mc_flushed = 1;
 799         }
 800
 801         ret = mrvl_mtu_set(dev, dev->data->mtu);
 802         if (ret)
 803                 MRVL_LOG(ERR, "Failed to set MTU to %d", dev->data->mtu);
 804
 805         if (!rte_is_zero_ether_addr(&dev->data->mac_addrs[0]))
 806                 mrvl_mac_addr_set(dev, &dev->data->mac_addrs[0]);
 807
 808         for (i = 1; i < MRVL_MAC_ADDRS_MAX; i++) {
 809                 mac_addr = &dev->data->mac_addrs[i];
 810
 811                 /* skip zero address */
 812                 if (rte_is_zero_ether_addr(mac_addr))
 813                         continue;
 814
 815                 mrvl_mac_addr_add(dev, mac_addr, i, 0);
 816         }
 817
 818         if (dev->data->all_multicast == 1)
 819                 mrvl_allmulticast_enable(dev);
 820
 821         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_VLAN_FILTER) {
 822                 ret = mrvl_populate_vlan_table(dev, 1);
 823                 if (ret) {
 824                         MRVL_LOG(ERR, "Failed to populate VLAN table");
 825                         goto out;
 826                 }
 827         }
 828
 829         /* For default QoS config, don't start classifier. */
 830         if (mrvl_cfg  &&
 831             mrvl_cfg->port[dev->data->port_id].use_global_defaults == 0) {
 832                 ret = mrvl_start_qos_mapping(priv);
 833                 if (ret) {
 834                         MRVL_LOG(ERR, "Failed to setup QoS mapping");
 835                         goto out;
 836                 }
 837         }
 838
 839         ret = pp2_ppio_set_loopback(priv->ppio, dev->data->dev_conf.lpbk_mode);
 840         if (ret) {
 841                 MRVL_LOG(ERR, "Failed to set loopback");
 842                 goto out;
 843         }
 844
 845         if (dev->data->promiscuous == 1)
 846                 mrvl_promiscuous_enable(dev);
 847
 848         if (dev->data->dev_link.link_status == ETH_LINK_UP) {
 849                 ret = mrvl_dev_set_link_up(dev);
 850                 if (ret) {
 851                         MRVL_LOG(ERR, "Failed to set link up");
 852                         dev->data->dev_link.link_status = ETH_LINK_DOWN;
 853                         goto out;
 854                 }
 855         }
 856
 857         /* start tx queues */
 858         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 859                 struct mrvl_txq *txq = dev->data->tx_queues[i];
 860
 861                 dev->data->tx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
 862
 863                 if (!txq->tx_deferred_start)
 864                         continue;
 865
 866                 /*
 867                  * All txqs are started by default. Stop them
 868                  * so that tx_deferred_start works as expected.
 869                  */
 870                 ret = mrvl_tx_queue_stop(dev, i);
 871                 if (ret)
 872                         goto out;
 873         }
 874
 875         mrvl_flow_init(dev);
 876         mrvl_mtr_init(dev);
 877         mrvl_set_tx_function(dev);
 878
 879         return 0;
 880 out:
 881         MRVL_LOG(ERR, "Failed to start device");
 882         pp2_ppio_deinit(priv->ppio);
 883         return ret;
 884 }
 885
 886 /**
 887  * Flush receive queues.
 888  *
 889  * @param dev
 890  *   Pointer to Ethernet device structure.
 891  */
 892 static void
 893 mrvl_flush_rx_queues(struct rte_eth_dev *dev)
 894 {
 895         int i;
 896
 897         MRVL_LOG(INFO, "Flushing rx queues");
 898         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 899                 int ret, num;
 900
 901                 do {
 902                         struct mrvl_rxq *q = dev->data->rx_queues[i];
 903                         struct pp2_ppio_desc descs[MRVL_PP2_RXD_MAX];
 904
 905                         num = MRVL_PP2_RXD_MAX;
 906                         ret = pp2_ppio_recv(q->priv->ppio,
 907                                             q->priv->rxq_map[q->queue_id].tc,
 908                                             q->priv->rxq_map[q->queue_id].inq,
 909                                             descs, (uint16_t *)&num);
 910                 } while (ret == 0 && num);
 911         }
 912 }
 913
 914 /**
 915  * Flush transmit shadow queues.
 916  *
 917  * @param dev
 918  *   Pointer to Ethernet device structure.
 919  */
 920 static void
 921 mrvl_flush_tx_shadow_queues(struct rte_eth_dev *dev)
 922 {
 923         int i, j;
 924         struct mrvl_txq *txq;
 925
 926         MRVL_LOG(INFO, "Flushing tx shadow queues");
 927         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 928                 txq = (struct mrvl_txq *)dev->data->tx_queues[i];
 929
 930                 for (j = 0; j < RTE_MAX_LCORE; j++) {
 931                         struct mrvl_shadow_txq *sq;
 932
 933                         if (!hifs[j])
 934                                 continue;
 935
 936                         sq = &txq->shadow_txqs[j];
 937                         mrvl_free_sent_buffers(txq->priv->ppio,
 938                                 hifs[j], j, sq, txq->queue_id, 1);
 939                         while (sq->tail != sq->head) {
 940                                 uint64_t addr = cookie_addr_high |
 941                                         sq->ent[sq->tail].buff.cookie;
 942                                 rte_pktmbuf_free(
 943                                         (struct rte_mbuf *)addr);
 944                                 sq->tail = (sq->tail + 1) &
 945                                             MRVL_PP2_TX_SHADOWQ_MASK;
 946                         }
 947                         memset(sq, 0, sizeof(*sq));
 948                 }
 949         }
 950 }
 951
 952 /**
 953  * Flush hardware bpool (buffer-pool).
 954  *
 955  * @param dev
 956  *   Pointer to Ethernet device structure.
 957  */
 958 static void
 959 mrvl_flush_bpool(struct rte_eth_dev *dev)
 960 {
 961         struct mrvl_priv *priv = dev->data->dev_private;
 962         struct pp2_hif *hif;
 963         uint32_t num;
 964         int ret;
 965         unsigned int core_id = rte_lcore_id();
 966
 967         if (core_id == LCORE_ID_ANY)
 968                 core_id = rte_get_main_lcore();
 969
 970         hif = mrvl_get_hif(priv, core_id);
 971
 972         ret = pp2_bpool_get_num_buffs(priv->bpool, &num);
 973         if (ret) {
 974                 MRVL_LOG(ERR, "Failed to get bpool buffers number");
 975                 return;
 976         }
 977
 978         while (num--) {
 979                 struct pp2_buff_inf inf;
 980                 uint64_t addr;
 981
 982                 ret = pp2_bpool_get_buff(hif, priv->bpool, &inf);
 983                 if (ret)
 984                         break;
 985
 986                 addr = cookie_addr_high | inf.cookie;
 987                 rte_pktmbuf_free((struct rte_mbuf *)addr);
 988         }
 989 }
 990
 991 /**
 992  * DPDK callback to stop the device.
 993  *
 994  * @param dev
 995  *   Pointer to Ethernet device structure.
 996  */
 997 static int
 998 mrvl_dev_stop(struct rte_eth_dev *dev)
 999 {
1000         return mrvl_dev_set_link_down(dev);
1001 }
1002
1003 /**
1004  * DPDK callback to close the device.
1005  *
1006  * @param dev
1007  *   Pointer to Ethernet device structure.
1008  */
1009 static int
1010 mrvl_dev_close(struct rte_eth_dev *dev)
1011 {
1012         struct mrvl_priv *priv = dev->data->dev_private;
1013         size_t i;
1014
1015         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
1016                 return 0;
1017
1018         mrvl_flush_rx_queues(dev);
1019         mrvl_flush_tx_shadow_queues(dev);
1020         mrvl_flow_deinit(dev);
1021         mrvl_mtr_deinit(dev);
1022
1023         for (i = 0; i < priv->ppio_params.inqs_params.num_tcs; ++i) {
1024                 struct pp2_ppio_tc_params *tc_params =
1025                         &priv->ppio_params.inqs_params.tcs_params[i];
1026
1027                 if (tc_params->inqs_params) {
1028                         rte_free(tc_params->inqs_params);
1029                         tc_params->inqs_params = NULL;
1030                 }
1031         }
1032
1033         if (priv->cls_tbl) {
1034                 pp2_cls_tbl_deinit(priv->cls_tbl);
1035                 priv->cls_tbl = NULL;
1036         }
1037
1038         if (priv->qos_tbl) {
1039                 pp2_cls_qos_tbl_deinit(priv->qos_tbl);
1040                 priv->qos_tbl = NULL;
1041         }
1042
1043         mrvl_flush_bpool(dev);
1044         mrvl_tm_deinit(dev);
1045
1046         if (priv->ppio) {
1047                 pp2_ppio_deinit(priv->ppio);
1048                 priv->ppio = NULL;
1049         }
1050
1051         /* policer must be released after ppio deinitialization */
1052         if (priv->default_policer) {
1053                 pp2_cls_plcr_deinit(priv->default_policer);
1054                 priv->default_policer = NULL;
1055         }
1056
1057
1058         if (priv->bpool) {
1059                 pp2_bpool_deinit(priv->bpool);
1060                 used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
1061                 priv->bpool = NULL;
1062         }
1063
1064         mrvl_dev_num--;
1065
1066         if (mrvl_dev_num == 0) {
1067                 MRVL_LOG(INFO, "Perform MUSDK deinit");
1068                 mrvl_deinit_hifs();
1069                 mrvl_deinit_pp2();
1070                 rte_mvep_deinit(MVEP_MOD_T_PP2);
1071         }
1072
1073         return 0;
1074 }
1075
1076 /**
1077  * DPDK callback to retrieve physical link information.
1078  *
1079  * @param dev
1080  *   Pointer to Ethernet device structure.
1081  * @param wait_to_complete
1082  *   Wait for request completion (ignored).
1083  *
1084  * @return
1085  *   0 on success, negative error value otherwise.
1086  */
1087 static int
1088 mrvl_link_update(struct rte_eth_dev *dev, int wait_to_complete __rte_unused)
1089 {
1090         /*
1091          * TODO
1092          * once MUSDK provides necessary API use it here
1093          */
1094         struct mrvl_priv *priv = dev->data->dev_private;
1095         struct ethtool_cmd edata;
1096         struct ifreq req;
1097         int ret, fd, link_up;
1098
1099         if (!priv->ppio)
1100                 return -EPERM;
1101
1102         edata.cmd = ETHTOOL_GSET;
1103
1104         strcpy(req.ifr_name, dev->data->name);
1105         req.ifr_data = (void *)&edata;
1106
1107         fd = socket(AF_INET, SOCK_DGRAM, 0);
1108         if (fd == -1)
1109                 return -EFAULT;
1110
1111         ret = ioctl(fd, SIOCETHTOOL, &req);
1112         if (ret == -1) {
1113                 close(fd);
1114                 return -EFAULT;
1115         }
1116
1117         close(fd);
1118
1119         switch (ethtool_cmd_speed(&edata)) {
1120         case SPEED_10:
1121                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10M;
1122                 break;
1123         case SPEED_100:
1124                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_100M;
1125                 break;
1126         case SPEED_1000:
1127                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_1G;
1128                 break;
1129         case SPEED_10000:
1130                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_10G;
1131                 break;
1132         default:
1133                 dev->data->dev_link.link_speed = ETH_SPEED_NUM_NONE;
1134         }
1135
1136         dev->data->dev_link.link_duplex = edata.duplex ? ETH_LINK_FULL_DUPLEX :
1137                                                          ETH_LINK_HALF_DUPLEX;
1138         dev->data->dev_link.link_autoneg = edata.autoneg ? ETH_LINK_AUTONEG :
1139                                                            ETH_LINK_FIXED;
1140         pp2_ppio_get_link_state(priv->ppio, &link_up);
1141         dev->data->dev_link.link_status = link_up ? ETH_LINK_UP : ETH_LINK_DOWN;
1142
1143         return 0;
1144 }
1145
1146 /**
1147  * DPDK callback to enable promiscuous mode.
1148  *
1149  * @param dev
1150  *   Pointer to Ethernet device structure.
1151  *
1152  * @return
1153  *   0 on success, negative error value otherwise.
1154  */
1155 static int
1156 mrvl_promiscuous_enable(struct rte_eth_dev *dev)
1157 {
1158         struct mrvl_priv *priv = dev->data->dev_private;
1159         int ret;
1160
1161         if (priv->isolated)
1162                 return -ENOTSUP;
1163
1164         if (!priv->ppio)
1165                 return 0;
1166
1167         ret = pp2_ppio_set_promisc(priv->ppio, 1);
1168         if (ret) {
1169                 MRVL_LOG(ERR, "Failed to enable promiscuous mode");
1170                 return -EAGAIN;
1171         }
1172
1173         return 0;
1174 }
1175
1176 /**
1177  * DPDK callback to enable allmulti mode.
1178  *
1179  * @param dev
1180  *   Pointer to Ethernet device structure.
1181  *
1182  * @return
1183  *   0 on success, negative error value otherwise.
1184  */
1185 static int
1186 mrvl_allmulticast_enable(struct rte_eth_dev *dev)
1187 {
1188         struct mrvl_priv *priv = dev->data->dev_private;
1189         int ret;
1190
1191         if (priv->isolated)
1192                 return -ENOTSUP;
1193
1194         if (!priv->ppio)
1195                 return 0;
1196
1197         ret = pp2_ppio_set_mc_promisc(priv->ppio, 1);
1198         if (ret) {
1199                 MRVL_LOG(ERR, "Failed enable all-multicast mode");
1200                 return -EAGAIN;
1201         }
1202
1203         return 0;
1204 }
1205
1206 /**
1207  * DPDK callback to disable promiscuous mode.
1208  *
1209  * @param dev
1210  *   Pointer to Ethernet device structure.
1211  *
1212  * @return
1213  *   0 on success, negative error value otherwise.
1214  */
1215 static int
1216 mrvl_promiscuous_disable(struct rte_eth_dev *dev)
1217 {
1218         struct mrvl_priv *priv = dev->data->dev_private;
1219         int ret;
1220
1221         if (priv->isolated)
1222                 return -ENOTSUP;
1223
1224         if (!priv->ppio)
1225                 return 0;
1226
1227         ret = pp2_ppio_set_promisc(priv->ppio, 0);
1228         if (ret) {
1229                 MRVL_LOG(ERR, "Failed to disable promiscuous mode");
1230                 return -EAGAIN;
1231         }
1232
1233         return 0;
1234 }
1235
1236 /**
1237  * DPDK callback to disable allmulticast mode.
1238  *
1239  * @param dev
1240  *   Pointer to Ethernet device structure.
1241  *
1242  * @return
1243  *   0 on success, negative error value otherwise.
1244  */
1245 static int
1246 mrvl_allmulticast_disable(struct rte_eth_dev *dev)
1247 {
1248         struct mrvl_priv *priv = dev->data->dev_private;
1249         int ret;
1250
1251         if (priv->isolated)
1252                 return -ENOTSUP;
1253
1254         if (!priv->ppio)
1255                 return 0;
1256
1257         ret = pp2_ppio_set_mc_promisc(priv->ppio, 0);
1258         if (ret) {
1259                 MRVL_LOG(ERR, "Failed to disable all-multicast mode");
1260                 return -EAGAIN;
1261         }
1262
1263         return 0;
1264 }
1265
1266 /**
1267  * DPDK callback to remove a MAC address.
1268  *
1269  * @param dev
1270  *   Pointer to Ethernet device structure.
1271  * @param index
1272  *   MAC address index.
1273  */
1274 static void
1275 mrvl_mac_addr_remove(struct rte_eth_dev *dev, uint32_t index)
1276 {
1277         struct mrvl_priv *priv = dev->data->dev_private;
1278         char buf[RTE_ETHER_ADDR_FMT_SIZE];
1279         int ret;
1280
1281         if (priv->isolated)
1282                 return;
1283
1284         if (!priv->ppio)
1285                 return;
1286
1287         ret = pp2_ppio_remove_mac_addr(priv->ppio,
1288                                        dev->data->mac_addrs[index].addr_bytes);
1289         if (ret) {
1290                 rte_ether_format_addr(buf, sizeof(buf),
1291                                   &dev->data->mac_addrs[index]);
1292                 MRVL_LOG(ERR, "Failed to remove mac %s", buf);
1293         }
1294 }
1295
1296 /**
1297  * DPDK callback to add a MAC address.
1298  *
1299  * @param dev
1300  *   Pointer to Ethernet device structure.
1301  * @param mac_addr
1302  *   MAC address to register.
1303  * @param index
1304  *   MAC address index.
1305  * @param vmdq
1306  *   VMDq pool index to associate address with (unused).
1307  *
1308  * @return
1309  *   0 on success, negative error value otherwise.
1310  */
1311 static int
1312 mrvl_mac_addr_add(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr,
1313                   uint32_t index, uint32_t vmdq __rte_unused)
1314 {
1315         struct mrvl_priv *priv = dev->data->dev_private;
1316         char buf[RTE_ETHER_ADDR_FMT_SIZE];
1317         int ret;
1318
1319         if (priv->isolated)
1320                 return -ENOTSUP;
1321
1322         if (!priv->ppio)
1323                 return 0;
1324
1325         if (index == 0)
1326                 /* For setting index 0, mrvl_mac_addr_set() should be used.*/
1327                 return -1;
1328
1329         /*
1330          * Maximum number of uc addresses can be tuned via kernel module mvpp2x
1331          * parameter uc_filter_max. Maximum number of mc addresses is then
1332          * MRVL_MAC_ADDRS_MAX - uc_filter_max. Currently it defaults to 4 and
1333          * 21 respectively.
1334          *
1335          * If more than uc_filter_max uc addresses were added to filter list
1336          * then NIC will switch to promiscuous mode automatically.
1337          *
1338          * If more than MRVL_MAC_ADDRS_MAX - uc_filter_max number mc addresses
1339          * were added to filter list then NIC will switch to all-multicast mode
1340          * automatically.
1341          */
1342         ret = pp2_ppio_add_mac_addr(priv->ppio, mac_addr->addr_bytes);
1343         if (ret) {
1344                 rte_ether_format_addr(buf, sizeof(buf), mac_addr);
1345                 MRVL_LOG(ERR, "Failed to add mac %s", buf);
1346                 return -1;
1347         }
1348
1349         return 0;
1350 }
1351
1352 /**
1353  * DPDK callback to set the primary MAC address.
1354  *
1355  * @param dev
1356  *   Pointer to Ethernet device structure.
1357  * @param mac_addr
1358  *   MAC address to register.
1359  *
1360  * @return
1361  *   0 on success, negative error value otherwise.
1362  */
1363 static int
1364 mrvl_mac_addr_set(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr)
1365 {
1366         struct mrvl_priv *priv = dev->data->dev_private;
1367         int ret;
1368
1369         if (priv->isolated)
1370                 return -ENOTSUP;
1371
1372         if (!priv->ppio)
1373                 return 0;
1374
1375         ret = pp2_ppio_set_mac_addr(priv->ppio, mac_addr->addr_bytes);
1376         if (ret) {
1377                 char buf[RTE_ETHER_ADDR_FMT_SIZE];
1378                 rte_ether_format_addr(buf, sizeof(buf), mac_addr);
1379                 MRVL_LOG(ERR, "Failed to set mac to %s", buf);
1380         }
1381
1382         return ret;
1383 }
1384
1385 /**
1386  * DPDK callback to get device statistics.
1387  *
1388  * @param dev
1389  *   Pointer to Ethernet device structure.
1390  * @param stats
1391  *   Stats structure output buffer.
1392  *
1393  * @return
1394  *   0 on success, negative error value otherwise.
1395  */
1396 static int
1397 mrvl_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
1398 {
1399         struct mrvl_priv *priv = dev->data->dev_private;
1400         struct pp2_ppio_statistics ppio_stats;
1401         uint64_t drop_mac = 0;
1402         unsigned int i, idx, ret;
1403
1404         if (!priv->ppio)
1405                 return -EPERM;
1406
1407         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1408                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1409                 struct pp2_ppio_inq_statistics rx_stats;
1410
1411                 if (!rxq)
1412                         continue;
1413
1414                 idx = rxq->queue_id;
1415                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1416                         MRVL_LOG(ERR,
1417                                 "rx queue %d stats out of range (0 - %d)",
1418                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1419                         continue;
1420                 }
1421
1422                 ret = pp2_ppio_inq_get_statistics(priv->ppio,
1423                                                   priv->rxq_map[idx].tc,
1424                                                   priv->rxq_map[idx].inq,
1425                                                   &rx_stats, 0);
1426                 if (unlikely(ret)) {
1427                         MRVL_LOG(ERR,
1428                                 "Failed to update rx queue %d stats", idx);
1429                         break;
1430                 }
1431
1432                 stats->q_ibytes[idx] = rxq->bytes_recv;
1433                 stats->q_ipackets[idx] = rx_stats.enq_desc - rxq->drop_mac;
1434                 stats->q_errors[idx] = rx_stats.drop_early +
1435                                        rx_stats.drop_fullq +
1436                                        rx_stats.drop_bm +
1437                                        rxq->drop_mac;
1438                 stats->ibytes += rxq->bytes_recv;
1439                 drop_mac += rxq->drop_mac;
1440         }
1441
1442         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1443                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1444                 struct pp2_ppio_outq_statistics tx_stats;
1445
1446                 if (!txq)
1447                         continue;
1448
1449                 idx = txq->queue_id;
1450                 if (unlikely(idx >= RTE_ETHDEV_QUEUE_STAT_CNTRS)) {
1451                         MRVL_LOG(ERR,
1452                                 "tx queue %d stats out of range (0 - %d)",
1453                                 idx, RTE_ETHDEV_QUEUE_STAT_CNTRS - 1);
1454                 }
1455
1456                 ret = pp2_ppio_outq_get_statistics(priv->ppio, idx,
1457                                                    &tx_stats, 0);
1458                 if (unlikely(ret)) {
1459                         MRVL_LOG(ERR,
1460                                 "Failed to update tx queue %d stats", idx);
1461                         break;
1462                 }
1463
1464                 stats->q_opackets[idx] = tx_stats.deq_desc;
1465                 stats->q_obytes[idx] = txq->bytes_sent;
1466                 stats->obytes += txq->bytes_sent;
1467         }
1468
1469         ret = pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1470         if (unlikely(ret)) {
1471                 MRVL_LOG(ERR, "Failed to update port statistics");
1472                 return ret;
1473         }
1474
1475         stats->ipackets += ppio_stats.rx_packets - drop_mac;
1476         stats->opackets += ppio_stats.tx_packets;
1477         stats->imissed += ppio_stats.rx_fullq_dropped +
1478                           ppio_stats.rx_bm_dropped +
1479                           ppio_stats.rx_early_dropped +
1480                           ppio_stats.rx_fifo_dropped +
1481                           ppio_stats.rx_cls_dropped;
1482         stats->ierrors = drop_mac;
1483
1484         return 0;
1485 }
1486
1487 /**
1488  * DPDK callback to clear device statistics.
1489  *
1490  * @param dev
1491  *   Pointer to Ethernet device structure.
1492  *
1493  * @return
1494  *   0 on success, negative error value otherwise.
1495  */
1496 static int
1497 mrvl_stats_reset(struct rte_eth_dev *dev)
1498 {
1499         struct mrvl_priv *priv = dev->data->dev_private;
1500         int i;
1501
1502         if (!priv->ppio)
1503                 return 0;
1504
1505         for (i = 0; i < dev->data->nb_rx_queues; i++) {
1506                 struct mrvl_rxq *rxq = dev->data->rx_queues[i];
1507
1508                 pp2_ppio_inq_get_statistics(priv->ppio, priv->rxq_map[i].tc,
1509                                             priv->rxq_map[i].inq, NULL, 1);
1510                 rxq->bytes_recv = 0;
1511                 rxq->drop_mac = 0;
1512         }
1513
1514         for (i = 0; i < dev->data->nb_tx_queues; i++) {
1515                 struct mrvl_txq *txq = dev->data->tx_queues[i];
1516
1517                 pp2_ppio_outq_get_statistics(priv->ppio, i, NULL, 1);
1518                 txq->bytes_sent = 0;
1519         }
1520
1521         return pp2_ppio_get_statistics(priv->ppio, NULL, 1);
1522 }
1523
1524 /**
1525  * DPDK callback to get extended statistics.
1526  *
1527  * @param dev
1528  *   Pointer to Ethernet device structure.
1529  * @param stats
1530  *   Pointer to xstats table.
1531  * @param n
1532  *   Number of entries in xstats table.
1533  * @return
1534  *   Negative value on error, number of read xstats otherwise.
1535  */
1536 static int
1537 mrvl_xstats_get(struct rte_eth_dev *dev,
1538                 struct rte_eth_xstat *stats, unsigned int n)
1539 {
1540         struct mrvl_priv *priv = dev->data->dev_private;
1541         struct pp2_ppio_statistics ppio_stats;
1542         unsigned int i;
1543
1544         if (!stats)
1545                 return 0;
1546
1547         pp2_ppio_get_statistics(priv->ppio, &ppio_stats, 0);
1548         for (i = 0; i < n && i < RTE_DIM(mrvl_xstats_tbl); i++) {
1549                 uint64_t val;
1550
1551                 if (mrvl_xstats_tbl[i].size == sizeof(uint32_t))
1552                         val = *(uint32_t *)((uint8_t *)&ppio_stats +
1553                                             mrvl_xstats_tbl[i].offset);
1554                 else if (mrvl_xstats_tbl[i].size == sizeof(uint64_t))
1555                         val = *(uint64_t *)((uint8_t *)&ppio_stats +
1556                                             mrvl_xstats_tbl[i].offset);
1557                 else
1558                         return -EINVAL;
1559
1560                 stats[i].id = i;
1561                 stats[i].value = val;
1562         }
1563
1564         return n;
1565 }
1566
1567 /**
1568  * DPDK callback to reset extended statistics.
1569  *
1570  * @param dev
1571  *   Pointer to Ethernet device structure.
1572  *
1573  * @return
1574  *   0 on success, negative error value otherwise.
1575  */
1576 static int
1577 mrvl_xstats_reset(struct rte_eth_dev *dev)
1578 {
1579         return mrvl_stats_reset(dev);
1580 }
1581
1582 /**
1583  * DPDK callback to get extended statistics names.
1584  *
1585  * @param dev (unused)
1586  *   Pointer to Ethernet device structure.
1587  * @param xstats_names
1588  *   Pointer to xstats names table.
1589  * @param size
1590  *   Size of the xstats names table.
1591  * @return
1592  *   Number of read names.
1593  */
1594 static int
1595 mrvl_xstats_get_names(struct rte_eth_dev *dev __rte_unused,
1596                       struct rte_eth_xstat_name *xstats_names,
1597                       unsigned int size)
1598 {
1599         unsigned int i;
1600
1601         if (!xstats_names)
1602                 return RTE_DIM(mrvl_xstats_tbl);
1603
1604         for (i = 0; i < size && i < RTE_DIM(mrvl_xstats_tbl); i++)
1605                 strlcpy(xstats_names[i].name, mrvl_xstats_tbl[i].name,
1606                         RTE_ETH_XSTATS_NAME_SIZE);
1607
1608         return size;
1609 }
1610
1611 /**
1612  * DPDK callback to get information about the device.
1613  *
1614  * @param dev
1615  *   Pointer to Ethernet device structure (unused).
1616  * @param info
1617  *   Info structure output buffer.
1618  */
1619 static int
1620 mrvl_dev_infos_get(struct rte_eth_dev *dev __rte_unused,
1621                    struct rte_eth_dev_info *info)
1622 {
1623         info->speed_capa = ETH_LINK_SPEED_10M |
1624                            ETH_LINK_SPEED_100M |
1625                            ETH_LINK_SPEED_1G |
1626                            ETH_LINK_SPEED_10G;
1627
1628         info->max_rx_queues = MRVL_PP2_RXQ_MAX;
1629         info->max_tx_queues = MRVL_PP2_TXQ_MAX;
1630         info->max_mac_addrs = MRVL_MAC_ADDRS_MAX;
1631
1632         info->rx_desc_lim.nb_max = MRVL_PP2_RXD_MAX;
1633         info->rx_desc_lim.nb_min = MRVL_PP2_RXD_MIN;
1634         info->rx_desc_lim.nb_align = MRVL_PP2_RXD_ALIGN;
1635
1636         info->tx_desc_lim.nb_max = MRVL_PP2_TXD_MAX;
1637         info->tx_desc_lim.nb_min = MRVL_PP2_TXD_MIN;
1638         info->tx_desc_lim.nb_align = MRVL_PP2_TXD_ALIGN;
1639
1640         info->rx_offload_capa = MRVL_RX_OFFLOADS;
1641         info->rx_queue_offload_capa = MRVL_RX_OFFLOADS;
1642
1643         info->tx_offload_capa = MRVL_TX_OFFLOADS;
1644         info->tx_queue_offload_capa = MRVL_TX_OFFLOADS;
1645
1646         info->flow_type_rss_offloads = ETH_RSS_IPV4 |
1647                                        ETH_RSS_NONFRAG_IPV4_TCP |
1648                                        ETH_RSS_NONFRAG_IPV4_UDP;
1649
1650         /* By default packets are dropped if no descriptors are available */
1651         info->default_rxconf.rx_drop_en = 1;
1652
1653         info->max_rx_pktlen = MRVL_PKT_SIZE_MAX;
1654
1655         return 0;
1656 }
1657
1658 /**
1659  * Return supported packet types.
1660  *
1661  * @param dev
1662  *   Pointer to Ethernet device structure (unused).
1663  *
1664  * @return
1665  *   Const pointer to the table with supported packet types.
1666  */
1667 static const uint32_t *
1668 mrvl_dev_supported_ptypes_get(struct rte_eth_dev *dev __rte_unused)
1669 {
1670         static const uint32_t ptypes[] = {
1671                 RTE_PTYPE_L2_ETHER,
1672                 RTE_PTYPE_L2_ETHER_VLAN,
1673                 RTE_PTYPE_L2_ETHER_QINQ,
1674                 RTE_PTYPE_L3_IPV4,
1675                 RTE_PTYPE_L3_IPV4_EXT,
1676                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
1677                 RTE_PTYPE_L3_IPV6,
1678                 RTE_PTYPE_L3_IPV6_EXT,
1679                 RTE_PTYPE_L2_ETHER_ARP,
1680                 RTE_PTYPE_L4_TCP,
1681                 RTE_PTYPE_L4_UDP
1682         };
1683
1684         return ptypes;
1685 }
1686
1687 /**
1688  * DPDK callback to get information about specific receive queue.
1689  *
1690  * @param dev
1691  *   Pointer to Ethernet device structure.
1692  * @param rx_queue_id
1693  *   Receive queue index.
1694  * @param qinfo
1695  *   Receive queue information structure.
1696  */
1697 static void mrvl_rxq_info_get(struct rte_eth_dev *dev, uint16_t rx_queue_id,
1698                               struct rte_eth_rxq_info *qinfo)
1699 {
1700         struct mrvl_rxq *q = dev->data->rx_queues[rx_queue_id];
1701         struct mrvl_priv *priv = dev->data->dev_private;
1702         int inq = priv->rxq_map[rx_queue_id].inq;
1703         int tc = priv->rxq_map[rx_queue_id].tc;
1704         struct pp2_ppio_tc_params *tc_params =
1705                 &priv->ppio_params.inqs_params.tcs_params[tc];
1706
1707         qinfo->mp = q->mp;
1708         qinfo->nb_desc = tc_params->inqs_params[inq].size;
1709 }
1710
1711 /**
1712  * DPDK callback to get information about specific transmit queue.
1713  *
1714  * @param dev
1715  *   Pointer to Ethernet device structure.
1716  * @param tx_queue_id
1717  *   Transmit queue index.
1718  * @param qinfo
1719  *   Transmit queue information structure.
1720  */
1721 static void mrvl_txq_info_get(struct rte_eth_dev *dev, uint16_t tx_queue_id,
1722                               struct rte_eth_txq_info *qinfo)
1723 {
1724         struct mrvl_priv *priv = dev->data->dev_private;
1725         struct mrvl_txq *txq = dev->data->tx_queues[tx_queue_id];
1726
1727         qinfo->nb_desc =
1728                 priv->ppio_params.outqs_params.outqs_params[tx_queue_id].size;
1729         qinfo->conf.tx_deferred_start = txq->tx_deferred_start;
1730 }
1731
1732 /**
1733  * DPDK callback to Configure a VLAN filter.
1734  *
1735  * @param dev
1736  *   Pointer to Ethernet device structure.
1737  * @param vlan_id
1738  *   VLAN ID to filter.
1739  * @param on
1740  *   Toggle filter.
1741  *
1742  * @return
1743  *   0 on success, negative error value otherwise.
1744  */
1745 static int
1746 mrvl_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
1747 {
1748         struct mrvl_priv *priv = dev->data->dev_private;
1749
1750         if (priv->isolated)
1751                 return -ENOTSUP;
1752
1753         if (!priv->ppio)
1754                 return 0;
1755
1756         return on ? pp2_ppio_add_vlan(priv->ppio, vlan_id) :
1757                     pp2_ppio_remove_vlan(priv->ppio, vlan_id);
1758 }
1759
1760 /**
1761  * DPDK callback to Configure VLAN offload.
1762  *
1763  * @param dev
1764  *   Pointer to Ethernet device structure.
1765  * @param mask
1766  *   VLAN offload mask.
1767  *
1768  * @return
1769  *   0 on success, negative error value otherwise.
1770  */
1771 static int mrvl_vlan_offload_set(struct rte_eth_dev *dev, int mask)
1772 {
1773         uint64_t rx_offloads = dev->data->dev_conf.rxmode.offloads;
1774         int ret;
1775
1776         if (mask & ETH_VLAN_STRIP_MASK)
1777                 MRVL_LOG(ERR, "VLAN stripping is not supported\n");
1778
1779         if (mask & ETH_VLAN_FILTER_MASK) {
1780                 if (rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
1781                         ret = mrvl_populate_vlan_table(dev, 1);
1782                 else
1783                         ret = mrvl_populate_vlan_table(dev, 0);
1784
1785                 if (ret)
1786                         return ret;
1787         }
1788
1789         if (mask & ETH_VLAN_EXTEND_MASK)
1790                 MRVL_LOG(ERR, "Extend VLAN not supported\n");
1791
1792         return 0;
1793 }
1794
1795 /**
1796  * Release buffers to hardware bpool (buffer-pool)
1797  *
1798  * @param rxq
1799  *   Receive queue pointer.
1800  * @param num
1801  *   Number of buffers to release to bpool.
1802  *
1803  * @return
1804  *   0 on success, negative error value otherwise.
1805  */
1806 static int
1807 mrvl_fill_bpool(struct mrvl_rxq *rxq, int num)
1808 {
1809         struct buff_release_entry entries[num];
1810         struct rte_mbuf *mbufs[num];
1811         int i, ret;
1812         unsigned int core_id;
1813         struct pp2_hif *hif;
1814         struct pp2_bpool *bpool;
1815
1816         core_id = rte_lcore_id();
1817         if (core_id == LCORE_ID_ANY)
1818                 core_id = rte_get_main_lcore();
1819
1820         hif = mrvl_get_hif(rxq->priv, core_id);
1821         if (!hif)
1822                 return -1;
1823
1824         bpool = rxq->priv->bpool;
1825
1826         ret = rte_pktmbuf_alloc_bulk(rxq->mp, mbufs, num);
1827         if (ret)
1828                 return ret;
1829
1830         if (cookie_addr_high == MRVL_COOKIE_ADDR_INVALID)
1831                 cookie_addr_high =
1832                         (uint64_t)mbufs[0] & MRVL_COOKIE_HIGH_ADDR_MASK;
1833
1834         for (i = 0; i < num; i++) {
1835                 if (((uint64_t)mbufs[i] & MRVL_COOKIE_HIGH_ADDR_MASK)
1836                         != cookie_addr_high) {
1837                         MRVL_LOG(ERR,
1838                                 "mbuf virtual addr high is out of range "
1839                                 "0x%x instead of 0x%x\n",
1840                                 (uint32_t)((uint64_t)mbufs[i] >> 32),
1841                                 (uint32_t)(cookie_addr_high >> 32));
1842                         goto out;
1843                 }
1844
1845                 entries[i].buff.addr =
1846                         rte_mbuf_data_iova_default(mbufs[i]);
1847                 entries[i].buff.cookie = (uintptr_t)mbufs[i];
1848                 entries[i].bpool = bpool;
1849         }
1850
1851         pp2_bpool_put_buffs(hif, entries, (uint16_t *)&i);
1852         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] += i;
1853
1854         if (i != num)
1855                 goto out;
1856
1857         return 0;
1858 out:
1859         for (; i < num; i++)
1860                 rte_pktmbuf_free(mbufs[i]);
1861
1862         return -1;
1863 }
1864
1865 /**
1866  * DPDK callback to configure the receive queue.
1867  *
1868  * @param dev
1869  *   Pointer to Ethernet device structure.
1870  * @param idx
1871  *   RX queue index.
1872  * @param desc
1873  *   Number of descriptors to configure in queue.
1874  * @param socket
1875  *   NUMA socket on which memory must be allocated.
1876  * @param conf
1877  *   Thresholds parameters.
1878  * @param mp
1879  *   Memory pool for buffer allocations.
1880  *
1881  * @return
1882  *   0 on success, negative error value otherwise.
1883  */
1884 static int
1885 mrvl_rx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
1886                     unsigned int socket,
1887                     const struct rte_eth_rxconf *conf,
1888                     struct rte_mempool *mp)
1889 {
1890         struct mrvl_priv *priv = dev->data->dev_private;
1891         struct mrvl_rxq *rxq;
1892         uint32_t frame_size, buf_size = rte_pktmbuf_data_room_size(mp);
1893         uint32_t max_rx_pkt_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
1894         int ret, tc, inq;
1895         uint64_t offloads;
1896
1897         offloads = conf->offloads | dev->data->dev_conf.rxmode.offloads;
1898
1899         if (priv->rxq_map[idx].tc == MRVL_UNKNOWN_TC) {
1900                 /*
1901                  * Unknown TC mapping, mapping will not have a correct queue.
1902                  */
1903                 MRVL_LOG(ERR, "Unknown TC mapping for queue %hu eth%hhu",
1904                         idx, priv->ppio_id);
1905                 return -EFAULT;
1906         }
1907
1908         frame_size = buf_size - RTE_PKTMBUF_HEADROOM -
1909                      MRVL_PKT_EFFEC_OFFS + RTE_ETHER_CRC_LEN;
1910         if (frame_size < max_rx_pkt_len) {
1911                 MRVL_LOG(WARNING,
1912                         "Mbuf size must be increased to %u bytes to hold up "
1913                         "to %u bytes of data.",
1914                         buf_size + max_rx_pkt_len - frame_size,
1915                         max_rx_pkt_len);
1916                 dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
1917                 MRVL_LOG(INFO, "Setting max rx pkt len to %u",
1918                         dev->data->dev_conf.rxmode.max_rx_pkt_len);
1919         }
1920
1921         if (dev->data->rx_queues[idx]) {
1922                 rte_free(dev->data->rx_queues[idx]);
1923                 dev->data->rx_queues[idx] = NULL;
1924         }
1925
1926         rxq = rte_zmalloc_socket("rxq", sizeof(*rxq), 0, socket);
1927         if (!rxq)
1928                 return -ENOMEM;
1929
1930         rxq->priv = priv;
1931         rxq->mp = mp;
1932         rxq->cksum_enabled = offloads & DEV_RX_OFFLOAD_IPV4_CKSUM;
1933         rxq->queue_id = idx;
1934         rxq->port_id = dev->data->port_id;
1935         mrvl_port_to_bpool_lookup[rxq->port_id] = priv->bpool;
1936
1937         tc = priv->rxq_map[rxq->queue_id].tc,
1938         inq = priv->rxq_map[rxq->queue_id].inq;
1939         priv->ppio_params.inqs_params.tcs_params[tc].inqs_params[inq].size =
1940                 desc;
1941
1942         ret = mrvl_fill_bpool(rxq, desc);
1943         if (ret) {
1944                 rte_free(rxq);
1945                 return ret;
1946         }
1947
1948         priv->bpool_init_size += desc;
1949
1950         dev->data->rx_queues[idx] = rxq;
1951
1952         return 0;
1953 }
1954
1955 /**
1956  * DPDK callback to release the receive queue.
1957  *
1958  * @param rxq
1959  *   Generic receive queue pointer.
1960  */
1961 static void
1962 mrvl_rx_queue_release(void *rxq)
1963 {
1964         struct mrvl_rxq *q = rxq;
1965         struct pp2_ppio_tc_params *tc_params;
1966         int i, num, tc, inq;
1967         struct pp2_hif *hif;
1968         unsigned int core_id = rte_lcore_id();
1969
1970         if (core_id == LCORE_ID_ANY)
1971                 core_id = rte_get_main_lcore();
1972
1973         if (!q)
1974                 return;
1975
1976         hif = mrvl_get_hif(q->priv, core_id);
1977
1978         if (!hif)
1979                 return;
1980
1981         tc = q->priv->rxq_map[q->queue_id].tc;
1982         inq = q->priv->rxq_map[q->queue_id].inq;
1983         tc_params = &q->priv->ppio_params.inqs_params.tcs_params[tc];
1984         num = tc_params->inqs_params[inq].size;
1985         for (i = 0; i < num; i++) {
1986                 struct pp2_buff_inf inf;
1987                 uint64_t addr;
1988
1989                 pp2_bpool_get_buff(hif, q->priv->bpool, &inf);
1990                 addr = cookie_addr_high | inf.cookie;
1991                 rte_pktmbuf_free((struct rte_mbuf *)addr);
1992         }
1993
1994         rte_free(q);
1995 }
1996
1997 /**
1998  * DPDK callback to configure the transmit queue.
1999  *
2000  * @param dev
2001  *   Pointer to Ethernet device structure.
2002  * @param idx
2003  *   Transmit queue index.
2004  * @param desc
2005  *   Number of descriptors to configure in the queue.
2006  * @param socket
2007  *   NUMA socket on which memory must be allocated.
2008  * @param conf
2009  *   Tx queue configuration parameters.
2010  *
2011  * @return
2012  *   0 on success, negative error value otherwise.
2013  */
2014 static int
2015 mrvl_tx_queue_setup(struct rte_eth_dev *dev, uint16_t idx, uint16_t desc,
2016                     unsigned int socket,
2017                     const struct rte_eth_txconf *conf)
2018 {
2019         struct mrvl_priv *priv = dev->data->dev_private;
2020         struct mrvl_txq *txq;
2021
2022         if (dev->data->tx_queues[idx]) {
2023                 rte_free(dev->data->tx_queues[idx]);
2024                 dev->data->tx_queues[idx] = NULL;
2025         }
2026
2027         txq = rte_zmalloc_socket("txq", sizeof(*txq), 0, socket);
2028         if (!txq)
2029                 return -ENOMEM;
2030
2031         txq->priv = priv;
2032         txq->queue_id = idx;
2033         txq->port_id = dev->data->port_id;
2034         txq->tx_deferred_start = conf->tx_deferred_start;
2035         dev->data->tx_queues[idx] = txq;
2036
2037         priv->ppio_params.outqs_params.outqs_params[idx].size = desc;
2038
2039         return 0;
2040 }
2041
2042 /**
2043  * DPDK callback to release the transmit queue.
2044  *
2045  * @param txq
2046  *   Generic transmit queue pointer.
2047  */
2048 static void
2049 mrvl_tx_queue_release(void *txq)
2050 {
2051         struct mrvl_txq *q = txq;
2052
2053         if (!q)
2054                 return;
2055
2056         rte_free(q);
2057 }
2058
2059 /**
2060  * DPDK callback to get flow control configuration.
2061  *
2062  * @param dev
2063  *  Pointer to Ethernet device structure.
2064  * @param fc_conf
2065  *  Pointer to the flow control configuration.
2066  *
2067  * @return
2068  *  0 on success, negative error value otherwise.
2069  */
2070 static int
2071 mrvl_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2072 {
2073         struct mrvl_priv *priv = dev->data->dev_private;
2074         int ret, en;
2075
2076         if (!priv)
2077                 return -EPERM;
2078
2079         ret = pp2_ppio_get_rx_pause(priv->ppio, &en);
2080         if (ret) {
2081                 MRVL_LOG(ERR, "Failed to read rx pause state");
2082                 return ret;
2083         }
2084
2085         fc_conf->mode = en ? RTE_FC_RX_PAUSE : RTE_FC_NONE;
2086
2087         ret = pp2_ppio_get_tx_pause(priv->ppio, &en);
2088         if (ret) {
2089                 MRVL_LOG(ERR, "Failed to read tx pause state");
2090                 return ret;
2091         }
2092
2093         if (en) {
2094                 if (fc_conf->mode == RTE_FC_NONE)
2095                         fc_conf->mode = RTE_FC_TX_PAUSE;
2096                 else
2097                         fc_conf->mode = RTE_FC_FULL;
2098         }
2099
2100         return 0;
2101 }
2102
2103 /**
2104  * DPDK callback to set flow control configuration.
2105  *
2106  * @param dev
2107  *  Pointer to Ethernet device structure.
2108  * @param fc_conf
2109  *  Pointer to the flow control configuration.
2110  *
2111  * @return
2112  *  0 on success, negative error value otherwise.
2113  */
2114 static int
2115 mrvl_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
2116 {
2117         struct mrvl_priv *priv = dev->data->dev_private;
2118         struct pp2_ppio_tx_pause_params mrvl_pause_params;
2119         int ret;
2120         int rx_en, tx_en;
2121
2122         if (!priv)
2123                 return -EPERM;
2124
2125         if (fc_conf->high_water ||
2126             fc_conf->low_water ||
2127             fc_conf->pause_time ||
2128             fc_conf->mac_ctrl_frame_fwd ||
2129             fc_conf->autoneg) {
2130                 MRVL_LOG(ERR, "Flowctrl parameter is not supported");
2131
2132                 return -EINVAL;
2133         }
2134
2135         switch (fc_conf->mode) {
2136         case RTE_FC_FULL:
2137                 rx_en = 1;
2138                 tx_en = 1;
2139                 break;
2140         case RTE_FC_TX_PAUSE:
2141                 rx_en = 0;
2142                 tx_en = 1;
2143                 break;
2144         case RTE_FC_RX_PAUSE:
2145                 rx_en = 1;
2146                 tx_en = 0;
2147                 break;
2148         case RTE_FC_NONE:
2149                 rx_en = 0;
2150                 tx_en = 0;
2151                 break;
2152         default:
2153                 MRVL_LOG(ERR, "Incorrect Flow control flag (%d)",
2154                          fc_conf->mode);
2155                 return -EINVAL;
2156         }
2157
2158         /* Set RX flow control */
2159         ret = pp2_ppio_set_rx_pause(priv->ppio, rx_en);
2160         if (ret) {
2161                 MRVL_LOG(ERR, "Failed to change RX flowctrl");
2162                 return ret;
2163         }
2164
2165         /* Set TX flow control */
2166         mrvl_pause_params.en = tx_en;
2167         /* all inqs participate in xon/xoff decision */
2168         mrvl_pause_params.use_tc_pause_inqs = 0;
2169         ret = pp2_ppio_set_tx_pause(priv->ppio, &mrvl_pause_params);
2170         if (ret) {
2171                 MRVL_LOG(ERR, "Failed to change TX flowctrl");
2172                 return ret;
2173         }
2174
2175         return 0;
2176 }
2177
2178 /**
2179  * Update RSS hash configuration
2180  *
2181  * @param dev
2182  *   Pointer to Ethernet device structure.
2183  * @param rss_conf
2184  *   Pointer to RSS configuration.
2185  *
2186  * @return
2187  *   0 on success, negative error value otherwise.
2188  */
2189 static int
2190 mrvl_rss_hash_update(struct rte_eth_dev *dev,
2191                      struct rte_eth_rss_conf *rss_conf)
2192 {
2193         struct mrvl_priv *priv = dev->data->dev_private;
2194
2195         if (priv->isolated)
2196                 return -ENOTSUP;
2197
2198         return mrvl_configure_rss(priv, rss_conf);
2199 }
2200
2201 /**
2202  * DPDK callback to get RSS hash configuration.
2203  *
2204  * @param dev
2205  *   Pointer to Ethernet device structure.
2206  * @rss_conf
2207  *   Pointer to RSS configuration.
2208  *
2209  * @return
2210  *   Always 0.
2211  */
2212 static int
2213 mrvl_rss_hash_conf_get(struct rte_eth_dev *dev,
2214                        struct rte_eth_rss_conf *rss_conf)
2215 {
2216         struct mrvl_priv *priv = dev->data->dev_private;
2217         enum pp2_ppio_hash_type hash_type =
2218                 priv->ppio_params.inqs_params.hash_type;
2219
2220         rss_conf->rss_key = NULL;
2221
2222         if (hash_type == PP2_PPIO_HASH_T_NONE)
2223                 rss_conf->rss_hf = 0;
2224         else if (hash_type == PP2_PPIO_HASH_T_2_TUPLE)
2225                 rss_conf->rss_hf = ETH_RSS_IPV4;
2226         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && priv->rss_hf_tcp)
2227                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_TCP;
2228         else if (hash_type == PP2_PPIO_HASH_T_5_TUPLE && !priv->rss_hf_tcp)
2229                 rss_conf->rss_hf = ETH_RSS_NONFRAG_IPV4_UDP;
2230
2231         return 0;
2232 }
2233
2234 /**
2235  * DPDK callback to get rte_flow callbacks.
2236  *
2237  * @param dev
2238  *   Pointer to the device structure.
2239  * @param filer_type
2240  *   Flow filter type.
2241  * @param filter_op
2242  *   Flow filter operation.
2243  * @param arg
2244  *   Pointer to pass the flow ops.
2245  *
2246  * @return
2247  *   0 on success, negative error value otherwise.
2248  */
2249 static int
2250 mrvl_eth_filter_ctrl(struct rte_eth_dev *dev __rte_unused,
2251                      enum rte_filter_type filter_type,
2252                      enum rte_filter_op filter_op, void *arg)
2253 {
2254         switch (filter_type) {
2255         case RTE_ETH_FILTER_GENERIC:
2256                 if (filter_op != RTE_ETH_FILTER_GET)
2257                         return -EINVAL;
2258                 *(const void **)arg = &mrvl_flow_ops;
2259                 return 0;
2260         default:
2261                 MRVL_LOG(WARNING, "Filter type (%d) not supported",
2262                                 filter_type);
2263                 return -EINVAL;
2264         }
2265 }
2266
2267 /**
2268  * DPDK callback to get rte_mtr callbacks.
2269  *
2270  * @param dev
2271  *   Pointer to the device structure.
2272  * @param ops
2273  *   Pointer to pass the mtr ops.
2274  *
2275  * @return
2276  *   Always 0.
2277  */
2278 static int
2279 mrvl_mtr_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
2280 {
2281         *(const void **)ops = &mrvl_mtr_ops;
2282
2283         return 0;
2284 }
2285
2286 /**
2287  * DPDK callback to get rte_tm callbacks.
2288  *
2289  * @param dev
2290  *   Pointer to the device structure.
2291  * @param ops
2292  *   Pointer to pass the tm ops.
2293  *
2294  * @return
2295  *   Always 0.
2296  */
2297 static int
2298 mrvl_tm_ops_get(struct rte_eth_dev *dev __rte_unused, void *ops)
2299 {
2300         *(const void **)ops = &mrvl_tm_ops;
2301
2302         return 0;
2303 }
2304
2305 static const struct eth_dev_ops mrvl_ops = {
2306         .dev_configure = mrvl_dev_configure,
2307         .dev_start = mrvl_dev_start,
2308         .dev_stop = mrvl_dev_stop,
2309         .dev_set_link_up = mrvl_dev_set_link_up,
2310         .dev_set_link_down = mrvl_dev_set_link_down,
2311         .dev_close = mrvl_dev_close,
2312         .link_update = mrvl_link_update,
2313         .promiscuous_enable = mrvl_promiscuous_enable,
2314         .allmulticast_enable = mrvl_allmulticast_enable,
2315         .promiscuous_disable = mrvl_promiscuous_disable,
2316         .allmulticast_disable = mrvl_allmulticast_disable,
2317         .mac_addr_remove = mrvl_mac_addr_remove,
2318         .mac_addr_add = mrvl_mac_addr_add,
2319         .mac_addr_set = mrvl_mac_addr_set,
2320         .mtu_set = mrvl_mtu_set,
2321         .stats_get = mrvl_stats_get,
2322         .stats_reset = mrvl_stats_reset,
2323         .xstats_get = mrvl_xstats_get,
2324         .xstats_reset = mrvl_xstats_reset,
2325         .xstats_get_names = mrvl_xstats_get_names,
2326         .dev_infos_get = mrvl_dev_infos_get,
2327         .dev_supported_ptypes_get = mrvl_dev_supported_ptypes_get,
2328         .rxq_info_get = mrvl_rxq_info_get,
2329         .txq_info_get = mrvl_txq_info_get,
2330         .vlan_filter_set = mrvl_vlan_filter_set,
2331         .vlan_offload_set = mrvl_vlan_offload_set,
2332         .tx_queue_start = mrvl_tx_queue_start,
2333         .tx_queue_stop = mrvl_tx_queue_stop,
2334         .rx_queue_setup = mrvl_rx_queue_setup,
2335         .rx_queue_release = mrvl_rx_queue_release,
2336         .tx_queue_setup = mrvl_tx_queue_setup,
2337         .tx_queue_release = mrvl_tx_queue_release,
2338         .flow_ctrl_get = mrvl_flow_ctrl_get,
2339         .flow_ctrl_set = mrvl_flow_ctrl_set,
2340         .rss_hash_update = mrvl_rss_hash_update,
2341         .rss_hash_conf_get = mrvl_rss_hash_conf_get,
2342         .filter_ctrl = mrvl_eth_filter_ctrl,
2343         .mtr_ops_get = mrvl_mtr_ops_get,
2344         .tm_ops_get = mrvl_tm_ops_get,
2345 };
2346
2347 /**
2348  * Return packet type information and l3/l4 offsets.
2349  *
2350  * @param desc
2351  *   Pointer to the received packet descriptor.
2352  * @param l3_offset
2353  *   l3 packet offset.
2354  * @param l4_offset
2355  *   l4 packet offset.
2356  *
2357  * @return
2358  *   Packet type information.
2359  */
2360 static inline uint64_t
2361 mrvl_desc_to_packet_type_and_offset(struct pp2_ppio_desc *desc,
2362                                     uint8_t *l3_offset, uint8_t *l4_offset)
2363 {
2364         enum pp2_inq_l3_type l3_type;
2365         enum pp2_inq_l4_type l4_type;
2366         enum pp2_inq_vlan_tag vlan_tag;
2367         uint64_t packet_type;
2368
2369         pp2_ppio_inq_desc_get_l3_info(desc, &l3_type, l3_offset);
2370         pp2_ppio_inq_desc_get_l4_info(desc, &l4_type, l4_offset);
2371         pp2_ppio_inq_desc_get_vlan_tag(desc, &vlan_tag);
2372
2373         packet_type = RTE_PTYPE_L2_ETHER;
2374
2375         switch (vlan_tag) {
2376         case PP2_INQ_VLAN_TAG_SINGLE:
2377                 packet_type |= RTE_PTYPE_L2_ETHER_VLAN;
2378                 break;
2379         case PP2_INQ_VLAN_TAG_DOUBLE:
2380         case PP2_INQ_VLAN_TAG_TRIPLE:
2381                 packet_type |= RTE_PTYPE_L2_ETHER_QINQ;
2382                 break;
2383         default:
2384                 break;
2385         }
2386
2387         switch (l3_type) {
2388         case PP2_INQ_L3_TYPE_IPV4_NO_OPTS:
2389                 packet_type |= RTE_PTYPE_L3_IPV4;
2390                 break;
2391         case PP2_INQ_L3_TYPE_IPV4_OK:
2392                 packet_type |= RTE_PTYPE_L3_IPV4_EXT;
2393                 break;
2394         case PP2_INQ_L3_TYPE_IPV4_TTL_ZERO:
2395                 packet_type |= RTE_PTYPE_L3_IPV4_EXT_UNKNOWN;
2396                 break;
2397         case PP2_INQ_L3_TYPE_IPV6_NO_EXT:
2398                 packet_type |= RTE_PTYPE_L3_IPV6;
2399                 break;
2400         case PP2_INQ_L3_TYPE_IPV6_EXT:
2401                 packet_type |= RTE_PTYPE_L3_IPV6_EXT;
2402                 break;
2403         case PP2_INQ_L3_TYPE_ARP:
2404                 packet_type |= RTE_PTYPE_L2_ETHER_ARP;
2405                 /*
2406                  * In case of ARP l4_offset is set to wrong value.
2407                  * Set it to proper one so that later on mbuf->l3_len can be
2408                  * calculated subtracting l4_offset and l3_offset.
2409                  */
2410                 *l4_offset = *l3_offset + MRVL_ARP_LENGTH;
2411                 break;
2412         default:
2413                 break;
2414         }
2415
2416         switch (l4_type) {
2417         case PP2_INQ_L4_TYPE_TCP:
2418                 packet_type |= RTE_PTYPE_L4_TCP;
2419                 break;
2420         case PP2_INQ_L4_TYPE_UDP:
2421                 packet_type |= RTE_PTYPE_L4_UDP;
2422                 break;
2423         default:
2424                 break;
2425         }
2426
2427         return packet_type;
2428 }
2429
2430 /**
2431  * Get offload information from the received packet descriptor.
2432  *
2433  * @param desc
2434  *   Pointer to the received packet descriptor.
2435  *
2436  * @return
2437  *   Mbuf offload flags.
2438  */
2439 static inline uint64_t
2440 mrvl_desc_to_ol_flags(struct pp2_ppio_desc *desc)
2441 {
2442         uint64_t flags;
2443         enum pp2_inq_desc_status status;
2444
2445         status = pp2_ppio_inq_desc_get_l3_pkt_error(desc);
2446         if (unlikely(status != PP2_DESC_ERR_OK))
2447                 flags = PKT_RX_IP_CKSUM_BAD;
2448         else
2449                 flags = PKT_RX_IP_CKSUM_GOOD;
2450
2451         status = pp2_ppio_inq_desc_get_l4_pkt_error(desc);
2452         if (unlikely(status != PP2_DESC_ERR_OK))
2453                 flags |= PKT_RX_L4_CKSUM_BAD;
2454         else
2455                 flags |= PKT_RX_L4_CKSUM_GOOD;
2456
2457         return flags;
2458 }
2459
2460 /**
2461  * DPDK callback for receive.
2462  *
2463  * @param rxq
2464  *   Generic pointer to the receive queue.
2465  * @param rx_pkts
2466  *   Array to store received packets.
2467  * @param nb_pkts
2468  *   Maximum number of packets in array.
2469  *
2470  * @return
2471  *   Number of packets successfully received.
2472  */
2473 static uint16_t
2474 mrvl_rx_pkt_burst(void *rxq, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
2475 {
2476         struct mrvl_rxq *q = rxq;
2477         struct pp2_ppio_desc descs[nb_pkts];
2478         struct pp2_bpool *bpool;
2479         int i, ret, rx_done = 0;
2480         int num;
2481         struct pp2_hif *hif;
2482         unsigned int core_id = rte_lcore_id();
2483
2484         hif = mrvl_get_hif(q->priv, core_id);
2485
2486         if (unlikely(!q->priv->ppio || !hif))
2487                 return 0;
2488
2489         bpool = q->priv->bpool;
2490
2491         ret = pp2_ppio_recv(q->priv->ppio, q->priv->rxq_map[q->queue_id].tc,
2492                             q->priv->rxq_map[q->queue_id].inq, descs, &nb_pkts);
2493         if (unlikely(ret < 0))
2494                 return 0;
2495
2496         mrvl_port_bpool_size[bpool->pp2_id][bpool->id][core_id] -= nb_pkts;
2497
2498         for (i = 0; i < nb_pkts; i++) {
2499                 struct rte_mbuf *mbuf;
2500                 uint8_t l3_offset, l4_offset;
2501                 enum pp2_inq_desc_status status;
2502                 uint64_t addr;
2503
2504                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2505                         struct pp2_ppio_desc *pref_desc;
2506                         u64 pref_addr;
2507
2508                         pref_desc = &descs[i + MRVL_MUSDK_PREFETCH_SHIFT];
2509                         pref_addr = cookie_addr_high |
2510                                     pp2_ppio_inq_desc_get_cookie(pref_desc);
2511                         rte_mbuf_prefetch_part1((struct rte_mbuf *)(pref_addr));
2512                         rte_mbuf_prefetch_part2((struct rte_mbuf *)(pref_addr));
2513                 }
2514
2515                 addr = cookie_addr_high |
2516                        pp2_ppio_inq_desc_get_cookie(&descs[i]);
2517                 mbuf = (struct rte_mbuf *)addr;
2518                 rte_pktmbuf_reset(mbuf);
2519
2520                 /* drop packet in case of mac, overrun or resource error */
2521                 status = pp2_ppio_inq_desc_get_l2_pkt_error(&descs[i]);
2522                 if (unlikely(status != PP2_DESC_ERR_OK)) {
2523                         struct pp2_buff_inf binf = {
2524                                 .addr = rte_mbuf_data_iova_default(mbuf),
2525                                 .cookie = (uint64_t)mbuf,
2526                         };
2527
2528                         pp2_bpool_put_buff(hif, bpool, &binf);
2529                         mrvl_port_bpool_size
2530                                 [bpool->pp2_id][bpool->id][core_id]++;
2531                         q->drop_mac++;
2532                         continue;
2533                 }
2534
2535                 mbuf->data_off += MRVL_PKT_EFFEC_OFFS;
2536                 mbuf->pkt_len = pp2_ppio_inq_desc_get_pkt_len(&descs[i]);
2537                 mbuf->data_len = mbuf->pkt_len;
2538                 mbuf->port = q->port_id;
2539                 mbuf->packet_type =
2540                         mrvl_desc_to_packet_type_and_offset(&descs[i],
2541                                                             &l3_offset,
2542                                                             &l4_offset);
2543                 mbuf->l2_len = l3_offset;
2544                 mbuf->l3_len = l4_offset - l3_offset;
2545
2546                 if (likely(q->cksum_enabled))
2547                         mbuf->ol_flags = mrvl_desc_to_ol_flags(&descs[i]);
2548
2549                 rx_pkts[rx_done++] = mbuf;
2550                 q->bytes_recv += mbuf->pkt_len;
2551         }
2552
2553         if (rte_spinlock_trylock(&q->priv->lock) == 1) {
2554                 num = mrvl_get_bpool_size(bpool->pp2_id, bpool->id);
2555
2556                 if (unlikely(num <= q->priv->bpool_min_size ||
2557                              (!rx_done && num < q->priv->bpool_init_size))) {
2558                         mrvl_fill_bpool(q, MRVL_BURST_SIZE);
2559                 } else if (unlikely(num > q->priv->bpool_max_size)) {
2560                         int i;
2561                         int pkt_to_remove = num - q->priv->bpool_init_size;
2562                         struct rte_mbuf *mbuf;
2563                         struct pp2_buff_inf buff;
2564
2565                         for (i = 0; i < pkt_to_remove; i++) {
2566                                 ret = pp2_bpool_get_buff(hif, bpool, &buff);
2567                                 if (ret)
2568                                         break;
2569                                 mbuf = (struct rte_mbuf *)
2570                                         (cookie_addr_high | buff.cookie);
2571                                 rte_pktmbuf_free(mbuf);
2572                         }
2573                         mrvl_port_bpool_size
2574                                 [bpool->pp2_id][bpool->id][core_id] -= i;
2575                 }
2576                 rte_spinlock_unlock(&q->priv->lock);
2577         }
2578
2579         return rx_done;
2580 }
2581
2582 /**
2583  * Prepare offload information.
2584  *
2585  * @param ol_flags
2586  *   Offload flags.
2587  * @param l3_type
2588  *   Pointer to the pp2_ouq_l3_type structure.
2589  * @param l4_type
2590  *   Pointer to the pp2_outq_l4_type structure.
2591  * @param gen_l3_cksum
2592  *   Will be set to 1 in case l3 checksum is computed.
2593  * @param l4_cksum
2594  *   Will be set to 1 in case l4 checksum is computed.
2595  */
2596 static inline void
2597 mrvl_prepare_proto_info(uint64_t ol_flags,
2598                         enum pp2_outq_l3_type *l3_type,
2599                         enum pp2_outq_l4_type *l4_type,
2600                         int *gen_l3_cksum,
2601                         int *gen_l4_cksum)
2602 {
2603         /*
2604          * Based on ol_flags prepare information
2605          * for pp2_ppio_outq_desc_set_proto_info() which setups descriptor
2606          * for offloading.
2607          * in most of the checksum cases ipv4 must be set, so this is the
2608          * default value
2609          */
2610         *l3_type = PP2_OUTQ_L3_TYPE_IPV4;
2611         *gen_l3_cksum = ol_flags & PKT_TX_IP_CKSUM ? 1 : 0;
2612
2613         if (ol_flags & PKT_TX_IPV6) {
2614                 *l3_type = PP2_OUTQ_L3_TYPE_IPV6;
2615                 /* no checksum for ipv6 header */
2616                 *gen_l3_cksum = 0;
2617         }
2618
2619         if ((ol_flags & PKT_TX_L4_MASK) == PKT_TX_TCP_CKSUM) {
2620                 *l4_type = PP2_OUTQ_L4_TYPE_TCP;
2621                 *gen_l4_cksum = 1;
2622         } else if ((ol_flags & PKT_TX_L4_MASK) ==  PKT_TX_UDP_CKSUM) {
2623                 *l4_type = PP2_OUTQ_L4_TYPE_UDP;
2624                 *gen_l4_cksum = 1;
2625         } else {
2626                 *l4_type = PP2_OUTQ_L4_TYPE_OTHER;
2627                 /* no checksum for other type */
2628                 *gen_l4_cksum = 0;
2629         }
2630 }
2631
2632 /**
2633  * Release already sent buffers to bpool (buffer-pool).
2634  *
2635  * @param ppio
2636  *   Pointer to the port structure.
2637  * @param hif
2638  *   Pointer to the MUSDK hardware interface.
2639  * @param sq
2640  *   Pointer to the shadow queue.
2641  * @param qid
2642  *   Queue id number.
2643  * @param force
2644  *   Force releasing packets.
2645  */
2646 static inline void
2647 mrvl_free_sent_buffers(struct pp2_ppio *ppio, struct pp2_hif *hif,
2648                        unsigned int core_id, struct mrvl_shadow_txq *sq,
2649                        int qid, int force)
2650 {
2651         struct buff_release_entry *entry;
2652         uint16_t nb_done = 0, num = 0, skip_bufs = 0;
2653         int i;
2654
2655         pp2_ppio_get_num_outq_done(ppio, hif, qid, &nb_done);
2656
2657         sq->num_to_release += nb_done;
2658
2659         if (likely(!force &&
2660                    sq->num_to_release < MRVL_PP2_BUF_RELEASE_BURST_SIZE))
2661                 return;
2662
2663         nb_done = sq->num_to_release;
2664         sq->num_to_release = 0;
2665
2666         for (i = 0; i < nb_done; i++) {
2667                 entry = &sq->ent[sq->tail + num];
2668                 if (unlikely(!entry->buff.addr)) {
2669                         MRVL_LOG(ERR,
2670                                 "Shadow memory @%d: cookie(%lx), pa(%lx)!",
2671                                 sq->tail, (u64)entry->buff.cookie,
2672                                 (u64)entry->buff.addr);
2673                         skip_bufs = 1;
2674                         goto skip;
2675                 }
2676
2677                 if (unlikely(!entry->bpool)) {
2678                         struct rte_mbuf *mbuf;
2679
2680                         mbuf = (struct rte_mbuf *)entry->buff.cookie;
2681                         rte_pktmbuf_free(mbuf);
2682                         skip_bufs = 1;
2683                         goto skip;
2684                 }
2685
2686                 mrvl_port_bpool_size
2687                         [entry->bpool->pp2_id][entry->bpool->id][core_id]++;
2688                 num++;
2689                 if (unlikely(sq->tail + num == MRVL_PP2_TX_SHADOWQ_SIZE))
2690                         goto skip;
2691                 continue;
2692 skip:
2693                 if (likely(num))
2694                         pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2695                 num += skip_bufs;
2696                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2697                 sq->size -= num;
2698                 num = 0;
2699                 skip_bufs = 0;
2700         }
2701
2702         if (likely(num)) {
2703                 pp2_bpool_put_buffs(hif, &sq->ent[sq->tail], &num);
2704                 sq->tail = (sq->tail + num) & MRVL_PP2_TX_SHADOWQ_MASK;
2705                 sq->size -= num;
2706         }
2707 }
2708
2709 /**
2710  * DPDK callback for transmit.
2711  *
2712  * @param txq
2713  *   Generic pointer transmit queue.
2714  * @param tx_pkts
2715  *   Packets to transmit.
2716  * @param nb_pkts
2717  *   Number of packets in array.
2718  *
2719  * @return
2720  *   Number of packets successfully transmitted.
2721  */
2722 static uint16_t
2723 mrvl_tx_pkt_burst(void *txq, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)
2724 {
2725         struct mrvl_txq *q = txq;
2726         struct mrvl_shadow_txq *sq;
2727         struct pp2_hif *hif;
2728         struct pp2_ppio_desc descs[nb_pkts];
2729         unsigned int core_id = rte_lcore_id();
2730         int i, bytes_sent = 0;
2731         uint16_t num, sq_free_size;
2732         uint64_t addr;
2733
2734         hif = mrvl_get_hif(q->priv, core_id);
2735         sq = &q->shadow_txqs[core_id];
2736
2737         if (unlikely(!q->priv->ppio || !hif))
2738                 return 0;
2739
2740         if (sq->size)
2741                 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
2742                                        sq, q->queue_id, 0);
2743
2744         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
2745         if (unlikely(nb_pkts > sq_free_size))
2746                 nb_pkts = sq_free_size;
2747
2748         for (i = 0; i < nb_pkts; i++) {
2749                 struct rte_mbuf *mbuf = tx_pkts[i];
2750                 int gen_l3_cksum, gen_l4_cksum;
2751                 enum pp2_outq_l3_type l3_type;
2752                 enum pp2_outq_l4_type l4_type;
2753
2754                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2755                         struct rte_mbuf *pref_pkt_hdr;
2756
2757                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
2758                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
2759                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
2760                 }
2761
2762                 mrvl_fill_shadowq(sq, mbuf);
2763                 mrvl_fill_desc(&descs[i], mbuf);
2764
2765                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
2766                 /*
2767                  * in case unsupported ol_flags were passed
2768                  * do not update descriptor offload information
2769                  */
2770                 if (!(mbuf->ol_flags & MRVL_TX_PKT_OFFLOADS))
2771                         continue;
2772                 mrvl_prepare_proto_info(mbuf->ol_flags, &l3_type, &l4_type,
2773                                         &gen_l3_cksum, &gen_l4_cksum);
2774
2775                 pp2_ppio_outq_desc_set_proto_info(&descs[i], l3_type, l4_type,
2776                                                   mbuf->l2_len,
2777                                                   mbuf->l2_len + mbuf->l3_len,
2778                                                   gen_l3_cksum, gen_l4_cksum);
2779         }
2780
2781         num = nb_pkts;
2782         pp2_ppio_send(q->priv->ppio, hif, q->queue_id, descs, &nb_pkts);
2783         /* number of packets that were not sent */
2784         if (unlikely(num > nb_pkts)) {
2785                 for (i = nb_pkts; i < num; i++) {
2786                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
2787                                 MRVL_PP2_TX_SHADOWQ_MASK;
2788                         addr = sq->ent[sq->head].buff.cookie;
2789                         bytes_sent -=
2790                                 rte_pktmbuf_pkt_len((struct rte_mbuf *)addr);
2791                 }
2792                 sq->size -= num - nb_pkts;
2793         }
2794
2795         q->bytes_sent += bytes_sent;
2796
2797         return nb_pkts;
2798 }
2799
2800 /** DPDK callback for S/G transmit.
2801  *
2802  * @param txq
2803  *   Generic pointer transmit queue.
2804  * @param tx_pkts
2805  *   Packets to transmit.
2806  * @param nb_pkts
2807  *   Number of packets in array.
2808  *
2809  * @return
2810  *   Number of packets successfully transmitted.
2811  */
2812 static uint16_t
2813 mrvl_tx_sg_pkt_burst(void *txq, struct rte_mbuf **tx_pkts,
2814                      uint16_t nb_pkts)
2815 {
2816         struct mrvl_txq *q = txq;
2817         struct mrvl_shadow_txq *sq;
2818         struct pp2_hif *hif;
2819         struct pp2_ppio_desc descs[nb_pkts * PP2_PPIO_DESC_NUM_FRAGS];
2820         struct pp2_ppio_sg_pkts pkts;
2821         uint8_t frags[nb_pkts];
2822         unsigned int core_id = rte_lcore_id();
2823         int i, j, bytes_sent = 0;
2824         int tail, tail_first;
2825         uint16_t num, sq_free_size;
2826         uint16_t nb_segs, total_descs = 0;
2827         uint64_t addr;
2828
2829         hif = mrvl_get_hif(q->priv, core_id);
2830         sq = &q->shadow_txqs[core_id];
2831         pkts.frags = frags;
2832         pkts.num = 0;
2833
2834         if (unlikely(!q->priv->ppio || !hif))
2835                 return 0;
2836
2837         if (sq->size)
2838                 mrvl_free_sent_buffers(q->priv->ppio, hif, core_id,
2839                                        sq, q->queue_id, 0);
2840
2841         /* Save shadow queue free size */
2842         sq_free_size = MRVL_PP2_TX_SHADOWQ_SIZE - sq->size - 1;
2843
2844         tail = 0;
2845         for (i = 0; i < nb_pkts; i++) {
2846                 struct rte_mbuf *mbuf = tx_pkts[i];
2847                 struct rte_mbuf *seg = NULL;
2848                 int gen_l3_cksum, gen_l4_cksum;
2849                 enum pp2_outq_l3_type l3_type;
2850                 enum pp2_outq_l4_type l4_type;
2851
2852                 nb_segs = mbuf->nb_segs;
2853                 tail_first = tail;
2854                 total_descs += nb_segs;
2855
2856                 /*
2857                  * Check if total_descs does not exceed
2858                  * shadow queue free size
2859                  */
2860                 if (unlikely(total_descs > sq_free_size)) {
2861                         total_descs -= nb_segs;
2862                         break;
2863                 }
2864
2865                 /* Check if nb_segs does not exceed the max nb of desc per
2866                  * fragmented packet
2867                  */
2868                 if (nb_segs > PP2_PPIO_DESC_NUM_FRAGS) {
2869                         total_descs -= nb_segs;
2870                         RTE_LOG(ERR, PMD,
2871                                 "Too many segments. Packet won't be sent.\n");
2872                         break;
2873                 }
2874
2875                 if (likely(nb_pkts - i > MRVL_MUSDK_PREFETCH_SHIFT)) {
2876                         struct rte_mbuf *pref_pkt_hdr;
2877
2878                         pref_pkt_hdr = tx_pkts[i + MRVL_MUSDK_PREFETCH_SHIFT];
2879                         rte_mbuf_prefetch_part1(pref_pkt_hdr);
2880                         rte_mbuf_prefetch_part2(pref_pkt_hdr);
2881                 }
2882
2883                 pkts.frags[pkts.num] = nb_segs;
2884                 pkts.num++;
2885
2886                 seg = mbuf;
2887                 for (j = 0; j < nb_segs - 1; j++) {
2888                         /* For the subsequent segments, set shadow queue
2889                          * buffer to NULL
2890                          */
2891                         mrvl_fill_shadowq(sq, NULL);
2892                         mrvl_fill_desc(&descs[tail], seg);
2893
2894                         tail++;
2895                         seg = seg->next;
2896                 }
2897                 /* Put first mbuf info in last shadow queue entry */
2898                 mrvl_fill_shadowq(sq, mbuf);
2899                 /* Update descriptor with last segment */
2900                 mrvl_fill_desc(&descs[tail++], seg);
2901
2902                 bytes_sent += rte_pktmbuf_pkt_len(mbuf);
2903                 /* In case unsupported ol_flags were passed
2904                  * do not update descriptor offload information
2905                  */
2906                 if (!(mbuf->ol_flags & MRVL_TX_PKT_OFFLOADS))
2907                         continue;
2908                 mrvl_prepare_proto_info(mbuf->ol_flags, &l3_type, &l4_type,
2909                                         &gen_l3_cksum, &gen_l4_cksum);
2910
2911                 pp2_ppio_outq_desc_set_proto_info(&descs[tail_first], l3_type,
2912                                                   l4_type, mbuf->l2_len,
2913                                                   mbuf->l2_len + mbuf->l3_len,
2914                                                   gen_l3_cksum, gen_l4_cksum);
2915         }
2916
2917         num = total_descs;
2918         pp2_ppio_send_sg(q->priv->ppio, hif, q->queue_id, descs,
2919                          &total_descs, &pkts);
2920         /* number of packets that were not sent */
2921         if (unlikely(num > total_descs)) {
2922                 for (i = total_descs; i < num; i++) {
2923                         sq->head = (MRVL_PP2_TX_SHADOWQ_SIZE + sq->head - 1) &
2924                                 MRVL_PP2_TX_SHADOWQ_MASK;
2925
2926                         addr = sq->ent[sq->head].buff.cookie;
2927                         if (addr)
2928                                 bytes_sent -=
2929                                         rte_pktmbuf_pkt_len((struct rte_mbuf *)
2930                                                 (cookie_addr_high | addr));
2931                 }
2932                 sq->size -= num - total_descs;
2933                 nb_pkts = pkts.num;
2934         }
2935
2936         q->bytes_sent += bytes_sent;
2937
2938         return nb_pkts;
2939 }
2940
2941 /**
2942  * Create private device structure.
2943  *
2944  * @param dev_name
2945  *   Pointer to the port name passed in the initialization parameters.
2946  *
2947  * @return
2948  *   Pointer to the newly allocated private device structure.
2949  */
2950 static struct mrvl_priv *
2951 mrvl_priv_create(const char *dev_name)
2952 {
2953         struct pp2_bpool_params bpool_params;
2954         char match[MRVL_MATCH_LEN];
2955         struct mrvl_priv *priv;
2956         int ret, bpool_bit;
2957
2958         priv = rte_zmalloc_socket(dev_name, sizeof(*priv), 0, rte_socket_id());
2959         if (!priv)
2960                 return NULL;
2961
2962         ret = pp2_netdev_get_ppio_info((char *)(uintptr_t)dev_name,
2963                                        &priv->pp_id, &priv->ppio_id);
2964         if (ret)
2965                 goto out_free_priv;
2966
2967         bpool_bit = mrvl_reserve_bit(&used_bpools[priv->pp_id],
2968                                      PP2_BPOOL_NUM_POOLS);
2969         if (bpool_bit < 0)
2970                 goto out_free_priv;
2971         priv->bpool_bit = bpool_bit;
2972
2973         snprintf(match, sizeof(match), "pool-%d:%d", priv->pp_id,
2974                  priv->bpool_bit);
2975         memset(&bpool_params, 0, sizeof(bpool_params));
2976         bpool_params.match = match;
2977         bpool_params.buff_len = MRVL_PKT_SIZE_MAX + MRVL_PKT_EFFEC_OFFS;
2978         ret = pp2_bpool_init(&bpool_params, &priv->bpool);
2979         if (ret)
2980                 goto out_clear_bpool_bit;
2981
2982         priv->ppio_params.type = PP2_PPIO_T_NIC;
2983         rte_spinlock_init(&priv->lock);
2984
2985         return priv;
2986 out_clear_bpool_bit:
2987         used_bpools[priv->pp_id] &= ~(1 << priv->bpool_bit);
2988 out_free_priv:
2989         rte_free(priv);
2990         return NULL;
2991 }
2992
2993 /**
2994  * Create device representing Ethernet port.
2995  *
2996  * @param name
2997  *   Pointer to the port's name.
2998  *
2999  * @return
3000  *   0 on success, negative error value otherwise.
3001  */
3002 static int
3003 mrvl_eth_dev_create(struct rte_vdev_device *vdev, const char *name)
3004 {
3005         int ret, fd = socket(AF_INET, SOCK_DGRAM, 0);
3006         struct rte_eth_dev *eth_dev;
3007         struct mrvl_priv *priv;
3008         struct ifreq req;
3009
3010         eth_dev = rte_eth_dev_allocate(name);
3011         if (!eth_dev)
3012                 return -ENOMEM;
3013
3014         priv = mrvl_priv_create(name);
3015         if (!priv) {
3016                 ret = -ENOMEM;
3017                 goto out_free;
3018         }
3019         eth_dev->data->dev_private = priv;
3020
3021         eth_dev->data->mac_addrs =
3022                 rte_zmalloc("mac_addrs",
3023                             RTE_ETHER_ADDR_LEN * MRVL_MAC_ADDRS_MAX, 0);
3024         if (!eth_dev->data->mac_addrs) {
3025                 MRVL_LOG(ERR, "Failed to allocate space for eth addrs");
3026                 ret = -ENOMEM;
3027                 goto out_free;
3028         }
3029
3030         memset(&req, 0, sizeof(req));
3031         strcpy(req.ifr_name, name);
3032         ret = ioctl(fd, SIOCGIFHWADDR, &req);
3033         if (ret)
3034                 goto out_free;
3035
3036         memcpy(eth_dev->data->mac_addrs[0].addr_bytes,
3037                req.ifr_addr.sa_data, RTE_ETHER_ADDR_LEN);
3038
3039         eth_dev->device = &vdev->device;
3040         eth_dev->rx_pkt_burst = mrvl_rx_pkt_burst;
3041         mrvl_set_tx_function(eth_dev);
3042         eth_dev->dev_ops = &mrvl_ops;
3043         eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
3044
3045         eth_dev->data->dev_link.link_status = ETH_LINK_UP;
3046
3047         rte_eth_dev_probing_finish(eth_dev);
3048         return 0;
3049 out_free:
3050         rte_eth_dev_release_port(eth_dev);
3051
3052         return ret;
3053 }
3054
3055 /**
3056  * Callback used by rte_kvargs_process() during argument parsing.
3057  *
3058  * @param key
3059  *   Pointer to the parsed key (unused).
3060  * @param value
3061  *   Pointer to the parsed value.
3062  * @param extra_args
3063  *   Pointer to the extra arguments which contains address of the
3064  *   table of pointers to parsed interface names.
3065  *
3066  * @return
3067  *   Always 0.
3068  */
3069 static int
3070 mrvl_get_ifnames(const char *key __rte_unused, const char *value,
3071                  void *extra_args)
3072 {
3073         struct mrvl_ifnames *ifnames = extra_args;
3074
3075         ifnames->names[ifnames->idx++] = value;
3076
3077         return 0;
3078 }
3079
3080 /**
3081  * DPDK callback to register the virtual device.
3082  *
3083  * @param vdev
3084  *   Pointer to the virtual device.
3085  *
3086  * @return
3087  *   0 on success, negative error value otherwise.
3088  */
3089 static int
3090 rte_pmd_mrvl_probe(struct rte_vdev_device *vdev)
3091 {
3092         struct rte_kvargs *kvlist;
3093         struct mrvl_ifnames ifnames;
3094         int ret = -EINVAL;
3095         uint32_t i, ifnum, cfgnum;
3096         const char *params;
3097
3098         params = rte_vdev_device_args(vdev);
3099         if (!params)
3100                 return -EINVAL;
3101
3102         kvlist = rte_kvargs_parse(params, valid_args);
3103         if (!kvlist)
3104                 return -EINVAL;
3105
3106         ifnum = rte_kvargs_count(kvlist, MRVL_IFACE_NAME_ARG);
3107         if (ifnum > RTE_DIM(ifnames.names))
3108                 goto out_free_kvlist;
3109
3110         ifnames.idx = 0;
3111         rte_kvargs_process(kvlist, MRVL_IFACE_NAME_ARG,
3112                            mrvl_get_ifnames, &ifnames);
3113
3114
3115         /*
3116          * The below system initialization should be done only once,
3117          * on the first provided configuration file
3118          */
3119         if (!mrvl_cfg) {
3120                 cfgnum = rte_kvargs_count(kvlist, MRVL_CFG_ARG);
3121                 MRVL_LOG(INFO, "Parsing config file!");
3122                 if (cfgnum > 1) {
3123                         MRVL_LOG(ERR, "Cannot handle more than one config file!");
3124                         goto out_free_kvlist;
3125                 } else if (cfgnum == 1) {
3126                         rte_kvargs_process(kvlist, MRVL_CFG_ARG,
3127                                            mrvl_get_cfg, &mrvl_cfg);
3128                 }
3129         }
3130
3131         if (mrvl_dev_num)
3132                 goto init_devices;
3133
3134         MRVL_LOG(INFO, "Perform MUSDK initializations");
3135
3136         ret = rte_mvep_init(MVEP_MOD_T_PP2, kvlist);
3137         if (ret)
3138                 goto out_free_kvlist;
3139
3140         ret = mrvl_init_pp2();
3141         if (ret) {
3142                 MRVL_LOG(ERR, "Failed to init PP!");
3143                 rte_mvep_deinit(MVEP_MOD_T_PP2);
3144                 goto out_free_kvlist;
3145         }
3146
3147         memset(mrvl_port_bpool_size, 0, sizeof(mrvl_port_bpool_size));
3148         memset(mrvl_port_to_bpool_lookup, 0, sizeof(mrvl_port_to_bpool_lookup));
3149
3150         mrvl_lcore_first = RTE_MAX_LCORE;
3151         mrvl_lcore_last = 0;
3152
3153 init_devices:
3154         for (i = 0; i < ifnum; i++) {
3155                 MRVL_LOG(INFO, "Creating %s", ifnames.names[i]);
3156                 ret = mrvl_eth_dev_create(vdev, ifnames.names[i]);
3157                 if (ret)
3158                         goto out_cleanup;
3159                 mrvl_dev_num++;
3160         }
3161
3162         rte_kvargs_free(kvlist);
3163
3164         return 0;
3165 out_cleanup:
3166         rte_pmd_mrvl_remove(vdev);
3167
3168 out_free_kvlist:
3169         rte_kvargs_free(kvlist);
3170
3171         return ret;
3172 }
3173
3174 /**
3175  * DPDK callback to remove virtual device.
3176  *
3177  * @param vdev
3178  *   Pointer to the removed virtual device.
3179  *
3180  * @return
3181  *   0 on success, negative error value otherwise.
3182  */
3183 static int
3184 rte_pmd_mrvl_remove(struct rte_vdev_device *vdev)
3185 {
3186         uint16_t port_id;
3187         int ret = 0;
3188
3189         RTE_ETH_FOREACH_DEV(port_id) {
3190                 if (rte_eth_devices[port_id].device != &vdev->device)
3191                         continue;
3192                 ret |= rte_eth_dev_close(port_id);
3193         }
3194
3195         return ret == 0 ? 0 : -EIO;
3196 }
3197
3198 static struct rte_vdev_driver pmd_mrvl_drv = {
3199         .probe = rte_pmd_mrvl_probe,
3200         .remove = rte_pmd_mrvl_remove,
3201 };
3202
3203 RTE_PMD_REGISTER_VDEV(net_mvpp2, pmd_mrvl_drv);
3204 RTE_PMD_REGISTER_ALIAS(net_mvpp2, eth_mvpp2);
3205 RTE_LOG_REGISTER(mrvl_logtype, pmd.net.mvpp2, NOTICE);