drivers/net/dpaa/dpaa_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  *   Copyright 2016 Freescale Semiconductor, Inc. All rights reserved.
   4  *   Copyright 2017-2020 NXP
   5  *
   6  */
   7 /* System headers */
   8 #include <stdio.h>
   9 #include <inttypes.h>
  10 #include <unistd.h>
  11 #include <limits.h>
  12 #include <sched.h>
  13 #include <signal.h>
  14 #include <pthread.h>
  15 #include <sys/types.h>
  16 #include <sys/syscall.h>
  17
  18 #include <rte_string_fns.h>
  19 #include <rte_byteorder.h>
  20 #include <rte_common.h>
  21 #include <rte_interrupts.h>
  22 #include <rte_log.h>
  23 #include <rte_debug.h>
  24 #include <rte_pci.h>
  25 #include <rte_atomic.h>
  26 #include <rte_branch_prediction.h>
  27 #include <rte_memory.h>
  28 #include <rte_tailq.h>
  29 #include <rte_eal.h>
  30 #include <rte_alarm.h>
  31 #include <rte_ether.h>
  32 #include <ethdev_driver.h>
  33 #include <rte_malloc.h>
  34 #include <rte_ring.h>
  35
  36 #include <rte_dpaa_bus.h>
  37 #include <rte_dpaa_logs.h>
  38 #include <dpaa_mempool.h>
  39
  40 #include <dpaa_ethdev.h>
  41 #include <dpaa_rxtx.h>
  42 #include <dpaa_flow.h>
  43 #include <rte_pmd_dpaa.h>
  44
  45 #include <fsl_usd.h>
  46 #include <fsl_qman.h>
  47 #include <fsl_bman.h>
  48 #include <fsl_fman.h>
  49 #include <process.h>
  50 #include <fmlib/fm_ext.h>
  51
  52 #define CHECK_INTERVAL         100  /* 100ms */
  53 #define MAX_REPEAT_TIME        90   /* 9s (90 * 100ms) in total */
  54
  55 /* Supported Rx offloads */
  56 static uint64_t dev_rx_offloads_sup =
  57                 DEV_RX_OFFLOAD_JUMBO_FRAME |
  58                 DEV_RX_OFFLOAD_SCATTER;
  59
  60 /* Rx offloads which cannot be disabled */
  61 static uint64_t dev_rx_offloads_nodis =
  62                 DEV_RX_OFFLOAD_IPV4_CKSUM |
  63                 DEV_RX_OFFLOAD_UDP_CKSUM |
  64                 DEV_RX_OFFLOAD_TCP_CKSUM |
  65                 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
  66                 DEV_RX_OFFLOAD_RSS_HASH;
  67
  68 /* Supported Tx offloads */
  69 static uint64_t dev_tx_offloads_sup =
  70                 DEV_TX_OFFLOAD_MT_LOCKFREE |
  71                 DEV_TX_OFFLOAD_MBUF_FAST_FREE;
  72
  73 /* Tx offloads which cannot be disabled */
  74 static uint64_t dev_tx_offloads_nodis =
  75                 DEV_TX_OFFLOAD_IPV4_CKSUM |
  76                 DEV_TX_OFFLOAD_UDP_CKSUM |
  77                 DEV_TX_OFFLOAD_TCP_CKSUM |
  78                 DEV_TX_OFFLOAD_SCTP_CKSUM |
  79                 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
  80                 DEV_TX_OFFLOAD_MULTI_SEGS;
  81
  82 /* Keep track of whether QMAN and BMAN have been globally initialized */
  83 static int is_global_init;
  84 static int fmc_q = 1;   /* Indicates the use of static fmc for distribution */
  85 static int default_q;   /* use default queue - FMC is not executed*/
  86 /* At present we only allow up to 4 push mode queues as default - as each of
  87  * this queue need dedicated portal and we are short of portals.
  88  */
  89 #define DPAA_MAX_PUSH_MODE_QUEUE       8
  90 #define DPAA_DEFAULT_PUSH_MODE_QUEUE   4
  91
  92 static int dpaa_push_mode_max_queue = DPAA_DEFAULT_PUSH_MODE_QUEUE;
  93 static int dpaa_push_queue_idx; /* Queue index which are in push mode*/
  94
  95
  96 /* Per RX FQ Taildrop in frame count */
  97 static unsigned int td_threshold = CGR_RX_PERFQ_THRESH;
  98
  99 /* Per TX FQ Taildrop in frame count, disabled by default */
 100 static unsigned int td_tx_threshold;
 101
 102 struct rte_dpaa_xstats_name_off {
 103         char name[RTE_ETH_XSTATS_NAME_SIZE];
 104         uint32_t offset;
 105 };
 106
 107 static const struct rte_dpaa_xstats_name_off dpaa_xstats_strings[] = {
 108         {"rx_align_err",
 109                 offsetof(struct dpaa_if_stats, raln)},
 110         {"rx_valid_pause",
 111                 offsetof(struct dpaa_if_stats, rxpf)},
 112         {"rx_fcs_err",
 113                 offsetof(struct dpaa_if_stats, rfcs)},
 114         {"rx_vlan_frame",
 115                 offsetof(struct dpaa_if_stats, rvlan)},
 116         {"rx_frame_err",
 117                 offsetof(struct dpaa_if_stats, rerr)},
 118         {"rx_drop_err",
 119                 offsetof(struct dpaa_if_stats, rdrp)},
 120         {"rx_undersized",
 121                 offsetof(struct dpaa_if_stats, rund)},
 122         {"rx_oversize_err",
 123                 offsetof(struct dpaa_if_stats, rovr)},
 124         {"rx_fragment_pkt",
 125                 offsetof(struct dpaa_if_stats, rfrg)},
 126         {"tx_valid_pause",
 127                 offsetof(struct dpaa_if_stats, txpf)},
 128         {"tx_fcs_err",
 129                 offsetof(struct dpaa_if_stats, terr)},
 130         {"tx_vlan_frame",
 131                 offsetof(struct dpaa_if_stats, tvlan)},
 132         {"rx_undersized",
 133                 offsetof(struct dpaa_if_stats, tund)},
 134 };
 135
 136 static struct rte_dpaa_driver rte_dpaa_pmd;
 137
 138 static int
 139 dpaa_eth_dev_info(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info);
 140
 141 static int dpaa_eth_link_update(struct rte_eth_dev *dev,
 142                                 int wait_to_complete __rte_unused);
 143
 144 static void dpaa_interrupt_handler(void *param);
 145
 146 static inline void
 147 dpaa_poll_queue_default_config(struct qm_mcc_initfq *opts)
 148 {
 149         memset(opts, 0, sizeof(struct qm_mcc_initfq));
 150         opts->we_mask = QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_CONTEXTA;
 151         opts->fqd.fq_ctrl = QM_FQCTRL_AVOIDBLOCK | QM_FQCTRL_CTXASTASHING |
 152                            QM_FQCTRL_PREFERINCACHE;
 153         opts->fqd.context_a.stashing.exclusive = 0;
 154         if (dpaa_svr_family != SVR_LS1046A_FAMILY)
 155                 opts->fqd.context_a.stashing.annotation_cl =
 156                                                 DPAA_IF_RX_ANNOTATION_STASH;
 157         opts->fqd.context_a.stashing.data_cl = DPAA_IF_RX_DATA_STASH;
 158         opts->fqd.context_a.stashing.context_cl = DPAA_IF_RX_CONTEXT_STASH;
 159 }
 160
 161 static int
 162 dpaa_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
 163 {
 164         uint32_t frame_size = mtu + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN
 165                                 + VLAN_TAG_SIZE;
 166         uint32_t buffsz = dev->data->min_rx_buf_size - RTE_PKTMBUF_HEADROOM;
 167
 168         PMD_INIT_FUNC_TRACE();
 169
 170         if (mtu < RTE_ETHER_MIN_MTU || frame_size > DPAA_MAX_RX_PKT_LEN)
 171                 return -EINVAL;
 172         /*
 173          * Refuse mtu that requires the support of scattered packets
 174          * when this feature has not been enabled before.
 175          */
 176         if (dev->data->min_rx_buf_size &&
 177                 !dev->data->scattered_rx && frame_size > buffsz) {
 178                 DPAA_PMD_ERR("SG not enabled, will not fit in one buffer");
 179                 return -EINVAL;
 180         }
 181
 182         /* check <seg size> * <max_seg>  >= max_frame */
 183         if (dev->data->min_rx_buf_size && dev->data->scattered_rx &&
 184                 (frame_size > buffsz * DPAA_SGT_MAX_ENTRIES)) {
 185                 DPAA_PMD_ERR("Too big to fit for Max SG list %d",
 186                                 buffsz * DPAA_SGT_MAX_ENTRIES);
 187                 return -EINVAL;
 188         }
 189
 190         if (frame_size > DPAA_ETH_MAX_LEN)
 191                 dev->data->dev_conf.rxmode.offloads |=
 192                                                 DEV_RX_OFFLOAD_JUMBO_FRAME;
 193         else
 194                 dev->data->dev_conf.rxmode.offloads &=
 195                                                 ~DEV_RX_OFFLOAD_JUMBO_FRAME;
 196
 197         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
 198
 199         fman_if_set_maxfrm(dev->process_private, frame_size);
 200
 201         return 0;
 202 }
 203
 204 static int
 205 dpaa_eth_dev_configure(struct rte_eth_dev *dev)
 206 {
 207         struct rte_eth_conf *eth_conf = &dev->data->dev_conf;
 208         uint64_t rx_offloads = eth_conf->rxmode.offloads;
 209         uint64_t tx_offloads = eth_conf->txmode.offloads;
 210         struct rte_device *rdev = dev->device;
 211         struct rte_eth_link *link = &dev->data->dev_link;
 212         struct rte_dpaa_device *dpaa_dev;
 213         struct fman_if *fif = dev->process_private;
 214         struct __fman_if *__fif;
 215         struct rte_intr_handle *intr_handle;
 216         int speed, duplex;
 217         int ret;
 218
 219         PMD_INIT_FUNC_TRACE();
 220
 221         dpaa_dev = container_of(rdev, struct rte_dpaa_device, device);
 222         intr_handle = &dpaa_dev->intr_handle;
 223         __fif = container_of(fif, struct __fman_if, __if);
 224
 225         /* Rx offloads which are enabled by default */
 226         if (dev_rx_offloads_nodis & ~rx_offloads) {
 227                 DPAA_PMD_INFO(
 228                 "Some of rx offloads enabled by default - requested 0x%" PRIx64
 229                 " fixed are 0x%" PRIx64,
 230                 rx_offloads, dev_rx_offloads_nodis);
 231         }
 232
 233         /* Tx offloads which are enabled by default */
 234         if (dev_tx_offloads_nodis & ~tx_offloads) {
 235                 DPAA_PMD_INFO(
 236                 "Some of tx offloads enabled by default - requested 0x%" PRIx64
 237                 " fixed are 0x%" PRIx64,
 238                 tx_offloads, dev_tx_offloads_nodis);
 239         }
 240
 241         if (rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
 242                 uint32_t max_len;
 243
 244                 DPAA_PMD_DEBUG("enabling jumbo");
 245
 246                 if (dev->data->dev_conf.rxmode.max_rx_pkt_len <=
 247                     DPAA_MAX_RX_PKT_LEN)
 248                         max_len = dev->data->dev_conf.rxmode.max_rx_pkt_len;
 249                 else {
 250                         DPAA_PMD_INFO("enabling jumbo override conf max len=%d "
 251                                 "supported is %d",
 252                                 dev->data->dev_conf.rxmode.max_rx_pkt_len,
 253                                 DPAA_MAX_RX_PKT_LEN);
 254                         max_len = DPAA_MAX_RX_PKT_LEN;
 255                 }
 256
 257                 fman_if_set_maxfrm(dev->process_private, max_len);
 258                 dev->data->mtu = max_len
 259                         - RTE_ETHER_HDR_LEN - RTE_ETHER_CRC_LEN - VLAN_TAG_SIZE;
 260         }
 261
 262         if (rx_offloads & DEV_RX_OFFLOAD_SCATTER) {
 263                 DPAA_PMD_DEBUG("enabling scatter mode");
 264                 fman_if_set_sg(dev->process_private, 1);
 265                 dev->data->scattered_rx = 1;
 266         }
 267
 268         if (!(default_q || fmc_q)) {
 269                 if (dpaa_fm_config(dev,
 270                         eth_conf->rx_adv_conf.rss_conf.rss_hf)) {
 271                         dpaa_write_fm_config_to_file();
 272                         DPAA_PMD_ERR("FM port configuration: Failed\n");
 273                         return -1;
 274                 }
 275                 dpaa_write_fm_config_to_file();
 276         }
 277
 278         /* if the interrupts were configured on this devices*/
 279         if (intr_handle && intr_handle->fd) {
 280                 if (dev->data->dev_conf.intr_conf.lsc != 0)
 281                         rte_intr_callback_register(intr_handle,
 282                                            dpaa_interrupt_handler,
 283                                            (void *)dev);
 284
 285                 ret = dpaa_intr_enable(__fif->node_name, intr_handle->fd);
 286                 if (ret) {
 287                         if (dev->data->dev_conf.intr_conf.lsc != 0) {
 288                                 rte_intr_callback_unregister(intr_handle,
 289                                         dpaa_interrupt_handler,
 290                                         (void *)dev);
 291                                 if (ret == EINVAL)
 292                                         printf("Failed to enable interrupt: Not Supported\n");
 293                                 else
 294                                         printf("Failed to enable interrupt\n");
 295                         }
 296                         dev->data->dev_conf.intr_conf.lsc = 0;
 297                         dev->data->dev_flags &= ~RTE_ETH_DEV_INTR_LSC;
 298                 }
 299         }
 300
 301         /* Wait for link status to get updated */
 302         if (!link->link_status)
 303                 sleep(1);
 304
 305         /* Configure link only if link is UP*/
 306         if (link->link_status) {
 307                 if (eth_conf->link_speeds == ETH_LINK_SPEED_AUTONEG) {
 308                         /* Start autoneg only if link is not in autoneg mode */
 309                         if (!link->link_autoneg)
 310                                 dpaa_restart_link_autoneg(__fif->node_name);
 311                 } else if (eth_conf->link_speeds & ETH_LINK_SPEED_FIXED) {
 312                         switch (eth_conf->link_speeds & ~ETH_LINK_SPEED_FIXED) {
 313                         case ETH_LINK_SPEED_10M_HD:
 314                                 speed = ETH_SPEED_NUM_10M;
 315                                 duplex = ETH_LINK_HALF_DUPLEX;
 316                                 break;
 317                         case ETH_LINK_SPEED_10M:
 318                                 speed = ETH_SPEED_NUM_10M;
 319                                 duplex = ETH_LINK_FULL_DUPLEX;
 320                                 break;
 321                         case ETH_LINK_SPEED_100M_HD:
 322                                 speed = ETH_SPEED_NUM_100M;
 323                                 duplex = ETH_LINK_HALF_DUPLEX;
 324                                 break;
 325                         case ETH_LINK_SPEED_100M:
 326                                 speed = ETH_SPEED_NUM_100M;
 327                                 duplex = ETH_LINK_FULL_DUPLEX;
 328                                 break;
 329                         case ETH_LINK_SPEED_1G:
 330                                 speed = ETH_SPEED_NUM_1G;
 331                                 duplex = ETH_LINK_FULL_DUPLEX;
 332                                 break;
 333                         case ETH_LINK_SPEED_2_5G:
 334                                 speed = ETH_SPEED_NUM_2_5G;
 335                                 duplex = ETH_LINK_FULL_DUPLEX;
 336                                 break;
 337                         case ETH_LINK_SPEED_10G:
 338                                 speed = ETH_SPEED_NUM_10G;
 339                                 duplex = ETH_LINK_FULL_DUPLEX;
 340                                 break;
 341                         default:
 342                                 speed = ETH_SPEED_NUM_NONE;
 343                                 duplex = ETH_LINK_FULL_DUPLEX;
 344                                 break;
 345                         }
 346                         /* Set link speed */
 347                         dpaa_update_link_speed(__fif->node_name, speed, duplex);
 348                 } else {
 349                         /* Manual autoneg - custom advertisement speed. */
 350                         printf("Custom Advertisement speeds not supported\n");
 351                 }
 352         }
 353
 354         return 0;
 355 }
 356
 357 static const uint32_t *
 358 dpaa_supported_ptypes_get(struct rte_eth_dev *dev)
 359 {
 360         static const uint32_t ptypes[] = {
 361                 RTE_PTYPE_L2_ETHER,
 362                 RTE_PTYPE_L2_ETHER_VLAN,
 363                 RTE_PTYPE_L2_ETHER_ARP,
 364                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
 365                 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
 366                 RTE_PTYPE_L4_ICMP,
 367                 RTE_PTYPE_L4_TCP,
 368                 RTE_PTYPE_L4_UDP,
 369                 RTE_PTYPE_L4_FRAG,
 370                 RTE_PTYPE_L4_TCP,
 371                 RTE_PTYPE_L4_UDP,
 372                 RTE_PTYPE_L4_SCTP
 373         };
 374
 375         PMD_INIT_FUNC_TRACE();
 376
 377         if (dev->rx_pkt_burst == dpaa_eth_queue_rx)
 378                 return ptypes;
 379         return NULL;
 380 }
 381
 382 static void dpaa_interrupt_handler(void *param)
 383 {
 384         struct rte_eth_dev *dev = param;
 385         struct rte_device *rdev = dev->device;
 386         struct rte_dpaa_device *dpaa_dev;
 387         struct rte_intr_handle *intr_handle;
 388         uint64_t buf;
 389         int bytes_read;
 390
 391         dpaa_dev = container_of(rdev, struct rte_dpaa_device, device);
 392         intr_handle = &dpaa_dev->intr_handle;
 393
 394         bytes_read = read(intr_handle->fd, &buf, sizeof(uint64_t));
 395         if (bytes_read < 0)
 396                 DPAA_PMD_ERR("Error reading eventfd\n");
 397         dpaa_eth_link_update(dev, 0);
 398         rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC, NULL);
 399 }
 400
 401 static int dpaa_eth_dev_start(struct rte_eth_dev *dev)
 402 {
 403         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 404
 405         PMD_INIT_FUNC_TRACE();
 406
 407         if (!(default_q || fmc_q))
 408                 dpaa_write_fm_config_to_file();
 409
 410         /* Change tx callback to the real one */
 411         if (dpaa_intf->cgr_tx)
 412                 dev->tx_pkt_burst = dpaa_eth_queue_tx_slow;
 413         else
 414                 dev->tx_pkt_burst = dpaa_eth_queue_tx;
 415
 416         fman_if_enable_rx(dev->process_private);
 417
 418         return 0;
 419 }
 420
 421 static int dpaa_eth_dev_stop(struct rte_eth_dev *dev)
 422 {
 423         struct fman_if *fif = dev->process_private;
 424
 425         PMD_INIT_FUNC_TRACE();
 426         dev->data->dev_started = 0;
 427
 428         if (!fif->is_shared_mac)
 429                 fman_if_disable_rx(fif);
 430         dev->tx_pkt_burst = dpaa_eth_tx_drop_all;
 431
 432         return 0;
 433 }
 434
 435 static int dpaa_eth_dev_close(struct rte_eth_dev *dev)
 436 {
 437         struct fman_if *fif = dev->process_private;
 438         struct __fman_if *__fif;
 439         struct rte_device *rdev = dev->device;
 440         struct rte_dpaa_device *dpaa_dev;
 441         struct rte_intr_handle *intr_handle;
 442         struct rte_eth_link *link = &dev->data->dev_link;
 443         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 444         int loop;
 445         int ret;
 446
 447         PMD_INIT_FUNC_TRACE();
 448
 449         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 450                 return 0;
 451
 452         if (!dpaa_intf) {
 453                 DPAA_PMD_WARN("Already closed or not started");
 454                 return -1;
 455         }
 456
 457         /* DPAA FM deconfig */
 458         if (!(default_q || fmc_q)) {
 459                 if (dpaa_fm_deconfig(dpaa_intf, dev->process_private))
 460                         DPAA_PMD_WARN("DPAA FM deconfig failed\n");
 461         }
 462
 463         dpaa_dev = container_of(rdev, struct rte_dpaa_device, device);
 464         intr_handle = &dpaa_dev->intr_handle;
 465         __fif = container_of(fif, struct __fman_if, __if);
 466
 467         ret = dpaa_eth_dev_stop(dev);
 468
 469         /* Reset link to autoneg */
 470         if (link->link_status && !link->link_autoneg)
 471                 dpaa_restart_link_autoneg(__fif->node_name);
 472
 473         if (intr_handle && intr_handle->fd &&
 474             dev->data->dev_conf.intr_conf.lsc != 0) {
 475                 dpaa_intr_disable(__fif->node_name);
 476                 rte_intr_callback_unregister(intr_handle,
 477                                              dpaa_interrupt_handler,
 478                                              (void *)dev);
 479         }
 480
 481         /* release configuration memory */
 482         if (dpaa_intf->fc_conf)
 483                 rte_free(dpaa_intf->fc_conf);
 484
 485         /* Release RX congestion Groups */
 486         if (dpaa_intf->cgr_rx) {
 487                 for (loop = 0; loop < dpaa_intf->nb_rx_queues; loop++)
 488                         qman_delete_cgr(&dpaa_intf->cgr_rx[loop]);
 489
 490                 qman_release_cgrid_range(dpaa_intf->cgr_rx[loop].cgrid,
 491                                          dpaa_intf->nb_rx_queues);
 492         }
 493
 494         rte_free(dpaa_intf->cgr_rx);
 495         dpaa_intf->cgr_rx = NULL;
 496         /* Release TX congestion Groups */
 497         if (dpaa_intf->cgr_tx) {
 498                 for (loop = 0; loop < MAX_DPAA_CORES; loop++)
 499                         qman_delete_cgr(&dpaa_intf->cgr_tx[loop]);
 500
 501                 qman_release_cgrid_range(dpaa_intf->cgr_tx[loop].cgrid,
 502                                          MAX_DPAA_CORES);
 503                 rte_free(dpaa_intf->cgr_tx);
 504                 dpaa_intf->cgr_tx = NULL;
 505         }
 506
 507         rte_free(dpaa_intf->rx_queues);
 508         dpaa_intf->rx_queues = NULL;
 509
 510         rte_free(dpaa_intf->tx_queues);
 511         dpaa_intf->tx_queues = NULL;
 512
 513         return ret;
 514 }
 515
 516 static int
 517 dpaa_fw_version_get(struct rte_eth_dev *dev __rte_unused,
 518                      char *fw_version,
 519                      size_t fw_size)
 520 {
 521         int ret;
 522         FILE *svr_file = NULL;
 523         unsigned int svr_ver = 0;
 524
 525         PMD_INIT_FUNC_TRACE();
 526
 527         svr_file = fopen(DPAA_SOC_ID_FILE, "r");
 528         if (!svr_file) {
 529                 DPAA_PMD_ERR("Unable to open SoC device");
 530                 return -ENOTSUP; /* Not supported on this infra */
 531         }
 532         if (fscanf(svr_file, "svr:%x", &svr_ver) > 0)
 533                 dpaa_svr_family = svr_ver & SVR_MASK;
 534         else
 535                 DPAA_PMD_ERR("Unable to read SoC device");
 536
 537         fclose(svr_file);
 538
 539         ret = snprintf(fw_version, fw_size, "SVR:%x-fman-v%x",
 540                        svr_ver, fman_ip_rev);
 541         ret += 1; /* add the size of '\0' */
 542
 543         if (fw_size < (uint32_t)ret)
 544                 return ret;
 545         else
 546                 return 0;
 547 }
 548
 549 static int dpaa_eth_dev_info(struct rte_eth_dev *dev,
 550                              struct rte_eth_dev_info *dev_info)
 551 {
 552         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 553         struct fman_if *fif = dev->process_private;
 554
 555         DPAA_PMD_DEBUG(": %s", dpaa_intf->name);
 556
 557         dev_info->max_rx_queues = dpaa_intf->nb_rx_queues;
 558         dev_info->max_tx_queues = dpaa_intf->nb_tx_queues;
 559         dev_info->max_rx_pktlen = DPAA_MAX_RX_PKT_LEN;
 560         dev_info->max_mac_addrs = DPAA_MAX_MAC_FILTER;
 561         dev_info->max_hash_mac_addrs = 0;
 562         dev_info->max_vfs = 0;
 563         dev_info->max_vmdq_pools = ETH_16_POOLS;
 564         dev_info->flow_type_rss_offloads = DPAA_RSS_OFFLOAD_ALL;
 565
 566         if (fif->mac_type == fman_mac_1g) {
 567                 dev_info->speed_capa = ETH_LINK_SPEED_10M_HD
 568                                         | ETH_LINK_SPEED_10M
 569                                         | ETH_LINK_SPEED_100M_HD
 570                                         | ETH_LINK_SPEED_100M
 571                                         | ETH_LINK_SPEED_1G;
 572         } else if (fif->mac_type == fman_mac_2_5g) {
 573                 dev_info->speed_capa = ETH_LINK_SPEED_10M_HD
 574                                         | ETH_LINK_SPEED_10M
 575                                         | ETH_LINK_SPEED_100M_HD
 576                                         | ETH_LINK_SPEED_100M
 577                                         | ETH_LINK_SPEED_1G
 578                                         | ETH_LINK_SPEED_2_5G;
 579         } else if (fif->mac_type == fman_mac_10g) {
 580                 dev_info->speed_capa = ETH_LINK_SPEED_10M_HD
 581                                         | ETH_LINK_SPEED_10M
 582                                         | ETH_LINK_SPEED_100M_HD
 583                                         | ETH_LINK_SPEED_100M
 584                                         | ETH_LINK_SPEED_1G
 585                                         | ETH_LINK_SPEED_2_5G
 586                                         | ETH_LINK_SPEED_10G;
 587         } else {
 588                 DPAA_PMD_ERR("invalid link_speed: %s, %d",
 589                              dpaa_intf->name, fif->mac_type);
 590                 return -EINVAL;
 591         }
 592
 593         dev_info->rx_offload_capa = dev_rx_offloads_sup |
 594                                         dev_rx_offloads_nodis;
 595         dev_info->tx_offload_capa = dev_tx_offloads_sup |
 596                                         dev_tx_offloads_nodis;
 597         dev_info->default_rxportconf.burst_size = DPAA_DEF_RX_BURST_SIZE;
 598         dev_info->default_txportconf.burst_size = DPAA_DEF_TX_BURST_SIZE;
 599         dev_info->default_rxportconf.nb_queues = 1;
 600         dev_info->default_txportconf.nb_queues = 1;
 601         dev_info->default_txportconf.ring_size = CGR_TX_CGR_THRESH;
 602         dev_info->default_rxportconf.ring_size = CGR_RX_PERFQ_THRESH;
 603
 604         return 0;
 605 }
 606
 607 static int
 608 dpaa_dev_rx_burst_mode_get(struct rte_eth_dev *dev,
 609                         __rte_unused uint16_t queue_id,
 610                         struct rte_eth_burst_mode *mode)
 611 {
 612         struct rte_eth_conf *eth_conf = &dev->data->dev_conf;
 613         int ret = -EINVAL;
 614         unsigned int i;
 615         const struct burst_info {
 616                 uint64_t flags;
 617                 const char *output;
 618         } rx_offload_map[] = {
 619                         {DEV_RX_OFFLOAD_JUMBO_FRAME, " Jumbo frame,"},
 620                         {DEV_RX_OFFLOAD_SCATTER, " Scattered,"},
 621                         {DEV_RX_OFFLOAD_IPV4_CKSUM, " IPV4 csum,"},
 622                         {DEV_RX_OFFLOAD_UDP_CKSUM, " UDP csum,"},
 623                         {DEV_RX_OFFLOAD_TCP_CKSUM, " TCP csum,"},
 624                         {DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM, " Outer IPV4 csum,"},
 625                         {DEV_RX_OFFLOAD_RSS_HASH, " RSS,"}
 626         };
 627
 628         /* Update Rx offload info */
 629         for (i = 0; i < RTE_DIM(rx_offload_map); i++) {
 630                 if (eth_conf->rxmode.offloads & rx_offload_map[i].flags) {
 631                         snprintf(mode->info, sizeof(mode->info), "%s",
 632                                 rx_offload_map[i].output);
 633                         ret = 0;
 634                         break;
 635                 }
 636         }
 637         return ret;
 638 }
 639
 640 static int
 641 dpaa_dev_tx_burst_mode_get(struct rte_eth_dev *dev,
 642                         __rte_unused uint16_t queue_id,
 643                         struct rte_eth_burst_mode *mode)
 644 {
 645         struct rte_eth_conf *eth_conf = &dev->data->dev_conf;
 646         int ret = -EINVAL;
 647         unsigned int i;
 648         const struct burst_info {
 649                 uint64_t flags;
 650                 const char *output;
 651         } tx_offload_map[] = {
 652                         {DEV_TX_OFFLOAD_MT_LOCKFREE, " MT lockfree,"},
 653                         {DEV_TX_OFFLOAD_MBUF_FAST_FREE, " MBUF free disable,"},
 654                         {DEV_TX_OFFLOAD_IPV4_CKSUM, " IPV4 csum,"},
 655                         {DEV_TX_OFFLOAD_UDP_CKSUM, " UDP csum,"},
 656                         {DEV_TX_OFFLOAD_TCP_CKSUM, " TCP csum,"},
 657                         {DEV_TX_OFFLOAD_SCTP_CKSUM, " SCTP csum,"},
 658                         {DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM, " Outer IPV4 csum,"},
 659                         {DEV_TX_OFFLOAD_MULTI_SEGS, " Scattered,"}
 660         };
 661
 662         /* Update Tx offload info */
 663         for (i = 0; i < RTE_DIM(tx_offload_map); i++) {
 664                 if (eth_conf->txmode.offloads & tx_offload_map[i].flags) {
 665                         snprintf(mode->info, sizeof(mode->info), "%s",
 666                                 tx_offload_map[i].output);
 667                         ret = 0;
 668                         break;
 669                 }
 670         }
 671         return ret;
 672 }
 673
 674 static int dpaa_eth_link_update(struct rte_eth_dev *dev,
 675                                 int wait_to_complete)
 676 {
 677         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 678         struct rte_eth_link *link = &dev->data->dev_link;
 679         struct fman_if *fif = dev->process_private;
 680         struct __fman_if *__fif = container_of(fif, struct __fman_if, __if);
 681         int ret, ioctl_version;
 682         uint8_t count;
 683
 684         PMD_INIT_FUNC_TRACE();
 685
 686         ioctl_version = dpaa_get_ioctl_version_number();
 687
 688         if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC) {
 689                 for (count = 0; count <= MAX_REPEAT_TIME; count++) {
 690                         ret = dpaa_get_link_status(__fif->node_name, link);
 691                         if (ret)
 692                                 return ret;
 693                         if (link->link_status == ETH_LINK_DOWN &&
 694                             wait_to_complete)
 695                                 rte_delay_ms(CHECK_INTERVAL);
 696                         else
 697                                 break;
 698                 }
 699         } else {
 700                 link->link_status = dpaa_intf->valid;
 701         }
 702
 703         if (ioctl_version < 2) {
 704                 link->link_duplex = ETH_LINK_FULL_DUPLEX;
 705                 link->link_autoneg = ETH_LINK_AUTONEG;
 706
 707                 if (fif->mac_type == fman_mac_1g)
 708                         link->link_speed = ETH_SPEED_NUM_1G;
 709                 else if (fif->mac_type == fman_mac_2_5g)
 710                         link->link_speed = ETH_SPEED_NUM_2_5G;
 711                 else if (fif->mac_type == fman_mac_10g)
 712                         link->link_speed = ETH_SPEED_NUM_10G;
 713                 else
 714                         DPAA_PMD_ERR("invalid link_speed: %s, %d",
 715                                      dpaa_intf->name, fif->mac_type);
 716         }
 717
 718         DPAA_PMD_INFO("Port %d Link is %s\n", dev->data->port_id,
 719                       link->link_status ? "Up" : "Down");
 720         return 0;
 721 }
 722
 723 static int dpaa_eth_stats_get(struct rte_eth_dev *dev,
 724                                struct rte_eth_stats *stats)
 725 {
 726         PMD_INIT_FUNC_TRACE();
 727
 728         fman_if_stats_get(dev->process_private, stats);
 729         return 0;
 730 }
 731
 732 static int dpaa_eth_stats_reset(struct rte_eth_dev *dev)
 733 {
 734         PMD_INIT_FUNC_TRACE();
 735
 736         fman_if_stats_reset(dev->process_private);
 737
 738         return 0;
 739 }
 740
 741 static int
 742 dpaa_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
 743                     unsigned int n)
 744 {
 745         unsigned int i = 0, num = RTE_DIM(dpaa_xstats_strings);
 746         uint64_t values[sizeof(struct dpaa_if_stats) / 8];
 747
 748         if (n < num)
 749                 return num;
 750
 751         if (xstats == NULL)
 752                 return 0;
 753
 754         fman_if_stats_get_all(dev->process_private, values,
 755                               sizeof(struct dpaa_if_stats) / 8);
 756
 757         for (i = 0; i < num; i++) {
 758                 xstats[i].id = i;
 759                 xstats[i].value = values[dpaa_xstats_strings[i].offset / 8];
 760         }
 761         return i;
 762 }
 763
 764 static int
 765 dpaa_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
 766                       struct rte_eth_xstat_name *xstats_names,
 767                       unsigned int limit)
 768 {
 769         unsigned int i, stat_cnt = RTE_DIM(dpaa_xstats_strings);
 770
 771         if (limit < stat_cnt)
 772                 return stat_cnt;
 773
 774         if (xstats_names != NULL)
 775                 for (i = 0; i < stat_cnt; i++)
 776                         strlcpy(xstats_names[i].name,
 777                                 dpaa_xstats_strings[i].name,
 778                                 sizeof(xstats_names[i].name));
 779
 780         return stat_cnt;
 781 }
 782
 783 static int
 784 dpaa_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids,
 785                       uint64_t *values, unsigned int n)
 786 {
 787         unsigned int i, stat_cnt = RTE_DIM(dpaa_xstats_strings);
 788         uint64_t values_copy[sizeof(struct dpaa_if_stats) / 8];
 789
 790         if (!ids) {
 791                 if (n < stat_cnt)
 792                         return stat_cnt;
 793
 794                 if (!values)
 795                         return 0;
 796
 797                 fman_if_stats_get_all(dev->process_private, values_copy,
 798                                       sizeof(struct dpaa_if_stats) / 8);
 799
 800                 for (i = 0; i < stat_cnt; i++)
 801                         values[i] =
 802                                 values_copy[dpaa_xstats_strings[i].offset / 8];
 803
 804                 return stat_cnt;
 805         }
 806
 807         dpaa_xstats_get_by_id(dev, NULL, values_copy, stat_cnt);
 808
 809         for (i = 0; i < n; i++) {
 810                 if (ids[i] >= stat_cnt) {
 811                         DPAA_PMD_ERR("id value isn't valid");
 812                         return -1;
 813                 }
 814                 values[i] = values_copy[ids[i]];
 815         }
 816         return n;
 817 }
 818
 819 static int
 820 dpaa_xstats_get_names_by_id(
 821         struct rte_eth_dev *dev,
 822         struct rte_eth_xstat_name *xstats_names,
 823         const uint64_t *ids,
 824         unsigned int limit)
 825 {
 826         unsigned int i, stat_cnt = RTE_DIM(dpaa_xstats_strings);
 827         struct rte_eth_xstat_name xstats_names_copy[stat_cnt];
 828
 829         if (!ids)
 830                 return dpaa_xstats_get_names(dev, xstats_names, limit);
 831
 832         dpaa_xstats_get_names(dev, xstats_names_copy, limit);
 833
 834         for (i = 0; i < limit; i++) {
 835                 if (ids[i] >= stat_cnt) {
 836                         DPAA_PMD_ERR("id value isn't valid");
 837                         return -1;
 838                 }
 839                 strcpy(xstats_names[i].name, xstats_names_copy[ids[i]].name);
 840         }
 841         return limit;
 842 }
 843
 844 static int dpaa_eth_promiscuous_enable(struct rte_eth_dev *dev)
 845 {
 846         PMD_INIT_FUNC_TRACE();
 847
 848         fman_if_promiscuous_enable(dev->process_private);
 849
 850         return 0;
 851 }
 852
 853 static int dpaa_eth_promiscuous_disable(struct rte_eth_dev *dev)
 854 {
 855         PMD_INIT_FUNC_TRACE();
 856
 857         fman_if_promiscuous_disable(dev->process_private);
 858
 859         return 0;
 860 }
 861
 862 static int dpaa_eth_multicast_enable(struct rte_eth_dev *dev)
 863 {
 864         PMD_INIT_FUNC_TRACE();
 865
 866         fman_if_set_mcast_filter_table(dev->process_private);
 867
 868         return 0;
 869 }
 870
 871 static int dpaa_eth_multicast_disable(struct rte_eth_dev *dev)
 872 {
 873         PMD_INIT_FUNC_TRACE();
 874
 875         fman_if_reset_mcast_filter_table(dev->process_private);
 876
 877         return 0;
 878 }
 879
 880 static void dpaa_fman_if_pool_setup(struct rte_eth_dev *dev)
 881 {
 882         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 883         struct fman_if_ic_params icp;
 884         uint32_t fd_offset;
 885         uint32_t bp_size;
 886
 887         memset(&icp, 0, sizeof(icp));
 888         /* set ICEOF for to the default value , which is 0*/
 889         icp.iciof = DEFAULT_ICIOF;
 890         icp.iceof = DEFAULT_RX_ICEOF;
 891         icp.icsz = DEFAULT_ICSZ;
 892         fman_if_set_ic_params(dev->process_private, &icp);
 893
 894         fd_offset = RTE_PKTMBUF_HEADROOM + DPAA_HW_BUF_RESERVE;
 895         fman_if_set_fdoff(dev->process_private, fd_offset);
 896
 897         /* Buffer pool size should be equal to Dataroom Size*/
 898         bp_size = rte_pktmbuf_data_room_size(dpaa_intf->bp_info->mp);
 899
 900         fman_if_set_bp(dev->process_private,
 901                        dpaa_intf->bp_info->mp->size,
 902                        dpaa_intf->bp_info->bpid, bp_size);
 903 }
 904
 905 static inline int dpaa_eth_rx_queue_bp_check(struct rte_eth_dev *dev,
 906                                              int8_t vsp_id, uint32_t bpid)
 907 {
 908         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 909         struct fman_if *fif = dev->process_private;
 910
 911         if (fif->num_profiles) {
 912                 if (vsp_id < 0)
 913                         vsp_id = fif->base_profile_id;
 914         } else {
 915                 if (vsp_id < 0)
 916                         vsp_id = 0;
 917         }
 918
 919         if (dpaa_intf->vsp_bpid[vsp_id] &&
 920                 bpid != dpaa_intf->vsp_bpid[vsp_id]) {
 921                 DPAA_PMD_ERR("Various MPs are assigned to RXQs with same VSP");
 922
 923                 return -1;
 924         }
 925
 926         return 0;
 927 }
 928
 929 static
 930 int dpaa_eth_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 931                             uint16_t nb_desc,
 932                             unsigned int socket_id __rte_unused,
 933                             const struct rte_eth_rxconf *rx_conf,
 934                             struct rte_mempool *mp)
 935 {
 936         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 937         struct fman_if *fif = dev->process_private;
 938         struct qman_fq *rxq = &dpaa_intf->rx_queues[queue_idx];
 939         struct qm_mcc_initfq opts = {0};
 940         u32 flags = 0;
 941         int ret;
 942         u32 buffsz = rte_pktmbuf_data_room_size(mp) - RTE_PKTMBUF_HEADROOM;
 943
 944         PMD_INIT_FUNC_TRACE();
 945
 946         if (queue_idx >= dev->data->nb_rx_queues) {
 947                 rte_errno = EOVERFLOW;
 948                 DPAA_PMD_ERR("%p: queue index out of range (%u >= %u)",
 949                       (void *)dev, queue_idx, dev->data->nb_rx_queues);
 950                 return -rte_errno;
 951         }
 952
 953         /* Rx deferred start is not supported */
 954         if (rx_conf->rx_deferred_start) {
 955                 DPAA_PMD_ERR("%p:Rx deferred start not supported", (void *)dev);
 956                 return -EINVAL;
 957         }
 958         rxq->nb_desc = UINT16_MAX;
 959         rxq->offloads = rx_conf->offloads;
 960
 961         DPAA_PMD_INFO("Rx queue setup for queue index: %d fq_id (0x%x)",
 962                         queue_idx, rxq->fqid);
 963
 964         if (!fif->num_profiles) {
 965                 if (dpaa_intf->bp_info && dpaa_intf->bp_info->bp &&
 966                         dpaa_intf->bp_info->mp != mp) {
 967                         DPAA_PMD_WARN("Multiple pools on same interface not"
 968                                       " supported");
 969                         return -EINVAL;
 970                 }
 971         } else {
 972                 if (dpaa_eth_rx_queue_bp_check(dev, rxq->vsp_id,
 973                         DPAA_MEMPOOL_TO_POOL_INFO(mp)->bpid)) {
 974                         return -EINVAL;
 975                 }
 976         }
 977
 978         /* Max packet can fit in single buffer */
 979         if (dev->data->dev_conf.rxmode.max_rx_pkt_len <= buffsz) {
 980                 ;
 981         } else if (dev->data->dev_conf.rxmode.offloads &
 982                         DEV_RX_OFFLOAD_SCATTER) {
 983                 if (dev->data->dev_conf.rxmode.max_rx_pkt_len >
 984                         buffsz * DPAA_SGT_MAX_ENTRIES) {
 985                         DPAA_PMD_ERR("max RxPkt size %d too big to fit "
 986                                 "MaxSGlist %d",
 987                                 dev->data->dev_conf.rxmode.max_rx_pkt_len,
 988                                 buffsz * DPAA_SGT_MAX_ENTRIES);
 989                         rte_errno = EOVERFLOW;
 990                         return -rte_errno;
 991                 }
 992         } else {
 993                 DPAA_PMD_WARN("The requested maximum Rx packet size (%u) is"
 994                      " larger than a single mbuf (%u) and scattered"
 995                      " mode has not been requested",
 996                      dev->data->dev_conf.rxmode.max_rx_pkt_len,
 997                      buffsz - RTE_PKTMBUF_HEADROOM);
 998         }
 999
1000         dpaa_intf->bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp);
1001
1002         /* For shared interface, it's done in kernel, skip.*/
1003         if (!fif->is_shared_mac)
1004                 dpaa_fman_if_pool_setup(dev);
1005
1006         if (fif->num_profiles) {
1007                 int8_t vsp_id = rxq->vsp_id;
1008
1009                 if (vsp_id >= 0) {
1010                         ret = dpaa_port_vsp_update(dpaa_intf, fmc_q, vsp_id,
1011                                         DPAA_MEMPOOL_TO_POOL_INFO(mp)->bpid,
1012                                         fif);
1013                         if (ret) {
1014                                 DPAA_PMD_ERR("dpaa_port_vsp_update failed");
1015                                 return ret;
1016                         }
1017                 } else {
1018                         DPAA_PMD_INFO("Base profile is associated to"
1019                                 " RXQ fqid:%d\r\n", rxq->fqid);
1020                         if (fif->is_shared_mac) {
1021                                 DPAA_PMD_ERR("Fatal: Base profile is associated"
1022                                              " to shared interface on DPDK.");
1023                                 return -EINVAL;
1024                         }
1025                         dpaa_intf->vsp_bpid[fif->base_profile_id] =
1026                                 DPAA_MEMPOOL_TO_POOL_INFO(mp)->bpid;
1027                 }
1028         } else {
1029                 dpaa_intf->vsp_bpid[0] =
1030                         DPAA_MEMPOOL_TO_POOL_INFO(mp)->bpid;
1031         }
1032
1033         dpaa_intf->valid = 1;
1034         DPAA_PMD_DEBUG("if:%s sg_on = %d, max_frm =%d", dpaa_intf->name,
1035                 fman_if_get_sg_enable(fif),
1036                 dev->data->dev_conf.rxmode.max_rx_pkt_len);
1037         /* checking if push mode only, no error check for now */
1038         if (!rxq->is_static &&
1039             dpaa_push_mode_max_queue > dpaa_push_queue_idx) {
1040                 struct qman_portal *qp;
1041                 int q_fd;
1042
1043                 dpaa_push_queue_idx++;
1044                 opts.we_mask = QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_CONTEXTA;
1045                 opts.fqd.fq_ctrl = QM_FQCTRL_AVOIDBLOCK |
1046                                    QM_FQCTRL_CTXASTASHING |
1047                                    QM_FQCTRL_PREFERINCACHE;
1048                 opts.fqd.context_a.stashing.exclusive = 0;
1049                 /* In muticore scenario stashing becomes a bottleneck on LS1046.
1050                  * So do not enable stashing in this case
1051                  */
1052                 if (dpaa_svr_family != SVR_LS1046A_FAMILY)
1053                         opts.fqd.context_a.stashing.annotation_cl =
1054                                                 DPAA_IF_RX_ANNOTATION_STASH;
1055                 opts.fqd.context_a.stashing.data_cl = DPAA_IF_RX_DATA_STASH;
1056                 opts.fqd.context_a.stashing.context_cl =
1057                                                 DPAA_IF_RX_CONTEXT_STASH;
1058
1059                 /*Create a channel and associate given queue with the channel*/
1060                 qman_alloc_pool_range((u32 *)&rxq->ch_id, 1, 1, 0);
1061                 opts.we_mask = opts.we_mask | QM_INITFQ_WE_DESTWQ;
1062                 opts.fqd.dest.channel = rxq->ch_id;
1063                 opts.fqd.dest.wq = DPAA_IF_RX_PRIORITY;
1064                 flags = QMAN_INITFQ_FLAG_SCHED;
1065
1066                 /* Configure tail drop */
1067                 if (dpaa_intf->cgr_rx) {
1068                         opts.we_mask |= QM_INITFQ_WE_CGID;
1069                         opts.fqd.cgid = dpaa_intf->cgr_rx[queue_idx].cgrid;
1070                         opts.fqd.fq_ctrl |= QM_FQCTRL_CGE;
1071                 }
1072                 ret = qman_init_fq(rxq, flags, &opts);
1073                 if (ret) {
1074                         DPAA_PMD_ERR("Channel/Q association failed. fqid 0x%x "
1075                                 "ret:%d(%s)", rxq->fqid, ret, strerror(ret));
1076                         return ret;
1077                 }
1078                 if (dpaa_svr_family == SVR_LS1043A_FAMILY) {
1079                         rxq->cb.dqrr_dpdk_pull_cb = dpaa_rx_cb_no_prefetch;
1080                 } else {
1081                         rxq->cb.dqrr_dpdk_pull_cb = dpaa_rx_cb;
1082                         rxq->cb.dqrr_prepare = dpaa_rx_cb_prepare;
1083                 }
1084
1085                 rxq->is_static = true;
1086
1087                 /* Allocate qman specific portals */
1088                 qp = fsl_qman_fq_portal_create(&q_fd);
1089                 if (!qp) {
1090                         DPAA_PMD_ERR("Unable to alloc fq portal");
1091                         return -1;
1092                 }
1093                 rxq->qp = qp;
1094
1095                 /* Set up the device interrupt handler */
1096                 if (!dev->intr_handle) {
1097                         struct rte_dpaa_device *dpaa_dev;
1098                         struct rte_device *rdev = dev->device;
1099
1100                         dpaa_dev = container_of(rdev, struct rte_dpaa_device,
1101                                                 device);
1102                         dev->intr_handle = &dpaa_dev->intr_handle;
1103                         dev->intr_handle->intr_vec = rte_zmalloc(NULL,
1104                                         dpaa_push_mode_max_queue, 0);
1105                         if (!dev->intr_handle->intr_vec) {
1106                                 DPAA_PMD_ERR("intr_vec alloc failed");
1107                                 return -ENOMEM;
1108                         }
1109                         dev->intr_handle->nb_efd = dpaa_push_mode_max_queue;
1110                         dev->intr_handle->max_intr = dpaa_push_mode_max_queue;
1111                 }
1112
1113                 dev->intr_handle->type = RTE_INTR_HANDLE_EXT;
1114                 dev->intr_handle->intr_vec[queue_idx] = queue_idx + 1;
1115                 dev->intr_handle->efds[queue_idx] = q_fd;
1116                 rxq->q_fd = q_fd;
1117         }
1118         rxq->bp_array = rte_dpaa_bpid_info;
1119         dev->data->rx_queues[queue_idx] = rxq;
1120
1121         /* configure the CGR size as per the desc size */
1122         if (dpaa_intf->cgr_rx) {
1123                 struct qm_mcc_initcgr cgr_opts = {0};
1124
1125                 rxq->nb_desc = nb_desc;
1126                 /* Enable tail drop with cgr on this queue */
1127                 qm_cgr_cs_thres_set64(&cgr_opts.cgr.cs_thres, nb_desc, 0);
1128                 ret = qman_modify_cgr(dpaa_intf->cgr_rx, 0, &cgr_opts);
1129                 if (ret) {
1130                         DPAA_PMD_WARN(
1131                                 "rx taildrop modify fail on fqid %d (ret=%d)",
1132                                 rxq->fqid, ret);
1133                 }
1134         }
1135         /* Enable main queue to receive error packets also by default */
1136         fman_if_set_err_fqid(fif, rxq->fqid);
1137         return 0;
1138 }
1139
1140 int
1141 dpaa_eth_eventq_attach(const struct rte_eth_dev *dev,
1142                 int eth_rx_queue_id,
1143                 u16 ch_id,
1144                 const struct rte_event_eth_rx_adapter_queue_conf *queue_conf)
1145 {
1146         int ret;
1147         u32 flags = 0;
1148         struct dpaa_if *dpaa_intf = dev->data->dev_private;
1149         struct qman_fq *rxq = &dpaa_intf->rx_queues[eth_rx_queue_id];
1150         struct qm_mcc_initfq opts = {0};
1151
1152         if (dpaa_push_mode_max_queue)
1153                 DPAA_PMD_WARN("PUSH mode q and EVENTDEV are not compatible\n"
1154                               "PUSH mode already enabled for first %d queues.\n"
1155                               "To disable set DPAA_PUSH_QUEUES_NUMBER to 0\n",
1156                               dpaa_push_mode_max_queue);
1157
1158         dpaa_poll_queue_default_config(&opts);
1159
1160         switch (queue_conf->ev.sched_type) {
1161         case RTE_SCHED_TYPE_ATOMIC:
1162                 opts.fqd.fq_ctrl |= QM_FQCTRL_HOLDACTIVE;
1163                 /* Reset FQCTRL_AVOIDBLOCK bit as it is unnecessary
1164                  * configuration with HOLD_ACTIVE setting
1165                  */
1166                 opts.fqd.fq_ctrl &= (~QM_FQCTRL_AVOIDBLOCK);
1167                 rxq->cb.dqrr_dpdk_cb = dpaa_rx_cb_atomic;
1168                 break;
1169         case RTE_SCHED_TYPE_ORDERED:
1170                 DPAA_PMD_ERR("Ordered queue schedule type is not supported\n");
1171                 return -1;
1172         default:
1173                 opts.fqd.fq_ctrl |= QM_FQCTRL_AVOIDBLOCK;
1174                 rxq->cb.dqrr_dpdk_cb = dpaa_rx_cb_parallel;
1175                 break;
1176         }
1177
1178         opts.we_mask = opts.we_mask | QM_INITFQ_WE_DESTWQ;
1179         opts.fqd.dest.channel = ch_id;
1180         opts.fqd.dest.wq = queue_conf->ev.priority;
1181
1182         if (dpaa_intf->cgr_rx) {
1183                 opts.we_mask |= QM_INITFQ_WE_CGID;
1184                 opts.fqd.cgid = dpaa_intf->cgr_rx[eth_rx_queue_id].cgrid;
1185                 opts.fqd.fq_ctrl |= QM_FQCTRL_CGE;
1186         }
1187
1188         flags = QMAN_INITFQ_FLAG_SCHED;
1189
1190         ret = qman_init_fq(rxq, flags, &opts);
1191         if (ret) {
1192                 DPAA_PMD_ERR("Ev-Channel/Q association failed. fqid 0x%x "
1193                                 "ret:%d(%s)", rxq->fqid, ret, strerror(ret));
1194                 return ret;
1195         }
1196
1197         /* copy configuration which needs to be filled during dequeue */
1198         memcpy(&rxq->ev, &queue_conf->ev, sizeof(struct rte_event));
1199         dev->data->rx_queues[eth_rx_queue_id] = rxq;
1200
1201         return ret;
1202 }
1203
1204 int
1205 dpaa_eth_eventq_detach(const struct rte_eth_dev *dev,
1206                 int eth_rx_queue_id)
1207 {
1208         struct qm_mcc_initfq opts;
1209         int ret;
1210         u32 flags = 0;
1211         struct dpaa_if *dpaa_intf = dev->data->dev_private;
1212         struct qman_fq *rxq = &dpaa_intf->rx_queues[eth_rx_queue_id];
1213
1214         dpaa_poll_queue_default_config(&opts);
1215
1216         if (dpaa_intf->cgr_rx) {
1217                 opts.we_mask |= QM_INITFQ_WE_CGID;
1218                 opts.fqd.cgid = dpaa_intf->cgr_rx[eth_rx_queue_id].cgrid;
1219                 opts.fqd.fq_ctrl |= QM_FQCTRL_CGE;
1220         }
1221
1222         ret = qman_init_fq(rxq, flags, &opts);
1223         if (ret) {
1224                 DPAA_PMD_ERR("init rx fqid %d failed with ret: %d",
1225                              rxq->fqid, ret);
1226         }
1227
1228         rxq->cb.dqrr_dpdk_cb = NULL;
1229         dev->data->rx_queues[eth_rx_queue_id] = NULL;
1230
1231         return 0;
1232 }
1233
1234 static
1235 void dpaa_eth_rx_queue_release(void *rxq __rte_unused)
1236 {
1237         PMD_INIT_FUNC_TRACE();
1238 }
1239
1240 static
1241 int dpaa_eth_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
1242                             uint16_t nb_desc __rte_unused,
1243                 unsigned int socket_id __rte_unused,
1244                 const struct rte_eth_txconf *tx_conf)
1245 {
1246         struct dpaa_if *dpaa_intf = dev->data->dev_private;
1247         struct qman_fq *txq = &dpaa_intf->tx_queues[queue_idx];
1248
1249         PMD_INIT_FUNC_TRACE();
1250
1251         /* Tx deferred start is not supported */
1252         if (tx_conf->tx_deferred_start) {
1253                 DPAA_PMD_ERR("%p:Tx deferred start not supported", (void *)dev);
1254                 return -EINVAL;
1255         }
1256         txq->nb_desc = UINT16_MAX;
1257         txq->offloads = tx_conf->offloads;
1258
1259         if (queue_idx >= dev->data->nb_tx_queues) {
1260                 rte_errno = EOVERFLOW;
1261                 DPAA_PMD_ERR("%p: queue index out of range (%u >= %u)",
1262                       (void *)dev, queue_idx, dev->data->nb_tx_queues);
1263                 return -rte_errno;
1264         }
1265
1266         DPAA_PMD_INFO("Tx queue setup for queue index: %d fq_id (0x%x)",
1267                         queue_idx, txq->fqid);
1268         dev->data->tx_queues[queue_idx] = txq;
1269
1270         return 0;
1271 }
1272
1273 static void dpaa_eth_tx_queue_release(void *txq __rte_unused)
1274 {
1275         PMD_INIT_FUNC_TRACE();
1276 }
1277
1278 static uint32_t
1279 dpaa_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
1280 {
1281         struct dpaa_if *dpaa_intf = dev->data->dev_private;
1282         struct qman_fq *rxq = &dpaa_intf->rx_queues[rx_queue_id];
1283         u32 frm_cnt = 0;
1284
1285         PMD_INIT_FUNC_TRACE();
1286
1287         if (qman_query_fq_frm_cnt(rxq, &frm_cnt) == 0) {
1288                 DPAA_PMD_DEBUG("RX frame count for q(%d) is %u",
1289                                rx_queue_id, frm_cnt);
1290         }
1291         return frm_cnt;
1292 }
1293
1294 static int dpaa_link_down(struct rte_eth_dev *dev)
1295 {
1296         struct fman_if *fif = dev->process_private;
1297         struct __fman_if *__fif;
1298
1299         PMD_INIT_FUNC_TRACE();
1300
1301         __fif = container_of(fif, struct __fman_if, __if);
1302
1303         if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
1304                 dpaa_update_link_status(__fif->node_name, ETH_LINK_DOWN);
1305         else
1306                 return dpaa_eth_dev_stop(dev);
1307         return 0;
1308 }
1309
1310 static int dpaa_link_up(struct rte_eth_dev *dev)
1311 {
1312         struct fman_if *fif = dev->process_private;
1313         struct __fman_if *__fif;
1314
1315         PMD_INIT_FUNC_TRACE();
1316
1317         __fif = container_of(fif, struct __fman_if, __if);
1318
1319         if (dev->data->dev_flags & RTE_ETH_DEV_INTR_LSC)
1320                 dpaa_update_link_status(__fif->node_name, ETH_LINK_UP);
1321         else
1322                 dpaa_eth_dev_start(dev);
1323         return 0;
1324 }
1325
1326 static int
1327 dpaa_flow_ctrl_set(struct rte_eth_dev *dev,
1328                    struct rte_eth_fc_conf *fc_conf)
1329 {
1330         struct dpaa_if *dpaa_intf = dev->data->dev_private;
1331         struct rte_eth_fc_conf *net_fc;
1332
1333         PMD_INIT_FUNC_TRACE();
1334
1335         if (!(dpaa_intf->fc_conf)) {
1336                 dpaa_intf->fc_conf = rte_zmalloc(NULL,
1337                         sizeof(struct rte_eth_fc_conf), MAX_CACHELINE);
1338                 if (!dpaa_intf->fc_conf) {
1339                         DPAA_PMD_ERR("unable to save flow control info");
1340                         return -ENOMEM;
1341                 }
1342         }
1343         net_fc = dpaa_intf->fc_conf;
1344
1345         if (fc_conf->high_water < fc_conf->low_water) {
1346                 DPAA_PMD_ERR("Incorrect Flow Control Configuration");
1347                 return -EINVAL;
1348         }
1349
1350         if (fc_conf->mode == RTE_FC_NONE) {
1351                 return 0;
1352         } else if (fc_conf->mode == RTE_FC_TX_PAUSE ||
1353                  fc_conf->mode == RTE_FC_FULL) {
1354                 fman_if_set_fc_threshold(dev->process_private,
1355                                          fc_conf->high_water,
1356                                          fc_conf->low_water,
1357                                          dpaa_intf->bp_info->bpid);
1358                 if (fc_conf->pause_time)
1359                         fman_if_set_fc_quanta(dev->process_private,
1360                                               fc_conf->pause_time);
1361         }
1362
1363         /* Save the information in dpaa device */
1364         net_fc->pause_time = fc_conf->pause_time;
1365         net_fc->high_water = fc_conf->high_water;
1366         net_fc->low_water = fc_conf->low_water;
1367         net_fc->send_xon = fc_conf->send_xon;
1368         net_fc->mac_ctrl_frame_fwd = fc_conf->mac_ctrl_frame_fwd;
1369         net_fc->mode = fc_conf->mode;
1370         net_fc->autoneg = fc_conf->autoneg;
1371
1372         return 0;
1373 }
1374
1375 static int
1376 dpaa_flow_ctrl_get(struct rte_eth_dev *dev,
1377                    struct rte_eth_fc_conf *fc_conf)
1378 {
1379         struct dpaa_if *dpaa_intf = dev->data->dev_private;
1380         struct rte_eth_fc_conf *net_fc = dpaa_intf->fc_conf;
1381         int ret;
1382
1383         PMD_INIT_FUNC_TRACE();
1384
1385         if (net_fc) {
1386                 fc_conf->pause_time = net_fc->pause_time;
1387                 fc_conf->high_water = net_fc->high_water;
1388                 fc_conf->low_water = net_fc->low_water;
1389                 fc_conf->send_xon = net_fc->send_xon;
1390                 fc_conf->mac_ctrl_frame_fwd = net_fc->mac_ctrl_frame_fwd;
1391                 fc_conf->mode = net_fc->mode;
1392                 fc_conf->autoneg = net_fc->autoneg;
1393                 return 0;
1394         }
1395         ret = fman_if_get_fc_threshold(dev->process_private);
1396         if (ret) {
1397                 fc_conf->mode = RTE_FC_TX_PAUSE;
1398                 fc_conf->pause_time =
1399                         fman_if_get_fc_quanta(dev->process_private);
1400         } else {
1401                 fc_conf->mode = RTE_FC_NONE;
1402         }
1403
1404         return 0;
1405 }
1406
1407 static int
1408 dpaa_dev_add_mac_addr(struct rte_eth_dev *dev,
1409                              struct rte_ether_addr *addr,
1410                              uint32_t index,
1411                              __rte_unused uint32_t pool)
1412 {
1413         int ret;
1414
1415         PMD_INIT_FUNC_TRACE();
1416
1417         ret = fman_if_add_mac_addr(dev->process_private,
1418                                    addr->addr_bytes, index);
1419
1420         if (ret)
1421                 DPAA_PMD_ERR("Adding the MAC ADDR failed: err = %d", ret);
1422         return 0;
1423 }
1424
1425 static void
1426 dpaa_dev_remove_mac_addr(struct rte_eth_dev *dev,
1427                           uint32_t index)
1428 {
1429         PMD_INIT_FUNC_TRACE();
1430
1431         fman_if_clear_mac_addr(dev->process_private, index);
1432 }
1433
1434 static int
1435 dpaa_dev_set_mac_addr(struct rte_eth_dev *dev,
1436                        struct rte_ether_addr *addr)
1437 {
1438         int ret;
1439
1440         PMD_INIT_FUNC_TRACE();
1441
1442         ret = fman_if_add_mac_addr(dev->process_private, addr->addr_bytes, 0);
1443         if (ret)
1444                 DPAA_PMD_ERR("Setting the MAC ADDR failed %d", ret);
1445
1446         return ret;
1447 }
1448
1449 static int
1450 dpaa_dev_rss_hash_update(struct rte_eth_dev *dev,
1451                          struct rte_eth_rss_conf *rss_conf)
1452 {
1453         struct rte_eth_dev_data *data = dev->data;
1454         struct rte_eth_conf *eth_conf = &data->dev_conf;
1455
1456         PMD_INIT_FUNC_TRACE();
1457
1458         if (!(default_q || fmc_q)) {
1459                 if (dpaa_fm_config(dev, rss_conf->rss_hf)) {
1460                         DPAA_PMD_ERR("FM port configuration: Failed\n");
1461                         return -1;
1462                 }
1463                 eth_conf->rx_adv_conf.rss_conf.rss_hf = rss_conf->rss_hf;
1464         } else {
1465                 DPAA_PMD_ERR("Function not supported\n");
1466                 return -ENOTSUP;
1467         }
1468         return 0;
1469 }
1470
1471 static int
1472 dpaa_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
1473                            struct rte_eth_rss_conf *rss_conf)
1474 {
1475         struct rte_eth_dev_data *data = dev->data;
1476         struct rte_eth_conf *eth_conf = &data->dev_conf;
1477
1478         /* dpaa does not support rss_key, so length should be 0*/
1479         rss_conf->rss_key_len = 0;
1480         rss_conf->rss_hf = eth_conf->rx_adv_conf.rss_conf.rss_hf;
1481         return 0;
1482 }
1483
1484 static int dpaa_dev_queue_intr_enable(struct rte_eth_dev *dev,
1485                                       uint16_t queue_id)
1486 {
1487         struct dpaa_if *dpaa_intf = dev->data->dev_private;
1488         struct qman_fq *rxq = &dpaa_intf->rx_queues[queue_id];
1489
1490         if (!rxq->is_static)
1491                 return -EINVAL;
1492
1493         return qman_fq_portal_irqsource_add(rxq->qp, QM_PIRQ_DQRI);
1494 }
1495
1496 static int dpaa_dev_queue_intr_disable(struct rte_eth_dev *dev,
1497                                        uint16_t queue_id)
1498 {
1499         struct dpaa_if *dpaa_intf = dev->data->dev_private;
1500         struct qman_fq *rxq = &dpaa_intf->rx_queues[queue_id];
1501         uint32_t temp;
1502         ssize_t temp1;
1503
1504         if (!rxq->is_static)
1505                 return -EINVAL;
1506
1507         qman_fq_portal_irqsource_remove(rxq->qp, ~0);
1508
1509         temp1 = read(rxq->q_fd, &temp, sizeof(temp));
1510         if (temp1 != sizeof(temp))
1511                 DPAA_PMD_ERR("irq read error");
1512
1513         qman_fq_portal_thread_irq(rxq->qp);
1514
1515         return 0;
1516 }
1517
1518 static void
1519 dpaa_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
1520         struct rte_eth_rxq_info *qinfo)
1521 {
1522         struct dpaa_if *dpaa_intf = dev->data->dev_private;
1523         struct qman_fq *rxq;
1524
1525         rxq = dev->data->rx_queues[queue_id];
1526
1527         qinfo->mp = dpaa_intf->bp_info->mp;
1528         qinfo->scattered_rx = dev->data->scattered_rx;
1529         qinfo->nb_desc = rxq->nb_desc;
1530         qinfo->conf.rx_free_thresh = 1;
1531         qinfo->conf.rx_drop_en = 1;
1532         qinfo->conf.rx_deferred_start = 0;
1533         qinfo->conf.offloads = rxq->offloads;
1534 }
1535
1536 static void
1537 dpaa_txq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,
1538         struct rte_eth_txq_info *qinfo)
1539 {
1540         struct qman_fq *txq;
1541
1542         txq = dev->data->tx_queues[queue_id];
1543
1544         qinfo->nb_desc = txq->nb_desc;
1545         qinfo->conf.tx_thresh.pthresh = 0;
1546         qinfo->conf.tx_thresh.hthresh = 0;
1547         qinfo->conf.tx_thresh.wthresh = 0;
1548
1549         qinfo->conf.tx_free_thresh = 0;
1550         qinfo->conf.tx_rs_thresh = 0;
1551         qinfo->conf.offloads = txq->offloads;
1552         qinfo->conf.tx_deferred_start = 0;
1553 }
1554
1555 static struct eth_dev_ops dpaa_devops = {
1556         .dev_configure            = dpaa_eth_dev_configure,
1557         .dev_start                = dpaa_eth_dev_start,
1558         .dev_stop                 = dpaa_eth_dev_stop,
1559         .dev_close                = dpaa_eth_dev_close,
1560         .dev_infos_get            = dpaa_eth_dev_info,
1561         .dev_supported_ptypes_get = dpaa_supported_ptypes_get,
1562
1563         .rx_queue_setup           = dpaa_eth_rx_queue_setup,
1564         .tx_queue_setup           = dpaa_eth_tx_queue_setup,
1565         .rx_queue_release         = dpaa_eth_rx_queue_release,
1566         .tx_queue_release         = dpaa_eth_tx_queue_release,
1567         .rx_burst_mode_get        = dpaa_dev_rx_burst_mode_get,
1568         .tx_burst_mode_get        = dpaa_dev_tx_burst_mode_get,
1569         .rxq_info_get             = dpaa_rxq_info_get,
1570         .txq_info_get             = dpaa_txq_info_get,
1571
1572         .flow_ctrl_get            = dpaa_flow_ctrl_get,
1573         .flow_ctrl_set            = dpaa_flow_ctrl_set,
1574
1575         .link_update              = dpaa_eth_link_update,
1576         .stats_get                = dpaa_eth_stats_get,
1577         .xstats_get               = dpaa_dev_xstats_get,
1578         .xstats_get_by_id         = dpaa_xstats_get_by_id,
1579         .xstats_get_names_by_id   = dpaa_xstats_get_names_by_id,
1580         .xstats_get_names         = dpaa_xstats_get_names,
1581         .xstats_reset             = dpaa_eth_stats_reset,
1582         .stats_reset              = dpaa_eth_stats_reset,
1583         .promiscuous_enable       = dpaa_eth_promiscuous_enable,
1584         .promiscuous_disable      = dpaa_eth_promiscuous_disable,
1585         .allmulticast_enable      = dpaa_eth_multicast_enable,
1586         .allmulticast_disable     = dpaa_eth_multicast_disable,
1587         .mtu_set                  = dpaa_mtu_set,
1588         .dev_set_link_down        = dpaa_link_down,
1589         .dev_set_link_up          = dpaa_link_up,
1590         .mac_addr_add             = dpaa_dev_add_mac_addr,
1591         .mac_addr_remove          = dpaa_dev_remove_mac_addr,
1592         .mac_addr_set             = dpaa_dev_set_mac_addr,
1593
1594         .fw_version_get           = dpaa_fw_version_get,
1595
1596         .rx_queue_intr_enable     = dpaa_dev_queue_intr_enable,
1597         .rx_queue_intr_disable    = dpaa_dev_queue_intr_disable,
1598         .rss_hash_update          = dpaa_dev_rss_hash_update,
1599         .rss_hash_conf_get        = dpaa_dev_rss_hash_conf_get,
1600 };
1601
1602 static bool
1603 is_device_supported(struct rte_eth_dev *dev, struct rte_dpaa_driver *drv)
1604 {
1605         if (strcmp(dev->device->driver->name,
1606                    drv->driver.name))
1607                 return false;
1608
1609         return true;
1610 }
1611
1612 static bool
1613 is_dpaa_supported(struct rte_eth_dev *dev)
1614 {
1615         return is_device_supported(dev, &rte_dpaa_pmd);
1616 }
1617
1618 int
1619 rte_pmd_dpaa_set_tx_loopback(uint16_t port, uint8_t on)
1620 {
1621         struct rte_eth_dev *dev;
1622
1623         RTE_ETH_VALID_PORTID_OR_ERR_RET(port, -ENODEV);
1624
1625         dev = &rte_eth_devices[port];
1626
1627         if (!is_dpaa_supported(dev))
1628                 return -ENOTSUP;
1629
1630         if (on)
1631                 fman_if_loopback_enable(dev->process_private);
1632         else
1633                 fman_if_loopback_disable(dev->process_private);
1634
1635         return 0;
1636 }
1637
1638 static int dpaa_fc_set_default(struct dpaa_if *dpaa_intf,
1639                                struct fman_if *fman_intf)
1640 {
1641         struct rte_eth_fc_conf *fc_conf;
1642         int ret;
1643
1644         PMD_INIT_FUNC_TRACE();
1645
1646         if (!(dpaa_intf->fc_conf)) {
1647                 dpaa_intf->fc_conf = rte_zmalloc(NULL,
1648                         sizeof(struct rte_eth_fc_conf), MAX_CACHELINE);
1649                 if (!dpaa_intf->fc_conf) {
1650                         DPAA_PMD_ERR("unable to save flow control info");
1651                         return -ENOMEM;
1652                 }
1653         }
1654         fc_conf = dpaa_intf->fc_conf;
1655         ret = fman_if_get_fc_threshold(fman_intf);
1656         if (ret) {
1657                 fc_conf->mode = RTE_FC_TX_PAUSE;
1658                 fc_conf->pause_time = fman_if_get_fc_quanta(fman_intf);
1659         } else {
1660                 fc_conf->mode = RTE_FC_NONE;
1661         }
1662
1663         return 0;
1664 }
1665
1666 /* Initialise an Rx FQ */
1667 static int dpaa_rx_queue_init(struct qman_fq *fq, struct qman_cgr *cgr_rx,
1668                               uint32_t fqid)
1669 {
1670         struct qm_mcc_initfq opts = {0};
1671         int ret;
1672         u32 flags = QMAN_FQ_FLAG_NO_ENQUEUE;
1673         struct qm_mcc_initcgr cgr_opts = {
1674                 .we_mask = QM_CGR_WE_CS_THRES |
1675                                 QM_CGR_WE_CSTD_EN |
1676                                 QM_CGR_WE_MODE,
1677                 .cgr = {
1678                         .cstd_en = QM_CGR_EN,
1679                         .mode = QMAN_CGR_MODE_FRAME
1680                 }
1681         };
1682
1683         if (fmc_q || default_q) {
1684                 ret = qman_reserve_fqid(fqid);
1685                 if (ret) {
1686                         DPAA_PMD_ERR("reserve rx fqid 0x%x failed, ret: %d",
1687                                      fqid, ret);
1688                         return -EINVAL;
1689                 }
1690         }
1691
1692         DPAA_PMD_DEBUG("creating rx fq %p, fqid 0x%x", fq, fqid);
1693         ret = qman_create_fq(fqid, flags, fq);
1694         if (ret) {
1695                 DPAA_PMD_ERR("create rx fqid 0x%x failed with ret: %d",
1696                         fqid, ret);
1697                 return ret;
1698         }
1699         fq->is_static = false;
1700
1701         dpaa_poll_queue_default_config(&opts);
1702
1703         if (cgr_rx) {
1704                 /* Enable tail drop with cgr on this queue */
1705                 qm_cgr_cs_thres_set64(&cgr_opts.cgr.cs_thres, td_threshold, 0);
1706                 cgr_rx->cb = NULL;
1707                 ret = qman_create_cgr(cgr_rx, QMAN_CGR_FLAG_USE_INIT,
1708                                       &cgr_opts);
1709                 if (ret) {
1710                         DPAA_PMD_WARN(
1711                                 "rx taildrop init fail on rx fqid 0x%x(ret=%d)",
1712                                 fq->fqid, ret);
1713                         goto without_cgr;
1714                 }
1715                 opts.we_mask |= QM_INITFQ_WE_CGID;
1716                 opts.fqd.cgid = cgr_rx->cgrid;
1717                 opts.fqd.fq_ctrl |= QM_FQCTRL_CGE;
1718         }
1719 without_cgr:
1720         ret = qman_init_fq(fq, 0, &opts);
1721         if (ret)
1722                 DPAA_PMD_ERR("init rx fqid 0x%x failed with ret:%d", fqid, ret);
1723         return ret;
1724 }
1725
1726 /* Initialise a Tx FQ */
1727 static int dpaa_tx_queue_init(struct qman_fq *fq,
1728                               struct fman_if *fman_intf,
1729                               struct qman_cgr *cgr_tx)
1730 {
1731         struct qm_mcc_initfq opts = {0};
1732         struct qm_mcc_initcgr cgr_opts = {
1733                 .we_mask = QM_CGR_WE_CS_THRES |
1734                                 QM_CGR_WE_CSTD_EN |
1735                                 QM_CGR_WE_MODE,
1736                 .cgr = {
1737                         .cstd_en = QM_CGR_EN,
1738                         .mode = QMAN_CGR_MODE_FRAME
1739                 }
1740         };
1741         int ret;
1742
1743         ret = qman_create_fq(0, QMAN_FQ_FLAG_DYNAMIC_FQID |
1744                              QMAN_FQ_FLAG_TO_DCPORTAL, fq);
1745         if (ret) {
1746                 DPAA_PMD_ERR("create tx fq failed with ret: %d", ret);
1747                 return ret;
1748         }
1749         opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL |
1750                        QM_INITFQ_WE_CONTEXTB | QM_INITFQ_WE_CONTEXTA;
1751         opts.fqd.dest.channel = fman_intf->tx_channel_id;
1752         opts.fqd.dest.wq = DPAA_IF_TX_PRIORITY;
1753         opts.fqd.fq_ctrl = QM_FQCTRL_PREFERINCACHE;
1754         opts.fqd.context_b = 0;
1755         /* no tx-confirmation */
1756         opts.fqd.context_a.hi = 0x80000000 | fman_dealloc_bufs_mask_hi;
1757         opts.fqd.context_a.lo = 0 | fman_dealloc_bufs_mask_lo;
1758         DPAA_PMD_DEBUG("init tx fq %p, fqid 0x%x", fq, fq->fqid);
1759
1760         if (cgr_tx) {
1761                 /* Enable tail drop with cgr on this queue */
1762                 qm_cgr_cs_thres_set64(&cgr_opts.cgr.cs_thres,
1763                                       td_tx_threshold, 0);
1764                 cgr_tx->cb = NULL;
1765                 ret = qman_create_cgr(cgr_tx, QMAN_CGR_FLAG_USE_INIT,
1766                                       &cgr_opts);
1767                 if (ret) {
1768                         DPAA_PMD_WARN(
1769                                 "rx taildrop init fail on rx fqid 0x%x(ret=%d)",
1770                                 fq->fqid, ret);
1771                         goto without_cgr;
1772                 }
1773                 opts.we_mask |= QM_INITFQ_WE_CGID;
1774                 opts.fqd.cgid = cgr_tx->cgrid;
1775                 opts.fqd.fq_ctrl |= QM_FQCTRL_CGE;
1776                 DPAA_PMD_DEBUG("Tx FQ tail drop enabled, threshold = %d\n",
1777                                 td_tx_threshold);
1778         }
1779 without_cgr:
1780         ret = qman_init_fq(fq, QMAN_INITFQ_FLAG_SCHED, &opts);
1781         if (ret)
1782                 DPAA_PMD_ERR("init tx fqid 0x%x failed %d", fq->fqid, ret);
1783         return ret;
1784 }
1785
1786 #ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER
1787 /* Initialise a DEBUG FQ ([rt]x_error, rx_default). */
1788 static int dpaa_debug_queue_init(struct qman_fq *fq, uint32_t fqid)
1789 {
1790         struct qm_mcc_initfq opts = {0};
1791         int ret;
1792
1793         PMD_INIT_FUNC_TRACE();
1794
1795         ret = qman_reserve_fqid(fqid);
1796         if (ret) {
1797                 DPAA_PMD_ERR("Reserve debug fqid %d failed with ret: %d",
1798                         fqid, ret);
1799                 return -EINVAL;
1800         }
1801         /* "map" this Rx FQ to one of the interfaces Tx FQID */
1802         DPAA_PMD_DEBUG("Creating debug fq %p, fqid %d", fq, fqid);
1803         ret = qman_create_fq(fqid, QMAN_FQ_FLAG_NO_ENQUEUE, fq);
1804         if (ret) {
1805                 DPAA_PMD_ERR("create debug fqid %d failed with ret: %d",
1806                         fqid, ret);
1807                 return ret;
1808         }
1809         opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL;
1810         opts.fqd.dest.wq = DPAA_IF_DEBUG_PRIORITY;
1811         ret = qman_init_fq(fq, 0, &opts);
1812         if (ret)
1813                 DPAA_PMD_ERR("init debug fqid %d failed with ret: %d",
1814                             fqid, ret);
1815         return ret;
1816 }
1817 #endif
1818
1819 /* Initialise a network interface */
1820 static int
1821 dpaa_dev_init_secondary(struct rte_eth_dev *eth_dev)
1822 {
1823         struct rte_dpaa_device *dpaa_device;
1824         struct fm_eth_port_cfg *cfg;
1825         struct dpaa_if *dpaa_intf;
1826         struct fman_if *fman_intf;
1827         int dev_id;
1828
1829         PMD_INIT_FUNC_TRACE();
1830
1831         dpaa_device = DEV_TO_DPAA_DEVICE(eth_dev->device);
1832         dev_id = dpaa_device->id.dev_id;
1833         cfg = dpaa_get_eth_port_cfg(dev_id);
1834         fman_intf = cfg->fman_if;
1835         eth_dev->process_private = fman_intf;
1836
1837         /* Plugging of UCODE burst API not supported in Secondary */
1838         dpaa_intf = eth_dev->data->dev_private;
1839         eth_dev->rx_pkt_burst = dpaa_eth_queue_rx;
1840         if (dpaa_intf->cgr_tx)
1841                 eth_dev->tx_pkt_burst = dpaa_eth_queue_tx_slow;
1842         else
1843                 eth_dev->tx_pkt_burst = dpaa_eth_queue_tx;
1844 #ifdef CONFIG_FSL_QMAN_FQ_LOOKUP
1845         qman_set_fq_lookup_table(
1846                 dpaa_intf->rx_queues->qman_fq_lookup_table);
1847 #endif
1848
1849         return 0;
1850 }
1851
1852 /* Initialise a network interface */
1853 static int
1854 dpaa_dev_init(struct rte_eth_dev *eth_dev)
1855 {
1856         int num_rx_fqs, fqid;
1857         int loop, ret = 0;
1858         int dev_id;
1859         struct rte_dpaa_device *dpaa_device;
1860         struct dpaa_if *dpaa_intf;
1861         struct fm_eth_port_cfg *cfg;
1862         struct fman_if *fman_intf;
1863         struct fman_if_bpool *bp, *tmp_bp;
1864         uint32_t cgrid[DPAA_MAX_NUM_PCD_QUEUES];
1865         uint32_t cgrid_tx[MAX_DPAA_CORES];
1866         uint32_t dev_rx_fqids[DPAA_MAX_NUM_PCD_QUEUES];
1867         int8_t dev_vspids[DPAA_MAX_NUM_PCD_QUEUES];
1868         int8_t vsp_id = -1;
1869
1870         PMD_INIT_FUNC_TRACE();
1871
1872         dpaa_device = DEV_TO_DPAA_DEVICE(eth_dev->device);
1873         dev_id = dpaa_device->id.dev_id;
1874         dpaa_intf = eth_dev->data->dev_private;
1875         cfg = dpaa_get_eth_port_cfg(dev_id);
1876         fman_intf = cfg->fman_if;
1877
1878         dpaa_intf->name = dpaa_device->name;
1879
1880         /* save fman_if & cfg in the interface struture */
1881         eth_dev->process_private = fman_intf;
1882         dpaa_intf->ifid = dev_id;
1883         dpaa_intf->cfg = cfg;
1884
1885         memset((char *)dev_rx_fqids, 0,
1886                 sizeof(uint32_t) * DPAA_MAX_NUM_PCD_QUEUES);
1887
1888         memset(dev_vspids, -1, DPAA_MAX_NUM_PCD_QUEUES);
1889
1890         /* Initialize Rx FQ's */
1891         if (default_q) {
1892                 num_rx_fqs = DPAA_DEFAULT_NUM_PCD_QUEUES;
1893         } else if (fmc_q) {
1894                 num_rx_fqs = dpaa_port_fmc_init(fman_intf, dev_rx_fqids,
1895                                                 dev_vspids,
1896                                                 DPAA_MAX_NUM_PCD_QUEUES);
1897                 if (num_rx_fqs < 0) {
1898                         DPAA_PMD_ERR("%s FMC initializes failed!",
1899                                 dpaa_intf->name);
1900                         goto free_rx;
1901                 }
1902                 if (!num_rx_fqs) {
1903                         DPAA_PMD_WARN("%s is not configured by FMC.",
1904                                 dpaa_intf->name);
1905                 }
1906         } else {
1907                 /* FMCLESS mode, load balance to multiple cores.*/
1908                 num_rx_fqs = rte_lcore_count();
1909         }
1910
1911         /* Each device can not have more than DPAA_MAX_NUM_PCD_QUEUES RX
1912          * queues.
1913          */
1914         if (num_rx_fqs < 0 || num_rx_fqs > DPAA_MAX_NUM_PCD_QUEUES) {
1915                 DPAA_PMD_ERR("Invalid number of RX queues\n");
1916                 return -EINVAL;
1917         }
1918
1919         if (num_rx_fqs > 0) {
1920                 dpaa_intf->rx_queues = rte_zmalloc(NULL,
1921                         sizeof(struct qman_fq) * num_rx_fqs, MAX_CACHELINE);
1922                 if (!dpaa_intf->rx_queues) {
1923                         DPAA_PMD_ERR("Failed to alloc mem for RX queues\n");
1924                         return -ENOMEM;
1925                 }
1926         } else {
1927                 dpaa_intf->rx_queues = NULL;
1928         }
1929
1930         memset(cgrid, 0, sizeof(cgrid));
1931         memset(cgrid_tx, 0, sizeof(cgrid_tx));
1932
1933         /* if DPAA_TX_TAILDROP_THRESHOLD is set, use that value; if 0, it means
1934          * Tx tail drop is disabled.
1935          */
1936         if (getenv("DPAA_TX_TAILDROP_THRESHOLD")) {
1937                 td_tx_threshold = atoi(getenv("DPAA_TX_TAILDROP_THRESHOLD"));
1938                 DPAA_PMD_DEBUG("Tail drop threshold env configured: %u",
1939                                td_tx_threshold);
1940                 /* if a very large value is being configured */
1941                 if (td_tx_threshold > UINT16_MAX)
1942                         td_tx_threshold = CGR_RX_PERFQ_THRESH;
1943         }
1944
1945         /* If congestion control is enabled globally*/
1946         if (num_rx_fqs > 0 && td_threshold) {
1947                 dpaa_intf->cgr_rx = rte_zmalloc(NULL,
1948                         sizeof(struct qman_cgr) * num_rx_fqs, MAX_CACHELINE);
1949                 if (!dpaa_intf->cgr_rx) {
1950                         DPAA_PMD_ERR("Failed to alloc mem for cgr_rx\n");
1951                         ret = -ENOMEM;
1952                         goto free_rx;
1953                 }
1954
1955                 ret = qman_alloc_cgrid_range(&cgrid[0], num_rx_fqs, 1, 0);
1956                 if (ret != num_rx_fqs) {
1957                         DPAA_PMD_WARN("insufficient CGRIDs available");
1958                         ret = -EINVAL;
1959                         goto free_rx;
1960                 }
1961         } else {
1962                 dpaa_intf->cgr_rx = NULL;
1963         }
1964
1965         if (!fmc_q && !default_q) {
1966                 ret = qman_alloc_fqid_range(dev_rx_fqids, num_rx_fqs,
1967                                             num_rx_fqs, 0);
1968                 if (ret < 0) {
1969                         DPAA_PMD_ERR("Failed to alloc rx fqid's\n");
1970                         goto free_rx;
1971                 }
1972         }
1973
1974         for (loop = 0; loop < num_rx_fqs; loop++) {
1975                 if (default_q)
1976                         fqid = cfg->rx_def;
1977                 else
1978                         fqid = dev_rx_fqids[loop];
1979
1980                 vsp_id = dev_vspids[loop];
1981
1982                 if (dpaa_intf->cgr_rx)
1983                         dpaa_intf->cgr_rx[loop].cgrid = cgrid[loop];
1984
1985                 ret = dpaa_rx_queue_init(&dpaa_intf->rx_queues[loop],
1986                         dpaa_intf->cgr_rx ? &dpaa_intf->cgr_rx[loop] : NULL,
1987                         fqid);
1988                 if (ret)
1989                         goto free_rx;
1990                 dpaa_intf->rx_queues[loop].vsp_id = vsp_id;
1991                 dpaa_intf->rx_queues[loop].dpaa_intf = dpaa_intf;
1992         }
1993         dpaa_intf->nb_rx_queues = num_rx_fqs;
1994
1995         /* Initialise Tx FQs.free_rx Have as many Tx FQ's as number of cores */
1996         dpaa_intf->tx_queues = rte_zmalloc(NULL, sizeof(struct qman_fq) *
1997                 MAX_DPAA_CORES, MAX_CACHELINE);
1998         if (!dpaa_intf->tx_queues) {
1999                 DPAA_PMD_ERR("Failed to alloc mem for TX queues\n");
2000                 ret = -ENOMEM;
2001                 goto free_rx;
2002         }
2003
2004         /* If congestion control is enabled globally*/
2005         if (td_tx_threshold) {
2006                 dpaa_intf->cgr_tx = rte_zmalloc(NULL,
2007                         sizeof(struct qman_cgr) * MAX_DPAA_CORES,
2008                         MAX_CACHELINE);
2009                 if (!dpaa_intf->cgr_tx) {
2010                         DPAA_PMD_ERR("Failed to alloc mem for cgr_tx\n");
2011                         ret = -ENOMEM;
2012                         goto free_rx;
2013                 }
2014
2015                 ret = qman_alloc_cgrid_range(&cgrid_tx[0], MAX_DPAA_CORES,
2016                                              1, 0);
2017                 if (ret != MAX_DPAA_CORES) {
2018                         DPAA_PMD_WARN("insufficient CGRIDs available");
2019                         ret = -EINVAL;
2020                         goto free_rx;
2021                 }
2022         } else {
2023                 dpaa_intf->cgr_tx = NULL;
2024         }
2025
2026
2027         for (loop = 0; loop < MAX_DPAA_CORES; loop++) {
2028                 if (dpaa_intf->cgr_tx)
2029                         dpaa_intf->cgr_tx[loop].cgrid = cgrid_tx[loop];
2030
2031                 ret = dpaa_tx_queue_init(&dpaa_intf->tx_queues[loop],
2032                         fman_intf,
2033                         dpaa_intf->cgr_tx ? &dpaa_intf->cgr_tx[loop] : NULL);
2034                 if (ret)
2035                         goto free_tx;
2036                 dpaa_intf->tx_queues[loop].dpaa_intf = dpaa_intf;
2037         }
2038         dpaa_intf->nb_tx_queues = MAX_DPAA_CORES;
2039
2040 #ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER
2041         ret = dpaa_debug_queue_init(&dpaa_intf->debug_queues
2042                         [DPAA_DEBUG_FQ_RX_ERROR], fman_intf->fqid_rx_err);
2043         if (ret) {
2044                 DPAA_PMD_ERR("DPAA RX ERROR queue init failed!");
2045                 goto free_tx;
2046         }
2047         dpaa_intf->debug_queues[DPAA_DEBUG_FQ_RX_ERROR].dpaa_intf = dpaa_intf;
2048         ret = dpaa_debug_queue_init(&dpaa_intf->debug_queues
2049                         [DPAA_DEBUG_FQ_TX_ERROR], fman_intf->fqid_tx_err);
2050         if (ret) {
2051                 DPAA_PMD_ERR("DPAA TX ERROR queue init failed!");
2052                 goto free_tx;
2053         }
2054         dpaa_intf->debug_queues[DPAA_DEBUG_FQ_TX_ERROR].dpaa_intf = dpaa_intf;
2055 #endif
2056
2057         DPAA_PMD_DEBUG("All frame queues created");
2058
2059         /* Get the initial configuration for flow control */
2060         dpaa_fc_set_default(dpaa_intf, fman_intf);
2061
2062         /* reset bpool list, initialize bpool dynamically */
2063         list_for_each_entry_safe(bp, tmp_bp, &cfg->fman_if->bpool_list, node) {
2064                 list_del(&bp->node);
2065                 rte_free(bp);
2066         }
2067
2068         /* Populate ethdev structure */
2069         eth_dev->dev_ops = &dpaa_devops;
2070         eth_dev->rx_queue_count = dpaa_dev_rx_queue_count;
2071         eth_dev->rx_pkt_burst = dpaa_eth_queue_rx;
2072         eth_dev->tx_pkt_burst = dpaa_eth_tx_drop_all;
2073
2074         /* Allocate memory for storing MAC addresses */
2075         eth_dev->data->mac_addrs = rte_zmalloc("mac_addr",
2076                 RTE_ETHER_ADDR_LEN * DPAA_MAX_MAC_FILTER, 0);
2077         if (eth_dev->data->mac_addrs == NULL) {
2078                 DPAA_PMD_ERR("Failed to allocate %d bytes needed to "
2079                                                 "store MAC addresses",
2080                                 RTE_ETHER_ADDR_LEN * DPAA_MAX_MAC_FILTER);
2081                 ret = -ENOMEM;
2082                 goto free_tx;
2083         }
2084
2085         /* copy the primary mac address */
2086         rte_ether_addr_copy(&fman_intf->mac_addr, &eth_dev->data->mac_addrs[0]);
2087
2088         RTE_LOG(INFO, PMD, "net: dpaa: %s: %02x:%02x:%02x:%02x:%02x:%02x\n",
2089                 dpaa_device->name,
2090                 fman_intf->mac_addr.addr_bytes[0],
2091                 fman_intf->mac_addr.addr_bytes[1],
2092                 fman_intf->mac_addr.addr_bytes[2],
2093                 fman_intf->mac_addr.addr_bytes[3],
2094                 fman_intf->mac_addr.addr_bytes[4],
2095                 fman_intf->mac_addr.addr_bytes[5]);
2096
2097         if (!fman_intf->is_shared_mac) {
2098                 /* Configure error packet handling */
2099                 fman_if_receive_rx_errors(fman_intf,
2100                         FM_FD_RX_STATUS_ERR_MASK);
2101                 /* Disable RX mode */
2102                 fman_if_disable_rx(fman_intf);
2103                 /* Disable promiscuous mode */
2104                 fman_if_promiscuous_disable(fman_intf);
2105                 /* Disable multicast */
2106                 fman_if_reset_mcast_filter_table(fman_intf);
2107                 /* Reset interface statistics */
2108                 fman_if_stats_reset(fman_intf);
2109                 /* Disable SG by default */
2110                 fman_if_set_sg(fman_intf, 0);
2111                 fman_if_set_maxfrm(fman_intf,
2112                                    RTE_ETHER_MAX_LEN + VLAN_TAG_SIZE);
2113         }
2114
2115         return 0;
2116
2117 free_tx:
2118         rte_free(dpaa_intf->tx_queues);
2119         dpaa_intf->tx_queues = NULL;
2120         dpaa_intf->nb_tx_queues = 0;
2121
2122 free_rx:
2123         rte_free(dpaa_intf->cgr_rx);
2124         rte_free(dpaa_intf->cgr_tx);
2125         rte_free(dpaa_intf->rx_queues);
2126         dpaa_intf->rx_queues = NULL;
2127         dpaa_intf->nb_rx_queues = 0;
2128         return ret;
2129 }
2130
2131 static int
2132 rte_dpaa_probe(struct rte_dpaa_driver *dpaa_drv,
2133                struct rte_dpaa_device *dpaa_dev)
2134 {
2135         int diag;
2136         int ret;
2137         struct rte_eth_dev *eth_dev;
2138
2139         PMD_INIT_FUNC_TRACE();
2140
2141         if ((DPAA_MBUF_HW_ANNOTATION + DPAA_FD_PTA_SIZE) >
2142                 RTE_PKTMBUF_HEADROOM) {
2143                 DPAA_PMD_ERR(
2144                 "RTE_PKTMBUF_HEADROOM(%d) shall be > DPAA Annotation req(%d)",
2145                 RTE_PKTMBUF_HEADROOM,
2146                 DPAA_MBUF_HW_ANNOTATION + DPAA_FD_PTA_SIZE);
2147
2148                 return -1;
2149         }
2150
2151         /* In case of secondary process, the device is already configured
2152          * and no further action is required, except portal initialization
2153          * and verifying secondary attachment to port name.
2154          */
2155         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2156                 eth_dev = rte_eth_dev_attach_secondary(dpaa_dev->name);
2157                 if (!eth_dev)
2158                         return -ENOMEM;
2159                 eth_dev->device = &dpaa_dev->device;
2160                 eth_dev->dev_ops = &dpaa_devops;
2161
2162                 ret = dpaa_dev_init_secondary(eth_dev);
2163                 if (ret != 0) {
2164                         RTE_LOG(ERR, PMD, "secondary dev init failed\n");
2165                         return ret;
2166                 }
2167
2168                 rte_eth_dev_probing_finish(eth_dev);
2169                 return 0;
2170         }
2171
2172         if (!is_global_init && (rte_eal_process_type() == RTE_PROC_PRIMARY)) {
2173                 if (access("/tmp/fmc.bin", F_OK) == -1) {
2174                         DPAA_PMD_INFO("* FMC not configured.Enabling default mode");
2175                         default_q = 1;
2176                 }
2177
2178                 if (!(default_q || fmc_q)) {
2179                         if (dpaa_fm_init()) {
2180                                 DPAA_PMD_ERR("FM init failed\n");
2181                                 return -1;
2182                         }
2183                 }
2184
2185                 /* disabling the default push mode for LS1043 */
2186                 if (dpaa_svr_family == SVR_LS1043A_FAMILY)
2187                         dpaa_push_mode_max_queue = 0;
2188
2189                 /* if push mode queues to be enabled. Currenly we are allowing
2190                  * only one queue per thread.
2191                  */
2192                 if (getenv("DPAA_PUSH_QUEUES_NUMBER")) {
2193                         dpaa_push_mode_max_queue =
2194                                         atoi(getenv("DPAA_PUSH_QUEUES_NUMBER"));
2195                         if (dpaa_push_mode_max_queue > DPAA_MAX_PUSH_MODE_QUEUE)
2196                             dpaa_push_mode_max_queue = DPAA_MAX_PUSH_MODE_QUEUE;
2197                 }
2198
2199                 is_global_init = 1;
2200         }
2201
2202         if (unlikely(!DPAA_PER_LCORE_PORTAL)) {
2203                 ret = rte_dpaa_portal_init((void *)1);
2204                 if (ret) {
2205                         DPAA_PMD_ERR("Unable to initialize portal");
2206                         return ret;
2207                 }
2208         }
2209
2210         eth_dev = rte_eth_dev_allocate(dpaa_dev->name);
2211         if (!eth_dev)
2212                 return -ENOMEM;
2213
2214         eth_dev->data->dev_private =
2215                         rte_zmalloc("ethdev private structure",
2216                                         sizeof(struct dpaa_if),
2217                                         RTE_CACHE_LINE_SIZE);
2218         if (!eth_dev->data->dev_private) {
2219                 DPAA_PMD_ERR("Cannot allocate memzone for port data");
2220                 rte_eth_dev_release_port(eth_dev);
2221                 return -ENOMEM;
2222         }
2223
2224         eth_dev->device = &dpaa_dev->device;
2225         dpaa_dev->eth_dev = eth_dev;
2226
2227         qman_ern_register_cb(dpaa_free_mbuf);
2228
2229         if (dpaa_drv->drv_flags & RTE_DPAA_DRV_INTR_LSC)
2230                 eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC;
2231
2232         eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
2233
2234         /* Invoke PMD device initialization function */
2235         diag = dpaa_dev_init(eth_dev);
2236         if (diag == 0) {
2237                 rte_eth_dev_probing_finish(eth_dev);
2238                 return 0;
2239         }
2240
2241         rte_eth_dev_release_port(eth_dev);
2242         return diag;
2243 }
2244
2245 static int
2246 rte_dpaa_remove(struct rte_dpaa_device *dpaa_dev)
2247 {
2248         struct rte_eth_dev *eth_dev;
2249         int ret;
2250
2251         PMD_INIT_FUNC_TRACE();
2252
2253         eth_dev = dpaa_dev->eth_dev;
2254         dpaa_eth_dev_close(eth_dev);
2255         ret = rte_eth_dev_release_port(eth_dev);
2256
2257         return ret;
2258 }
2259
2260 static void __attribute__((destructor(102))) dpaa_finish(void)
2261 {
2262         /* For secondary, primary will do all the cleanup */
2263         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2264                 return;
2265
2266         if (!(default_q || fmc_q)) {
2267                 unsigned int i;
2268
2269                 for (i = 0; i < RTE_MAX_ETHPORTS; i++) {
2270                         if (rte_eth_devices[i].dev_ops == &dpaa_devops) {
2271                                 struct rte_eth_dev *dev = &rte_eth_devices[i];
2272                                 struct dpaa_if *dpaa_intf =
2273                                         dev->data->dev_private;
2274                                 struct fman_if *fif =
2275                                         dev->process_private;
2276                                 if (dpaa_intf->port_handle)
2277                                         if (dpaa_fm_deconfig(dpaa_intf, fif))
2278                                                 DPAA_PMD_WARN("DPAA FM "
2279                                                         "deconfig failed\n");
2280                                 if (fif->num_profiles) {
2281                                         if (dpaa_port_vsp_cleanup(dpaa_intf,
2282                                                                   fif))
2283                                                 DPAA_PMD_WARN("DPAA FM vsp cleanup failed\n");
2284                                 }
2285                         }
2286                 }
2287                 if (is_global_init)
2288                         if (dpaa_fm_term())
2289                                 DPAA_PMD_WARN("DPAA FM term failed\n");
2290
2291                 is_global_init = 0;
2292
2293                 DPAA_PMD_INFO("DPAA fman cleaned up");
2294         }
2295 }
2296
2297 static struct rte_dpaa_driver rte_dpaa_pmd = {
2298         .drv_flags = RTE_DPAA_DRV_INTR_LSC,
2299         .drv_type = FSL_DPAA_ETH,
2300         .probe = rte_dpaa_probe,
2301         .remove = rte_dpaa_remove,
2302 };
2303
2304 RTE_PMD_REGISTER_DPAA(net_dpaa, rte_dpaa_pmd);
2305 RTE_LOG_REGISTER(dpaa_logtype_pmd, pmd.net.dpaa, NOTICE);