drivers/net/dpaa2/dpaa2_ethdev.c

   1 /* * SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  *   Copyright (c) 2016 Freescale Semiconductor, Inc. All rights reserved.
   4  *   Copyright 2016-2020 NXP
   5  *
   6  */
   7
   8 #include <time.h>
   9 #include <net/if.h>
  10
  11 #include <rte_mbuf.h>
  12 #include <rte_ethdev_driver.h>
  13 #include <rte_malloc.h>
  14 #include <rte_memcpy.h>
  15 #include <rte_string_fns.h>
  16 #include <rte_cycles.h>
  17 #include <rte_kvargs.h>
  18 #include <rte_dev.h>
  19 #include <rte_fslmc.h>
  20 #include <rte_flow_driver.h>
  21
  22 #include "dpaa2_pmd_logs.h"
  23 #include <fslmc_vfio.h>
  24 #include <dpaa2_hw_pvt.h>
  25 #include <dpaa2_hw_mempool.h>
  26 #include <dpaa2_hw_dpio.h>
  27 #include <mc/fsl_dpmng.h>
  28 #include "dpaa2_ethdev.h"
  29 #include "dpaa2_sparser.h"
  30 #include <fsl_qbman_debug.h>
  31
  32 #define DRIVER_LOOPBACK_MODE "drv_loopback"
  33 #define DRIVER_NO_PREFETCH_MODE "drv_no_prefetch"
  34
  35 /* Supported Rx offloads */
  36 static uint64_t dev_rx_offloads_sup =
  37                 DEV_RX_OFFLOAD_CHECKSUM |
  38                 DEV_RX_OFFLOAD_SCTP_CKSUM |
  39                 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM |
  40                 DEV_RX_OFFLOAD_OUTER_UDP_CKSUM |
  41                 DEV_RX_OFFLOAD_VLAN_STRIP |
  42                 DEV_RX_OFFLOAD_VLAN_FILTER |
  43                 DEV_RX_OFFLOAD_JUMBO_FRAME |
  44                 DEV_RX_OFFLOAD_TIMESTAMP;
  45
  46 /* Rx offloads which cannot be disabled */
  47 static uint64_t dev_rx_offloads_nodis =
  48                 DEV_RX_OFFLOAD_RSS_HASH |
  49                 DEV_RX_OFFLOAD_SCATTER;
  50
  51 /* Supported Tx offloads */
  52 static uint64_t dev_tx_offloads_sup =
  53                 DEV_TX_OFFLOAD_VLAN_INSERT |
  54                 DEV_TX_OFFLOAD_IPV4_CKSUM |
  55                 DEV_TX_OFFLOAD_UDP_CKSUM |
  56                 DEV_TX_OFFLOAD_TCP_CKSUM |
  57                 DEV_TX_OFFLOAD_SCTP_CKSUM |
  58                 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM |
  59                 DEV_TX_OFFLOAD_MT_LOCKFREE |
  60                 DEV_TX_OFFLOAD_MBUF_FAST_FREE;
  61
  62 /* Tx offloads which cannot be disabled */
  63 static uint64_t dev_tx_offloads_nodis =
  64                 DEV_TX_OFFLOAD_MULTI_SEGS;
  65
  66 /* enable timestamp in mbuf */
  67 bool dpaa2_enable_ts;
  68
  69 struct rte_dpaa2_xstats_name_off {
  70         char name[RTE_ETH_XSTATS_NAME_SIZE];
  71         uint8_t page_id; /* dpni statistics page id */
  72         uint8_t stats_id; /* stats id in the given page */
  73 };
  74
  75 static const struct rte_dpaa2_xstats_name_off dpaa2_xstats_strings[] = {
  76         {"ingress_multicast_frames", 0, 2},
  77         {"ingress_multicast_bytes", 0, 3},
  78         {"ingress_broadcast_frames", 0, 4},
  79         {"ingress_broadcast_bytes", 0, 5},
  80         {"egress_multicast_frames", 1, 2},
  81         {"egress_multicast_bytes", 1, 3},
  82         {"egress_broadcast_frames", 1, 4},
  83         {"egress_broadcast_bytes", 1, 5},
  84         {"ingress_filtered_frames", 2, 0},
  85         {"ingress_discarded_frames", 2, 1},
  86         {"ingress_nobuffer_discards", 2, 2},
  87         {"egress_discarded_frames", 2, 3},
  88         {"egress_confirmed_frames", 2, 4},
  89         {"cgr_reject_frames", 4, 0},
  90         {"cgr_reject_bytes", 4, 1},
  91 };
  92
  93 static const enum rte_filter_op dpaa2_supported_filter_ops[] = {
  94         RTE_ETH_FILTER_ADD,
  95         RTE_ETH_FILTER_DELETE,
  96         RTE_ETH_FILTER_UPDATE,
  97         RTE_ETH_FILTER_FLUSH,
  98         RTE_ETH_FILTER_GET
  99 };
 100
 101 static struct rte_dpaa2_driver rte_dpaa2_pmd;
 102 static int dpaa2_dev_uninit(struct rte_eth_dev *eth_dev);
 103 static int dpaa2_dev_link_update(struct rte_eth_dev *dev,
 104                                  int wait_to_complete);
 105 static int dpaa2_dev_set_link_up(struct rte_eth_dev *dev);
 106 static int dpaa2_dev_set_link_down(struct rte_eth_dev *dev);
 107 static int dpaa2_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu);
 108
 109 static int
 110 dpaa2_vlan_filter_set(struct rte_eth_dev *dev, uint16_t vlan_id, int on)
 111 {
 112         int ret;
 113         struct dpaa2_dev_priv *priv = dev->data->dev_private;
 114         struct fsl_mc_io *dpni = dev->process_private;
 115
 116         PMD_INIT_FUNC_TRACE();
 117
 118         if (dpni == NULL) {
 119                 DPAA2_PMD_ERR("dpni is NULL");
 120                 return -1;
 121         }
 122
 123         if (on)
 124                 ret = dpni_add_vlan_id(dpni, CMD_PRI_LOW, priv->token,
 125                                        vlan_id, 0, 0, 0);
 126         else
 127                 ret = dpni_remove_vlan_id(dpni, CMD_PRI_LOW,
 128                                           priv->token, vlan_id);
 129
 130         if (ret < 0)
 131                 DPAA2_PMD_ERR("ret = %d Unable to add/rem vlan %d hwid =%d",
 132                               ret, vlan_id, priv->hw_id);
 133
 134         return ret;
 135 }
 136
 137 static int
 138 dpaa2_vlan_offload_set(struct rte_eth_dev *dev, int mask)
 139 {
 140         struct dpaa2_dev_priv *priv = dev->data->dev_private;
 141         struct fsl_mc_io *dpni = dev->process_private;
 142         int ret = 0;
 143
 144         PMD_INIT_FUNC_TRACE();
 145
 146         if (mask & ETH_VLAN_FILTER_MASK) {
 147                 /* VLAN Filter not avaialble */
 148                 if (!priv->max_vlan_filters) {
 149                         DPAA2_PMD_INFO("VLAN filter not available");
 150                         return -ENOTSUP;
 151                 }
 152
 153                 if (dev->data->dev_conf.rxmode.offloads &
 154                         DEV_RX_OFFLOAD_VLAN_FILTER)
 155                         ret = dpni_enable_vlan_filter(dpni, CMD_PRI_LOW,
 156                                                       priv->token, true);
 157                 else
 158                         ret = dpni_enable_vlan_filter(dpni, CMD_PRI_LOW,
 159                                                       priv->token, false);
 160                 if (ret < 0)
 161                         DPAA2_PMD_INFO("Unable to set vlan filter = %d", ret);
 162         }
 163
 164         return ret;
 165 }
 166
 167 static int
 168 dpaa2_vlan_tpid_set(struct rte_eth_dev *dev,
 169                       enum rte_vlan_type vlan_type __rte_unused,
 170                       uint16_t tpid)
 171 {
 172         struct dpaa2_dev_priv *priv = dev->data->dev_private;
 173         struct fsl_mc_io *dpni = dev->process_private;
 174         int ret = -ENOTSUP;
 175
 176         PMD_INIT_FUNC_TRACE();
 177
 178         /* nothing to be done for standard vlan tpids */
 179         if (tpid == 0x8100 || tpid == 0x88A8)
 180                 return 0;
 181
 182         ret = dpni_add_custom_tpid(dpni, CMD_PRI_LOW,
 183                                    priv->token, tpid);
 184         if (ret < 0)
 185                 DPAA2_PMD_INFO("Unable to set vlan tpid = %d", ret);
 186         /* if already configured tpids, remove them first */
 187         if (ret == -EBUSY) {
 188                 struct dpni_custom_tpid_cfg tpid_list = {0};
 189
 190                 ret = dpni_get_custom_tpid(dpni, CMD_PRI_LOW,
 191                                    priv->token, &tpid_list);
 192                 if (ret < 0)
 193                         goto fail;
 194                 ret = dpni_remove_custom_tpid(dpni, CMD_PRI_LOW,
 195                                    priv->token, tpid_list.tpid1);
 196                 if (ret < 0)
 197                         goto fail;
 198                 ret = dpni_add_custom_tpid(dpni, CMD_PRI_LOW,
 199                                            priv->token, tpid);
 200         }
 201 fail:
 202         return ret;
 203 }
 204
 205 static int
 206 dpaa2_fw_version_get(struct rte_eth_dev *dev,
 207                      char *fw_version,
 208                      size_t fw_size)
 209 {
 210         int ret;
 211         struct fsl_mc_io *dpni = dev->process_private;
 212         struct mc_soc_version mc_plat_info = {0};
 213         struct mc_version mc_ver_info = {0};
 214
 215         PMD_INIT_FUNC_TRACE();
 216
 217         if (mc_get_soc_version(dpni, CMD_PRI_LOW, &mc_plat_info))
 218                 DPAA2_PMD_WARN("\tmc_get_soc_version failed");
 219
 220         if (mc_get_version(dpni, CMD_PRI_LOW, &mc_ver_info))
 221                 DPAA2_PMD_WARN("\tmc_get_version failed");
 222
 223         ret = snprintf(fw_version, fw_size,
 224                        "%x-%d.%d.%d",
 225                        mc_plat_info.svr,
 226                        mc_ver_info.major,
 227                        mc_ver_info.minor,
 228                        mc_ver_info.revision);
 229
 230         ret += 1; /* add the size of '\0' */
 231         if (fw_size < (uint32_t)ret)
 232                 return ret;
 233         else
 234                 return 0;
 235 }
 236
 237 static int
 238 dpaa2_dev_info_get(struct rte_eth_dev *dev, struct rte_eth_dev_info *dev_info)
 239 {
 240         struct dpaa2_dev_priv *priv = dev->data->dev_private;
 241
 242         PMD_INIT_FUNC_TRACE();
 243
 244         dev_info->if_index = priv->hw_id;
 245
 246         dev_info->max_mac_addrs = priv->max_mac_filters;
 247         dev_info->max_rx_pktlen = DPAA2_MAX_RX_PKT_LEN;
 248         dev_info->min_rx_bufsize = DPAA2_MIN_RX_BUF_SIZE;
 249         dev_info->max_rx_queues = (uint16_t)priv->nb_rx_queues;
 250         dev_info->max_tx_queues = (uint16_t)priv->nb_tx_queues;
 251         dev_info->rx_offload_capa = dev_rx_offloads_sup |
 252                                         dev_rx_offloads_nodis;
 253         dev_info->tx_offload_capa = dev_tx_offloads_sup |
 254                                         dev_tx_offloads_nodis;
 255         dev_info->speed_capa = ETH_LINK_SPEED_1G |
 256                         ETH_LINK_SPEED_2_5G |
 257                         ETH_LINK_SPEED_10G;
 258
 259         dev_info->max_hash_mac_addrs = 0;
 260         dev_info->max_vfs = 0;
 261         dev_info->max_vmdq_pools = ETH_16_POOLS;
 262         dev_info->flow_type_rss_offloads = DPAA2_RSS_OFFLOAD_ALL;
 263
 264         dev_info->default_rxportconf.burst_size = dpaa2_dqrr_size;
 265         /* same is rx size for best perf */
 266         dev_info->default_txportconf.burst_size = dpaa2_dqrr_size;
 267
 268         dev_info->default_rxportconf.nb_queues = 1;
 269         dev_info->default_txportconf.nb_queues = 1;
 270         dev_info->default_txportconf.ring_size = CONG_ENTER_TX_THRESHOLD;
 271         dev_info->default_rxportconf.ring_size = DPAA2_RX_DEFAULT_NBDESC;
 272
 273         if (dpaa2_svr_family == SVR_LX2160A) {
 274                 dev_info->speed_capa |= ETH_LINK_SPEED_25G |
 275                                 ETH_LINK_SPEED_40G |
 276                                 ETH_LINK_SPEED_50G |
 277                                 ETH_LINK_SPEED_100G;
 278         }
 279
 280         return 0;
 281 }
 282
 283 static int
 284 dpaa2_alloc_rx_tx_queues(struct rte_eth_dev *dev)
 285 {
 286         struct dpaa2_dev_priv *priv = dev->data->dev_private;
 287         uint16_t dist_idx;
 288         uint32_t vq_id;
 289         uint8_t num_rxqueue_per_tc;
 290         struct dpaa2_queue *mc_q, *mcq;
 291         uint32_t tot_queues;
 292         int i;
 293         struct dpaa2_queue *dpaa2_q;
 294
 295         PMD_INIT_FUNC_TRACE();
 296
 297         num_rxqueue_per_tc = (priv->nb_rx_queues / priv->num_rx_tc);
 298         if (priv->tx_conf_en)
 299                 tot_queues = priv->nb_rx_queues + 2 * priv->nb_tx_queues;
 300         else
 301                 tot_queues = priv->nb_rx_queues + priv->nb_tx_queues;
 302         mc_q = rte_malloc(NULL, sizeof(struct dpaa2_queue) * tot_queues,
 303                           RTE_CACHE_LINE_SIZE);
 304         if (!mc_q) {
 305                 DPAA2_PMD_ERR("Memory allocation failed for rx/tx queues");
 306                 return -1;
 307         }
 308
 309         for (i = 0; i < priv->nb_rx_queues; i++) {
 310                 mc_q->eth_data = dev->data;
 311                 priv->rx_vq[i] = mc_q++;
 312                 dpaa2_q = (struct dpaa2_queue *)priv->rx_vq[i];
 313                 dpaa2_q->q_storage = rte_malloc("dq_storage",
 314                                         sizeof(struct queue_storage_info_t),
 315                                         RTE_CACHE_LINE_SIZE);
 316                 if (!dpaa2_q->q_storage)
 317                         goto fail;
 318
 319                 memset(dpaa2_q->q_storage, 0,
 320                        sizeof(struct queue_storage_info_t));
 321                 if (dpaa2_alloc_dq_storage(dpaa2_q->q_storage))
 322                         goto fail;
 323         }
 324
 325         for (i = 0; i < priv->nb_tx_queues; i++) {
 326                 mc_q->eth_data = dev->data;
 327                 mc_q->flow_id = 0xffff;
 328                 priv->tx_vq[i] = mc_q++;
 329                 dpaa2_q = (struct dpaa2_queue *)priv->tx_vq[i];
 330                 dpaa2_q->cscn = rte_malloc(NULL,
 331                                            sizeof(struct qbman_result), 16);
 332                 if (!dpaa2_q->cscn)
 333                         goto fail_tx;
 334         }
 335
 336         if (priv->tx_conf_en) {
 337                 /*Setup tx confirmation queues*/
 338                 for (i = 0; i < priv->nb_tx_queues; i++) {
 339                         mc_q->eth_data = dev->data;
 340                         mc_q->tc_index = i;
 341                         mc_q->flow_id = 0;
 342                         priv->tx_conf_vq[i] = mc_q++;
 343                         dpaa2_q = (struct dpaa2_queue *)priv->tx_conf_vq[i];
 344                         dpaa2_q->q_storage =
 345                                 rte_malloc("dq_storage",
 346                                         sizeof(struct queue_storage_info_t),
 347                                         RTE_CACHE_LINE_SIZE);
 348                         if (!dpaa2_q->q_storage)
 349                                 goto fail_tx_conf;
 350
 351                         memset(dpaa2_q->q_storage, 0,
 352                                sizeof(struct queue_storage_info_t));
 353                         if (dpaa2_alloc_dq_storage(dpaa2_q->q_storage))
 354                                 goto fail_tx_conf;
 355                 }
 356         }
 357
 358         vq_id = 0;
 359         for (dist_idx = 0; dist_idx < priv->nb_rx_queues; dist_idx++) {
 360                 mcq = (struct dpaa2_queue *)priv->rx_vq[vq_id];
 361                 mcq->tc_index = dist_idx / num_rxqueue_per_tc;
 362                 mcq->flow_id = dist_idx % num_rxqueue_per_tc;
 363                 vq_id++;
 364         }
 365
 366         return 0;
 367 fail_tx_conf:
 368         i -= 1;
 369         while (i >= 0) {
 370                 dpaa2_q = (struct dpaa2_queue *)priv->tx_conf_vq[i];
 371                 rte_free(dpaa2_q->q_storage);
 372                 priv->tx_conf_vq[i--] = NULL;
 373         }
 374         i = priv->nb_tx_queues;
 375 fail_tx:
 376         i -= 1;
 377         while (i >= 0) {
 378                 dpaa2_q = (struct dpaa2_queue *)priv->tx_vq[i];
 379                 rte_free(dpaa2_q->cscn);
 380                 priv->tx_vq[i--] = NULL;
 381         }
 382         i = priv->nb_rx_queues;
 383 fail:
 384         i -= 1;
 385         mc_q = priv->rx_vq[0];
 386         while (i >= 0) {
 387                 dpaa2_q = (struct dpaa2_queue *)priv->rx_vq[i];
 388                 dpaa2_free_dq_storage(dpaa2_q->q_storage);
 389                 rte_free(dpaa2_q->q_storage);
 390                 priv->rx_vq[i--] = NULL;
 391         }
 392         rte_free(mc_q);
 393         return -1;
 394 }
 395
 396 static void
 397 dpaa2_free_rx_tx_queues(struct rte_eth_dev *dev)
 398 {
 399         struct dpaa2_dev_priv *priv = dev->data->dev_private;
 400         struct dpaa2_queue *dpaa2_q;
 401         int i;
 402
 403         PMD_INIT_FUNC_TRACE();
 404
 405         /* Queue allocation base */
 406         if (priv->rx_vq[0]) {
 407                 /* cleaning up queue storage */
 408                 for (i = 0; i < priv->nb_rx_queues; i++) {
 409                         dpaa2_q = (struct dpaa2_queue *)priv->rx_vq[i];
 410                         if (dpaa2_q->q_storage)
 411                                 rte_free(dpaa2_q->q_storage);
 412                 }
 413                 /* cleanup tx queue cscn */
 414                 for (i = 0; i < priv->nb_tx_queues; i++) {
 415                         dpaa2_q = (struct dpaa2_queue *)priv->tx_vq[i];
 416                         rte_free(dpaa2_q->cscn);
 417                 }
 418                 if (priv->tx_conf_en) {
 419                         /* cleanup tx conf queue storage */
 420                         for (i = 0; i < priv->nb_tx_queues; i++) {
 421                                 dpaa2_q = (struct dpaa2_queue *)
 422                                                 priv->tx_conf_vq[i];
 423                                 rte_free(dpaa2_q->q_storage);
 424                         }
 425                 }
 426                 /*free memory for all queues (RX+TX) */
 427                 rte_free(priv->rx_vq[0]);
 428                 priv->rx_vq[0] = NULL;
 429         }
 430 }
 431
 432 static int
 433 dpaa2_eth_dev_configure(struct rte_eth_dev *dev)
 434 {
 435         struct dpaa2_dev_priv *priv = dev->data->dev_private;
 436         struct fsl_mc_io *dpni = dev->process_private;
 437         struct rte_eth_conf *eth_conf = &dev->data->dev_conf;
 438         uint64_t rx_offloads = eth_conf->rxmode.offloads;
 439         uint64_t tx_offloads = eth_conf->txmode.offloads;
 440         int rx_l3_csum_offload = false;
 441         int rx_l4_csum_offload = false;
 442         int tx_l3_csum_offload = false;
 443         int tx_l4_csum_offload = false;
 444         int ret, tc_index;
 445
 446         PMD_INIT_FUNC_TRACE();
 447
 448         /* Rx offloads which are enabled by default */
 449         if (dev_rx_offloads_nodis & ~rx_offloads) {
 450                 DPAA2_PMD_INFO(
 451                 "Some of rx offloads enabled by default - requested 0x%" PRIx64
 452                 " fixed are 0x%" PRIx64,
 453                 rx_offloads, dev_rx_offloads_nodis);
 454         }
 455
 456         /* Tx offloads which are enabled by default */
 457         if (dev_tx_offloads_nodis & ~tx_offloads) {
 458                 DPAA2_PMD_INFO(
 459                 "Some of tx offloads enabled by default - requested 0x%" PRIx64
 460                 " fixed are 0x%" PRIx64,
 461                 tx_offloads, dev_tx_offloads_nodis);
 462         }
 463
 464         if (rx_offloads & DEV_RX_OFFLOAD_JUMBO_FRAME) {
 465                 if (eth_conf->rxmode.max_rx_pkt_len <= DPAA2_MAX_RX_PKT_LEN) {
 466                         ret = dpni_set_max_frame_length(dpni, CMD_PRI_LOW,
 467                                 priv->token, eth_conf->rxmode.max_rx_pkt_len
 468                                 - RTE_ETHER_CRC_LEN);
 469                         if (ret) {
 470                                 DPAA2_PMD_ERR(
 471                                         "Unable to set mtu. check config");
 472                                 return ret;
 473                         }
 474                         dev->data->mtu =
 475                                 dev->data->dev_conf.rxmode.max_rx_pkt_len -
 476                                 RTE_ETHER_HDR_LEN - RTE_ETHER_CRC_LEN -
 477                                 VLAN_TAG_SIZE;
 478                 } else {
 479                         return -1;
 480                 }
 481         }
 482
 483         if (eth_conf->rxmode.mq_mode == ETH_MQ_RX_RSS) {
 484                 for (tc_index = 0; tc_index < priv->num_rx_tc; tc_index++) {
 485                         ret = dpaa2_setup_flow_dist(dev,
 486                                         eth_conf->rx_adv_conf.rss_conf.rss_hf,
 487                                         tc_index);
 488                         if (ret) {
 489                                 DPAA2_PMD_ERR(
 490                                         "Unable to set flow distribution on tc%d."
 491                                         "Check queue config", tc_index);
 492                                 return ret;
 493                         }
 494                 }
 495         }
 496
 497         if (rx_offloads & DEV_RX_OFFLOAD_IPV4_CKSUM)
 498                 rx_l3_csum_offload = true;
 499
 500         if ((rx_offloads & DEV_RX_OFFLOAD_UDP_CKSUM) ||
 501                 (rx_offloads & DEV_RX_OFFLOAD_TCP_CKSUM) ||
 502                 (rx_offloads & DEV_RX_OFFLOAD_SCTP_CKSUM))
 503                 rx_l4_csum_offload = true;
 504
 505         ret = dpni_set_offload(dpni, CMD_PRI_LOW, priv->token,
 506                                DPNI_OFF_RX_L3_CSUM, rx_l3_csum_offload);
 507         if (ret) {
 508                 DPAA2_PMD_ERR("Error to set RX l3 csum:Error = %d", ret);
 509                 return ret;
 510         }
 511
 512         ret = dpni_set_offload(dpni, CMD_PRI_LOW, priv->token,
 513                                DPNI_OFF_RX_L4_CSUM, rx_l4_csum_offload);
 514         if (ret) {
 515                 DPAA2_PMD_ERR("Error to get RX l4 csum:Error = %d", ret);
 516                 return ret;
 517         }
 518
 519 #if !defined(RTE_LIBRTE_IEEE1588)
 520         if (rx_offloads & DEV_RX_OFFLOAD_TIMESTAMP)
 521 #endif
 522         dpaa2_enable_ts = true;
 523
 524         if (tx_offloads & DEV_TX_OFFLOAD_IPV4_CKSUM)
 525                 tx_l3_csum_offload = true;
 526
 527         if ((tx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) ||
 528                 (tx_offloads & DEV_TX_OFFLOAD_TCP_CKSUM) ||
 529                 (tx_offloads & DEV_TX_OFFLOAD_SCTP_CKSUM))
 530                 tx_l4_csum_offload = true;
 531
 532         ret = dpni_set_offload(dpni, CMD_PRI_LOW, priv->token,
 533                                DPNI_OFF_TX_L3_CSUM, tx_l3_csum_offload);
 534         if (ret) {
 535                 DPAA2_PMD_ERR("Error to set TX l3 csum:Error = %d", ret);
 536                 return ret;
 537         }
 538
 539         ret = dpni_set_offload(dpni, CMD_PRI_LOW, priv->token,
 540                                DPNI_OFF_TX_L4_CSUM, tx_l4_csum_offload);
 541         if (ret) {
 542                 DPAA2_PMD_ERR("Error to get TX l4 csum:Error = %d", ret);
 543                 return ret;
 544         }
 545
 546         /* Enabling hash results in FD requires setting DPNI_FLCTYPE_HASH in
 547          * dpni_set_offload API. Setting this FLCTYPE for DPNI sets the FD[SC]
 548          * to 0 for LS2 in the hardware thus disabling data/annotation
 549          * stashing. For LX2 this is fixed in hardware and thus hash result and
 550          * parse results can be received in FD using this option.
 551          */
 552         if (dpaa2_svr_family == SVR_LX2160A) {
 553                 ret = dpni_set_offload(dpni, CMD_PRI_LOW, priv->token,
 554                                        DPNI_FLCTYPE_HASH, true);
 555                 if (ret) {
 556                         DPAA2_PMD_ERR("Error setting FLCTYPE: Err = %d", ret);
 557                         return ret;
 558                 }
 559         }
 560
 561         if (rx_offloads & DEV_RX_OFFLOAD_VLAN_FILTER)
 562                 dpaa2_vlan_offload_set(dev, ETH_VLAN_FILTER_MASK);
 563
 564         return 0;
 565 }
 566
 567 /* Function to setup RX flow information. It contains traffic class ID,
 568  * flow ID, destination configuration etc.
 569  */
 570 static int
 571 dpaa2_dev_rx_queue_setup(struct rte_eth_dev *dev,
 572                          uint16_t rx_queue_id,
 573                          uint16_t nb_rx_desc,
 574                          unsigned int socket_id __rte_unused,
 575                          const struct rte_eth_rxconf *rx_conf __rte_unused,
 576                          struct rte_mempool *mb_pool)
 577 {
 578         struct dpaa2_dev_priv *priv = dev->data->dev_private;
 579         struct fsl_mc_io *dpni = (struct fsl_mc_io *)dev->process_private;
 580         struct dpaa2_queue *dpaa2_q;
 581         struct dpni_queue cfg;
 582         uint8_t options = 0;
 583         uint8_t flow_id;
 584         uint32_t bpid;
 585         int i, ret;
 586
 587         PMD_INIT_FUNC_TRACE();
 588
 589         DPAA2_PMD_DEBUG("dev =%p, queue =%d, pool = %p, conf =%p",
 590                         dev, rx_queue_id, mb_pool, rx_conf);
 591
 592         if (!priv->bp_list || priv->bp_list->mp != mb_pool) {
 593                 bpid = mempool_to_bpid(mb_pool);
 594                 ret = dpaa2_attach_bp_list(priv,
 595                                            rte_dpaa2_bpid_info[bpid].bp_list);
 596                 if (ret)
 597                         return ret;
 598         }
 599         dpaa2_q = (struct dpaa2_queue *)priv->rx_vq[rx_queue_id];
 600         dpaa2_q->mb_pool = mb_pool; /**< mbuf pool to populate RX ring. */
 601         dpaa2_q->bp_array = rte_dpaa2_bpid_info;
 602
 603         /*Get the flow id from given VQ id*/
 604         flow_id = dpaa2_q->flow_id;
 605         memset(&cfg, 0, sizeof(struct dpni_queue));
 606
 607         options = options | DPNI_QUEUE_OPT_USER_CTX;
 608         cfg.user_context = (size_t)(dpaa2_q);
 609
 610         /* check if a private cgr available. */
 611         for (i = 0; i < priv->max_cgs; i++) {
 612                 if (!priv->cgid_in_use[i]) {
 613                         priv->cgid_in_use[i] = 1;
 614                         break;
 615                 }
 616         }
 617
 618         if (i < priv->max_cgs) {
 619                 options |= DPNI_QUEUE_OPT_SET_CGID;
 620                 cfg.cgid = i;
 621                 dpaa2_q->cgid = cfg.cgid;
 622         } else {
 623                 dpaa2_q->cgid = 0xff;
 624         }
 625
 626         /*if ls2088 or rev2 device, enable the stashing */
 627
 628         if ((dpaa2_svr_family & 0xffff0000) != SVR_LS2080A) {
 629                 options |= DPNI_QUEUE_OPT_FLC;
 630                 cfg.flc.stash_control = true;
 631                 cfg.flc.value &= 0xFFFFFFFFFFFFFFC0;
 632                 /* 00 00 00 - last 6 bit represent annotation, context stashing,
 633                  * data stashing setting 01 01 00 (0x14)
 634                  * (in following order ->DS AS CS)
 635                  * to enable 1 line data, 1 line annotation.
 636                  * For LX2, this setting should be 01 00 00 (0x10)
 637                  */
 638                 if ((dpaa2_svr_family & 0xffff0000) == SVR_LX2160A)
 639                         cfg.flc.value |= 0x10;
 640                 else
 641                         cfg.flc.value |= 0x14;
 642         }
 643         ret = dpni_set_queue(dpni, CMD_PRI_LOW, priv->token, DPNI_QUEUE_RX,
 644                              dpaa2_q->tc_index, flow_id, options, &cfg);
 645         if (ret) {
 646                 DPAA2_PMD_ERR("Error in setting the rx flow: = %d", ret);
 647                 return -1;
 648         }
 649
 650         if (!(priv->flags & DPAA2_RX_TAILDROP_OFF)) {
 651                 struct dpni_taildrop taildrop;
 652
 653                 taildrop.enable = 1;
 654
 655                 /* Private CGR will use tail drop length as nb_rx_desc.
 656                  * for rest cases we can use standard byte based tail drop.
 657                  * There is no HW restriction, but number of CGRs are limited,
 658                  * hence this restriction is placed.
 659                  */
 660                 if (dpaa2_q->cgid != 0xff) {
 661                         /*enabling per rx queue congestion control */
 662                         taildrop.threshold = nb_rx_desc;
 663                         taildrop.units = DPNI_CONGESTION_UNIT_FRAMES;
 664                         taildrop.oal = 0;
 665                         DPAA2_PMD_DEBUG("Enabling CG Tail Drop on queue = %d",
 666                                         rx_queue_id);
 667                         ret = dpni_set_taildrop(dpni, CMD_PRI_LOW, priv->token,
 668                                                 DPNI_CP_CONGESTION_GROUP,
 669                                                 DPNI_QUEUE_RX,
 670                                                 dpaa2_q->tc_index,
 671                                                 dpaa2_q->cgid, &taildrop);
 672                 } else {
 673                         /*enabling per rx queue congestion control */
 674                         taildrop.threshold = CONG_THRESHOLD_RX_BYTES_Q;
 675                         taildrop.units = DPNI_CONGESTION_UNIT_BYTES;
 676                         taildrop.oal = CONG_RX_OAL;
 677                         DPAA2_PMD_DEBUG("Enabling Byte based Drop on queue= %d",
 678                                         rx_queue_id);
 679                         ret = dpni_set_taildrop(dpni, CMD_PRI_LOW, priv->token,
 680                                                 DPNI_CP_QUEUE, DPNI_QUEUE_RX,
 681                                                 dpaa2_q->tc_index, flow_id,
 682                                                 &taildrop);
 683                 }
 684                 if (ret) {
 685                         DPAA2_PMD_ERR("Error in setting taildrop. err=(%d)",
 686                                       ret);
 687                         return -1;
 688                 }
 689         } else { /* Disable tail Drop */
 690                 struct dpni_taildrop taildrop = {0};
 691                 DPAA2_PMD_INFO("Tail drop is disabled on queue");
 692
 693                 taildrop.enable = 0;
 694                 if (dpaa2_q->cgid != 0xff) {
 695                         ret = dpni_set_taildrop(dpni, CMD_PRI_LOW, priv->token,
 696                                         DPNI_CP_CONGESTION_GROUP, DPNI_QUEUE_RX,
 697                                         dpaa2_q->tc_index,
 698                                         dpaa2_q->cgid, &taildrop);
 699                 } else {
 700                         ret = dpni_set_taildrop(dpni, CMD_PRI_LOW, priv->token,
 701                                         DPNI_CP_QUEUE, DPNI_QUEUE_RX,
 702                                         dpaa2_q->tc_index, flow_id, &taildrop);
 703                 }
 704                 if (ret) {
 705                         DPAA2_PMD_ERR("Error in setting taildrop. err=(%d)",
 706                                       ret);
 707                         return -1;
 708                 }
 709         }
 710
 711         dev->data->rx_queues[rx_queue_id] = dpaa2_q;
 712         return 0;
 713 }
 714
 715 static int
 716 dpaa2_dev_tx_queue_setup(struct rte_eth_dev *dev,
 717                          uint16_t tx_queue_id,
 718                          uint16_t nb_tx_desc __rte_unused,
 719                          unsigned int socket_id __rte_unused,
 720                          const struct rte_eth_txconf *tx_conf __rte_unused)
 721 {
 722         struct dpaa2_dev_priv *priv = dev->data->dev_private;
 723         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)
 724                 priv->tx_vq[tx_queue_id];
 725         struct dpaa2_queue *dpaa2_tx_conf_q = (struct dpaa2_queue *)
 726                 priv->tx_conf_vq[tx_queue_id];
 727         struct fsl_mc_io *dpni = dev->process_private;
 728         struct dpni_queue tx_conf_cfg;
 729         struct dpni_queue tx_flow_cfg;
 730         uint8_t options = 0, flow_id;
 731         struct dpni_queue_id qid;
 732         uint32_t tc_id;
 733         int ret;
 734
 735         PMD_INIT_FUNC_TRACE();
 736
 737         /* Return if queue already configured */
 738         if (dpaa2_q->flow_id != 0xffff) {
 739                 dev->data->tx_queues[tx_queue_id] = dpaa2_q;
 740                 return 0;
 741         }
 742
 743         memset(&tx_conf_cfg, 0, sizeof(struct dpni_queue));
 744         memset(&tx_flow_cfg, 0, sizeof(struct dpni_queue));
 745
 746         tc_id = tx_queue_id;
 747         flow_id = 0;
 748
 749         ret = dpni_set_queue(dpni, CMD_PRI_LOW, priv->token, DPNI_QUEUE_TX,
 750                         tc_id, flow_id, options, &tx_flow_cfg);
 751         if (ret) {
 752                 DPAA2_PMD_ERR("Error in setting the tx flow: "
 753                         "tc_id=%d, flow=%d err=%d",
 754                         tc_id, flow_id, ret);
 755                         return -1;
 756         }
 757
 758         dpaa2_q->flow_id = flow_id;
 759
 760         if (tx_queue_id == 0) {
 761                 /*Set tx-conf and error configuration*/
 762                 if (priv->tx_conf_en)
 763                         ret = dpni_set_tx_confirmation_mode(dpni, CMD_PRI_LOW,
 764                                                             priv->token,
 765                                                             DPNI_CONF_AFFINE);
 766                 else
 767                         ret = dpni_set_tx_confirmation_mode(dpni, CMD_PRI_LOW,
 768                                                             priv->token,
 769                                                             DPNI_CONF_DISABLE);
 770                 if (ret) {
 771                         DPAA2_PMD_ERR("Error in set tx conf mode settings: "
 772                                       "err=%d", ret);
 773                         return -1;
 774                 }
 775         }
 776         dpaa2_q->tc_index = tc_id;
 777
 778         ret = dpni_get_queue(dpni, CMD_PRI_LOW, priv->token,
 779                              DPNI_QUEUE_TX, dpaa2_q->tc_index,
 780                              dpaa2_q->flow_id, &tx_flow_cfg, &qid);
 781         if (ret) {
 782                 DPAA2_PMD_ERR("Error in getting LFQID err=%d", ret);
 783                 return -1;
 784         }
 785         dpaa2_q->fqid = qid.fqid;
 786
 787         if (!(priv->flags & DPAA2_TX_CGR_OFF)) {
 788                 struct dpni_congestion_notification_cfg cong_notif_cfg = {0};
 789
 790                 cong_notif_cfg.units = DPNI_CONGESTION_UNIT_FRAMES;
 791                 cong_notif_cfg.threshold_entry = CONG_ENTER_TX_THRESHOLD;
 792                 /* Notify that the queue is not congested when the data in
 793                  * the queue is below this thershold.
 794                  */
 795                 cong_notif_cfg.threshold_exit = CONG_EXIT_TX_THRESHOLD;
 796                 cong_notif_cfg.message_ctx = 0;
 797                 cong_notif_cfg.message_iova =
 798                                 (size_t)DPAA2_VADDR_TO_IOVA(dpaa2_q->cscn);
 799                 cong_notif_cfg.dest_cfg.dest_type = DPNI_DEST_NONE;
 800                 cong_notif_cfg.notification_mode =
 801                                          DPNI_CONG_OPT_WRITE_MEM_ON_ENTER |
 802                                          DPNI_CONG_OPT_WRITE_MEM_ON_EXIT |
 803                                          DPNI_CONG_OPT_COHERENT_WRITE;
 804                 cong_notif_cfg.cg_point = DPNI_CP_QUEUE;
 805
 806                 ret = dpni_set_congestion_notification(dpni, CMD_PRI_LOW,
 807                                                        priv->token,
 808                                                        DPNI_QUEUE_TX,
 809                                                        tc_id,
 810                                                        &cong_notif_cfg);
 811                 if (ret) {
 812                         DPAA2_PMD_ERR(
 813                            "Error in setting tx congestion notification: "
 814                            "err=%d", ret);
 815                         return -ret;
 816                 }
 817         }
 818         dpaa2_q->cb_eqresp_free = dpaa2_dev_free_eqresp_buf;
 819         dev->data->tx_queues[tx_queue_id] = dpaa2_q;
 820
 821         if (priv->tx_conf_en) {
 822                 dpaa2_q->tx_conf_queue = dpaa2_tx_conf_q;
 823                 options = options | DPNI_QUEUE_OPT_USER_CTX;
 824                 tx_conf_cfg.user_context = (size_t)(dpaa2_q);
 825                 ret = dpni_set_queue(dpni, CMD_PRI_LOW, priv->token,
 826                              DPNI_QUEUE_TX_CONFIRM, dpaa2_tx_conf_q->tc_index,
 827                              dpaa2_tx_conf_q->flow_id, options, &tx_conf_cfg);
 828                 if (ret) {
 829                         DPAA2_PMD_ERR("Error in setting the tx conf flow: "
 830                               "tc_index=%d, flow=%d err=%d",
 831                               dpaa2_tx_conf_q->tc_index,
 832                               dpaa2_tx_conf_q->flow_id, ret);
 833                         return -1;
 834                 }
 835
 836                 ret = dpni_get_queue(dpni, CMD_PRI_LOW, priv->token,
 837                              DPNI_QUEUE_TX_CONFIRM, dpaa2_tx_conf_q->tc_index,
 838                              dpaa2_tx_conf_q->flow_id, &tx_conf_cfg, &qid);
 839                 if (ret) {
 840                         DPAA2_PMD_ERR("Error in getting LFQID err=%d", ret);
 841                         return -1;
 842                 }
 843                 dpaa2_tx_conf_q->fqid = qid.fqid;
 844         }
 845         return 0;
 846 }
 847
 848 static void
 849 dpaa2_dev_rx_queue_release(void *q __rte_unused)
 850 {
 851         struct dpaa2_queue *dpaa2_q = (struct dpaa2_queue *)q;
 852         struct dpaa2_dev_priv *priv = dpaa2_q->eth_data->dev_private;
 853         struct fsl_mc_io *dpni =
 854                 (struct fsl_mc_io *)priv->eth_dev->process_private;
 855         uint8_t options = 0;
 856         int ret;
 857         struct dpni_queue cfg;
 858
 859         memset(&cfg, 0, sizeof(struct dpni_queue));
 860         PMD_INIT_FUNC_TRACE();
 861         if (dpaa2_q->cgid != 0xff) {
 862                 options = DPNI_QUEUE_OPT_CLEAR_CGID;
 863                 cfg.cgid = dpaa2_q->cgid;
 864
 865                 ret = dpni_set_queue(dpni, CMD_PRI_LOW, priv->token,
 866                                      DPNI_QUEUE_RX,
 867                                      dpaa2_q->tc_index, dpaa2_q->flow_id,
 868                                      options, &cfg);
 869                 if (ret)
 870                         DPAA2_PMD_ERR("Unable to clear CGR from q=%u err=%d",
 871                                         dpaa2_q->fqid, ret);
 872                 priv->cgid_in_use[dpaa2_q->cgid] = 0;
 873                 dpaa2_q->cgid = 0xff;
 874         }
 875 }
 876
 877 static void
 878 dpaa2_dev_tx_queue_release(void *q __rte_unused)
 879 {
 880         PMD_INIT_FUNC_TRACE();
 881 }
 882
 883 static uint32_t
 884 dpaa2_dev_rx_queue_count(struct rte_eth_dev *dev, uint16_t rx_queue_id)
 885 {
 886         int32_t ret;
 887         struct dpaa2_dev_priv *priv = dev->data->dev_private;
 888         struct dpaa2_queue *dpaa2_q;
 889         struct qbman_swp *swp;
 890         struct qbman_fq_query_np_rslt state;
 891         uint32_t frame_cnt = 0;
 892
 893         if (unlikely(!DPAA2_PER_LCORE_DPIO)) {
 894                 ret = dpaa2_affine_qbman_swp();
 895                 if (ret) {
 896                         DPAA2_PMD_ERR(
 897                                 "Failed to allocate IO portal, tid: %d\n",
 898                                 rte_gettid());
 899                         return -EINVAL;
 900                 }
 901         }
 902         swp = DPAA2_PER_LCORE_PORTAL;
 903
 904         dpaa2_q = (struct dpaa2_queue *)priv->rx_vq[rx_queue_id];
 905
 906         if (qbman_fq_query_state(swp, dpaa2_q->fqid, &state) == 0) {
 907                 frame_cnt = qbman_fq_state_frame_count(&state);
 908                 DPAA2_PMD_DP_DEBUG("RX frame count for q(%d) is %u",
 909                                 rx_queue_id, frame_cnt);
 910         }
 911         return frame_cnt;
 912 }
 913
 914 static const uint32_t *
 915 dpaa2_supported_ptypes_get(struct rte_eth_dev *dev)
 916 {
 917         static const uint32_t ptypes[] = {
 918                 /*todo -= add more types */
 919                 RTE_PTYPE_L2_ETHER,
 920                 RTE_PTYPE_L3_IPV4,
 921                 RTE_PTYPE_L3_IPV4_EXT,
 922                 RTE_PTYPE_L3_IPV6,
 923                 RTE_PTYPE_L3_IPV6_EXT,
 924                 RTE_PTYPE_L4_TCP,
 925                 RTE_PTYPE_L4_UDP,
 926                 RTE_PTYPE_L4_SCTP,
 927                 RTE_PTYPE_L4_ICMP,
 928                 RTE_PTYPE_UNKNOWN
 929         };
 930
 931         if (dev->rx_pkt_burst == dpaa2_dev_prefetch_rx ||
 932                 dev->rx_pkt_burst == dpaa2_dev_rx ||
 933                 dev->rx_pkt_burst == dpaa2_dev_loopback_rx)
 934                 return ptypes;
 935         return NULL;
 936 }
 937
 938 /**
 939  * Dpaa2 link Interrupt handler
 940  *
 941  * @param param
 942  *  The address of parameter (struct rte_eth_dev *) regsitered before.
 943  *
 944  * @return
 945  *  void
 946  */
 947 static void
 948 dpaa2_interrupt_handler(void *param)
 949 {
 950         struct rte_eth_dev *dev = param;
 951         struct dpaa2_dev_priv *priv = dev->data->dev_private;
 952         struct fsl_mc_io *dpni = (struct fsl_mc_io *)dev->process_private;
 953         int ret;
 954         int irq_index = DPNI_IRQ_INDEX;
 955         unsigned int status = 0, clear = 0;
 956
 957         PMD_INIT_FUNC_TRACE();
 958
 959         if (dpni == NULL) {
 960                 DPAA2_PMD_ERR("dpni is NULL");
 961                 return;
 962         }
 963
 964         ret = dpni_get_irq_status(dpni, CMD_PRI_LOW, priv->token,
 965                                   irq_index, &status);
 966         if (unlikely(ret)) {
 967                 DPAA2_PMD_ERR("Can't get irq status (err %d)", ret);
 968                 clear = 0xffffffff;
 969                 goto out;
 970         }
 971
 972         if (status & DPNI_IRQ_EVENT_LINK_CHANGED) {
 973                 clear = DPNI_IRQ_EVENT_LINK_CHANGED;
 974                 dpaa2_dev_link_update(dev, 0);
 975                 /* calling all the apps registered for link status event */
 976                 _rte_eth_dev_callback_process(dev, RTE_ETH_EVENT_INTR_LSC,
 977                                               NULL);
 978         }
 979 out:
 980         ret = dpni_clear_irq_status(dpni, CMD_PRI_LOW, priv->token,
 981                                     irq_index, clear);
 982         if (unlikely(ret))
 983                 DPAA2_PMD_ERR("Can't clear irq status (err %d)", ret);
 984 }
 985
 986 static int
 987 dpaa2_eth_setup_irqs(struct rte_eth_dev *dev, int enable)
 988 {
 989         int err = 0;
 990         struct dpaa2_dev_priv *priv = dev->data->dev_private;
 991         struct fsl_mc_io *dpni = (struct fsl_mc_io *)dev->process_private;
 992         int irq_index = DPNI_IRQ_INDEX;
 993         unsigned int mask = DPNI_IRQ_EVENT_LINK_CHANGED;
 994
 995         PMD_INIT_FUNC_TRACE();
 996
 997         err = dpni_set_irq_mask(dpni, CMD_PRI_LOW, priv->token,
 998                                 irq_index, mask);
 999         if (err < 0) {
1000                 DPAA2_PMD_ERR("Error: dpni_set_irq_mask():%d (%s)", err,
1001                               strerror(-err));
1002                 return err;
1003         }
1004
1005         err = dpni_set_irq_enable(dpni, CMD_PRI_LOW, priv->token,
1006                                   irq_index, enable);
1007         if (err < 0)
1008                 DPAA2_PMD_ERR("Error: dpni_set_irq_enable():%d (%s)", err,
1009                               strerror(-err));
1010
1011         return err;
1012 }
1013
1014 static int
1015 dpaa2_dev_start(struct rte_eth_dev *dev)
1016 {
1017         struct rte_device *rdev = dev->device;
1018         struct rte_dpaa2_device *dpaa2_dev;
1019         struct rte_eth_dev_data *data = dev->data;
1020         struct dpaa2_dev_priv *priv = data->dev_private;
1021         struct fsl_mc_io *dpni = (struct fsl_mc_io *)dev->process_private;
1022         struct dpni_queue cfg;
1023         struct dpni_error_cfg   err_cfg;
1024         uint16_t qdid;
1025         struct dpni_queue_id qid;
1026         struct dpaa2_queue *dpaa2_q;
1027         int ret, i;
1028         struct rte_intr_handle *intr_handle;
1029
1030         dpaa2_dev = container_of(rdev, struct rte_dpaa2_device, device);
1031         intr_handle = &dpaa2_dev->intr_handle;
1032
1033         PMD_INIT_FUNC_TRACE();
1034
1035         ret = dpni_enable(dpni, CMD_PRI_LOW, priv->token);
1036         if (ret) {
1037                 DPAA2_PMD_ERR("Failure in enabling dpni %d device: err=%d",
1038                               priv->hw_id, ret);
1039                 return ret;
1040         }
1041
1042         /* Power up the phy. Needed to make the link go UP */
1043         dpaa2_dev_set_link_up(dev);
1044
1045         ret = dpni_get_qdid(dpni, CMD_PRI_LOW, priv->token,
1046                             DPNI_QUEUE_TX, &qdid);
1047         if (ret) {
1048                 DPAA2_PMD_ERR("Error in getting qdid: err=%d", ret);
1049                 return ret;
1050         }
1051         priv->qdid = qdid;
1052
1053         for (i = 0; i < data->nb_rx_queues; i++) {
1054                 dpaa2_q = (struct dpaa2_queue *)data->rx_queues[i];
1055                 ret = dpni_get_queue(dpni, CMD_PRI_LOW, priv->token,
1056                                      DPNI_QUEUE_RX, dpaa2_q->tc_index,
1057                                        dpaa2_q->flow_id, &cfg, &qid);
1058                 if (ret) {
1059                         DPAA2_PMD_ERR("Error in getting flow information: "
1060                                       "err=%d", ret);
1061                         return ret;
1062                 }
1063                 dpaa2_q->fqid = qid.fqid;
1064         }
1065
1066         /*checksum errors, send them to normal path and set it in annotation */
1067         err_cfg.errors = DPNI_ERROR_L3CE | DPNI_ERROR_L4CE;
1068         err_cfg.errors |= DPNI_ERROR_PHE;
1069
1070         err_cfg.error_action = DPNI_ERROR_ACTION_CONTINUE;
1071         err_cfg.set_frame_annotation = true;
1072
1073         ret = dpni_set_errors_behavior(dpni, CMD_PRI_LOW,
1074                                        priv->token, &err_cfg);
1075         if (ret) {
1076                 DPAA2_PMD_ERR("Error to dpni_set_errors_behavior: code = %d",
1077                               ret);
1078                 return ret;
1079         }
1080
1081         /* if the interrupts were configured on this devices*/
1082         if (intr_handle && (intr_handle->fd) &&
1083             (dev->data->dev_conf.intr_conf.lsc != 0)) {
1084                 /* Registering LSC interrupt handler */
1085                 rte_intr_callback_register(intr_handle,
1086                                            dpaa2_interrupt_handler,
1087                                            (void *)dev);
1088
1089                 /* enable vfio intr/eventfd mapping
1090                  * Interrupt index 0 is required, so we can not use
1091                  * rte_intr_enable.
1092                  */
1093                 rte_dpaa2_intr_enable(intr_handle, DPNI_IRQ_INDEX);
1094
1095                 /* enable dpni_irqs */
1096                 dpaa2_eth_setup_irqs(dev, 1);
1097         }
1098
1099         /* Change the tx burst function if ordered queues are used */
1100         if (priv->en_ordered)
1101                 dev->tx_pkt_burst = dpaa2_dev_tx_ordered;
1102
1103         return 0;
1104 }
1105
1106 /**
1107  *  This routine disables all traffic on the adapter by issuing a
1108  *  global reset on the MAC.
1109  */
1110 static void
1111 dpaa2_dev_stop(struct rte_eth_dev *dev)
1112 {
1113         struct dpaa2_dev_priv *priv = dev->data->dev_private;
1114         struct fsl_mc_io *dpni = (struct fsl_mc_io *)dev->process_private;
1115         int ret;
1116         struct rte_eth_link link;
1117         struct rte_intr_handle *intr_handle = dev->intr_handle;
1118
1119         PMD_INIT_FUNC_TRACE();
1120
1121         /* reset interrupt callback  */
1122         if (intr_handle && (intr_handle->fd) &&
1123             (dev->data->dev_conf.intr_conf.lsc != 0)) {
1124                 /*disable dpni irqs */
1125                 dpaa2_eth_setup_irqs(dev, 0);
1126
1127                 /* disable vfio intr before callback unregister */
1128                 rte_dpaa2_intr_disable(intr_handle, DPNI_IRQ_INDEX);
1129
1130                 /* Unregistering LSC interrupt handler */
1131                 rte_intr_callback_unregister(intr_handle,
1132                                              dpaa2_interrupt_handler,
1133                                              (void *)dev);
1134         }
1135
1136         dpaa2_dev_set_link_down(dev);
1137
1138         ret = dpni_disable(dpni, CMD_PRI_LOW, priv->token);
1139         if (ret) {
1140                 DPAA2_PMD_ERR("Failure (ret %d) in disabling dpni %d dev",
1141                               ret, priv->hw_id);
1142                 return;
1143         }
1144
1145         /* clear the recorded link status */
1146         memset(&link, 0, sizeof(link));
1147         rte_eth_linkstatus_set(dev, &link);
1148 }
1149
1150 static void
1151 dpaa2_dev_close(struct rte_eth_dev *dev)
1152 {
1153         struct dpaa2_dev_priv *priv = dev->data->dev_private;
1154         struct fsl_mc_io *dpni = (struct fsl_mc_io *)dev->process_private;
1155         int ret;
1156         struct rte_eth_link link;
1157
1158         PMD_INIT_FUNC_TRACE();
1159
1160         dpaa2_flow_clean(dev);
1161
1162         /* Clean the device first */
1163         ret = dpni_reset(dpni, CMD_PRI_LOW, priv->token);
1164         if (ret) {
1165                 DPAA2_PMD_ERR("Failure cleaning dpni device: err=%d", ret);
1166                 return;
1167         }
1168
1169         memset(&link, 0, sizeof(link));
1170         rte_eth_linkstatus_set(dev, &link);
1171 }
1172
1173 static int
1174 dpaa2_dev_promiscuous_enable(
1175                 struct rte_eth_dev *dev)
1176 {
1177         int ret;
1178         struct dpaa2_dev_priv *priv = dev->data->dev_private;
1179         struct fsl_mc_io *dpni = (struct fsl_mc_io *)dev->process_private;
1180
1181         PMD_INIT_FUNC_TRACE();
1182
1183         if (dpni == NULL) {
1184                 DPAA2_PMD_ERR("dpni is NULL");
1185                 return -ENODEV;
1186         }
1187
1188         ret = dpni_set_unicast_promisc(dpni, CMD_PRI_LOW, priv->token, true);
1189         if (ret < 0)
1190                 DPAA2_PMD_ERR("Unable to enable U promisc mode %d", ret);
1191
1192         ret = dpni_set_multicast_promisc(dpni, CMD_PRI_LOW, priv->token, true);
1193         if (ret < 0)
1194                 DPAA2_PMD_ERR("Unable to enable M promisc mode %d", ret);
1195
1196         return ret;
1197 }
1198
1199 static int
1200 dpaa2_dev_promiscuous_disable(
1201                 struct rte_eth_dev *dev)
1202 {
1203         int ret;
1204         struct dpaa2_dev_priv *priv = dev->data->dev_private;
1205         struct fsl_mc_io *dpni = (struct fsl_mc_io *)dev->process_private;
1206
1207         PMD_INIT_FUNC_TRACE();
1208
1209         if (dpni == NULL) {
1210                 DPAA2_PMD_ERR("dpni is NULL");
1211                 return -ENODEV;
1212         }
1213
1214         ret = dpni_set_unicast_promisc(dpni, CMD_PRI_LOW, priv->token, false);
1215         if (ret < 0)
1216                 DPAA2_PMD_ERR("Unable to disable U promisc mode %d", ret);
1217
1218         if (dev->data->all_multicast == 0) {
1219                 ret = dpni_set_multicast_promisc(dpni, CMD_PRI_LOW,
1220                                                  priv->token, false);
1221                 if (ret < 0)
1222                         DPAA2_PMD_ERR("Unable to disable M promisc mode %d",
1223                                       ret);
1224         }
1225
1226         return ret;
1227 }
1228
1229 static int
1230 dpaa2_dev_allmulticast_enable(
1231                 struct rte_eth_dev *dev)
1232 {
1233         int ret;
1234         struct dpaa2_dev_priv *priv = dev->data->dev_private;
1235         struct fsl_mc_io *dpni = (struct fsl_mc_io *)dev->process_private;
1236
1237         PMD_INIT_FUNC_TRACE();
1238
1239         if (dpni == NULL) {
1240                 DPAA2_PMD_ERR("dpni is NULL");
1241                 return -ENODEV;
1242         }
1243
1244         ret = dpni_set_multicast_promisc(dpni, CMD_PRI_LOW, priv->token, true);
1245         if (ret < 0)
1246                 DPAA2_PMD_ERR("Unable to enable multicast mode %d", ret);
1247
1248         return ret;
1249 }
1250
1251 static int
1252 dpaa2_dev_allmulticast_disable(struct rte_eth_dev *dev)
1253 {
1254         int ret;
1255         struct dpaa2_dev_priv *priv = dev->data->dev_private;
1256         struct fsl_mc_io *dpni = (struct fsl_mc_io *)dev->process_private;
1257
1258         PMD_INIT_FUNC_TRACE();
1259
1260         if (dpni == NULL) {
1261                 DPAA2_PMD_ERR("dpni is NULL");
1262                 return -ENODEV;
1263         }
1264
1265         /* must remain on for all promiscuous */
1266         if (dev->data->promiscuous == 1)
1267                 return 0;
1268
1269         ret = dpni_set_multicast_promisc(dpni, CMD_PRI_LOW, priv->token, false);
1270         if (ret < 0)
1271                 DPAA2_PMD_ERR("Unable to disable multicast mode %d", ret);
1272
1273         return ret;
1274 }
1275
1276 static int
1277 dpaa2_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
1278 {
1279         int ret;
1280         struct dpaa2_dev_priv *priv = dev->data->dev_private;
1281         struct fsl_mc_io *dpni = (struct fsl_mc_io *)dev->process_private;
1282         uint32_t frame_size = mtu + RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN
1283                                 + VLAN_TAG_SIZE;
1284
1285         PMD_INIT_FUNC_TRACE();
1286
1287         if (dpni == NULL) {
1288                 DPAA2_PMD_ERR("dpni is NULL");
1289                 return -EINVAL;
1290         }
1291
1292         /* check that mtu is within the allowed range */
1293         if (mtu < RTE_ETHER_MIN_MTU || frame_size > DPAA2_MAX_RX_PKT_LEN)
1294                 return -EINVAL;
1295
1296         if (frame_size > RTE_ETHER_MAX_LEN)
1297                 dev->data->dev_conf.rxmode.offloads |=
1298                                                 DEV_RX_OFFLOAD_JUMBO_FRAME;
1299         else
1300                 dev->data->dev_conf.rxmode.offloads &=
1301                                                 ~DEV_RX_OFFLOAD_JUMBO_FRAME;
1302
1303         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
1304
1305         /* Set the Max Rx frame length as 'mtu' +
1306          * Maximum Ethernet header length
1307          */
1308         ret = dpni_set_max_frame_length(dpni, CMD_PRI_LOW, priv->token,
1309                                         frame_size - RTE_ETHER_CRC_LEN);
1310         if (ret) {
1311                 DPAA2_PMD_ERR("Setting the max frame length failed");
1312                 return -1;
1313         }
1314         DPAA2_PMD_INFO("MTU configured for the device: %d", mtu);
1315         return 0;
1316 }
1317
1318 static int
1319 dpaa2_dev_add_mac_addr(struct rte_eth_dev *dev,
1320                        struct rte_ether_addr *addr,
1321                        __rte_unused uint32_t index,
1322                        __rte_unused uint32_t pool)
1323 {
1324         int ret;
1325         struct dpaa2_dev_priv *priv = dev->data->dev_private;
1326         struct fsl_mc_io *dpni = (struct fsl_mc_io *)dev->process_private;
1327
1328         PMD_INIT_FUNC_TRACE();
1329
1330         if (dpni == NULL) {
1331                 DPAA2_PMD_ERR("dpni is NULL");
1332                 return -1;
1333         }
1334
1335         ret = dpni_add_mac_addr(dpni, CMD_PRI_LOW, priv->token,
1336                                 addr->addr_bytes, 0, 0, 0);
1337         if (ret)
1338                 DPAA2_PMD_ERR(
1339                         "error: Adding the MAC ADDR failed: err = %d", ret);
1340         return 0;
1341 }
1342
1343 static void
1344 dpaa2_dev_remove_mac_addr(struct rte_eth_dev *dev,
1345                           uint32_t index)
1346 {
1347         int ret;
1348         struct dpaa2_dev_priv *priv = dev->data->dev_private;
1349         struct fsl_mc_io *dpni = (struct fsl_mc_io *)dev->process_private;
1350         struct rte_eth_dev_data *data = dev->data;
1351         struct rte_ether_addr *macaddr;
1352
1353         PMD_INIT_FUNC_TRACE();
1354
1355         macaddr = &data->mac_addrs[index];
1356
1357         if (dpni == NULL) {
1358                 DPAA2_PMD_ERR("dpni is NULL");
1359                 return;
1360         }
1361
1362         ret = dpni_remove_mac_addr(dpni, CMD_PRI_LOW,
1363                                    priv->token, macaddr->addr_bytes);
1364         if (ret)
1365                 DPAA2_PMD_ERR(
1366                         "error: Removing the MAC ADDR failed: err = %d", ret);
1367 }
1368
1369 static int
1370 dpaa2_dev_set_mac_addr(struct rte_eth_dev *dev,
1371                        struct rte_ether_addr *addr)
1372 {
1373         int ret;
1374         struct dpaa2_dev_priv *priv = dev->data->dev_private;
1375         struct fsl_mc_io *dpni = (struct fsl_mc_io *)dev->process_private;
1376
1377         PMD_INIT_FUNC_TRACE();
1378
1379         if (dpni == NULL) {
1380                 DPAA2_PMD_ERR("dpni is NULL");
1381                 return -EINVAL;
1382         }
1383
1384         ret = dpni_set_primary_mac_addr(dpni, CMD_PRI_LOW,
1385                                         priv->token, addr->addr_bytes);
1386
1387         if (ret)
1388                 DPAA2_PMD_ERR(
1389                         "error: Setting the MAC ADDR failed %d", ret);
1390
1391         return ret;
1392 }
1393
1394 static
1395 int dpaa2_dev_stats_get(struct rte_eth_dev *dev,
1396                          struct rte_eth_stats *stats)
1397 {
1398         struct dpaa2_dev_priv *priv = dev->data->dev_private;
1399         struct fsl_mc_io *dpni = (struct fsl_mc_io *)dev->process_private;
1400         int32_t  retcode;
1401         uint8_t page0 = 0, page1 = 1, page2 = 2;
1402         union dpni_statistics value;
1403         int i;
1404         struct dpaa2_queue *dpaa2_rxq, *dpaa2_txq;
1405
1406         memset(&value, 0, sizeof(union dpni_statistics));
1407
1408         PMD_INIT_FUNC_TRACE();
1409
1410         if (!dpni) {
1411                 DPAA2_PMD_ERR("dpni is NULL");
1412                 return -EINVAL;
1413         }
1414
1415         if (!stats) {
1416                 DPAA2_PMD_ERR("stats is NULL");
1417                 return -EINVAL;
1418         }
1419
1420         /*Get Counters from page_0*/
1421         retcode = dpni_get_statistics(dpni, CMD_PRI_LOW, priv->token,
1422                                       page0, 0, &value);
1423         if (retcode)
1424                 goto err;
1425
1426         stats->ipackets = value.page_0.ingress_all_frames;
1427         stats->ibytes = value.page_0.ingress_all_bytes;
1428
1429         /*Get Counters from page_1*/
1430         retcode = dpni_get_statistics(dpni, CMD_PRI_LOW, priv->token,
1431                                       page1, 0, &value);
1432         if (retcode)
1433                 goto err;
1434
1435         stats->opackets = value.page_1.egress_all_frames;
1436         stats->obytes = value.page_1.egress_all_bytes;
1437
1438         /*Get Counters from page_2*/
1439         retcode = dpni_get_statistics(dpni, CMD_PRI_LOW, priv->token,
1440                                       page2, 0, &value);
1441         if (retcode)
1442                 goto err;
1443
1444         /* Ingress drop frame count due to configured rules */
1445         stats->ierrors = value.page_2.ingress_filtered_frames;
1446         /* Ingress drop frame count due to error */
1447         stats->ierrors += value.page_2.ingress_discarded_frames;
1448
1449         stats->oerrors = value.page_2.egress_discarded_frames;
1450         stats->imissed = value.page_2.ingress_nobuffer_discards;
1451
1452         /* Fill in per queue stats */
1453         for (i = 0; (i < RTE_ETHDEV_QUEUE_STAT_CNTRS) &&
1454                 (i < priv->nb_rx_queues || i < priv->nb_tx_queues); ++i) {
1455                 dpaa2_rxq = (struct dpaa2_queue *)priv->rx_vq[i];
1456                 dpaa2_txq = (struct dpaa2_queue *)priv->tx_vq[i];
1457                 if (dpaa2_rxq)
1458                         stats->q_ipackets[i] = dpaa2_rxq->rx_pkts;
1459                 if (dpaa2_txq)
1460                         stats->q_opackets[i] = dpaa2_txq->tx_pkts;
1461
1462                 /* Byte counting is not implemented */
1463                 stats->q_ibytes[i]   = 0;
1464                 stats->q_obytes[i]   = 0;
1465         }
1466
1467         return 0;
1468
1469 err:
1470         DPAA2_PMD_ERR("Operation not completed:Error Code = %d", retcode);
1471         return retcode;
1472 };
1473
1474 static int
1475 dpaa2_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
1476                      unsigned int n)
1477 {
1478         struct dpaa2_dev_priv *priv = dev->data->dev_private;
1479         struct fsl_mc_io *dpni = (struct fsl_mc_io *)dev->process_private;
1480         int32_t  retcode;
1481         union dpni_statistics value[5] = {};
1482         unsigned int i = 0, num = RTE_DIM(dpaa2_xstats_strings);
1483
1484         if (n < num)
1485                 return num;
1486
1487         if (xstats == NULL)
1488                 return 0;
1489
1490         /* Get Counters from page_0*/
1491         retcode = dpni_get_statistics(dpni, CMD_PRI_LOW, priv->token,
1492                                       0, 0, &value[0]);
1493         if (retcode)
1494                 goto err;
1495
1496         /* Get Counters from page_1*/
1497         retcode = dpni_get_statistics(dpni, CMD_PRI_LOW, priv->token,
1498                                       1, 0, &value[1]);
1499         if (retcode)
1500                 goto err;
1501
1502         /* Get Counters from page_2*/
1503         retcode = dpni_get_statistics(dpni, CMD_PRI_LOW, priv->token,
1504                                       2, 0, &value[2]);
1505         if (retcode)
1506                 goto err;
1507
1508         for (i = 0; i < priv->max_cgs; i++) {
1509                 if (!priv->cgid_in_use[i]) {
1510                         /* Get Counters from page_4*/
1511                         retcode = dpni_get_statistics(dpni, CMD_PRI_LOW,
1512                                                       priv->token,
1513                                                       4, 0, &value[4]);
1514                         if (retcode)
1515                                 goto err;
1516                         break;
1517                 }
1518         }
1519
1520         for (i = 0; i < num; i++) {
1521                 xstats[i].id = i;
1522                 xstats[i].value = value[dpaa2_xstats_strings[i].page_id].
1523                         raw.counter[dpaa2_xstats_strings[i].stats_id];
1524         }
1525         return i;
1526 err:
1527         DPAA2_PMD_ERR("Error in obtaining extended stats (%d)", retcode);
1528         return retcode;
1529 }
1530
1531 static int
1532 dpaa2_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
1533                        struct rte_eth_xstat_name *xstats_names,
1534                        unsigned int limit)
1535 {
1536         unsigned int i, stat_cnt = RTE_DIM(dpaa2_xstats_strings);
1537
1538         if (limit < stat_cnt)
1539                 return stat_cnt;
1540
1541         if (xstats_names != NULL)
1542                 for (i = 0; i < stat_cnt; i++)
1543                         strlcpy(xstats_names[i].name,
1544                                 dpaa2_xstats_strings[i].name,
1545                                 sizeof(xstats_names[i].name));
1546
1547         return stat_cnt;
1548 }
1549
1550 static int
1551 dpaa2_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids,
1552                        uint64_t *values, unsigned int n)
1553 {
1554         unsigned int i, stat_cnt = RTE_DIM(dpaa2_xstats_strings);
1555         uint64_t values_copy[stat_cnt];
1556
1557         if (!ids) {
1558                 struct dpaa2_dev_priv *priv = dev->data->dev_private;
1559                 struct fsl_mc_io *dpni =
1560                         (struct fsl_mc_io *)dev->process_private;
1561                 int32_t  retcode;
1562                 union dpni_statistics value[5] = {};
1563
1564                 if (n < stat_cnt)
1565                         return stat_cnt;
1566
1567                 if (!values)
1568                         return 0;
1569
1570                 /* Get Counters from page_0*/
1571                 retcode = dpni_get_statistics(dpni, CMD_PRI_LOW, priv->token,
1572                                               0, 0, &value[0]);
1573                 if (retcode)
1574                         return 0;
1575
1576                 /* Get Counters from page_1*/
1577                 retcode = dpni_get_statistics(dpni, CMD_PRI_LOW, priv->token,
1578                                               1, 0, &value[1]);
1579                 if (retcode)
1580                         return 0;
1581
1582                 /* Get Counters from page_2*/
1583                 retcode = dpni_get_statistics(dpni, CMD_PRI_LOW, priv->token,
1584                                               2, 0, &value[2]);
1585                 if (retcode)
1586                         return 0;
1587
1588                 /* Get Counters from page_4*/
1589                 retcode = dpni_get_statistics(dpni, CMD_PRI_LOW, priv->token,
1590                                               4, 0, &value[4]);
1591                 if (retcode)
1592                         return 0;
1593
1594                 for (i = 0; i < stat_cnt; i++) {
1595                         values[i] = value[dpaa2_xstats_strings[i].page_id].
1596                                 raw.counter[dpaa2_xstats_strings[i].stats_id];
1597                 }
1598                 return stat_cnt;
1599         }
1600
1601         dpaa2_xstats_get_by_id(dev, NULL, values_copy, stat_cnt);
1602
1603         for (i = 0; i < n; i++) {
1604                 if (ids[i] >= stat_cnt) {
1605                         DPAA2_PMD_ERR("xstats id value isn't valid");
1606                         return -1;
1607                 }
1608                 values[i] = values_copy[ids[i]];
1609         }
1610         return n;
1611 }
1612
1613 static int
1614 dpaa2_xstats_get_names_by_id(
1615         struct rte_eth_dev *dev,
1616         struct rte_eth_xstat_name *xstats_names,
1617         const uint64_t *ids,
1618         unsigned int limit)
1619 {
1620         unsigned int i, stat_cnt = RTE_DIM(dpaa2_xstats_strings);
1621         struct rte_eth_xstat_name xstats_names_copy[stat_cnt];
1622
1623         if (!ids)
1624                 return dpaa2_xstats_get_names(dev, xstats_names, limit);
1625
1626         dpaa2_xstats_get_names(dev, xstats_names_copy, limit);
1627
1628         for (i = 0; i < limit; i++) {
1629                 if (ids[i] >= stat_cnt) {
1630                         DPAA2_PMD_ERR("xstats id value isn't valid");
1631                         return -1;
1632                 }
1633                 strcpy(xstats_names[i].name, xstats_names_copy[ids[i]].name);
1634         }
1635         return limit;
1636 }
1637
1638 static int
1639 dpaa2_dev_stats_reset(struct rte_eth_dev *dev)
1640 {
1641         struct dpaa2_dev_priv *priv = dev->data->dev_private;
1642         struct fsl_mc_io *dpni = (struct fsl_mc_io *)dev->process_private;
1643         int retcode;
1644         int i;
1645         struct dpaa2_queue *dpaa2_q;
1646
1647         PMD_INIT_FUNC_TRACE();
1648
1649         if (dpni == NULL) {
1650                 DPAA2_PMD_ERR("dpni is NULL");
1651                 return -EINVAL;
1652         }
1653
1654         retcode =  dpni_reset_statistics(dpni, CMD_PRI_LOW, priv->token);
1655         if (retcode)
1656                 goto error;
1657
1658         /* Reset the per queue stats in dpaa2_queue structure */
1659         for (i = 0; i < priv->nb_rx_queues; i++) {
1660                 dpaa2_q = (struct dpaa2_queue *)priv->rx_vq[i];
1661                 if (dpaa2_q)
1662                         dpaa2_q->rx_pkts = 0;
1663         }
1664
1665         for (i = 0; i < priv->nb_tx_queues; i++) {
1666                 dpaa2_q = (struct dpaa2_queue *)priv->tx_vq[i];
1667                 if (dpaa2_q)
1668                         dpaa2_q->tx_pkts = 0;
1669         }
1670
1671         return 0;
1672
1673 error:
1674         DPAA2_PMD_ERR("Operation not completed:Error Code = %d", retcode);
1675         return retcode;
1676 };
1677
1678 /* return 0 means link status changed, -1 means not changed */
1679 static int
1680 dpaa2_dev_link_update(struct rte_eth_dev *dev,
1681                         int wait_to_complete __rte_unused)
1682 {
1683         int ret;
1684         struct dpaa2_dev_priv *priv = dev->data->dev_private;
1685         struct fsl_mc_io *dpni = (struct fsl_mc_io *)dev->process_private;
1686         struct rte_eth_link link;
1687         struct dpni_link_state state = {0};
1688
1689         if (dpni == NULL) {
1690                 DPAA2_PMD_ERR("dpni is NULL");
1691                 return 0;
1692         }
1693
1694         ret = dpni_get_link_state(dpni, CMD_PRI_LOW, priv->token, &state);
1695         if (ret < 0) {
1696                 DPAA2_PMD_DEBUG("error: dpni_get_link_state %d", ret);
1697                 return -1;
1698         }
1699
1700         memset(&link, 0, sizeof(struct rte_eth_link));
1701         link.link_status = state.up;
1702         link.link_speed = state.rate;
1703
1704         if (state.options & DPNI_LINK_OPT_HALF_DUPLEX)
1705                 link.link_duplex = ETH_LINK_HALF_DUPLEX;
1706         else
1707                 link.link_duplex = ETH_LINK_FULL_DUPLEX;
1708
1709         ret = rte_eth_linkstatus_set(dev, &link);
1710         if (ret == -1)
1711                 DPAA2_PMD_DEBUG("No change in status");
1712         else
1713                 DPAA2_PMD_INFO("Port %d Link is %s\n", dev->data->port_id,
1714                                link.link_status ? "Up" : "Down");
1715
1716         return ret;
1717 }
1718
1719 /**
1720  * Toggle the DPNI to enable, if not already enabled.
1721  * This is not strictly PHY up/down - it is more of logical toggling.
1722  */
1723 static int
1724 dpaa2_dev_set_link_up(struct rte_eth_dev *dev)
1725 {
1726         int ret = -EINVAL;
1727         struct dpaa2_dev_priv *priv;
1728         struct fsl_mc_io *dpni;
1729         int en = 0;
1730         struct dpni_link_state state = {0};
1731
1732         priv = dev->data->dev_private;
1733         dpni = (struct fsl_mc_io *)dev->process_private;
1734
1735         if (dpni == NULL) {
1736                 DPAA2_PMD_ERR("dpni is NULL");
1737                 return ret;
1738         }
1739
1740         /* Check if DPNI is currently enabled */
1741         ret = dpni_is_enabled(dpni, CMD_PRI_LOW, priv->token, &en);
1742         if (ret) {
1743                 /* Unable to obtain dpni status; Not continuing */
1744                 DPAA2_PMD_ERR("Interface Link UP failed (%d)", ret);
1745                 return -EINVAL;
1746         }
1747
1748         /* Enable link if not already enabled */
1749         if (!en) {
1750                 ret = dpni_enable(dpni, CMD_PRI_LOW, priv->token);
1751                 if (ret) {
1752                         DPAA2_PMD_ERR("Interface Link UP failed (%d)", ret);
1753                         return -EINVAL;
1754                 }
1755         }
1756         ret = dpni_get_link_state(dpni, CMD_PRI_LOW, priv->token, &state);
1757         if (ret < 0) {
1758                 DPAA2_PMD_DEBUG("Unable to get link state (%d)", ret);
1759                 return -1;
1760         }
1761
1762         /* changing tx burst function to start enqueues */
1763         dev->tx_pkt_burst = dpaa2_dev_tx;
1764         dev->data->dev_link.link_status = state.up;
1765         dev->data->dev_link.link_speed = state.rate;
1766
1767         if (state.up)
1768                 DPAA2_PMD_INFO("Port %d Link is Up", dev->data->port_id);
1769         else
1770                 DPAA2_PMD_INFO("Port %d Link is Down", dev->data->port_id);
1771         return ret;
1772 }
1773
1774 /**
1775  * Toggle the DPNI to disable, if not already disabled.
1776  * This is not strictly PHY up/down - it is more of logical toggling.
1777  */
1778 static int
1779 dpaa2_dev_set_link_down(struct rte_eth_dev *dev)
1780 {
1781         int ret = -EINVAL;
1782         struct dpaa2_dev_priv *priv;
1783         struct fsl_mc_io *dpni;
1784         int dpni_enabled = 0;
1785         int retries = 10;
1786
1787         PMD_INIT_FUNC_TRACE();
1788
1789         priv = dev->data->dev_private;
1790         dpni = (struct fsl_mc_io *)dev->process_private;
1791
1792         if (dpni == NULL) {
1793                 DPAA2_PMD_ERR("Device has not yet been configured");
1794                 return ret;
1795         }
1796
1797         /*changing  tx burst function to avoid any more enqueues */
1798         dev->tx_pkt_burst = dummy_dev_tx;
1799
1800         /* Loop while dpni_disable() attempts to drain the egress FQs
1801          * and confirm them back to us.
1802          */
1803         do {
1804                 ret = dpni_disable(dpni, 0, priv->token);
1805                 if (ret) {
1806                         DPAA2_PMD_ERR("dpni disable failed (%d)", ret);
1807                         return ret;
1808                 }
1809                 ret = dpni_is_enabled(dpni, 0, priv->token, &dpni_enabled);
1810                 if (ret) {
1811                         DPAA2_PMD_ERR("dpni enable check failed (%d)", ret);
1812                         return ret;
1813                 }
1814                 if (dpni_enabled)
1815                         /* Allow the MC some slack */
1816                         rte_delay_us(100 * 1000);
1817         } while (dpni_enabled && --retries);
1818
1819         if (!retries) {
1820                 DPAA2_PMD_WARN("Retry count exceeded disabling dpni");
1821                 /* todo- we may have to manually cleanup queues.
1822                  */
1823         } else {
1824                 DPAA2_PMD_INFO("Port %d Link DOWN successful",
1825                                dev->data->port_id);
1826         }
1827
1828         dev->data->dev_link.link_status = 0;
1829
1830         return ret;
1831 }
1832
1833 static int
1834 dpaa2_flow_ctrl_get(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1835 {
1836         int ret = -EINVAL;
1837         struct dpaa2_dev_priv *priv;
1838         struct fsl_mc_io *dpni;
1839         struct dpni_link_state state = {0};
1840
1841         PMD_INIT_FUNC_TRACE();
1842
1843         priv = dev->data->dev_private;
1844         dpni = (struct fsl_mc_io *)dev->process_private;
1845
1846         if (dpni == NULL || fc_conf == NULL) {
1847                 DPAA2_PMD_ERR("device not configured");
1848                 return ret;
1849         }
1850
1851         ret = dpni_get_link_state(dpni, CMD_PRI_LOW, priv->token, &state);
1852         if (ret) {
1853                 DPAA2_PMD_ERR("error: dpni_get_link_state %d", ret);
1854                 return ret;
1855         }
1856
1857         memset(fc_conf, 0, sizeof(struct rte_eth_fc_conf));
1858         if (state.options & DPNI_LINK_OPT_PAUSE) {
1859                 /* DPNI_LINK_OPT_PAUSE set
1860                  *  if ASYM_PAUSE not set,
1861                  *      RX Side flow control (handle received Pause frame)
1862                  *      TX side flow control (send Pause frame)
1863                  *  if ASYM_PAUSE set,
1864                  *      RX Side flow control (handle received Pause frame)
1865                  *      No TX side flow control (send Pause frame disabled)
1866                  */
1867                 if (!(state.options & DPNI_LINK_OPT_ASYM_PAUSE))
1868                         fc_conf->mode = RTE_FC_FULL;
1869                 else
1870                         fc_conf->mode = RTE_FC_RX_PAUSE;
1871         } else {
1872                 /* DPNI_LINK_OPT_PAUSE not set
1873                  *  if ASYM_PAUSE set,
1874                  *      TX side flow control (send Pause frame)
1875                  *      No RX side flow control (No action on pause frame rx)
1876                  *  if ASYM_PAUSE not set,
1877                  *      Flow control disabled
1878                  */
1879                 if (state.options & DPNI_LINK_OPT_ASYM_PAUSE)
1880                         fc_conf->mode = RTE_FC_TX_PAUSE;
1881                 else
1882                         fc_conf->mode = RTE_FC_NONE;
1883         }
1884
1885         return ret;
1886 }
1887
1888 static int
1889 dpaa2_flow_ctrl_set(struct rte_eth_dev *dev, struct rte_eth_fc_conf *fc_conf)
1890 {
1891         int ret = -EINVAL;
1892         struct dpaa2_dev_priv *priv;
1893         struct fsl_mc_io *dpni;
1894         struct dpni_link_state state = {0};
1895         struct dpni_link_cfg cfg = {0};
1896
1897         PMD_INIT_FUNC_TRACE();
1898
1899         priv = dev->data->dev_private;
1900         dpni = (struct fsl_mc_io *)dev->process_private;
1901
1902         if (dpni == NULL) {
1903                 DPAA2_PMD_ERR("dpni is NULL");
1904                 return ret;
1905         }
1906
1907         /* It is necessary to obtain the current state before setting fc_conf
1908          * as MC would return error in case rate, autoneg or duplex values are
1909          * different.
1910          */
1911         ret = dpni_get_link_state(dpni, CMD_PRI_LOW, priv->token, &state);
1912         if (ret) {
1913                 DPAA2_PMD_ERR("Unable to get link state (err=%d)", ret);
1914                 return -1;
1915         }
1916
1917         /* Disable link before setting configuration */
1918         dpaa2_dev_set_link_down(dev);
1919
1920         /* Based on fc_conf, update cfg */
1921         cfg.rate = state.rate;
1922         cfg.options = state.options;
1923
1924         /* update cfg with fc_conf */
1925         switch (fc_conf->mode) {
1926         case RTE_FC_FULL:
1927                 /* Full flow control;
1928                  * OPT_PAUSE set, ASYM_PAUSE not set
1929                  */
1930                 cfg.options |= DPNI_LINK_OPT_PAUSE;
1931                 cfg.options &= ~DPNI_LINK_OPT_ASYM_PAUSE;
1932                 break;
1933         case RTE_FC_TX_PAUSE:
1934                 /* Enable RX flow control
1935                  * OPT_PAUSE not set;
1936                  * ASYM_PAUSE set;
1937                  */
1938                 cfg.options |= DPNI_LINK_OPT_ASYM_PAUSE;
1939                 cfg.options &= ~DPNI_LINK_OPT_PAUSE;
1940                 break;
1941         case RTE_FC_RX_PAUSE:
1942                 /* Enable TX Flow control
1943                  * OPT_PAUSE set
1944                  * ASYM_PAUSE set
1945                  */
1946                 cfg.options |= DPNI_LINK_OPT_PAUSE;
1947                 cfg.options |= DPNI_LINK_OPT_ASYM_PAUSE;
1948                 break;
1949         case RTE_FC_NONE:
1950                 /* Disable Flow control
1951                  * OPT_PAUSE not set
1952                  * ASYM_PAUSE not set
1953                  */
1954                 cfg.options &= ~DPNI_LINK_OPT_PAUSE;
1955                 cfg.options &= ~DPNI_LINK_OPT_ASYM_PAUSE;
1956                 break;
1957         default:
1958                 DPAA2_PMD_ERR("Incorrect Flow control flag (%d)",
1959                               fc_conf->mode);
1960                 return -1;
1961         }
1962
1963         ret = dpni_set_link_cfg(dpni, CMD_PRI_LOW, priv->token, &cfg);
1964         if (ret)
1965                 DPAA2_PMD_ERR("Unable to set Link configuration (err=%d)",
1966                               ret);
1967
1968         /* Enable link */
1969         dpaa2_dev_set_link_up(dev);
1970
1971         return ret;
1972 }
1973
1974 static int
1975 dpaa2_dev_rss_hash_update(struct rte_eth_dev *dev,
1976                           struct rte_eth_rss_conf *rss_conf)
1977 {
1978         struct rte_eth_dev_data *data = dev->data;
1979         struct dpaa2_dev_priv *priv = data->dev_private;
1980         struct rte_eth_conf *eth_conf = &data->dev_conf;
1981         int ret, tc_index;
1982
1983         PMD_INIT_FUNC_TRACE();
1984
1985         if (rss_conf->rss_hf) {
1986                 for (tc_index = 0; tc_index < priv->num_rx_tc; tc_index++) {
1987                         ret = dpaa2_setup_flow_dist(dev, rss_conf->rss_hf,
1988                                 tc_index);
1989                         if (ret) {
1990                                 DPAA2_PMD_ERR("Unable to set flow dist on tc%d",
1991                                         tc_index);
1992                                 return ret;
1993                         }
1994                 }
1995         } else {
1996                 for (tc_index = 0; tc_index < priv->num_rx_tc; tc_index++) {
1997                         ret = dpaa2_remove_flow_dist(dev, tc_index);
1998                         if (ret) {
1999                                 DPAA2_PMD_ERR(
2000                                         "Unable to remove flow dist on tc%d",
2001                                         tc_index);
2002                                 return ret;
2003                         }
2004                 }
2005         }
2006         eth_conf->rx_adv_conf.rss_conf.rss_hf = rss_conf->rss_hf;
2007         return 0;
2008 }
2009
2010 static int
2011 dpaa2_dev_rss_hash_conf_get(struct rte_eth_dev *dev,
2012                             struct rte_eth_rss_conf *rss_conf)
2013 {
2014         struct rte_eth_dev_data *data = dev->data;
2015         struct rte_eth_conf *eth_conf = &data->dev_conf;
2016
2017         /* dpaa2 does not support rss_key, so length should be 0*/
2018         rss_conf->rss_key_len = 0;
2019         rss_conf->rss_hf = eth_conf->rx_adv_conf.rss_conf.rss_hf;
2020         return 0;
2021 }
2022
2023 int dpaa2_eth_eventq_attach(const struct rte_eth_dev *dev,
2024                 int eth_rx_queue_id,
2025                 struct dpaa2_dpcon_dev *dpcon,
2026                 const struct rte_event_eth_rx_adapter_queue_conf *queue_conf)
2027 {
2028         struct dpaa2_dev_priv *eth_priv = dev->data->dev_private;
2029         struct fsl_mc_io *dpni = (struct fsl_mc_io *)dev->process_private;
2030         struct dpaa2_queue *dpaa2_ethq = eth_priv->rx_vq[eth_rx_queue_id];
2031         uint8_t flow_id = dpaa2_ethq->flow_id;
2032         struct dpni_queue cfg;
2033         uint8_t options, priority;
2034         int ret;
2035
2036         if (queue_conf->ev.sched_type == RTE_SCHED_TYPE_PARALLEL)
2037                 dpaa2_ethq->cb = dpaa2_dev_process_parallel_event;
2038         else if (queue_conf->ev.sched_type == RTE_SCHED_TYPE_ATOMIC)
2039                 dpaa2_ethq->cb = dpaa2_dev_process_atomic_event;
2040         else if (queue_conf->ev.sched_type == RTE_SCHED_TYPE_ORDERED)
2041                 dpaa2_ethq->cb = dpaa2_dev_process_ordered_event;
2042         else
2043                 return -EINVAL;
2044
2045         priority = (RTE_EVENT_DEV_PRIORITY_LOWEST / queue_conf->ev.priority) *
2046                    (dpcon->num_priorities - 1);
2047
2048         memset(&cfg, 0, sizeof(struct dpni_queue));
2049         options = DPNI_QUEUE_OPT_DEST;
2050         cfg.destination.type = DPNI_DEST_DPCON;
2051         cfg.destination.id = dpcon->dpcon_id;
2052         cfg.destination.priority = priority;
2053
2054         if (queue_conf->ev.sched_type == RTE_SCHED_TYPE_ATOMIC) {
2055                 options |= DPNI_QUEUE_OPT_HOLD_ACTIVE;
2056                 cfg.destination.hold_active = 1;
2057         }
2058
2059         if (queue_conf->ev.sched_type == RTE_SCHED_TYPE_ORDERED &&
2060                         !eth_priv->en_ordered) {
2061                 struct opr_cfg ocfg;
2062
2063                 /* Restoration window size = 256 frames */
2064                 ocfg.oprrws = 3;
2065                 /* Restoration window size = 512 frames for LX2 */
2066                 if (dpaa2_svr_family == SVR_LX2160A)
2067                         ocfg.oprrws = 4;
2068                 /* Auto advance NESN window enabled */
2069                 ocfg.oa = 1;
2070                 /* Late arrival window size disabled */
2071                 ocfg.olws = 0;
2072                 /* ORL resource exhaustaion advance NESN disabled */
2073                 ocfg.oeane = 0;
2074                 /* Loose ordering enabled */
2075                 ocfg.oloe = 1;
2076                 eth_priv->en_loose_ordered = 1;
2077                 /* Strict ordering enabled if explicitly set */
2078                 if (getenv("DPAA2_STRICT_ORDERING_ENABLE")) {
2079                         ocfg.oloe = 0;
2080                         eth_priv->en_loose_ordered = 0;
2081                 }
2082
2083                 ret = dpni_set_opr(dpni, CMD_PRI_LOW, eth_priv->token,
2084                                    dpaa2_ethq->tc_index, flow_id,
2085                                    OPR_OPT_CREATE, &ocfg);
2086                 if (ret) {
2087                         DPAA2_PMD_ERR("Error setting opr: ret: %d\n", ret);
2088                         return ret;
2089                 }
2090
2091                 eth_priv->en_ordered = 1;
2092         }
2093
2094         options |= DPNI_QUEUE_OPT_USER_CTX;
2095         cfg.user_context = (size_t)(dpaa2_ethq);
2096
2097         ret = dpni_set_queue(dpni, CMD_PRI_LOW, eth_priv->token, DPNI_QUEUE_RX,
2098                              dpaa2_ethq->tc_index, flow_id, options, &cfg);
2099         if (ret) {
2100                 DPAA2_PMD_ERR("Error in dpni_set_queue: ret: %d", ret);
2101                 return ret;
2102         }
2103
2104         memcpy(&dpaa2_ethq->ev, &queue_conf->ev, sizeof(struct rte_event));
2105
2106         return 0;
2107 }
2108
2109 int dpaa2_eth_eventq_detach(const struct rte_eth_dev *dev,
2110                 int eth_rx_queue_id)
2111 {
2112         struct dpaa2_dev_priv *eth_priv = dev->data->dev_private;
2113         struct fsl_mc_io *dpni = (struct fsl_mc_io *)dev->process_private;
2114         struct dpaa2_queue *dpaa2_ethq = eth_priv->rx_vq[eth_rx_queue_id];
2115         uint8_t flow_id = dpaa2_ethq->flow_id;
2116         struct dpni_queue cfg;
2117         uint8_t options;
2118         int ret;
2119
2120         memset(&cfg, 0, sizeof(struct dpni_queue));
2121         options = DPNI_QUEUE_OPT_DEST;
2122         cfg.destination.type = DPNI_DEST_NONE;
2123
2124         ret = dpni_set_queue(dpni, CMD_PRI_LOW, eth_priv->token, DPNI_QUEUE_RX,
2125                              dpaa2_ethq->tc_index, flow_id, options, &cfg);
2126         if (ret)
2127                 DPAA2_PMD_ERR("Error in dpni_set_queue: ret: %d", ret);
2128
2129         return ret;
2130 }
2131
2132 static inline int
2133 dpaa2_dev_verify_filter_ops(enum rte_filter_op filter_op)
2134 {
2135         unsigned int i;
2136
2137         for (i = 0; i < RTE_DIM(dpaa2_supported_filter_ops); i++) {
2138                 if (dpaa2_supported_filter_ops[i] == filter_op)
2139                         return 0;
2140         }
2141         return -ENOTSUP;
2142 }
2143
2144 static int
2145 dpaa2_dev_flow_ctrl(struct rte_eth_dev *dev,
2146                     enum rte_filter_type filter_type,
2147                                  enum rte_filter_op filter_op,
2148                                  void *arg)
2149 {
2150         int ret = 0;
2151
2152         if (!dev)
2153                 return -ENODEV;
2154
2155         switch (filter_type) {
2156         case RTE_ETH_FILTER_GENERIC:
2157                 if (dpaa2_dev_verify_filter_ops(filter_op) < 0) {
2158                         ret = -ENOTSUP;
2159                         break;
2160                 }
2161                 *(const void **)arg = &dpaa2_flow_ops;
2162                 dpaa2_filter_type |= filter_type;
2163                 break;
2164         default:
2165                 RTE_LOG(ERR, PMD, "Filter type (%d) not supported",
2166                         filter_type);
2167                 ret = -ENOTSUP;
2168                 break;
2169         }
2170         return ret;
2171 }
2172
2173 static struct eth_dev_ops dpaa2_ethdev_ops = {
2174         .dev_configure    = dpaa2_eth_dev_configure,
2175         .dev_start            = dpaa2_dev_start,
2176         .dev_stop             = dpaa2_dev_stop,
2177         .dev_close            = dpaa2_dev_close,
2178         .promiscuous_enable   = dpaa2_dev_promiscuous_enable,
2179         .promiscuous_disable  = dpaa2_dev_promiscuous_disable,
2180         .allmulticast_enable  = dpaa2_dev_allmulticast_enable,
2181         .allmulticast_disable = dpaa2_dev_allmulticast_disable,
2182         .dev_set_link_up      = dpaa2_dev_set_link_up,
2183         .dev_set_link_down    = dpaa2_dev_set_link_down,
2184         .link_update       = dpaa2_dev_link_update,
2185         .stats_get             = dpaa2_dev_stats_get,
2186         .xstats_get            = dpaa2_dev_xstats_get,
2187         .xstats_get_by_id     = dpaa2_xstats_get_by_id,
2188         .xstats_get_names_by_id = dpaa2_xstats_get_names_by_id,
2189         .xstats_get_names      = dpaa2_xstats_get_names,
2190         .stats_reset       = dpaa2_dev_stats_reset,
2191         .xstats_reset         = dpaa2_dev_stats_reset,
2192         .fw_version_get    = dpaa2_fw_version_get,
2193         .dev_infos_get     = dpaa2_dev_info_get,
2194         .dev_supported_ptypes_get = dpaa2_supported_ptypes_get,
2195         .mtu_set           = dpaa2_dev_mtu_set,
2196         .vlan_filter_set      = dpaa2_vlan_filter_set,
2197         .vlan_offload_set     = dpaa2_vlan_offload_set,
2198         .vlan_tpid_set        = dpaa2_vlan_tpid_set,
2199         .rx_queue_setup    = dpaa2_dev_rx_queue_setup,
2200         .rx_queue_release  = dpaa2_dev_rx_queue_release,
2201         .tx_queue_setup    = dpaa2_dev_tx_queue_setup,
2202         .tx_queue_release  = dpaa2_dev_tx_queue_release,
2203         .rx_queue_count       = dpaa2_dev_rx_queue_count,
2204         .flow_ctrl_get        = dpaa2_flow_ctrl_get,
2205         .flow_ctrl_set        = dpaa2_flow_ctrl_set,
2206         .mac_addr_add         = dpaa2_dev_add_mac_addr,
2207         .mac_addr_remove      = dpaa2_dev_remove_mac_addr,
2208         .mac_addr_set         = dpaa2_dev_set_mac_addr,
2209         .rss_hash_update      = dpaa2_dev_rss_hash_update,
2210         .rss_hash_conf_get    = dpaa2_dev_rss_hash_conf_get,
2211         .filter_ctrl          = dpaa2_dev_flow_ctrl,
2212 #if defined(RTE_LIBRTE_IEEE1588)
2213         .timesync_enable      = dpaa2_timesync_enable,
2214         .timesync_disable     = dpaa2_timesync_disable,
2215         .timesync_read_time   = dpaa2_timesync_read_time,
2216         .timesync_write_time  = dpaa2_timesync_write_time,
2217         .timesync_adjust_time = dpaa2_timesync_adjust_time,
2218         .timesync_read_rx_timestamp = dpaa2_timesync_read_rx_timestamp,
2219         .timesync_read_tx_timestamp = dpaa2_timesync_read_tx_timestamp,
2220 #endif
2221 };
2222
2223 /* Populate the mac address from physically available (u-boot/firmware) and/or
2224  * one set by higher layers like MC (restool) etc.
2225  * Returns the table of MAC entries (multiple entries)
2226  */
2227 static int
2228 populate_mac_addr(struct fsl_mc_io *dpni_dev, struct dpaa2_dev_priv *priv,
2229                   struct rte_ether_addr *mac_entry)
2230 {
2231         int ret;
2232         struct rte_ether_addr phy_mac, prime_mac;
2233
2234         memset(&phy_mac, 0, sizeof(struct rte_ether_addr));
2235         memset(&prime_mac, 0, sizeof(struct rte_ether_addr));
2236
2237         /* Get the physical device MAC address */
2238         ret = dpni_get_port_mac_addr(dpni_dev, CMD_PRI_LOW, priv->token,
2239                                      phy_mac.addr_bytes);
2240         if (ret) {
2241                 DPAA2_PMD_ERR("DPNI get physical port MAC failed: %d", ret);
2242                 goto cleanup;
2243         }
2244
2245         ret = dpni_get_primary_mac_addr(dpni_dev, CMD_PRI_LOW, priv->token,
2246                                         prime_mac.addr_bytes);
2247         if (ret) {
2248                 DPAA2_PMD_ERR("DPNI get Prime port MAC failed: %d", ret);
2249                 goto cleanup;
2250         }
2251
2252         /* Now that both MAC have been obtained, do:
2253          *  if not_empty_mac(phy) && phy != Prime, overwrite prime with Phy
2254          *     and return phy
2255          *  If empty_mac(phy), return prime.
2256          *  if both are empty, create random MAC, set as prime and return
2257          */
2258         if (!rte_is_zero_ether_addr(&phy_mac)) {
2259                 /* If the addresses are not same, overwrite prime */
2260                 if (!rte_is_same_ether_addr(&phy_mac, &prime_mac)) {
2261                         ret = dpni_set_primary_mac_addr(dpni_dev, CMD_PRI_LOW,
2262                                                         priv->token,
2263                                                         phy_mac.addr_bytes);
2264                         if (ret) {
2265                                 DPAA2_PMD_ERR("Unable to set MAC Address: %d",
2266                                               ret);
2267                                 goto cleanup;
2268                         }
2269                         memcpy(&prime_mac, &phy_mac,
2270                                 sizeof(struct rte_ether_addr));
2271                 }
2272         } else if (rte_is_zero_ether_addr(&prime_mac)) {
2273                 /* In case phys and prime, both are zero, create random MAC */
2274                 rte_eth_random_addr(prime_mac.addr_bytes);
2275                 ret = dpni_set_primary_mac_addr(dpni_dev, CMD_PRI_LOW,
2276                                                 priv->token,
2277                                                 prime_mac.addr_bytes);
2278                 if (ret) {
2279                         DPAA2_PMD_ERR("Unable to set MAC Address: %d", ret);
2280                         goto cleanup;
2281                 }
2282         }
2283
2284         /* prime_mac the final MAC address */
2285         memcpy(mac_entry, &prime_mac, sizeof(struct rte_ether_addr));
2286         return 0;
2287
2288 cleanup:
2289         return -1;
2290 }
2291
2292 static int
2293 check_devargs_handler(__rte_unused const char *key, const char *value,
2294                       __rte_unused void *opaque)
2295 {
2296         if (strcmp(value, "1"))
2297                 return -1;
2298
2299         return 0;
2300 }
2301
2302 static int
2303 dpaa2_get_devargs(struct rte_devargs *devargs, const char *key)
2304 {
2305         struct rte_kvargs *kvlist;
2306
2307         if (!devargs)
2308                 return 0;
2309
2310         kvlist = rte_kvargs_parse(devargs->args, NULL);
2311         if (!kvlist)
2312                 return 0;
2313
2314         if (!rte_kvargs_count(kvlist, key)) {
2315                 rte_kvargs_free(kvlist);
2316                 return 0;
2317         }
2318
2319         if (rte_kvargs_process(kvlist, key,
2320                                check_devargs_handler, NULL) < 0) {
2321                 rte_kvargs_free(kvlist);
2322                 return 0;
2323         }
2324         rte_kvargs_free(kvlist);
2325
2326         return 1;
2327 }
2328
2329 static int
2330 dpaa2_dev_init(struct rte_eth_dev *eth_dev)
2331 {
2332         struct rte_device *dev = eth_dev->device;
2333         struct rte_dpaa2_device *dpaa2_dev;
2334         struct fsl_mc_io *dpni_dev;
2335         struct dpni_attr attr;
2336         struct dpaa2_dev_priv *priv = eth_dev->data->dev_private;
2337         struct dpni_buffer_layout layout;
2338         int ret, hw_id, i;
2339
2340         PMD_INIT_FUNC_TRACE();
2341
2342         dpni_dev = rte_malloc(NULL, sizeof(struct fsl_mc_io), 0);
2343         if (!dpni_dev) {
2344                 DPAA2_PMD_ERR("Memory allocation failed for dpni device");
2345                 return -1;
2346         }
2347         dpni_dev->regs = dpaa2_get_mcp_ptr(MC_PORTAL_INDEX);
2348         eth_dev->process_private = (void *)dpni_dev;
2349
2350         /* For secondary processes, the primary has done all the work */
2351         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
2352                 /* In case of secondary, only burst and ops API need to be
2353                  * plugged.
2354                  */
2355                 eth_dev->dev_ops = &dpaa2_ethdev_ops;
2356                 if (dpaa2_get_devargs(dev->devargs, DRIVER_LOOPBACK_MODE))
2357                         eth_dev->rx_pkt_burst = dpaa2_dev_loopback_rx;
2358                 else if (dpaa2_get_devargs(dev->devargs,
2359                                         DRIVER_NO_PREFETCH_MODE))
2360                         eth_dev->rx_pkt_burst = dpaa2_dev_rx;
2361                 else
2362                         eth_dev->rx_pkt_burst = dpaa2_dev_prefetch_rx;
2363                 eth_dev->tx_pkt_burst = dpaa2_dev_tx;
2364                 return 0;
2365         }
2366
2367         dpaa2_dev = container_of(dev, struct rte_dpaa2_device, device);
2368
2369         hw_id = dpaa2_dev->object_id;
2370         ret = dpni_open(dpni_dev, CMD_PRI_LOW, hw_id, &priv->token);
2371         if (ret) {
2372                 DPAA2_PMD_ERR(
2373                              "Failure in opening dpni@%d with err code %d",
2374                              hw_id, ret);
2375                 rte_free(dpni_dev);
2376                 return -1;
2377         }
2378
2379         /* Clean the device first */
2380         ret = dpni_reset(dpni_dev, CMD_PRI_LOW, priv->token);
2381         if (ret) {
2382                 DPAA2_PMD_ERR("Failure cleaning dpni@%d with err code %d",
2383                               hw_id, ret);
2384                 goto init_err;
2385         }
2386
2387         ret = dpni_get_attributes(dpni_dev, CMD_PRI_LOW, priv->token, &attr);
2388         if (ret) {
2389                 DPAA2_PMD_ERR(
2390                              "Failure in get dpni@%d attribute, err code %d",
2391                              hw_id, ret);
2392                 goto init_err;
2393         }
2394
2395         priv->num_rx_tc = attr.num_rx_tcs;
2396         priv->qos_entries = attr.qos_entries;
2397         priv->fs_entries = attr.fs_entries;
2398         priv->dist_queues = attr.num_queues;
2399
2400         /* only if the custom CG is enabled */
2401         if (attr.options & DPNI_OPT_CUSTOM_CG)
2402                 priv->max_cgs = attr.num_cgs;
2403         else
2404                 priv->max_cgs = 0;
2405
2406         for (i = 0; i < priv->max_cgs; i++)
2407                 priv->cgid_in_use[i] = 0;
2408
2409         for (i = 0; i < attr.num_rx_tcs; i++)
2410                 priv->nb_rx_queues += attr.num_queues;
2411
2412         /* Using number of TX queues as number of TX TCs */
2413         priv->nb_tx_queues = attr.num_tx_tcs;
2414
2415         DPAA2_PMD_DEBUG("RX-TC= %d, rx_queues= %d, tx_queues=%d, max_cgs=%d",
2416                         priv->num_rx_tc, priv->nb_rx_queues,
2417                         priv->nb_tx_queues, priv->max_cgs);
2418
2419         priv->hw = dpni_dev;
2420         priv->hw_id = hw_id;
2421         priv->options = attr.options;
2422         priv->max_mac_filters = attr.mac_filter_entries;
2423         priv->max_vlan_filters = attr.vlan_filter_entries;
2424         priv->flags = 0;
2425 #if defined(RTE_LIBRTE_IEEE1588)
2426         priv->tx_conf_en = 1;
2427 #else
2428         priv->tx_conf_en = 0;
2429 #endif
2430
2431         /* Allocate memory for hardware structure for queues */
2432         ret = dpaa2_alloc_rx_tx_queues(eth_dev);
2433         if (ret) {
2434                 DPAA2_PMD_ERR("Queue allocation Failed");
2435                 goto init_err;
2436         }
2437
2438         /* Allocate memory for storing MAC addresses.
2439          * Table of mac_filter_entries size is allocated so that RTE ether lib
2440          * can add MAC entries when rte_eth_dev_mac_addr_add is called.
2441          */
2442         eth_dev->data->mac_addrs = rte_zmalloc("dpni",
2443                 RTE_ETHER_ADDR_LEN * attr.mac_filter_entries, 0);
2444         if (eth_dev->data->mac_addrs == NULL) {
2445                 DPAA2_PMD_ERR(
2446                    "Failed to allocate %d bytes needed to store MAC addresses",
2447                    RTE_ETHER_ADDR_LEN * attr.mac_filter_entries);
2448                 ret = -ENOMEM;
2449                 goto init_err;
2450         }
2451
2452         ret = populate_mac_addr(dpni_dev, priv, &eth_dev->data->mac_addrs[0]);
2453         if (ret) {
2454                 DPAA2_PMD_ERR("Unable to fetch MAC Address for device");
2455                 rte_free(eth_dev->data->mac_addrs);
2456                 eth_dev->data->mac_addrs = NULL;
2457                 goto init_err;
2458         }
2459
2460         /* ... tx buffer layout ... */
2461         memset(&layout, 0, sizeof(struct dpni_buffer_layout));
2462         if (priv->tx_conf_en) {
2463                 layout.options = DPNI_BUF_LAYOUT_OPT_FRAME_STATUS |
2464                                  DPNI_BUF_LAYOUT_OPT_TIMESTAMP;
2465                 layout.pass_timestamp = true;
2466         } else {
2467                 layout.options = DPNI_BUF_LAYOUT_OPT_FRAME_STATUS;
2468         }
2469         layout.pass_frame_status = 1;
2470         ret = dpni_set_buffer_layout(dpni_dev, CMD_PRI_LOW, priv->token,
2471                                      DPNI_QUEUE_TX, &layout);
2472         if (ret) {
2473                 DPAA2_PMD_ERR("Error (%d) in setting tx buffer layout", ret);
2474                 goto init_err;
2475         }
2476
2477         /* ... tx-conf and error buffer layout ... */
2478         memset(&layout, 0, sizeof(struct dpni_buffer_layout));
2479         if (priv->tx_conf_en) {
2480                 layout.options = DPNI_BUF_LAYOUT_OPT_FRAME_STATUS |
2481                                  DPNI_BUF_LAYOUT_OPT_TIMESTAMP;
2482                 layout.pass_timestamp = true;
2483         } else {
2484                 layout.options = DPNI_BUF_LAYOUT_OPT_FRAME_STATUS;
2485         }
2486         layout.pass_frame_status = 1;
2487         ret = dpni_set_buffer_layout(dpni_dev, CMD_PRI_LOW, priv->token,
2488                                      DPNI_QUEUE_TX_CONFIRM, &layout);
2489         if (ret) {
2490                 DPAA2_PMD_ERR("Error (%d) in setting tx-conf buffer layout",
2491                              ret);
2492                 goto init_err;
2493         }
2494
2495         eth_dev->dev_ops = &dpaa2_ethdev_ops;
2496
2497         if (dpaa2_get_devargs(dev->devargs, DRIVER_LOOPBACK_MODE)) {
2498                 eth_dev->rx_pkt_burst = dpaa2_dev_loopback_rx;
2499                 DPAA2_PMD_INFO("Loopback mode");
2500         } else if (dpaa2_get_devargs(dev->devargs, DRIVER_NO_PREFETCH_MODE)) {
2501                 eth_dev->rx_pkt_burst = dpaa2_dev_rx;
2502                 DPAA2_PMD_INFO("No Prefetch mode");
2503         } else {
2504                 eth_dev->rx_pkt_burst = dpaa2_dev_prefetch_rx;
2505         }
2506         eth_dev->tx_pkt_burst = dpaa2_dev_tx;
2507
2508         /*Init fields w.r.t. classficaition*/
2509         memset(&priv->extract.qos_key_extract, 0,
2510                 sizeof(struct dpaa2_key_extract));
2511         priv->extract.qos_extract_param = (size_t)rte_malloc(NULL, 256, 64);
2512         if (!priv->extract.qos_extract_param) {
2513                 DPAA2_PMD_ERR(" Error(%d) in allocation resources for flow "
2514                             " classificaiton ", ret);
2515                 goto init_err;
2516         }
2517         priv->extract.qos_key_extract.key_info.ipv4_src_offset =
2518                 IP_ADDRESS_OFFSET_INVALID;
2519         priv->extract.qos_key_extract.key_info.ipv4_dst_offset =
2520                 IP_ADDRESS_OFFSET_INVALID;
2521         priv->extract.qos_key_extract.key_info.ipv6_src_offset =
2522                 IP_ADDRESS_OFFSET_INVALID;
2523         priv->extract.qos_key_extract.key_info.ipv6_dst_offset =
2524                 IP_ADDRESS_OFFSET_INVALID;
2525
2526         for (i = 0; i < MAX_TCS; i++) {
2527                 memset(&priv->extract.tc_key_extract[i], 0,
2528                         sizeof(struct dpaa2_key_extract));
2529                 priv->extract.tc_extract_param[i] =
2530                         (size_t)rte_malloc(NULL, 256, 64);
2531                 if (!priv->extract.tc_extract_param[i]) {
2532                         DPAA2_PMD_ERR(" Error(%d) in allocation resources for flow classificaiton",
2533                                      ret);
2534                         goto init_err;
2535                 }
2536                 priv->extract.tc_key_extract[i].key_info.ipv4_src_offset =
2537                         IP_ADDRESS_OFFSET_INVALID;
2538                 priv->extract.tc_key_extract[i].key_info.ipv4_dst_offset =
2539                         IP_ADDRESS_OFFSET_INVALID;
2540                 priv->extract.tc_key_extract[i].key_info.ipv6_src_offset =
2541                         IP_ADDRESS_OFFSET_INVALID;
2542                 priv->extract.tc_key_extract[i].key_info.ipv6_dst_offset =
2543                         IP_ADDRESS_OFFSET_INVALID;
2544         }
2545
2546         ret = dpni_set_max_frame_length(dpni_dev, CMD_PRI_LOW, priv->token,
2547                                         RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN
2548                                         + VLAN_TAG_SIZE);
2549         if (ret) {
2550                 DPAA2_PMD_ERR("Unable to set mtu. check config");
2551                 goto init_err;
2552         }
2553
2554         /*TODO To enable soft parser support DPAA2 driver needs to integrate
2555          * with external entity to receive byte code for software sequence
2556          * and same will be offload to the H/W using MC interface.
2557          * Currently it is assumed that DPAA2 driver has byte code by some
2558          * mean and same if offloaded to H/W.
2559          */
2560         if (getenv("DPAA2_ENABLE_SOFT_PARSER")) {
2561                 WRIOP_SS_INITIALIZER(priv);
2562                 ret = dpaa2_eth_load_wriop_soft_parser(priv, DPNI_SS_INGRESS);
2563                 if (ret < 0) {
2564                         DPAA2_PMD_ERR(" Error(%d) in loading softparser\n",
2565                                       ret);
2566                         return ret;
2567                 }
2568
2569                 ret = dpaa2_eth_enable_wriop_soft_parser(priv,
2570                                                          DPNI_SS_INGRESS);
2571                 if (ret < 0) {
2572                         DPAA2_PMD_ERR(" Error(%d) in enabling softparser\n",
2573                                       ret);
2574                         return ret;
2575                 }
2576         }
2577         RTE_LOG(INFO, PMD, "%s: netdev created\n", eth_dev->data->name);
2578         return 0;
2579 init_err:
2580         dpaa2_dev_uninit(eth_dev);
2581         return ret;
2582 }
2583
2584 static int
2585 dpaa2_dev_uninit(struct rte_eth_dev *eth_dev)
2586 {
2587         struct dpaa2_dev_priv *priv = eth_dev->data->dev_private;
2588         struct fsl_mc_io *dpni = (struct fsl_mc_io *)eth_dev->process_private;
2589         int i, ret;
2590
2591         PMD_INIT_FUNC_TRACE();
2592
2593         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
2594                 return 0;
2595
2596         if (!dpni) {
2597                 DPAA2_PMD_WARN("Already closed or not started");
2598                 return -1;
2599         }
2600
2601         dpaa2_dev_close(eth_dev);
2602
2603         dpaa2_free_rx_tx_queues(eth_dev);
2604
2605         /* Close the device at underlying layer*/
2606         ret = dpni_close(dpni, CMD_PRI_LOW, priv->token);
2607         if (ret) {
2608                 DPAA2_PMD_ERR(
2609                              "Failure closing dpni device with err code %d",
2610                              ret);
2611         }
2612
2613         /* Free the allocated memory for ethernet private data and dpni*/
2614         priv->hw = NULL;
2615         eth_dev->process_private = NULL;
2616         rte_free(dpni);
2617
2618         for (i = 0; i < MAX_TCS; i++)
2619                 rte_free((void *)(size_t)priv->extract.tc_extract_param[i]);
2620
2621         if (priv->extract.qos_extract_param)
2622                 rte_free((void *)(size_t)priv->extract.qos_extract_param);
2623
2624         eth_dev->dev_ops = NULL;
2625         eth_dev->rx_pkt_burst = NULL;
2626         eth_dev->tx_pkt_burst = NULL;
2627
2628         DPAA2_PMD_INFO("%s: netdev deleted", eth_dev->data->name);
2629         return 0;
2630 }
2631
2632 static int
2633 rte_dpaa2_probe(struct rte_dpaa2_driver *dpaa2_drv,
2634                 struct rte_dpaa2_device *dpaa2_dev)
2635 {
2636         struct rte_eth_dev *eth_dev;
2637         struct dpaa2_dev_priv *dev_priv;
2638         int diag;
2639
2640         if ((DPAA2_MBUF_HW_ANNOTATION + DPAA2_FD_PTA_SIZE) >
2641                 RTE_PKTMBUF_HEADROOM) {
2642                 DPAA2_PMD_ERR(
2643                 "RTE_PKTMBUF_HEADROOM(%d) shall be > DPAA2 Annotation req(%d)",
2644                 RTE_PKTMBUF_HEADROOM,
2645                 DPAA2_MBUF_HW_ANNOTATION + DPAA2_FD_PTA_SIZE);
2646
2647                 return -1;
2648         }
2649
2650         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
2651                 eth_dev = rte_eth_dev_allocate(dpaa2_dev->device.name);
2652                 if (!eth_dev)
2653                         return -ENODEV;
2654                 dev_priv = rte_zmalloc("ethdev private structure",
2655                                        sizeof(struct dpaa2_dev_priv),
2656                                        RTE_CACHE_LINE_SIZE);
2657                 if (dev_priv == NULL) {
2658                         DPAA2_PMD_CRIT(
2659                                 "Unable to allocate memory for private data");
2660                         rte_eth_dev_release_port(eth_dev);
2661                         return -ENOMEM;
2662                 }
2663                 eth_dev->data->dev_private = (void *)dev_priv;
2664                 /* Store a pointer to eth_dev in dev_private */
2665                 dev_priv->eth_dev = eth_dev;
2666                 dev_priv->tx_conf_en = 0;
2667         } else {
2668                 eth_dev = rte_eth_dev_attach_secondary(dpaa2_dev->device.name);
2669                 if (!eth_dev) {
2670                         DPAA2_PMD_DEBUG("returning enodev");
2671                         return -ENODEV;
2672                 }
2673         }
2674
2675         eth_dev->device = &dpaa2_dev->device;
2676
2677         dpaa2_dev->eth_dev = eth_dev;
2678         eth_dev->data->rx_mbuf_alloc_failed = 0;
2679
2680         if (dpaa2_drv->drv_flags & RTE_DPAA2_DRV_INTR_LSC)
2681                 eth_dev->data->dev_flags |= RTE_ETH_DEV_INTR_LSC;
2682
2683         /* Invoke PMD device initialization function */
2684         diag = dpaa2_dev_init(eth_dev);
2685         if (diag == 0) {
2686                 rte_eth_dev_probing_finish(eth_dev);
2687                 return 0;
2688         }
2689
2690         rte_eth_dev_release_port(eth_dev);
2691         return diag;
2692 }
2693
2694 static int
2695 rte_dpaa2_remove(struct rte_dpaa2_device *dpaa2_dev)
2696 {
2697         struct rte_eth_dev *eth_dev;
2698
2699         eth_dev = dpaa2_dev->eth_dev;
2700         dpaa2_dev_uninit(eth_dev);
2701
2702         rte_eth_dev_release_port(eth_dev);
2703
2704         return 0;
2705 }
2706
2707 static struct rte_dpaa2_driver rte_dpaa2_pmd = {
2708         .drv_flags = RTE_DPAA2_DRV_INTR_LSC | RTE_DPAA2_DRV_IOVA_AS_VA,
2709         .drv_type = DPAA2_ETH,
2710         .probe = rte_dpaa2_probe,
2711         .remove = rte_dpaa2_remove,
2712 };
2713
2714 RTE_PMD_REGISTER_DPAA2(net_dpaa2, rte_dpaa2_pmd);
2715 RTE_PMD_REGISTER_PARAM_STRING(net_dpaa2,
2716                 DRIVER_LOOPBACK_MODE "=<int> "
2717                 DRIVER_NO_PREFETCH_MODE "=<int>");
2718 RTE_LOG_REGISTER(dpaa2_logtype_pmd, pmd.net.dpaa2, NOTICE);