drivers/net/dpaa/dpaa_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  *   Copyright 2016 Freescale Semiconductor, Inc. All rights reserved.
   4  *   Copyright 2017 NXP
   5  *
   6  */
   7 /* System headers */
   8 #include <stdio.h>
   9 #include <inttypes.h>
  10 #include <unistd.h>
  11 #include <limits.h>
  12 #include <sched.h>
  13 #include <signal.h>
  14 #include <pthread.h>
  15 #include <sys/types.h>
  16 #include <sys/syscall.h>
  17
  18 #include <rte_byteorder.h>
  19 #include <rte_common.h>
  20 #include <rte_interrupts.h>
  21 #include <rte_log.h>
  22 #include <rte_debug.h>
  23 #include <rte_pci.h>
  24 #include <rte_atomic.h>
  25 #include <rte_branch_prediction.h>
  26 #include <rte_memory.h>
  27 #include <rte_tailq.h>
  28 #include <rte_eal.h>
  29 #include <rte_alarm.h>
  30 #include <rte_ether.h>
  31 #include <rte_ethdev.h>
  32 #include <rte_malloc.h>
  33 #include <rte_ring.h>
  34
  35 #include <rte_dpaa_bus.h>
  36 #include <rte_dpaa_logs.h>
  37 #include <dpaa_mempool.h>
  38
  39 #include <dpaa_ethdev.h>
  40 #include <dpaa_rxtx.h>
  41
  42 #include <fsl_usd.h>
  43 #include <fsl_qman.h>
  44 #include <fsl_bman.h>
  45 #include <fsl_fman.h>
  46
  47 /* Keep track of whether QMAN and BMAN have been globally initialized */
  48 static int is_global_init;
  49
  50 struct rte_dpaa_xstats_name_off {
  51         char name[RTE_ETH_XSTATS_NAME_SIZE];
  52         uint32_t offset;
  53 };
  54
  55 static const struct rte_dpaa_xstats_name_off dpaa_xstats_strings[] = {
  56         {"rx_align_err",
  57                 offsetof(struct dpaa_if_stats, raln)},
  58         {"rx_valid_pause",
  59                 offsetof(struct dpaa_if_stats, rxpf)},
  60         {"rx_fcs_err",
  61                 offsetof(struct dpaa_if_stats, rfcs)},
  62         {"rx_vlan_frame",
  63                 offsetof(struct dpaa_if_stats, rvlan)},
  64         {"rx_frame_err",
  65                 offsetof(struct dpaa_if_stats, rerr)},
  66         {"rx_drop_err",
  67                 offsetof(struct dpaa_if_stats, rdrp)},
  68         {"rx_undersized",
  69                 offsetof(struct dpaa_if_stats, rund)},
  70         {"rx_oversize_err",
  71                 offsetof(struct dpaa_if_stats, rovr)},
  72         {"rx_fragment_pkt",
  73                 offsetof(struct dpaa_if_stats, rfrg)},
  74         {"tx_valid_pause",
  75                 offsetof(struct dpaa_if_stats, txpf)},
  76         {"tx_fcs_err",
  77                 offsetof(struct dpaa_if_stats, terr)},
  78         {"tx_vlan_frame",
  79                 offsetof(struct dpaa_if_stats, tvlan)},
  80         {"rx_undersized",
  81                 offsetof(struct dpaa_if_stats, tund)},
  82 };
  83
  84 static int
  85 dpaa_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)
  86 {
  87         struct dpaa_if *dpaa_intf = dev->data->dev_private;
  88         uint32_t frame_size = mtu + ETHER_HDR_LEN + ETHER_CRC_LEN
  89                                 + VLAN_TAG_SIZE;
  90
  91         PMD_INIT_FUNC_TRACE();
  92
  93         if (mtu < ETHER_MIN_MTU || frame_size > DPAA_MAX_RX_PKT_LEN)
  94                 return -EINVAL;
  95         if (frame_size > ETHER_MAX_LEN)
  96                 dev->data->dev_conf.rxmode.jumbo_frame = 1;
  97         else
  98                 dev->data->dev_conf.rxmode.jumbo_frame = 0;
  99
 100         dev->data->dev_conf.rxmode.max_rx_pkt_len = frame_size;
 101
 102         fman_if_set_maxfrm(dpaa_intf->fif, frame_size);
 103
 104         return 0;
 105 }
 106
 107 static int
 108 dpaa_eth_dev_configure(struct rte_eth_dev *dev __rte_unused)
 109 {
 110         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 111
 112         PMD_INIT_FUNC_TRACE();
 113
 114         if (dev->data->dev_conf.rxmode.jumbo_frame == 1) {
 115                 if (dev->data->dev_conf.rxmode.max_rx_pkt_len <=
 116                     DPAA_MAX_RX_PKT_LEN) {
 117                         fman_if_set_maxfrm(dpaa_intf->fif,
 118                                 dev->data->dev_conf.rxmode.max_rx_pkt_len);
 119                         return 0;
 120                 } else {
 121                         return -1;
 122                 }
 123         }
 124         return 0;
 125 }
 126
 127 static const uint32_t *
 128 dpaa_supported_ptypes_get(struct rte_eth_dev *dev)
 129 {
 130         static const uint32_t ptypes[] = {
 131                 /*todo -= add more types */
 132                 RTE_PTYPE_L2_ETHER,
 133                 RTE_PTYPE_L3_IPV4,
 134                 RTE_PTYPE_L3_IPV4_EXT,
 135                 RTE_PTYPE_L3_IPV6,
 136                 RTE_PTYPE_L3_IPV6_EXT,
 137                 RTE_PTYPE_L4_TCP,
 138                 RTE_PTYPE_L4_UDP,
 139                 RTE_PTYPE_L4_SCTP
 140         };
 141
 142         PMD_INIT_FUNC_TRACE();
 143
 144         if (dev->rx_pkt_burst == dpaa_eth_queue_rx)
 145                 return ptypes;
 146         return NULL;
 147 }
 148
 149 static int dpaa_eth_dev_start(struct rte_eth_dev *dev)
 150 {
 151         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 152
 153         PMD_INIT_FUNC_TRACE();
 154
 155         /* Change tx callback to the real one */
 156         dev->tx_pkt_burst = dpaa_eth_queue_tx;
 157         fman_if_enable_rx(dpaa_intf->fif);
 158
 159         return 0;
 160 }
 161
 162 static void dpaa_eth_dev_stop(struct rte_eth_dev *dev)
 163 {
 164         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 165
 166         PMD_INIT_FUNC_TRACE();
 167
 168         fman_if_disable_rx(dpaa_intf->fif);
 169         dev->tx_pkt_burst = dpaa_eth_tx_drop_all;
 170 }
 171
 172 static void dpaa_eth_dev_close(struct rte_eth_dev *dev)
 173 {
 174         PMD_INIT_FUNC_TRACE();
 175
 176         dpaa_eth_dev_stop(dev);
 177 }
 178
 179 static int
 180 dpaa_fw_version_get(struct rte_eth_dev *dev __rte_unused,
 181                      char *fw_version,
 182                      size_t fw_size)
 183 {
 184         int ret;
 185         FILE *svr_file = NULL;
 186         unsigned int svr_ver = 0;
 187
 188         PMD_INIT_FUNC_TRACE();
 189
 190         svr_file = fopen(DPAA_SOC_ID_FILE, "r");
 191         if (!svr_file) {
 192                 DPAA_PMD_ERR("Unable to open SoC device");
 193                 return -ENOTSUP; /* Not supported on this infra */
 194         }
 195         if (fscanf(svr_file, "svr:%x", &svr_ver) > 0)
 196                 dpaa_svr_family = svr_ver & SVR_MASK;
 197         else
 198                 DPAA_PMD_ERR("Unable to read SoC device");
 199
 200         fclose(svr_file);
 201
 202         ret = snprintf(fw_version, fw_size, "SVR:%x-fman-v%x",
 203                        svr_ver, fman_ip_rev);
 204         ret += 1; /* add the size of '\0' */
 205
 206         if (fw_size < (uint32_t)ret)
 207                 return ret;
 208         else
 209                 return 0;
 210 }
 211
 212 static void dpaa_eth_dev_info(struct rte_eth_dev *dev,
 213                               struct rte_eth_dev_info *dev_info)
 214 {
 215         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 216
 217         PMD_INIT_FUNC_TRACE();
 218
 219         dev_info->max_rx_queues = dpaa_intf->nb_rx_queues;
 220         dev_info->max_tx_queues = dpaa_intf->nb_tx_queues;
 221         dev_info->min_rx_bufsize = DPAA_MIN_RX_BUF_SIZE;
 222         dev_info->max_rx_pktlen = DPAA_MAX_RX_PKT_LEN;
 223         dev_info->max_mac_addrs = DPAA_MAX_MAC_FILTER;
 224         dev_info->max_hash_mac_addrs = 0;
 225         dev_info->max_vfs = 0;
 226         dev_info->max_vmdq_pools = ETH_16_POOLS;
 227         dev_info->flow_type_rss_offloads = DPAA_RSS_OFFLOAD_ALL;
 228         dev_info->speed_capa = (ETH_LINK_SPEED_1G |
 229                                 ETH_LINK_SPEED_10G);
 230         dev_info->rx_offload_capa =
 231                 (DEV_RX_OFFLOAD_IPV4_CKSUM |
 232                 DEV_RX_OFFLOAD_UDP_CKSUM   |
 233                 DEV_RX_OFFLOAD_TCP_CKSUM);
 234         dev_info->tx_offload_capa =
 235                 (DEV_TX_OFFLOAD_IPV4_CKSUM  |
 236                 DEV_TX_OFFLOAD_UDP_CKSUM   |
 237                 DEV_TX_OFFLOAD_TCP_CKSUM);
 238 }
 239
 240 static int dpaa_eth_link_update(struct rte_eth_dev *dev,
 241                                 int wait_to_complete __rte_unused)
 242 {
 243         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 244         struct rte_eth_link *link = &dev->data->dev_link;
 245
 246         PMD_INIT_FUNC_TRACE();
 247
 248         if (dpaa_intf->fif->mac_type == fman_mac_1g)
 249                 link->link_speed = 1000;
 250         else if (dpaa_intf->fif->mac_type == fman_mac_10g)
 251                 link->link_speed = 10000;
 252         else
 253                 DPAA_PMD_ERR("invalid link_speed: %s, %d",
 254                              dpaa_intf->name, dpaa_intf->fif->mac_type);
 255
 256         link->link_status = dpaa_intf->valid;
 257         link->link_duplex = ETH_LINK_FULL_DUPLEX;
 258         link->link_autoneg = ETH_LINK_AUTONEG;
 259         return 0;
 260 }
 261
 262 static int dpaa_eth_stats_get(struct rte_eth_dev *dev,
 263                                struct rte_eth_stats *stats)
 264 {
 265         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 266
 267         PMD_INIT_FUNC_TRACE();
 268
 269         fman_if_stats_get(dpaa_intf->fif, stats);
 270         return 0;
 271 }
 272
 273 static void dpaa_eth_stats_reset(struct rte_eth_dev *dev)
 274 {
 275         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 276
 277         PMD_INIT_FUNC_TRACE();
 278
 279         fman_if_stats_reset(dpaa_intf->fif);
 280 }
 281
 282 static int
 283 dpaa_dev_xstats_get(struct rte_eth_dev *dev, struct rte_eth_xstat *xstats,
 284                     unsigned int n)
 285 {
 286         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 287         unsigned int i = 0, num = RTE_DIM(dpaa_xstats_strings);
 288         uint64_t values[sizeof(struct dpaa_if_stats) / 8];
 289
 290         if (xstats == NULL)
 291                 return 0;
 292
 293         if (n < num)
 294                 return num;
 295
 296         fman_if_stats_get_all(dpaa_intf->fif, values,
 297                               sizeof(struct dpaa_if_stats) / 8);
 298
 299         for (i = 0; i < num; i++) {
 300                 xstats[i].id = i;
 301                 xstats[i].value = values[dpaa_xstats_strings[i].offset / 8];
 302         }
 303         return i;
 304 }
 305
 306 static int
 307 dpaa_xstats_get_names(__rte_unused struct rte_eth_dev *dev,
 308                       struct rte_eth_xstat_name *xstats_names,
 309                       __rte_unused unsigned int limit)
 310 {
 311         unsigned int i, stat_cnt = RTE_DIM(dpaa_xstats_strings);
 312
 313         if (xstats_names != NULL)
 314                 for (i = 0; i < stat_cnt; i++)
 315                         snprintf(xstats_names[i].name,
 316                                  sizeof(xstats_names[i].name),
 317                                  "%s",
 318                                  dpaa_xstats_strings[i].name);
 319
 320         return stat_cnt;
 321 }
 322
 323 static int
 324 dpaa_xstats_get_by_id(struct rte_eth_dev *dev, const uint64_t *ids,
 325                       uint64_t *values, unsigned int n)
 326 {
 327         unsigned int i, stat_cnt = RTE_DIM(dpaa_xstats_strings);
 328         uint64_t values_copy[sizeof(struct dpaa_if_stats) / 8];
 329
 330         if (!ids) {
 331                 struct dpaa_if *dpaa_intf = dev->data->dev_private;
 332
 333                 if (n < stat_cnt)
 334                         return stat_cnt;
 335
 336                 if (!values)
 337                         return 0;
 338
 339                 fman_if_stats_get_all(dpaa_intf->fif, values_copy,
 340                                       sizeof(struct dpaa_if_stats));
 341
 342                 for (i = 0; i < stat_cnt; i++)
 343                         values[i] =
 344                                 values_copy[dpaa_xstats_strings[i].offset / 8];
 345
 346                 return stat_cnt;
 347         }
 348
 349         dpaa_xstats_get_by_id(dev, NULL, values_copy, stat_cnt);
 350
 351         for (i = 0; i < n; i++) {
 352                 if (ids[i] >= stat_cnt) {
 353                         DPAA_PMD_ERR("id value isn't valid");
 354                         return -1;
 355                 }
 356                 values[i] = values_copy[ids[i]];
 357         }
 358         return n;
 359 }
 360
 361 static int
 362 dpaa_xstats_get_names_by_id(
 363         struct rte_eth_dev *dev,
 364         struct rte_eth_xstat_name *xstats_names,
 365         const uint64_t *ids,
 366         unsigned int limit)
 367 {
 368         unsigned int i, stat_cnt = RTE_DIM(dpaa_xstats_strings);
 369         struct rte_eth_xstat_name xstats_names_copy[stat_cnt];
 370
 371         if (!ids)
 372                 return dpaa_xstats_get_names(dev, xstats_names, limit);
 373
 374         dpaa_xstats_get_names(dev, xstats_names_copy, limit);
 375
 376         for (i = 0; i < limit; i++) {
 377                 if (ids[i] >= stat_cnt) {
 378                         DPAA_PMD_ERR("id value isn't valid");
 379                         return -1;
 380                 }
 381                 strcpy(xstats_names[i].name, xstats_names_copy[ids[i]].name);
 382         }
 383         return limit;
 384 }
 385
 386 static void dpaa_eth_promiscuous_enable(struct rte_eth_dev *dev)
 387 {
 388         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 389
 390         PMD_INIT_FUNC_TRACE();
 391
 392         fman_if_promiscuous_enable(dpaa_intf->fif);
 393 }
 394
 395 static void dpaa_eth_promiscuous_disable(struct rte_eth_dev *dev)
 396 {
 397         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 398
 399         PMD_INIT_FUNC_TRACE();
 400
 401         fman_if_promiscuous_disable(dpaa_intf->fif);
 402 }
 403
 404 static void dpaa_eth_multicast_enable(struct rte_eth_dev *dev)
 405 {
 406         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 407
 408         PMD_INIT_FUNC_TRACE();
 409
 410         fman_if_set_mcast_filter_table(dpaa_intf->fif);
 411 }
 412
 413 static void dpaa_eth_multicast_disable(struct rte_eth_dev *dev)
 414 {
 415         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 416
 417         PMD_INIT_FUNC_TRACE();
 418
 419         fman_if_reset_mcast_filter_table(dpaa_intf->fif);
 420 }
 421
 422 static
 423 int dpaa_eth_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 424                             uint16_t nb_desc __rte_unused,
 425                             unsigned int socket_id __rte_unused,
 426                             const struct rte_eth_rxconf *rx_conf __rte_unused,
 427                             struct rte_mempool *mp)
 428 {
 429         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 430
 431         PMD_INIT_FUNC_TRACE();
 432
 433         DPAA_PMD_INFO("Rx queue setup for queue index: %d", queue_idx);
 434
 435         if (!dpaa_intf->bp_info || dpaa_intf->bp_info->mp != mp) {
 436                 struct fman_if_ic_params icp;
 437                 uint32_t fd_offset;
 438                 uint32_t bp_size;
 439
 440                 if (!mp->pool_data) {
 441                         DPAA_PMD_ERR("Not an offloaded buffer pool!");
 442                         return -1;
 443                 }
 444                 dpaa_intf->bp_info = DPAA_MEMPOOL_TO_POOL_INFO(mp);
 445
 446                 memset(&icp, 0, sizeof(icp));
 447                 /* set ICEOF for to the default value , which is 0*/
 448                 icp.iciof = DEFAULT_ICIOF;
 449                 icp.iceof = DEFAULT_RX_ICEOF;
 450                 icp.icsz = DEFAULT_ICSZ;
 451                 fman_if_set_ic_params(dpaa_intf->fif, &icp);
 452
 453                 fd_offset = RTE_PKTMBUF_HEADROOM + DPAA_HW_BUF_RESERVE;
 454                 fman_if_set_fdoff(dpaa_intf->fif, fd_offset);
 455
 456                 /* Buffer pool size should be equal to Dataroom Size*/
 457                 bp_size = rte_pktmbuf_data_room_size(mp);
 458                 fman_if_set_bp(dpaa_intf->fif, mp->size,
 459                                dpaa_intf->bp_info->bpid, bp_size);
 460                 dpaa_intf->valid = 1;
 461                 DPAA_PMD_INFO("if =%s - fd_offset = %d offset = %d",
 462                             dpaa_intf->name, fd_offset,
 463                         fman_if_get_fdoff(dpaa_intf->fif));
 464         }
 465         dev->data->rx_queues[queue_idx] = &dpaa_intf->rx_queues[queue_idx];
 466
 467         return 0;
 468 }
 469
 470 static
 471 void dpaa_eth_rx_queue_release(void *rxq __rte_unused)
 472 {
 473         PMD_INIT_FUNC_TRACE();
 474 }
 475
 476 static
 477 int dpaa_eth_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,
 478                             uint16_t nb_desc __rte_unused,
 479                 unsigned int socket_id __rte_unused,
 480                 const struct rte_eth_txconf *tx_conf __rte_unused)
 481 {
 482         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 483
 484         PMD_INIT_FUNC_TRACE();
 485
 486         DPAA_PMD_INFO("Tx queue setup for queue index: %d", queue_idx);
 487         dev->data->tx_queues[queue_idx] = &dpaa_intf->tx_queues[queue_idx];
 488         return 0;
 489 }
 490
 491 static void dpaa_eth_tx_queue_release(void *txq __rte_unused)
 492 {
 493         PMD_INIT_FUNC_TRACE();
 494 }
 495
 496 static int dpaa_link_down(struct rte_eth_dev *dev)
 497 {
 498         PMD_INIT_FUNC_TRACE();
 499
 500         dpaa_eth_dev_stop(dev);
 501         return 0;
 502 }
 503
 504 static int dpaa_link_up(struct rte_eth_dev *dev)
 505 {
 506         PMD_INIT_FUNC_TRACE();
 507
 508         dpaa_eth_dev_start(dev);
 509         return 0;
 510 }
 511
 512 static int
 513 dpaa_flow_ctrl_set(struct rte_eth_dev *dev,
 514                    struct rte_eth_fc_conf *fc_conf)
 515 {
 516         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 517         struct rte_eth_fc_conf *net_fc;
 518
 519         PMD_INIT_FUNC_TRACE();
 520
 521         if (!(dpaa_intf->fc_conf)) {
 522                 dpaa_intf->fc_conf = rte_zmalloc(NULL,
 523                         sizeof(struct rte_eth_fc_conf), MAX_CACHELINE);
 524                 if (!dpaa_intf->fc_conf) {
 525                         DPAA_PMD_ERR("unable to save flow control info");
 526                         return -ENOMEM;
 527                 }
 528         }
 529         net_fc = dpaa_intf->fc_conf;
 530
 531         if (fc_conf->high_water < fc_conf->low_water) {
 532                 DPAA_PMD_ERR("Incorrect Flow Control Configuration");
 533                 return -EINVAL;
 534         }
 535
 536         if (fc_conf->mode == RTE_FC_NONE) {
 537                 return 0;
 538         } else if (fc_conf->mode == RTE_FC_TX_PAUSE ||
 539                  fc_conf->mode == RTE_FC_FULL) {
 540                 fman_if_set_fc_threshold(dpaa_intf->fif, fc_conf->high_water,
 541                                          fc_conf->low_water,
 542                                 dpaa_intf->bp_info->bpid);
 543                 if (fc_conf->pause_time)
 544                         fman_if_set_fc_quanta(dpaa_intf->fif,
 545                                               fc_conf->pause_time);
 546         }
 547
 548         /* Save the information in dpaa device */
 549         net_fc->pause_time = fc_conf->pause_time;
 550         net_fc->high_water = fc_conf->high_water;
 551         net_fc->low_water = fc_conf->low_water;
 552         net_fc->send_xon = fc_conf->send_xon;
 553         net_fc->mac_ctrl_frame_fwd = fc_conf->mac_ctrl_frame_fwd;
 554         net_fc->mode = fc_conf->mode;
 555         net_fc->autoneg = fc_conf->autoneg;
 556
 557         return 0;
 558 }
 559
 560 static int
 561 dpaa_flow_ctrl_get(struct rte_eth_dev *dev,
 562                    struct rte_eth_fc_conf *fc_conf)
 563 {
 564         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 565         struct rte_eth_fc_conf *net_fc = dpaa_intf->fc_conf;
 566         int ret;
 567
 568         PMD_INIT_FUNC_TRACE();
 569
 570         if (net_fc) {
 571                 fc_conf->pause_time = net_fc->pause_time;
 572                 fc_conf->high_water = net_fc->high_water;
 573                 fc_conf->low_water = net_fc->low_water;
 574                 fc_conf->send_xon = net_fc->send_xon;
 575                 fc_conf->mac_ctrl_frame_fwd = net_fc->mac_ctrl_frame_fwd;
 576                 fc_conf->mode = net_fc->mode;
 577                 fc_conf->autoneg = net_fc->autoneg;
 578                 return 0;
 579         }
 580         ret = fman_if_get_fc_threshold(dpaa_intf->fif);
 581         if (ret) {
 582                 fc_conf->mode = RTE_FC_TX_PAUSE;
 583                 fc_conf->pause_time = fman_if_get_fc_quanta(dpaa_intf->fif);
 584         } else {
 585                 fc_conf->mode = RTE_FC_NONE;
 586         }
 587
 588         return 0;
 589 }
 590
 591 static int
 592 dpaa_dev_add_mac_addr(struct rte_eth_dev *dev,
 593                              struct ether_addr *addr,
 594                              uint32_t index,
 595                              __rte_unused uint32_t pool)
 596 {
 597         int ret;
 598         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 599
 600         PMD_INIT_FUNC_TRACE();
 601
 602         ret = fman_if_add_mac_addr(dpaa_intf->fif, addr->addr_bytes, index);
 603
 604         if (ret)
 605                 RTE_LOG(ERR, PMD, "error: Adding the MAC ADDR failed:"
 606                         " err = %d", ret);
 607         return 0;
 608 }
 609
 610 static void
 611 dpaa_dev_remove_mac_addr(struct rte_eth_dev *dev,
 612                           uint32_t index)
 613 {
 614         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 615
 616         PMD_INIT_FUNC_TRACE();
 617
 618         fman_if_clear_mac_addr(dpaa_intf->fif, index);
 619 }
 620
 621 static void
 622 dpaa_dev_set_mac_addr(struct rte_eth_dev *dev,
 623                        struct ether_addr *addr)
 624 {
 625         int ret;
 626         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 627
 628         PMD_INIT_FUNC_TRACE();
 629
 630         ret = fman_if_add_mac_addr(dpaa_intf->fif, addr->addr_bytes, 0);
 631         if (ret)
 632                 RTE_LOG(ERR, PMD, "error: Setting the MAC ADDR failed %d", ret);
 633 }
 634
 635 static struct eth_dev_ops dpaa_devops = {
 636         .dev_configure            = dpaa_eth_dev_configure,
 637         .dev_start                = dpaa_eth_dev_start,
 638         .dev_stop                 = dpaa_eth_dev_stop,
 639         .dev_close                = dpaa_eth_dev_close,
 640         .dev_infos_get            = dpaa_eth_dev_info,
 641         .dev_supported_ptypes_get = dpaa_supported_ptypes_get,
 642
 643         .rx_queue_setup           = dpaa_eth_rx_queue_setup,
 644         .tx_queue_setup           = dpaa_eth_tx_queue_setup,
 645         .rx_queue_release         = dpaa_eth_rx_queue_release,
 646         .tx_queue_release         = dpaa_eth_tx_queue_release,
 647
 648         .flow_ctrl_get            = dpaa_flow_ctrl_get,
 649         .flow_ctrl_set            = dpaa_flow_ctrl_set,
 650
 651         .link_update              = dpaa_eth_link_update,
 652         .stats_get                = dpaa_eth_stats_get,
 653         .xstats_get               = dpaa_dev_xstats_get,
 654         .xstats_get_by_id         = dpaa_xstats_get_by_id,
 655         .xstats_get_names_by_id   = dpaa_xstats_get_names_by_id,
 656         .xstats_get_names         = dpaa_xstats_get_names,
 657         .xstats_reset             = dpaa_eth_stats_reset,
 658         .stats_reset              = dpaa_eth_stats_reset,
 659         .promiscuous_enable       = dpaa_eth_promiscuous_enable,
 660         .promiscuous_disable      = dpaa_eth_promiscuous_disable,
 661         .allmulticast_enable      = dpaa_eth_multicast_enable,
 662         .allmulticast_disable     = dpaa_eth_multicast_disable,
 663         .mtu_set                  = dpaa_mtu_set,
 664         .dev_set_link_down        = dpaa_link_down,
 665         .dev_set_link_up          = dpaa_link_up,
 666         .mac_addr_add             = dpaa_dev_add_mac_addr,
 667         .mac_addr_remove          = dpaa_dev_remove_mac_addr,
 668         .mac_addr_set             = dpaa_dev_set_mac_addr,
 669
 670         .fw_version_get           = dpaa_fw_version_get,
 671 };
 672
 673 static int dpaa_fc_set_default(struct dpaa_if *dpaa_intf)
 674 {
 675         struct rte_eth_fc_conf *fc_conf;
 676         int ret;
 677
 678         PMD_INIT_FUNC_TRACE();
 679
 680         if (!(dpaa_intf->fc_conf)) {
 681                 dpaa_intf->fc_conf = rte_zmalloc(NULL,
 682                         sizeof(struct rte_eth_fc_conf), MAX_CACHELINE);
 683                 if (!dpaa_intf->fc_conf) {
 684                         DPAA_PMD_ERR("unable to save flow control info");
 685                         return -ENOMEM;
 686                 }
 687         }
 688         fc_conf = dpaa_intf->fc_conf;
 689         ret = fman_if_get_fc_threshold(dpaa_intf->fif);
 690         if (ret) {
 691                 fc_conf->mode = RTE_FC_TX_PAUSE;
 692                 fc_conf->pause_time = fman_if_get_fc_quanta(dpaa_intf->fif);
 693         } else {
 694                 fc_conf->mode = RTE_FC_NONE;
 695         }
 696
 697         return 0;
 698 }
 699
 700 /* Initialise an Rx FQ */
 701 static int dpaa_rx_queue_init(struct qman_fq *fq,
 702                               uint32_t fqid)
 703 {
 704         struct qm_mcc_initfq opts = {0};
 705         int ret;
 706
 707         PMD_INIT_FUNC_TRACE();
 708
 709         ret = qman_reserve_fqid(fqid);
 710         if (ret) {
 711                 DPAA_PMD_ERR("reserve rx fqid %d failed with ret: %d",
 712                              fqid, ret);
 713                 return -EINVAL;
 714         }
 715
 716         DPAA_PMD_DEBUG("creating rx fq %p, fqid %d", fq, fqid);
 717         ret = qman_create_fq(fqid, QMAN_FQ_FLAG_NO_ENQUEUE, fq);
 718         if (ret) {
 719                 DPAA_PMD_ERR("create rx fqid %d failed with ret: %d",
 720                         fqid, ret);
 721                 return ret;
 722         }
 723
 724         opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL |
 725                        QM_INITFQ_WE_CONTEXTA;
 726
 727         opts.fqd.dest.wq = DPAA_IF_RX_PRIORITY;
 728         opts.fqd.fq_ctrl = QM_FQCTRL_AVOIDBLOCK | QM_FQCTRL_CTXASTASHING |
 729                            QM_FQCTRL_PREFERINCACHE;
 730         opts.fqd.context_a.stashing.exclusive = 0;
 731         opts.fqd.context_a.stashing.annotation_cl = DPAA_IF_RX_ANNOTATION_STASH;
 732         opts.fqd.context_a.stashing.data_cl = DPAA_IF_RX_DATA_STASH;
 733         opts.fqd.context_a.stashing.context_cl = DPAA_IF_RX_CONTEXT_STASH;
 734
 735         /*Enable tail drop */
 736         opts.we_mask = opts.we_mask | QM_INITFQ_WE_TDTHRESH;
 737         opts.fqd.fq_ctrl = opts.fqd.fq_ctrl | QM_FQCTRL_TDE;
 738         qm_fqd_taildrop_set(&opts.fqd.td, CONG_THRESHOLD_RX_Q, 1);
 739
 740         ret = qman_init_fq(fq, 0, &opts);
 741         if (ret)
 742                 DPAA_PMD_ERR("init rx fqid %d failed with ret: %d", fqid, ret);
 743         return ret;
 744 }
 745
 746 /* Initialise a Tx FQ */
 747 static int dpaa_tx_queue_init(struct qman_fq *fq,
 748                               struct fman_if *fman_intf)
 749 {
 750         struct qm_mcc_initfq opts = {0};
 751         int ret;
 752
 753         PMD_INIT_FUNC_TRACE();
 754
 755         ret = qman_create_fq(0, QMAN_FQ_FLAG_DYNAMIC_FQID |
 756                              QMAN_FQ_FLAG_TO_DCPORTAL, fq);
 757         if (ret) {
 758                 DPAA_PMD_ERR("create tx fq failed with ret: %d", ret);
 759                 return ret;
 760         }
 761         opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL |
 762                        QM_INITFQ_WE_CONTEXTB | QM_INITFQ_WE_CONTEXTA;
 763         opts.fqd.dest.channel = fman_intf->tx_channel_id;
 764         opts.fqd.dest.wq = DPAA_IF_TX_PRIORITY;
 765         opts.fqd.fq_ctrl = QM_FQCTRL_PREFERINCACHE;
 766         opts.fqd.context_b = 0;
 767         /* no tx-confirmation */
 768         opts.fqd.context_a.hi = 0x80000000 | fman_dealloc_bufs_mask_hi;
 769         opts.fqd.context_a.lo = 0 | fman_dealloc_bufs_mask_lo;
 770         DPAA_PMD_DEBUG("init tx fq %p, fqid %d", fq, fq->fqid);
 771         ret = qman_init_fq(fq, QMAN_INITFQ_FLAG_SCHED, &opts);
 772         if (ret)
 773                 DPAA_PMD_ERR("init tx fqid %d failed %d", fq->fqid, ret);
 774         return ret;
 775 }
 776
 777 #ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER
 778 /* Initialise a DEBUG FQ ([rt]x_error, rx_default). */
 779 static int dpaa_debug_queue_init(struct qman_fq *fq, uint32_t fqid)
 780 {
 781         struct qm_mcc_initfq opts = {0};
 782         int ret;
 783
 784         PMD_INIT_FUNC_TRACE();
 785
 786         ret = qman_reserve_fqid(fqid);
 787         if (ret) {
 788                 DPAA_PMD_ERR("Reserve debug fqid %d failed with ret: %d",
 789                         fqid, ret);
 790                 return -EINVAL;
 791         }
 792         /* "map" this Rx FQ to one of the interfaces Tx FQID */
 793         DPAA_PMD_DEBUG("Creating debug fq %p, fqid %d", fq, fqid);
 794         ret = qman_create_fq(fqid, QMAN_FQ_FLAG_NO_ENQUEUE, fq);
 795         if (ret) {
 796                 DPAA_PMD_ERR("create debug fqid %d failed with ret: %d",
 797                         fqid, ret);
 798                 return ret;
 799         }
 800         opts.we_mask = QM_INITFQ_WE_DESTWQ | QM_INITFQ_WE_FQCTRL;
 801         opts.fqd.dest.wq = DPAA_IF_DEBUG_PRIORITY;
 802         ret = qman_init_fq(fq, 0, &opts);
 803         if (ret)
 804                 DPAA_PMD_ERR("init debug fqid %d failed with ret: %d",
 805                             fqid, ret);
 806         return ret;
 807 }
 808 #endif
 809
 810 /* Initialise a network interface */
 811 static int
 812 dpaa_dev_init(struct rte_eth_dev *eth_dev)
 813 {
 814         int num_cores, num_rx_fqs, fqid;
 815         int loop, ret = 0;
 816         int dev_id;
 817         struct rte_dpaa_device *dpaa_device;
 818         struct dpaa_if *dpaa_intf;
 819         struct fm_eth_port_cfg *cfg;
 820         struct fman_if *fman_intf;
 821         struct fman_if_bpool *bp, *tmp_bp;
 822
 823         PMD_INIT_FUNC_TRACE();
 824
 825         /* For secondary processes, the primary has done all the work */
 826         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 827                 return 0;
 828
 829         dpaa_device = DEV_TO_DPAA_DEVICE(eth_dev->device);
 830         dev_id = dpaa_device->id.dev_id;
 831         dpaa_intf = eth_dev->data->dev_private;
 832         cfg = &dpaa_netcfg->port_cfg[dev_id];
 833         fman_intf = cfg->fman_if;
 834
 835         dpaa_intf->name = dpaa_device->name;
 836
 837         /* save fman_if & cfg in the interface struture */
 838         dpaa_intf->fif = fman_intf;
 839         dpaa_intf->ifid = dev_id;
 840         dpaa_intf->cfg = cfg;
 841
 842         /* Initialize Rx FQ's */
 843         if (getenv("DPAA_NUM_RX_QUEUES"))
 844                 num_rx_fqs = atoi(getenv("DPAA_NUM_RX_QUEUES"));
 845         else
 846                 num_rx_fqs = DPAA_DEFAULT_NUM_PCD_QUEUES;
 847
 848         /* Each device can not have more than DPAA_PCD_FQID_MULTIPLIER RX
 849          * queues.
 850          */
 851         if (num_rx_fqs <= 0 || num_rx_fqs > DPAA_PCD_FQID_MULTIPLIER) {
 852                 DPAA_PMD_ERR("Invalid number of RX queues\n");
 853                 return -EINVAL;
 854         }
 855
 856         dpaa_intf->rx_queues = rte_zmalloc(NULL,
 857                 sizeof(struct qman_fq) * num_rx_fqs, MAX_CACHELINE);
 858         for (loop = 0; loop < num_rx_fqs; loop++) {
 859                 fqid = DPAA_PCD_FQID_START + dpaa_intf->ifid *
 860                         DPAA_PCD_FQID_MULTIPLIER + loop;
 861                 ret = dpaa_rx_queue_init(&dpaa_intf->rx_queues[loop], fqid);
 862                 if (ret)
 863                         return ret;
 864                 dpaa_intf->rx_queues[loop].dpaa_intf = dpaa_intf;
 865         }
 866         dpaa_intf->nb_rx_queues = num_rx_fqs;
 867
 868         /* Initialise Tx FQs. Have as many Tx FQ's as number of cores */
 869         num_cores = rte_lcore_count();
 870         dpaa_intf->tx_queues = rte_zmalloc(NULL, sizeof(struct qman_fq) *
 871                 num_cores, MAX_CACHELINE);
 872         if (!dpaa_intf->tx_queues)
 873                 return -ENOMEM;
 874
 875         for (loop = 0; loop < num_cores; loop++) {
 876                 ret = dpaa_tx_queue_init(&dpaa_intf->tx_queues[loop],
 877                                          fman_intf);
 878                 if (ret)
 879                         return ret;
 880                 dpaa_intf->tx_queues[loop].dpaa_intf = dpaa_intf;
 881         }
 882         dpaa_intf->nb_tx_queues = num_cores;
 883
 884 #ifdef RTE_LIBRTE_DPAA_DEBUG_DRIVER
 885         dpaa_debug_queue_init(&dpaa_intf->debug_queues[
 886                 DPAA_DEBUG_FQ_RX_ERROR], fman_intf->fqid_rx_err);
 887         dpaa_intf->debug_queues[DPAA_DEBUG_FQ_RX_ERROR].dpaa_intf = dpaa_intf;
 888         dpaa_debug_queue_init(&dpaa_intf->debug_queues[
 889                 DPAA_DEBUG_FQ_TX_ERROR], fman_intf->fqid_tx_err);
 890         dpaa_intf->debug_queues[DPAA_DEBUG_FQ_TX_ERROR].dpaa_intf = dpaa_intf;
 891 #endif
 892
 893         DPAA_PMD_DEBUG("All frame queues created");
 894
 895         /* Get the initial configuration for flow control */
 896         dpaa_fc_set_default(dpaa_intf);
 897
 898         /* reset bpool list, initialize bpool dynamically */
 899         list_for_each_entry_safe(bp, tmp_bp, &cfg->fman_if->bpool_list, node) {
 900                 list_del(&bp->node);
 901                 free(bp);
 902         }
 903
 904         /* Populate ethdev structure */
 905         eth_dev->dev_ops = &dpaa_devops;
 906         eth_dev->rx_pkt_burst = dpaa_eth_queue_rx;
 907         eth_dev->tx_pkt_burst = dpaa_eth_tx_drop_all;
 908
 909         /* Allocate memory for storing MAC addresses */
 910         eth_dev->data->mac_addrs = rte_zmalloc("mac_addr",
 911                 ETHER_ADDR_LEN * DPAA_MAX_MAC_FILTER, 0);
 912         if (eth_dev->data->mac_addrs == NULL) {
 913                 DPAA_PMD_ERR("Failed to allocate %d bytes needed to "
 914                                                 "store MAC addresses",
 915                                 ETHER_ADDR_LEN * DPAA_MAX_MAC_FILTER);
 916                 rte_free(dpaa_intf->rx_queues);
 917                 rte_free(dpaa_intf->tx_queues);
 918                 dpaa_intf->rx_queues = NULL;
 919                 dpaa_intf->tx_queues = NULL;
 920                 dpaa_intf->nb_rx_queues = 0;
 921                 dpaa_intf->nb_tx_queues = 0;
 922                 return -ENOMEM;
 923         }
 924
 925         /* copy the primary mac address */
 926         ether_addr_copy(&fman_intf->mac_addr, &eth_dev->data->mac_addrs[0]);
 927
 928         RTE_LOG(INFO, PMD, "net: dpaa: %s: %02x:%02x:%02x:%02x:%02x:%02x\n",
 929                 dpaa_device->name,
 930                 fman_intf->mac_addr.addr_bytes[0],
 931                 fman_intf->mac_addr.addr_bytes[1],
 932                 fman_intf->mac_addr.addr_bytes[2],
 933                 fman_intf->mac_addr.addr_bytes[3],
 934                 fman_intf->mac_addr.addr_bytes[4],
 935                 fman_intf->mac_addr.addr_bytes[5]);
 936
 937         /* Disable RX mode */
 938         fman_if_discard_rx_errors(fman_intf);
 939         fman_if_disable_rx(fman_intf);
 940         /* Disable promiscuous mode */
 941         fman_if_promiscuous_disable(fman_intf);
 942         /* Disable multicast */
 943         fman_if_reset_mcast_filter_table(fman_intf);
 944         /* Reset interface statistics */
 945         fman_if_stats_reset(fman_intf);
 946
 947         return 0;
 948 }
 949
 950 static int
 951 dpaa_dev_uninit(struct rte_eth_dev *dev)
 952 {
 953         struct dpaa_if *dpaa_intf = dev->data->dev_private;
 954
 955         PMD_INIT_FUNC_TRACE();
 956
 957         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 958                 return -EPERM;
 959
 960         if (!dpaa_intf) {
 961                 DPAA_PMD_WARN("Already closed or not started");
 962                 return -1;
 963         }
 964
 965         dpaa_eth_dev_close(dev);
 966
 967         /* release configuration memory */
 968         if (dpaa_intf->fc_conf)
 969                 rte_free(dpaa_intf->fc_conf);
 970
 971         rte_free(dpaa_intf->rx_queues);
 972         dpaa_intf->rx_queues = NULL;
 973
 974         rte_free(dpaa_intf->tx_queues);
 975         dpaa_intf->tx_queues = NULL;
 976
 977         /* free memory for storing MAC addresses */
 978         rte_free(dev->data->mac_addrs);
 979         dev->data->mac_addrs = NULL;
 980
 981         dev->dev_ops = NULL;
 982         dev->rx_pkt_burst = NULL;
 983         dev->tx_pkt_burst = NULL;
 984
 985         return 0;
 986 }
 987
 988 static int
 989 rte_dpaa_probe(struct rte_dpaa_driver *dpaa_drv,
 990                struct rte_dpaa_device *dpaa_dev)
 991 {
 992         int diag;
 993         int ret;
 994         struct rte_eth_dev *eth_dev;
 995
 996         PMD_INIT_FUNC_TRACE();
 997
 998         /* In case of secondary process, the device is already configured
 999          * and no further action is required, except portal initialization
1000          * and verifying secondary attachment to port name.
1001          */
1002         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
1003                 eth_dev = rte_eth_dev_attach_secondary(dpaa_dev->name);
1004                 if (!eth_dev)
1005                         return -ENOMEM;
1006                 return 0;
1007         }
1008
1009         if (!is_global_init) {
1010                 /* One time load of Qman/Bman drivers */
1011                 ret = qman_global_init();
1012                 if (ret) {
1013                         DPAA_PMD_ERR("QMAN initialization failed: %d",
1014                                      ret);
1015                         return ret;
1016                 }
1017                 ret = bman_global_init();
1018                 if (ret) {
1019                         DPAA_PMD_ERR("BMAN initialization failed: %d",
1020                                      ret);
1021                         return ret;
1022                 }
1023
1024                 is_global_init = 1;
1025         }
1026
1027         ret = rte_dpaa_portal_init((void *)1);
1028         if (ret) {
1029                 DPAA_PMD_ERR("Unable to initialize portal");
1030                 return ret;
1031         }
1032
1033         eth_dev = rte_eth_dev_allocate(dpaa_dev->name);
1034         if (eth_dev == NULL)
1035                 return -ENOMEM;
1036
1037         eth_dev->data->dev_private = rte_zmalloc(
1038                                         "ethdev private structure",
1039                                         sizeof(struct dpaa_if),
1040                                         RTE_CACHE_LINE_SIZE);
1041         if (!eth_dev->data->dev_private) {
1042                 DPAA_PMD_ERR("Cannot allocate memzone for port data");
1043                 rte_eth_dev_release_port(eth_dev);
1044                 return -ENOMEM;
1045         }
1046
1047         eth_dev->device = &dpaa_dev->device;
1048         eth_dev->device->driver = &dpaa_drv->driver;
1049         dpaa_dev->eth_dev = eth_dev;
1050
1051         /* Invoke PMD device initialization function */
1052         diag = dpaa_dev_init(eth_dev);
1053         if (diag == 0)
1054                 return 0;
1055
1056         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
1057                 rte_free(eth_dev->data->dev_private);
1058
1059         rte_eth_dev_release_port(eth_dev);
1060         return diag;
1061 }
1062
1063 static int
1064 rte_dpaa_remove(struct rte_dpaa_device *dpaa_dev)
1065 {
1066         struct rte_eth_dev *eth_dev;
1067
1068         PMD_INIT_FUNC_TRACE();
1069
1070         eth_dev = dpaa_dev->eth_dev;
1071         dpaa_dev_uninit(eth_dev);
1072
1073         if (rte_eal_process_type() == RTE_PROC_PRIMARY)
1074                 rte_free(eth_dev->data->dev_private);
1075
1076         rte_eth_dev_release_port(eth_dev);
1077
1078         return 0;
1079 }
1080
1081 static struct rte_dpaa_driver rte_dpaa_pmd = {
1082         .drv_type = FSL_DPAA_ETH,
1083         .probe = rte_dpaa_probe,
1084         .remove = rte_dpaa_remove,
1085 };
1086
1087 RTE_PMD_REGISTER_DPAA(net_dpaa, rte_dpaa_pmd);