drivers/net/ark/ark_ethdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2015-2018 Atomic Rules LLC
   3  */
   4
   5 #include <unistd.h>
   6 #include <sys/stat.h>
   7 #include <dlfcn.h>
   8
   9 #include <rte_bus_pci.h>
  10 #include <ethdev_pci.h>
  11 #include <rte_kvargs.h>
  12
  13 #include "rte_pmd_ark.h"
  14 #include "ark_global.h"
  15 #include "ark_logs.h"
  16 #include "ark_ethdev_tx.h"
  17 #include "ark_ethdev_rx.h"
  18 #include "ark_mpu.h"
  19 #include "ark_ddm.h"
  20 #include "ark_udm.h"
  21 #include "ark_rqp.h"
  22 #include "ark_pktdir.h"
  23 #include "ark_pktgen.h"
  24 #include "ark_pktchkr.h"
  25
  26 /*  Internal prototypes */
  27 static int eth_ark_check_args(struct ark_adapter *ark, const char *params);
  28 static int eth_ark_dev_init(struct rte_eth_dev *dev);
  29 static int ark_config_device(struct rte_eth_dev *dev);
  30 static int eth_ark_dev_uninit(struct rte_eth_dev *eth_dev);
  31 static int eth_ark_dev_configure(struct rte_eth_dev *dev);
  32 static int eth_ark_dev_start(struct rte_eth_dev *dev);
  33 static int eth_ark_dev_stop(struct rte_eth_dev *dev);
  34 static int eth_ark_dev_close(struct rte_eth_dev *dev);
  35 static int eth_ark_dev_info_get(struct rte_eth_dev *dev,
  36                                 struct rte_eth_dev_info *dev_info);
  37 static int eth_ark_dev_link_update(struct rte_eth_dev *dev,
  38                                    int wait_to_complete);
  39 static int eth_ark_dev_set_link_up(struct rte_eth_dev *dev);
  40 static int eth_ark_dev_set_link_down(struct rte_eth_dev *dev);
  41 static int eth_ark_dev_stats_get(struct rte_eth_dev *dev,
  42                                   struct rte_eth_stats *stats);
  43 static int eth_ark_dev_stats_reset(struct rte_eth_dev *dev);
  44 static int eth_ark_set_default_mac_addr(struct rte_eth_dev *dev,
  45                                          struct rte_ether_addr *mac_addr);
  46 static int eth_ark_macaddr_add(struct rte_eth_dev *dev,
  47                                struct rte_ether_addr *mac_addr,
  48                                uint32_t index,
  49                                uint32_t pool);
  50 static void eth_ark_macaddr_remove(struct rte_eth_dev *dev,
  51                                    uint32_t index);
  52 static int  eth_ark_set_mtu(struct rte_eth_dev *dev, uint16_t size);
  53
  54 /*
  55  * The packet generator is a functional block used to generate packet
  56  * patterns for testing.  It is not intended for nominal use.
  57  */
  58 #define ARK_PKTGEN_ARG "Pkt_gen"
  59
  60 /*
  61  * The packet checker is a functional block used to verify packet
  62  * patterns for testing.  It is not intended for nominal use.
  63  */
  64 #define ARK_PKTCHKR_ARG "Pkt_chkr"
  65
  66 /*
  67  * The packet director is used to select the internal ingress and
  68  * egress packets paths during testing.  It is not intended for
  69  * nominal use.
  70  */
  71 #define ARK_PKTDIR_ARG "Pkt_dir"
  72
  73 /* Devinfo configurations */
  74 #define ARK_RX_MAX_QUEUE (4096 * 4)
  75 #define ARK_RX_MIN_QUEUE (512)
  76 #define ARK_RX_MAX_PKT_LEN ((16 * 1024) - 128)
  77 #define ARK_RX_MIN_BUFSIZE (1024)
  78
  79 #define ARK_TX_MAX_QUEUE (4096 * 4)
  80 #define ARK_TX_MIN_QUEUE (256)
  81
  82 uint64_t ark_timestamp_rx_dynflag;
  83 int ark_timestamp_dynfield_offset = -1;
  84
  85 int rte_pmd_ark_rx_userdata_dynfield_offset = -1;
  86 int rte_pmd_ark_tx_userdata_dynfield_offset = -1;
  87
  88 static const char * const valid_arguments[] = {
  89         ARK_PKTGEN_ARG,
  90         ARK_PKTCHKR_ARG,
  91         ARK_PKTDIR_ARG,
  92         NULL
  93 };
  94
  95 static const struct rte_pci_id pci_id_ark_map[] = {
  96         {RTE_PCI_DEVICE(0x1d6c, 0x100d)},
  97         {RTE_PCI_DEVICE(0x1d6c, 0x100e)},
  98         {RTE_PCI_DEVICE(0x1d6c, 0x100f)},
  99         {RTE_PCI_DEVICE(0x1d6c, 0x1010)},
 100         {RTE_PCI_DEVICE(0x1d6c, 0x1017)},
 101         {RTE_PCI_DEVICE(0x1d6c, 0x1018)},
 102         {RTE_PCI_DEVICE(0x1d6c, 0x1019)},
 103         {.vendor_id = 0, /* sentinel */ },
 104 };
 105
 106 static int
 107 eth_ark_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 108                 struct rte_pci_device *pci_dev)
 109 {
 110         struct rte_eth_dev *eth_dev;
 111         int ret;
 112
 113         eth_dev = rte_eth_dev_pci_allocate(pci_dev, sizeof(struct ark_adapter));
 114
 115         if (eth_dev == NULL)
 116                 return -ENOMEM;
 117
 118         ret = eth_ark_dev_init(eth_dev);
 119         if (ret)
 120                 rte_eth_dev_release_port(eth_dev);
 121
 122         return ret;
 123 }
 124
 125 static int
 126 eth_ark_pci_remove(struct rte_pci_device *pci_dev)
 127 {
 128         return rte_eth_dev_pci_generic_remove(pci_dev, eth_ark_dev_uninit);
 129 }
 130
 131 static struct rte_pci_driver rte_ark_pmd = {
 132         .id_table = pci_id_ark_map,
 133         .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
 134         .probe = eth_ark_pci_probe,
 135         .remove = eth_ark_pci_remove,
 136 };
 137
 138 static const struct eth_dev_ops ark_eth_dev_ops = {
 139         .dev_configure = eth_ark_dev_configure,
 140         .dev_start = eth_ark_dev_start,
 141         .dev_stop = eth_ark_dev_stop,
 142         .dev_close = eth_ark_dev_close,
 143
 144         .dev_infos_get = eth_ark_dev_info_get,
 145
 146         .rx_queue_setup = eth_ark_dev_rx_queue_setup,
 147         .tx_queue_setup = eth_ark_tx_queue_setup,
 148
 149         .link_update = eth_ark_dev_link_update,
 150         .dev_set_link_up = eth_ark_dev_set_link_up,
 151         .dev_set_link_down = eth_ark_dev_set_link_down,
 152
 153         .rx_queue_start = eth_ark_rx_start_queue,
 154         .rx_queue_stop = eth_ark_rx_stop_queue,
 155
 156         .tx_queue_start = eth_ark_tx_queue_start,
 157         .tx_queue_stop = eth_ark_tx_queue_stop,
 158
 159         .stats_get = eth_ark_dev_stats_get,
 160         .stats_reset = eth_ark_dev_stats_reset,
 161
 162         .mac_addr_add = eth_ark_macaddr_add,
 163         .mac_addr_remove = eth_ark_macaddr_remove,
 164         .mac_addr_set = eth_ark_set_default_mac_addr,
 165
 166         .mtu_set = eth_ark_set_mtu,
 167 };
 168
 169 static int
 170 check_for_ext(struct ark_adapter *ark)
 171 {
 172         int found = 0;
 173
 174         /* Get the env */
 175         const char *dllpath = getenv("ARK_EXT_PATH");
 176
 177         if (dllpath == NULL) {
 178                 ARK_PMD_LOG(DEBUG, "EXT NO dll path specified\n");
 179                 return 0;
 180         }
 181         ARK_PMD_LOG(NOTICE, "EXT found dll path at %s\n", dllpath);
 182
 183         /* Open and load the .so */
 184         ark->d_handle = dlopen(dllpath, RTLD_LOCAL | RTLD_LAZY);
 185         if (ark->d_handle == NULL) {
 186                 ARK_PMD_LOG(ERR, "Could not load user extension %s\n",
 187                             dllpath);
 188                 return -1;
 189         }
 190         ARK_PMD_LOG(DEBUG, "SUCCESS: loaded user extension %s\n",
 191                             dllpath);
 192
 193         /* Get the entry points */
 194         ark->user_ext.dev_init =
 195                 (void *(*)(struct rte_eth_dev *, void *, int))
 196                 dlsym(ark->d_handle, "dev_init");
 197         ARK_PMD_LOG(DEBUG, "device ext init pointer = %p\n",
 198                       ark->user_ext.dev_init);
 199         ark->user_ext.dev_get_port_count =
 200                 (int (*)(struct rte_eth_dev *, void *))
 201                 dlsym(ark->d_handle, "dev_get_port_count");
 202         ark->user_ext.dev_uninit =
 203                 (void (*)(struct rte_eth_dev *, void *))
 204                 dlsym(ark->d_handle, "dev_uninit");
 205         ark->user_ext.dev_configure =
 206                 (int (*)(struct rte_eth_dev *, void *))
 207                 dlsym(ark->d_handle, "dev_configure");
 208         ark->user_ext.dev_start =
 209                 (int (*)(struct rte_eth_dev *, void *))
 210                 dlsym(ark->d_handle, "dev_start");
 211         ark->user_ext.dev_stop =
 212                 (void (*)(struct rte_eth_dev *, void *))
 213                 dlsym(ark->d_handle, "dev_stop");
 214         ark->user_ext.dev_close =
 215                 (void (*)(struct rte_eth_dev *, void *))
 216                 dlsym(ark->d_handle, "dev_close");
 217         ark->user_ext.link_update =
 218                 (int (*)(struct rte_eth_dev *, int, void *))
 219                 dlsym(ark->d_handle, "link_update");
 220         ark->user_ext.dev_set_link_up =
 221                 (int (*)(struct rte_eth_dev *, void *))
 222                 dlsym(ark->d_handle, "dev_set_link_up");
 223         ark->user_ext.dev_set_link_down =
 224                 (int (*)(struct rte_eth_dev *, void *))
 225                 dlsym(ark->d_handle, "dev_set_link_down");
 226         ark->user_ext.stats_get =
 227                 (int (*)(struct rte_eth_dev *, struct rte_eth_stats *,
 228                           void *))
 229                 dlsym(ark->d_handle, "stats_get");
 230         ark->user_ext.stats_reset =
 231                 (void (*)(struct rte_eth_dev *, void *))
 232                 dlsym(ark->d_handle, "stats_reset");
 233         ark->user_ext.mac_addr_add =
 234                 (void (*)(struct rte_eth_dev *, struct rte_ether_addr *,
 235                         uint32_t, uint32_t, void *))
 236                 dlsym(ark->d_handle, "mac_addr_add");
 237         ark->user_ext.mac_addr_remove =
 238                 (void (*)(struct rte_eth_dev *, uint32_t, void *))
 239                 dlsym(ark->d_handle, "mac_addr_remove");
 240         ark->user_ext.mac_addr_set =
 241                 (void (*)(struct rte_eth_dev *, struct rte_ether_addr *,
 242                           void *))
 243                 dlsym(ark->d_handle, "mac_addr_set");
 244         ark->user_ext.set_mtu =
 245                 (int (*)(struct rte_eth_dev *, uint16_t,
 246                           void *))
 247                 dlsym(ark->d_handle, "set_mtu");
 248
 249         return found;
 250 }
 251
 252 static int
 253 eth_ark_dev_init(struct rte_eth_dev *dev)
 254 {
 255         struct ark_adapter *ark = dev->data->dev_private;
 256         struct rte_pci_device *pci_dev;
 257         int ret;
 258         int port_count = 1;
 259         int p;
 260         static const struct rte_mbuf_dynfield ark_tx_userdata_dynfield_desc = {
 261                 .name = RTE_PMD_ARK_TX_USERDATA_DYNFIELD_NAME,
 262                 .size = sizeof(rte_pmd_ark_tx_userdata_t),
 263                 .align = __alignof__(rte_pmd_ark_tx_userdata_t),
 264         };
 265         static const struct rte_mbuf_dynfield ark_rx_userdata_dynfield_desc = {
 266                 .name = RTE_PMD_ARK_RX_USERDATA_DYNFIELD_NAME,
 267                 .size = sizeof(rte_pmd_ark_rx_userdata_t),
 268                 .align = __alignof__(rte_pmd_ark_rx_userdata_t),
 269         };
 270
 271         ark->eth_dev = dev;
 272
 273         ARK_PMD_LOG(DEBUG, "\n");
 274
 275         /* Check to see if there is an extension that we need to load */
 276         ret = check_for_ext(ark);
 277         if (ret)
 278                 return ret;
 279
 280         /* Extra mbuf fields for user data */
 281         if (RTE_PMD_ARK_TX_USERDATA_ENABLE) {
 282                 rte_pmd_ark_tx_userdata_dynfield_offset =
 283                     rte_mbuf_dynfield_register(&ark_tx_userdata_dynfield_desc);
 284                 if (rte_pmd_ark_tx_userdata_dynfield_offset < 0) {
 285                         ARK_PMD_LOG(ERR,
 286                                     "Failed to register mbuf field for tx userdata\n");
 287                         return -rte_errno;
 288                 }
 289                 ARK_PMD_LOG(INFO, "Registered TX-meta dynamic field at %d\n",
 290                             rte_pmd_ark_tx_userdata_dynfield_offset);
 291         }
 292         if (RTE_PMD_ARK_RX_USERDATA_ENABLE) {
 293                 rte_pmd_ark_rx_userdata_dynfield_offset =
 294                     rte_mbuf_dynfield_register(&ark_rx_userdata_dynfield_desc);
 295                 if (rte_pmd_ark_rx_userdata_dynfield_offset < 0) {
 296                         ARK_PMD_LOG(ERR,
 297                                     "Failed to register mbuf field for rx userdata\n");
 298                         return -rte_errno;
 299                 }
 300                 ARK_PMD_LOG(INFO, "Registered RX-meta dynamic field at %d\n",
 301                             rte_pmd_ark_rx_userdata_dynfield_offset);
 302         }
 303
 304         pci_dev = RTE_ETH_DEV_TO_PCI(dev);
 305         rte_eth_copy_pci_info(dev, pci_dev);
 306         dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
 307
 308         /* Use dummy function until setup */
 309         dev->rx_pkt_burst = &eth_ark_recv_pkts_noop;
 310         dev->tx_pkt_burst = &eth_ark_xmit_pkts_noop;
 311
 312         ark->bar0 = (uint8_t *)pci_dev->mem_resource[0].addr;
 313         ark->a_bar = (uint8_t *)pci_dev->mem_resource[2].addr;
 314
 315         ark->sysctrl.v  = (void *)&ark->bar0[ARK_SYSCTRL_BASE];
 316         ark->mpurx.v  = (void *)&ark->bar0[ARK_MPU_RX_BASE];
 317         ark->udm.v  = (void *)&ark->bar0[ARK_UDM_BASE];
 318         ark->mputx.v  = (void *)&ark->bar0[ARK_MPU_TX_BASE];
 319         ark->ddm.v  = (void *)&ark->bar0[ARK_DDM_BASE];
 320         ark->cmac.v  = (void *)&ark->bar0[ARK_CMAC_BASE];
 321         ark->external.v  = (void *)&ark->bar0[ARK_EXTERNAL_BASE];
 322         ark->pktdir.v  = (void *)&ark->bar0[ARK_PKTDIR_BASE];
 323         ark->pktgen.v  = (void *)&ark->bar0[ARK_PKTGEN_BASE];
 324         ark->pktchkr.v  = (void *)&ark->bar0[ARK_PKTCHKR_BASE];
 325
 326         ark->rqpacing =
 327                 (struct ark_rqpace_t *)(ark->bar0 + ARK_RCPACING_BASE);
 328         ark->started = 0;
 329         ark->pkt_dir_v = ARK_PKT_DIR_INIT_VAL;
 330
 331         ARK_PMD_LOG(INFO, "Sys Ctrl Const = 0x%x  HW Commit_ID: %08x\n",
 332                       ark->sysctrl.t32[4],
 333                       rte_be_to_cpu_32(ark->sysctrl.t32[0x20 / 4]));
 334         ARK_PMD_LOG(NOTICE, "Arkville HW Commit_ID: %08x\n",
 335                     rte_be_to_cpu_32(ark->sysctrl.t32[0x20 / 4]));
 336
 337         /* If HW sanity test fails, return an error */
 338         if (ark->sysctrl.t32[4] != 0xcafef00d) {
 339                 ARK_PMD_LOG(ERR,
 340                             "HW Sanity test has failed, expected constant"
 341                             " 0x%x, read 0x%x (%s)\n",
 342                             0xcafef00d,
 343                             ark->sysctrl.t32[4], __func__);
 344                 return -1;
 345         }
 346         if (ark->sysctrl.t32[3] != 0) {
 347                 if (ark_rqp_lasped(ark->rqpacing)) {
 348                         ARK_PMD_LOG(ERR, "Arkville Evaluation System - "
 349                                     "Timer has Expired\n");
 350                         return -1;
 351                 }
 352                 ARK_PMD_LOG(WARNING, "Arkville Evaluation System - "
 353                             "Timer is Running\n");
 354         }
 355
 356         ARK_PMD_LOG(DEBUG,
 357                     "HW Sanity test has PASSED, expected constant"
 358                     " 0x%x, read 0x%x (%s)\n",
 359                     0xcafef00d, ark->sysctrl.t32[4], __func__);
 360
 361         /* We are a single function multi-port device. */
 362         ret = ark_config_device(dev);
 363         if (ret)
 364                 return -1;
 365
 366         dev->dev_ops = &ark_eth_dev_ops;
 367         dev->rx_queue_count = eth_ark_dev_rx_queue_count;
 368
 369         dev->data->mac_addrs = rte_zmalloc("ark", RTE_ETHER_ADDR_LEN, 0);
 370         if (!dev->data->mac_addrs) {
 371                 ARK_PMD_LOG(ERR,
 372                             "Failed to allocated memory for storing mac address"
 373                             );
 374         }
 375
 376         if (ark->user_ext.dev_init) {
 377                 ark->user_data[dev->data->port_id] =
 378                         ark->user_ext.dev_init(dev, ark->a_bar, 0);
 379                 if (!ark->user_data[dev->data->port_id]) {
 380                         ARK_PMD_LOG(WARNING,
 381                                     "Failed to initialize PMD extension!"
 382                                     " continuing without it\n");
 383                         memset(&ark->user_ext, 0, sizeof(struct ark_user_ext));
 384                         dlclose(ark->d_handle);
 385                 }
 386         }
 387
 388         if (pci_dev->device.devargs)
 389                 ret = eth_ark_check_args(ark, pci_dev->device.devargs->args);
 390         else
 391                 ARK_PMD_LOG(INFO, "No Device args found\n");
 392
 393         if (ret)
 394                 goto error;
 395         /*
 396          * We will create additional devices based on the number of requested
 397          * ports
 398          */
 399         if (ark->user_ext.dev_get_port_count)
 400                 port_count =
 401                         ark->user_ext.dev_get_port_count(dev,
 402                                  ark->user_data[dev->data->port_id]);
 403         ark->num_ports = port_count;
 404
 405         for (p = 0; p < port_count; p++) {
 406                 struct rte_eth_dev *eth_dev;
 407                 char name[RTE_ETH_NAME_MAX_LEN];
 408
 409                 snprintf(name, sizeof(name), "arketh%d",
 410                          dev->data->port_id + p);
 411
 412                 if (p == 0) {
 413                         /* First port is already allocated by DPDK */
 414                         eth_dev = ark->eth_dev;
 415                         rte_eth_dev_probing_finish(eth_dev);
 416                         continue;
 417                 }
 418
 419                 /* reserve an ethdev entry */
 420                 eth_dev = rte_eth_dev_allocate(name);
 421                 if (!eth_dev) {
 422                         ARK_PMD_LOG(ERR,
 423                                     "Could not allocate eth_dev for port %d\n",
 424                                     p);
 425                         goto error;
 426                 }
 427
 428                 eth_dev->device = &pci_dev->device;
 429                 eth_dev->data->dev_private = ark;
 430                 eth_dev->dev_ops = ark->eth_dev->dev_ops;
 431                 eth_dev->tx_pkt_burst = ark->eth_dev->tx_pkt_burst;
 432                 eth_dev->rx_pkt_burst = ark->eth_dev->rx_pkt_burst;
 433
 434                 rte_eth_copy_pci_info(eth_dev, pci_dev);
 435                 eth_dev->data->dev_flags |= RTE_ETH_DEV_AUTOFILL_QUEUE_XSTATS;
 436
 437                 eth_dev->data->mac_addrs = rte_zmalloc(name,
 438                                                 RTE_ETHER_ADDR_LEN, 0);
 439                 if (!eth_dev->data->mac_addrs) {
 440                         ARK_PMD_LOG(ERR,
 441                                     "Memory allocation for MAC failed!"
 442                                     " Exiting.\n");
 443                         goto error;
 444                 }
 445
 446                 if (ark->user_ext.dev_init) {
 447                         ark->user_data[eth_dev->data->port_id] =
 448                                 ark->user_ext.dev_init(dev, ark->a_bar, p);
 449                 }
 450
 451                 rte_eth_dev_probing_finish(eth_dev);
 452         }
 453
 454         return ret;
 455
 456 error:
 457         rte_free(dev->data->mac_addrs);
 458         dev->data->mac_addrs = NULL;
 459         return -1;
 460 }
 461
 462 /*
 463  *Initial device configuration when device is opened
 464  * setup the DDM, and UDM
 465  * Called once per PCIE device
 466  */
 467 static int
 468 ark_config_device(struct rte_eth_dev *dev)
 469 {
 470         struct ark_adapter *ark = dev->data->dev_private;
 471         uint16_t num_q, i;
 472         struct ark_mpu_t *mpu;
 473
 474         /*
 475          * Make sure that the packet director, generator and checker are in a
 476          * known state
 477          */
 478         ark->start_pg = 0;
 479         ark->pg = ark_pktgen_init(ark->pktgen.v, 0, 1);
 480         if (ark->pg == NULL)
 481                 return -1;
 482         ark_pktgen_reset(ark->pg);
 483         ark->pc = ark_pktchkr_init(ark->pktchkr.v, 0, 1);
 484         if (ark->pc == NULL)
 485                 return -1;
 486         ark_pktchkr_stop(ark->pc);
 487         ark->pd = ark_pktdir_init(ark->pktdir.v);
 488         if (ark->pd == NULL)
 489                 return -1;
 490
 491         /* Verify HW */
 492         if (ark_udm_verify(ark->udm.v))
 493                 return -1;
 494         if (ark_ddm_verify(ark->ddm.v))
 495                 return -1;
 496
 497         /* UDM */
 498         if (ark_udm_reset(ark->udm.v)) {
 499                 ARK_PMD_LOG(ERR, "Unable to stop and reset UDM\n");
 500                 return -1;
 501         }
 502         /* Keep in reset until the MPU are cleared */
 503
 504         /* MPU reset */
 505         mpu = ark->mpurx.v;
 506         num_q = ark_api_num_queues(mpu);
 507         ark->rx_queues = num_q;
 508         for (i = 0; i < num_q; i++) {
 509                 ark_mpu_reset(mpu);
 510                 mpu = RTE_PTR_ADD(mpu, ARK_MPU_QOFFSET);
 511         }
 512
 513         ark_udm_stop(ark->udm.v, 0);
 514         ark_udm_configure(ark->udm.v,
 515                           RTE_PKTMBUF_HEADROOM,
 516                           RTE_MBUF_DEFAULT_DATAROOM,
 517                           ARK_RX_WRITE_TIME_NS);
 518         ark_udm_stats_reset(ark->udm.v);
 519         ark_udm_stop(ark->udm.v, 0);
 520
 521         /* TX -- DDM */
 522         if (ark_ddm_stop(ark->ddm.v, 1))
 523                 ARK_PMD_LOG(ERR, "Unable to stop DDM\n");
 524
 525         mpu = ark->mputx.v;
 526         num_q = ark_api_num_queues(mpu);
 527         ark->tx_queues = num_q;
 528         for (i = 0; i < num_q; i++) {
 529                 ark_mpu_reset(mpu);
 530                 mpu = RTE_PTR_ADD(mpu, ARK_MPU_QOFFSET);
 531         }
 532
 533         ark_ddm_reset(ark->ddm.v);
 534         ark_ddm_stats_reset(ark->ddm.v);
 535
 536         ark_ddm_stop(ark->ddm.v, 0);
 537         ark_rqp_stats_reset(ark->rqpacing);
 538
 539         return 0;
 540 }
 541
 542 static int
 543 eth_ark_dev_uninit(struct rte_eth_dev *dev)
 544 {
 545         struct ark_adapter *ark = dev->data->dev_private;
 546
 547         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 548                 return 0;
 549
 550         if (ark->user_ext.dev_uninit)
 551                 ark->user_ext.dev_uninit(dev,
 552                          ark->user_data[dev->data->port_id]);
 553
 554         ark_pktgen_uninit(ark->pg);
 555         ark_pktchkr_uninit(ark->pc);
 556
 557         return 0;
 558 }
 559
 560 static int
 561 eth_ark_dev_configure(struct rte_eth_dev *dev)
 562 {
 563         struct ark_adapter *ark = dev->data->dev_private;
 564         int ret;
 565
 566         if (dev->data->dev_conf.rxmode.offloads & DEV_RX_OFFLOAD_TIMESTAMP) {
 567                 ret = rte_mbuf_dyn_rx_timestamp_register(
 568                                 &ark_timestamp_dynfield_offset,
 569                                 &ark_timestamp_rx_dynflag);
 570                 if (ret != 0) {
 571                         ARK_PMD_LOG(ERR,
 572                                 "Failed to register Rx timestamp field/flag\n");
 573                         return -rte_errno;
 574                 }
 575         }
 576
 577         eth_ark_dev_set_link_up(dev);
 578         if (ark->user_ext.dev_configure)
 579                 return ark->user_ext.dev_configure(dev,
 580                            ark->user_data[dev->data->port_id]);
 581         return 0;
 582 }
 583
 584 static void *
 585 delay_pg_start(void *arg)
 586 {
 587         struct ark_adapter *ark = (struct ark_adapter *)arg;
 588
 589         /* This function is used exclusively for regression testing, We
 590          * perform a blind sleep here to ensure that the external test
 591          * application has time to setup the test before we generate packets
 592          */
 593         usleep(100000);
 594         ark_pktgen_run(ark->pg);
 595         return NULL;
 596 }
 597
 598 static int
 599 eth_ark_dev_start(struct rte_eth_dev *dev)
 600 {
 601         struct ark_adapter *ark = dev->data->dev_private;
 602         int i;
 603
 604         /* RX Side */
 605         /* start UDM */
 606         ark_udm_start(ark->udm.v);
 607
 608         for (i = 0; i < dev->data->nb_rx_queues; i++)
 609                 eth_ark_rx_start_queue(dev, i);
 610
 611         /* TX Side */
 612         for (i = 0; i < dev->data->nb_tx_queues; i++)
 613                 eth_ark_tx_queue_start(dev, i);
 614
 615         /* start DDM */
 616         ark_ddm_start(ark->ddm.v);
 617
 618         ark->started = 1;
 619         /* set xmit and receive function */
 620         dev->rx_pkt_burst = &eth_ark_recv_pkts;
 621         dev->tx_pkt_burst = &eth_ark_xmit_pkts;
 622
 623         if (ark->start_pg)
 624                 ark_pktchkr_run(ark->pc);
 625
 626         if (ark->start_pg && (dev->data->port_id == 0)) {
 627                 pthread_t thread;
 628
 629                 /* Delay packet generatpr start allow the hardware to be ready
 630                  * This is only used for sanity checking with internal generator
 631                  */
 632                 if (pthread_create(&thread, NULL, delay_pg_start, ark)) {
 633                         ARK_PMD_LOG(ERR, "Could not create pktgen "
 634                                     "starter thread\n");
 635                         return -1;
 636                 }
 637         }
 638
 639         if (ark->user_ext.dev_start)
 640                 ark->user_ext.dev_start(dev,
 641                         ark->user_data[dev->data->port_id]);
 642
 643         return 0;
 644 }
 645
 646 static int
 647 eth_ark_dev_stop(struct rte_eth_dev *dev)
 648 {
 649         uint16_t i;
 650         int status;
 651         struct ark_adapter *ark = dev->data->dev_private;
 652         struct ark_mpu_t *mpu;
 653
 654         if (ark->started == 0)
 655                 return 0;
 656         ark->started = 0;
 657         dev->data->dev_started = 0;
 658
 659         /* Stop the extension first */
 660         if (ark->user_ext.dev_stop)
 661                 ark->user_ext.dev_stop(dev,
 662                        ark->user_data[dev->data->port_id]);
 663
 664         /* Stop the packet generator */
 665         if (ark->start_pg)
 666                 ark_pktgen_pause(ark->pg);
 667
 668         dev->rx_pkt_burst = &eth_ark_recv_pkts_noop;
 669         dev->tx_pkt_burst = &eth_ark_xmit_pkts_noop;
 670
 671         /* STOP TX Side */
 672         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 673                 status = eth_ark_tx_queue_stop(dev, i);
 674                 if (status != 0) {
 675                         uint16_t port = dev->data->port_id;
 676                         ARK_PMD_LOG(ERR,
 677                                     "tx_queue stop anomaly"
 678                                     " port %u, queue %u\n",
 679                                     port, i);
 680                 }
 681         }
 682
 683         /* Stop DDM */
 684         /* Wait up to 0.1 second.  each stop is up to 1000 * 10 useconds */
 685         for (i = 0; i < 10; i++) {
 686                 status = ark_ddm_stop(ark->ddm.v, 1);
 687                 if (status == 0)
 688                         break;
 689         }
 690         if (status || i != 0) {
 691                 ARK_PMD_LOG(ERR, "DDM stop anomaly. status:"
 692                             " %d iter: %u. (%s)\n",
 693                             status,
 694                             i,
 695                             __func__);
 696                 ark_ddm_dump(ark->ddm.v, "Stop anomaly");
 697
 698                 mpu = ark->mputx.v;
 699                 for (i = 0; i < ark->tx_queues; i++) {
 700                         ark_mpu_dump(mpu, "DDM failure dump", i);
 701                         mpu = RTE_PTR_ADD(mpu, ARK_MPU_QOFFSET);
 702                 }
 703         }
 704
 705         /* STOP RX Side */
 706         /* Stop UDM  multiple tries attempted */
 707         for (i = 0; i < 10; i++) {
 708                 status = ark_udm_stop(ark->udm.v, 1);
 709                 if (status == 0)
 710                         break;
 711         }
 712         if (status || i != 0) {
 713                 ARK_PMD_LOG(ERR, "UDM stop anomaly. status %d iter: %u. (%s)\n",
 714                             status, i, __func__);
 715                 ark_udm_dump(ark->udm.v, "Stop anomaly");
 716
 717                 mpu = ark->mpurx.v;
 718                 for (i = 0; i < ark->rx_queues; i++) {
 719                         ark_mpu_dump(mpu, "UDM Stop anomaly", i);
 720                         mpu = RTE_PTR_ADD(mpu, ARK_MPU_QOFFSET);
 721                 }
 722         }
 723
 724         ark_udm_dump_stats(ark->udm.v, "Post stop");
 725         ark_udm_dump_perf(ark->udm.v, "Post stop");
 726
 727         for (i = 0; i < dev->data->nb_rx_queues; i++)
 728                 eth_ark_rx_dump_queue(dev, i, __func__);
 729
 730         /* Stop the packet checker if it is running */
 731         if (ark->start_pg) {
 732                 ark_pktchkr_dump_stats(ark->pc);
 733                 ark_pktchkr_stop(ark->pc);
 734         }
 735
 736         return 0;
 737 }
 738
 739 static int
 740 eth_ark_dev_close(struct rte_eth_dev *dev)
 741 {
 742         struct ark_adapter *ark = dev->data->dev_private;
 743         uint16_t i;
 744
 745         if (rte_eal_process_type() != RTE_PROC_PRIMARY)
 746                 return 0;
 747
 748         if (ark->user_ext.dev_close)
 749                 ark->user_ext.dev_close(dev,
 750                  ark->user_data[dev->data->port_id]);
 751
 752         eth_ark_dev_stop(dev);
 753         eth_ark_udm_force_close(dev);
 754
 755         /*
 756          * TODO This should only be called once for the device during shutdown
 757          */
 758         ark_rqp_dump(ark->rqpacing);
 759
 760         for (i = 0; i < dev->data->nb_tx_queues; i++) {
 761                 eth_ark_tx_queue_release(dev->data->tx_queues[i]);
 762                 dev->data->tx_queues[i] = 0;
 763         }
 764
 765         for (i = 0; i < dev->data->nb_rx_queues; i++) {
 766                 eth_ark_dev_rx_queue_release(dev->data->rx_queues[i]);
 767                 dev->data->rx_queues[i] = 0;
 768         }
 769
 770         return 0;
 771 }
 772
 773 static int
 774 eth_ark_dev_info_get(struct rte_eth_dev *dev,
 775                      struct rte_eth_dev_info *dev_info)
 776 {
 777         struct ark_adapter *ark = dev->data->dev_private;
 778         struct ark_mpu_t *tx_mpu = RTE_PTR_ADD(ark->bar0, ARK_MPU_TX_BASE);
 779         struct ark_mpu_t *rx_mpu = RTE_PTR_ADD(ark->bar0, ARK_MPU_RX_BASE);
 780         uint16_t ports = ark->num_ports;
 781
 782         dev_info->max_rx_pktlen = ARK_RX_MAX_PKT_LEN;
 783         dev_info->min_rx_bufsize = ARK_RX_MIN_BUFSIZE;
 784
 785         dev_info->max_rx_queues = ark_api_num_queues_per_port(rx_mpu, ports);
 786         dev_info->max_tx_queues = ark_api_num_queues_per_port(tx_mpu, ports);
 787
 788         dev_info->rx_desc_lim = (struct rte_eth_desc_lim) {
 789                 .nb_max = ARK_RX_MAX_QUEUE,
 790                 .nb_min = ARK_RX_MIN_QUEUE,
 791                 .nb_align = ARK_RX_MIN_QUEUE}; /* power of 2 */
 792
 793         dev_info->tx_desc_lim = (struct rte_eth_desc_lim) {
 794                 .nb_max = ARK_TX_MAX_QUEUE,
 795                 .nb_min = ARK_TX_MIN_QUEUE,
 796                 .nb_align = ARK_TX_MIN_QUEUE}; /* power of 2 */
 797
 798         /* ARK PMD supports all line rates, how do we indicate that here ?? */
 799         dev_info->speed_capa = (ETH_LINK_SPEED_1G |
 800                                 ETH_LINK_SPEED_10G |
 801                                 ETH_LINK_SPEED_25G |
 802                                 ETH_LINK_SPEED_40G |
 803                                 ETH_LINK_SPEED_50G |
 804                                 ETH_LINK_SPEED_100G);
 805
 806         dev_info->rx_offload_capa = DEV_RX_OFFLOAD_TIMESTAMP;
 807
 808         return 0;
 809 }
 810
 811 static int
 812 eth_ark_dev_link_update(struct rte_eth_dev *dev, int wait_to_complete)
 813 {
 814         ARK_PMD_LOG(DEBUG, "link status = %d\n",
 815                         dev->data->dev_link.link_status);
 816         struct ark_adapter *ark = dev->data->dev_private;
 817
 818         if (ark->user_ext.link_update) {
 819                 return ark->user_ext.link_update
 820                         (dev, wait_to_complete,
 821                          ark->user_data[dev->data->port_id]);
 822         }
 823         return 0;
 824 }
 825
 826 static int
 827 eth_ark_dev_set_link_up(struct rte_eth_dev *dev)
 828 {
 829         dev->data->dev_link.link_status = 1;
 830         struct ark_adapter *ark = dev->data->dev_private;
 831
 832         if (ark->user_ext.dev_set_link_up)
 833                 return ark->user_ext.dev_set_link_up(dev,
 834                              ark->user_data[dev->data->port_id]);
 835         return 0;
 836 }
 837
 838 static int
 839 eth_ark_dev_set_link_down(struct rte_eth_dev *dev)
 840 {
 841         dev->data->dev_link.link_status = 0;
 842         struct ark_adapter *ark = dev->data->dev_private;
 843
 844         if (ark->user_ext.dev_set_link_down)
 845                 return ark->user_ext.dev_set_link_down(dev,
 846                        ark->user_data[dev->data->port_id]);
 847         return 0;
 848 }
 849
 850 static int
 851 eth_ark_dev_stats_get(struct rte_eth_dev *dev, struct rte_eth_stats *stats)
 852 {
 853         uint16_t i;
 854         struct ark_adapter *ark = dev->data->dev_private;
 855
 856         stats->ipackets = 0;
 857         stats->ibytes = 0;
 858         stats->opackets = 0;
 859         stats->obytes = 0;
 860         stats->imissed = 0;
 861         stats->oerrors = 0;
 862
 863         for (i = 0; i < dev->data->nb_tx_queues; i++)
 864                 eth_tx_queue_stats_get(dev->data->tx_queues[i], stats);
 865         for (i = 0; i < dev->data->nb_rx_queues; i++)
 866                 eth_rx_queue_stats_get(dev->data->rx_queues[i], stats);
 867         if (ark->user_ext.stats_get)
 868                 return ark->user_ext.stats_get(dev, stats,
 869                         ark->user_data[dev->data->port_id]);
 870         return 0;
 871 }
 872
 873 static int
 874 eth_ark_dev_stats_reset(struct rte_eth_dev *dev)
 875 {
 876         uint16_t i;
 877         struct ark_adapter *ark = dev->data->dev_private;
 878
 879         for (i = 0; i < dev->data->nb_tx_queues; i++)
 880                 eth_tx_queue_stats_reset(dev->data->tx_queues[i]);
 881         for (i = 0; i < dev->data->nb_rx_queues; i++)
 882                 eth_rx_queue_stats_reset(dev->data->rx_queues[i]);
 883         if (ark->user_ext.stats_reset)
 884                 ark->user_ext.stats_reset(dev,
 885                           ark->user_data[dev->data->port_id]);
 886
 887         return 0;
 888 }
 889
 890 static int
 891 eth_ark_macaddr_add(struct rte_eth_dev *dev,
 892                     struct rte_ether_addr *mac_addr,
 893                     uint32_t index,
 894                     uint32_t pool)
 895 {
 896         struct ark_adapter *ark = dev->data->dev_private;
 897
 898         if (ark->user_ext.mac_addr_add) {
 899                 ark->user_ext.mac_addr_add(dev,
 900                                            mac_addr,
 901                                            index,
 902                                            pool,
 903                            ark->user_data[dev->data->port_id]);
 904                 return 0;
 905         }
 906         return -ENOTSUP;
 907 }
 908
 909 static void
 910 eth_ark_macaddr_remove(struct rte_eth_dev *dev, uint32_t index)
 911 {
 912         struct ark_adapter *ark = dev->data->dev_private;
 913
 914         if (ark->user_ext.mac_addr_remove)
 915                 ark->user_ext.mac_addr_remove(dev, index,
 916                               ark->user_data[dev->data->port_id]);
 917 }
 918
 919 static int
 920 eth_ark_set_default_mac_addr(struct rte_eth_dev *dev,
 921                              struct rte_ether_addr *mac_addr)
 922 {
 923         struct ark_adapter *ark = dev->data->dev_private;
 924
 925         if (ark->user_ext.mac_addr_set) {
 926                 ark->user_ext.mac_addr_set(dev, mac_addr,
 927                            ark->user_data[dev->data->port_id]);
 928                 return 0;
 929         }
 930         return -ENOTSUP;
 931 }
 932
 933 static int
 934 eth_ark_set_mtu(struct rte_eth_dev *dev, uint16_t  size)
 935 {
 936         struct ark_adapter *ark = dev->data->dev_private;
 937
 938         if (ark->user_ext.set_mtu)
 939                 return ark->user_ext.set_mtu(dev, size,
 940                              ark->user_data[dev->data->port_id]);
 941
 942         return -ENOTSUP;
 943 }
 944
 945 static inline int
 946 process_pktdir_arg(const char *key, const char *value,
 947                    void *extra_args)
 948 {
 949         ARK_PMD_LOG(DEBUG, "key = %s, value = %s\n",
 950                     key, value);
 951         struct ark_adapter *ark =
 952                 (struct ark_adapter *)extra_args;
 953
 954         ark->pkt_dir_v = strtol(value, NULL, 16);
 955         ARK_PMD_LOG(DEBUG, "pkt_dir_v = 0x%x\n", ark->pkt_dir_v);
 956         return 0;
 957 }
 958
 959 static inline int
 960 process_file_args(const char *key, const char *value, void *extra_args)
 961 {
 962         ARK_PMD_LOG(DEBUG, "key = %s, value = %s\n",
 963                     key, value);
 964         char *args = (char *)extra_args;
 965
 966         /* Open the configuration file */
 967         FILE *file = fopen(value, "r");
 968         char line[ARK_MAX_ARG_LEN];
 969         int  size = 0;
 970         int first = 1;
 971
 972         if (file == NULL) {
 973                 ARK_PMD_LOG(ERR, "Unable to open "
 974                             "config file %s\n", value);
 975                 return -1;
 976         }
 977
 978         while (fgets(line, sizeof(line), file)) {
 979                 size += strlen(line);
 980                 if (size >= ARK_MAX_ARG_LEN) {
 981                         ARK_PMD_LOG(ERR, "Unable to parse file %s args, "
 982                                     "parameter list is too long\n", value);
 983                         fclose(file);
 984                         return -1;
 985                 }
 986                 if (first) {
 987                         strncpy(args, line, ARK_MAX_ARG_LEN);
 988                         first = 0;
 989                 } else {
 990                         strncat(args, line, ARK_MAX_ARG_LEN);
 991                 }
 992         }
 993         ARK_PMD_LOG(DEBUG, "file = %s\n", args);
 994         fclose(file);
 995         return 0;
 996 }
 997
 998 static int
 999 eth_ark_check_args(struct ark_adapter *ark, const char *params)
1000 {
1001         struct rte_kvargs *kvlist;
1002         unsigned int k_idx;
1003         struct rte_kvargs_pair *pair = NULL;
1004         int ret = -1;
1005
1006         kvlist = rte_kvargs_parse(params, valid_arguments);
1007         if (kvlist == NULL)
1008                 return 0;
1009
1010         ark->pkt_gen_args[0] = 0;
1011         ark->pkt_chkr_args[0] = 0;
1012
1013         for (k_idx = 0; k_idx < kvlist->count; k_idx++) {
1014                 pair = &kvlist->pairs[k_idx];
1015                 ARK_PMD_LOG(DEBUG, "**** Arg passed to PMD = %s:%s\n",
1016                              pair->key,
1017                              pair->value);
1018         }
1019
1020         if (rte_kvargs_process(kvlist,
1021                                ARK_PKTDIR_ARG,
1022                                &process_pktdir_arg,
1023                                ark) != 0) {
1024                 ARK_PMD_LOG(ERR, "Unable to parse arg %s\n", ARK_PKTDIR_ARG);
1025                 goto free_kvlist;
1026         }
1027
1028         if (rte_kvargs_process(kvlist,
1029                                ARK_PKTGEN_ARG,
1030                                &process_file_args,
1031                                ark->pkt_gen_args) != 0) {
1032                 ARK_PMD_LOG(ERR, "Unable to parse arg %s\n", ARK_PKTGEN_ARG);
1033                 goto free_kvlist;
1034         }
1035
1036         if (rte_kvargs_process(kvlist,
1037                                ARK_PKTCHKR_ARG,
1038                                &process_file_args,
1039                                ark->pkt_chkr_args) != 0) {
1040                 ARK_PMD_LOG(ERR, "Unable to parse arg %s\n", ARK_PKTCHKR_ARG);
1041                 goto free_kvlist;
1042         }
1043
1044         ARK_PMD_LOG(INFO, "packet director set to 0x%x\n", ark->pkt_dir_v);
1045         /* Setup the packet director */
1046         ark_pktdir_setup(ark->pd, ark->pkt_dir_v);
1047
1048         /* Setup the packet generator */
1049         if (ark->pkt_gen_args[0]) {
1050                 ARK_PMD_LOG(DEBUG, "Setting up the packet generator\n");
1051                 ark_pktgen_parse(ark->pkt_gen_args);
1052                 ark_pktgen_reset(ark->pg);
1053                 ark_pktgen_setup(ark->pg);
1054                 ark->start_pg = 1;
1055         }
1056
1057         /* Setup the packet checker */
1058         if (ark->pkt_chkr_args[0]) {
1059                 ark_pktchkr_parse(ark->pkt_chkr_args);
1060                 ark_pktchkr_setup(ark->pc);
1061         }
1062
1063         ret = 0;
1064
1065 free_kvlist:
1066         rte_kvargs_free(kvlist);
1067
1068         return ret;
1069 }
1070
1071 RTE_PMD_REGISTER_PCI(net_ark, rte_ark_pmd);
1072 RTE_PMD_REGISTER_KMOD_DEP(net_ark, "* igb_uio | uio_pci_generic ");
1073 RTE_PMD_REGISTER_PCI_TABLE(net_ark, pci_id_ark_map);
1074 RTE_PMD_REGISTER_PARAM_STRING(net_ark,
1075                               ARK_PKTGEN_ARG "=<filename> "
1076                               ARK_PKTCHKR_ARG "=<filename> "
1077                               ARK_PKTDIR_ARG "=<bitmap>");
1078 RTE_LOG_REGISTER(ark_logtype, pmd.net.ark, NOTICE);