drivers/event/dlb/pf/dlb_pf.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2016-2020 Intel Corporation
   3  */
   4
   5 #include <stdint.h>
   6 #include <stdbool.h>
   7 #include <stdio.h>
   8 #include <sys/mman.h>
   9 #include <sys/fcntl.h>
  10 #include <sys/time.h>
  11 #include <errno.h>
  12 #include <assert.h>
  13 #include <unistd.h>
  14 #include <string.h>
  15 #include <rte_debug.h>
  16 #include <rte_log.h>
  17 #include <rte_dev.h>
  18 #include <rte_devargs.h>
  19 #include <rte_mbuf.h>
  20 #include <rte_ring.h>
  21 #include <rte_errno.h>
  22 #include <rte_kvargs.h>
  23 #include <rte_malloc.h>
  24 #include <rte_cycles.h>
  25 #include <rte_io.h>
  26 #include <rte_memory.h>
  27 #include <rte_string_fns.h>
  28
  29 #include "../dlb_priv.h"
  30 #include "../dlb_iface.h"
  31 #include "../dlb_inline_fns.h"
  32 #include "dlb_main.h"
  33 #include "base/dlb_hw_types.h"
  34 #include "base/dlb_osdep.h"
  35 #include "base/dlb_resource.h"
  36
  37 static void
  38 dlb_pf_low_level_io_init(struct dlb_eventdev *dlb __rte_unused)
  39 {
  40         int i;
  41
  42         /* Addresses will be initialized at port create */
  43         for (i = 0; i < DLB_MAX_NUM_PORTS; i++) {
  44                 /* First directed ports */
  45
  46                 /* producer port */
  47                 dlb_port[i][DLB_DIR].pp_addr = NULL;
  48
  49                 /* popcount */
  50                 dlb_port[i][DLB_DIR].ldb_popcount = NULL;
  51                 dlb_port[i][DLB_DIR].dir_popcount = NULL;
  52
  53                 /* consumer queue */
  54                 dlb_port[i][DLB_DIR].cq_base = NULL;
  55                 dlb_port[i][DLB_DIR].mmaped = true;
  56
  57                 /* Now load balanced ports */
  58
  59                 /* producer port */
  60                 dlb_port[i][DLB_LDB].pp_addr = NULL;
  61
  62                 /* popcount */
  63                 dlb_port[i][DLB_LDB].ldb_popcount = NULL;
  64                 dlb_port[i][DLB_LDB].dir_popcount = NULL;
  65
  66                 /* consumer queue */
  67                 dlb_port[i][DLB_LDB].cq_base = NULL;
  68                 dlb_port[i][DLB_LDB].mmaped = true;
  69         }
  70 }
  71
  72 static int
  73 dlb_pf_open(struct dlb_hw_dev *handle, const char *name)
  74 {
  75         RTE_SET_USED(handle);
  76         RTE_SET_USED(name);
  77
  78         return 0;
  79 }
  80
  81 static void
  82 dlb_pf_domain_close(struct dlb_eventdev *dlb)
  83 {
  84         struct dlb_dev *dlb_dev = (struct dlb_dev *)dlb->qm_instance.pf_dev;
  85         int ret;
  86
  87         ret = dlb_reset_domain(&dlb_dev->hw, dlb->qm_instance.domain_id);
  88         if (ret)
  89                 DLB_LOG_ERR("dlb_pf_reset_domain err %d", ret);
  90 }
  91
  92 static int
  93 dlb_pf_get_device_version(struct dlb_hw_dev *handle,
  94                           uint8_t *revision)
  95 {
  96         struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
  97
  98         *revision = dlb_dev->revision;
  99
 100         return 0;
 101 }
 102
 103 static int
 104 dlb_pf_get_num_resources(struct dlb_hw_dev *handle,
 105                          struct dlb_get_num_resources_args *rsrcs)
 106 {
 107         struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
 108
 109         dlb_hw_get_num_resources(&dlb_dev->hw, rsrcs);
 110
 111         return 0;
 112 }
 113
 114 static int
 115 dlb_pf_sched_domain_create(struct dlb_hw_dev *handle,
 116                            struct dlb_create_sched_domain_args *arg)
 117 {
 118         struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
 119         struct dlb_cmd_response response = {0};
 120         int ret;
 121
 122         DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
 123
 124         if (dlb_dev->domain_reset_failed) {
 125                 response.status = DLB_ST_DOMAIN_RESET_FAILED;
 126                 ret = -EINVAL;
 127                 goto done;
 128         }
 129
 130         ret = dlb_hw_create_sched_domain(&dlb_dev->hw, arg, &response);
 131         if (ret)
 132                 goto done;
 133
 134 done:
 135
 136         *(struct dlb_cmd_response *)arg->response = response;
 137
 138         DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
 139
 140         return ret;
 141 }
 142
 143 static int
 144 dlb_pf_ldb_credit_pool_create(struct dlb_hw_dev *handle,
 145                               struct dlb_create_ldb_pool_args *cfg)
 146 {
 147         struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
 148         struct dlb_cmd_response response = {0};
 149         int ret;
 150
 151         DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
 152
 153         ret = dlb_hw_create_ldb_pool(&dlb_dev->hw,
 154                                      handle->domain_id,
 155                                      cfg,
 156                                      &response);
 157
 158         *(struct dlb_cmd_response *)cfg->response = response;
 159
 160         DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
 161
 162         return ret;
 163 }
 164
 165 static int
 166 dlb_pf_dir_credit_pool_create(struct dlb_hw_dev *handle,
 167                               struct dlb_create_dir_pool_args *cfg)
 168 {
 169         struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
 170         struct dlb_cmd_response response = {0};
 171         int ret;
 172
 173         DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
 174
 175         ret = dlb_hw_create_dir_pool(&dlb_dev->hw,
 176                                      handle->domain_id,
 177                                      cfg,
 178                                      &response);
 179
 180         *(struct dlb_cmd_response *)cfg->response = response;
 181
 182         DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
 183
 184         return ret;
 185 }
 186
 187 static int
 188 dlb_pf_get_cq_poll_mode(struct dlb_hw_dev *handle,
 189                         enum dlb_cq_poll_modes *mode)
 190 {
 191         struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
 192
 193         if (dlb_dev->revision >= DLB_REV_B0)
 194                 *mode = DLB_CQ_POLL_MODE_SPARSE;
 195         else
 196                 *mode = DLB_CQ_POLL_MODE_STD;
 197
 198         return 0;
 199 }
 200
 201 static int
 202 dlb_pf_ldb_queue_create(struct dlb_hw_dev *handle,
 203                         struct dlb_create_ldb_queue_args *cfg)
 204 {
 205         struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
 206         struct dlb_cmd_response response = {0};
 207         int ret;
 208
 209         DLB_INFO(dev->dlb_device, "Entering %s()\n", __func__);
 210
 211         ret = dlb_hw_create_ldb_queue(&dlb_dev->hw,
 212                                       handle->domain_id,
 213                                       cfg,
 214                                       &response);
 215
 216         *(struct dlb_cmd_response *)cfg->response = response;
 217
 218         DLB_INFO(dev->dlb_device, "Exiting %s() with ret=%d\n", __func__, ret);
 219
 220         return ret;
 221 }
 222
 223 static int
 224 dlb_pf_get_sn_allocation(struct dlb_hw_dev *handle,
 225                          struct dlb_get_sn_allocation_args *args)
 226 {
 227         struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
 228         struct dlb_cmd_response response = {0};
 229         int ret;
 230
 231         ret = dlb_get_group_sequence_numbers(&dlb_dev->hw, args->group);
 232
 233         response.id = ret;
 234         response.status = 0;
 235
 236         *(struct dlb_cmd_response *)args->response = response;
 237
 238         return ret;
 239 }
 240
 241 static int
 242 dlb_pf_set_sn_allocation(struct dlb_hw_dev *handle,
 243                          struct dlb_set_sn_allocation_args *args)
 244 {
 245         struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
 246         struct dlb_cmd_response response = {0};
 247         int ret;
 248
 249         ret = dlb_set_group_sequence_numbers(&dlb_dev->hw, args->group,
 250                                              args->num);
 251
 252         response.status = 0;
 253
 254         *(struct dlb_cmd_response *)args->response = response;
 255
 256         return ret;
 257 }
 258
 259 static int
 260 dlb_pf_get_sn_occupancy(struct dlb_hw_dev *handle,
 261                         struct dlb_get_sn_occupancy_args *args)
 262 {
 263         struct dlb_dev *dlb_dev = (struct dlb_dev *)handle->pf_dev;
 264         struct dlb_cmd_response response = {0};
 265         int ret;
 266
 267         ret = dlb_get_group_sequence_number_occupancy(&dlb_dev->hw,
 268                                                       args->group);
 269
 270         response.id = ret;
 271         response.status = 0;
 272
 273         *(struct dlb_cmd_response *)args->response = response;
 274
 275         return ret;
 276 }
 277
 278 static void
 279 dlb_pf_iface_fn_ptrs_init(void)
 280 {
 281         dlb_iface_low_level_io_init = dlb_pf_low_level_io_init;
 282         dlb_iface_open = dlb_pf_open;
 283         dlb_iface_domain_close = dlb_pf_domain_close;
 284         dlb_iface_get_device_version = dlb_pf_get_device_version;
 285         dlb_iface_get_num_resources = dlb_pf_get_num_resources;
 286         dlb_iface_sched_domain_create = dlb_pf_sched_domain_create;
 287         dlb_iface_ldb_credit_pool_create = dlb_pf_ldb_credit_pool_create;
 288         dlb_iface_dir_credit_pool_create = dlb_pf_dir_credit_pool_create;
 289         dlb_iface_ldb_queue_create = dlb_pf_ldb_queue_create;
 290         dlb_iface_get_cq_poll_mode = dlb_pf_get_cq_poll_mode;
 291         dlb_iface_get_sn_allocation = dlb_pf_get_sn_allocation;
 292         dlb_iface_set_sn_allocation = dlb_pf_set_sn_allocation;
 293         dlb_iface_get_sn_occupancy = dlb_pf_get_sn_occupancy;
 294 }
 295
 296 /* PCI DEV HOOKS */
 297 static int
 298 dlb_eventdev_pci_init(struct rte_eventdev *eventdev)
 299 {
 300         int ret = 0;
 301         struct rte_pci_device *pci_dev;
 302         struct dlb_devargs dlb_args = {
 303                 .socket_id = rte_socket_id(),
 304                 .max_num_events = DLB_MAX_NUM_LDB_CREDITS,
 305                 .num_dir_credits_override = -1,
 306                 .defer_sched = 0,
 307                 .num_atm_inflights = DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE,
 308         };
 309         struct dlb_eventdev *dlb;
 310
 311         DLB_LOG_DBG("Enter with dev_id=%d socket_id=%d",
 312                     eventdev->data->dev_id, eventdev->data->socket_id);
 313
 314         dlb_entry_points_init(eventdev);
 315
 316         dlb_pf_iface_fn_ptrs_init();
 317
 318         pci_dev = RTE_DEV_TO_PCI(eventdev->dev);
 319
 320         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 321                 dlb = dlb_pmd_priv(eventdev); /* rte_zmalloc_socket mem */
 322
 323                 /* Probe the DLB PF layer */
 324                 dlb->qm_instance.pf_dev = dlb_probe(pci_dev);
 325
 326                 if (dlb->qm_instance.pf_dev == NULL) {
 327                         DLB_LOG_ERR("DLB PF Probe failed with error %d\n",
 328                                     rte_errno);
 329                         ret = -rte_errno;
 330                         goto dlb_probe_failed;
 331                 }
 332
 333                 /* Were we invoked with runtime parameters? */
 334                 if (pci_dev->device.devargs) {
 335                         ret = dlb_parse_params(pci_dev->device.devargs->args,
 336                                                pci_dev->device.devargs->name,
 337                                                &dlb_args);
 338                         if (ret) {
 339                                 DLB_LOG_ERR("PFPMD failed to parse args ret=%d, errno=%d\n",
 340                                             ret, rte_errno);
 341                                 goto dlb_probe_failed;
 342                         }
 343                 }
 344
 345                 ret = dlb_primary_eventdev_probe(eventdev,
 346                                                  EVDEV_DLB_NAME_PMD_STR,
 347                                                  &dlb_args);
 348         } else {
 349                 ret = dlb_secondary_eventdev_probe(eventdev,
 350                                                    EVDEV_DLB_NAME_PMD_STR);
 351         }
 352         if (ret)
 353                 goto dlb_probe_failed;
 354
 355         DLB_LOG_INFO("DLB PF Probe success\n");
 356
 357         return 0;
 358
 359 dlb_probe_failed:
 360
 361         DLB_LOG_INFO("DLB PF Probe failed, ret=%d\n", ret);
 362
 363         return ret;
 364 }
 365
 366 #define EVENTDEV_INTEL_VENDOR_ID 0x8086
 367
 368 static const struct rte_pci_id pci_id_dlb_map[] = {
 369         {
 370                 RTE_PCI_DEVICE(EVENTDEV_INTEL_VENDOR_ID,
 371                                DLB_PF_DEV_ID)
 372         },
 373         {
 374                 .vendor_id = 0,
 375         },
 376 };
 377
 378 static int
 379 event_dlb_pci_probe(struct rte_pci_driver *pci_drv,
 380                     struct rte_pci_device *pci_dev)
 381 {
 382         return rte_event_pmd_pci_probe_named(pci_drv, pci_dev,
 383                 sizeof(struct dlb_eventdev), dlb_eventdev_pci_init,
 384                 EVDEV_DLB_NAME_PMD_STR);
 385 }
 386
 387 static int
 388 event_dlb_pci_remove(struct rte_pci_device *pci_dev)
 389 {
 390         return rte_event_pmd_pci_remove(pci_dev, NULL);
 391 }
 392
 393 static struct rte_pci_driver pci_eventdev_dlb_pmd = {
 394         .id_table = pci_id_dlb_map,
 395         .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
 396         .probe = event_dlb_pci_probe,
 397         .remove = event_dlb_pci_remove,
 398 };
 399
 400 RTE_PMD_REGISTER_PCI(event_dlb_pf, pci_eventdev_dlb_pmd);
 401 RTE_PMD_REGISTER_PCI_TABLE(event_dlb_pf, pci_id_dlb_map);