drivers/common/qat/qat_device.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018-2020 Intel Corporation
   3  */
   4
   5 #include <rte_string_fns.h>
   6 #include <rte_devargs.h>
   7 #include <ctype.h>
   8
   9 #include "qat_device.h"
  10 #include "adf_transport_access_macros.h"
  11 #include "qat_sym_pmd.h"
  12 #include "qat_comp_pmd.h"
  13
  14 /* Hardware device information per generation */
  15 __extension__
  16 struct qat_gen_hw_data qat_gen_config[] =  {
  17         [QAT_GEN1] = {
  18                 .dev_gen = QAT_GEN1,
  19                 .qp_hw_data = qat_gen1_qps,
  20                 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN1
  21         },
  22         [QAT_GEN2] = {
  23                 .dev_gen = QAT_GEN2,
  24                 .qp_hw_data = qat_gen1_qps,
  25                 /* gen2 has same ring layout as gen1 */
  26                 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN2
  27         },
  28         [QAT_GEN3] = {
  29                 .dev_gen = QAT_GEN3,
  30                 .qp_hw_data = qat_gen3_qps,
  31                 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3
  32         },
  33         [QAT_GEN4] = {
  34                 .dev_gen = QAT_GEN4,
  35                 .qp_hw_data = NULL,
  36                 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3
  37         },
  38 };
  39
  40 /* per-process array of device data */
  41 struct qat_device_info qat_pci_devs[RTE_PMD_QAT_MAX_PCI_DEVICES];
  42 static int qat_nb_pci_devices;
  43
  44 /*
  45  * The set of PCI devices this driver supports
  46  */
  47
  48 static const struct rte_pci_id pci_id_qat_map[] = {
  49                 {
  50                         RTE_PCI_DEVICE(0x8086, 0x0443),
  51                 },
  52                 {
  53                         RTE_PCI_DEVICE(0x8086, 0x37c9),
  54                 },
  55                 {
  56                         RTE_PCI_DEVICE(0x8086, 0x19e3),
  57                 },
  58                 {
  59                         RTE_PCI_DEVICE(0x8086, 0x6f55),
  60                 },
  61                 {
  62                         RTE_PCI_DEVICE(0x8086, 0x18ef),
  63                 },
  64                 {
  65                         RTE_PCI_DEVICE(0x8086, 0x18a1),
  66                 },
  67                 {
  68                         RTE_PCI_DEVICE(0x8086, 0x4941),
  69                 },
  70                 {.device_id = 0},
  71 };
  72
  73 static struct qat_pci_device *
  74 qat_pci_get_named_dev(const char *name)
  75 {
  76         unsigned int i;
  77
  78         if (name == NULL)
  79                 return NULL;
  80
  81         for (i = 0; i < RTE_PMD_QAT_MAX_PCI_DEVICES; i++) {
  82                 if (qat_pci_devs[i].mz &&
  83                                 (strcmp(((struct qat_pci_device *)
  84                                 qat_pci_devs[i].mz->addr)->name, name)
  85                                 == 0))
  86                         return (struct qat_pci_device *)
  87                                 qat_pci_devs[i].mz->addr;
  88         }
  89
  90         return NULL;
  91 }
  92
  93 static uint8_t
  94 qat_pci_find_free_device_index(void)
  95 {
  96                 uint8_t dev_id;
  97
  98                 for (dev_id = 0; dev_id < RTE_PMD_QAT_MAX_PCI_DEVICES;
  99                                 dev_id++) {
 100                         if (qat_pci_devs[dev_id].mz == NULL)
 101                                 break;
 102                 }
 103                 return dev_id;
 104 }
 105
 106 struct qat_pci_device *
 107 qat_get_qat_dev_from_pci_dev(struct rte_pci_device *pci_dev)
 108 {
 109         char name[QAT_DEV_NAME_MAX_LEN];
 110
 111         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 112
 113         return qat_pci_get_named_dev(name);
 114 }
 115
 116 static void qat_dev_parse_cmd(const char *str, struct qat_dev_cmd_param
 117                 *qat_dev_cmd_param)
 118 {
 119         int i = 0;
 120         const char *param;
 121
 122         while (1) {
 123                 char value_str[4] = { };
 124
 125                 param = qat_dev_cmd_param[i].name;
 126                 if (param == NULL)
 127                         return;
 128                 long value = 0;
 129                 const char *arg = strstr(str, param);
 130                 const char *arg2 = NULL;
 131
 132                 if (arg) {
 133                         arg2 = arg + strlen(param);
 134                         if (*arg2 != '=') {
 135                                 QAT_LOG(DEBUG, "parsing error '=' sign"
 136                                                 " should immediately follow %s",
 137                                                 param);
 138                                 arg2 = NULL;
 139                         } else
 140                                 arg2++;
 141                 } else {
 142                         QAT_LOG(DEBUG, "%s not provided", param);
 143                 }
 144                 if (arg2) {
 145                         int iter = 0;
 146                         while (iter < 2) {
 147                                 if (!isdigit(*(arg2 + iter)))
 148                                         break;
 149                                 iter++;
 150                         }
 151                         if (!iter) {
 152                                 QAT_LOG(DEBUG, "parsing error %s"
 153                                                " no number provided",
 154                                                param);
 155                         } else {
 156                                 memcpy(value_str, arg2, iter);
 157                                 value = strtol(value_str, NULL, 10);
 158                                 if (value > MAX_QP_THRESHOLD_SIZE) {
 159                                         QAT_LOG(DEBUG, "Exceeded max size of"
 160                                                 " threshold, setting to %d",
 161                                                 MAX_QP_THRESHOLD_SIZE);
 162                                         value = MAX_QP_THRESHOLD_SIZE;
 163                                 }
 164                                 QAT_LOG(DEBUG, "parsing %s = %ld",
 165                                                 param, value);
 166                         }
 167                 }
 168                 qat_dev_cmd_param[i].val = value;
 169                 i++;
 170         }
 171 }
 172
 173 struct qat_pci_device *
 174 qat_pci_device_allocate(struct rte_pci_device *pci_dev,
 175                 struct qat_dev_cmd_param *qat_dev_cmd_param)
 176 {
 177         struct qat_pci_device *qat_dev;
 178         uint8_t qat_dev_id = 0;
 179         char name[QAT_DEV_NAME_MAX_LEN];
 180         struct rte_devargs *devargs = pci_dev->device.devargs;
 181
 182         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 183         snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat");
 184
 185         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 186                 const struct rte_memzone *mz = rte_memzone_lookup(name);
 187
 188                 if (mz == NULL) {
 189                         QAT_LOG(ERR,
 190                                 "Secondary can't find %s mz, did primary create device?",
 191                                 name);
 192                         return NULL;
 193                 }
 194                 qat_dev = mz->addr;
 195                 qat_pci_devs[qat_dev->qat_dev_id].mz = mz;
 196                 qat_pci_devs[qat_dev->qat_dev_id].pci_dev = pci_dev;
 197                 qat_nb_pci_devices++;
 198                 QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d",
 199                         qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices);
 200                 return qat_dev;
 201         }
 202
 203         if (qat_pci_get_named_dev(name) != NULL) {
 204                 QAT_LOG(ERR, "QAT device with name %s already allocated!",
 205                                 name);
 206                 return NULL;
 207         }
 208
 209         qat_dev_id = qat_pci_find_free_device_index();
 210         if (qat_dev_id == RTE_PMD_QAT_MAX_PCI_DEVICES) {
 211                 QAT_LOG(ERR, "Reached maximum number of QAT devices");
 212                 return NULL;
 213         }
 214
 215         qat_pci_devs[qat_dev_id].mz = rte_memzone_reserve(name,
 216                 sizeof(struct qat_pci_device),
 217                 rte_socket_id(), 0);
 218
 219         if (qat_pci_devs[qat_dev_id].mz == NULL) {
 220                 QAT_LOG(ERR, "Error when allocating memzone for QAT_%d",
 221                         qat_dev_id);
 222                 return NULL;
 223         }
 224
 225         qat_dev = qat_pci_devs[qat_dev_id].mz->addr;
 226         memset(qat_dev, 0, sizeof(*qat_dev));
 227         strlcpy(qat_dev->name, name, QAT_DEV_NAME_MAX_LEN);
 228         qat_dev->qat_dev_id = qat_dev_id;
 229         qat_pci_devs[qat_dev_id].pci_dev = pci_dev;
 230         switch (pci_dev->id.device_id) {
 231         case 0x0443:
 232                 qat_dev->qat_dev_gen = QAT_GEN1;
 233                 break;
 234         case 0x37c9:
 235         case 0x19e3:
 236         case 0x6f55:
 237         case 0x18ef:
 238                 qat_dev->qat_dev_gen = QAT_GEN2;
 239                 break;
 240         case 0x18a1:
 241                 qat_dev->qat_dev_gen = QAT_GEN3;
 242                 break;
 243         case 0x4941:
 244                 qat_dev->qat_dev_gen = QAT_GEN4;
 245                 break;
 246         default:
 247                 QAT_LOG(ERR, "Invalid dev_id, can't determine generation");
 248                 rte_memzone_free(qat_pci_devs[qat_dev->qat_dev_id].mz);
 249                 return NULL;
 250         }
 251
 252         if (devargs && devargs->drv_str)
 253                 qat_dev_parse_cmd(devargs->drv_str, qat_dev_cmd_param);
 254
 255         if (qat_dev->qat_dev_gen >= QAT_GEN4) {
 256                 int ret = qat_read_qp_config(qat_dev, qat_dev->qat_dev_gen);
 257
 258                 if (ret) {
 259                         QAT_LOG(ERR,
 260                                 "Cannot acquire ring configuration for QAT_%d",
 261                                 qat_dev_id);
 262                         return NULL;
 263                 }
 264         }
 265
 266         rte_spinlock_init(&qat_dev->arb_csr_lock);
 267         qat_nb_pci_devices++;
 268
 269         QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d",
 270                         qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices);
 271
 272         return qat_dev;
 273 }
 274
 275 static int
 276 qat_pci_device_release(struct rte_pci_device *pci_dev)
 277 {
 278         struct qat_pci_device *qat_dev;
 279         char name[QAT_DEV_NAME_MAX_LEN];
 280         int busy = 0;
 281
 282         if (pci_dev == NULL)
 283                 return -EINVAL;
 284
 285         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 286         snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat");
 287         qat_dev = qat_pci_get_named_dev(name);
 288         if (qat_dev != NULL) {
 289
 290                 struct qat_device_info *inst =
 291                                 &qat_pci_devs[qat_dev->qat_dev_id];
 292                 /* Check that there are no service devs still on pci device */
 293
 294                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 295                         if (qat_dev->sym_dev != NULL) {
 296                                 QAT_LOG(DEBUG, "QAT sym device %s is busy",
 297                                         name);
 298                                 busy = 1;
 299                         }
 300                         if (qat_dev->asym_dev != NULL) {
 301                                 QAT_LOG(DEBUG, "QAT asym device %s is busy",
 302                                         name);
 303                                 busy = 1;
 304                         }
 305                         if (qat_dev->comp_dev != NULL) {
 306                                 QAT_LOG(DEBUG, "QAT comp device %s is busy",
 307                                         name);
 308                                 busy = 1;
 309                         }
 310                         if (busy)
 311                                 return -EBUSY;
 312                         rte_memzone_free(inst->mz);
 313                 }
 314                 memset(inst, 0, sizeof(struct qat_device_info));
 315                 qat_nb_pci_devices--;
 316                 QAT_LOG(DEBUG, "QAT device %s released, total QATs %d",
 317                                         name, qat_nb_pci_devices);
 318         }
 319         return 0;
 320 }
 321
 322 static int
 323 qat_pci_dev_destroy(struct qat_pci_device *qat_pci_dev,
 324                 struct rte_pci_device *pci_dev)
 325 {
 326         qat_sym_dev_destroy(qat_pci_dev);
 327         qat_comp_dev_destroy(qat_pci_dev);
 328         qat_asym_dev_destroy(qat_pci_dev);
 329         return qat_pci_device_release(pci_dev);
 330 }
 331
 332 static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 333                 struct rte_pci_device *pci_dev)
 334 {
 335         int sym_ret = 0, asym_ret = 0, comp_ret = 0;
 336         int num_pmds_created = 0;
 337         struct qat_pci_device *qat_pci_dev;
 338         struct qat_dev_cmd_param qat_dev_cmd_param[] = {
 339                         { SYM_ENQ_THRESHOLD_NAME, 0 },
 340                         { ASYM_ENQ_THRESHOLD_NAME, 0 },
 341                         { COMP_ENQ_THRESHOLD_NAME, 0 },
 342                         { NULL, 0 },
 343         };
 344
 345         QAT_LOG(DEBUG, "Found QAT device at %02x:%02x.%x",
 346                         pci_dev->addr.bus,
 347                         pci_dev->addr.devid,
 348                         pci_dev->addr.function);
 349
 350         qat_pci_dev = qat_pci_device_allocate(pci_dev, qat_dev_cmd_param);
 351         if (qat_pci_dev == NULL)
 352                 return -ENODEV;
 353
 354         sym_ret = qat_sym_dev_create(qat_pci_dev, qat_dev_cmd_param);
 355         if (sym_ret == 0) {
 356                 num_pmds_created++;
 357
 358         }
 359         else
 360                 QAT_LOG(WARNING,
 361                                 "Failed to create QAT SYM PMD on device %s",
 362                                 qat_pci_dev->name);
 363
 364         comp_ret = qat_comp_dev_create(qat_pci_dev, qat_dev_cmd_param);
 365         if (comp_ret == 0)
 366                 num_pmds_created++;
 367         else
 368                 QAT_LOG(WARNING,
 369                                 "Failed to create QAT COMP PMD on device %s",
 370                                 qat_pci_dev->name);
 371
 372         asym_ret = qat_asym_dev_create(qat_pci_dev, qat_dev_cmd_param);
 373         if (asym_ret == 0)
 374                 num_pmds_created++;
 375         else
 376                 QAT_LOG(WARNING,
 377                                 "Failed to create QAT ASYM PMD on device %s",
 378                                 qat_pci_dev->name);
 379
 380         if (num_pmds_created == 0)
 381                 qat_pci_dev_destroy(qat_pci_dev, pci_dev);
 382
 383         return 0;
 384 }
 385
 386 static int qat_pci_remove(struct rte_pci_device *pci_dev)
 387 {
 388         struct qat_pci_device *qat_pci_dev;
 389
 390         if (pci_dev == NULL)
 391                 return -EINVAL;
 392
 393         qat_pci_dev = qat_get_qat_dev_from_pci_dev(pci_dev);
 394         if (qat_pci_dev == NULL)
 395                 return 0;
 396
 397         return qat_pci_dev_destroy(qat_pci_dev, pci_dev);
 398 }
 399
 400 static struct rte_pci_driver rte_qat_pmd = {
 401         .id_table = pci_id_qat_map,
 402         .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
 403         .probe = qat_pci_probe,
 404         .remove = qat_pci_remove
 405 };
 406
 407 __rte_weak int
 408 qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
 409                 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 410 {
 411         return 0;
 412 }
 413
 414 __rte_weak int
 415 qat_asym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
 416                 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 417 {
 418         return 0;
 419 }
 420
 421 __rte_weak int
 422 qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
 423 {
 424         return 0;
 425 }
 426
 427 __rte_weak int
 428 qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
 429 {
 430         return 0;
 431 }
 432
 433 __rte_weak int
 434 qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
 435                 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 436 {
 437         return 0;
 438 }
 439
 440 __rte_weak int
 441 qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
 442 {
 443         return 0;
 444 }
 445
 446 RTE_PMD_REGISTER_PCI(QAT_PCI_NAME, rte_qat_pmd);
 447 RTE_PMD_REGISTER_PCI_TABLE(QAT_PCI_NAME, pci_id_qat_map);
 448 RTE_PMD_REGISTER_KMOD_DEP(QAT_PCI_NAME, "* igb_uio | uio_pci_generic | vfio-pci");