drivers/common/qat/qat_device.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018-2020 Intel Corporation
   3  */
   4
   5 #include <rte_string_fns.h>
   6 #include <rte_devargs.h>
   7 #include <ctype.h>
   8
   9 #include "qat_device.h"
  10 #include "adf_transport_access_macros.h"
  11 #include "qat_sym_pmd.h"
  12 #include "qat_comp_pmd.h"
  13 #include "adf_pf2vf_msg.h"
  14
  15 /* pv2vf data Gen 4*/
  16 struct qat_pf2vf_dev qat_pf2vf_gen4 = {
  17         .pf2vf_offset = ADF_4XXXIOV_PF2VM_OFFSET,
  18         .vf2pf_offset = ADF_4XXXIOV_VM2PF_OFFSET,
  19         .pf2vf_type_shift = ADF_PFVF_2X_MSGTYPE_SHIFT,
  20         .pf2vf_type_mask = ADF_PFVF_2X_MSGTYPE_MASK,
  21         .pf2vf_data_shift = ADF_PFVF_2X_MSGDATA_SHIFT,
  22         .pf2vf_data_mask = ADF_PFVF_2X_MSGDATA_MASK,
  23 };
  24
  25 /* Hardware device information per generation */
  26 __extension__
  27 struct qat_gen_hw_data qat_gen_config[] =  {
  28         [QAT_GEN1] = {
  29                 .dev_gen = QAT_GEN1,
  30                 .qp_hw_data = qat_gen1_qps,
  31                 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN1
  32         },
  33         [QAT_GEN2] = {
  34                 .dev_gen = QAT_GEN2,
  35                 .qp_hw_data = qat_gen1_qps,
  36                 /* gen2 has same ring layout as gen1 */
  37                 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN2
  38         },
  39         [QAT_GEN3] = {
  40                 .dev_gen = QAT_GEN3,
  41                 .qp_hw_data = qat_gen3_qps,
  42                 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3
  43         },
  44         [QAT_GEN4] = {
  45                 .dev_gen = QAT_GEN4,
  46                 .qp_hw_data = NULL,
  47                 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3,
  48                 .pf2vf_dev = &qat_pf2vf_gen4
  49         },
  50 };
  51
  52 /* per-process array of device data */
  53 struct qat_device_info qat_pci_devs[RTE_PMD_QAT_MAX_PCI_DEVICES];
  54 static int qat_nb_pci_devices;
  55
  56 /*
  57  * The set of PCI devices this driver supports
  58  */
  59
  60 static const struct rte_pci_id pci_id_qat_map[] = {
  61                 {
  62                         RTE_PCI_DEVICE(0x8086, 0x0443),
  63                 },
  64                 {
  65                         RTE_PCI_DEVICE(0x8086, 0x37c9),
  66                 },
  67                 {
  68                         RTE_PCI_DEVICE(0x8086, 0x19e3),
  69                 },
  70                 {
  71                         RTE_PCI_DEVICE(0x8086, 0x6f55),
  72                 },
  73                 {
  74                         RTE_PCI_DEVICE(0x8086, 0x18ef),
  75                 },
  76                 {
  77                         RTE_PCI_DEVICE(0x8086, 0x18a1),
  78                 },
  79                 {
  80                         RTE_PCI_DEVICE(0x8086, 0x4941),
  81                 },
  82                 {.device_id = 0},
  83 };
  84
  85 static struct qat_pci_device *
  86 qat_pci_get_named_dev(const char *name)
  87 {
  88         unsigned int i;
  89
  90         if (name == NULL)
  91                 return NULL;
  92
  93         for (i = 0; i < RTE_PMD_QAT_MAX_PCI_DEVICES; i++) {
  94                 if (qat_pci_devs[i].mz &&
  95                                 (strcmp(((struct qat_pci_device *)
  96                                 qat_pci_devs[i].mz->addr)->name, name)
  97                                 == 0))
  98                         return (struct qat_pci_device *)
  99                                 qat_pci_devs[i].mz->addr;
 100         }
 101
 102         return NULL;
 103 }
 104
 105 static uint8_t
 106 qat_pci_find_free_device_index(void)
 107 {
 108                 uint8_t dev_id;
 109
 110                 for (dev_id = 0; dev_id < RTE_PMD_QAT_MAX_PCI_DEVICES;
 111                                 dev_id++) {
 112                         if (qat_pci_devs[dev_id].mz == NULL)
 113                                 break;
 114                 }
 115                 return dev_id;
 116 }
 117
 118 struct qat_pci_device *
 119 qat_get_qat_dev_from_pci_dev(struct rte_pci_device *pci_dev)
 120 {
 121         char name[QAT_DEV_NAME_MAX_LEN];
 122
 123         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 124
 125         return qat_pci_get_named_dev(name);
 126 }
 127
 128 static void qat_dev_parse_cmd(const char *str, struct qat_dev_cmd_param
 129                 *qat_dev_cmd_param)
 130 {
 131         int i = 0;
 132         const char *param;
 133
 134         while (1) {
 135                 char value_str[4] = { };
 136
 137                 param = qat_dev_cmd_param[i].name;
 138                 if (param == NULL)
 139                         return;
 140                 long value = 0;
 141                 const char *arg = strstr(str, param);
 142                 const char *arg2 = NULL;
 143
 144                 if (arg) {
 145                         arg2 = arg + strlen(param);
 146                         if (*arg2 != '=') {
 147                                 QAT_LOG(DEBUG, "parsing error '=' sign"
 148                                                 " should immediately follow %s",
 149                                                 param);
 150                                 arg2 = NULL;
 151                         } else
 152                                 arg2++;
 153                 } else {
 154                         QAT_LOG(DEBUG, "%s not provided", param);
 155                 }
 156                 if (arg2) {
 157                         int iter = 0;
 158                         while (iter < 2) {
 159                                 if (!isdigit(*(arg2 + iter)))
 160                                         break;
 161                                 iter++;
 162                         }
 163                         if (!iter) {
 164                                 QAT_LOG(DEBUG, "parsing error %s"
 165                                                " no number provided",
 166                                                param);
 167                         } else {
 168                                 memcpy(value_str, arg2, iter);
 169                                 value = strtol(value_str, NULL, 10);
 170                                 if (value > MAX_QP_THRESHOLD_SIZE) {
 171                                         QAT_LOG(DEBUG, "Exceeded max size of"
 172                                                 " threshold, setting to %d",
 173                                                 MAX_QP_THRESHOLD_SIZE);
 174                                         value = MAX_QP_THRESHOLD_SIZE;
 175                                 }
 176                                 QAT_LOG(DEBUG, "parsing %s = %ld",
 177                                                 param, value);
 178                         }
 179                 }
 180                 qat_dev_cmd_param[i].val = value;
 181                 i++;
 182         }
 183 }
 184
 185 struct qat_pci_device *
 186 qat_pci_device_allocate(struct rte_pci_device *pci_dev,
 187                 struct qat_dev_cmd_param *qat_dev_cmd_param)
 188 {
 189         struct qat_pci_device *qat_dev;
 190         uint8_t qat_dev_id = 0;
 191         char name[QAT_DEV_NAME_MAX_LEN];
 192         struct rte_devargs *devargs = pci_dev->device.devargs;
 193
 194         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 195         snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat");
 196
 197         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 198                 const struct rte_memzone *mz = rte_memzone_lookup(name);
 199
 200                 if (mz == NULL) {
 201                         QAT_LOG(ERR,
 202                                 "Secondary can't find %s mz, did primary create device?",
 203                                 name);
 204                         return NULL;
 205                 }
 206                 qat_dev = mz->addr;
 207                 qat_pci_devs[qat_dev->qat_dev_id].mz = mz;
 208                 qat_pci_devs[qat_dev->qat_dev_id].pci_dev = pci_dev;
 209                 qat_nb_pci_devices++;
 210                 QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d",
 211                         qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices);
 212                 return qat_dev;
 213         }
 214
 215         if (qat_pci_get_named_dev(name) != NULL) {
 216                 QAT_LOG(ERR, "QAT device with name %s already allocated!",
 217                                 name);
 218                 return NULL;
 219         }
 220
 221         qat_dev_id = qat_pci_find_free_device_index();
 222         if (qat_dev_id == RTE_PMD_QAT_MAX_PCI_DEVICES) {
 223                 QAT_LOG(ERR, "Reached maximum number of QAT devices");
 224                 return NULL;
 225         }
 226
 227         qat_pci_devs[qat_dev_id].mz = rte_memzone_reserve(name,
 228                 sizeof(struct qat_pci_device),
 229                 rte_socket_id(), 0);
 230
 231         if (qat_pci_devs[qat_dev_id].mz == NULL) {
 232                 QAT_LOG(ERR, "Error when allocating memzone for QAT_%d",
 233                         qat_dev_id);
 234                 return NULL;
 235         }
 236
 237         qat_dev = qat_pci_devs[qat_dev_id].mz->addr;
 238         memset(qat_dev, 0, sizeof(*qat_dev));
 239         strlcpy(qat_dev->name, name, QAT_DEV_NAME_MAX_LEN);
 240         qat_dev->qat_dev_id = qat_dev_id;
 241         qat_pci_devs[qat_dev_id].pci_dev = pci_dev;
 242         switch (pci_dev->id.device_id) {
 243         case 0x0443:
 244                 qat_dev->qat_dev_gen = QAT_GEN1;
 245                 break;
 246         case 0x37c9:
 247         case 0x19e3:
 248         case 0x6f55:
 249         case 0x18ef:
 250                 qat_dev->qat_dev_gen = QAT_GEN2;
 251                 break;
 252         case 0x18a1:
 253                 qat_dev->qat_dev_gen = QAT_GEN3;
 254                 break;
 255         case 0x4941:
 256                 qat_dev->qat_dev_gen = QAT_GEN4;
 257                 break;
 258         default:
 259                 QAT_LOG(ERR, "Invalid dev_id, can't determine generation");
 260                 rte_memzone_free(qat_pci_devs[qat_dev->qat_dev_id].mz);
 261                 return NULL;
 262         }
 263
 264         if (qat_dev->qat_dev_gen == QAT_GEN4) {
 265                 qat_dev->misc_bar_io_addr = pci_dev->mem_resource[2].addr;
 266                 if (qat_dev->misc_bar_io_addr == NULL) {
 267                         QAT_LOG(ERR, "QAT cannot get access to VF misc bar");
 268                         return NULL;
 269                 }
 270         }
 271
 272         if (devargs && devargs->drv_str)
 273                 qat_dev_parse_cmd(devargs->drv_str, qat_dev_cmd_param);
 274
 275         if (qat_dev->qat_dev_gen >= QAT_GEN4) {
 276                 int ret = qat_read_qp_config(qat_dev, qat_dev->qat_dev_gen);
 277
 278                 if (ret) {
 279                         QAT_LOG(ERR,
 280                                 "Cannot acquire ring configuration for QAT_%d",
 281                                 qat_dev_id);
 282                         return NULL;
 283                 }
 284         }
 285
 286         rte_spinlock_init(&qat_dev->arb_csr_lock);
 287         qat_nb_pci_devices++;
 288
 289         QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d",
 290                         qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices);
 291
 292         return qat_dev;
 293 }
 294
 295 static int
 296 qat_pci_device_release(struct rte_pci_device *pci_dev)
 297 {
 298         struct qat_pci_device *qat_dev;
 299         char name[QAT_DEV_NAME_MAX_LEN];
 300         int busy = 0;
 301
 302         if (pci_dev == NULL)
 303                 return -EINVAL;
 304
 305         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 306         snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat");
 307         qat_dev = qat_pci_get_named_dev(name);
 308         if (qat_dev != NULL) {
 309
 310                 struct qat_device_info *inst =
 311                                 &qat_pci_devs[qat_dev->qat_dev_id];
 312                 /* Check that there are no service devs still on pci device */
 313
 314                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 315                         if (qat_dev->sym_dev != NULL) {
 316                                 QAT_LOG(DEBUG, "QAT sym device %s is busy",
 317                                         name);
 318                                 busy = 1;
 319                         }
 320                         if (qat_dev->asym_dev != NULL) {
 321                                 QAT_LOG(DEBUG, "QAT asym device %s is busy",
 322                                         name);
 323                                 busy = 1;
 324                         }
 325                         if (qat_dev->comp_dev != NULL) {
 326                                 QAT_LOG(DEBUG, "QAT comp device %s is busy",
 327                                         name);
 328                                 busy = 1;
 329                         }
 330                         if (busy)
 331                                 return -EBUSY;
 332                         rte_memzone_free(inst->mz);
 333                 }
 334                 memset(inst, 0, sizeof(struct qat_device_info));
 335                 qat_nb_pci_devices--;
 336                 QAT_LOG(DEBUG, "QAT device %s released, total QATs %d",
 337                                         name, qat_nb_pci_devices);
 338         }
 339         return 0;
 340 }
 341
 342 static int
 343 qat_pci_dev_destroy(struct qat_pci_device *qat_pci_dev,
 344                 struct rte_pci_device *pci_dev)
 345 {
 346         qat_sym_dev_destroy(qat_pci_dev);
 347         qat_comp_dev_destroy(qat_pci_dev);
 348         qat_asym_dev_destroy(qat_pci_dev);
 349         return qat_pci_device_release(pci_dev);
 350 }
 351
 352 static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 353                 struct rte_pci_device *pci_dev)
 354 {
 355         int sym_ret = 0, asym_ret = 0, comp_ret = 0;
 356         int num_pmds_created = 0;
 357         struct qat_pci_device *qat_pci_dev;
 358         struct qat_dev_cmd_param qat_dev_cmd_param[] = {
 359                         { SYM_ENQ_THRESHOLD_NAME, 0 },
 360                         { ASYM_ENQ_THRESHOLD_NAME, 0 },
 361                         { COMP_ENQ_THRESHOLD_NAME, 0 },
 362                         { NULL, 0 },
 363         };
 364
 365         QAT_LOG(DEBUG, "Found QAT device at %02x:%02x.%x",
 366                         pci_dev->addr.bus,
 367                         pci_dev->addr.devid,
 368                         pci_dev->addr.function);
 369
 370         qat_pci_dev = qat_pci_device_allocate(pci_dev, qat_dev_cmd_param);
 371         if (qat_pci_dev == NULL)
 372                 return -ENODEV;
 373
 374         sym_ret = qat_sym_dev_create(qat_pci_dev, qat_dev_cmd_param);
 375         if (sym_ret == 0) {
 376                 num_pmds_created++;
 377
 378         }
 379         else
 380                 QAT_LOG(WARNING,
 381                                 "Failed to create QAT SYM PMD on device %s",
 382                                 qat_pci_dev->name);
 383
 384         comp_ret = qat_comp_dev_create(qat_pci_dev, qat_dev_cmd_param);
 385         if (comp_ret == 0)
 386                 num_pmds_created++;
 387         else
 388                 QAT_LOG(WARNING,
 389                                 "Failed to create QAT COMP PMD on device %s",
 390                                 qat_pci_dev->name);
 391
 392         asym_ret = qat_asym_dev_create(qat_pci_dev, qat_dev_cmd_param);
 393         if (asym_ret == 0)
 394                 num_pmds_created++;
 395         else
 396                 QAT_LOG(WARNING,
 397                                 "Failed to create QAT ASYM PMD on device %s",
 398                                 qat_pci_dev->name);
 399
 400         if (num_pmds_created == 0)
 401                 qat_pci_dev_destroy(qat_pci_dev, pci_dev);
 402
 403         return 0;
 404 }
 405
 406 static int qat_pci_remove(struct rte_pci_device *pci_dev)
 407 {
 408         struct qat_pci_device *qat_pci_dev;
 409
 410         if (pci_dev == NULL)
 411                 return -EINVAL;
 412
 413         qat_pci_dev = qat_get_qat_dev_from_pci_dev(pci_dev);
 414         if (qat_pci_dev == NULL)
 415                 return 0;
 416
 417         return qat_pci_dev_destroy(qat_pci_dev, pci_dev);
 418 }
 419
 420 static struct rte_pci_driver rte_qat_pmd = {
 421         .id_table = pci_id_qat_map,
 422         .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
 423         .probe = qat_pci_probe,
 424         .remove = qat_pci_remove
 425 };
 426
 427 __rte_weak int
 428 qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
 429                 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 430 {
 431         return 0;
 432 }
 433
 434 __rte_weak int
 435 qat_asym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
 436                 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 437 {
 438         return 0;
 439 }
 440
 441 __rte_weak int
 442 qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
 443 {
 444         return 0;
 445 }
 446
 447 __rte_weak int
 448 qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
 449 {
 450         return 0;
 451 }
 452
 453 __rte_weak int
 454 qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
 455                 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 456 {
 457         return 0;
 458 }
 459
 460 __rte_weak int
 461 qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
 462 {
 463         return 0;
 464 }
 465
 466 RTE_PMD_REGISTER_PCI(QAT_PCI_NAME, rte_qat_pmd);
 467 RTE_PMD_REGISTER_PCI_TABLE(QAT_PCI_NAME, pci_id_qat_map);
 468 RTE_PMD_REGISTER_KMOD_DEP(QAT_PCI_NAME, "* igb_uio | uio_pci_generic | vfio-pci");