drivers/common/qat/qat_device.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018 Intel Corporation
   3  */
   4
   5 #include <rte_string_fns.h>
   6 #include <rte_devargs.h>
   7 #include <ctype.h>
   8
   9 #include "qat_device.h"
  10 #include "adf_transport_access_macros.h"
  11 #include "qat_sym_pmd.h"
  12 #include "qat_comp_pmd.h"
  13
  14 /* Hardware device information per generation */
  15 __extension__
  16 struct qat_gen_hw_data qat_gen_config[] =  {
  17         [QAT_GEN1] = {
  18                 .dev_gen = QAT_GEN1,
  19                 .qp_hw_data = qat_gen1_qps,
  20                 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN1
  21         },
  22         [QAT_GEN2] = {
  23                 .dev_gen = QAT_GEN2,
  24                 .qp_hw_data = qat_gen1_qps,
  25                 /* gen2 has same ring layout as gen1 */
  26                 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN2
  27         },
  28         [QAT_GEN3] = {
  29                 .dev_gen = QAT_GEN3,
  30                 .qp_hw_data = qat_gen3_qps,
  31                 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3
  32         },
  33 };
  34
  35 /* per-process array of device data */
  36 struct qat_device_info qat_pci_devs[RTE_PMD_QAT_MAX_PCI_DEVICES];
  37 static int qat_nb_pci_devices;
  38
  39 /*
  40  * The set of PCI devices this driver supports
  41  */
  42
  43 static const struct rte_pci_id pci_id_qat_map[] = {
  44                 {
  45                         RTE_PCI_DEVICE(0x8086, 0x0443),
  46                 },
  47                 {
  48                         RTE_PCI_DEVICE(0x8086, 0x37c9),
  49                 },
  50                 {
  51                         RTE_PCI_DEVICE(0x8086, 0x19e3),
  52                 },
  53                 {
  54                         RTE_PCI_DEVICE(0x8086, 0x6f55),
  55                 },
  56                 {
  57                         RTE_PCI_DEVICE(0x8086, 0x18a1),
  58                 },
  59                 {.device_id = 0},
  60 };
  61
  62 static struct qat_pci_device *
  63 qat_pci_get_named_dev(const char *name)
  64 {
  65         unsigned int i;
  66
  67         if (name == NULL)
  68                 return NULL;
  69
  70         for (i = 0; i < RTE_PMD_QAT_MAX_PCI_DEVICES; i++) {
  71                 if (qat_pci_devs[i].mz &&
  72                                 (strcmp(((struct qat_pci_device *)
  73                                 qat_pci_devs[i].mz->addr)->name, name)
  74                                 == 0))
  75                         return (struct qat_pci_device *)
  76                                 qat_pci_devs[i].mz->addr;
  77         }
  78
  79         return NULL;
  80 }
  81
  82 static uint8_t
  83 qat_pci_find_free_device_index(void)
  84 {
  85                 uint8_t dev_id;
  86
  87                 for (dev_id = 0; dev_id < RTE_PMD_QAT_MAX_PCI_DEVICES;
  88                                 dev_id++) {
  89                         if (qat_pci_devs[dev_id].mz == NULL)
  90                                 break;
  91                 }
  92                 return dev_id;
  93 }
  94
  95 struct qat_pci_device *
  96 qat_get_qat_dev_from_pci_dev(struct rte_pci_device *pci_dev)
  97 {
  98         char name[QAT_DEV_NAME_MAX_LEN];
  99
 100         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 101
 102         return qat_pci_get_named_dev(name);
 103 }
 104
 105 static void qat_dev_parse_cmd(const char *str, struct qat_dev_cmd_param
 106                 *qat_dev_cmd_param)
 107 {
 108         int i = 0;
 109         const char *param;
 110
 111         while (1) {
 112                 char value_str[4] = { };
 113
 114                 param = qat_dev_cmd_param[i].name;
 115                 if (param == NULL)
 116                         return;
 117                 long value = 0;
 118                 const char *arg = strstr(str, param);
 119                 const char *arg2 = NULL;
 120
 121                 if (arg) {
 122                         arg2 = arg + strlen(param);
 123                         if (*arg2 != '=') {
 124                                 QAT_LOG(DEBUG, "parsing error '=' sign"
 125                                                 " should immediately follow %s",
 126                                                 param);
 127                                 arg2 = NULL;
 128                         } else
 129                                 arg2++;
 130                 } else {
 131                         QAT_LOG(DEBUG, "%s not provided", param);
 132                 }
 133                 if (arg2) {
 134                         int iter = 0;
 135                         while (iter < 2) {
 136                                 if (!isdigit(*(arg2 + iter)))
 137                                         break;
 138                                 iter++;
 139                         }
 140                         if (!iter) {
 141                                 QAT_LOG(DEBUG, "parsing error %s"
 142                                                " no number provided",
 143                                                param);
 144                         } else {
 145                                 memcpy(value_str, arg2, iter);
 146                                 value = strtol(value_str, NULL, 10);
 147                                 if (value > MAX_QP_THRESHOLD_SIZE) {
 148                                         QAT_LOG(DEBUG, "Exceeded max size of"
 149                                                 " threshold, setting to %d",
 150                                                 MAX_QP_THRESHOLD_SIZE);
 151                                         value = MAX_QP_THRESHOLD_SIZE;
 152                                 }
 153                                 QAT_LOG(DEBUG, "parsing %s = %ld",
 154                                                 param, value);
 155                         }
 156                 }
 157                 qat_dev_cmd_param[i].val = value;
 158                 i++;
 159         }
 160 }
 161
 162 struct qat_pci_device *
 163 qat_pci_device_allocate(struct rte_pci_device *pci_dev,
 164                 struct qat_dev_cmd_param *qat_dev_cmd_param)
 165 {
 166         struct qat_pci_device *qat_dev;
 167         uint8_t qat_dev_id = 0;
 168         char name[QAT_DEV_NAME_MAX_LEN];
 169         struct rte_devargs *devargs = pci_dev->device.devargs;
 170
 171         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 172         snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat");
 173
 174         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 175                 const struct rte_memzone *mz = rte_memzone_lookup(name);
 176
 177                 if (mz == NULL) {
 178                         QAT_LOG(ERR,
 179                                 "Secondary can't find %s mz, did primary create device?",
 180                                 name);
 181                         return NULL;
 182                 }
 183                 qat_dev = mz->addr;
 184                 qat_pci_devs[qat_dev->qat_dev_id].mz = mz;
 185                 qat_pci_devs[qat_dev->qat_dev_id].pci_dev = pci_dev;
 186                 qat_nb_pci_devices++;
 187                 QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d",
 188                         qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices);
 189                 return qat_dev;
 190         }
 191
 192         if (qat_pci_get_named_dev(name) != NULL) {
 193                 QAT_LOG(ERR, "QAT device with name %s already allocated!",
 194                                 name);
 195                 return NULL;
 196         }
 197
 198         qat_dev_id = qat_pci_find_free_device_index();
 199         if (qat_dev_id == RTE_PMD_QAT_MAX_PCI_DEVICES) {
 200                 QAT_LOG(ERR, "Reached maximum number of QAT devices");
 201                 return NULL;
 202         }
 203
 204         qat_pci_devs[qat_dev_id].mz = rte_memzone_reserve(name,
 205                 sizeof(struct qat_pci_device),
 206                 rte_socket_id(), 0);
 207
 208         if (qat_pci_devs[qat_dev_id].mz == NULL) {
 209                 QAT_LOG(ERR, "Error when allocating memzone for QAT_%d",
 210                         qat_dev_id);
 211                 return NULL;
 212         }
 213
 214         qat_dev = qat_pci_devs[qat_dev_id].mz->addr;
 215         memset(qat_dev, 0, sizeof(*qat_dev));
 216         strlcpy(qat_dev->name, name, QAT_DEV_NAME_MAX_LEN);
 217         qat_dev->qat_dev_id = qat_dev_id;
 218         qat_pci_devs[qat_dev_id].pci_dev = pci_dev;
 219         switch (pci_dev->id.device_id) {
 220         case 0x0443:
 221                 qat_dev->qat_dev_gen = QAT_GEN1;
 222                 break;
 223         case 0x37c9:
 224         case 0x19e3:
 225         case 0x6f55:
 226                 qat_dev->qat_dev_gen = QAT_GEN2;
 227                 break;
 228         case 0x18a1:
 229                 qat_dev->qat_dev_gen = QAT_GEN3;
 230                 break;
 231         default:
 232                 QAT_LOG(ERR, "Invalid dev_id, can't determine generation");
 233                 rte_memzone_free(qat_pci_devs[qat_dev->qat_dev_id].mz);
 234                 return NULL;
 235         }
 236
 237         if (devargs && devargs->drv_str)
 238                 qat_dev_parse_cmd(devargs->drv_str, qat_dev_cmd_param);
 239
 240         rte_spinlock_init(&qat_dev->arb_csr_lock);
 241         qat_nb_pci_devices++;
 242
 243         QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d",
 244                         qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices);
 245
 246         return qat_dev;
 247 }
 248
 249 static int
 250 qat_pci_device_release(struct rte_pci_device *pci_dev)
 251 {
 252         struct qat_pci_device *qat_dev;
 253         char name[QAT_DEV_NAME_MAX_LEN];
 254         int busy = 0;
 255
 256         if (pci_dev == NULL)
 257                 return -EINVAL;
 258
 259         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 260         snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat");
 261         qat_dev = qat_pci_get_named_dev(name);
 262         if (qat_dev != NULL) {
 263
 264                 struct qat_device_info *inst =
 265                                 &qat_pci_devs[qat_dev->qat_dev_id];
 266                 /* Check that there are no service devs still on pci device */
 267
 268                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 269                         if (qat_dev->sym_dev != NULL) {
 270                                 QAT_LOG(DEBUG, "QAT sym device %s is busy",
 271                                         name);
 272                                 busy = 1;
 273                         }
 274                         if (qat_dev->asym_dev != NULL) {
 275                                 QAT_LOG(DEBUG, "QAT asym device %s is busy",
 276                                         name);
 277                                 busy = 1;
 278                         }
 279                         if (qat_dev->comp_dev != NULL) {
 280                                 QAT_LOG(DEBUG, "QAT comp device %s is busy",
 281                                         name);
 282                                 busy = 1;
 283                         }
 284                         if (busy)
 285                                 return -EBUSY;
 286                         rte_memzone_free(inst->mz);
 287                 }
 288                 memset(inst, 0, sizeof(struct qat_device_info));
 289                 qat_nb_pci_devices--;
 290                 QAT_LOG(DEBUG, "QAT device %s released, total QATs %d",
 291                                         name, qat_nb_pci_devices);
 292         }
 293         return 0;
 294 }
 295
 296 static int
 297 qat_pci_dev_destroy(struct qat_pci_device *qat_pci_dev,
 298                 struct rte_pci_device *pci_dev)
 299 {
 300         qat_sym_dev_destroy(qat_pci_dev);
 301         qat_comp_dev_destroy(qat_pci_dev);
 302         qat_asym_dev_destroy(qat_pci_dev);
 303         return qat_pci_device_release(pci_dev);
 304 }
 305
 306 static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 307                 struct rte_pci_device *pci_dev)
 308 {
 309         int sym_ret = 0, asym_ret = 0, comp_ret = 0;
 310         int num_pmds_created = 0;
 311         struct qat_pci_device *qat_pci_dev;
 312         struct qat_dev_cmd_param qat_dev_cmd_param[] = {
 313                         { SYM_ENQ_THRESHOLD_NAME, 0 },
 314                         { ASYM_ENQ_THRESHOLD_NAME, 0 },
 315                         { COMP_ENQ_THRESHOLD_NAME, 0 },
 316                         { NULL, 0 },
 317         };
 318
 319         QAT_LOG(DEBUG, "Found QAT device at %02x:%02x.%x",
 320                         pci_dev->addr.bus,
 321                         pci_dev->addr.devid,
 322                         pci_dev->addr.function);
 323
 324         qat_pci_dev = qat_pci_device_allocate(pci_dev, qat_dev_cmd_param);
 325         if (qat_pci_dev == NULL)
 326                 return -ENODEV;
 327
 328         sym_ret = qat_sym_dev_create(qat_pci_dev, qat_dev_cmd_param);
 329         if (sym_ret == 0) {
 330                 num_pmds_created++;
 331
 332         }
 333         else
 334                 QAT_LOG(WARNING,
 335                                 "Failed to create QAT SYM PMD on device %s",
 336                                 qat_pci_dev->name);
 337
 338         comp_ret = qat_comp_dev_create(qat_pci_dev, qat_dev_cmd_param);
 339         if (comp_ret == 0)
 340                 num_pmds_created++;
 341         else
 342                 QAT_LOG(WARNING,
 343                                 "Failed to create QAT COMP PMD on device %s",
 344                                 qat_pci_dev->name);
 345
 346         asym_ret = qat_asym_dev_create(qat_pci_dev, qat_dev_cmd_param);
 347         if (asym_ret == 0)
 348                 num_pmds_created++;
 349         else
 350                 QAT_LOG(WARNING,
 351                                 "Failed to create QAT ASYM PMD on device %s",
 352                                 qat_pci_dev->name);
 353
 354         if (num_pmds_created == 0)
 355                 qat_pci_dev_destroy(qat_pci_dev, pci_dev);
 356
 357         return 0;
 358 }
 359
 360 static int qat_pci_remove(struct rte_pci_device *pci_dev)
 361 {
 362         struct qat_pci_device *qat_pci_dev;
 363
 364         if (pci_dev == NULL)
 365                 return -EINVAL;
 366
 367         qat_pci_dev = qat_get_qat_dev_from_pci_dev(pci_dev);
 368         if (qat_pci_dev == NULL)
 369                 return 0;
 370
 371         return qat_pci_dev_destroy(qat_pci_dev, pci_dev);
 372 }
 373
 374 static struct rte_pci_driver rte_qat_pmd = {
 375         .id_table = pci_id_qat_map,
 376         .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
 377         .probe = qat_pci_probe,
 378         .remove = qat_pci_remove
 379 };
 380
 381 __rte_weak int
 382 qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
 383                 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 384 {
 385         return 0;
 386 }
 387
 388 __rte_weak int
 389 qat_asym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
 390                 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 391 {
 392         return 0;
 393 }
 394
 395 __rte_weak int
 396 qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
 397 {
 398         return 0;
 399 }
 400
 401 __rte_weak int
 402 qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
 403 {
 404         return 0;
 405 }
 406
 407 __rte_weak int
 408 qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
 409                 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 410 {
 411         return 0;
 412 }
 413
 414 __rte_weak int
 415 qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
 416 {
 417         return 0;
 418 }
 419
 420 RTE_PMD_REGISTER_PCI(QAT_PCI_NAME, rte_qat_pmd);
 421 RTE_PMD_REGISTER_PCI_TABLE(QAT_PCI_NAME, pci_id_qat_map);