drivers/common/qat/qat_device.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018-2020 Intel Corporation
   3  */
   4
   5 #include <rte_string_fns.h>
   6 #include <rte_devargs.h>
   7 #include <ctype.h>
   8
   9 #include "qat_device.h"
  10 #include "adf_transport_access_macros.h"
  11 #include "qat_sym_pmd.h"
  12 #include "qat_comp_pmd.h"
  13
  14 /* Hardware device information per generation */
  15 __extension__
  16 struct qat_gen_hw_data qat_gen_config[] =  {
  17         [QAT_GEN1] = {
  18                 .dev_gen = QAT_GEN1,
  19                 .qp_hw_data = qat_gen1_qps,
  20                 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN1
  21         },
  22         [QAT_GEN2] = {
  23                 .dev_gen = QAT_GEN2,
  24                 .qp_hw_data = qat_gen1_qps,
  25                 /* gen2 has same ring layout as gen1 */
  26                 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN2
  27         },
  28         [QAT_GEN3] = {
  29                 .dev_gen = QAT_GEN3,
  30                 .qp_hw_data = qat_gen3_qps,
  31                 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3
  32         },
  33 };
  34
  35 /* per-process array of device data */
  36 struct qat_device_info qat_pci_devs[RTE_PMD_QAT_MAX_PCI_DEVICES];
  37 static int qat_nb_pci_devices;
  38
  39 /*
  40  * The set of PCI devices this driver supports
  41  */
  42
  43 static const struct rte_pci_id pci_id_qat_map[] = {
  44                 {
  45                         RTE_PCI_DEVICE(0x8086, 0x0443),
  46                 },
  47                 {
  48                         RTE_PCI_DEVICE(0x8086, 0x37c9),
  49                 },
  50                 {
  51                         RTE_PCI_DEVICE(0x8086, 0x19e3),
  52                 },
  53                 {
  54                         RTE_PCI_DEVICE(0x8086, 0x6f55),
  55                 },
  56                 {
  57                         RTE_PCI_DEVICE(0x8086, 0x18ef),
  58                 },
  59                 {
  60                         RTE_PCI_DEVICE(0x8086, 0x18a1),
  61                 },
  62                 {.device_id = 0},
  63 };
  64
  65 static struct qat_pci_device *
  66 qat_pci_get_named_dev(const char *name)
  67 {
  68         unsigned int i;
  69
  70         if (name == NULL)
  71                 return NULL;
  72
  73         for (i = 0; i < RTE_PMD_QAT_MAX_PCI_DEVICES; i++) {
  74                 if (qat_pci_devs[i].mz &&
  75                                 (strcmp(((struct qat_pci_device *)
  76                                 qat_pci_devs[i].mz->addr)->name, name)
  77                                 == 0))
  78                         return (struct qat_pci_device *)
  79                                 qat_pci_devs[i].mz->addr;
  80         }
  81
  82         return NULL;
  83 }
  84
  85 static uint8_t
  86 qat_pci_find_free_device_index(void)
  87 {
  88                 uint8_t dev_id;
  89
  90                 for (dev_id = 0; dev_id < RTE_PMD_QAT_MAX_PCI_DEVICES;
  91                                 dev_id++) {
  92                         if (qat_pci_devs[dev_id].mz == NULL)
  93                                 break;
  94                 }
  95                 return dev_id;
  96 }
  97
  98 struct qat_pci_device *
  99 qat_get_qat_dev_from_pci_dev(struct rte_pci_device *pci_dev)
 100 {
 101         char name[QAT_DEV_NAME_MAX_LEN];
 102
 103         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 104
 105         return qat_pci_get_named_dev(name);
 106 }
 107
 108 static void qat_dev_parse_cmd(const char *str, struct qat_dev_cmd_param
 109                 *qat_dev_cmd_param)
 110 {
 111         int i = 0;
 112         const char *param;
 113
 114         while (1) {
 115                 char value_str[4] = { };
 116
 117                 param = qat_dev_cmd_param[i].name;
 118                 if (param == NULL)
 119                         return;
 120                 long value = 0;
 121                 const char *arg = strstr(str, param);
 122                 const char *arg2 = NULL;
 123
 124                 if (arg) {
 125                         arg2 = arg + strlen(param);
 126                         if (*arg2 != '=') {
 127                                 QAT_LOG(DEBUG, "parsing error '=' sign"
 128                                                 " should immediately follow %s",
 129                                                 param);
 130                                 arg2 = NULL;
 131                         } else
 132                                 arg2++;
 133                 } else {
 134                         QAT_LOG(DEBUG, "%s not provided", param);
 135                 }
 136                 if (arg2) {
 137                         int iter = 0;
 138                         while (iter < 2) {
 139                                 if (!isdigit(*(arg2 + iter)))
 140                                         break;
 141                                 iter++;
 142                         }
 143                         if (!iter) {
 144                                 QAT_LOG(DEBUG, "parsing error %s"
 145                                                " no number provided",
 146                                                param);
 147                         } else {
 148                                 memcpy(value_str, arg2, iter);
 149                                 value = strtol(value_str, NULL, 10);
 150                                 if (value > MAX_QP_THRESHOLD_SIZE) {
 151                                         QAT_LOG(DEBUG, "Exceeded max size of"
 152                                                 " threshold, setting to %d",
 153                                                 MAX_QP_THRESHOLD_SIZE);
 154                                         value = MAX_QP_THRESHOLD_SIZE;
 155                                 }
 156                                 QAT_LOG(DEBUG, "parsing %s = %ld",
 157                                                 param, value);
 158                         }
 159                 }
 160                 qat_dev_cmd_param[i].val = value;
 161                 i++;
 162         }
 163 }
 164
 165 struct qat_pci_device *
 166 qat_pci_device_allocate(struct rte_pci_device *pci_dev,
 167                 struct qat_dev_cmd_param *qat_dev_cmd_param)
 168 {
 169         struct qat_pci_device *qat_dev;
 170         uint8_t qat_dev_id = 0;
 171         char name[QAT_DEV_NAME_MAX_LEN];
 172         struct rte_devargs *devargs = pci_dev->device.devargs;
 173
 174         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 175         snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat");
 176
 177         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 178                 const struct rte_memzone *mz = rte_memzone_lookup(name);
 179
 180                 if (mz == NULL) {
 181                         QAT_LOG(ERR,
 182                                 "Secondary can't find %s mz, did primary create device?",
 183                                 name);
 184                         return NULL;
 185                 }
 186                 qat_dev = mz->addr;
 187                 qat_pci_devs[qat_dev->qat_dev_id].mz = mz;
 188                 qat_pci_devs[qat_dev->qat_dev_id].pci_dev = pci_dev;
 189                 qat_nb_pci_devices++;
 190                 QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d",
 191                         qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices);
 192                 return qat_dev;
 193         }
 194
 195         if (qat_pci_get_named_dev(name) != NULL) {
 196                 QAT_LOG(ERR, "QAT device with name %s already allocated!",
 197                                 name);
 198                 return NULL;
 199         }
 200
 201         qat_dev_id = qat_pci_find_free_device_index();
 202         if (qat_dev_id == RTE_PMD_QAT_MAX_PCI_DEVICES) {
 203                 QAT_LOG(ERR, "Reached maximum number of QAT devices");
 204                 return NULL;
 205         }
 206
 207         qat_pci_devs[qat_dev_id].mz = rte_memzone_reserve(name,
 208                 sizeof(struct qat_pci_device),
 209                 rte_socket_id(), 0);
 210
 211         if (qat_pci_devs[qat_dev_id].mz == NULL) {
 212                 QAT_LOG(ERR, "Error when allocating memzone for QAT_%d",
 213                         qat_dev_id);
 214                 return NULL;
 215         }
 216
 217         qat_dev = qat_pci_devs[qat_dev_id].mz->addr;
 218         memset(qat_dev, 0, sizeof(*qat_dev));
 219         strlcpy(qat_dev->name, name, QAT_DEV_NAME_MAX_LEN);
 220         qat_dev->qat_dev_id = qat_dev_id;
 221         qat_pci_devs[qat_dev_id].pci_dev = pci_dev;
 222         switch (pci_dev->id.device_id) {
 223         case 0x0443:
 224                 qat_dev->qat_dev_gen = QAT_GEN1;
 225                 break;
 226         case 0x37c9:
 227         case 0x19e3:
 228         case 0x6f55:
 229         case 0x18ef:
 230                 qat_dev->qat_dev_gen = QAT_GEN2;
 231                 break;
 232         case 0x18a1:
 233                 qat_dev->qat_dev_gen = QAT_GEN3;
 234                 break;
 235         default:
 236                 QAT_LOG(ERR, "Invalid dev_id, can't determine generation");
 237                 rte_memzone_free(qat_pci_devs[qat_dev->qat_dev_id].mz);
 238                 return NULL;
 239         }
 240
 241         if (devargs && devargs->drv_str)
 242                 qat_dev_parse_cmd(devargs->drv_str, qat_dev_cmd_param);
 243
 244         rte_spinlock_init(&qat_dev->arb_csr_lock);
 245         qat_nb_pci_devices++;
 246
 247         QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d",
 248                         qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices);
 249
 250         return qat_dev;
 251 }
 252
 253 static int
 254 qat_pci_device_release(struct rte_pci_device *pci_dev)
 255 {
 256         struct qat_pci_device *qat_dev;
 257         char name[QAT_DEV_NAME_MAX_LEN];
 258         int busy = 0;
 259
 260         if (pci_dev == NULL)
 261                 return -EINVAL;
 262
 263         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 264         snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat");
 265         qat_dev = qat_pci_get_named_dev(name);
 266         if (qat_dev != NULL) {
 267
 268                 struct qat_device_info *inst =
 269                                 &qat_pci_devs[qat_dev->qat_dev_id];
 270                 /* Check that there are no service devs still on pci device */
 271
 272                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 273                         if (qat_dev->sym_dev != NULL) {
 274                                 QAT_LOG(DEBUG, "QAT sym device %s is busy",
 275                                         name);
 276                                 busy = 1;
 277                         }
 278                         if (qat_dev->asym_dev != NULL) {
 279                                 QAT_LOG(DEBUG, "QAT asym device %s is busy",
 280                                         name);
 281                                 busy = 1;
 282                         }
 283                         if (qat_dev->comp_dev != NULL) {
 284                                 QAT_LOG(DEBUG, "QAT comp device %s is busy",
 285                                         name);
 286                                 busy = 1;
 287                         }
 288                         if (busy)
 289                                 return -EBUSY;
 290                         rte_memzone_free(inst->mz);
 291                 }
 292                 memset(inst, 0, sizeof(struct qat_device_info));
 293                 qat_nb_pci_devices--;
 294                 QAT_LOG(DEBUG, "QAT device %s released, total QATs %d",
 295                                         name, qat_nb_pci_devices);
 296         }
 297         return 0;
 298 }
 299
 300 static int
 301 qat_pci_dev_destroy(struct qat_pci_device *qat_pci_dev,
 302                 struct rte_pci_device *pci_dev)
 303 {
 304         qat_sym_dev_destroy(qat_pci_dev);
 305         qat_comp_dev_destroy(qat_pci_dev);
 306         qat_asym_dev_destroy(qat_pci_dev);
 307         return qat_pci_device_release(pci_dev);
 308 }
 309
 310 static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 311                 struct rte_pci_device *pci_dev)
 312 {
 313         int sym_ret = 0, asym_ret = 0, comp_ret = 0;
 314         int num_pmds_created = 0;
 315         struct qat_pci_device *qat_pci_dev;
 316         struct qat_dev_cmd_param qat_dev_cmd_param[] = {
 317                         { SYM_ENQ_THRESHOLD_NAME, 0 },
 318                         { ASYM_ENQ_THRESHOLD_NAME, 0 },
 319                         { COMP_ENQ_THRESHOLD_NAME, 0 },
 320                         { NULL, 0 },
 321         };
 322
 323         QAT_LOG(DEBUG, "Found QAT device at %02x:%02x.%x",
 324                         pci_dev->addr.bus,
 325                         pci_dev->addr.devid,
 326                         pci_dev->addr.function);
 327
 328         qat_pci_dev = qat_pci_device_allocate(pci_dev, qat_dev_cmd_param);
 329         if (qat_pci_dev == NULL)
 330                 return -ENODEV;
 331
 332         sym_ret = qat_sym_dev_create(qat_pci_dev, qat_dev_cmd_param);
 333         if (sym_ret == 0) {
 334                 num_pmds_created++;
 335
 336         }
 337         else
 338                 QAT_LOG(WARNING,
 339                                 "Failed to create QAT SYM PMD on device %s",
 340                                 qat_pci_dev->name);
 341
 342         comp_ret = qat_comp_dev_create(qat_pci_dev, qat_dev_cmd_param);
 343         if (comp_ret == 0)
 344                 num_pmds_created++;
 345         else
 346                 QAT_LOG(WARNING,
 347                                 "Failed to create QAT COMP PMD on device %s",
 348                                 qat_pci_dev->name);
 349
 350         asym_ret = qat_asym_dev_create(qat_pci_dev, qat_dev_cmd_param);
 351         if (asym_ret == 0)
 352                 num_pmds_created++;
 353         else
 354                 QAT_LOG(WARNING,
 355                                 "Failed to create QAT ASYM PMD on device %s",
 356                                 qat_pci_dev->name);
 357
 358         if (num_pmds_created == 0)
 359                 qat_pci_dev_destroy(qat_pci_dev, pci_dev);
 360
 361         return 0;
 362 }
 363
 364 static int qat_pci_remove(struct rte_pci_device *pci_dev)
 365 {
 366         struct qat_pci_device *qat_pci_dev;
 367
 368         if (pci_dev == NULL)
 369                 return -EINVAL;
 370
 371         qat_pci_dev = qat_get_qat_dev_from_pci_dev(pci_dev);
 372         if (qat_pci_dev == NULL)
 373                 return 0;
 374
 375         return qat_pci_dev_destroy(qat_pci_dev, pci_dev);
 376 }
 377
 378 static struct rte_pci_driver rte_qat_pmd = {
 379         .id_table = pci_id_qat_map,
 380         .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
 381         .probe = qat_pci_probe,
 382         .remove = qat_pci_remove
 383 };
 384
 385 __rte_weak int
 386 qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
 387                 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 388 {
 389         return 0;
 390 }
 391
 392 __rte_weak int
 393 qat_asym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
 394                 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 395 {
 396         return 0;
 397 }
 398
 399 __rte_weak int
 400 qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
 401 {
 402         return 0;
 403 }
 404
 405 __rte_weak int
 406 qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
 407 {
 408         return 0;
 409 }
 410
 411 __rte_weak int
 412 qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
 413                 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 414 {
 415         return 0;
 416 }
 417
 418 __rte_weak int
 419 qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
 420 {
 421         return 0;
 422 }
 423
 424 RTE_PMD_REGISTER_PCI(QAT_PCI_NAME, rte_qat_pmd);
 425 RTE_PMD_REGISTER_PCI_TABLE(QAT_PCI_NAME, pci_id_qat_map);