drivers/common/qat/qat_device.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2018-2020 Intel Corporation
   3  */
   4
   5 #include <rte_string_fns.h>
   6 #include <rte_devargs.h>
   7 #include <ctype.h>
   8
   9 #include "qat_device.h"
  10 #include "adf_transport_access_macros.h"
  11 #include "qat_sym_pmd.h"
  12 #include "qat_comp_pmd.h"
  13 #include "adf_pf2vf_msg.h"
  14 #include "qat_pf2vf.h"
  15
  16 /* pv2vf data Gen 4*/
  17 struct qat_pf2vf_dev qat_pf2vf_gen4 = {
  18         .pf2vf_offset = ADF_4XXXIOV_PF2VM_OFFSET,
  19         .vf2pf_offset = ADF_4XXXIOV_VM2PF_OFFSET,
  20         .pf2vf_type_shift = ADF_PFVF_2X_MSGTYPE_SHIFT,
  21         .pf2vf_type_mask = ADF_PFVF_2X_MSGTYPE_MASK,
  22         .pf2vf_data_shift = ADF_PFVF_2X_MSGDATA_SHIFT,
  23         .pf2vf_data_mask = ADF_PFVF_2X_MSGDATA_MASK,
  24 };
  25
  26 /* Hardware device information per generation */
  27 __extension__
  28 struct qat_gen_hw_data qat_gen_config[] =  {
  29         [QAT_GEN1] = {
  30                 .dev_gen = QAT_GEN1,
  31                 .qp_hw_data = qat_gen1_qps,
  32                 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN1
  33         },
  34         [QAT_GEN2] = {
  35                 .dev_gen = QAT_GEN2,
  36                 .qp_hw_data = qat_gen1_qps,
  37                 /* gen2 has same ring layout as gen1 */
  38                 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN2
  39         },
  40         [QAT_GEN3] = {
  41                 .dev_gen = QAT_GEN3,
  42                 .qp_hw_data = qat_gen3_qps,
  43                 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3
  44         },
  45         [QAT_GEN4] = {
  46                 .dev_gen = QAT_GEN4,
  47                 .qp_hw_data = NULL,
  48                 .comp_num_im_bufs_required = QAT_NUM_INTERM_BUFS_GEN3,
  49                 .pf2vf_dev = &qat_pf2vf_gen4
  50         },
  51 };
  52
  53 /* per-process array of device data */
  54 struct qat_device_info qat_pci_devs[RTE_PMD_QAT_MAX_PCI_DEVICES];
  55 static int qat_nb_pci_devices;
  56
  57 /*
  58  * The set of PCI devices this driver supports
  59  */
  60
  61 static const struct rte_pci_id pci_id_qat_map[] = {
  62                 {
  63                         RTE_PCI_DEVICE(0x8086, 0x0443),
  64                 },
  65                 {
  66                         RTE_PCI_DEVICE(0x8086, 0x37c9),
  67                 },
  68                 {
  69                         RTE_PCI_DEVICE(0x8086, 0x19e3),
  70                 },
  71                 {
  72                         RTE_PCI_DEVICE(0x8086, 0x6f55),
  73                 },
  74                 {
  75                         RTE_PCI_DEVICE(0x8086, 0x18ef),
  76                 },
  77                 {
  78                         RTE_PCI_DEVICE(0x8086, 0x18a1),
  79                 },
  80                 {
  81                         RTE_PCI_DEVICE(0x8086, 0x4941),
  82                 },
  83                 {.device_id = 0},
  84 };
  85
  86 static struct qat_pci_device *
  87 qat_pci_get_named_dev(const char *name)
  88 {
  89         unsigned int i;
  90
  91         if (name == NULL)
  92                 return NULL;
  93
  94         for (i = 0; i < RTE_PMD_QAT_MAX_PCI_DEVICES; i++) {
  95                 if (qat_pci_devs[i].mz &&
  96                                 (strcmp(((struct qat_pci_device *)
  97                                 qat_pci_devs[i].mz->addr)->name, name)
  98                                 == 0))
  99                         return (struct qat_pci_device *)
 100                                 qat_pci_devs[i].mz->addr;
 101         }
 102
 103         return NULL;
 104 }
 105
 106 static uint8_t
 107 qat_pci_find_free_device_index(void)
 108 {
 109                 uint8_t dev_id;
 110
 111                 for (dev_id = 0; dev_id < RTE_PMD_QAT_MAX_PCI_DEVICES;
 112                                 dev_id++) {
 113                         if (qat_pci_devs[dev_id].mz == NULL)
 114                                 break;
 115                 }
 116                 return dev_id;
 117 }
 118
 119 struct qat_pci_device *
 120 qat_get_qat_dev_from_pci_dev(struct rte_pci_device *pci_dev)
 121 {
 122         char name[QAT_DEV_NAME_MAX_LEN];
 123
 124         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 125
 126         return qat_pci_get_named_dev(name);
 127 }
 128
 129 static int
 130 qat_gen4_reset_ring_pair(struct qat_pci_device *qat_pci_dev)
 131 {
 132         int ret = 0, i;
 133         uint8_t data[4];
 134         struct qat_pf2vf_msg pf2vf_msg;
 135
 136         pf2vf_msg.msg_type = ADF_VF2PF_MSGTYPE_RP_RESET;
 137         pf2vf_msg.block_hdr = -1;
 138         for (i = 0; i < QAT_GEN4_BUNDLE_NUM; i++) {
 139                 pf2vf_msg.msg_data = i;
 140                 ret = qat_pf2vf_exch_msg(qat_pci_dev, pf2vf_msg, 1, data);
 141                 if (ret) {
 142                         QAT_LOG(ERR, "QAT error when reset bundle no %d",
 143                                 i);
 144                         return ret;
 145                 }
 146         }
 147
 148         return 0;
 149 }
 150
 151 int qat_query_svc(struct qat_pci_device *qat_dev, uint8_t *val)
 152 {
 153         int ret = -(EINVAL);
 154         struct qat_pf2vf_msg pf2vf_msg;
 155
 156         if (qat_dev->qat_dev_gen == QAT_GEN4) {
 157                 pf2vf_msg.msg_type = ADF_VF2PF_MSGTYPE_GET_SMALL_BLOCK_REQ;
 158                 pf2vf_msg.block_hdr = ADF_VF2PF_BLOCK_MSG_GET_RING_TO_SVC_REQ;
 159                 pf2vf_msg.msg_data = 2;
 160                 ret = qat_pf2vf_exch_msg(qat_dev, pf2vf_msg, 2, val);
 161         }
 162
 163         return ret;
 164 }
 165
 166
 167 static void qat_dev_parse_cmd(const char *str, struct qat_dev_cmd_param
 168                 *qat_dev_cmd_param)
 169 {
 170         int i = 0;
 171         const char *param;
 172
 173         while (1) {
 174                 char value_str[4] = { };
 175
 176                 param = qat_dev_cmd_param[i].name;
 177                 if (param == NULL)
 178                         return;
 179                 long value = 0;
 180                 const char *arg = strstr(str, param);
 181                 const char *arg2 = NULL;
 182
 183                 if (arg) {
 184                         arg2 = arg + strlen(param);
 185                         if (*arg2 != '=') {
 186                                 QAT_LOG(DEBUG, "parsing error '=' sign"
 187                                                 " should immediately follow %s",
 188                                                 param);
 189                                 arg2 = NULL;
 190                         } else
 191                                 arg2++;
 192                 } else {
 193                         QAT_LOG(DEBUG, "%s not provided", param);
 194                 }
 195                 if (arg2) {
 196                         int iter = 0;
 197                         while (iter < 2) {
 198                                 if (!isdigit(*(arg2 + iter)))
 199                                         break;
 200                                 iter++;
 201                         }
 202                         if (!iter) {
 203                                 QAT_LOG(DEBUG, "parsing error %s"
 204                                                " no number provided",
 205                                                param);
 206                         } else {
 207                                 memcpy(value_str, arg2, iter);
 208                                 value = strtol(value_str, NULL, 10);
 209                                 if (value > MAX_QP_THRESHOLD_SIZE) {
 210                                         QAT_LOG(DEBUG, "Exceeded max size of"
 211                                                 " threshold, setting to %d",
 212                                                 MAX_QP_THRESHOLD_SIZE);
 213                                         value = MAX_QP_THRESHOLD_SIZE;
 214                                 }
 215                                 QAT_LOG(DEBUG, "parsing %s = %ld",
 216                                                 param, value);
 217                         }
 218                 }
 219                 qat_dev_cmd_param[i].val = value;
 220                 i++;
 221         }
 222 }
 223
 224 struct qat_pci_device *
 225 qat_pci_device_allocate(struct rte_pci_device *pci_dev,
 226                 struct qat_dev_cmd_param *qat_dev_cmd_param)
 227 {
 228         struct qat_pci_device *qat_dev;
 229         uint8_t qat_dev_id = 0;
 230         char name[QAT_DEV_NAME_MAX_LEN];
 231         struct rte_devargs *devargs = pci_dev->device.devargs;
 232
 233         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 234         snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat");
 235
 236         if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
 237                 const struct rte_memzone *mz = rte_memzone_lookup(name);
 238
 239                 if (mz == NULL) {
 240                         QAT_LOG(ERR,
 241                                 "Secondary can't find %s mz, did primary create device?",
 242                                 name);
 243                         return NULL;
 244                 }
 245                 qat_dev = mz->addr;
 246                 qat_pci_devs[qat_dev->qat_dev_id].mz = mz;
 247                 qat_pci_devs[qat_dev->qat_dev_id].pci_dev = pci_dev;
 248                 qat_nb_pci_devices++;
 249                 QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d",
 250                         qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices);
 251                 return qat_dev;
 252         }
 253
 254         if (qat_pci_get_named_dev(name) != NULL) {
 255                 QAT_LOG(ERR, "QAT device with name %s already allocated!",
 256                                 name);
 257                 return NULL;
 258         }
 259
 260         qat_dev_id = qat_pci_find_free_device_index();
 261         if (qat_dev_id == RTE_PMD_QAT_MAX_PCI_DEVICES) {
 262                 QAT_LOG(ERR, "Reached maximum number of QAT devices");
 263                 return NULL;
 264         }
 265
 266         qat_pci_devs[qat_dev_id].mz = rte_memzone_reserve(name,
 267                 sizeof(struct qat_pci_device),
 268                 rte_socket_id(), 0);
 269
 270         if (qat_pci_devs[qat_dev_id].mz == NULL) {
 271                 QAT_LOG(ERR, "Error when allocating memzone for QAT_%d",
 272                         qat_dev_id);
 273                 return NULL;
 274         }
 275
 276         qat_dev = qat_pci_devs[qat_dev_id].mz->addr;
 277         memset(qat_dev, 0, sizeof(*qat_dev));
 278         strlcpy(qat_dev->name, name, QAT_DEV_NAME_MAX_LEN);
 279         qat_dev->qat_dev_id = qat_dev_id;
 280         qat_pci_devs[qat_dev_id].pci_dev = pci_dev;
 281         switch (pci_dev->id.device_id) {
 282         case 0x0443:
 283                 qat_dev->qat_dev_gen = QAT_GEN1;
 284                 break;
 285         case 0x37c9:
 286         case 0x19e3:
 287         case 0x6f55:
 288         case 0x18ef:
 289                 qat_dev->qat_dev_gen = QAT_GEN2;
 290                 break;
 291         case 0x18a1:
 292                 qat_dev->qat_dev_gen = QAT_GEN3;
 293                 break;
 294         case 0x4941:
 295                 qat_dev->qat_dev_gen = QAT_GEN4;
 296                 break;
 297         default:
 298                 QAT_LOG(ERR, "Invalid dev_id, can't determine generation");
 299                 rte_memzone_free(qat_pci_devs[qat_dev->qat_dev_id].mz);
 300                 return NULL;
 301         }
 302
 303         if (qat_dev->qat_dev_gen == QAT_GEN4) {
 304                 qat_dev->misc_bar_io_addr = pci_dev->mem_resource[2].addr;
 305                 if (qat_dev->misc_bar_io_addr == NULL) {
 306                         QAT_LOG(ERR, "QAT cannot get access to VF misc bar");
 307                         return NULL;
 308                 }
 309         }
 310
 311         if (devargs && devargs->drv_str)
 312                 qat_dev_parse_cmd(devargs->drv_str, qat_dev_cmd_param);
 313
 314         if (qat_dev->qat_dev_gen >= QAT_GEN4) {
 315                 if (qat_read_qp_config(qat_dev)) {
 316                         QAT_LOG(ERR,
 317                                 "Cannot acquire ring configuration for QAT_%d",
 318                                 qat_dev_id);
 319                         return NULL;
 320                 }
 321         }
 322
 323         rte_spinlock_init(&qat_dev->arb_csr_lock);
 324         qat_nb_pci_devices++;
 325
 326         QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d",
 327                         qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices);
 328
 329         return qat_dev;
 330 }
 331
 332 static int
 333 qat_pci_device_release(struct rte_pci_device *pci_dev)
 334 {
 335         struct qat_pci_device *qat_dev;
 336         char name[QAT_DEV_NAME_MAX_LEN];
 337         int busy = 0;
 338
 339         if (pci_dev == NULL)
 340                 return -EINVAL;
 341
 342         rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
 343         snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat");
 344         qat_dev = qat_pci_get_named_dev(name);
 345         if (qat_dev != NULL) {
 346
 347                 struct qat_device_info *inst =
 348                                 &qat_pci_devs[qat_dev->qat_dev_id];
 349                 /* Check that there are no service devs still on pci device */
 350
 351                 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 352                         if (qat_dev->sym_dev != NULL) {
 353                                 QAT_LOG(DEBUG, "QAT sym device %s is busy",
 354                                         name);
 355                                 busy = 1;
 356                         }
 357                         if (qat_dev->asym_dev != NULL) {
 358                                 QAT_LOG(DEBUG, "QAT asym device %s is busy",
 359                                         name);
 360                                 busy = 1;
 361                         }
 362                         if (qat_dev->comp_dev != NULL) {
 363                                 QAT_LOG(DEBUG, "QAT comp device %s is busy",
 364                                         name);
 365                                 busy = 1;
 366                         }
 367                         if (busy)
 368                                 return -EBUSY;
 369                         rte_memzone_free(inst->mz);
 370                 }
 371                 memset(inst, 0, sizeof(struct qat_device_info));
 372                 qat_nb_pci_devices--;
 373                 QAT_LOG(DEBUG, "QAT device %s released, total QATs %d",
 374                                         name, qat_nb_pci_devices);
 375         }
 376         return 0;
 377 }
 378
 379 static int
 380 qat_pci_dev_destroy(struct qat_pci_device *qat_pci_dev,
 381                 struct rte_pci_device *pci_dev)
 382 {
 383         qat_sym_dev_destroy(qat_pci_dev);
 384         qat_comp_dev_destroy(qat_pci_dev);
 385         qat_asym_dev_destroy(qat_pci_dev);
 386         return qat_pci_device_release(pci_dev);
 387 }
 388
 389 static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
 390                 struct rte_pci_device *pci_dev)
 391 {
 392         int sym_ret = 0, asym_ret = 0, comp_ret = 0;
 393         int num_pmds_created = 0;
 394         struct qat_pci_device *qat_pci_dev;
 395         struct qat_dev_cmd_param qat_dev_cmd_param[] = {
 396                         { SYM_ENQ_THRESHOLD_NAME, 0 },
 397                         { ASYM_ENQ_THRESHOLD_NAME, 0 },
 398                         { COMP_ENQ_THRESHOLD_NAME, 0 },
 399                         { NULL, 0 },
 400         };
 401
 402         QAT_LOG(DEBUG, "Found QAT device at %02x:%02x.%x",
 403                         pci_dev->addr.bus,
 404                         pci_dev->addr.devid,
 405                         pci_dev->addr.function);
 406
 407         qat_pci_dev = qat_pci_device_allocate(pci_dev, qat_dev_cmd_param);
 408         if (qat_pci_dev == NULL)
 409                 return -ENODEV;
 410
 411         if (qat_pci_dev->qat_dev_gen == QAT_GEN4) {
 412                 if (qat_gen4_reset_ring_pair(qat_pci_dev)) {
 413                         QAT_LOG(ERR,
 414                                 "Cannot reset ring pairs, does pf driver supports pf2vf comms?"
 415                                 );
 416                         return -ENODEV;
 417                 }
 418         }
 419
 420         sym_ret = qat_sym_dev_create(qat_pci_dev, qat_dev_cmd_param);
 421         if (sym_ret == 0) {
 422                 num_pmds_created++;
 423
 424         }
 425         else
 426                 QAT_LOG(WARNING,
 427                                 "Failed to create QAT SYM PMD on device %s",
 428                                 qat_pci_dev->name);
 429
 430         comp_ret = qat_comp_dev_create(qat_pci_dev, qat_dev_cmd_param);
 431         if (comp_ret == 0)
 432                 num_pmds_created++;
 433         else
 434                 QAT_LOG(WARNING,
 435                                 "Failed to create QAT COMP PMD on device %s",
 436                                 qat_pci_dev->name);
 437
 438         asym_ret = qat_asym_dev_create(qat_pci_dev, qat_dev_cmd_param);
 439         if (asym_ret == 0)
 440                 num_pmds_created++;
 441         else
 442                 QAT_LOG(WARNING,
 443                                 "Failed to create QAT ASYM PMD on device %s",
 444                                 qat_pci_dev->name);
 445
 446         if (num_pmds_created == 0)
 447                 qat_pci_dev_destroy(qat_pci_dev, pci_dev);
 448
 449         return 0;
 450 }
 451
 452 static int qat_pci_remove(struct rte_pci_device *pci_dev)
 453 {
 454         struct qat_pci_device *qat_pci_dev;
 455
 456         if (pci_dev == NULL)
 457                 return -EINVAL;
 458
 459         qat_pci_dev = qat_get_qat_dev_from_pci_dev(pci_dev);
 460         if (qat_pci_dev == NULL)
 461                 return 0;
 462
 463         return qat_pci_dev_destroy(qat_pci_dev, pci_dev);
 464 }
 465
 466 static struct rte_pci_driver rte_qat_pmd = {
 467         .id_table = pci_id_qat_map,
 468         .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
 469         .probe = qat_pci_probe,
 470         .remove = qat_pci_remove
 471 };
 472
 473 __rte_weak int
 474 qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
 475                 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 476 {
 477         return 0;
 478 }
 479
 480 __rte_weak int
 481 qat_asym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
 482                 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 483 {
 484         return 0;
 485 }
 486
 487 __rte_weak int
 488 qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
 489 {
 490         return 0;
 491 }
 492
 493 __rte_weak int
 494 qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
 495 {
 496         return 0;
 497 }
 498
 499 __rte_weak int
 500 qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
 501                 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
 502 {
 503         return 0;
 504 }
 505
 506 __rte_weak int
 507 qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
 508 {
 509         return 0;
 510 }
 511
 512 RTE_PMD_REGISTER_PCI(QAT_PCI_NAME, rte_qat_pmd);
 513 RTE_PMD_REGISTER_PCI_TABLE(QAT_PCI_NAME, pci_id_qat_map);
 514 RTE_PMD_REGISTER_KMOD_DEP(QAT_PCI_NAME, "* igb_uio | uio_pci_generic | vfio-pci");