1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018-2020 Intel Corporation
5 #include <rte_string_fns.h>
6 #include <rte_devargs.h>
9 #include "qat_device.h"
10 #include "adf_transport_access_macros.h"
11 #include "qat_sym_pmd.h"
12 #include "qat_comp_pmd.h"
13 #include "adf_pf2vf_msg.h"
14 #include "qat_pf2vf.h"
16 /* Hardware device information per generation */
17 struct qat_gen_hw_data qat_gen_config[QAT_N_GENS];
18 struct qat_dev_hw_spec_funcs *qat_dev_hw_spec[QAT_N_GENS];
20 /* per-process array of device data */
21 struct qat_device_info qat_pci_devs[RTE_PMD_QAT_MAX_PCI_DEVICES];
22 static int qat_nb_pci_devices;
25 * The set of PCI devices this driver supports
28 static const struct rte_pci_id pci_id_qat_map[] = {
30 RTE_PCI_DEVICE(0x8086, 0x0443),
33 RTE_PCI_DEVICE(0x8086, 0x37c9),
36 RTE_PCI_DEVICE(0x8086, 0x19e3),
39 RTE_PCI_DEVICE(0x8086, 0x6f55),
42 RTE_PCI_DEVICE(0x8086, 0x18ef),
45 RTE_PCI_DEVICE(0x8086, 0x18a1),
48 RTE_PCI_DEVICE(0x8086, 0x4941),
54 qat_pci_get_extra_size(enum qat_device_gen qat_dev_gen)
56 struct qat_dev_hw_spec_funcs *ops_hw =
57 qat_dev_hw_spec[qat_dev_gen];
58 RTE_FUNC_PTR_OR_ERR_RET(ops_hw->qat_dev_get_extra_size,
60 return ops_hw->qat_dev_get_extra_size();
63 static struct qat_pci_device *
64 qat_pci_get_named_dev(const char *name)
71 for (i = 0; i < RTE_PMD_QAT_MAX_PCI_DEVICES; i++) {
72 if (qat_pci_devs[i].mz &&
73 (strcmp(((struct qat_pci_device *)
74 qat_pci_devs[i].mz->addr)->name, name)
76 return (struct qat_pci_device *)
77 qat_pci_devs[i].mz->addr;
84 qat_pci_find_free_device_index(void)
88 for (dev_id = 0; dev_id < RTE_PMD_QAT_MAX_PCI_DEVICES;
90 if (qat_pci_devs[dev_id].mz == NULL)
96 struct qat_pci_device *
97 qat_get_qat_dev_from_pci_dev(struct rte_pci_device *pci_dev)
99 char name[QAT_DEV_NAME_MAX_LEN];
101 rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
103 return qat_pci_get_named_dev(name);
107 qat_dev_parse_cmd(const char *str, struct qat_dev_cmd_param
114 char value_str[4] = { };
116 param = qat_dev_cmd_param[i].name;
120 const char *arg = strstr(str, param);
121 const char *arg2 = NULL;
124 arg2 = arg + strlen(param);
126 QAT_LOG(DEBUG, "parsing error '=' sign"
127 " should immediately follow %s",
133 QAT_LOG(DEBUG, "%s not provided", param);
138 if (!isdigit(*(arg2 + iter)))
143 QAT_LOG(DEBUG, "parsing error %s"
144 " no number provided",
147 memcpy(value_str, arg2, iter);
148 value = strtol(value_str, NULL, 10);
149 if (value > MAX_QP_THRESHOLD_SIZE) {
150 QAT_LOG(DEBUG, "Exceeded max size of"
151 " threshold, setting to %d",
152 MAX_QP_THRESHOLD_SIZE);
153 value = MAX_QP_THRESHOLD_SIZE;
155 QAT_LOG(DEBUG, "parsing %s = %ld",
159 qat_dev_cmd_param[i].val = value;
164 struct qat_pci_device *
165 qat_pci_device_allocate(struct rte_pci_device *pci_dev,
166 struct qat_dev_cmd_param *qat_dev_cmd_param)
168 struct qat_pci_device *qat_dev;
169 enum qat_device_gen qat_dev_gen;
170 uint8_t qat_dev_id = 0;
171 char name[QAT_DEV_NAME_MAX_LEN];
172 struct rte_devargs *devargs = pci_dev->device.devargs;
173 struct qat_dev_hw_spec_funcs *ops_hw;
174 struct rte_mem_resource *mem_resource;
175 const struct rte_memzone *qat_dev_mz;
176 int qat_dev_size, extra_size;
178 rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
179 snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat");
181 switch (pci_dev->id.device_id) {
183 qat_dev_gen = QAT_GEN1;
189 qat_dev_gen = QAT_GEN2;
192 qat_dev_gen = QAT_GEN3;
195 qat_dev_gen = QAT_GEN4;
198 QAT_LOG(ERR, "Invalid dev_id, can't determine generation");
202 if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
203 const struct rte_memzone *mz = rte_memzone_lookup(name);
207 "Secondary can't find %s mz, did primary create device?",
212 qat_pci_devs[qat_dev->qat_dev_id].mz = mz;
213 qat_pci_devs[qat_dev->qat_dev_id].pci_dev = pci_dev;
214 qat_nb_pci_devices++;
215 QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d",
216 qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices);
220 if (qat_pci_get_named_dev(name) != NULL) {
221 QAT_LOG(ERR, "QAT device with name %s already allocated!",
226 qat_dev_id = qat_pci_find_free_device_index();
227 if (qat_dev_id == RTE_PMD_QAT_MAX_PCI_DEVICES) {
228 QAT_LOG(ERR, "Reached maximum number of QAT devices");
232 extra_size = qat_pci_get_extra_size(qat_dev_gen);
233 if (extra_size < 0) {
234 QAT_LOG(ERR, "QAT internal error: no pci pointer for gen %d",
239 qat_dev_size = sizeof(struct qat_pci_device) + extra_size;
240 qat_dev_mz = rte_memzone_reserve(name, qat_dev_size,
243 if (qat_dev_mz == NULL) {
244 QAT_LOG(ERR, "Error when allocating memzone for QAT_%d",
249 qat_dev = qat_dev_mz->addr;
250 memset(qat_dev, 0, qat_dev_size);
251 qat_dev->dev_private = qat_dev + 1;
252 strlcpy(qat_dev->name, name, QAT_DEV_NAME_MAX_LEN);
253 qat_dev->qat_dev_id = qat_dev_id;
254 qat_pci_devs[qat_dev_id].pci_dev = pci_dev;
255 qat_dev->qat_dev_gen = qat_dev_gen;
257 ops_hw = qat_dev_hw_spec[qat_dev->qat_dev_gen];
258 if (ops_hw->qat_dev_get_misc_bar == NULL) {
259 QAT_LOG(ERR, "qat_dev_get_misc_bar function pointer not set");
260 rte_memzone_free(qat_dev_mz);
263 if (ops_hw->qat_dev_get_misc_bar(&mem_resource, pci_dev) == 0) {
264 if (mem_resource->addr == NULL) {
265 QAT_LOG(ERR, "QAT cannot get access to VF misc bar");
266 rte_memzone_free(qat_dev_mz);
269 qat_dev->misc_bar_io_addr = mem_resource->addr;
271 qat_dev->misc_bar_io_addr = NULL;
273 if (devargs && devargs->drv_str)
274 qat_dev_parse_cmd(devargs->drv_str, qat_dev_cmd_param);
276 if (qat_read_qp_config(qat_dev)) {
278 "Cannot acquire ring configuration for QAT_%d",
280 rte_memzone_free(qat_dev_mz);
284 /* No errors when allocating, attach memzone with
285 * qat_dev to list of devices
287 qat_pci_devs[qat_dev_id].mz = qat_dev_mz;
289 rte_spinlock_init(&qat_dev->arb_csr_lock);
290 qat_nb_pci_devices++;
292 QAT_LOG(DEBUG, "QAT device %d found, name %s, total QATs %d",
293 qat_dev->qat_dev_id, qat_dev->name, qat_nb_pci_devices);
299 qat_pci_device_release(struct rte_pci_device *pci_dev)
301 struct qat_pci_device *qat_dev;
302 char name[QAT_DEV_NAME_MAX_LEN];
308 rte_pci_device_name(&pci_dev->addr, name, sizeof(name));
309 snprintf(name+strlen(name), QAT_DEV_NAME_MAX_LEN-strlen(name), "_qat");
310 qat_dev = qat_pci_get_named_dev(name);
311 if (qat_dev != NULL) {
313 struct qat_device_info *inst =
314 &qat_pci_devs[qat_dev->qat_dev_id];
315 /* Check that there are no service devs still on pci device */
317 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
318 if (qat_dev->sym_dev != NULL) {
319 QAT_LOG(DEBUG, "QAT sym device %s is busy",
323 if (qat_dev->asym_dev != NULL) {
324 QAT_LOG(DEBUG, "QAT asym device %s is busy",
328 if (qat_dev->comp_dev != NULL) {
329 QAT_LOG(DEBUG, "QAT comp device %s is busy",
335 rte_memzone_free(inst->mz);
337 memset(inst, 0, sizeof(struct qat_device_info));
338 qat_nb_pci_devices--;
339 QAT_LOG(DEBUG, "QAT device %s released, total QATs %d",
340 name, qat_nb_pci_devices);
346 qat_pci_dev_destroy(struct qat_pci_device *qat_pci_dev,
347 struct rte_pci_device *pci_dev)
349 qat_sym_dev_destroy(qat_pci_dev);
350 qat_comp_dev_destroy(qat_pci_dev);
351 qat_asym_dev_destroy(qat_pci_dev);
352 return qat_pci_device_release(pci_dev);
355 static int qat_pci_probe(struct rte_pci_driver *pci_drv __rte_unused,
356 struct rte_pci_device *pci_dev)
358 int sym_ret = 0, asym_ret = 0, comp_ret = 0;
359 int num_pmds_created = 0;
360 struct qat_pci_device *qat_pci_dev;
361 struct qat_dev_hw_spec_funcs *ops_hw;
362 struct qat_dev_cmd_param qat_dev_cmd_param[] = {
363 { SYM_ENQ_THRESHOLD_NAME, 0 },
364 { ASYM_ENQ_THRESHOLD_NAME, 0 },
365 { COMP_ENQ_THRESHOLD_NAME, 0 },
369 QAT_LOG(DEBUG, "Found QAT device at %02x:%02x.%x",
372 pci_dev->addr.function);
374 qat_pci_dev = qat_pci_device_allocate(pci_dev, qat_dev_cmd_param);
375 if (qat_pci_dev == NULL)
378 ops_hw = qat_dev_hw_spec[qat_pci_dev->qat_dev_gen];
379 RTE_FUNC_PTR_OR_ERR_RET(ops_hw->qat_dev_reset_ring_pairs,
381 if (ops_hw->qat_dev_reset_ring_pairs(qat_pci_dev)) {
383 "Cannot reset ring pairs, does pf driver supports pf2vf comms?"
388 sym_ret = qat_sym_dev_create(qat_pci_dev, qat_dev_cmd_param);
395 "Failed to create QAT SYM PMD on device %s",
398 comp_ret = qat_comp_dev_create(qat_pci_dev, qat_dev_cmd_param);
403 "Failed to create QAT COMP PMD on device %s",
406 asym_ret = qat_asym_dev_create(qat_pci_dev, qat_dev_cmd_param);
411 "Failed to create QAT ASYM PMD on device %s",
414 if (num_pmds_created == 0)
415 qat_pci_dev_destroy(qat_pci_dev, pci_dev);
421 qat_pci_remove(struct rte_pci_device *pci_dev)
423 struct qat_pci_device *qat_pci_dev;
428 qat_pci_dev = qat_get_qat_dev_from_pci_dev(pci_dev);
429 if (qat_pci_dev == NULL)
432 return qat_pci_dev_destroy(qat_pci_dev, pci_dev);
435 static struct rte_pci_driver rte_qat_pmd = {
436 .id_table = pci_id_qat_map,
437 .drv_flags = RTE_PCI_DRV_NEED_MAPPING,
438 .probe = qat_pci_probe,
439 .remove = qat_pci_remove
443 qat_sym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
444 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
450 qat_asym_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
451 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
457 qat_sym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
463 qat_asym_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
469 qat_comp_dev_create(struct qat_pci_device *qat_pci_dev __rte_unused,
470 struct qat_dev_cmd_param *qat_dev_cmd_param __rte_unused)
476 qat_comp_dev_destroy(struct qat_pci_device *qat_pci_dev __rte_unused)
481 RTE_PMD_REGISTER_PCI(QAT_PCI_NAME, rte_qat_pmd);
482 RTE_PMD_REGISTER_PCI_TABLE(QAT_PCI_NAME, pci_id_qat_map);
483 RTE_PMD_REGISTER_KMOD_DEP(QAT_PCI_NAME, "* igb_uio | uio_pci_generic | vfio-pci");