bus/pci: support Windows with bifurcated drivers
authorTal Shnaiderman <talshn@mellanox.com>
Mon, 29 Jun 2020 12:37:40 +0000 (15:37 +0300)
committerThomas Monjalon <thomas@monjalon.net>
Mon, 29 Jun 2020 22:02:54 +0000 (00:02 +0200)
Uses SetupAPI.h functions to scan PCI tree.
Uses DEVPKEY_Device_Numa_Node to get the PCI NUMA node.
Uses SPDRP_BUSNUMBER and SPDRP_BUSNUMBER to get the BDF.
scanning currently supports types RTE_KDRV_NONE.

Signed-off-by: Tal Shnaiderman <talshn@mellanox.com>
drivers/bus/pci/windows/pci.c
lib/librte_eal/rte_eal_exports.def
lib/librte_eal/windows/include/rte_windows.h

index b1d34ae..489aa79 100644 (file)
@@ -1,14 +1,27 @@
 /* SPDX-License-Identifier: BSD-3-Clause
  * Copyright 2020 Mellanox Technologies, Ltd
  */
+#include <rte_windows.h>
 #include <rte_errno.h>
 #include <rte_log.h>
-
-#include <rte_string_fns.h>
 #include <rte_eal_memconfig.h>
+#include <rte_eal.h>
 
 #include "private.h"
 
+#include <devpkey.h>
+
+#ifdef RTE_TOOLCHAIN_GCC
+#include <devpropdef.h>
+DEFINE_DEVPROPKEY(DEVPKEY_Device_Numa_Node, 0x540b947e, 0x8b40, 0x45bc,
+       0xa8, 0xa2, 0x6a, 0x0b, 0x89, 0x4c, 0xbd, 0xa2, 3);
+#endif
+
+/*
+ * This code is used to simulate a PCI probe by parsing information in
+ * the registry hive for PCI devices.
+ */
+
 /* The functions below are not implemented on Windows,
  * but need to be defined for compilation purposes
  */
@@ -146,7 +159,6 @@ rte_pci_ioport_write(struct rte_pci_ioport *p __rte_unused,
         */
 }
 
-
 /* remap the PCI resource of a PCI device in anonymous virtual memory */
 int
 pci_uio_remap_resource(struct rte_pci_device *dev __rte_unused)
@@ -158,6 +170,210 @@ pci_uio_remap_resource(struct rte_pci_device *dev __rte_unused)
         */
        return -1;
 }
+
+static int
+get_device_pci_address(HDEVINFO dev_info,
+       PSP_DEVINFO_DATA device_info_data, struct rte_pci_addr *addr)
+{
+       BOOL  res;
+       ULONG bus_num, dev_and_func;
+
+       res = SetupDiGetDeviceRegistryProperty(dev_info, device_info_data,
+               SPDRP_BUSNUMBER, NULL, (PBYTE)&bus_num, sizeof(bus_num), NULL);
+       if (!res) {
+               RTE_LOG_WIN32_ERR(
+                       "SetupDiGetDeviceRegistryProperty(SPDRP_BUSNUMBER)");
+               return -1;
+       }
+
+       res = SetupDiGetDeviceRegistryProperty(dev_info, device_info_data,
+               SPDRP_ADDRESS, NULL, (PBYTE)&dev_and_func, sizeof(dev_and_func),
+               NULL);
+       if (!res) {
+               RTE_LOG_WIN32_ERR(
+                       "SetupDiGetDeviceRegistryProperty(SPDRP_ADDRESS)");
+               return -1;
+       }
+
+       addr->domain = 0;
+       addr->bus = bus_num;
+       addr->devid = dev_and_func >> 16;
+       addr->function = dev_and_func & 0xffff;
+       return 0;
+}
+
+static int
+get_device_resource_info(HDEVINFO dev_info,
+       PSP_DEVINFO_DATA dev_info_data, struct rte_pci_device *dev)
+{
+       DEVPROPTYPE property_type;
+       DWORD numa_node;
+       BOOL  res;
+
+       switch (dev->kdrv) {
+       case RTE_KDRV_NONE:
+               /* Get NUMA node using DEVPKEY_Device_Numa_Node */
+               res = SetupDiGetDevicePropertyW(dev_info, dev_info_data,
+                       &DEVPKEY_Device_Numa_Node, &property_type,
+                       (BYTE *)&numa_node, sizeof(numa_node), NULL, 0);
+               if (!res) {
+                       RTE_LOG_WIN32_ERR(
+                               "SetupDiGetDevicePropertyW"
+                               "(DEVPKEY_Device_Numa_Node)");
+                       return -1;
+               }
+               dev->device.numa_node = numa_node;
+               /* mem_resource - Unneeded for RTE_KDRV_NONE */
+               dev->mem_resource[0].phys_addr = 0;
+               dev->mem_resource[0].len = 0;
+               dev->mem_resource[0].addr = NULL;
+               break;
+       default:
+               /* kernel driver type is unsupported */
+               RTE_LOG(DEBUG, EAL,
+                       "Kernel driver type for PCI device " PCI_PRI_FMT ","
+                       " is unsupported",
+                       dev->addr.domain, dev->addr.bus,
+                       dev->addr.devid, dev->addr.function);
+               return -1;
+       }
+
+       return ERROR_SUCCESS;
+}
+
+/*
+ * get string that contains the list of hardware IDs for a device
+ */
+static int
+get_pci_hardware_id(HDEVINFO dev_info, PSP_DEVINFO_DATA device_info_data,
+       char *pci_device_info, size_t pci_device_info_len)
+{
+       BOOL  res;
+
+       /* Retrieve PCI device IDs */
+       res = SetupDiGetDeviceRegistryPropertyA(dev_info, device_info_data,
+                       SPDRP_HARDWAREID, NULL, (BYTE *)pci_device_info,
+                       pci_device_info_len, NULL);
+       if (!res) {
+               RTE_LOG_WIN32_ERR(
+                       "SetupDiGetDeviceRegistryPropertyA(SPDRP_HARDWAREID)");
+               return -1;
+       }
+
+       return 0;
+}
+
+/*
+ * parse the SPDRP_HARDWAREID output and assign to rte_pci_id
+ */
+static int
+parse_pci_hardware_id(const char *buf, struct rte_pci_id *pci_id)
+{
+       int ids = 0;
+       uint16_t vendor_id, device_id, subvendor_id = 0;
+
+       ids = sscanf_s(buf, "PCI\\VEN_%x&DEV_%x&SUBSYS_%x", &vendor_id,
+               &device_id, &subvendor_id);
+       if (ids != 3)
+               return -1;
+
+       pci_id->vendor_id = vendor_id;
+       pci_id->device_id = device_id;
+       pci_id->subsystem_vendor_id = subvendor_id >> 16;
+       pci_id->subsystem_device_id = subvendor_id & 0xffff;
+       return 0;
+}
+
+static void
+get_kernel_driver_type(struct rte_pci_device *dev)
+{
+       /*
+        * If another kernel driver is supported the relevant checking
+        * functions should be here
+        */
+       dev->kdrv = RTE_KDRV_NONE;
+}
+
+static int
+pci_scan_one(HDEVINFO dev_info, PSP_DEVINFO_DATA device_info_data)
+{
+       struct rte_pci_device *dev;
+       int ret = -1;
+       char  pci_device_info[PATH_MAX];
+       struct rte_pci_addr addr;
+       struct rte_pci_id pci_id;
+
+       dev = malloc(sizeof(*dev));
+       if (dev == NULL)
+               goto end;
+
+       memset(dev, 0, sizeof(*dev));
+
+       ret = get_pci_hardware_id(dev_info, device_info_data,
+               pci_device_info, PATH_MAX);
+       if (ret != 0)
+               goto end;
+
+       ret = parse_pci_hardware_id((const char *)&pci_device_info, &pci_id);
+       if (ret != 0) {
+               /*
+                * We won't add this device, but we want to continue
+                * looking for supported devices
+                */
+               ret = ERROR_CONTINUE;
+               goto end;
+       }
+
+       ret = get_device_pci_address(dev_info, device_info_data, &addr);
+       if (ret != 0)
+               goto end;
+
+       dev->addr = addr;
+       dev->id = pci_id;
+       dev->max_vfs = 0; /* TODO: get max_vfs */
+
+       pci_name_set(dev);
+
+       get_kernel_driver_type(dev);
+
+       /* get resources */
+       if (get_device_resource_info(dev_info, device_info_data, dev)
+                       != ERROR_SUCCESS) {
+               goto end;
+       }
+
+       /* device is valid, add in list (sorted) */
+       if (TAILQ_EMPTY(&rte_pci_bus.device_list)) {
+               rte_pci_add_device(dev);
+       } else {
+               struct rte_pci_device *dev2 = NULL;
+               int ret;
+
+               TAILQ_FOREACH(dev2, &rte_pci_bus.device_list, next) {
+                       ret = rte_pci_addr_cmp(&dev->addr, &dev2->addr);
+                       if (ret > 0) {
+                               continue;
+                       } else if (ret < 0) {
+                               rte_pci_insert_device(dev2, dev);
+                       } else { /* already registered */
+                               dev2->kdrv = dev->kdrv;
+                               dev2->max_vfs = dev->max_vfs;
+                               memmove(dev2->mem_resource, dev->mem_resource,
+                                       sizeof(dev->mem_resource));
+                               free(dev);
+                       }
+                       return 0;
+               }
+               rte_pci_add_device(dev);
+       }
+
+       return 0;
+end:
+       if (dev)
+               free(dev);
+       return ret;
+}
+
 /*
  * Scan the contents of the PCI bus
  * and add all network class devices into the devices list.
@@ -165,5 +381,44 @@ pci_uio_remap_resource(struct rte_pci_device *dev __rte_unused)
 int
 rte_pci_scan(void)
 {
-       return 0;
+       int   ret = -1;
+       DWORD device_index = 0, found_device = 0;
+       HDEVINFO dev_info;
+       SP_DEVINFO_DATA device_info_data;
+
+       /* for debug purposes, PCI can be disabled */
+       if (!rte_eal_has_pci())
+               return 0;
+
+       dev_info = SetupDiGetClassDevs(&GUID_DEVCLASS_NET, TEXT("PCI"), NULL,
+                               DIGCF_PRESENT);
+       if (dev_info == INVALID_HANDLE_VALUE) {
+               RTE_LOG_WIN32_ERR("SetupDiGetClassDevs(pci_scan)");
+               RTE_LOG(ERR, EAL, "Unable to enumerate PCI devices.\n");
+               goto end;
+       }
+
+       device_info_data.cbSize = sizeof(SP_DEVINFO_DATA);
+       device_index = 0;
+
+       while (SetupDiEnumDeviceInfo(dev_info, device_index,
+           &device_info_data)) {
+               device_index++;
+               ret = pci_scan_one(dev_info, &device_info_data);
+               if (ret == ERROR_SUCCESS)
+                       found_device++;
+               else if (ret != ERROR_CONTINUE)
+                       goto end;
+
+               memset(&device_info_data, 0, sizeof(SP_DEVINFO_DATA));
+               device_info_data.cbSize = sizeof(SP_DEVINFO_DATA);
+       }
+
+       RTE_LOG(DEBUG, EAL, "PCI scan found %lu devices\n", found_device);
+       ret = 0;
+end:
+       if (dev_info != INVALID_HANDLE_VALUE)
+               SetupDiDestroyDeviceInfoList(dev_info);
+
+       return ret;
 }
index 919a92f..374e654 100644 (file)
@@ -9,6 +9,7 @@ EXPORTS
        rte_devargs_remove
        rte_eal_get_configuration
        rte_eal_has_hugepages
+       rte_eal_has_pci
        rte_eal_init
        rte_eal_iova_mode
        rte_eal_mp_remote_launch
index 899ed7d..725ac4f 100644 (file)
@@ -25,6 +25,7 @@
 #include <psapi.h>
 #include <setupapi.h>
 #include <winioctl.h>
+#include <devguid.h>
 
 /* Have GUIDs defined. */
 #ifndef INITGUID