From cb4a1d14bf3e0f1191b79c684c42a749fb0f8039 Mon Sep 17 00:00:00 2001 From: Eoin Breen Date: Tue, 20 Sep 2016 01:05:01 +0100 Subject: [PATCH] tools: bind crypto devices Adding the support to bind/unbind crypto devices with dpdk-devbind.py script, as now it is not restricted to network devices anymore. Signed-off-by: Eoin Breen Signed-off-by: Pablo de Lara Acked-by: Deepak Kumar Jain --- doc/guides/cryptodevs/qat.rst | 8 +++ tools/dpdk-devbind.py | 94 +++++++++++++++++++++++++++++++---- tools/dpdk-setup.sh | 32 ++++++------ 3 files changed, 108 insertions(+), 26 deletions(-) diff --git a/doc/guides/cryptodevs/qat.rst b/doc/guides/cryptodevs/qat.rst index 483433f86f..558c1f4039 100644 --- a/doc/guides/cryptodevs/qat.rst +++ b/doc/guides/cryptodevs/qat.rst @@ -399,3 +399,11 @@ if yours are different adjust the unbind command below:: echo "8086 19e3" > /sys/bus/pci/drivers/igb_uio/new_id You can use ``lspci -vvd:19e3`` to confirm that all devices are now in use by igb_uio kernel driver. + + +The other way to bind the VFs to the DPDK UIO driver is by using the ``dpdk-devbind.py`` script: + +.. code-block:: console + + cd $RTE_SDK + ./tools/dpdk-devbind.py -b igb_uio 0000:03:01.1 diff --git a/tools/dpdk-devbind.py b/tools/dpdk-devbind.py index 34be4953db..f1d374d6b0 100755 --- a/tools/dpdk-devbind.py +++ b/tools/dpdk-devbind.py @@ -40,6 +40,7 @@ from os.path import exists, abspath, dirname, basename # The PCI base class for NETWORK devices NETWORK_BASE_CLASS = "02" +CRYPTO_BASE_CLASS = "0b" # global dict ethernet devices present. Dictionary indexed by PCI address. # Each device within this is itself a dictionary of device properties @@ -72,7 +73,7 @@ Options: Display usage information and quit -s, --status: - Print the current status of all known network interfaces. + Print the current status of all known network and crypto devices. For each device, it displays the PCI domain, bus, slot and function, along with a text description of the device. Depending upon whether the device is being used by a kernel driver, the igb_uio driver, or no @@ -92,7 +93,7 @@ Options: Unbind a device (Equivalent to \"-b none\") --force: - By default, devices which are used by Linux - as indicated by having + By default, network devices which are used by Linux - as indicated by having routes in the routing table - cannot be modified. Using the --force flag overrides this behavior, allowing active links to be forcibly unbound. @@ -300,6 +301,54 @@ def get_nic_details(): devices[d]["Module_str"] = ",".join(modules) +def get_crypto_details(): + '''This function populates the "devices" dictionary. The keys used are + the pci addresses (domain:bus:slot.func). The values are themselves + dictionaries - one for each NIC.''' + global devices + global dpdk_drivers + + # clear any old data + # devices = {} + # first loop through and read details for all devices + # request machine readable format, with numeric IDs + dev = {} + dev_lines = check_output(["lspci", "-Dvmmn"]).splitlines() + for dev_line in dev_lines: + if (len(dev_line) == 0): + if (dev["Class"][0:2] == CRYPTO_BASE_CLASS): + # convert device and vendor ids to numbers, then add to global + dev["Vendor"] = int(dev["Vendor"], 16) + dev["Device"] = int(dev["Device"], 16) + # use dict to make copy of dev + devices[dev["Slot"]] = dict(dev) + else: + name, value = dev_line.decode().split("\t", 1) + dev[name.rstrip(":")] = value + + # based on the basic info, get extended text details + for d in devices.keys(): + # get additional info and add it to existing data + devices[d] = devices[d].copy() + devices[d].update(get_pci_device_details(d).items()) + + # add igb_uio to list of supporting modules if needed + if "Module_str" in devices[d]: + for driver in dpdk_drivers: + if driver not in devices[d]["Module_str"]: + devices[d]["Module_str"] = \ + devices[d]["Module_str"] + ",%s" % driver + else: + devices[d]["Module_str"] = ",".join(dpdk_drivers) + + # make sure the driver and module strings do not have any duplicates + if has_driver(d): + modules = devices[d]["Module_str"].split(",") + if devices[d]["Driver_str"] in modules: + modules.remove(devices[d]["Driver_str"]) + devices[d]["Module_str"] = ",".join(modules) + + def dev_id_from_dev_name(dev_name): '''Take a device "name" - a string passed in by user to identify a NIC device, and determine the device id - i.e. the domain:bus:slot.func - for @@ -481,15 +530,16 @@ def show_status(): dpdk_drv = [] no_drv = [] - # split our list of devices into the three categories above + # split our list of network devices into the three categories above for d in devices.keys(): - if not has_driver(d): - no_drv.append(devices[d]) - continue - if devices[d]["Driver_str"] in dpdk_drivers: - dpdk_drv.append(devices[d]) - else: - kernel_drv.append(devices[d]) + if (NETWORK_BASE_CLASS in devices[d]["Class"]): + if not has_driver(d): + no_drv.append(devices[d]) + continue + if devices[d]["Driver_str"] in dpdk_drivers: + dpdk_drv.append(devices[d]) + else: + kernel_drv.append(devices[d]) # print each category separately, so we can clearly see what's used by DPDK display_devices("Network devices using DPDK-compatible driver", dpdk_drv, @@ -499,6 +549,28 @@ def show_status(): "unused=%(Module_str)s %(Active)s") display_devices("Other network devices", no_drv, "unused=%(Module_str)s") + # split our list of crypto devices into the three categories above + kernel_drv = [] + dpdk_drv = [] + no_drv = [] + + for d in devices.keys(): + if (CRYPTO_BASE_CLASS in devices[d]["Class"]): + if not has_driver(d): + no_drv.append(devices[d]) + continue + if devices[d]["Driver_str"] in dpdk_drivers: + dpdk_drv.append(devices[d]) + else: + kernel_drv.append(devices[d]) + + display_devices("Crypto devices using DPDK-compatible driver", dpdk_drv, + "drv=%(Driver_str)s unused=%(Module_str)s") + display_devices("Crypto devices using kernel driver", kernel_drv, + "drv=%(Driver_str)s " + "unused=%(Module_str)s") + display_devices("Other crypto devices", no_drv, "unused=%(Module_str)s") + def parse_args(): '''Parses the command-line arguments given by the user and takes the @@ -563,6 +635,7 @@ def do_arg_actions(): if status_flag: if b_flag is not None: get_nic_details() # refresh if we have changed anything + get_crypto_details() # refresh if we have changed anything show_status() @@ -571,6 +644,7 @@ def main(): parse_args() check_modules() get_nic_details() + get_crypto_details() do_arg_actions() if __name__ == "__main__": diff --git a/tools/dpdk-setup.sh b/tools/dpdk-setup.sh index ac81b2e486..14ed590707 100755 --- a/tools/dpdk-setup.sh +++ b/tools/dpdk-setup.sh @@ -422,23 +422,23 @@ grep_meminfo() } # -# Calls dpdk-devbind.py --status to show the NIC and what they +# Calls dpdk-devbind.py --status to show the devices and what they # are all bound to, in terms of drivers. # -show_nics() +show_devices() { if [ -d /sys/module/vfio_pci -o -d /sys/module/igb_uio ]; then ${RTE_SDK}/tools/dpdk-devbind.py --status else echo "# Please load the 'igb_uio' or 'vfio-pci' kernel module before " - echo "# querying or adjusting NIC device bindings" + echo "# querying or adjusting device bindings" fi } # # Uses dpdk-devbind.py to move devices to work with vfio-pci # -bind_nics_to_vfio() +bind_devices_to_vfio() { if [ -d /sys/module/vfio_pci ]; then ${RTE_SDK}/tools/dpdk-devbind.py --status @@ -449,14 +449,14 @@ bind_nics_to_vfio() echo "OK" else echo "# Please load the 'vfio-pci' kernel module before querying or " - echo "# adjusting NIC device bindings" + echo "# adjusting device bindings" fi } # # Uses dpdk-devbind.py to move devices to work with igb_uio # -bind_nics_to_igb_uio() +bind_devices_to_igb_uio() { if [ -d /sys/module/igb_uio ]; then ${RTE_SDK}/tools/dpdk-devbind.py --status @@ -466,14 +466,14 @@ bind_nics_to_igb_uio() sudo ${RTE_SDK}/tools/dpdk-devbind.py -b igb_uio $PCI_PATH && echo "OK" else echo "# Please load the 'igb_uio' kernel module before querying or " - echo "# adjusting NIC device bindings" + echo "# adjusting device bindings" fi } # # Uses dpdk-devbind.py to move devices to work with kernel drivers again # -unbind_nics() +unbind_devices() { ${RTE_SDK}/tools/dpdk-devbind.py --status echo "" @@ -525,14 +525,14 @@ step2_func() TEXT[5]="Setup hugepage mappings for NUMA systems" FUNC[5]="set_numa_pages" - TEXT[6]="Display current Ethernet device settings" - FUNC[6]="show_nics" + TEXT[6]="Display current Ethernet/Crypto device settings" + FUNC[6]="show_devices" - TEXT[7]="Bind Ethernet device to IGB UIO module" - FUNC[7]="bind_nics_to_igb_uio" + TEXT[7]="Bind Ethernet/Crypto device to IGB UIO module" + FUNC[7]="bind_devices_to_igb_uio" - TEXT[8]="Bind Ethernet device to VFIO module" - FUNC[8]="bind_nics_to_vfio" + TEXT[8]="Bind Ethernet/Crypto device to VFIO module" + FUNC[8]="bind_devices_to_vfio" TEXT[9]="Setup VFIO permissions" FUNC[9]="set_vfio_permissions" @@ -571,8 +571,8 @@ step5_func() { TITLE="Uninstall and system cleanup" - TEXT[1]="Unbind NICs from IGB UIO or VFIO driver" - FUNC[1]="unbind_nics" + TEXT[1]="Unbind devices from IGB UIO or VFIO driver" + FUNC[1]="unbind_devices" TEXT[2]="Remove IGB UIO module" FUNC[2]="remove_igb_uio_module" -- 2.20.1