From: Intel Date: Wed, 19 Dec 2012 23:00:00 +0000 (+0100) Subject: pci: make it possible to keep devices bound to uio X-Git-Tag: spdx-start~11373 X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=d6537e6a7432ea9cf39fc4ab2112d4bce0e9fe57;p=dpdk.git pci: make it possible to keep devices bound to uio Binding code has been refactored as well. Signed-off-by: Intel --- diff --git a/config/defconfig_i686-default-linuxapp-gcc b/config/defconfig_i686-default-linuxapp-gcc index 4033194d02..c4e192c201 100644 --- a/config/defconfig_i686-default-linuxapp-gcc +++ b/config/defconfig_i686-default-linuxapp-gcc @@ -108,6 +108,7 @@ CONFIG_RTE_LOG_HISTORY=256 CONFIG_RTE_LIBEAL_USE_HPET=y CONFIG_RTE_EAL_ALLOW_INV_SOCKET_ID=n CONFIG_RTE_EAL_ALWAYS_PANIC_ON_ERROR=n +CONFIG_RTE_EAL_UNBIND_PORTS=n # # Compile Environment Abstraction Layer for linux diff --git a/config/defconfig_i686-default-linuxapp-icc b/config/defconfig_i686-default-linuxapp-icc index 2850b6e815..bdedf73f9e 100644 --- a/config/defconfig_i686-default-linuxapp-icc +++ b/config/defconfig_i686-default-linuxapp-icc @@ -108,6 +108,7 @@ CONFIG_RTE_LOG_HISTORY=256 CONFIG_RTE_LIBEAL_USE_HPET=y CONFIG_RTE_EAL_ALLOW_INV_SOCKET_ID=n CONFIG_RTE_EAL_ALWAYS_PANIC_ON_ERROR=n +CONFIG_RTE_EAL_UNBIND_PORTS=n # # Compile Environment Abstraction Layer for linux diff --git a/config/defconfig_x86_64-default-linuxapp-gcc b/config/defconfig_x86_64-default-linuxapp-gcc index 5fc5ad0478..c64fd8f70f 100644 --- a/config/defconfig_x86_64-default-linuxapp-gcc +++ b/config/defconfig_x86_64-default-linuxapp-gcc @@ -108,6 +108,7 @@ CONFIG_RTE_LOG_HISTORY=256 CONFIG_RTE_LIBEAL_USE_HPET=y CONFIG_RTE_EAL_ALLOW_INV_SOCKET_ID=n CONFIG_RTE_EAL_ALWAYS_PANIC_ON_ERROR=n +CONFIG_RTE_EAL_UNBIND_PORTS=n # # Compile Environment Abstraction Layer for linux diff --git a/config/defconfig_x86_64-default-linuxapp-icc b/config/defconfig_x86_64-default-linuxapp-icc index fd2f21ceb2..65411c6988 100644 --- a/config/defconfig_x86_64-default-linuxapp-icc +++ b/config/defconfig_x86_64-default-linuxapp-icc @@ -108,6 +108,7 @@ CONFIG_RTE_LOG_HISTORY=256 CONFIG_RTE_LIBEAL_USE_HPET=y CONFIG_RTE_EAL_ALLOW_INV_SOCKET_ID=n CONFIG_RTE_EAL_ALWAYS_PANIC_ON_ERROR=n +CONFIG_RTE_EAL_UNBIND_PORTS=n # # Compile Environment Abstraction Layer for linux diff --git a/lib/librte_eal/common/eal_common_pci.c b/lib/librte_eal/common/eal_common_pci.c index e5e9b85f1f..a368799bf9 100644 --- a/lib/librte_eal/common/eal_common_pci.c +++ b/lib/librte_eal/common/eal_common_pci.c @@ -34,10 +34,12 @@ #include #include +#include #include #include #include +#include #include #include #include @@ -100,7 +102,10 @@ rte_eal_pci_probe(void) TAILQ_FOREACH(dev, &device_list, next) pci_probe_all_drivers(dev); - + #ifdef RTE_EAL_UNBIND_PORTS + if (atexit(rte_eal_pci_exit) != 0) + RTE_LOG(ERR, EAL, "atexit failure\n"); + #endif return 0; } diff --git a/lib/librte_eal/common/include/rte_pci.h b/lib/librte_eal/common/include/rte_pci.h index 4be38d22c8..53944e7b72 100644 --- a/lib/librte_eal/common/include/rte_pci.h +++ b/lib/librte_eal/common/include/rte_pci.h @@ -45,6 +45,7 @@ extern "C" { #endif +#include #include #include #include @@ -110,6 +111,7 @@ struct rte_pci_device { struct rte_pci_id id; /**< PCI ID. */ struct rte_pci_resource mem_resource; /**< PCI Memory Resource */ struct rte_intr_handle intr_handle; /**< Interrupt handle */ + char previous_dr[PATH_MAX]; /**< path for pre-dpdk driver*/ const struct rte_pci_driver *driver; /**< Associated driver */ unsigned int blacklisted:1; /**< Device is blacklisted */ }; @@ -154,6 +156,11 @@ struct rte_pci_driver { /**< Device needs igb_uio kernel module */ #define RTE_PCI_DRV_NEED_IGB_UIO 0x0001 +/** + * Perform clean up of pci drivers on application exits. + * */ +void rte_eal_pci_exit(void); + /** * Probe the PCI bus for registered drivers. * diff --git a/lib/librte_eal/linuxapp/eal/eal_pci.c b/lib/librte_eal/linuxapp/eal/eal_pci.c index 0a60bb3cff..ec5de8da68 100644 --- a/lib/librte_eal/linuxapp/eal/eal_pci.c +++ b/lib/librte_eal/linuxapp/eal/eal_pci.c @@ -66,7 +66,6 @@ #include #include -#include "eal_internal_cfg.h" #include "eal_filesystem.h" #include "eal_private.h" @@ -87,8 +86,7 @@ #define IGB_UIO_NAME "igb_uio" -#define UIO_NEWID "/sys/bus/pci/drivers/%s/new_id" -#define UIO_BIND "/sys/bus/pci/drivers/%s/bind" +#define UIO_DRV_PATH "/sys/bus/pci/drivers/%s" /* maximum time to wait that /dev/uioX appears */ #define UIO_DEV_WAIT_TIMEOUT 3 /* seconds */ @@ -127,7 +125,8 @@ pci_uio_check_module(const char *module_name) f = fopen(PROC_MODULES, "r"); if (f == NULL) { - RTE_LOG(ERR, EAL, "Cannot open "PROC_MODULES"\n"); + RTE_LOG(ERR, EAL, "Cannot open "PROC_MODULES": %s\n", + strerror(errno)); return -1; } @@ -144,11 +143,51 @@ pci_uio_check_module(const char *module_name) } } fclose(f); - RTE_LOG(ERR, EAL, "Cannot find %s in "PROC_MODULES"\n", module_name); return -1; } /* bind a PCI to the kernel module driver */ +static int +pci_bind_device(struct rte_pci_device *dev, char dr_path[]) +{ + FILE *f; + int n; + char buf[BUFSIZ]; + char dev_bind[PATH_MAX]; + struct rte_pci_addr *loc = &dev->addr; + + RTE_LOG(DEBUG, EAL, "bind PCI device "PCI_PRI_FMT"\n", + loc->domain, loc->bus, loc->devid, loc->function); + + n = rte_snprintf(dev_bind, sizeof(dev_bind), "%s/bind", dr_path); + if ((n < 0) || (n >= (int)sizeof(buf))) { + RTE_LOG(ERR, EAL, "Cannot rte_snprintf device bind path\n"); + return -1; + } + + f = fopen(dev_bind, "w"); + if (f == NULL) { + RTE_LOG(ERR, EAL, "Cannot open %s\n", dev->previous_dr); + return -1; + } + n = rte_snprintf(buf, sizeof(buf), PCI_PRI_FMT "\n", + loc->domain, loc->bus, loc->devid, loc->function); + if ((n < 0) || (n >= (int)sizeof(buf))) { + RTE_LOG(ERR, EAL, "Cannot rte_snprintf PCI infos\n"); + fclose(f); + return -1; + } + if (fwrite(buf, n, 1, f) == 0) { + fclose(f); + return -1; + } + + RTE_LOG(DEBUG, EAL, "Device bound\n"); + + fclose(f); + return 0; +} + static int pci_uio_bind_device(struct rte_pci_device *dev, const char *module_name) { @@ -157,17 +196,14 @@ pci_uio_bind_device(struct rte_pci_device *dev, const char *module_name) char buf[BUFSIZ]; char uio_newid[PATH_MAX]; char uio_bind[PATH_MAX]; - struct rte_pci_addr *loc = &dev->addr; - - RTE_LOG(DEBUG, EAL, "bind PCI device "PCI_PRI_FMT" to %s driver\n", - loc->domain, loc->bus, loc->devid, loc->function, module_name); - n = rte_snprintf(uio_newid, sizeof(uio_newid), UIO_NEWID, module_name); + n = rte_snprintf(uio_newid, sizeof(uio_newid), UIO_DRV_PATH "/new_id", module_name); if ((n < 0) || (n >= (int)sizeof(uio_newid))) { RTE_LOG(ERR, EAL, "Cannot rte_snprintf uio_newid name\n"); return -1; } - n = rte_snprintf(uio_bind, sizeof(uio_bind), UIO_BIND, module_name); + + n = rte_snprintf(uio_bind, sizeof(uio_bind), UIO_DRV_PATH, module_name); if ((n < 0) || (n >= (int)sizeof(uio_bind))) { RTE_LOG(ERR, EAL, "Cannot rte_snprintf uio_bind name\n"); return -1; @@ -191,29 +227,11 @@ pci_uio_bind_device(struct rte_pci_device *dev, const char *module_name) } fclose(f); - f = fopen(uio_bind, "w"); - if (f == NULL) { - RTE_LOG(ERR, EAL, "Cannot open %s\n", uio_bind); - return -1; - } - n = rte_snprintf(buf, sizeof(buf), PCI_PRI_FMT "\n", - loc->domain, loc->bus, loc->devid, loc->function); - if ((n < 0) || (n >= (int)sizeof(buf))) { - RTE_LOG(ERR, EAL, "Cannot rte_snprintf PCI infos\n"); - fclose(f); - return -1; - } - if (fwrite(buf, n, 1, f) == 0) { - fclose(f); - return -1; - } - - RTE_LOG(DEBUG, EAL, "Device bound\n"); - - fclose(f); + pci_bind_device(dev, uio_bind); return 0; } + /* map a particular resource from a file */ static void * pci_map_resource(struct rte_pci_device *dev, void *requested_addr, const char *devname, @@ -391,7 +409,7 @@ pci_uio_map_resource(struct rte_pci_device *dev) /* save the mapping details for secondary processes*/ uio_res = rte_malloc("UIO_RES", sizeof(*uio_res), 0); - if (uio_res == NULL){ + if (uio_res == NULL) { RTE_LOG(ERR, EAL, "%s(): cannot store uio mmap details\n", __func__); return -1; } @@ -467,6 +485,20 @@ error: } +/* Compare two PCI device addresses. */ +static int +pci_addr_comparison(struct rte_pci_addr *addr, struct rte_pci_addr *addr2) +{ + uint64_t dev_addr = (addr->domain << 24) + (addr->bus << 16) + (addr->devid << 8) + addr->function; + uint64_t dev_addr2 = (addr2->domain << 24) + (addr2->bus << 16) + (addr2->devid << 8) + addr2->function; + + if (dev_addr > dev_addr2) + return 1; + else + return 0; +} + + /* Scan one pci sysfs entry, and fill the devices list from it. */ static int pci_scan_one(const char *dirname, uint16_t domain, uint8_t bus, @@ -529,9 +561,24 @@ pci_scan_one(const char *dirname, uint16_t domain, uint8_t bus, return -1; } - /* device is valid, add in list */ - TAILQ_INSERT_TAIL(&device_list, dev, next); + /* device is valid, add in list (sorted) */ + if (TAILQ_EMPTY(&device_list)) { + TAILQ_INSERT_TAIL(&device_list, dev, next); + } + else { + struct rte_pci_device *dev2 = NULL; + TAILQ_FOREACH(dev2, &device_list, next) { + if (pci_addr_comparison(&dev->addr, &dev2->addr)) + continue; + else { + TAILQ_INSERT_BEFORE(dev2, dev, next); + return 0; + } + } + TAILQ_INSERT_TAIL(&device_list, dev, next); + } + return 0; } @@ -650,8 +697,11 @@ pci_unbind_kernel_driver(struct rte_pci_device *dev) RTE_LOG(ERR, EAL, "%s(): rte_snprintf failed\n", __func__); goto error; } - if (fwrite(buf, n, 1, f) == 0) + if (fwrite(buf, n, 1, f) == 0) { + RTE_LOG(ERR, EAL, "%s(): could not write to %s\n", __func__, + filename); goto error; + } fclose(f); return 0; @@ -661,6 +711,63 @@ error: return -1; } +static int +pci_exit_process(struct rte_pci_device *dev) +{ + if (munmap(dev->mem_resource.addr, dev->mem_resource.len) == -1){ + RTE_LOG(ERR, EAL, "Error with munmap\n"); + return -1; + } + if (close(dev->intr_handle.fd) == -1){ + RTE_LOG(ERR, EAL, "Error closing interrupt handle\n"); + return -1; + } + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + if (pci_unbind_kernel_driver(dev) < 0){ + RTE_LOG(ERR, EAL, "Error unbinding\n"); + return -1; + } + if (pci_bind_device(dev, dev->previous_dr) < 0){ + RTE_LOG(ERR, EAL, "Error binding\n"); + return -1; + } + } + + return 0; +} + +static void +pci_get_previous_driver_path(struct rte_pci_device *dev) +{ + int n; + char dev_path[PATH_MAX]; + char dr_path[PATH_MAX]; + struct rte_pci_addr *loc = &dev->addr; + + n = rte_snprintf(dev_path, sizeof(dev_path), SYSFS_PCI_DEVICES "/" + PCI_PRI_FMT "/driver", loc->domain, loc->bus, loc->devid, loc->function ); + if ((n < 0) || (n >= (int)sizeof(dev_path))) + RTE_LOG(ERR, EAL, "Cannot rte_snprintf device filepath\n"); + + n = readlink(dev_path, dr_path, sizeof(dr_path)); + if ((n < 0) || (n >= (int)sizeof(dr_path))){ + RTE_LOG(ERR, EAL, "Cannot readlink driver filepath\n"); + dev->previous_dr[0] = '\0'; + return; + } + dr_path[n] = '\0'; + + if (dr_path[0] != '/') + n = rte_snprintf(dev->previous_dr, sizeof(dev->previous_dr), + SYSFS_PCI_DEVICES "/" PCI_PRI_FMT "/%s", + loc->domain, loc->bus, loc->devid, loc->function, dr_path); + else + n = rte_snprintf(dev->previous_dr, sizeof(dev->previous_dr), + "%s", dr_path); + if ((n < 0) || (n >= (int)sizeof(dev_path))) + RTE_LOG(ERR, EAL, "Cannot rte_snprintf driver filepath\n"); +} + /* * If vendor/device ID match, call the devinit() function of the * driver. @@ -675,37 +782,44 @@ rte_eal_pci_probe_one_driver(struct rte_pci_driver *dr, struct rte_pci_device *d if (dr->drv_flags & RTE_PCI_DRV_NEED_IGB_UIO) module_name = IGB_UIO_NAME; - uio_status = pci_uio_check_module(module_name); - for (id_table = dr->id_table ; id_table->vendor_id != 0; id_table++) { /* check if device's identifiers match the driver's ones */ if (id_table->vendor_id != dev->id.vendor_id && - id_table->vendor_id != PCI_ANY_ID) + id_table->vendor_id != PCI_ANY_ID) continue; if (id_table->device_id != dev->id.device_id && - id_table->device_id != PCI_ANY_ID) + id_table->device_id != PCI_ANY_ID) continue; if (id_table->subsystem_vendor_id != dev->id.subsystem_vendor_id && - id_table->subsystem_vendor_id != PCI_ANY_ID) + id_table->subsystem_vendor_id != PCI_ANY_ID) continue; if (id_table->subsystem_device_id != dev->id.subsystem_device_id && - id_table->subsystem_device_id != PCI_ANY_ID) + id_table->subsystem_device_id != PCI_ANY_ID) continue; - RTE_LOG(DEBUG, EAL, "probe driver: %x:%x %s\n", - dev->id.vendor_id, dev->id.device_id, dr->name); + RTE_LOG(DEBUG, EAL, "probe driver: %x:%x %s\n", dev->id.vendor_id, + dev->id.device_id, dr->name); - if ((!dev->blacklisted) && (uio_status != 0)) { - rte_exit(1, "The %s module is required by the %s driver\n", - module_name, dr->name); - } + /* reference driver structure */ + dev->driver = dr; + + /* no initialization when blacklisted, return without error */ + if (dev->blacklisted) + return 0; /* Unbind PCI devices if needed */ - if ((!dev->blacklisted) && - (module_name != NULL)) { + if (module_name != NULL) { + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { - /* unbind current driver, bind ours */ + /* check that our driver is loaded */ + if (uio_status != 0 && + (uio_status = pci_uio_check_module(module_name)) != 0) + rte_exit(EXIT_FAILURE, "The %s module is required by the " + "%s driver\n", module_name, dr->name); + + /* unbind current driver, bind ours */ + pci_get_previous_driver_path(dev); if (pci_unbind_kernel_driver(dev) < 0) return -1; if (pci_uio_bind_device(dev, module_name) < 0) @@ -716,20 +830,26 @@ rte_eal_pci_probe_one_driver(struct rte_pci_driver *dr, struct rte_pci_device *d return -1; } - /* reference driver structure */ - dev->driver = dr; - - /* no initialization when blacklisted */ - if (dev->blacklisted) - return -1; - /* call the driver devinit() function */ return dr->devinit(dr, dev); - } return -1; } +/*Start the exit process for each dev in use*/ +void +rte_eal_pci_exit(void) +{ + struct rte_pci_device *dev = NULL; + + TAILQ_FOREACH(dev, &device_list, next){ + if (dev->previous_dr[0] == '\0') + continue; + if(pci_exit_process(dev) == 1) + RTE_LOG(ERR, EAL, "Exit process failure\n"); + } +} + /* Init the PCI EAL subsystem */ int rte_eal_pci_init(void) diff --git a/tools/pci_unbind.py b/tools/pci_unbind.py new file mode 100755 index 0000000000..bb80660b59 --- /dev/null +++ b/tools/pci_unbind.py @@ -0,0 +1,479 @@ +#! /usr/bin/python +# +# BSD LICENSE +# +# Copyright(c) 2010-2012 Intel Corporation. All rights reserved. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Intel Corporation nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# + +import sys, os, getopt, subprocess +from os.path import exists, abspath, dirname, basename + + +# The PCI device class for ETHERNET devices +ETHERNET_CLASS = "0200" + +# global dict ethernet devices present. Dictionary indexed by PCI address. +# Each device within this is itself a dictionary of device properties +devices = {} +# list of vendor:device pairs (again stored as dictionary) supported by igb_uio +module_dev_ids = [] + +def usage(): + '''Print usage information for the program''' + argv0 = basename(sys.argv[0]) + print """ +Usage: +------ + + %(argv0)s [options] DEVICE1 DEVICE2 .... + +where DEVICE1, DEVICE2 etc, are specified via PCI "domain:bus:slot.func" syntax +or "bus:slot.func" syntax. For devices bound to Linux kernel drivers, they may +also be referred to by Linux interface name e.g. eth0, eth1, em0, em1, etc. + +Options: + --help, --usage: + Display usage information and quit + + --status: + Print the current status of all known network interfaces. + For each device, it displays the PCI domain, bus, slot and function, + along with a text description of the device. Depending upon whether the + device is being used by a kernel driver, the igb_uio driver, or no + driver, other relevant information will be displayed: + * the Linux interface name e.g. if=eth0 + * the driver being used e.g. drv=igb_uio + * any suitable drivers not currently using that device + e.g. unused=igb_uio + NOTE: if this flag is passed along with a bind/unbind option, the status + display will always occur after the other operations have taken place. + + -b driver, --bind=driver: + Select the driver to use or \"none\" to unbind the device + + -u, --unbind: + Unbind a device (Equivalent to \"-b none\") + + --force: + By default, devices which are used by Linux - as indicated by having + routes in the routing table - cannot be modified. Using the --force + flag overrides this behavior, allowing active links to be forcibly + unbound. + WARNING: This can lead to loss of network connection and should be used + with caution. + +Examples: +--------- + +To display current device status: + %(argv0)s --status + +To bind eth1 from the current driver and move to use igb_uio + %(argv0)s --bind=igb_uio eth1 + +To unbind 0000:01:00.0 from using any driver + %(argv0)s -u 0000:01:00.0 + +To bind 0000:02:00.0 and 0000:02:00.1 to the ixgbe kernel driver + %(argv0)s -b ixgbe 02:00.0 02:00.1 + + """ % locals() # replace items from local variables + +# This is roughly compatible with check_output function in subprocess module +# which is only available in python 2.7. +def check_output(args, stderr=None): + '''Run a command and capture its output''' + return subprocess.Popen(args, stdout=subprocess.PIPE, + stderr=stderr).communicate()[0] + +def find_module(mod): + '''find the .ko file for kernel module named mod. + Searches the $RTE_SDK/$RTE_TARGET directory, the kernel + modules directory and finally under the parent directory of + the script ''' + # check $RTE_SDK/$RTE_TARGET directory + if 'RTE_SDK' in os.environ and 'RTE_TARGET' in os.environ: + path = "%s/%s/kmod/%s.ko" % (os.environ['RTE_SDK'],\ + os.environ['RTE_TARGET'], mod) + if exists(path): + return path + + # check using depmod + try: + depmod_out = check_output(["modinfo", "-n", mod], \ + stderr=subprocess.STDOUT).lower() + if "error" not in depmod_out: + path = depmod_out.strip() + if exists(path): + return path + except: # if modinfo can't find module, it fails, so continue + pass + + # check for a copy based off current path + tools_dir = dirname(abspath(sys.argv[0])) + if (tools_dir.endswith("tools")): + base_dir = dirname(tools_dir) + find_out = check_output(["find", base_dir, "-name", mod + ".ko"]) + if len(find_out) > 0: #something matched + path = find_out.splitlines()[0] + if exists(path): + return path + +def check_modules(): + '''Checks that the needed modules (igb_uio) is loaded, and then + determine from the .ko file, what its supported device ids are''' + global module_dev_ids + + fd = file("/proc/modules") + loaded_mods = fd.readlines() + fd.close() + mod = "igb_uio" + + # first check if module is loaded + found = False + for line in loaded_mods: + if line.startswith(mod): + found = True + break + if not found: + print "Error - module %s not loaded" %mod + sys.exit(1) + + # now find the .ko and get list of supported vendor/dev-ids + modpath = find_module(mod) + if modpath is None: + print "Cannot find module file %s" % (mod + ".ko") + sys.exit(1) + depmod_output = check_output(["depmod", "-n", modpath]).splitlines() + for line in depmod_output: + if not line.startswith(mod): + continue + if line.endswith(mod+".ko:"): + continue + lineparts = line.split() + module_dev_ids.append({"Vendor": int(lineparts[1],0), + "Device": int(lineparts[2],0)}) + +def is_supported_device(dev_id): + '''return true if device is supported by igb_uio, false otherwise''' + for dev in module_dev_ids: + if (dev["Vendor"] == devices[dev_id]["Vendor"] and + dev["Device"] == devices[dev_id]["Device"]): + return True + return False + +def has_driver(dev_id): + '''return true if a device is assigned to a driver. False otherwise''' + return "Driver_str" in devices[dev_id] + +def get_nic_details(): + '''This function populates the "devices" dictionary. The keys used are + the pci addresses (domain:bus:slot.func). The values are themselves + dictionaries - one for each NIC.''' + global devices + + # clear any old data + devices = {} + # first loop through and read details for all devices + # request machine readable format, with numeric IDs + dev = {}; + dev_lines = check_output(["lspci", "-Dvmmn"]).splitlines() + for dev_line in dev_lines: + if (len(dev_line) == 0): + if dev["Class"] == ETHERNET_CLASS: + #convert device and vendor ids to numbers, then add to global + dev["Vendor"] = int(dev["Vendor"],16) + dev["Device"] = int(dev["Device"],16) + devices[dev["Slot"]] = dict(dev) # use dict to make copy of dev + else: + name, value = dev_line.split("\t", 1) + dev[name.rstrip(":")] = value + + # check what is the interface if any for an ssh connection if + # any to this host, so we can mark it later. + ssh_if = [] + route = check_output(["ip", "-o", "route"]) + # filter out all lines for 169.254 routes + route = "\n".join(filter(lambda ln: not ln.startswith("169.254"), + route.splitlines())) + rt_info = route.split() + for i in xrange(len(rt_info) - 1): + if rt_info[i] == "dev": + ssh_if.append(rt_info[i+1]) + + # based on the basic info, get extended text details + for d in devices.keys(): + extra_info = check_output(["lspci", "-vmmks", d]).splitlines() + # parse lspci details + for line in extra_info: + if len(line) == 0: + continue + name, value = line.split("\t", 1) + name = name.strip(":") + "_str" + devices[d][name] = value + # check for a unix interface name + sys_path = "/sys/bus/pci/devices/%s/net/" % d + if exists(sys_path): + devices[d]["Interface"] = ",".join(os.listdir(sys_path)) + else: + devices[d]["Interface"] = "" + # check if a port is used for ssh connection + devices[d]["Ssh_if"] = False + devices[d]["Active"] = "" + for _if in ssh_if: + if _if in devices[d]["Interface"].split(","): + devices[d]["Ssh_if"] = True + devices[d]["Active"] = "*Active*" + break; + + # add igb_uio to list of supporting modules if needed + if is_supported_device(d): + if "Module_str" in devices[d]: + if "igb_uio" not in devices[d]["Module_str"]: + devices[d]["Module_str"] = devices[d]["Module_str"] + ",igb_uio" + else: + devices[d]["Module_str"] = "igb_uio" + if "Module_str" not in devices[d]: + devices[d]["Module_str"] = "" + # make sure the driver and module strings do not have any duplicates + if has_driver(d): + modules = devices[d]["Module_str"].split(",") + if devices[d]["Driver_str"] in modules: + modules.remove(devices[d]["Driver_str"]) + devices[d]["Module_str"] = ",".join(modules) + +def dev_id_from_dev_name(dev_name): + '''Take a device "name" - a string passed in by user to identify a NIC + device, and determine the device id - i.e. the domain:bus:slot.func - for + it, which can then be used to index into the devices array''' + dev = None + # check if it's already a suitable index + if dev_name in devices: + return dev_name + # check if it's an index just missing the domain part + elif "0000:" + dev_name in devices: + return "0000:" + dev_name + else: + # check if it's an interface name, e.g. eth1 + for d in devices.keys(): + if dev_name in devices[d]["Interface"].split(","): + return devices[d]["Slot"] + # if nothing else matches - error + print "Unknown device: %s. " \ + "Please specify device in \"bus:slot.func\" format" % dev_name + sys.exit(1) + +def unbind_one(dev_id, force): + '''Unbind the device identified by "dev_id" from its current driver''' + dev = devices[dev_id] + if not has_driver(dev_id): + print "%s %s %s is not currently managed by any driver\n" % \ + (dev["Slot"], dev["Device_str"], dev["Interface"]) + return + + # prevent us disconnecting ourselves + if dev["Ssh_if"] and not force: + print "Routing table indicates that interface %s is active" \ + ". Skipping unbind" % (dev_id) + return + + # write to /sys to unbind + filename = "/sys/bus/pci/drivers/%s/unbind" % dev["Driver_str"] + try: + f = open(filename, "a") + except: + print "Error: unbind failed for %s - Cannot open %s" % (dev_id, filename) + sys/exit(1) + f.write(dev_id) + f.close() + +def bind_one(dev_id, driver, force): + '''Bind the device given by "dev_id" to the driver "driver". If the device + is already bound to a different driver, it will be unbound first''' + dev = devices[dev_id] + saved_driver = None # used to rollback any unbind in case of failure + + # prevent disconnection of our ssh session + if dev["Ssh_if"] and not force: + print "Routing table indicates that interface %s is active" \ + ". Not modifying" % (dev_id) + return + + # unbind any existing drivers we don't want + if has_driver(dev_id): + if dev["Driver_str"] == driver: + print "%s already bound to driver %s, skipping\n" % (dev_id, driver) + return + else: + saved_driver = dev["Driver_str"] + unbind_one(dev_id, force) + dev["Driver_str"] = "" # clear driver string + + # do the bind by writing to /sys + filename = "/sys/bus/pci/drivers/%s/bind" % driver + try: + f = open(filename, "a") + except: + print "Error: bind failed for %s - Cannot open %s" % (dev_id, filename) + if saved_driver is not None: # restore any previous driver + bind_one(dev_id, saved_driver, force) + return + try: + f.write(dev_id) + f.close() + except: + print "Error: bind failed for %s - Cannot bind to driver %s" % (dev_id, driver) + if saved_driver is not None: # restore any previous driver + bind_one(dev_id, saved_driver, force) + return + + +def unbind_all(dev_list, force=False): + """Unbind method, takes a list of device locations""" + dev_list = map(dev_id_from_dev_name, dev_list) + for d in dev_list: + unbind_one(d, force) + +def bind_all(dev_list, driver, force=False): + """Unbind method, takes a list of device locations""" + dev_list = map(dev_id_from_dev_name, dev_list) + for d in dev_list: + bind_one(d, driver, force) + +def display_devices(title, dev_list, extra_params = None): + '''Displays to the user the details of a list of devices given in "dev_list" + The "extra_params" parameter, if given, should contain a string with + %()s fields in it for replacement by the named fields in each device's + dictionary.''' + strings = [] # this holds the strings to print. We sort before printing + print "\n%s" % title + print "="*len(title) + if len(dev_list) == 0: + strings.append("") + else: + for dev in dev_list: + if extra_params is not None: + strings.append("%s '%s' %s" % (dev["Slot"], \ + dev["Device_str"], extra_params % dev)) + else: + strings.append("%s '%s'" % (dev["Slot"], dev["Device_str"])) + # sort before printing, so that the entries appear in PCI order + strings.sort() + print "\n".join(strings) # print one per line + +def show_status(): + '''Function called when the script is passed the "--status" option. Displays + to the user what devices are bound to the igb_uio driver, the kernel driver + or to no driver''' + kernel_drv = [] + uio_drv = [] + no_drv = [] + # split our list of devices into the three categories above + for d in devices.keys(): + if not has_driver(d): + no_drv.append(devices[d]) + continue + if devices[d]["Driver_str"] == "igb_uio": + uio_drv.append(devices[d]) + else: + kernel_drv.append(devices[d]) + + # print each category separately, so we can clearly see what's used by DPDK + display_devices("Network devices using IGB_UIO driver", uio_drv, \ + "drv=%(Driver_str)s unused=%(Module_str)s") + display_devices("Network devices using kernel driver", kernel_drv, + "if=%(Interface)s drv=%(Driver_str)s unused=%(Module_str)s %(Active)s") + display_devices("Other network devices", no_drv,\ + "unused=%(Module_str)s") + +def parse_args(): + '''Parses the command-line arguments given by the user and takes the + appropriate action for each''' + b_flag = None + status_flag = False + force_flag = False + if len(sys.argv) <= 1: + usage() + sys.exit(0) + + try: + opts, args = getopt.getopt(sys.argv[1:], "b:u", + ["help", "usage", "status", "force", + "bind=", "unbind"]) + except getopt.GetoptError, error: + print str(error) + print "Run '%s --usage' for further information" % sys.argv[0] + sys.exit(1) + + for opt, arg in opts: + if opt == "--help" or opt == "--usage": + usage() + sys.exit(0) + if opt == "--status": + status_flag = True + if opt == "--force": + force_flag = True + if opt == "-b" or opt == "-u" or opt == "--bind" or opt == "--unbind": + if b_flag is not None: + print "Error - Only one bind or unbind may be specified\n" + sys.exit(1) + if opt == "-u" or opt == "--unbind": + b_flag = "none" + else: + b_flag = arg + + if b_flag is None and not status_flag: + print "Error: No action specified for devices. Please give a -b or -u option" + print "Run '%s --usage' for further information" % sys.argv[0] + sys.exit(1) + + if b_flag is not None and len(args) == 0: + print "Error: No devices specified." + print "Run '%s --usage' for further information" % sys.argv[0] + sys.exit(1) + + if b_flag == "none" or b_flag == "None": + unbind_all(args, force_flag) + elif b_flag is not None: + bind_all(args, b_flag, force_flag) + if status_flag: + if b_flag is not None: + get_nic_details() # refresh if we have changed anything + show_status() + +def main(): + '''program main function''' + check_modules() + get_nic_details() + parse_args() + +if __name__ == "__main__": + main()