From eeae544a51f56e43f0b885e4884e58376175dc78 Mon Sep 17 00:00:00 2001 From: Helin Zhang Date: Thu, 5 Jun 2014 13:09:09 +0800 Subject: [PATCH] pci: access to specific bits via sysfs Enabling 'Extended Tag' and resetting 'Max Read Request Size' in PCI config space have big impacts to i40e performance. They cannot be changed on some BIOS implementations, though can on others. Two sys files of 'extended_tag' and 'max_read_request_size' are added to support changing them by 'echo' in user space. Signed-off-by: Helin Zhang Signed-off-by: Jing Chen Acked-by: Cunming Liang Acked-by: Jijiang Liu Acked-by: Jingjing Wu Acked-by: Heqing Zhu Tested-by: Waterman Cao --- config/common_linuxapp | 7 ++ lib/librte_eal/linuxapp/eal/eal_pci.c | 97 ++++++++++++++++++++ lib/librte_eal/linuxapp/igb_uio/igb_uio.c | 105 ++++++++++++++++++++++ 3 files changed, 209 insertions(+) diff --git a/config/common_linuxapp b/config/common_linuxapp index 130e370e21..416e26d418 100644 --- a/config/common_linuxapp +++ b/config/common_linuxapp @@ -126,6 +126,13 @@ CONFIG_RTE_EAL_ALWAYS_PANIC_ON_ERROR=n CONFIG_RTE_EAL_IGB_UIO=y CONFIG_RTE_EAL_VFIO=y +# +# Special configurations in PCI Config Space for high performance +# +CONFIG_RTE_PCI_CONFIG=n +CONFIG_RTE_PCI_EXTENDED_TAG="" +CONFIG_RTE_PCI_MAX_READ_REQUEST_SIZE=0 + # # Compile Environment Abstraction Layer for linux # diff --git a/lib/librte_eal/linuxapp/eal/eal_pci.c b/lib/librte_eal/linuxapp/eal/eal_pci.c index 3b94b6fb54..b20c510c3e 100644 --- a/lib/librte_eal/linuxapp/eal/eal_pci.c +++ b/lib/librte_eal/linuxapp/eal/eal_pci.c @@ -393,6 +393,96 @@ error: return -1; } +#ifdef RTE_PCI_CONFIG +static int +pci_config_extended_tag(struct rte_pci_device *dev) +{ + struct rte_pci_addr *loc = &dev->addr; + char filename[PATH_MAX]; + char buf[BUFSIZ]; + FILE *f; + + /* not configured, let it as is */ + if (strncmp(RTE_PCI_EXTENDED_TAG, "on", 2) != 0 && + strncmp(RTE_PCI_EXTENDED_TAG, "off", 3) != 0) + return 0; + + rte_snprintf(filename, sizeof(filename), + SYSFS_PCI_DEVICES "/" PCI_PRI_FMT "/" "extended_tag", + loc->domain, loc->bus, loc->devid, loc->function); + f = fopen(filename, "rw+"); + if (!f) + return -1; + + fgets(buf, sizeof(buf), f); + if (strncmp(RTE_PCI_EXTENDED_TAG, "on", 2) == 0) { + /* enable Extended Tag*/ + if (strncmp(buf, "on", 2) != 0) { + fseek(f, 0, SEEK_SET); + fputs("on", f); + } + } else { + /* disable Extended Tag */ + if (strncmp(buf, "off", 3) != 0) { + fseek(f, 0, SEEK_SET); + fputs("off", f); + } + } + fclose(f); + + return 0; +} + +static int +pci_config_max_read_request_size(struct rte_pci_device *dev) +{ + struct rte_pci_addr *loc = &dev->addr; + char filename[PATH_MAX]; + char buf[BUFSIZ], param[BUFSIZ]; + FILE *f; + /* size can be 128, 256, 512, 1024, 2048, 4096 */ + uint32_t max_size = RTE_PCI_MAX_READ_REQUEST_SIZE; + + /* not configured, let it as is */ + if (!max_size) + return 0; + + rte_snprintf(filename, sizeof(filename), + SYSFS_PCI_DEVICES "/" PCI_PRI_FMT "/" "max_read_request_size", + loc->domain, loc->bus, loc->devid, loc->function); + f = fopen(filename, "rw+"); + if (!f) + return -1; + + fgets(buf, sizeof(buf), f); + rte_snprintf(param, sizeof(param), "%d", max_size); + + /* check if the size to be set is the same as current */ + if (strcmp(buf, param) == 0) { + fclose(f); + return 0; + } + fseek(f, 0, SEEK_SET); + fputs(param, f); + fclose(f); + + return 0; +} + +static void +pci_config_space_set(struct rte_pci_device *dev) +{ + if (rte_eal_process_type() != RTE_PROC_PRIMARY) + return; + + /* configure extended tag */ + pci_config_extended_tag(dev); + + /* configure max read request size */ + pci_config_max_read_request_size(dev); +} +#endif + static int pci_map_device(struct rte_pci_device *dev) { @@ -460,6 +550,13 @@ rte_eal_pci_probe_one_driver(struct rte_pci_driver *dr, struct rte_pci_device *d } if (dr->drv_flags & RTE_PCI_DRV_NEED_MAPPING) { +#ifdef RTE_PCI_CONFIG + /* + * Set PCIe config space for high performance. + * Return value can be ignored. + */ + pci_config_space_set(dev); +#endif /* map resources for devices that use igb_uio */ ret = pci_map_device(dev); if (ret != 0) diff --git a/lib/librte_eal/linuxapp/igb_uio/igb_uio.c b/lib/librte_eal/linuxapp/igb_uio/igb_uio.c index 60b8ca461a..1bd95f5424 100644 --- a/lib/librte_eal/linuxapp/igb_uio/igb_uio.c +++ b/lib/librte_eal/linuxapp/igb_uio/igb_uio.c @@ -48,6 +48,15 @@ #define PCI_MSIX_ENTRY_CTRL_MASKBIT 1 #endif +#ifdef RTE_PCI_CONFIG +#define PCI_SYS_FILE_BUF_SIZE 10 +#define PCI_DEV_CAP_REG 0xA4 +#define PCI_DEV_CTRL_REG 0xA8 +#define PCI_DEV_CAP_EXT_TAG_MASK 0x20 +#define PCI_DEV_CTRL_EXT_TAG_SHIFT 8 +#define PCI_DEV_CTRL_EXT_TAG_MASK (1 << PCI_DEV_CTRL_EXT_TAG_SHIFT) +#endif + #define IGBUIO_NUM_MSI_VECTORS 1 /** @@ -123,9 +132,105 @@ store_max_vfs(struct device *dev, struct device_attribute *attr, return err ? err : count; } +#ifdef RTE_PCI_CONFIG +static ssize_t +show_extended_tag(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct pci_dev *pci_dev = container_of(dev, struct pci_dev, dev); + uint32_t val = 0; + + pci_read_config_dword(pci_dev, PCI_DEV_CAP_REG, &val); + if (!(val & PCI_DEV_CAP_EXT_TAG_MASK)) /* Not supported */ + return snprintf(buf, PCI_SYS_FILE_BUF_SIZE, "%s\n", "invalid"); + + val = 0; + pci_bus_read_config_dword(pci_dev->bus, pci_dev->devfn, + PCI_DEV_CTRL_REG, &val); + + return snprintf(buf, PCI_SYS_FILE_BUF_SIZE, "%s\n", + (val & PCI_DEV_CTRL_EXT_TAG_MASK) ? "on" : "off"); +} + +static ssize_t +store_extended_tag(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct pci_dev *pci_dev = container_of(dev, struct pci_dev, dev); + uint32_t val = 0, enable; + + if (strncmp(buf, "on", 2) == 0) + enable = 1; + else if (strncmp(buf, "off", 3) == 0) + enable = 0; + else + return -EINVAL; + + pci_bus_read_config_dword(pci_dev->bus, pci_dev->devfn, + PCI_DEV_CAP_REG, &val); + if (!(val & PCI_DEV_CAP_EXT_TAG_MASK)) /* Not supported */ + return -EPERM; + + val = 0; + pci_bus_read_config_dword(pci_dev->bus, pci_dev->devfn, + PCI_DEV_CTRL_REG, &val); + if (enable) + val |= PCI_DEV_CTRL_EXT_TAG_MASK; + else + val &= ~PCI_DEV_CTRL_EXT_TAG_MASK; + pci_bus_write_config_dword(pci_dev->bus, pci_dev->devfn, + PCI_DEV_CTRL_REG, val); + + return count; +} + +static ssize_t +show_max_read_request_size(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct pci_dev *pci_dev = container_of(dev, struct pci_dev, dev); + int val = pcie_get_readrq(pci_dev); + + return snprintf(buf, PCI_SYS_FILE_BUF_SIZE, "%d\n", val); +} + +static ssize_t +store_max_read_request_size(struct device *dev, + struct device_attribute *attr, + const char *buf, + size_t count) +{ + struct pci_dev *pci_dev = container_of(dev, struct pci_dev, dev); + unsigned long size = 0; + int ret; + + if (strict_strtoul(buf, 0, &size) != 0) + return -EINVAL; + + ret = pcie_set_readrq(pci_dev, (int)size); + if (ret < 0) + return ret; + + return count; +} +#endif + static DEVICE_ATTR(max_vfs, S_IRUGO | S_IWUSR, show_max_vfs, store_max_vfs); +#ifdef RTE_PCI_CONFIG +static DEVICE_ATTR(extended_tag, S_IRUGO | S_IWUSR, show_extended_tag, \ + store_extended_tag); +static DEVICE_ATTR(max_read_request_size, S_IRUGO | S_IWUSR, \ + show_max_read_request_size, store_max_read_request_size); +#endif + static struct attribute *dev_attrs[] = { &dev_attr_max_vfs.attr, +#ifdef RTE_PCI_CONFIG + &dev_attr_extended_tag.attr, + &dev_attr_max_read_request_size.attr, +#endif NULL, }; -- 2.20.1