From: Pavan Nikhilesh Date: Fri, 26 Jan 2018 05:04:49 +0000 (+0530) Subject: eal: introduce integer divide through reciprocal X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=0b037e8b02b7ced8c18a650b708f2fd6e3364d83;p=dpdk.git eal: introduce integer divide through reciprocal In some use cases of integer division, denominator remains constant and numerator varies. It is possible to optimize division for such specific scenarios. The librte_sched uses rte_reciprocal to optimize division so, moving it to eal/common would allow other libraries and applications to use it. Signed-off-by: Pavan Nikhilesh --- diff --git a/lib/librte_eal/bsdapp/eal/Makefile b/lib/librte_eal/bsdapp/eal/Makefile index c6940760f9..028e81dd8e 100644 --- a/lib/librte_eal/bsdapp/eal/Makefile +++ b/lib/librte_eal/bsdapp/eal/Makefile @@ -57,6 +57,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_elem.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += malloc_heap.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_keepalive.c SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_service.c +SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_reciprocal.c # from arch dir SRCS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += rte_cpuflags.c diff --git a/lib/librte_eal/common/Makefile b/lib/librte_eal/common/Makefile index 96904dea1a..ea824a3a8c 100644 --- a/lib/librte_eal/common/Makefile +++ b/lib/librte_eal/common/Makefile @@ -16,6 +16,7 @@ INC += rte_pci_dev_feature_defs.h rte_pci_dev_features.h INC += rte_malloc.h rte_keepalive.h rte_time.h INC += rte_service.h rte_service_component.h INC += rte_bitmap.h rte_vfio.h rte_hypervisor.h rte_test.h +INC += rte_reciprocal.h GENERIC_INC := rte_atomic.h rte_byteorder.h rte_cycles.h rte_prefetch.h GENERIC_INC += rte_spinlock.h rte_memcpy.h rte_cpuflags.h rte_rwlock.h diff --git a/lib/librte_eal/common/include/rte_reciprocal.h b/lib/librte_eal/common/include/rte_reciprocal.h new file mode 100644 index 0000000000..5e21f096fa --- /dev/null +++ b/lib/librte_eal/common/include/rte_reciprocal.h @@ -0,0 +1,41 @@ +/* + * Reciprocal divide + * + * Used with permission from original authors + * Hannes Frederic Sowa and Daniel Borkmann + * + * This algorithm is based on the paper "Division by Invariant + * Integers Using Multiplication" by Torbjörn Granlund and Peter + * L. Montgomery. + * + * The assembler implementation from Agner Fog, which this code is + * based on, can be found here: + * http://www.agner.org/optimize/asmlib.zip + * + * This optimization for A/B is helpful if the divisor B is mostly + * runtime invariant. The reciprocal of B is calculated in the + * slow-path with reciprocal_value(). The fast-path can then just use + * a much faster multiplication operation with a variable dividend A + * to calculate the division A/B. + */ + +#ifndef _RTE_RECIPROCAL_H_ +#define _RTE_RECIPROCAL_H_ + +#include + +struct rte_reciprocal { + uint32_t m; + uint8_t sh1, sh2; +}; + +static inline uint32_t rte_reciprocal_divide(uint32_t a, struct rte_reciprocal R) +{ + uint32_t t = (uint32_t)(((uint64_t)a * R.m) >> 32); + + return (t + ((a - t) >> R.sh1)) >> R.sh2; +} + +struct rte_reciprocal rte_reciprocal_value(uint32_t d); + +#endif /* _RTE_RECIPROCAL_H_ */ diff --git a/lib/librte_eal/common/rte_reciprocal.c b/lib/librte_eal/common/rte_reciprocal.c new file mode 100644 index 0000000000..652f0237aa --- /dev/null +++ b/lib/librte_eal/common/rte_reciprocal.c @@ -0,0 +1,72 @@ +/*- + * BSD LICENSE + * + * Copyright(c) Hannes Frederic Sowa + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include + +#include + +#include "rte_reciprocal.h" + +/* find largest set bit. + * portable and slow but does not matter for this usage. + */ +static inline int fls(uint32_t x) +{ + int b; + + for (b = 31; b >= 0; --b) { + if (x & (1u << b)) + return b + 1; + } + + return 0; +} + +struct rte_reciprocal rte_reciprocal_value(uint32_t d) +{ + struct rte_reciprocal R; + uint64_t m; + int l; + + l = fls(d - 1); + m = ((1ULL << 32) * ((1ULL << l) - d)); + m /= d; + + ++m; + R.m = m; + R.sh1 = RTE_MIN(l, 1); + R.sh2 = RTE_MAX(l - 1, 0); + + return R; +} diff --git a/lib/librte_eal/linuxapp/eal/Makefile b/lib/librte_eal/linuxapp/eal/Makefile index 7bf278f3bf..e50b73a69c 100644 --- a/lib/librte_eal/linuxapp/eal/Makefile +++ b/lib/librte_eal/linuxapp/eal/Makefile @@ -64,6 +64,7 @@ SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += malloc_elem.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += malloc_heap.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_keepalive.c SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_service.c +SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_reciprocal.c # from arch dir SRCS-$(CONFIG_RTE_EXEC_ENV_LINUXAPP) += rte_cpuflags.c diff --git a/lib/librte_eal/rte_eal_version.map b/lib/librte_eal/rte_eal_version.map index 1a8b1b5efd..a179f5e8d8 100644 --- a/lib/librte_eal/rte_eal_version.map +++ b/lib/librte_eal/rte_eal_version.map @@ -206,7 +206,7 @@ DPDK_18.02 { rte_hypervisor_get; rte_hypervisor_get_name; rte_vfio_clear_group; - + rte_reciprocal_value; } DPDK_17.11; EXPERIMENTAL { diff --git a/lib/librte_sched/Makefile b/lib/librte_sched/Makefile index 73c9d1a37b..55d9c69899 100644 --- a/lib/librte_sched/Makefile +++ b/lib/librte_sched/Makefile @@ -26,10 +26,8 @@ LIBABIVER := 1 # all source are stored in SRCS-y # SRCS-$(CONFIG_RTE_LIBRTE_SCHED) += rte_sched.c rte_red.c rte_approx.c -SRCS-$(CONFIG_RTE_LIBRTE_SCHED) += rte_reciprocal.c # install includes SYMLINK-$(CONFIG_RTE_LIBRTE_SCHED)-include := rte_sched.h rte_sched_common.h rte_red.h rte_approx.h -SYMLINK-$(CONFIG_RTE_LIBRTE_SCHED)-include += rte_reciprocal.h include $(RTE_SDK)/mk/rte.lib.mk diff --git a/lib/librte_sched/rte_reciprocal.c b/lib/librte_sched/rte_reciprocal.c deleted file mode 100644 index 652f0237aa..0000000000 --- a/lib/librte_sched/rte_reciprocal.c +++ /dev/null @@ -1,72 +0,0 @@ -/*- - * BSD LICENSE - * - * Copyright(c) Hannes Frederic Sowa - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include -#include - -#include - -#include "rte_reciprocal.h" - -/* find largest set bit. - * portable and slow but does not matter for this usage. - */ -static inline int fls(uint32_t x) -{ - int b; - - for (b = 31; b >= 0; --b) { - if (x & (1u << b)) - return b + 1; - } - - return 0; -} - -struct rte_reciprocal rte_reciprocal_value(uint32_t d) -{ - struct rte_reciprocal R; - uint64_t m; - int l; - - l = fls(d - 1); - m = ((1ULL << 32) * ((1ULL << l) - d)); - m /= d; - - ++m; - R.m = m; - R.sh1 = RTE_MIN(l, 1); - R.sh2 = RTE_MAX(l - 1, 0); - - return R; -} diff --git a/lib/librte_sched/rte_reciprocal.h b/lib/librte_sched/rte_reciprocal.h deleted file mode 100644 index 5e21f096fa..0000000000 --- a/lib/librte_sched/rte_reciprocal.h +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Reciprocal divide - * - * Used with permission from original authors - * Hannes Frederic Sowa and Daniel Borkmann - * - * This algorithm is based on the paper "Division by Invariant - * Integers Using Multiplication" by Torbjörn Granlund and Peter - * L. Montgomery. - * - * The assembler implementation from Agner Fog, which this code is - * based on, can be found here: - * http://www.agner.org/optimize/asmlib.zip - * - * This optimization for A/B is helpful if the divisor B is mostly - * runtime invariant. The reciprocal of B is calculated in the - * slow-path with reciprocal_value(). The fast-path can then just use - * a much faster multiplication operation with a variable dividend A - * to calculate the division A/B. - */ - -#ifndef _RTE_RECIPROCAL_H_ -#define _RTE_RECIPROCAL_H_ - -#include - -struct rte_reciprocal { - uint32_t m; - uint8_t sh1, sh2; -}; - -static inline uint32_t rte_reciprocal_divide(uint32_t a, struct rte_reciprocal R) -{ - uint32_t t = (uint32_t)(((uint64_t)a * R.m) >> 32); - - return (t + ((a - t) >> R.sh1)) >> R.sh2; -} - -struct rte_reciprocal rte_reciprocal_value(uint32_t d); - -#endif /* _RTE_RECIPROCAL_H_ */ diff --git a/lib/librte_sched/rte_sched.c b/lib/librte_sched/rte_sched.c index ad2f7c6d59..634486c80a 100644 --- a/lib/librte_sched/rte_sched.c +++ b/lib/librte_sched/rte_sched.c @@ -14,11 +14,11 @@ #include #include #include +#include #include "rte_sched.h" #include "rte_sched_common.h" #include "rte_approx.h" -#include "rte_reciprocal.h" #ifdef __INTEL_COMPILER #pragma warning(disable:2259) /* conversion may lose significant bits */