No need to keep the same code duplicated for 32 and 64bits x86.
Signed-off-by: David Marchand <david.marchand@6wind.com>
Acked-by: Chao Zhu <bjzhuc@cn.ibm.com>
Acked-by: Thomas Monjalon <thomas.monjalon@6wind.com>
+++ /dev/null
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-/*
- * Inspired from FreeBSD src/sys/i386/include/atomic.h
- * Copyright (c) 1998 Doug Rabson
- * All rights reserved.
- */
-
-#ifndef _RTE_ATOMIC_I686_H_
-#define _RTE_ATOMIC_I686_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <emmintrin.h>
-#include "generic/rte_atomic.h"
-
-#if RTE_MAX_LCORE == 1
-#define MPLOCKED /**< No need to insert MP lock prefix. */
-#else
-#define MPLOCKED "lock ; " /**< Insert MP lock prefix. */
-#endif
-
-#define rte_mb() _mm_mfence()
-
-#define rte_wmb() _mm_sfence()
-
-#define rte_rmb() _mm_lfence()
-
-/*------------------------- 16 bit atomic operations -------------------------*/
-
-#ifndef RTE_FORCE_INTRINSICS
-static inline int
-rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
-{
- uint8_t res;
-
- asm volatile(
- MPLOCKED
- "cmpxchgw %[src], %[dst];"
- "sete %[res];"
- : [res] "=a" (res), /* output */
- [dst] "=m" (*dst)
- : [src] "r" (src), /* input */
- "a" (exp),
- "m" (*dst)
- : "memory"); /* no-clobber list */
- return res;
-}
-
-static inline int rte_atomic16_test_and_set(rte_atomic16_t *v)
-{
- return rte_atomic16_cmpset((volatile uint16_t *)&v->cnt, 0, 1);
-}
-
-static inline void
-rte_atomic16_inc(rte_atomic16_t *v)
-{
- asm volatile(
- MPLOCKED
- "incw %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : "m" (v->cnt) /* input */
- );
-}
-
-static inline void
-rte_atomic16_dec(rte_atomic16_t *v)
-{
- asm volatile(
- MPLOCKED
- "decw %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : "m" (v->cnt) /* input */
- );
-}
-
-static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
-{
- uint8_t ret;
-
- asm volatile(
- MPLOCKED
- "incw %[cnt] ; "
- "sete %[ret]"
- : [cnt] "+m" (v->cnt), /* output */
- [ret] "=qm" (ret)
- );
- return (ret != 0);
-}
-
-static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
-{
- uint8_t ret;
-
- asm volatile(MPLOCKED
- "decw %[cnt] ; "
- "sete %[ret]"
- : [cnt] "+m" (v->cnt), /* output */
- [ret] "=qm" (ret)
- );
- return (ret != 0);
-}
-
-/*------------------------- 32 bit atomic operations -------------------------*/
-
-static inline int
-rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
-{
- uint8_t res;
-
- asm volatile(
- MPLOCKED
- "cmpxchgl %[src], %[dst];"
- "sete %[res];"
- : [res] "=a" (res), /* output */
- [dst] "=m" (*dst)
- : [src] "r" (src), /* input */
- "a" (exp),
- "m" (*dst)
- : "memory"); /* no-clobber list */
- return res;
-}
-
-static inline int rte_atomic32_test_and_set(rte_atomic32_t *v)
-{
- return rte_atomic32_cmpset((volatile uint32_t *)&v->cnt, 0, 1);
-}
-
-static inline void
-rte_atomic32_inc(rte_atomic32_t *v)
-{
- asm volatile(
- MPLOCKED
- "incl %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : "m" (v->cnt) /* input */
- );
-}
-
-static inline void
-rte_atomic32_dec(rte_atomic32_t *v)
-{
- asm volatile(
- MPLOCKED
- "decl %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : "m" (v->cnt) /* input */
- );
-}
-
-static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
-{
- uint8_t ret;
-
- asm volatile(
- MPLOCKED
- "incl %[cnt] ; "
- "sete %[ret]"
- : [cnt] "+m" (v->cnt), /* output */
- [ret] "=qm" (ret)
- );
- return (ret != 0);
-}
-
-static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
-{
- uint8_t ret;
-
- asm volatile(MPLOCKED
- "decl %[cnt] ; "
- "sete %[ret]"
- : [cnt] "+m" (v->cnt), /* output */
- [ret] "=qm" (ret)
- );
- return (ret != 0);
-}
-
-/*------------------------- 64 bit atomic operations -------------------------*/
-
-static inline int
-rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
-{
- uint8_t res;
- union {
- struct {
- uint32_t l32;
- uint32_t h32;
- };
- uint64_t u64;
- } _exp, _src;
-
- _exp.u64 = exp;
- _src.u64 = src;
-
-#ifndef __PIC__
- asm volatile (
- MPLOCKED
- "cmpxchg8b (%[dst]);"
- "setz %[res];"
- : [res] "=a" (res) /* result in eax */
- : [dst] "S" (dst), /* esi */
- "b" (_src.l32), /* ebx */
- "c" (_src.h32), /* ecx */
- "a" (_exp.l32), /* eax */
- "d" (_exp.h32) /* edx */
- : "memory" ); /* no-clobber list */
-#else
- asm volatile (
- "mov %%ebx, %%edi\n"
- MPLOCKED
- "cmpxchg8b (%[dst]);"
- "setz %[res];"
- "xchgl %%ebx, %%edi;\n"
- : [res] "=a" (res) /* result in eax */
- : [dst] "S" (dst), /* esi */
- "D" (_src.l32), /* ebx */
- "c" (_src.h32), /* ecx */
- "a" (_exp.l32), /* eax */
- "d" (_exp.h32) /* edx */
- : "memory" ); /* no-clobber list */
-#endif
-
- return res;
-}
-
-static inline void
-rte_atomic64_init(rte_atomic64_t *v)
-{
- int success = 0;
- uint64_t tmp;
-
- while (success == 0) {
- tmp = v->cnt;
- success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
- tmp, 0);
- }
-}
-
-static inline int64_t
-rte_atomic64_read(rte_atomic64_t *v)
-{
- int success = 0;
- uint64_t tmp;
-
- while (success == 0) {
- tmp = v->cnt;
- /* replace the value by itself */
- success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
- tmp, tmp);
- }
- return tmp;
-}
-
-static inline void
-rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
-{
- int success = 0;
- uint64_t tmp;
-
- while (success == 0) {
- tmp = v->cnt;
- success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
- tmp, new_value);
- }
-}
-
-static inline void
-rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
-{
- int success = 0;
- uint64_t tmp;
-
- while (success == 0) {
- tmp = v->cnt;
- success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
- tmp, tmp + inc);
- }
-}
-
-static inline void
-rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
-{
- int success = 0;
- uint64_t tmp;
-
- while (success == 0) {
- tmp = v->cnt;
- success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
- tmp, tmp - dec);
- }
-}
-
-static inline void
-rte_atomic64_inc(rte_atomic64_t *v)
-{
- rte_atomic64_add(v, 1);
-}
-
-static inline void
-rte_atomic64_dec(rte_atomic64_t *v)
-{
- rte_atomic64_sub(v, 1);
-}
-
-static inline int64_t
-rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
-{
- int success = 0;
- uint64_t tmp;
-
- while (success == 0) {
- tmp = v->cnt;
- success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
- tmp, tmp + inc);
- }
-
- return tmp + inc;
-}
-
-static inline int64_t
-rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
-{
- int success = 0;
- uint64_t tmp;
-
- while (success == 0) {
- tmp = v->cnt;
- success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
- tmp, tmp - dec);
- }
-
- return tmp - dec;
-}
-
-static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
-{
- return rte_atomic64_add_return(v, 1) == 0;
-}
-
-static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
-{
- return rte_atomic64_sub_return(v, 1) == 0;
-}
-
-static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
-{
- return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
-}
-
-static inline void rte_atomic64_clear(rte_atomic64_t *v)
-{
- rte_atomic64_set(v, 0);
-}
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_ATOMIC_I686_H_ */
+++ /dev/null
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _RTE_BYTEORDER_I686_H_
-#define _RTE_BYTEORDER_I686_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "generic/rte_byteorder.h"
-
-/*
- * An architecture-optimized byte swap for a 16-bit value.
- *
- * Do not use this function directly. The preferred function is rte_bswap16().
- */
-static inline uint16_t rte_arch_bswap16(uint16_t _x)
-{
- register uint16_t x = _x;
- asm volatile ("xchgb %b[x1],%h[x2]"
- : [x1] "=Q" (x)
- : [x2] "0" (x)
- );
- return x;
-}
-
-/*
- * An architecture-optimized byte swap for a 32-bit value.
- *
- * Do not use this function directly. The preferred function is rte_bswap32().
- */
-static inline uint32_t rte_arch_bswap32(uint32_t _x)
-{
- register uint32_t x = _x;
- asm volatile ("bswap %[x]"
- : [x] "+r" (x)
- );
- return x;
-}
-
-/*
- * An architecture-optimized byte swap for a 64-bit value.
- *
- * Do not use this function directly. The preferred function is rte_bswap64().
- */
-/* Compat./Leg. mode */
-static inline uint64_t rte_arch_bswap64(uint64_t x)
-{
- uint64_t ret = 0;
- ret |= ((uint64_t)rte_arch_bswap32(x & 0xffffffffUL) << 32);
- ret |= ((uint64_t)rte_arch_bswap32((x >> 32) & 0xffffffffUL));
- return ret;
-}
-
-#ifndef RTE_FORCE_INTRINSICS
-#define rte_bswap16(x) ((uint16_t)(__builtin_constant_p(x) ? \
- rte_constant_bswap16(x) : \
- rte_arch_bswap16(x)))
-
-#define rte_bswap32(x) ((uint32_t)(__builtin_constant_p(x) ? \
- rte_constant_bswap32(x) : \
- rte_arch_bswap32(x)))
-
-#define rte_bswap64(x) ((uint64_t)(__builtin_constant_p(x) ? \
- rte_constant_bswap64(x) : \
- rte_arch_bswap64(x)))
-#else
-/*
- * __builtin_bswap16 is only available gcc 4.8 and upwards
- */
-#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 8)
-#define rte_bswap16(x) ((uint16_t)(__builtin_constant_p(x) ? \
- rte_constant_bswap16(x) : \
- rte_arch_bswap16(x)))
-#endif
-#endif
-
-#define rte_cpu_to_le_16(x) (x)
-#define rte_cpu_to_le_32(x) (x)
-#define rte_cpu_to_le_64(x) (x)
-
-#define rte_cpu_to_be_16(x) rte_bswap16(x)
-#define rte_cpu_to_be_32(x) rte_bswap32(x)
-#define rte_cpu_to_be_64(x) rte_bswap64(x)
-
-#define rte_le_to_cpu_16(x) (x)
-#define rte_le_to_cpu_32(x) (x)
-#define rte_le_to_cpu_64(x) (x)
-
-#define rte_be_to_cpu_16(x) rte_bswap16(x)
-#define rte_be_to_cpu_32(x) rte_bswap32(x)
-#define rte_be_to_cpu_64(x) rte_bswap64(x)
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_BYTEORDER_I686_H_ */
+++ /dev/null
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _RTE_CPUFLAGS_I686_H_
-#define _RTE_CPUFLAGS_I686_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <errno.h>
-#include <stdint.h>
-
-#include "generic/rte_cpuflags.h"
-
-enum rte_cpu_flag_t {
- /* (EAX 01h) ECX features*/
- RTE_CPUFLAG_SSE3 = 0, /**< SSE3 */
- RTE_CPUFLAG_PCLMULQDQ, /**< PCLMULQDQ */
- RTE_CPUFLAG_DTES64, /**< DTES64 */
- RTE_CPUFLAG_MONITOR, /**< MONITOR */
- RTE_CPUFLAG_DS_CPL, /**< DS_CPL */
- RTE_CPUFLAG_VMX, /**< VMX */
- RTE_CPUFLAG_SMX, /**< SMX */
- RTE_CPUFLAG_EIST, /**< EIST */
- RTE_CPUFLAG_TM2, /**< TM2 */
- RTE_CPUFLAG_SSSE3, /**< SSSE3 */
- RTE_CPUFLAG_CNXT_ID, /**< CNXT_ID */
- RTE_CPUFLAG_FMA, /**< FMA */
- RTE_CPUFLAG_CMPXCHG16B, /**< CMPXCHG16B */
- RTE_CPUFLAG_XTPR, /**< XTPR */
- RTE_CPUFLAG_PDCM, /**< PDCM */
- RTE_CPUFLAG_PCID, /**< PCID */
- RTE_CPUFLAG_DCA, /**< DCA */
- RTE_CPUFLAG_SSE4_1, /**< SSE4_1 */
- RTE_CPUFLAG_SSE4_2, /**< SSE4_2 */
- RTE_CPUFLAG_X2APIC, /**< X2APIC */
- RTE_CPUFLAG_MOVBE, /**< MOVBE */
- RTE_CPUFLAG_POPCNT, /**< POPCNT */
- RTE_CPUFLAG_TSC_DEADLINE, /**< TSC_DEADLINE */
- RTE_CPUFLAG_AES, /**< AES */
- RTE_CPUFLAG_XSAVE, /**< XSAVE */
- RTE_CPUFLAG_OSXSAVE, /**< OSXSAVE */
- RTE_CPUFLAG_AVX, /**< AVX */
- RTE_CPUFLAG_F16C, /**< F16C */
- RTE_CPUFLAG_RDRAND, /**< RDRAND */
-
- /* (EAX 01h) EDX features */
- RTE_CPUFLAG_FPU, /**< FPU */
- RTE_CPUFLAG_VME, /**< VME */
- RTE_CPUFLAG_DE, /**< DE */
- RTE_CPUFLAG_PSE, /**< PSE */
- RTE_CPUFLAG_TSC, /**< TSC */
- RTE_CPUFLAG_MSR, /**< MSR */
- RTE_CPUFLAG_PAE, /**< PAE */
- RTE_CPUFLAG_MCE, /**< MCE */
- RTE_CPUFLAG_CX8, /**< CX8 */
- RTE_CPUFLAG_APIC, /**< APIC */
- RTE_CPUFLAG_SEP, /**< SEP */
- RTE_CPUFLAG_MTRR, /**< MTRR */
- RTE_CPUFLAG_PGE, /**< PGE */
- RTE_CPUFLAG_MCA, /**< MCA */
- RTE_CPUFLAG_CMOV, /**< CMOV */
- RTE_CPUFLAG_PAT, /**< PAT */
- RTE_CPUFLAG_PSE36, /**< PSE36 */
- RTE_CPUFLAG_PSN, /**< PSN */
- RTE_CPUFLAG_CLFSH, /**< CLFSH */
- RTE_CPUFLAG_DS, /**< DS */
- RTE_CPUFLAG_ACPI, /**< ACPI */
- RTE_CPUFLAG_MMX, /**< MMX */
- RTE_CPUFLAG_FXSR, /**< FXSR */
- RTE_CPUFLAG_SSE, /**< SSE */
- RTE_CPUFLAG_SSE2, /**< SSE2 */
- RTE_CPUFLAG_SS, /**< SS */
- RTE_CPUFLAG_HTT, /**< HTT */
- RTE_CPUFLAG_TM, /**< TM */
- RTE_CPUFLAG_PBE, /**< PBE */
-
- /* (EAX 06h) EAX features */
- RTE_CPUFLAG_DIGTEMP, /**< DIGTEMP */
- RTE_CPUFLAG_TRBOBST, /**< TRBOBST */
- RTE_CPUFLAG_ARAT, /**< ARAT */
- RTE_CPUFLAG_PLN, /**< PLN */
- RTE_CPUFLAG_ECMD, /**< ECMD */
- RTE_CPUFLAG_PTM, /**< PTM */
-
- /* (EAX 06h) ECX features */
- RTE_CPUFLAG_MPERF_APERF_MSR, /**< MPERF_APERF_MSR */
- RTE_CPUFLAG_ACNT2, /**< ACNT2 */
- RTE_CPUFLAG_ENERGY_EFF, /**< ENERGY_EFF */
-
- /* (EAX 07h, ECX 0h) EBX features */
- RTE_CPUFLAG_FSGSBASE, /**< FSGSBASE */
- RTE_CPUFLAG_BMI1, /**< BMI1 */
- RTE_CPUFLAG_HLE, /**< Hardware Lock elision */
- RTE_CPUFLAG_AVX2, /**< AVX2 */
- RTE_CPUFLAG_SMEP, /**< SMEP */
- RTE_CPUFLAG_BMI2, /**< BMI2 */
- RTE_CPUFLAG_ERMS, /**< ERMS */
- RTE_CPUFLAG_INVPCID, /**< INVPCID */
- RTE_CPUFLAG_RTM, /**< Transactional memory */
-
- /* (EAX 80000001h) ECX features */
- RTE_CPUFLAG_LAHF_SAHF, /**< LAHF_SAHF */
- RTE_CPUFLAG_LZCNT, /**< LZCNT */
-
- /* (EAX 80000001h) EDX features */
- RTE_CPUFLAG_SYSCALL, /**< SYSCALL */
- RTE_CPUFLAG_XD, /**< XD */
- RTE_CPUFLAG_1GB_PG, /**< 1GB_PG */
- RTE_CPUFLAG_RDTSCP, /**< RDTSCP */
- RTE_CPUFLAG_EM64T, /**< EM64T */
-
- /* (EAX 80000007h) EDX features */
- RTE_CPUFLAG_INVTSC, /**< INVTSC */
-
- /* The last item */
- RTE_CPUFLAG_NUMFLAGS, /**< This should always be the last! */
-};
-
-enum cpu_register_t {
- REG_EAX = 0,
- REG_EBX,
- REG_ECX,
- REG_EDX,
-};
-
-static const struct feature_entry cpu_feature_table[] = {
- FEAT_DEF(SSE3, 0x00000001, 0, REG_ECX, 0)
- FEAT_DEF(PCLMULQDQ, 0x00000001, 0, REG_ECX, 1)
- FEAT_DEF(DTES64, 0x00000001, 0, REG_ECX, 2)
- FEAT_DEF(MONITOR, 0x00000001, 0, REG_ECX, 3)
- FEAT_DEF(DS_CPL, 0x00000001, 0, REG_ECX, 4)
- FEAT_DEF(VMX, 0x00000001, 0, REG_ECX, 5)
- FEAT_DEF(SMX, 0x00000001, 0, REG_ECX, 6)
- FEAT_DEF(EIST, 0x00000001, 0, REG_ECX, 7)
- FEAT_DEF(TM2, 0x00000001, 0, REG_ECX, 8)
- FEAT_DEF(SSSE3, 0x00000001, 0, REG_ECX, 9)
- FEAT_DEF(CNXT_ID, 0x00000001, 0, REG_ECX, 10)
- FEAT_DEF(FMA, 0x00000001, 0, REG_ECX, 12)
- FEAT_DEF(CMPXCHG16B, 0x00000001, 0, REG_ECX, 13)
- FEAT_DEF(XTPR, 0x00000001, 0, REG_ECX, 14)
- FEAT_DEF(PDCM, 0x00000001, 0, REG_ECX, 15)
- FEAT_DEF(PCID, 0x00000001, 0, REG_ECX, 17)
- FEAT_DEF(DCA, 0x00000001, 0, REG_ECX, 18)
- FEAT_DEF(SSE4_1, 0x00000001, 0, REG_ECX, 19)
- FEAT_DEF(SSE4_2, 0x00000001, 0, REG_ECX, 20)
- FEAT_DEF(X2APIC, 0x00000001, 0, REG_ECX, 21)
- FEAT_DEF(MOVBE, 0x00000001, 0, REG_ECX, 22)
- FEAT_DEF(POPCNT, 0x00000001, 0, REG_ECX, 23)
- FEAT_DEF(TSC_DEADLINE, 0x00000001, 0, REG_ECX, 24)
- FEAT_DEF(AES, 0x00000001, 0, REG_ECX, 25)
- FEAT_DEF(XSAVE, 0x00000001, 0, REG_ECX, 26)
- FEAT_DEF(OSXSAVE, 0x00000001, 0, REG_ECX, 27)
- FEAT_DEF(AVX, 0x00000001, 0, REG_ECX, 28)
- FEAT_DEF(F16C, 0x00000001, 0, REG_ECX, 29)
- FEAT_DEF(RDRAND, 0x00000001, 0, REG_ECX, 30)
-
- FEAT_DEF(FPU, 0x00000001, 0, REG_EDX, 0)
- FEAT_DEF(VME, 0x00000001, 0, REG_EDX, 1)
- FEAT_DEF(DE, 0x00000001, 0, REG_EDX, 2)
- FEAT_DEF(PSE, 0x00000001, 0, REG_EDX, 3)
- FEAT_DEF(TSC, 0x00000001, 0, REG_EDX, 4)
- FEAT_DEF(MSR, 0x00000001, 0, REG_EDX, 5)
- FEAT_DEF(PAE, 0x00000001, 0, REG_EDX, 6)
- FEAT_DEF(MCE, 0x00000001, 0, REG_EDX, 7)
- FEAT_DEF(CX8, 0x00000001, 0, REG_EDX, 8)
- FEAT_DEF(APIC, 0x00000001, 0, REG_EDX, 9)
- FEAT_DEF(SEP, 0x00000001, 0, REG_EDX, 11)
- FEAT_DEF(MTRR, 0x00000001, 0, REG_EDX, 12)
- FEAT_DEF(PGE, 0x00000001, 0, REG_EDX, 13)
- FEAT_DEF(MCA, 0x00000001, 0, REG_EDX, 14)
- FEAT_DEF(CMOV, 0x00000001, 0, REG_EDX, 15)
- FEAT_DEF(PAT, 0x00000001, 0, REG_EDX, 16)
- FEAT_DEF(PSE36, 0x00000001, 0, REG_EDX, 17)
- FEAT_DEF(PSN, 0x00000001, 0, REG_EDX, 18)
- FEAT_DEF(CLFSH, 0x00000001, 0, REG_EDX, 19)
- FEAT_DEF(DS, 0x00000001, 0, REG_EDX, 21)
- FEAT_DEF(ACPI, 0x00000001, 0, REG_EDX, 22)
- FEAT_DEF(MMX, 0x00000001, 0, REG_EDX, 23)
- FEAT_DEF(FXSR, 0x00000001, 0, REG_EDX, 24)
- FEAT_DEF(SSE, 0x00000001, 0, REG_EDX, 25)
- FEAT_DEF(SSE2, 0x00000001, 0, REG_EDX, 26)
- FEAT_DEF(SS, 0x00000001, 0, REG_EDX, 27)
- FEAT_DEF(HTT, 0x00000001, 0, REG_EDX, 28)
- FEAT_DEF(TM, 0x00000001, 0, REG_EDX, 29)
- FEAT_DEF(PBE, 0x00000001, 0, REG_EDX, 31)
-
- FEAT_DEF(DIGTEMP, 0x00000006, 0, REG_EAX, 0)
- FEAT_DEF(TRBOBST, 0x00000006, 0, REG_EAX, 1)
- FEAT_DEF(ARAT, 0x00000006, 0, REG_EAX, 2)
- FEAT_DEF(PLN, 0x00000006, 0, REG_EAX, 4)
- FEAT_DEF(ECMD, 0x00000006, 0, REG_EAX, 5)
- FEAT_DEF(PTM, 0x00000006, 0, REG_EAX, 6)
-
- FEAT_DEF(MPERF_APERF_MSR, 0x00000006, 0, REG_ECX, 0)
- FEAT_DEF(ACNT2, 0x00000006, 0, REG_ECX, 1)
- FEAT_DEF(ENERGY_EFF, 0x00000006, 0, REG_ECX, 3)
-
- FEAT_DEF(FSGSBASE, 0x00000007, 0, REG_EBX, 0)
- FEAT_DEF(BMI1, 0x00000007, 0, REG_EBX, 2)
- FEAT_DEF(HLE, 0x00000007, 0, REG_EBX, 4)
- FEAT_DEF(AVX2, 0x00000007, 0, REG_EBX, 5)
- FEAT_DEF(SMEP, 0x00000007, 0, REG_EBX, 6)
- FEAT_DEF(BMI2, 0x00000007, 0, REG_EBX, 7)
- FEAT_DEF(ERMS, 0x00000007, 0, REG_EBX, 8)
- FEAT_DEF(INVPCID, 0x00000007, 0, REG_EBX, 10)
- FEAT_DEF(RTM, 0x00000007, 0, REG_EBX, 11)
-
- FEAT_DEF(LAHF_SAHF, 0x80000001, 0, REG_ECX, 0)
- FEAT_DEF(LZCNT, 0x80000001, 0, REG_ECX, 4)
-
- FEAT_DEF(SYSCALL, 0x80000001, 0, REG_EDX, 11)
- FEAT_DEF(XD, 0x80000001, 0, REG_EDX, 20)
- FEAT_DEF(1GB_PG, 0x80000001, 0, REG_EDX, 26)
- FEAT_DEF(RDTSCP, 0x80000001, 0, REG_EDX, 27)
- FEAT_DEF(EM64T, 0x80000001, 0, REG_EDX, 29)
-
- FEAT_DEF(INVTSC, 0x80000007, 0, REG_EDX, 8)
-};
-
-static inline void
-rte_cpu_get_features(uint32_t leaf, uint32_t subleaf, cpuid_registers_t out)
-{
-#if defined(__i386__) && defined(__PIC__)
- /* %ebx is a forbidden register if we compile with -fPIC or -fPIE */
- asm volatile("movl %%ebx,%0 ; cpuid ; xchgl %%ebx,%0"
- : "=r" (out[REG_EBX]),
- "=a" (out[REG_EAX]),
- "=c" (out[REG_ECX]),
- "=d" (out[REG_EDX])
- : "a" (leaf), "c" (subleaf));
-#else
-
- asm volatile("cpuid"
- : "=a" (out[REG_EAX]),
- "=b" (out[REG_EBX]),
- "=c" (out[REG_ECX]),
- "=d" (out[REG_EDX])
- : "a" (leaf), "c" (subleaf));
-
-#endif
-}
-
-static inline int
-rte_cpu_get_flag_enabled(enum rte_cpu_flag_t feature)
-{
- const struct feature_entry *feat;
- cpuid_registers_t regs;
-
-
- if (feature >= RTE_CPUFLAG_NUMFLAGS)
- /* Flag does not match anything in the feature tables */
- return -ENOENT;
-
- feat = &cpu_feature_table[feature];
-
- if (!feat->leaf)
- /* This entry in the table wasn't filled out! */
- return -EFAULT;
-
- rte_cpu_get_features(feat->leaf & 0xffff0000, 0, regs);
- if (((regs[REG_EAX] ^ feat->leaf) & 0xffff0000) ||
- regs[REG_EAX] < feat->leaf)
- return 0;
-
- /* get the cpuid leaf containing the desired feature */
- rte_cpu_get_features(feat->leaf, feat->subleaf, regs);
-
- /* check if the feature is enabled */
- return (regs[feat->reg] >> feat->bit) & 1;
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_CPUFLAGS_I686_H_ */
+++ /dev/null
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-/* BSD LICENSE
- *
- * Copyright(c) 2013 6WIND.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of 6WIND S.A. nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _RTE_CYCLES_I686_H_
-#define _RTE_CYCLES_I686_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "generic/rte_cycles.h"
-
-#ifdef RTE_LIBRTE_EAL_VMWARE_TSC_MAP_SUPPORT
-/* Global switch to use VMWARE mapping of TSC instead of RDTSC */
-extern int rte_cycles_vmware_tsc_map;
-#include <rte_branch_prediction.h>
-#endif
-
-static inline uint64_t
-rte_rdtsc(void)
-{
- union {
- uint64_t tsc_64;
- struct {
- uint32_t lo_32;
- uint32_t hi_32;
- };
- } tsc;
-
-#ifdef RTE_LIBRTE_EAL_VMWARE_TSC_MAP_SUPPORT
- if (unlikely(rte_cycles_vmware_tsc_map)) {
- /* ecx = 0x10000 corresponds to the physical TSC for VMware */
- asm volatile("rdpmc" :
- "=a" (tsc.lo_32),
- "=d" (tsc.hi_32) :
- "c"(0x10000));
- return tsc.tsc_64;
- }
-#endif
-
- asm volatile("rdtsc" :
- "=a" (tsc.lo_32),
- "=d" (tsc.hi_32));
- return tsc.tsc_64;
-}
-
-static inline uint64_t
-rte_rdtsc_precise(void)
-{
- rte_mb();
- return rte_rdtsc();
-}
-
-static inline uint64_t
-rte_get_tsc_cycles(void) { return rte_rdtsc(); }
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_CYCLES_I686_H_ */
+++ /dev/null
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _RTE_MEMCPY_I686_H_
-#define _RTE_MEMCPY_I686_H_
-
-#include <stdint.h>
-#include <string.h>
-#include <emmintrin.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "generic/rte_memcpy.h"
-
-#ifdef __INTEL_COMPILER
-#pragma warning(disable:593) /* Stop unused variable warning (reg_a etc). */
-#endif
-
-static inline void
-rte_mov16(uint8_t *dst, const uint8_t *src)
-{
- __m128i reg_a;
- asm volatile (
- "movdqu (%[src]), %[reg_a]\n\t"
- "movdqu %[reg_a], (%[dst])\n\t"
- : [reg_a] "=x" (reg_a)
- : [src] "r" (src),
- [dst] "r"(dst)
- : "memory"
- );
-}
-
-static inline void
-rte_mov32(uint8_t *dst, const uint8_t *src)
-{
- __m128i reg_a, reg_b;
- asm volatile (
- "movdqu (%[src]), %[reg_a]\n\t"
- "movdqu 16(%[src]), %[reg_b]\n\t"
- "movdqu %[reg_a], (%[dst])\n\t"
- "movdqu %[reg_b], 16(%[dst])\n\t"
- : [reg_a] "=x" (reg_a),
- [reg_b] "=x" (reg_b)
- : [src] "r" (src),
- [dst] "r"(dst)
- : "memory"
- );
-}
-
-static inline void
-rte_mov48(uint8_t *dst, const uint8_t *src)
-{
- __m128i reg_a, reg_b, reg_c;
- asm volatile (
- "movdqu (%[src]), %[reg_a]\n\t"
- "movdqu 16(%[src]), %[reg_b]\n\t"
- "movdqu 32(%[src]), %[reg_c]\n\t"
- "movdqu %[reg_a], (%[dst])\n\t"
- "movdqu %[reg_b], 16(%[dst])\n\t"
- "movdqu %[reg_c], 32(%[dst])\n\t"
- : [reg_a] "=x" (reg_a),
- [reg_b] "=x" (reg_b),
- [reg_c] "=x" (reg_c)
- : [src] "r" (src),
- [dst] "r"(dst)
- : "memory"
- );
-}
-
-static inline void
-rte_mov64(uint8_t *dst, const uint8_t *src)
-{
- __m128i reg_a, reg_b, reg_c, reg_d;
- asm volatile (
- "movdqu (%[src]), %[reg_a]\n\t"
- "movdqu 16(%[src]), %[reg_b]\n\t"
- "movdqu 32(%[src]), %[reg_c]\n\t"
- "movdqu 48(%[src]), %[reg_d]\n\t"
- "movdqu %[reg_a], (%[dst])\n\t"
- "movdqu %[reg_b], 16(%[dst])\n\t"
- "movdqu %[reg_c], 32(%[dst])\n\t"
- "movdqu %[reg_d], 48(%[dst])\n\t"
- : [reg_a] "=x" (reg_a),
- [reg_b] "=x" (reg_b),
- [reg_c] "=x" (reg_c),
- [reg_d] "=x" (reg_d)
- : [src] "r" (src),
- [dst] "r"(dst)
- : "memory"
- );
-}
-
-static inline void
-rte_mov128(uint8_t *dst, const uint8_t *src)
-{
- __m128i reg_a, reg_b, reg_c, reg_d, reg_e, reg_f, reg_g, reg_h;
- asm volatile (
- "movdqu (%[src]), %[reg_a]\n\t"
- "movdqu 16(%[src]), %[reg_b]\n\t"
- "movdqu 32(%[src]), %[reg_c]\n\t"
- "movdqu 48(%[src]), %[reg_d]\n\t"
- "movdqu 64(%[src]), %[reg_e]\n\t"
- "movdqu 80(%[src]), %[reg_f]\n\t"
- "movdqu 96(%[src]), %[reg_g]\n\t"
- "movdqu 112(%[src]), %[reg_h]\n\t"
- "movdqu %[reg_a], (%[dst])\n\t"
- "movdqu %[reg_b], 16(%[dst])\n\t"
- "movdqu %[reg_c], 32(%[dst])\n\t"
- "movdqu %[reg_d], 48(%[dst])\n\t"
- "movdqu %[reg_e], 64(%[dst])\n\t"
- "movdqu %[reg_f], 80(%[dst])\n\t"
- "movdqu %[reg_g], 96(%[dst])\n\t"
- "movdqu %[reg_h], 112(%[dst])\n\t"
- : [reg_a] "=x" (reg_a),
- [reg_b] "=x" (reg_b),
- [reg_c] "=x" (reg_c),
- [reg_d] "=x" (reg_d),
- [reg_e] "=x" (reg_e),
- [reg_f] "=x" (reg_f),
- [reg_g] "=x" (reg_g),
- [reg_h] "=x" (reg_h)
- : [src] "r" (src),
- [dst] "r"(dst)
- : "memory"
- );
-}
-
-#ifdef __INTEL_COMPILER
-#pragma warning(enable:593)
-#endif
-
-static inline void
-rte_mov256(uint8_t *dst, const uint8_t *src)
-{
- rte_mov128(dst, src);
- rte_mov128(dst + 128, src + 128);
-}
-
-#define rte_memcpy(dst, src, n) \
- ((__builtin_constant_p(n)) ? \
- memcpy((dst), (src), (n)) : \
- rte_memcpy_func((dst), (src), (n)))
-
-static inline void *
-rte_memcpy_func(void *dst, const void *src, size_t n)
-{
- void *ret = dst;
-
- /* We can't copy < 16 bytes using XMM registers so do it manually. */
- if (n < 16) {
- if (n & 0x01) {
- *(uint8_t *)dst = *(const uint8_t *)src;
- dst = (uint8_t *)dst + 1;
- src = (const uint8_t *)src + 1;
- }
- if (n & 0x02) {
- *(uint16_t *)dst = *(const uint16_t *)src;
- dst = (uint16_t *)dst + 1;
- src = (const uint16_t *)src + 1;
- }
- if (n & 0x04) {
- *(uint32_t *)dst = *(const uint32_t *)src;
- dst = (uint32_t *)dst + 1;
- src = (const uint32_t *)src + 1;
- }
- if (n & 0x08) {
- *(uint64_t *)dst = *(const uint64_t *)src;
- }
- return ret;
- }
-
- /* Special fast cases for <= 128 bytes */
- if (n <= 32) {
- rte_mov16((uint8_t *)dst, (const uint8_t *)src);
- rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n);
- return ret;
- }
-
- if (n <= 64) {
- rte_mov32((uint8_t *)dst, (const uint8_t *)src);
- rte_mov32((uint8_t *)dst - 32 + n, (const uint8_t *)src - 32 + n);
- return ret;
- }
-
- if (n <= 128) {
- rte_mov64((uint8_t *)dst, (const uint8_t *)src);
- rte_mov64((uint8_t *)dst - 64 + n, (const uint8_t *)src - 64 + n);
- return ret;
- }
-
- /*
- * For large copies > 128 bytes. This combination of 256, 64 and 16 byte
- * copies was found to be faster than doing 128 and 32 byte copies as
- * well.
- */
- for ( ; n >= 256; n -= 256) {
- rte_mov256((uint8_t *)dst, (const uint8_t *)src);
- dst = (uint8_t *)dst + 256;
- src = (const uint8_t *)src + 256;
- }
-
- /*
- * We split the remaining bytes (which will be less than 256) into
- * 64byte (2^6) chunks.
- * Using incrementing integers in the case labels of a switch statement
- * enourages the compiler to use a jump table. To get incrementing
- * integers, we shift the 2 relevant bits to the LSB position to first
- * get decrementing integers, and then subtract.
- */
- switch (3 - (n >> 6)) {
- case 0x00:
- rte_mov64((uint8_t *)dst, (const uint8_t *)src);
- n -= 64;
- dst = (uint8_t *)dst + 64;
- src = (const uint8_t *)src + 64; /* fallthrough */
- case 0x01:
- rte_mov64((uint8_t *)dst, (const uint8_t *)src);
- n -= 64;
- dst = (uint8_t *)dst + 64;
- src = (const uint8_t *)src + 64; /* fallthrough */
- case 0x02:
- rte_mov64((uint8_t *)dst, (const uint8_t *)src);
- n -= 64;
- dst = (uint8_t *)dst + 64;
- src = (const uint8_t *)src + 64; /* fallthrough */
- default:
- ;
- }
-
- /*
- * We split the remaining bytes (which will be less than 64) into
- * 16byte (2^4) chunks, using the same switch structure as above.
- */
- switch (3 - (n >> 4)) {
- case 0x00:
- rte_mov16((uint8_t *)dst, (const uint8_t *)src);
- n -= 16;
- dst = (uint8_t *)dst + 16;
- src = (const uint8_t *)src + 16; /* fallthrough */
- case 0x01:
- rte_mov16((uint8_t *)dst, (const uint8_t *)src);
- n -= 16;
- dst = (uint8_t *)dst + 16;
- src = (const uint8_t *)src + 16; /* fallthrough */
- case 0x02:
- rte_mov16((uint8_t *)dst, (const uint8_t *)src);
- n -= 16;
- dst = (uint8_t *)dst + 16;
- src = (const uint8_t *)src + 16; /* fallthrough */
- default:
- ;
- }
-
- /* Copy any remaining bytes, without going beyond end of buffers */
- if (n != 0) {
- rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n);
- }
- return ret;
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_MEMCPY_I686_H_ */
+++ /dev/null
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _RTE_PREFETCH_I686_H_
-#define _RTE_PREFETCH_I686_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "generic/rte_prefetch.h"
-
-static inline void rte_prefetch0(volatile void *p)
-{
- asm volatile ("prefetcht0 %[p]" : [p] "+m" (*(volatile char *)p));
-}
-
-static inline void rte_prefetch1(volatile void *p)
-{
- asm volatile ("prefetcht1 %[p]" : [p] "+m" (*(volatile char *)p));
-}
-
-static inline void rte_prefetch2(volatile void *p)
-{
- asm volatile ("prefetcht2 %[p]" : [p] "+m" (*(volatile char *)p));
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_PREFETCH_I686_H_ */
+++ /dev/null
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _RTE_SPINLOCK_I686_H_
-#define _RTE_SPINLOCK_I686_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "generic/rte_spinlock.h"
-
-#ifndef RTE_FORCE_INTRINSICS
-static inline void
-rte_spinlock_lock(rte_spinlock_t *sl)
-{
- int lock_val = 1;
- asm volatile (
- "1:\n"
- "xchg %[locked], %[lv]\n"
- "test %[lv], %[lv]\n"
- "jz 3f\n"
- "2:\n"
- "pause\n"
- "cmpl $0, %[locked]\n"
- "jnz 2b\n"
- "jmp 1b\n"
- "3:\n"
- : [locked] "=m" (sl->locked), [lv] "=q" (lock_val)
- : "[lv]" (lock_val)
- : "memory");
-}
-
-static inline void
-rte_spinlock_unlock (rte_spinlock_t *sl)
-{
- int unlock_val = 0;
- asm volatile (
- "xchg %[locked], %[ulv]\n"
- : [locked] "=m" (sl->locked), [ulv] "=q" (unlock_val)
- : "[ulv]" (unlock_val)
- : "memory");
-}
-
-static inline int
-rte_spinlock_trylock (rte_spinlock_t *sl)
-{
- int lockval = 1;
-
- asm volatile (
- "xchg %[locked], %[lockval]"
- : [locked] "=m" (sl->locked), [lockval] "=q" (lockval)
- : "[lockval]" (lockval)
- : "memory");
-
- return (lockval == 0);
-}
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_SPINLOCK_I686_H_ */
--- /dev/null
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_ATOMIC_X86_H_
+#define _RTE_ATOMIC_X86_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <emmintrin.h>
+#include "generic/rte_atomic.h"
+
+#if RTE_MAX_LCORE == 1
+#define MPLOCKED /**< No need to insert MP lock prefix. */
+#else
+#define MPLOCKED "lock ; " /**< Insert MP lock prefix. */
+#endif
+
+#define rte_mb() _mm_mfence()
+
+#define rte_wmb() _mm_sfence()
+
+#define rte_rmb() _mm_lfence()
+
+/*------------------------- 16 bit atomic operations -------------------------*/
+
+#ifndef RTE_FORCE_INTRINSICS
+static inline int
+rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
+{
+ uint8_t res;
+
+ asm volatile(
+ MPLOCKED
+ "cmpxchgw %[src], %[dst];"
+ "sete %[res];"
+ : [res] "=a" (res), /* output */
+ [dst] "=m" (*dst)
+ : [src] "r" (src), /* input */
+ "a" (exp),
+ "m" (*dst)
+ : "memory"); /* no-clobber list */
+ return res;
+}
+
+static inline int rte_atomic16_test_and_set(rte_atomic16_t *v)
+{
+ return rte_atomic16_cmpset((volatile uint16_t *)&v->cnt, 0, 1);
+}
+
+static inline void
+rte_atomic16_inc(rte_atomic16_t *v)
+{
+ asm volatile(
+ MPLOCKED
+ "incw %[cnt]"
+ : [cnt] "=m" (v->cnt) /* output */
+ : "m" (v->cnt) /* input */
+ );
+}
+
+static inline void
+rte_atomic16_dec(rte_atomic16_t *v)
+{
+ asm volatile(
+ MPLOCKED
+ "decw %[cnt]"
+ : [cnt] "=m" (v->cnt) /* output */
+ : "m" (v->cnt) /* input */
+ );
+}
+
+static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
+{
+ uint8_t ret;
+
+ asm volatile(
+ MPLOCKED
+ "incw %[cnt] ; "
+ "sete %[ret]"
+ : [cnt] "+m" (v->cnt), /* output */
+ [ret] "=qm" (ret)
+ );
+ return (ret != 0);
+}
+
+static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
+{
+ uint8_t ret;
+
+ asm volatile(MPLOCKED
+ "decw %[cnt] ; "
+ "sete %[ret]"
+ : [cnt] "+m" (v->cnt), /* output */
+ [ret] "=qm" (ret)
+ );
+ return (ret != 0);
+}
+
+/*------------------------- 32 bit atomic operations -------------------------*/
+
+static inline int
+rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
+{
+ uint8_t res;
+
+ asm volatile(
+ MPLOCKED
+ "cmpxchgl %[src], %[dst];"
+ "sete %[res];"
+ : [res] "=a" (res), /* output */
+ [dst] "=m" (*dst)
+ : [src] "r" (src), /* input */
+ "a" (exp),
+ "m" (*dst)
+ : "memory"); /* no-clobber list */
+ return res;
+}
+
+static inline int rte_atomic32_test_and_set(rte_atomic32_t *v)
+{
+ return rte_atomic32_cmpset((volatile uint32_t *)&v->cnt, 0, 1);
+}
+
+static inline void
+rte_atomic32_inc(rte_atomic32_t *v)
+{
+ asm volatile(
+ MPLOCKED
+ "incl %[cnt]"
+ : [cnt] "=m" (v->cnt) /* output */
+ : "m" (v->cnt) /* input */
+ );
+}
+
+static inline void
+rte_atomic32_dec(rte_atomic32_t *v)
+{
+ asm volatile(
+ MPLOCKED
+ "decl %[cnt]"
+ : [cnt] "=m" (v->cnt) /* output */
+ : "m" (v->cnt) /* input */
+ );
+}
+
+static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
+{
+ uint8_t ret;
+
+ asm volatile(
+ MPLOCKED
+ "incl %[cnt] ; "
+ "sete %[ret]"
+ : [cnt] "+m" (v->cnt), /* output */
+ [ret] "=qm" (ret)
+ );
+ return (ret != 0);
+}
+
+static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
+{
+ uint8_t ret;
+
+ asm volatile(MPLOCKED
+ "decl %[cnt] ; "
+ "sete %[ret]"
+ : [cnt] "+m" (v->cnt), /* output */
+ [ret] "=qm" (ret)
+ );
+ return (ret != 0);
+}
+#endif
+
+#ifdef RTE_ARCH_I686
+#include "rte_atomic_32.h"
+#else
+#include "rte_atomic_64.h"
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_ATOMIC_X86_H_ */
--- /dev/null
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Inspired from FreeBSD src/sys/i386/include/atomic.h
+ * Copyright (c) 1998 Doug Rabson
+ * All rights reserved.
+ */
+
+#ifndef _RTE_ATOMIC_I686_H_
+#define _RTE_ATOMIC_I686_H_
+
+/*------------------------- 64 bit atomic operations -------------------------*/
+
+#ifndef RTE_FORCE_INTRINSICS
+static inline int
+rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
+{
+ uint8_t res;
+ union {
+ struct {
+ uint32_t l32;
+ uint32_t h32;
+ };
+ uint64_t u64;
+ } _exp, _src;
+
+ _exp.u64 = exp;
+ _src.u64 = src;
+
+#ifndef __PIC__
+ asm volatile (
+ MPLOCKED
+ "cmpxchg8b (%[dst]);"
+ "setz %[res];"
+ : [res] "=a" (res) /* result in eax */
+ : [dst] "S" (dst), /* esi */
+ "b" (_src.l32), /* ebx */
+ "c" (_src.h32), /* ecx */
+ "a" (_exp.l32), /* eax */
+ "d" (_exp.h32) /* edx */
+ : "memory" ); /* no-clobber list */
+#else
+ asm volatile (
+ "mov %%ebx, %%edi\n"
+ MPLOCKED
+ "cmpxchg8b (%[dst]);"
+ "setz %[res];"
+ "xchgl %%ebx, %%edi;\n"
+ : [res] "=a" (res) /* result in eax */
+ : [dst] "S" (dst), /* esi */
+ "D" (_src.l32), /* ebx */
+ "c" (_src.h32), /* ecx */
+ "a" (_exp.l32), /* eax */
+ "d" (_exp.h32) /* edx */
+ : "memory" ); /* no-clobber list */
+#endif
+
+ return res;
+}
+
+static inline void
+rte_atomic64_init(rte_atomic64_t *v)
+{
+ int success = 0;
+ uint64_t tmp;
+
+ while (success == 0) {
+ tmp = v->cnt;
+ success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
+ tmp, 0);
+ }
+}
+
+static inline int64_t
+rte_atomic64_read(rte_atomic64_t *v)
+{
+ int success = 0;
+ uint64_t tmp;
+
+ while (success == 0) {
+ tmp = v->cnt;
+ /* replace the value by itself */
+ success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
+ tmp, tmp);
+ }
+ return tmp;
+}
+
+static inline void
+rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
+{
+ int success = 0;
+ uint64_t tmp;
+
+ while (success == 0) {
+ tmp = v->cnt;
+ success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
+ tmp, new_value);
+ }
+}
+
+static inline void
+rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
+{
+ int success = 0;
+ uint64_t tmp;
+
+ while (success == 0) {
+ tmp = v->cnt;
+ success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
+ tmp, tmp + inc);
+ }
+}
+
+static inline void
+rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
+{
+ int success = 0;
+ uint64_t tmp;
+
+ while (success == 0) {
+ tmp = v->cnt;
+ success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
+ tmp, tmp - dec);
+ }
+}
+
+static inline void
+rte_atomic64_inc(rte_atomic64_t *v)
+{
+ rte_atomic64_add(v, 1);
+}
+
+static inline void
+rte_atomic64_dec(rte_atomic64_t *v)
+{
+ rte_atomic64_sub(v, 1);
+}
+
+static inline int64_t
+rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
+{
+ int success = 0;
+ uint64_t tmp;
+
+ while (success == 0) {
+ tmp = v->cnt;
+ success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
+ tmp, tmp + inc);
+ }
+
+ return tmp + inc;
+}
+
+static inline int64_t
+rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
+{
+ int success = 0;
+ uint64_t tmp;
+
+ while (success == 0) {
+ tmp = v->cnt;
+ success = rte_atomic64_cmpset((volatile uint64_t *)&v->cnt,
+ tmp, tmp - dec);
+ }
+
+ return tmp - dec;
+}
+
+static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
+{
+ return rte_atomic64_add_return(v, 1) == 0;
+}
+
+static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
+{
+ return rte_atomic64_sub_return(v, 1) == 0;
+}
+
+static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
+{
+ return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
+}
+
+static inline void rte_atomic64_clear(rte_atomic64_t *v)
+{
+ rte_atomic64_set(v, 0);
+}
+#endif
+
+#endif /* _RTE_ATOMIC_I686_H_ */
--- /dev/null
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * Inspired from FreeBSD src/sys/amd64/include/atomic.h
+ * Copyright (c) 1998 Doug Rabson
+ * All rights reserved.
+ */
+
+#ifndef _RTE_ATOMIC_X86_64_H_
+#define _RTE_ATOMIC_X86_64_H_
+
+/*------------------------- 64 bit atomic operations -------------------------*/
+
+#ifndef RTE_FORCE_INTRINSICS
+static inline int
+rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
+{
+ uint8_t res;
+
+
+ asm volatile(
+ MPLOCKED
+ "cmpxchgq %[src], %[dst];"
+ "sete %[res];"
+ : [res] "=a" (res), /* output */
+ [dst] "=m" (*dst)
+ : [src] "r" (src), /* input */
+ "a" (exp),
+ "m" (*dst)
+ : "memory"); /* no-clobber list */
+
+ return res;
+}
+
+static inline void
+rte_atomic64_init(rte_atomic64_t *v)
+{
+ v->cnt = 0;
+}
+
+static inline int64_t
+rte_atomic64_read(rte_atomic64_t *v)
+{
+ return v->cnt;
+}
+
+static inline void
+rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
+{
+ v->cnt = new_value;
+}
+
+static inline void
+rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
+{
+ asm volatile(
+ MPLOCKED
+ "addq %[inc], %[cnt]"
+ : [cnt] "=m" (v->cnt) /* output */
+ : [inc] "ir" (inc), /* input */
+ "m" (v->cnt)
+ );
+}
+
+static inline void
+rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
+{
+ asm volatile(
+ MPLOCKED
+ "subq %[dec], %[cnt]"
+ : [cnt] "=m" (v->cnt) /* output */
+ : [dec] "ir" (dec), /* input */
+ "m" (v->cnt)
+ );
+}
+
+static inline void
+rte_atomic64_inc(rte_atomic64_t *v)
+{
+ asm volatile(
+ MPLOCKED
+ "incq %[cnt]"
+ : [cnt] "=m" (v->cnt) /* output */
+ : "m" (v->cnt) /* input */
+ );
+}
+
+static inline void
+rte_atomic64_dec(rte_atomic64_t *v)
+{
+ asm volatile(
+ MPLOCKED
+ "decq %[cnt]"
+ : [cnt] "=m" (v->cnt) /* output */
+ : "m" (v->cnt) /* input */
+ );
+}
+
+static inline int64_t
+rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
+{
+ int64_t prev = inc;
+
+ asm volatile(
+ MPLOCKED
+ "xaddq %[prev], %[cnt]"
+ : [prev] "+r" (prev), /* output */
+ [cnt] "=m" (v->cnt)
+ : "m" (v->cnt) /* input */
+ );
+ return prev + inc;
+}
+
+static inline int64_t
+rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
+{
+ return rte_atomic64_add_return(v, -dec);
+}
+
+static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
+{
+ uint8_t ret;
+
+ asm volatile(
+ MPLOCKED
+ "incq %[cnt] ; "
+ "sete %[ret]"
+ : [cnt] "+m" (v->cnt), /* output */
+ [ret] "=qm" (ret)
+ );
+
+ return ret != 0;
+}
+
+static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
+{
+ uint8_t ret;
+
+ asm volatile(
+ MPLOCKED
+ "decq %[cnt] ; "
+ "sete %[ret]"
+ : [cnt] "+m" (v->cnt), /* output */
+ [ret] "=qm" (ret)
+ );
+ return ret != 0;
+}
+
+static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
+{
+ return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
+}
+
+static inline void rte_atomic64_clear(rte_atomic64_t *v)
+{
+ v->cnt = 0;
+}
+#endif
+
+#endif /* _RTE_ATOMIC_X86_64_H_ */
--- /dev/null
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_BYTEORDER_X86_H_
+#define _RTE_BYTEORDER_X86_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "generic/rte_byteorder.h"
+
+/*
+ * An architecture-optimized byte swap for a 16-bit value.
+ *
+ * Do not use this function directly. The preferred function is rte_bswap16().
+ */
+static inline uint16_t rte_arch_bswap16(uint16_t _x)
+{
+ register uint16_t x = _x;
+ asm volatile ("xchgb %b[x1],%h[x2]"
+ : [x1] "=Q" (x)
+ : [x2] "0" (x)
+ );
+ return x;
+}
+
+/*
+ * An architecture-optimized byte swap for a 32-bit value.
+ *
+ * Do not use this function directly. The preferred function is rte_bswap32().
+ */
+static inline uint32_t rte_arch_bswap32(uint32_t _x)
+{
+ register uint32_t x = _x;
+ asm volatile ("bswap %[x]"
+ : [x] "+r" (x)
+ );
+ return x;
+}
+
+#ifndef RTE_FORCE_INTRINSICS
+#define rte_bswap16(x) ((uint16_t)(__builtin_constant_p(x) ? \
+ rte_constant_bswap16(x) : \
+ rte_arch_bswap16(x)))
+
+#define rte_bswap32(x) ((uint32_t)(__builtin_constant_p(x) ? \
+ rte_constant_bswap32(x) : \
+ rte_arch_bswap32(x)))
+
+#define rte_bswap64(x) ((uint64_t)(__builtin_constant_p(x) ? \
+ rte_constant_bswap64(x) : \
+ rte_arch_bswap64(x)))
+#else
+/*
+ * __builtin_bswap16 is only available gcc 4.8 and upwards
+ */
+#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 8)
+#define rte_bswap16(x) ((uint16_t)(__builtin_constant_p(x) ? \
+ rte_constant_bswap16(x) : \
+ rte_arch_bswap16(x)))
+#endif
+#endif
+
+#define rte_cpu_to_le_16(x) (x)
+#define rte_cpu_to_le_32(x) (x)
+#define rte_cpu_to_le_64(x) (x)
+
+#define rte_cpu_to_be_16(x) rte_bswap16(x)
+#define rte_cpu_to_be_32(x) rte_bswap32(x)
+#define rte_cpu_to_be_64(x) rte_bswap64(x)
+
+#define rte_le_to_cpu_16(x) (x)
+#define rte_le_to_cpu_32(x) (x)
+#define rte_le_to_cpu_64(x) (x)
+
+#define rte_be_to_cpu_16(x) rte_bswap16(x)
+#define rte_be_to_cpu_32(x) rte_bswap32(x)
+#define rte_be_to_cpu_64(x) rte_bswap64(x)
+
+#ifdef RTE_ARCH_I686
+#include "rte_byteorder_32.h"
+#else
+#include "rte_byteorder_64.h"
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_BYTEORDER_X86_H_ */
--- /dev/null
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_BYTEORDER_I686_H_
+#define _RTE_BYTEORDER_I686_H_
+
+/*
+ * An architecture-optimized byte swap for a 64-bit value.
+ *
+ * Do not use this function directly. The preferred function is rte_bswap64().
+ */
+/* Compat./Leg. mode */
+static inline uint64_t rte_arch_bswap64(uint64_t x)
+{
+ uint64_t ret = 0;
+ ret |= ((uint64_t)rte_arch_bswap32(x & 0xffffffffUL) << 32);
+ ret |= ((uint64_t)rte_arch_bswap32((x >> 32) & 0xffffffffUL));
+ return ret;
+}
+
+#endif /* _RTE_BYTEORDER_I686_H_ */
--- /dev/null
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_BYTEORDER_X86_64_H_
+#define _RTE_BYTEORDER_X86_64_H_
+
+/*
+ * An architecture-optimized byte swap for a 64-bit value.
+ *
+ * Do not use this function directly. The preferred function is rte_bswap64().
+ */
+/* 64-bit mode */
+static inline uint64_t rte_arch_bswap64(uint64_t _x)
+{
+ register uint64_t x = _x;
+ asm volatile ("bswap %[x]"
+ : [x] "+r" (x)
+ );
+ return x;
+}
+
+#endif /* _RTE_BYTEORDER_X86_64_H_ */
--- /dev/null
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_CPUFLAGS_X86_64_H_
+#define _RTE_CPUFLAGS_X86_64_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <errno.h>
+#include <stdint.h>
+
+#include "generic/rte_cpuflags.h"
+
+enum rte_cpu_flag_t {
+ /* (EAX 01h) ECX features*/
+ RTE_CPUFLAG_SSE3 = 0, /**< SSE3 */
+ RTE_CPUFLAG_PCLMULQDQ, /**< PCLMULQDQ */
+ RTE_CPUFLAG_DTES64, /**< DTES64 */
+ RTE_CPUFLAG_MONITOR, /**< MONITOR */
+ RTE_CPUFLAG_DS_CPL, /**< DS_CPL */
+ RTE_CPUFLAG_VMX, /**< VMX */
+ RTE_CPUFLAG_SMX, /**< SMX */
+ RTE_CPUFLAG_EIST, /**< EIST */
+ RTE_CPUFLAG_TM2, /**< TM2 */
+ RTE_CPUFLAG_SSSE3, /**< SSSE3 */
+ RTE_CPUFLAG_CNXT_ID, /**< CNXT_ID */
+ RTE_CPUFLAG_FMA, /**< FMA */
+ RTE_CPUFLAG_CMPXCHG16B, /**< CMPXCHG16B */
+ RTE_CPUFLAG_XTPR, /**< XTPR */
+ RTE_CPUFLAG_PDCM, /**< PDCM */
+ RTE_CPUFLAG_PCID, /**< PCID */
+ RTE_CPUFLAG_DCA, /**< DCA */
+ RTE_CPUFLAG_SSE4_1, /**< SSE4_1 */
+ RTE_CPUFLAG_SSE4_2, /**< SSE4_2 */
+ RTE_CPUFLAG_X2APIC, /**< X2APIC */
+ RTE_CPUFLAG_MOVBE, /**< MOVBE */
+ RTE_CPUFLAG_POPCNT, /**< POPCNT */
+ RTE_CPUFLAG_TSC_DEADLINE, /**< TSC_DEADLINE */
+ RTE_CPUFLAG_AES, /**< AES */
+ RTE_CPUFLAG_XSAVE, /**< XSAVE */
+ RTE_CPUFLAG_OSXSAVE, /**< OSXSAVE */
+ RTE_CPUFLAG_AVX, /**< AVX */
+ RTE_CPUFLAG_F16C, /**< F16C */
+ RTE_CPUFLAG_RDRAND, /**< RDRAND */
+
+ /* (EAX 01h) EDX features */
+ RTE_CPUFLAG_FPU, /**< FPU */
+ RTE_CPUFLAG_VME, /**< VME */
+ RTE_CPUFLAG_DE, /**< DE */
+ RTE_CPUFLAG_PSE, /**< PSE */
+ RTE_CPUFLAG_TSC, /**< TSC */
+ RTE_CPUFLAG_MSR, /**< MSR */
+ RTE_CPUFLAG_PAE, /**< PAE */
+ RTE_CPUFLAG_MCE, /**< MCE */
+ RTE_CPUFLAG_CX8, /**< CX8 */
+ RTE_CPUFLAG_APIC, /**< APIC */
+ RTE_CPUFLAG_SEP, /**< SEP */
+ RTE_CPUFLAG_MTRR, /**< MTRR */
+ RTE_CPUFLAG_PGE, /**< PGE */
+ RTE_CPUFLAG_MCA, /**< MCA */
+ RTE_CPUFLAG_CMOV, /**< CMOV */
+ RTE_CPUFLAG_PAT, /**< PAT */
+ RTE_CPUFLAG_PSE36, /**< PSE36 */
+ RTE_CPUFLAG_PSN, /**< PSN */
+ RTE_CPUFLAG_CLFSH, /**< CLFSH */
+ RTE_CPUFLAG_DS, /**< DS */
+ RTE_CPUFLAG_ACPI, /**< ACPI */
+ RTE_CPUFLAG_MMX, /**< MMX */
+ RTE_CPUFLAG_FXSR, /**< FXSR */
+ RTE_CPUFLAG_SSE, /**< SSE */
+ RTE_CPUFLAG_SSE2, /**< SSE2 */
+ RTE_CPUFLAG_SS, /**< SS */
+ RTE_CPUFLAG_HTT, /**< HTT */
+ RTE_CPUFLAG_TM, /**< TM */
+ RTE_CPUFLAG_PBE, /**< PBE */
+
+ /* (EAX 06h) EAX features */
+ RTE_CPUFLAG_DIGTEMP, /**< DIGTEMP */
+ RTE_CPUFLAG_TRBOBST, /**< TRBOBST */
+ RTE_CPUFLAG_ARAT, /**< ARAT */
+ RTE_CPUFLAG_PLN, /**< PLN */
+ RTE_CPUFLAG_ECMD, /**< ECMD */
+ RTE_CPUFLAG_PTM, /**< PTM */
+
+ /* (EAX 06h) ECX features */
+ RTE_CPUFLAG_MPERF_APERF_MSR, /**< MPERF_APERF_MSR */
+ RTE_CPUFLAG_ACNT2, /**< ACNT2 */
+ RTE_CPUFLAG_ENERGY_EFF, /**< ENERGY_EFF */
+
+ /* (EAX 07h, ECX 0h) EBX features */
+ RTE_CPUFLAG_FSGSBASE, /**< FSGSBASE */
+ RTE_CPUFLAG_BMI1, /**< BMI1 */
+ RTE_CPUFLAG_HLE, /**< Hardware Lock elision */
+ RTE_CPUFLAG_AVX2, /**< AVX2 */
+ RTE_CPUFLAG_SMEP, /**< SMEP */
+ RTE_CPUFLAG_BMI2, /**< BMI2 */
+ RTE_CPUFLAG_ERMS, /**< ERMS */
+ RTE_CPUFLAG_INVPCID, /**< INVPCID */
+ RTE_CPUFLAG_RTM, /**< Transactional memory */
+
+ /* (EAX 80000001h) ECX features */
+ RTE_CPUFLAG_LAHF_SAHF, /**< LAHF_SAHF */
+ RTE_CPUFLAG_LZCNT, /**< LZCNT */
+
+ /* (EAX 80000001h) EDX features */
+ RTE_CPUFLAG_SYSCALL, /**< SYSCALL */
+ RTE_CPUFLAG_XD, /**< XD */
+ RTE_CPUFLAG_1GB_PG, /**< 1GB_PG */
+ RTE_CPUFLAG_RDTSCP, /**< RDTSCP */
+ RTE_CPUFLAG_EM64T, /**< EM64T */
+
+ /* (EAX 80000007h) EDX features */
+ RTE_CPUFLAG_INVTSC, /**< INVTSC */
+
+ /* The last item */
+ RTE_CPUFLAG_NUMFLAGS, /**< This should always be the last! */
+};
+
+enum cpu_register_t {
+ REG_EAX = 0,
+ REG_EBX,
+ REG_ECX,
+ REG_EDX,
+};
+
+static const struct feature_entry cpu_feature_table[] = {
+ FEAT_DEF(SSE3, 0x00000001, 0, REG_ECX, 0)
+ FEAT_DEF(PCLMULQDQ, 0x00000001, 0, REG_ECX, 1)
+ FEAT_DEF(DTES64, 0x00000001, 0, REG_ECX, 2)
+ FEAT_DEF(MONITOR, 0x00000001, 0, REG_ECX, 3)
+ FEAT_DEF(DS_CPL, 0x00000001, 0, REG_ECX, 4)
+ FEAT_DEF(VMX, 0x00000001, 0, REG_ECX, 5)
+ FEAT_DEF(SMX, 0x00000001, 0, REG_ECX, 6)
+ FEAT_DEF(EIST, 0x00000001, 0, REG_ECX, 7)
+ FEAT_DEF(TM2, 0x00000001, 0, REG_ECX, 8)
+ FEAT_DEF(SSSE3, 0x00000001, 0, REG_ECX, 9)
+ FEAT_DEF(CNXT_ID, 0x00000001, 0, REG_ECX, 10)
+ FEAT_DEF(FMA, 0x00000001, 0, REG_ECX, 12)
+ FEAT_DEF(CMPXCHG16B, 0x00000001, 0, REG_ECX, 13)
+ FEAT_DEF(XTPR, 0x00000001, 0, REG_ECX, 14)
+ FEAT_DEF(PDCM, 0x00000001, 0, REG_ECX, 15)
+ FEAT_DEF(PCID, 0x00000001, 0, REG_ECX, 17)
+ FEAT_DEF(DCA, 0x00000001, 0, REG_ECX, 18)
+ FEAT_DEF(SSE4_1, 0x00000001, 0, REG_ECX, 19)
+ FEAT_DEF(SSE4_2, 0x00000001, 0, REG_ECX, 20)
+ FEAT_DEF(X2APIC, 0x00000001, 0, REG_ECX, 21)
+ FEAT_DEF(MOVBE, 0x00000001, 0, REG_ECX, 22)
+ FEAT_DEF(POPCNT, 0x00000001, 0, REG_ECX, 23)
+ FEAT_DEF(TSC_DEADLINE, 0x00000001, 0, REG_ECX, 24)
+ FEAT_DEF(AES, 0x00000001, 0, REG_ECX, 25)
+ FEAT_DEF(XSAVE, 0x00000001, 0, REG_ECX, 26)
+ FEAT_DEF(OSXSAVE, 0x00000001, 0, REG_ECX, 27)
+ FEAT_DEF(AVX, 0x00000001, 0, REG_ECX, 28)
+ FEAT_DEF(F16C, 0x00000001, 0, REG_ECX, 29)
+ FEAT_DEF(RDRAND, 0x00000001, 0, REG_ECX, 30)
+
+ FEAT_DEF(FPU, 0x00000001, 0, REG_EDX, 0)
+ FEAT_DEF(VME, 0x00000001, 0, REG_EDX, 1)
+ FEAT_DEF(DE, 0x00000001, 0, REG_EDX, 2)
+ FEAT_DEF(PSE, 0x00000001, 0, REG_EDX, 3)
+ FEAT_DEF(TSC, 0x00000001, 0, REG_EDX, 4)
+ FEAT_DEF(MSR, 0x00000001, 0, REG_EDX, 5)
+ FEAT_DEF(PAE, 0x00000001, 0, REG_EDX, 6)
+ FEAT_DEF(MCE, 0x00000001, 0, REG_EDX, 7)
+ FEAT_DEF(CX8, 0x00000001, 0, REG_EDX, 8)
+ FEAT_DEF(APIC, 0x00000001, 0, REG_EDX, 9)
+ FEAT_DEF(SEP, 0x00000001, 0, REG_EDX, 11)
+ FEAT_DEF(MTRR, 0x00000001, 0, REG_EDX, 12)
+ FEAT_DEF(PGE, 0x00000001, 0, REG_EDX, 13)
+ FEAT_DEF(MCA, 0x00000001, 0, REG_EDX, 14)
+ FEAT_DEF(CMOV, 0x00000001, 0, REG_EDX, 15)
+ FEAT_DEF(PAT, 0x00000001, 0, REG_EDX, 16)
+ FEAT_DEF(PSE36, 0x00000001, 0, REG_EDX, 17)
+ FEAT_DEF(PSN, 0x00000001, 0, REG_EDX, 18)
+ FEAT_DEF(CLFSH, 0x00000001, 0, REG_EDX, 19)
+ FEAT_DEF(DS, 0x00000001, 0, REG_EDX, 21)
+ FEAT_DEF(ACPI, 0x00000001, 0, REG_EDX, 22)
+ FEAT_DEF(MMX, 0x00000001, 0, REG_EDX, 23)
+ FEAT_DEF(FXSR, 0x00000001, 0, REG_EDX, 24)
+ FEAT_DEF(SSE, 0x00000001, 0, REG_EDX, 25)
+ FEAT_DEF(SSE2, 0x00000001, 0, REG_EDX, 26)
+ FEAT_DEF(SS, 0x00000001, 0, REG_EDX, 27)
+ FEAT_DEF(HTT, 0x00000001, 0, REG_EDX, 28)
+ FEAT_DEF(TM, 0x00000001, 0, REG_EDX, 29)
+ FEAT_DEF(PBE, 0x00000001, 0, REG_EDX, 31)
+
+ FEAT_DEF(DIGTEMP, 0x00000006, 0, REG_EAX, 0)
+ FEAT_DEF(TRBOBST, 0x00000006, 0, REG_EAX, 1)
+ FEAT_DEF(ARAT, 0x00000006, 0, REG_EAX, 2)
+ FEAT_DEF(PLN, 0x00000006, 0, REG_EAX, 4)
+ FEAT_DEF(ECMD, 0x00000006, 0, REG_EAX, 5)
+ FEAT_DEF(PTM, 0x00000006, 0, REG_EAX, 6)
+
+ FEAT_DEF(MPERF_APERF_MSR, 0x00000006, 0, REG_ECX, 0)
+ FEAT_DEF(ACNT2, 0x00000006, 0, REG_ECX, 1)
+ FEAT_DEF(ENERGY_EFF, 0x00000006, 0, REG_ECX, 3)
+
+ FEAT_DEF(FSGSBASE, 0x00000007, 0, REG_EBX, 0)
+ FEAT_DEF(BMI1, 0x00000007, 0, REG_EBX, 2)
+ FEAT_DEF(HLE, 0x00000007, 0, REG_EBX, 4)
+ FEAT_DEF(AVX2, 0x00000007, 0, REG_EBX, 5)
+ FEAT_DEF(SMEP, 0x00000007, 0, REG_EBX, 6)
+ FEAT_DEF(BMI2, 0x00000007, 0, REG_EBX, 7)
+ FEAT_DEF(ERMS, 0x00000007, 0, REG_EBX, 8)
+ FEAT_DEF(INVPCID, 0x00000007, 0, REG_EBX, 10)
+ FEAT_DEF(RTM, 0x00000007, 0, REG_EBX, 11)
+
+ FEAT_DEF(LAHF_SAHF, 0x80000001, 0, REG_ECX, 0)
+ FEAT_DEF(LZCNT, 0x80000001, 0, REG_ECX, 4)
+
+ FEAT_DEF(SYSCALL, 0x80000001, 0, REG_EDX, 11)
+ FEAT_DEF(XD, 0x80000001, 0, REG_EDX, 20)
+ FEAT_DEF(1GB_PG, 0x80000001, 0, REG_EDX, 26)
+ FEAT_DEF(RDTSCP, 0x80000001, 0, REG_EDX, 27)
+ FEAT_DEF(EM64T, 0x80000001, 0, REG_EDX, 29)
+
+ FEAT_DEF(INVTSC, 0x80000007, 0, REG_EDX, 8)
+};
+
+static inline void
+rte_cpu_get_features(uint32_t leaf, uint32_t subleaf, cpuid_registers_t out)
+{
+#if defined(__i386__) && defined(__PIC__)
+ /* %ebx is a forbidden register if we compile with -fPIC or -fPIE */
+ asm volatile("movl %%ebx,%0 ; cpuid ; xchgl %%ebx,%0"
+ : "=r" (out[REG_EBX]),
+ "=a" (out[REG_EAX]),
+ "=c" (out[REG_ECX]),
+ "=d" (out[REG_EDX])
+ : "a" (leaf), "c" (subleaf));
+#else
+
+ asm volatile("cpuid"
+ : "=a" (out[REG_EAX]),
+ "=b" (out[REG_EBX]),
+ "=c" (out[REG_ECX]),
+ "=d" (out[REG_EDX])
+ : "a" (leaf), "c" (subleaf));
+
+#endif
+}
+
+static inline int
+rte_cpu_get_flag_enabled(enum rte_cpu_flag_t feature)
+{
+ const struct feature_entry *feat;
+ cpuid_registers_t regs;
+
+
+ if (feature >= RTE_CPUFLAG_NUMFLAGS)
+ /* Flag does not match anything in the feature tables */
+ return -ENOENT;
+
+ feat = &cpu_feature_table[feature];
+
+ if (!feat->leaf)
+ /* This entry in the table wasn't filled out! */
+ return -EFAULT;
+
+ rte_cpu_get_features(feat->leaf & 0xffff0000, 0, regs);
+ if (((regs[REG_EAX] ^ feat->leaf) & 0xffff0000) ||
+ regs[REG_EAX] < feat->leaf)
+ return 0;
+
+ /* get the cpuid leaf containing the desired feature */
+ rte_cpu_get_features(feat->leaf, feat->subleaf, regs);
+
+ /* check if the feature is enabled */
+ return (regs[feat->reg] >> feat->bit) & 1;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_CPUFLAGS_X86_64_H_ */
--- /dev/null
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+/* BSD LICENSE
+ *
+ * Copyright(c) 2013 6WIND.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of 6WIND S.A. nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_CYCLES_X86_64_H_
+#define _RTE_CYCLES_X86_64_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "generic/rte_cycles.h"
+
+#ifdef RTE_LIBRTE_EAL_VMWARE_TSC_MAP_SUPPORT
+/* Global switch to use VMWARE mapping of TSC instead of RDTSC */
+extern int rte_cycles_vmware_tsc_map;
+#include <rte_branch_prediction.h>
+#endif
+
+static inline uint64_t
+rte_rdtsc(void)
+{
+ union {
+ uint64_t tsc_64;
+ struct {
+ uint32_t lo_32;
+ uint32_t hi_32;
+ };
+ } tsc;
+
+#ifdef RTE_LIBRTE_EAL_VMWARE_TSC_MAP_SUPPORT
+ if (unlikely(rte_cycles_vmware_tsc_map)) {
+ /* ecx = 0x10000 corresponds to the physical TSC for VMware */
+ asm volatile("rdpmc" :
+ "=a" (tsc.lo_32),
+ "=d" (tsc.hi_32) :
+ "c"(0x10000));
+ return tsc.tsc_64;
+ }
+#endif
+
+ asm volatile("rdtsc" :
+ "=a" (tsc.lo_32),
+ "=d" (tsc.hi_32));
+ return tsc.tsc_64;
+}
+
+static inline uint64_t
+rte_rdtsc_precise(void)
+{
+ rte_mb();
+ return rte_rdtsc();
+}
+
+static inline uint64_t
+rte_get_tsc_cycles(void) { return rte_rdtsc(); }
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_CYCLES_X86_64_H_ */
--- /dev/null
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_MEMCPY_X86_64_H_
+#define _RTE_MEMCPY_X86_64_H_
+
+#include <stdint.h>
+#include <string.h>
+#include <emmintrin.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "generic/rte_memcpy.h"
+
+#ifdef __INTEL_COMPILER
+#pragma warning(disable:593) /* Stop unused variable warning (reg_a etc). */
+#endif
+
+static inline void
+rte_mov16(uint8_t *dst, const uint8_t *src)
+{
+ __m128i reg_a;
+ asm volatile (
+ "movdqu (%[src]), %[reg_a]\n\t"
+ "movdqu %[reg_a], (%[dst])\n\t"
+ : [reg_a] "=x" (reg_a)
+ : [src] "r" (src),
+ [dst] "r"(dst)
+ : "memory"
+ );
+}
+
+static inline void
+rte_mov32(uint8_t *dst, const uint8_t *src)
+{
+ __m128i reg_a, reg_b;
+ asm volatile (
+ "movdqu (%[src]), %[reg_a]\n\t"
+ "movdqu 16(%[src]), %[reg_b]\n\t"
+ "movdqu %[reg_a], (%[dst])\n\t"
+ "movdqu %[reg_b], 16(%[dst])\n\t"
+ : [reg_a] "=x" (reg_a),
+ [reg_b] "=x" (reg_b)
+ : [src] "r" (src),
+ [dst] "r"(dst)
+ : "memory"
+ );
+}
+
+static inline void
+rte_mov48(uint8_t *dst, const uint8_t *src)
+{
+ __m128i reg_a, reg_b, reg_c;
+ asm volatile (
+ "movdqu (%[src]), %[reg_a]\n\t"
+ "movdqu 16(%[src]), %[reg_b]\n\t"
+ "movdqu 32(%[src]), %[reg_c]\n\t"
+ "movdqu %[reg_a], (%[dst])\n\t"
+ "movdqu %[reg_b], 16(%[dst])\n\t"
+ "movdqu %[reg_c], 32(%[dst])\n\t"
+ : [reg_a] "=x" (reg_a),
+ [reg_b] "=x" (reg_b),
+ [reg_c] "=x" (reg_c)
+ : [src] "r" (src),
+ [dst] "r"(dst)
+ : "memory"
+ );
+}
+
+static inline void
+rte_mov64(uint8_t *dst, const uint8_t *src)
+{
+ __m128i reg_a, reg_b, reg_c, reg_d;
+ asm volatile (
+ "movdqu (%[src]), %[reg_a]\n\t"
+ "movdqu 16(%[src]), %[reg_b]\n\t"
+ "movdqu 32(%[src]), %[reg_c]\n\t"
+ "movdqu 48(%[src]), %[reg_d]\n\t"
+ "movdqu %[reg_a], (%[dst])\n\t"
+ "movdqu %[reg_b], 16(%[dst])\n\t"
+ "movdqu %[reg_c], 32(%[dst])\n\t"
+ "movdqu %[reg_d], 48(%[dst])\n\t"
+ : [reg_a] "=x" (reg_a),
+ [reg_b] "=x" (reg_b),
+ [reg_c] "=x" (reg_c),
+ [reg_d] "=x" (reg_d)
+ : [src] "r" (src),
+ [dst] "r"(dst)
+ : "memory"
+ );
+}
+
+static inline void
+rte_mov128(uint8_t *dst, const uint8_t *src)
+{
+ __m128i reg_a, reg_b, reg_c, reg_d, reg_e, reg_f, reg_g, reg_h;
+ asm volatile (
+ "movdqu (%[src]), %[reg_a]\n\t"
+ "movdqu 16(%[src]), %[reg_b]\n\t"
+ "movdqu 32(%[src]), %[reg_c]\n\t"
+ "movdqu 48(%[src]), %[reg_d]\n\t"
+ "movdqu 64(%[src]), %[reg_e]\n\t"
+ "movdqu 80(%[src]), %[reg_f]\n\t"
+ "movdqu 96(%[src]), %[reg_g]\n\t"
+ "movdqu 112(%[src]), %[reg_h]\n\t"
+ "movdqu %[reg_a], (%[dst])\n\t"
+ "movdqu %[reg_b], 16(%[dst])\n\t"
+ "movdqu %[reg_c], 32(%[dst])\n\t"
+ "movdqu %[reg_d], 48(%[dst])\n\t"
+ "movdqu %[reg_e], 64(%[dst])\n\t"
+ "movdqu %[reg_f], 80(%[dst])\n\t"
+ "movdqu %[reg_g], 96(%[dst])\n\t"
+ "movdqu %[reg_h], 112(%[dst])\n\t"
+ : [reg_a] "=x" (reg_a),
+ [reg_b] "=x" (reg_b),
+ [reg_c] "=x" (reg_c),
+ [reg_d] "=x" (reg_d),
+ [reg_e] "=x" (reg_e),
+ [reg_f] "=x" (reg_f),
+ [reg_g] "=x" (reg_g),
+ [reg_h] "=x" (reg_h)
+ : [src] "r" (src),
+ [dst] "r"(dst)
+ : "memory"
+ );
+}
+
+#ifdef __INTEL_COMPILER
+#pragma warning(enable:593)
+#endif
+
+static inline void
+rte_mov256(uint8_t *dst, const uint8_t *src)
+{
+ rte_mov128(dst, src);
+ rte_mov128(dst + 128, src + 128);
+}
+
+#define rte_memcpy(dst, src, n) \
+ ((__builtin_constant_p(n)) ? \
+ memcpy((dst), (src), (n)) : \
+ rte_memcpy_func((dst), (src), (n)))
+
+static inline void *
+rte_memcpy_func(void *dst, const void *src, size_t n)
+{
+ void *ret = dst;
+
+ /* We can't copy < 16 bytes using XMM registers so do it manually. */
+ if (n < 16) {
+ if (n & 0x01) {
+ *(uint8_t *)dst = *(const uint8_t *)src;
+ dst = (uint8_t *)dst + 1;
+ src = (const uint8_t *)src + 1;
+ }
+ if (n & 0x02) {
+ *(uint16_t *)dst = *(const uint16_t *)src;
+ dst = (uint16_t *)dst + 1;
+ src = (const uint16_t *)src + 1;
+ }
+ if (n & 0x04) {
+ *(uint32_t *)dst = *(const uint32_t *)src;
+ dst = (uint32_t *)dst + 1;
+ src = (const uint32_t *)src + 1;
+ }
+ if (n & 0x08) {
+ *(uint64_t *)dst = *(const uint64_t *)src;
+ }
+ return ret;
+ }
+
+ /* Special fast cases for <= 128 bytes */
+ if (n <= 32) {
+ rte_mov16((uint8_t *)dst, (const uint8_t *)src);
+ rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n);
+ return ret;
+ }
+
+ if (n <= 64) {
+ rte_mov32((uint8_t *)dst, (const uint8_t *)src);
+ rte_mov32((uint8_t *)dst - 32 + n, (const uint8_t *)src - 32 + n);
+ return ret;
+ }
+
+ if (n <= 128) {
+ rte_mov64((uint8_t *)dst, (const uint8_t *)src);
+ rte_mov64((uint8_t *)dst - 64 + n, (const uint8_t *)src - 64 + n);
+ return ret;
+ }
+
+ /*
+ * For large copies > 128 bytes. This combination of 256, 64 and 16 byte
+ * copies was found to be faster than doing 128 and 32 byte copies as
+ * well.
+ */
+ for ( ; n >= 256; n -= 256) {
+ rte_mov256((uint8_t *)dst, (const uint8_t *)src);
+ dst = (uint8_t *)dst + 256;
+ src = (const uint8_t *)src + 256;
+ }
+
+ /*
+ * We split the remaining bytes (which will be less than 256) into
+ * 64byte (2^6) chunks.
+ * Using incrementing integers in the case labels of a switch statement
+ * enourages the compiler to use a jump table. To get incrementing
+ * integers, we shift the 2 relevant bits to the LSB position to first
+ * get decrementing integers, and then subtract.
+ */
+ switch (3 - (n >> 6)) {
+ case 0x00:
+ rte_mov64((uint8_t *)dst, (const uint8_t *)src);
+ n -= 64;
+ dst = (uint8_t *)dst + 64;
+ src = (const uint8_t *)src + 64; /* fallthrough */
+ case 0x01:
+ rte_mov64((uint8_t *)dst, (const uint8_t *)src);
+ n -= 64;
+ dst = (uint8_t *)dst + 64;
+ src = (const uint8_t *)src + 64; /* fallthrough */
+ case 0x02:
+ rte_mov64((uint8_t *)dst, (const uint8_t *)src);
+ n -= 64;
+ dst = (uint8_t *)dst + 64;
+ src = (const uint8_t *)src + 64; /* fallthrough */
+ default:
+ ;
+ }
+
+ /*
+ * We split the remaining bytes (which will be less than 64) into
+ * 16byte (2^4) chunks, using the same switch structure as above.
+ */
+ switch (3 - (n >> 4)) {
+ case 0x00:
+ rte_mov16((uint8_t *)dst, (const uint8_t *)src);
+ n -= 16;
+ dst = (uint8_t *)dst + 16;
+ src = (const uint8_t *)src + 16; /* fallthrough */
+ case 0x01:
+ rte_mov16((uint8_t *)dst, (const uint8_t *)src);
+ n -= 16;
+ dst = (uint8_t *)dst + 16;
+ src = (const uint8_t *)src + 16; /* fallthrough */
+ case 0x02:
+ rte_mov16((uint8_t *)dst, (const uint8_t *)src);
+ n -= 16;
+ dst = (uint8_t *)dst + 16;
+ src = (const uint8_t *)src + 16; /* fallthrough */
+ default:
+ ;
+ }
+
+ /* Copy any remaining bytes, without going beyond end of buffers */
+ if (n != 0) {
+ rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n);
+ }
+ return ret;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_MEMCPY_X86_64_H_ */
--- /dev/null
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_PREFETCH_X86_64_H_
+#define _RTE_PREFETCH_X86_64_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "generic/rte_prefetch.h"
+
+static inline void rte_prefetch0(volatile void *p)
+{
+ asm volatile ("prefetcht0 %[p]" : [p] "+m" (*(volatile char *)p));
+}
+
+static inline void rte_prefetch1(volatile void *p)
+{
+ asm volatile ("prefetcht1 %[p]" : [p] "+m" (*(volatile char *)p));
+}
+
+static inline void rte_prefetch2(volatile void *p)
+{
+ asm volatile ("prefetcht2 %[p]" : [p] "+m" (*(volatile char *)p));
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_PREFETCH_X86_64_H_ */
--- /dev/null
+/*-
+ * BSD LICENSE
+ *
+ * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _RTE_SPINLOCK_X86_64_H_
+#define _RTE_SPINLOCK_X86_64_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "generic/rte_spinlock.h"
+
+#ifndef RTE_FORCE_INTRINSICS
+static inline void
+rte_spinlock_lock(rte_spinlock_t *sl)
+{
+ int lock_val = 1;
+ asm volatile (
+ "1:\n"
+ "xchg %[locked], %[lv]\n"
+ "test %[lv], %[lv]\n"
+ "jz 3f\n"
+ "2:\n"
+ "pause\n"
+ "cmpl $0, %[locked]\n"
+ "jnz 2b\n"
+ "jmp 1b\n"
+ "3:\n"
+ : [locked] "=m" (sl->locked), [lv] "=q" (lock_val)
+ : "[lv]" (lock_val)
+ : "memory");
+}
+
+static inline void
+rte_spinlock_unlock (rte_spinlock_t *sl)
+{
+ int unlock_val = 0;
+ asm volatile (
+ "xchg %[locked], %[ulv]\n"
+ : [locked] "=m" (sl->locked), [ulv] "=q" (unlock_val)
+ : "[ulv]" (unlock_val)
+ : "memory");
+}
+
+static inline int
+rte_spinlock_trylock (rte_spinlock_t *sl)
+{
+ int lockval = 1;
+
+ asm volatile (
+ "xchg %[locked], %[lockval]"
+ : [locked] "=m" (sl->locked), [lockval] "=q" (lockval)
+ : "[lockval]" (lockval)
+ : "memory");
+
+ return (lockval == 0);
+}
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_SPINLOCK_X86_64_H_ */
+++ /dev/null
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-/*
- * Inspired from FreeBSD src/sys/amd64/include/atomic.h
- * Copyright (c) 1998 Doug Rabson
- * All rights reserved.
- */
-
-#ifndef _RTE_ATOMIC_X86_64_H_
-#define _RTE_ATOMIC_X86_64_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <emmintrin.h>
-#include "generic/rte_atomic.h"
-
-#if RTE_MAX_LCORE == 1
-#define MPLOCKED /**< No need to insert MP lock prefix. */
-#else
-#define MPLOCKED "lock ; " /**< Insert MP lock prefix. */
-#endif
-
-#define rte_mb() _mm_mfence()
-
-#define rte_wmb() _mm_sfence()
-
-#define rte_rmb() _mm_lfence()
-
-/*------------------------- 16 bit atomic operations -------------------------*/
-
-#ifndef RTE_FORCE_INTRINSICS
-static inline int
-rte_atomic16_cmpset(volatile uint16_t *dst, uint16_t exp, uint16_t src)
-{
- uint8_t res;
-
- asm volatile(
- MPLOCKED
- "cmpxchgw %[src], %[dst];"
- "sete %[res];"
- : [res] "=a" (res), /* output */
- [dst] "=m" (*dst)
- : [src] "r" (src), /* input */
- "a" (exp),
- "m" (*dst)
- : "memory"); /* no-clobber list */
- return res;
-}
-
-static inline int rte_atomic16_test_and_set(rte_atomic16_t *v)
-{
- return rte_atomic16_cmpset((volatile uint16_t *)&v->cnt, 0, 1);
-}
-
-static inline void
-rte_atomic16_inc(rte_atomic16_t *v)
-{
- asm volatile(
- MPLOCKED
- "incw %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : "m" (v->cnt) /* input */
- );
-}
-
-static inline void
-rte_atomic16_dec(rte_atomic16_t *v)
-{
- asm volatile(
- MPLOCKED
- "decw %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : "m" (v->cnt) /* input */
- );
-}
-
-static inline int rte_atomic16_inc_and_test(rte_atomic16_t *v)
-{
- uint8_t ret;
-
- asm volatile(
- MPLOCKED
- "incw %[cnt] ; "
- "sete %[ret]"
- : [cnt] "+m" (v->cnt), /* output */
- [ret] "=qm" (ret)
- );
- return (ret != 0);
-}
-
-static inline int rte_atomic16_dec_and_test(rte_atomic16_t *v)
-{
- uint8_t ret;
-
- asm volatile(MPLOCKED
- "decw %[cnt] ; "
- "sete %[ret]"
- : [cnt] "+m" (v->cnt), /* output */
- [ret] "=qm" (ret)
- );
- return (ret != 0);
-}
-
-/*------------------------- 32 bit atomic operations -------------------------*/
-
-static inline int
-rte_atomic32_cmpset(volatile uint32_t *dst, uint32_t exp, uint32_t src)
-{
- uint8_t res;
-
- asm volatile(
- MPLOCKED
- "cmpxchgl %[src], %[dst];"
- "sete %[res];"
- : [res] "=a" (res), /* output */
- [dst] "=m" (*dst)
- : [src] "r" (src), /* input */
- "a" (exp),
- "m" (*dst)
- : "memory"); /* no-clobber list */
- return res;
-}
-
-static inline int rte_atomic32_test_and_set(rte_atomic32_t *v)
-{
- return rte_atomic32_cmpset((volatile uint32_t *)&v->cnt, 0, 1);
-}
-
-static inline void
-rte_atomic32_inc(rte_atomic32_t *v)
-{
- asm volatile(
- MPLOCKED
- "incl %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : "m" (v->cnt) /* input */
- );
-}
-
-static inline void
-rte_atomic32_dec(rte_atomic32_t *v)
-{
- asm volatile(
- MPLOCKED
- "decl %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : "m" (v->cnt) /* input */
- );
-}
-
-static inline int rte_atomic32_inc_and_test(rte_atomic32_t *v)
-{
- uint8_t ret;
-
- asm volatile(
- MPLOCKED
- "incl %[cnt] ; "
- "sete %[ret]"
- : [cnt] "+m" (v->cnt), /* output */
- [ret] "=qm" (ret)
- );
- return (ret != 0);
-}
-
-static inline int rte_atomic32_dec_and_test(rte_atomic32_t *v)
-{
- uint8_t ret;
-
- asm volatile(MPLOCKED
- "decl %[cnt] ; "
- "sete %[ret]"
- : [cnt] "+m" (v->cnt), /* output */
- [ret] "=qm" (ret)
- );
- return (ret != 0);
-}
-
-/*------------------------- 64 bit atomic operations -------------------------*/
-
-static inline int
-rte_atomic64_cmpset(volatile uint64_t *dst, uint64_t exp, uint64_t src)
-{
- uint8_t res;
-
-
- asm volatile(
- MPLOCKED
- "cmpxchgq %[src], %[dst];"
- "sete %[res];"
- : [res] "=a" (res), /* output */
- [dst] "=m" (*dst)
- : [src] "r" (src), /* input */
- "a" (exp),
- "m" (*dst)
- : "memory"); /* no-clobber list */
-
- return res;
-}
-
-static inline void
-rte_atomic64_init(rte_atomic64_t *v)
-{
- v->cnt = 0;
-}
-
-static inline int64_t
-rte_atomic64_read(rte_atomic64_t *v)
-{
- return v->cnt;
-}
-
-static inline void
-rte_atomic64_set(rte_atomic64_t *v, int64_t new_value)
-{
- v->cnt = new_value;
-}
-
-static inline void
-rte_atomic64_add(rte_atomic64_t *v, int64_t inc)
-{
- asm volatile(
- MPLOCKED
- "addq %[inc], %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : [inc] "ir" (inc), /* input */
- "m" (v->cnt)
- );
-}
-
-static inline void
-rte_atomic64_sub(rte_atomic64_t *v, int64_t dec)
-{
- asm volatile(
- MPLOCKED
- "subq %[dec], %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : [dec] "ir" (dec), /* input */
- "m" (v->cnt)
- );
-}
-
-static inline void
-rte_atomic64_inc(rte_atomic64_t *v)
-{
- asm volatile(
- MPLOCKED
- "incq %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : "m" (v->cnt) /* input */
- );
-}
-
-static inline void
-rte_atomic64_dec(rte_atomic64_t *v)
-{
- asm volatile(
- MPLOCKED
- "decq %[cnt]"
- : [cnt] "=m" (v->cnt) /* output */
- : "m" (v->cnt) /* input */
- );
-}
-
-static inline int64_t
-rte_atomic64_add_return(rte_atomic64_t *v, int64_t inc)
-{
- int64_t prev = inc;
-
- asm volatile(
- MPLOCKED
- "xaddq %[prev], %[cnt]"
- : [prev] "+r" (prev), /* output */
- [cnt] "=m" (v->cnt)
- : "m" (v->cnt) /* input */
- );
- return prev + inc;
-}
-
-static inline int64_t
-rte_atomic64_sub_return(rte_atomic64_t *v, int64_t dec)
-{
- return rte_atomic64_add_return(v, -dec);
-}
-
-static inline int rte_atomic64_inc_and_test(rte_atomic64_t *v)
-{
- uint8_t ret;
-
- asm volatile(
- MPLOCKED
- "incq %[cnt] ; "
- "sete %[ret]"
- : [cnt] "+m" (v->cnt), /* output */
- [ret] "=qm" (ret)
- );
-
- return ret != 0;
-}
-
-static inline int rte_atomic64_dec_and_test(rte_atomic64_t *v)
-{
- uint8_t ret;
-
- asm volatile(
- MPLOCKED
- "decq %[cnt] ; "
- "sete %[ret]"
- : [cnt] "+m" (v->cnt), /* output */
- [ret] "=qm" (ret)
- );
- return ret != 0;
-}
-
-static inline int rte_atomic64_test_and_set(rte_atomic64_t *v)
-{
- return rte_atomic64_cmpset((volatile uint64_t *)&v->cnt, 0, 1);
-}
-
-static inline void rte_atomic64_clear(rte_atomic64_t *v)
-{
- v->cnt = 0;
-}
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_ATOMIC_X86_64_H_ */
+++ /dev/null
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _RTE_BYTEORDER_X86_64_H_
-#define _RTE_BYTEORDER_X86_64_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "generic/rte_byteorder.h"
-
-/*
- * An architecture-optimized byte swap for a 16-bit value.
- *
- * Do not use this function directly. The preferred function is rte_bswap16().
- */
-static inline uint16_t rte_arch_bswap16(uint16_t _x)
-{
- register uint16_t x = _x;
- asm volatile ("xchgb %b[x1],%h[x2]"
- : [x1] "=Q" (x)
- : [x2] "0" (x)
- );
- return x;
-}
-
-/*
- * An architecture-optimized byte swap for a 32-bit value.
- *
- * Do not use this function directly. The preferred function is rte_bswap32().
- */
-static inline uint32_t rte_arch_bswap32(uint32_t _x)
-{
- register uint32_t x = _x;
- asm volatile ("bswap %[x]"
- : [x] "+r" (x)
- );
- return x;
-}
-
-/*
- * An architecture-optimized byte swap for a 64-bit value.
- *
- * Do not use this function directly. The preferred function is rte_bswap64().
- */
-/* 64-bit mode */
-static inline uint64_t rte_arch_bswap64(uint64_t _x)
-{
- register uint64_t x = _x;
- asm volatile ("bswap %[x]"
- : [x] "+r" (x)
- );
- return x;
-}
-
-#ifndef RTE_FORCE_INTRINSICS
-#define rte_bswap16(x) ((uint16_t)(__builtin_constant_p(x) ? \
- rte_constant_bswap16(x) : \
- rte_arch_bswap16(x)))
-
-#define rte_bswap32(x) ((uint32_t)(__builtin_constant_p(x) ? \
- rte_constant_bswap32(x) : \
- rte_arch_bswap32(x)))
-
-#define rte_bswap64(x) ((uint64_t)(__builtin_constant_p(x) ? \
- rte_constant_bswap64(x) : \
- rte_arch_bswap64(x)))
-#else
-/*
- * __builtin_bswap16 is only available gcc 4.8 and upwards
- */
-#if __GNUC__ < 4 || (__GNUC__ == 4 && __GNUC_MINOR__ < 8)
-#define rte_bswap16(x) ((uint16_t)(__builtin_constant_p(x) ? \
- rte_constant_bswap16(x) : \
- rte_arch_bswap16(x)))
-#endif
-#endif
-
-#define rte_cpu_to_le_16(x) (x)
-#define rte_cpu_to_le_32(x) (x)
-#define rte_cpu_to_le_64(x) (x)
-
-#define rte_cpu_to_be_16(x) rte_bswap16(x)
-#define rte_cpu_to_be_32(x) rte_bswap32(x)
-#define rte_cpu_to_be_64(x) rte_bswap64(x)
-
-#define rte_le_to_cpu_16(x) (x)
-#define rte_le_to_cpu_32(x) (x)
-#define rte_le_to_cpu_64(x) (x)
-
-#define rte_be_to_cpu_16(x) rte_bswap16(x)
-#define rte_be_to_cpu_32(x) rte_bswap32(x)
-#define rte_be_to_cpu_64(x) rte_bswap64(x)
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_BYTEORDER_X86_64_H_ */
+++ /dev/null
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _RTE_CPUFLAGS_X86_64_H_
-#define _RTE_CPUFLAGS_X86_64_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdlib.h>
-#include <stdio.h>
-#include <errno.h>
-#include <stdint.h>
-
-#include "generic/rte_cpuflags.h"
-
-enum rte_cpu_flag_t {
- /* (EAX 01h) ECX features*/
- RTE_CPUFLAG_SSE3 = 0, /**< SSE3 */
- RTE_CPUFLAG_PCLMULQDQ, /**< PCLMULQDQ */
- RTE_CPUFLAG_DTES64, /**< DTES64 */
- RTE_CPUFLAG_MONITOR, /**< MONITOR */
- RTE_CPUFLAG_DS_CPL, /**< DS_CPL */
- RTE_CPUFLAG_VMX, /**< VMX */
- RTE_CPUFLAG_SMX, /**< SMX */
- RTE_CPUFLAG_EIST, /**< EIST */
- RTE_CPUFLAG_TM2, /**< TM2 */
- RTE_CPUFLAG_SSSE3, /**< SSSE3 */
- RTE_CPUFLAG_CNXT_ID, /**< CNXT_ID */
- RTE_CPUFLAG_FMA, /**< FMA */
- RTE_CPUFLAG_CMPXCHG16B, /**< CMPXCHG16B */
- RTE_CPUFLAG_XTPR, /**< XTPR */
- RTE_CPUFLAG_PDCM, /**< PDCM */
- RTE_CPUFLAG_PCID, /**< PCID */
- RTE_CPUFLAG_DCA, /**< DCA */
- RTE_CPUFLAG_SSE4_1, /**< SSE4_1 */
- RTE_CPUFLAG_SSE4_2, /**< SSE4_2 */
- RTE_CPUFLAG_X2APIC, /**< X2APIC */
- RTE_CPUFLAG_MOVBE, /**< MOVBE */
- RTE_CPUFLAG_POPCNT, /**< POPCNT */
- RTE_CPUFLAG_TSC_DEADLINE, /**< TSC_DEADLINE */
- RTE_CPUFLAG_AES, /**< AES */
- RTE_CPUFLAG_XSAVE, /**< XSAVE */
- RTE_CPUFLAG_OSXSAVE, /**< OSXSAVE */
- RTE_CPUFLAG_AVX, /**< AVX */
- RTE_CPUFLAG_F16C, /**< F16C */
- RTE_CPUFLAG_RDRAND, /**< RDRAND */
-
- /* (EAX 01h) EDX features */
- RTE_CPUFLAG_FPU, /**< FPU */
- RTE_CPUFLAG_VME, /**< VME */
- RTE_CPUFLAG_DE, /**< DE */
- RTE_CPUFLAG_PSE, /**< PSE */
- RTE_CPUFLAG_TSC, /**< TSC */
- RTE_CPUFLAG_MSR, /**< MSR */
- RTE_CPUFLAG_PAE, /**< PAE */
- RTE_CPUFLAG_MCE, /**< MCE */
- RTE_CPUFLAG_CX8, /**< CX8 */
- RTE_CPUFLAG_APIC, /**< APIC */
- RTE_CPUFLAG_SEP, /**< SEP */
- RTE_CPUFLAG_MTRR, /**< MTRR */
- RTE_CPUFLAG_PGE, /**< PGE */
- RTE_CPUFLAG_MCA, /**< MCA */
- RTE_CPUFLAG_CMOV, /**< CMOV */
- RTE_CPUFLAG_PAT, /**< PAT */
- RTE_CPUFLAG_PSE36, /**< PSE36 */
- RTE_CPUFLAG_PSN, /**< PSN */
- RTE_CPUFLAG_CLFSH, /**< CLFSH */
- RTE_CPUFLAG_DS, /**< DS */
- RTE_CPUFLAG_ACPI, /**< ACPI */
- RTE_CPUFLAG_MMX, /**< MMX */
- RTE_CPUFLAG_FXSR, /**< FXSR */
- RTE_CPUFLAG_SSE, /**< SSE */
- RTE_CPUFLAG_SSE2, /**< SSE2 */
- RTE_CPUFLAG_SS, /**< SS */
- RTE_CPUFLAG_HTT, /**< HTT */
- RTE_CPUFLAG_TM, /**< TM */
- RTE_CPUFLAG_PBE, /**< PBE */
-
- /* (EAX 06h) EAX features */
- RTE_CPUFLAG_DIGTEMP, /**< DIGTEMP */
- RTE_CPUFLAG_TRBOBST, /**< TRBOBST */
- RTE_CPUFLAG_ARAT, /**< ARAT */
- RTE_CPUFLAG_PLN, /**< PLN */
- RTE_CPUFLAG_ECMD, /**< ECMD */
- RTE_CPUFLAG_PTM, /**< PTM */
-
- /* (EAX 06h) ECX features */
- RTE_CPUFLAG_MPERF_APERF_MSR, /**< MPERF_APERF_MSR */
- RTE_CPUFLAG_ACNT2, /**< ACNT2 */
- RTE_CPUFLAG_ENERGY_EFF, /**< ENERGY_EFF */
-
- /* (EAX 07h, ECX 0h) EBX features */
- RTE_CPUFLAG_FSGSBASE, /**< FSGSBASE */
- RTE_CPUFLAG_BMI1, /**< BMI1 */
- RTE_CPUFLAG_HLE, /**< Hardware Lock elision */
- RTE_CPUFLAG_AVX2, /**< AVX2 */
- RTE_CPUFLAG_SMEP, /**< SMEP */
- RTE_CPUFLAG_BMI2, /**< BMI2 */
- RTE_CPUFLAG_ERMS, /**< ERMS */
- RTE_CPUFLAG_INVPCID, /**< INVPCID */
- RTE_CPUFLAG_RTM, /**< Transactional memory */
-
- /* (EAX 80000001h) ECX features */
- RTE_CPUFLAG_LAHF_SAHF, /**< LAHF_SAHF */
- RTE_CPUFLAG_LZCNT, /**< LZCNT */
-
- /* (EAX 80000001h) EDX features */
- RTE_CPUFLAG_SYSCALL, /**< SYSCALL */
- RTE_CPUFLAG_XD, /**< XD */
- RTE_CPUFLAG_1GB_PG, /**< 1GB_PG */
- RTE_CPUFLAG_RDTSCP, /**< RDTSCP */
- RTE_CPUFLAG_EM64T, /**< EM64T */
-
- /* (EAX 80000007h) EDX features */
- RTE_CPUFLAG_INVTSC, /**< INVTSC */
-
- /* The last item */
- RTE_CPUFLAG_NUMFLAGS, /**< This should always be the last! */
-};
-
-enum cpu_register_t {
- REG_EAX = 0,
- REG_EBX,
- REG_ECX,
- REG_EDX,
-};
-
-static const struct feature_entry cpu_feature_table[] = {
- FEAT_DEF(SSE3, 0x00000001, 0, REG_ECX, 0)
- FEAT_DEF(PCLMULQDQ, 0x00000001, 0, REG_ECX, 1)
- FEAT_DEF(DTES64, 0x00000001, 0, REG_ECX, 2)
- FEAT_DEF(MONITOR, 0x00000001, 0, REG_ECX, 3)
- FEAT_DEF(DS_CPL, 0x00000001, 0, REG_ECX, 4)
- FEAT_DEF(VMX, 0x00000001, 0, REG_ECX, 5)
- FEAT_DEF(SMX, 0x00000001, 0, REG_ECX, 6)
- FEAT_DEF(EIST, 0x00000001, 0, REG_ECX, 7)
- FEAT_DEF(TM2, 0x00000001, 0, REG_ECX, 8)
- FEAT_DEF(SSSE3, 0x00000001, 0, REG_ECX, 9)
- FEAT_DEF(CNXT_ID, 0x00000001, 0, REG_ECX, 10)
- FEAT_DEF(FMA, 0x00000001, 0, REG_ECX, 12)
- FEAT_DEF(CMPXCHG16B, 0x00000001, 0, REG_ECX, 13)
- FEAT_DEF(XTPR, 0x00000001, 0, REG_ECX, 14)
- FEAT_DEF(PDCM, 0x00000001, 0, REG_ECX, 15)
- FEAT_DEF(PCID, 0x00000001, 0, REG_ECX, 17)
- FEAT_DEF(DCA, 0x00000001, 0, REG_ECX, 18)
- FEAT_DEF(SSE4_1, 0x00000001, 0, REG_ECX, 19)
- FEAT_DEF(SSE4_2, 0x00000001, 0, REG_ECX, 20)
- FEAT_DEF(X2APIC, 0x00000001, 0, REG_ECX, 21)
- FEAT_DEF(MOVBE, 0x00000001, 0, REG_ECX, 22)
- FEAT_DEF(POPCNT, 0x00000001, 0, REG_ECX, 23)
- FEAT_DEF(TSC_DEADLINE, 0x00000001, 0, REG_ECX, 24)
- FEAT_DEF(AES, 0x00000001, 0, REG_ECX, 25)
- FEAT_DEF(XSAVE, 0x00000001, 0, REG_ECX, 26)
- FEAT_DEF(OSXSAVE, 0x00000001, 0, REG_ECX, 27)
- FEAT_DEF(AVX, 0x00000001, 0, REG_ECX, 28)
- FEAT_DEF(F16C, 0x00000001, 0, REG_ECX, 29)
- FEAT_DEF(RDRAND, 0x00000001, 0, REG_ECX, 30)
-
- FEAT_DEF(FPU, 0x00000001, 0, REG_EDX, 0)
- FEAT_DEF(VME, 0x00000001, 0, REG_EDX, 1)
- FEAT_DEF(DE, 0x00000001, 0, REG_EDX, 2)
- FEAT_DEF(PSE, 0x00000001, 0, REG_EDX, 3)
- FEAT_DEF(TSC, 0x00000001, 0, REG_EDX, 4)
- FEAT_DEF(MSR, 0x00000001, 0, REG_EDX, 5)
- FEAT_DEF(PAE, 0x00000001, 0, REG_EDX, 6)
- FEAT_DEF(MCE, 0x00000001, 0, REG_EDX, 7)
- FEAT_DEF(CX8, 0x00000001, 0, REG_EDX, 8)
- FEAT_DEF(APIC, 0x00000001, 0, REG_EDX, 9)
- FEAT_DEF(SEP, 0x00000001, 0, REG_EDX, 11)
- FEAT_DEF(MTRR, 0x00000001, 0, REG_EDX, 12)
- FEAT_DEF(PGE, 0x00000001, 0, REG_EDX, 13)
- FEAT_DEF(MCA, 0x00000001, 0, REG_EDX, 14)
- FEAT_DEF(CMOV, 0x00000001, 0, REG_EDX, 15)
- FEAT_DEF(PAT, 0x00000001, 0, REG_EDX, 16)
- FEAT_DEF(PSE36, 0x00000001, 0, REG_EDX, 17)
- FEAT_DEF(PSN, 0x00000001, 0, REG_EDX, 18)
- FEAT_DEF(CLFSH, 0x00000001, 0, REG_EDX, 19)
- FEAT_DEF(DS, 0x00000001, 0, REG_EDX, 21)
- FEAT_DEF(ACPI, 0x00000001, 0, REG_EDX, 22)
- FEAT_DEF(MMX, 0x00000001, 0, REG_EDX, 23)
- FEAT_DEF(FXSR, 0x00000001, 0, REG_EDX, 24)
- FEAT_DEF(SSE, 0x00000001, 0, REG_EDX, 25)
- FEAT_DEF(SSE2, 0x00000001, 0, REG_EDX, 26)
- FEAT_DEF(SS, 0x00000001, 0, REG_EDX, 27)
- FEAT_DEF(HTT, 0x00000001, 0, REG_EDX, 28)
- FEAT_DEF(TM, 0x00000001, 0, REG_EDX, 29)
- FEAT_DEF(PBE, 0x00000001, 0, REG_EDX, 31)
-
- FEAT_DEF(DIGTEMP, 0x00000006, 0, REG_EAX, 0)
- FEAT_DEF(TRBOBST, 0x00000006, 0, REG_EAX, 1)
- FEAT_DEF(ARAT, 0x00000006, 0, REG_EAX, 2)
- FEAT_DEF(PLN, 0x00000006, 0, REG_EAX, 4)
- FEAT_DEF(ECMD, 0x00000006, 0, REG_EAX, 5)
- FEAT_DEF(PTM, 0x00000006, 0, REG_EAX, 6)
-
- FEAT_DEF(MPERF_APERF_MSR, 0x00000006, 0, REG_ECX, 0)
- FEAT_DEF(ACNT2, 0x00000006, 0, REG_ECX, 1)
- FEAT_DEF(ENERGY_EFF, 0x00000006, 0, REG_ECX, 3)
-
- FEAT_DEF(FSGSBASE, 0x00000007, 0, REG_EBX, 0)
- FEAT_DEF(BMI1, 0x00000007, 0, REG_EBX, 2)
- FEAT_DEF(HLE, 0x00000007, 0, REG_EBX, 4)
- FEAT_DEF(AVX2, 0x00000007, 0, REG_EBX, 5)
- FEAT_DEF(SMEP, 0x00000007, 0, REG_EBX, 6)
- FEAT_DEF(BMI2, 0x00000007, 0, REG_EBX, 7)
- FEAT_DEF(ERMS, 0x00000007, 0, REG_EBX, 8)
- FEAT_DEF(INVPCID, 0x00000007, 0, REG_EBX, 10)
- FEAT_DEF(RTM, 0x00000007, 0, REG_EBX, 11)
-
- FEAT_DEF(LAHF_SAHF, 0x80000001, 0, REG_ECX, 0)
- FEAT_DEF(LZCNT, 0x80000001, 0, REG_ECX, 4)
-
- FEAT_DEF(SYSCALL, 0x80000001, 0, REG_EDX, 11)
- FEAT_DEF(XD, 0x80000001, 0, REG_EDX, 20)
- FEAT_DEF(1GB_PG, 0x80000001, 0, REG_EDX, 26)
- FEAT_DEF(RDTSCP, 0x80000001, 0, REG_EDX, 27)
- FEAT_DEF(EM64T, 0x80000001, 0, REG_EDX, 29)
-
- FEAT_DEF(INVTSC, 0x80000007, 0, REG_EDX, 8)
-};
-
-static inline void
-rte_cpu_get_features(uint32_t leaf, uint32_t subleaf, cpuid_registers_t out)
-{
-#if defined(__i386__) && defined(__PIC__)
- /* %ebx is a forbidden register if we compile with -fPIC or -fPIE */
- asm volatile("movl %%ebx,%0 ; cpuid ; xchgl %%ebx,%0"
- : "=r" (out[REG_EBX]),
- "=a" (out[REG_EAX]),
- "=c" (out[REG_ECX]),
- "=d" (out[REG_EDX])
- : "a" (leaf), "c" (subleaf));
-#else
-
- asm volatile("cpuid"
- : "=a" (out[REG_EAX]),
- "=b" (out[REG_EBX]),
- "=c" (out[REG_ECX]),
- "=d" (out[REG_EDX])
- : "a" (leaf), "c" (subleaf));
-
-#endif
-}
-
-static inline int
-rte_cpu_get_flag_enabled(enum rte_cpu_flag_t feature)
-{
- const struct feature_entry *feat;
- cpuid_registers_t regs;
-
-
- if (feature >= RTE_CPUFLAG_NUMFLAGS)
- /* Flag does not match anything in the feature tables */
- return -ENOENT;
-
- feat = &cpu_feature_table[feature];
-
- if (!feat->leaf)
- /* This entry in the table wasn't filled out! */
- return -EFAULT;
-
- rte_cpu_get_features(feat->leaf & 0xffff0000, 0, regs);
- if (((regs[REG_EAX] ^ feat->leaf) & 0xffff0000) ||
- regs[REG_EAX] < feat->leaf)
- return 0;
-
- /* get the cpuid leaf containing the desired feature */
- rte_cpu_get_features(feat->leaf, feat->subleaf, regs);
-
- /* check if the feature is enabled */
- return (regs[feat->reg] >> feat->bit) & 1;
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_CPUFLAGS_X86_64_H_ */
+++ /dev/null
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-/* BSD LICENSE
- *
- * Copyright(c) 2013 6WIND.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of 6WIND S.A. nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _RTE_CYCLES_X86_64_H_
-#define _RTE_CYCLES_X86_64_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "generic/rte_cycles.h"
-
-#ifdef RTE_LIBRTE_EAL_VMWARE_TSC_MAP_SUPPORT
-/* Global switch to use VMWARE mapping of TSC instead of RDTSC */
-extern int rte_cycles_vmware_tsc_map;
-#include <rte_branch_prediction.h>
-#endif
-
-static inline uint64_t
-rte_rdtsc(void)
-{
- union {
- uint64_t tsc_64;
- struct {
- uint32_t lo_32;
- uint32_t hi_32;
- };
- } tsc;
-
-#ifdef RTE_LIBRTE_EAL_VMWARE_TSC_MAP_SUPPORT
- if (unlikely(rte_cycles_vmware_tsc_map)) {
- /* ecx = 0x10000 corresponds to the physical TSC for VMware */
- asm volatile("rdpmc" :
- "=a" (tsc.lo_32),
- "=d" (tsc.hi_32) :
- "c"(0x10000));
- return tsc.tsc_64;
- }
-#endif
-
- asm volatile("rdtsc" :
- "=a" (tsc.lo_32),
- "=d" (tsc.hi_32));
- return tsc.tsc_64;
-}
-
-static inline uint64_t
-rte_rdtsc_precise(void)
-{
- rte_mb();
- return rte_rdtsc();
-}
-
-static inline uint64_t
-rte_get_tsc_cycles(void) { return rte_rdtsc(); }
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_CYCLES_X86_64_H_ */
+++ /dev/null
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _RTE_MEMCPY_X86_64_H_
-#define _RTE_MEMCPY_X86_64_H_
-
-#include <stdint.h>
-#include <string.h>
-#include <emmintrin.h>
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "generic/rte_memcpy.h"
-
-#ifdef __INTEL_COMPILER
-#pragma warning(disable:593) /* Stop unused variable warning (reg_a etc). */
-#endif
-
-static inline void
-rte_mov16(uint8_t *dst, const uint8_t *src)
-{
- __m128i reg_a;
- asm volatile (
- "movdqu (%[src]), %[reg_a]\n\t"
- "movdqu %[reg_a], (%[dst])\n\t"
- : [reg_a] "=x" (reg_a)
- : [src] "r" (src),
- [dst] "r"(dst)
- : "memory"
- );
-}
-
-static inline void
-rte_mov32(uint8_t *dst, const uint8_t *src)
-{
- __m128i reg_a, reg_b;
- asm volatile (
- "movdqu (%[src]), %[reg_a]\n\t"
- "movdqu 16(%[src]), %[reg_b]\n\t"
- "movdqu %[reg_a], (%[dst])\n\t"
- "movdqu %[reg_b], 16(%[dst])\n\t"
- : [reg_a] "=x" (reg_a),
- [reg_b] "=x" (reg_b)
- : [src] "r" (src),
- [dst] "r"(dst)
- : "memory"
- );
-}
-
-static inline void
-rte_mov48(uint8_t *dst, const uint8_t *src)
-{
- __m128i reg_a, reg_b, reg_c;
- asm volatile (
- "movdqu (%[src]), %[reg_a]\n\t"
- "movdqu 16(%[src]), %[reg_b]\n\t"
- "movdqu 32(%[src]), %[reg_c]\n\t"
- "movdqu %[reg_a], (%[dst])\n\t"
- "movdqu %[reg_b], 16(%[dst])\n\t"
- "movdqu %[reg_c], 32(%[dst])\n\t"
- : [reg_a] "=x" (reg_a),
- [reg_b] "=x" (reg_b),
- [reg_c] "=x" (reg_c)
- : [src] "r" (src),
- [dst] "r"(dst)
- : "memory"
- );
-}
-
-static inline void
-rte_mov64(uint8_t *dst, const uint8_t *src)
-{
- __m128i reg_a, reg_b, reg_c, reg_d;
- asm volatile (
- "movdqu (%[src]), %[reg_a]\n\t"
- "movdqu 16(%[src]), %[reg_b]\n\t"
- "movdqu 32(%[src]), %[reg_c]\n\t"
- "movdqu 48(%[src]), %[reg_d]\n\t"
- "movdqu %[reg_a], (%[dst])\n\t"
- "movdqu %[reg_b], 16(%[dst])\n\t"
- "movdqu %[reg_c], 32(%[dst])\n\t"
- "movdqu %[reg_d], 48(%[dst])\n\t"
- : [reg_a] "=x" (reg_a),
- [reg_b] "=x" (reg_b),
- [reg_c] "=x" (reg_c),
- [reg_d] "=x" (reg_d)
- : [src] "r" (src),
- [dst] "r"(dst)
- : "memory"
- );
-}
-
-static inline void
-rte_mov128(uint8_t *dst, const uint8_t *src)
-{
- __m128i reg_a, reg_b, reg_c, reg_d, reg_e, reg_f, reg_g, reg_h;
- asm volatile (
- "movdqu (%[src]), %[reg_a]\n\t"
- "movdqu 16(%[src]), %[reg_b]\n\t"
- "movdqu 32(%[src]), %[reg_c]\n\t"
- "movdqu 48(%[src]), %[reg_d]\n\t"
- "movdqu 64(%[src]), %[reg_e]\n\t"
- "movdqu 80(%[src]), %[reg_f]\n\t"
- "movdqu 96(%[src]), %[reg_g]\n\t"
- "movdqu 112(%[src]), %[reg_h]\n\t"
- "movdqu %[reg_a], (%[dst])\n\t"
- "movdqu %[reg_b], 16(%[dst])\n\t"
- "movdqu %[reg_c], 32(%[dst])\n\t"
- "movdqu %[reg_d], 48(%[dst])\n\t"
- "movdqu %[reg_e], 64(%[dst])\n\t"
- "movdqu %[reg_f], 80(%[dst])\n\t"
- "movdqu %[reg_g], 96(%[dst])\n\t"
- "movdqu %[reg_h], 112(%[dst])\n\t"
- : [reg_a] "=x" (reg_a),
- [reg_b] "=x" (reg_b),
- [reg_c] "=x" (reg_c),
- [reg_d] "=x" (reg_d),
- [reg_e] "=x" (reg_e),
- [reg_f] "=x" (reg_f),
- [reg_g] "=x" (reg_g),
- [reg_h] "=x" (reg_h)
- : [src] "r" (src),
- [dst] "r"(dst)
- : "memory"
- );
-}
-
-#ifdef __INTEL_COMPILER
-#pragma warning(enable:593)
-#endif
-
-static inline void
-rte_mov256(uint8_t *dst, const uint8_t *src)
-{
- rte_mov128(dst, src);
- rte_mov128(dst + 128, src + 128);
-}
-
-#define rte_memcpy(dst, src, n) \
- ((__builtin_constant_p(n)) ? \
- memcpy((dst), (src), (n)) : \
- rte_memcpy_func((dst), (src), (n)))
-
-static inline void *
-rte_memcpy_func(void *dst, const void *src, size_t n)
-{
- void *ret = dst;
-
- /* We can't copy < 16 bytes using XMM registers so do it manually. */
- if (n < 16) {
- if (n & 0x01) {
- *(uint8_t *)dst = *(const uint8_t *)src;
- dst = (uint8_t *)dst + 1;
- src = (const uint8_t *)src + 1;
- }
- if (n & 0x02) {
- *(uint16_t *)dst = *(const uint16_t *)src;
- dst = (uint16_t *)dst + 1;
- src = (const uint16_t *)src + 1;
- }
- if (n & 0x04) {
- *(uint32_t *)dst = *(const uint32_t *)src;
- dst = (uint32_t *)dst + 1;
- src = (const uint32_t *)src + 1;
- }
- if (n & 0x08) {
- *(uint64_t *)dst = *(const uint64_t *)src;
- }
- return ret;
- }
-
- /* Special fast cases for <= 128 bytes */
- if (n <= 32) {
- rte_mov16((uint8_t *)dst, (const uint8_t *)src);
- rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n);
- return ret;
- }
-
- if (n <= 64) {
- rte_mov32((uint8_t *)dst, (const uint8_t *)src);
- rte_mov32((uint8_t *)dst - 32 + n, (const uint8_t *)src - 32 + n);
- return ret;
- }
-
- if (n <= 128) {
- rte_mov64((uint8_t *)dst, (const uint8_t *)src);
- rte_mov64((uint8_t *)dst - 64 + n, (const uint8_t *)src - 64 + n);
- return ret;
- }
-
- /*
- * For large copies > 128 bytes. This combination of 256, 64 and 16 byte
- * copies was found to be faster than doing 128 and 32 byte copies as
- * well.
- */
- for ( ; n >= 256; n -= 256) {
- rte_mov256((uint8_t *)dst, (const uint8_t *)src);
- dst = (uint8_t *)dst + 256;
- src = (const uint8_t *)src + 256;
- }
-
- /*
- * We split the remaining bytes (which will be less than 256) into
- * 64byte (2^6) chunks.
- * Using incrementing integers in the case labels of a switch statement
- * enourages the compiler to use a jump table. To get incrementing
- * integers, we shift the 2 relevant bits to the LSB position to first
- * get decrementing integers, and then subtract.
- */
- switch (3 - (n >> 6)) {
- case 0x00:
- rte_mov64((uint8_t *)dst, (const uint8_t *)src);
- n -= 64;
- dst = (uint8_t *)dst + 64;
- src = (const uint8_t *)src + 64; /* fallthrough */
- case 0x01:
- rte_mov64((uint8_t *)dst, (const uint8_t *)src);
- n -= 64;
- dst = (uint8_t *)dst + 64;
- src = (const uint8_t *)src + 64; /* fallthrough */
- case 0x02:
- rte_mov64((uint8_t *)dst, (const uint8_t *)src);
- n -= 64;
- dst = (uint8_t *)dst + 64;
- src = (const uint8_t *)src + 64; /* fallthrough */
- default:
- ;
- }
-
- /*
- * We split the remaining bytes (which will be less than 64) into
- * 16byte (2^4) chunks, using the same switch structure as above.
- */
- switch (3 - (n >> 4)) {
- case 0x00:
- rte_mov16((uint8_t *)dst, (const uint8_t *)src);
- n -= 16;
- dst = (uint8_t *)dst + 16;
- src = (const uint8_t *)src + 16; /* fallthrough */
- case 0x01:
- rte_mov16((uint8_t *)dst, (const uint8_t *)src);
- n -= 16;
- dst = (uint8_t *)dst + 16;
- src = (const uint8_t *)src + 16; /* fallthrough */
- case 0x02:
- rte_mov16((uint8_t *)dst, (const uint8_t *)src);
- n -= 16;
- dst = (uint8_t *)dst + 16;
- src = (const uint8_t *)src + 16; /* fallthrough */
- default:
- ;
- }
-
- /* Copy any remaining bytes, without going beyond end of buffers */
- if (n != 0) {
- rte_mov16((uint8_t *)dst - 16 + n, (const uint8_t *)src - 16 + n);
- }
- return ret;
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_MEMCPY_X86_64_H_ */
+++ /dev/null
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _RTE_PREFETCH_X86_64_H_
-#define _RTE_PREFETCH_X86_64_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "generic/rte_prefetch.h"
-
-static inline void rte_prefetch0(volatile void *p)
-{
- asm volatile ("prefetcht0 %[p]" : [p] "+m" (*(volatile char *)p));
-}
-
-static inline void rte_prefetch1(volatile void *p)
-{
- asm volatile ("prefetcht1 %[p]" : [p] "+m" (*(volatile char *)p));
-}
-
-static inline void rte_prefetch2(volatile void *p)
-{
- asm volatile ("prefetcht2 %[p]" : [p] "+m" (*(volatile char *)p));
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_PREFETCH_X86_64_H_ */
+++ /dev/null
-/*-
- * BSD LICENSE
- *
- * Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * * Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * * Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in
- * the documentation and/or other materials provided with the
- * distribution.
- * * Neither the name of Intel Corporation nor the names of its
- * contributors may be used to endorse or promote products derived
- * from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
- * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#ifndef _RTE_SPINLOCK_X86_64_H_
-#define _RTE_SPINLOCK_X86_64_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "generic/rte_spinlock.h"
-
-#ifndef RTE_FORCE_INTRINSICS
-static inline void
-rte_spinlock_lock(rte_spinlock_t *sl)
-{
- int lock_val = 1;
- asm volatile (
- "1:\n"
- "xchg %[locked], %[lv]\n"
- "test %[lv], %[lv]\n"
- "jz 3f\n"
- "2:\n"
- "pause\n"
- "cmpl $0, %[locked]\n"
- "jnz 2b\n"
- "jmp 1b\n"
- "3:\n"
- : [locked] "=m" (sl->locked), [lv] "=q" (lock_val)
- : "[lv]" (lock_val)
- : "memory");
-}
-
-static inline void
-rte_spinlock_unlock (rte_spinlock_t *sl)
-{
- int unlock_val = 0;
- asm volatile (
- "xchg %[locked], %[ulv]\n"
- : [locked] "=m" (sl->locked), [ulv] "=q" (unlock_val)
- : "[ulv]" (unlock_val)
- : "memory");
-}
-
-static inline int
-rte_spinlock_trylock (rte_spinlock_t *sl)
-{
- int lockval = 1;
-
- asm volatile (
- "xchg %[locked], %[lockval]"
- : [locked] "=m" (sl->locked), [lockval] "=q" (lockval)
- : "[lockval]" (lockval)
- : "memory");
-
- return (lockval == 0);
-}
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_SPINLOCK_X86_64_H_ */
#
ARCH ?= i386
+# common arch dir in eal headers
+ARCH_DIR := x86
CROSS ?=
CPU_CFLAGS ?= -m32
#
ARCH ?= x86_64
+# common arch dir in eal headers
+ARCH_DIR := x86
CROSS ?=
CPU_CFLAGS ?= -m64
git.droids-corp.org / dpdk.git / commitdiff