X-Git-Url: http://git.droids-corp.org/?a=blobdiff_plain;f=lib%2Flibrte_efd%2Frte_efd.c;h=77f46809f866f261df6dd7c319988054304db9a6;hb=5b38d8cd4663;hp=2bcfd62a2126c7a13b12e17d69f9d33a023d90e8;hpb=56b6ef874f80891ff5fda833b92ff2cb7e8edfd7;p=dpdk.git diff --git a/lib/librte_efd/rte_efd.c b/lib/librte_efd/rte_efd.c index 2bcfd62a21..77f46809f8 100644 --- a/lib/librte_efd/rte_efd.c +++ b/lib/librte_efd/rte_efd.c @@ -1,34 +1,5 @@ -/*- - * BSD LICENSE - * - * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2016-2017 Intel Corporation */ #include #include @@ -36,22 +7,28 @@ #include #include #include -#include #include +#include #include #include #include #include -#include #include #include #include #include #include #include +#include +#include #include "rte_efd.h" +#if defined(RTE_ARCH_X86) +#include "rte_efd_x86.h" +#elif defined(RTE_ARCH_ARM64) +#include "rte_efd_arm64.h" +#endif #define EFD_KEY(key_idx, table) (table->keys + ((key_idx) * table->key_len)) /** Hash function used to determine chunk_id and bin_id for a group */ @@ -100,6 +77,8 @@ allocated memory /* All different internal lookup functions */ enum efd_lookup_internal_function { EFD_LOOKUP_SCALAR = 0, + EFD_LOOKUP_AVX2, + EFD_LOOKUP_NEON, EFD_LOOKUP_NUM }; @@ -203,7 +182,7 @@ struct efd_offline_group_rules { /**< Array with all values of the keys of the group. */ uint8_t bin_id[EFD_MAX_GROUP_NUM_RULES]; - /**< Stores the bin for each correspending key to + /**< Stores the bin for each corresponding key to * avoid having to recompute it */ }; @@ -229,7 +208,7 @@ struct efd_offline_chunk_rules { struct efd_online_group_entry { efd_hashfunc_t hash_idx[RTE_EFD_VALUE_NUM_BITS]; efd_lookuptbl_t lookup_table[RTE_EFD_VALUE_NUM_BITS]; -} __attribute__((__packed__)); +} __rte_packed; /** * A single chunk record, containing EFD_TARGET_CHUNK_NUM_RULES rules. @@ -245,7 +224,7 @@ struct efd_online_chunk { struct efd_online_group_entry groups[EFD_CHUNK_NUM_GROUPS]; /**< Array of all the groups in the chunk. */ -} __attribute__((__packed__)); +} __rte_packed; /** * EFD table structure @@ -553,9 +532,9 @@ rte_efd_create(const char *name, uint32_t max_num_rules, uint32_t key_len, num_chunks = rte_align32pow2((max_num_rules / EFD_TARGET_CHUNK_NUM_RULES) + 1); - num_chunks_shift = log2(num_chunks); + num_chunks_shift = rte_bsf32(num_chunks); - rte_rwlock_write_lock(RTE_EAL_TAILQ_RWLOCK); + rte_mcfg_tailq_write_lock(); /* * Guarantee there's no existing: this is normally already checked @@ -582,7 +561,7 @@ rte_efd_create(const char *name, uint32_t max_num_rules, uint32_t key_len, } /* Create a new EFD table management structure */ - table = (struct rte_efd_table *) rte_zmalloc_socket(NULL, + table = rte_zmalloc_socket(NULL, sizeof(struct rte_efd_table), RTE_CACHE_LINE_SIZE, offline_cpu_socket); @@ -604,7 +583,7 @@ rte_efd_create(const char *name, uint32_t max_num_rules, uint32_t key_len, table->key_len = key_len; /* key_array */ - key_array = (uint8_t *) rte_zmalloc_socket(NULL, + key_array = rte_zmalloc_socket(NULL, table->max_num_rules * table->key_len, RTE_CACHE_LINE_SIZE, offline_cpu_socket); @@ -615,7 +594,7 @@ rte_efd_create(const char *name, uint32_t max_num_rules, uint32_t key_len, goto error_unlock_exit; } table->keys = key_array; - snprintf(table->name, sizeof(table->name), "%s", name); + strlcpy(table->name, name, sizeof(table->name)); RTE_LOG(DEBUG, EFD, "Creating an EFD table with %u chunks," " which potentially supports %u entries\n", @@ -640,7 +619,7 @@ rte_efd_create(const char *name, uint32_t max_num_rules, uint32_t key_len, * as a continuous block */ table->chunks[socket_id] = - (struct efd_online_chunk *) rte_zmalloc_socket( + rte_zmalloc_socket( NULL, online_table_size, RTE_CACHE_LINE_SIZE, @@ -662,7 +641,29 @@ rte_efd_create(const char *name, uint32_t max_num_rules, uint32_t key_len, } } - table->lookup_fn = EFD_LOOKUP_SCALAR; +#if defined(RTE_ARCH_X86) + /* + * For less than 4 bits, scalar function performs better + * than vectorised version + */ + if (RTE_EFD_VALUE_NUM_BITS > 3 + && rte_cpu_get_flag_enabled(RTE_CPUFLAG_AVX2) + && rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_256) + table->lookup_fn = EFD_LOOKUP_AVX2; + else +#endif +#if defined(RTE_ARCH_ARM64) + /* + * For less than or equal to 16 bits, scalar function performs better + * than vectorised version + */ + if (RTE_EFD_VALUE_NUM_BITS > 16 && + rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON) && + rte_vect_get_max_simd_bitwidth() >= RTE_VECT_SIMD_128) + table->lookup_fn = EFD_LOOKUP_NEON; + else +#endif + table->lookup_fn = EFD_LOOKUP_SCALAR; /* * Allocate the EFD table offline portion (with the actual rules @@ -671,7 +672,7 @@ rte_efd_create(const char *name, uint32_t max_num_rules, uint32_t key_len, */ offline_table_size = num_chunks * sizeof(struct efd_offline_chunk_rules); table->offline_chunks = - (struct efd_offline_chunk_rules *) rte_zmalloc_socket(NULL, + rte_zmalloc_socket(NULL, offline_table_size, RTE_CACHE_LINE_SIZE, offline_cpu_socket); @@ -689,7 +690,7 @@ rte_efd_create(const char *name, uint32_t max_num_rules, uint32_t key_len, te->data = (void *) table; TAILQ_INSERT_TAIL(efd_list, te, next); - rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); + rte_mcfg_tailq_write_unlock(); snprintf(ring_name, sizeof(ring_name), "HT_%s", table->name); /* Create ring (Dummy slot index is not enqueued) */ @@ -697,7 +698,8 @@ rte_efd_create(const char *name, uint32_t max_num_rules, uint32_t key_len, offline_cpu_socket, 0); if (r == NULL) { RTE_LOG(ERR, EFD, "memory allocation failed\n"); - goto error_unlock_exit; + rte_efd_free(table); + return NULL; } /* Populate free slots ring. Entry zero is reserved for key misses. */ @@ -708,7 +710,8 @@ rte_efd_create(const char *name, uint32_t max_num_rules, uint32_t key_len, return table; error_unlock_exit: - rte_rwlock_write_unlock(RTE_EAL_TAILQ_RWLOCK); + rte_mcfg_tailq_write_unlock(); + rte_free(te); rte_efd_free(table); return NULL; @@ -723,7 +726,7 @@ rte_efd_find_existing(const char *name) efd_list = RTE_TAILQ_CAST(rte_efd_tailq.head, rte_efd_list); - rte_rwlock_read_lock(RTE_EAL_TAILQ_RWLOCK); + rte_mcfg_tailq_read_lock(); TAILQ_FOREACH(te, efd_list, next) { @@ -731,7 +734,7 @@ rte_efd_find_existing(const char *name) if (strncmp(name, table->name, RTE_EFD_NAMESIZE) == 0) break; } - rte_rwlock_read_unlock(RTE_EAL_TAILQ_RWLOCK); + rte_mcfg_tailq_read_unlock(); if (te == NULL) { rte_errno = ENOENT; @@ -744,6 +747,8 @@ void rte_efd_free(struct rte_efd_table *table) { uint8_t socket_id; + struct rte_efd_list *efd_list; + struct rte_tailq_entry *te, *temp; if (table == NULL) return; @@ -751,6 +756,18 @@ rte_efd_free(struct rte_efd_table *table) for (socket_id = 0; socket_id < RTE_MAX_NUMA_NODES; socket_id++) rte_free(table->chunks[socket_id]); + efd_list = RTE_TAILQ_CAST(rte_efd_tailq.head, rte_efd_list); + rte_mcfg_tailq_write_lock(); + + TAILQ_FOREACH_SAFE(te, efd_list, next, temp) { + if (te->data == (void *) table) { + TAILQ_REMOVE(efd_list, te, next); + rte_free(te); + break; + } + } + + rte_mcfg_tailq_write_unlock(); rte_ring_free(table->free_slots); rte_free(table->offline_chunks); rte_free(table->keys); @@ -928,7 +945,7 @@ revert_groups(struct efd_offline_group_rules *previous_group, * This operation was still successful, and entry contains a valid update * RTE_EFD_UPDATE_FAILED * Either the EFD failed to find a suitable perfect hash or the group was full - * This is a fatal error, and the table is now in an indeterminite state + * This is a fatal error, and the table is now in an indeterminate state * RTE_EFD_UPDATE_NO_CHANGE * Operation resulted in no change to the table (same value already exists) * 0 @@ -1253,6 +1270,22 @@ efd_lookup_internal(const struct efd_online_group_entry * const group, switch (lookup_fn) { +#if defined(RTE_ARCH_X86) && defined(CC_SUPPORT_AVX2) + case EFD_LOOKUP_AVX2: + return efd_lookup_internal_avx2(group->hash_idx, + group->lookup_table, + hash_val_a, + hash_val_b); + break; +#endif +#if defined(RTE_ARCH_ARM64) + case EFD_LOOKUP_NEON: + return efd_lookup_internal_neon(group->hash_idx, + group->lookup_table, + hash_val_a, + hash_val_b); + break; +#endif case EFD_LOOKUP_SCALAR: /* Fall-through */ default: