Use C11 atomic builtins with explicit ordering instead of rte_atomic
ops which enforce unnecessary barriers on aarch64.
Suggested-by: Olivier Matz <olivier.matz@6wind.com>
Suggested-by: Dodji Seketeli <dodji@redhat.com>
Signed-off-by: Phil Yang <phil.yang@arm.com>
Reviewed-by: Ruifeng Wang <ruifeng.wang@arm.com>
Acked-by: Olivier Matz <olivier.matz@6wind.com>
[suppress_type]
type_kind = struct
name = rte_epoll_event
[suppress_type]
type_kind = struct
name = rte_epoll_event
+; Ignore updates of rte_mbuf_ext_shared_info refcnt_atomic
+[suppress_type]
+ name = rte_mbuf_ext_shared_info
+ has_data_member_inserted_between = {offset_of(refcnt_atomic), offset_of(refcnt_atomic)}
;;;;;;;;;;;;;;;;;;;;;;
; Temporary exceptions till DPDK 20.11
;;;;;;;;;;;;;;;;;;;;;;
; Temporary exceptions till DPDK 20.11
#include <rte_eal.h>
#include <rte_per_lcore.h>
#include <rte_lcore.h>
#include <rte_eal.h>
#include <rte_per_lcore.h>
#include <rte_lcore.h>
#include <rte_branch_prediction.h>
#include <rte_mempool.h>
#include <rte_mbuf.h>
#include <rte_branch_prediction.h>
#include <rte_mempool.h>
#include <rte_mbuf.h>
#include <rte_config.h>
#include <rte_mempool.h>
#include <rte_memory.h>
#include <rte_config.h>
#include <rte_mempool.h>
#include <rte_memory.h>
#include <rte_prefetch.h>
#include <rte_branch_prediction.h>
#include <rte_byteorder.h>
#include <rte_prefetch.h>
#include <rte_branch_prediction.h>
#include <rte_byteorder.h>
static inline uint16_t
rte_mbuf_refcnt_read(const struct rte_mbuf *m)
{
static inline uint16_t
rte_mbuf_refcnt_read(const struct rte_mbuf *m)
{
- return (uint16_t)(rte_atomic16_read(&m->refcnt_atomic));
+ return __atomic_load_n(&m->refcnt, __ATOMIC_RELAXED);
static inline void
rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
{
static inline void
rte_mbuf_refcnt_set(struct rte_mbuf *m, uint16_t new_value)
{
- rte_atomic16_set(&m->refcnt_atomic, (int16_t)new_value);
+ __atomic_store_n(&m->refcnt, new_value, __ATOMIC_RELAXED);
}
/* internal */
static inline uint16_t
__rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
{
}
/* internal */
static inline uint16_t
__rte_mbuf_refcnt_update(struct rte_mbuf *m, int16_t value)
{
- return (uint16_t)(rte_atomic16_add_return(&m->refcnt_atomic, value));
+ return __atomic_add_fetch(&m->refcnt, (uint16_t)value,
+ __ATOMIC_ACQ_REL);
static inline uint16_t
rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo)
{
static inline uint16_t
rte_mbuf_ext_refcnt_read(const struct rte_mbuf_ext_shared_info *shinfo)
{
- return (uint16_t)(rte_atomic16_read(&shinfo->refcnt_atomic));
+ return __atomic_load_n(&shinfo->refcnt, __ATOMIC_RELAXED);
rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo,
uint16_t new_value)
{
rte_mbuf_ext_refcnt_set(struct rte_mbuf_ext_shared_info *shinfo,
uint16_t new_value)
{
- rte_atomic16_set(&shinfo->refcnt_atomic, (int16_t)new_value);
+ __atomic_store_n(&shinfo->refcnt, new_value, __ATOMIC_RELAXED);
return (uint16_t)value;
}
return (uint16_t)value;
}
- return (uint16_t)rte_atomic16_add_return(&shinfo->refcnt_atomic, value);
+ return __atomic_add_fetch(&shinfo->refcnt, (uint16_t)value,
+ __ATOMIC_ACQ_REL);
* Direct usage of add primitive to avoid
* duplication of comparing with one.
*/
* Direct usage of add primitive to avoid
* duplication of comparing with one.
*/
- if (likely(rte_atomic16_add_return
- (&shinfo->refcnt_atomic, -1)))
+ if (likely(__atomic_add_fetch(&shinfo->refcnt, (uint16_t)-1,
+ __ATOMIC_ACQ_REL)))
return 1;
/* Reinitialize counter before mbuf freeing. */
return 1;
/* Reinitialize counter before mbuf freeing. */
struct rte_mbuf_ext_shared_info {
rte_mbuf_extbuf_free_callback_t free_cb; /**< Free callback function */
void *fcb_opaque; /**< Free callback argument */
struct rte_mbuf_ext_shared_info {
rte_mbuf_extbuf_free_callback_t free_cb; /**< Free callback function */
void *fcb_opaque; /**< Free callback argument */
- rte_atomic16_t refcnt_atomic; /**< Atomically accessed refcnt */
+ RTE_STD_C11
+ union {
+ rte_atomic16_t refcnt_atomic; /**< Atomically accessed refcnt */
+ uint16_t refcnt;
+ };
};
/**< Maximum number of nb_segs allowed. */
};
/**< Maximum number of nb_segs allowed. */