MCSlock
M: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>
-F: lib/eal/include/generic/rte_mcslock.h
+F: lib/eal/include/rte_mcslock.h
F: app/test/test_mcslock.c
Sequence Lock
Ticketlock
M: Joyce Kong <joyce.kong@arm.com>
-F: lib/eal/include/generic/rte_ticketlock.h
+F: lib/eal/include/rte_ticketlock.h
F: app/test/test_ticketlock.c
Pseudo-random Number Generation
'rte_cycles.h',
'rte_io_64.h',
'rte_io.h',
- 'rte_mcslock.h',
'rte_memcpy_32.h',
'rte_memcpy_64.h',
'rte_memcpy.h',
'rte_pause_32.h',
'rte_pause_64.h',
'rte_pause.h',
- 'rte_pflock.h',
'rte_power_intrinsics.h',
'rte_prefetch_32.h',
'rte_prefetch_64.h',
'rte_prefetch.h',
'rte_rwlock.h',
'rte_spinlock.h',
- 'rte_ticketlock.h',
'rte_vect.h',
)
install_headers(arch_headers, subdir: get_option('include_subdir_arch'))
+++ /dev/null
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2019 Arm Limited
- */
-
-#ifndef _RTE_MCSLOCK_ARM_H_
-#define _RTE_MCSLOCK_ARM_H_
-
-#ifndef RTE_FORCE_INTRINSICS
-# error Platform must be built with RTE_FORCE_INTRINSICS
-#endif
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "generic/rte_mcslock.h"
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_MCSLOCK_ARM_H_ */
+++ /dev/null
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2021 Microsoft Corporation
- */
-
-#ifndef _RTE_PFLOCK_ARM_H_
-#define _RTE_PFLOCK_ARM_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "generic/rte_pflock.h"
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_PFLOCK_ARM_H_ */
+++ /dev/null
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2019 Arm Limited
- */
-
-#ifndef _RTE_TICKETLOCK_ARM_H_
-#define _RTE_TICKETLOCK_ARM_H_
-
-#ifndef RTE_FORCE_INTRINSICS
-# error Platform must be built with RTE_FORCE_INTRINSICS
-#endif
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "generic/rte_ticketlock.h"
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_TICKETLOCK_ARM_H_ */
+++ /dev/null
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2019 Arm Limited
- */
-
-#ifndef _RTE_MCSLOCK_H_
-#define _RTE_MCSLOCK_H_
-
-/**
- * @file
- *
- * RTE MCS lock
- *
- * This file defines the main data structure and APIs for MCS queued lock.
- *
- * The MCS lock (proposed by John M. Mellor-Crummey and Michael L. Scott)
- * provides scalability by spinning on a CPU/thread local variable which
- * avoids expensive cache bouncings. It provides fairness by maintaining
- * a list of acquirers and passing the lock to each CPU/thread in the order
- * they acquired the lock.
- */
-
-#include <rte_lcore.h>
-#include <rte_common.h>
-#include <rte_pause.h>
-#include <rte_branch_prediction.h>
-
-/**
- * The rte_mcslock_t type.
- */
-typedef struct rte_mcslock {
- struct rte_mcslock *next;
- int locked; /* 1 if the queue locked, 0 otherwise */
-} rte_mcslock_t;
-
-/**
- * Take the MCS lock.
- *
- * @param msl
- * A pointer to the pointer of a MCS lock.
- * When the lock is initialized or declared, the msl pointer should be
- * set to NULL.
- * @param me
- * A pointer to a new node of MCS lock. Each CPU/thread acquiring the
- * lock should use its 'own node'.
- */
-static inline void
-rte_mcslock_lock(rte_mcslock_t **msl, rte_mcslock_t *me)
-{
- rte_mcslock_t *prev;
-
- /* Init me node */
- __atomic_store_n(&me->locked, 1, __ATOMIC_RELAXED);
- __atomic_store_n(&me->next, NULL, __ATOMIC_RELAXED);
-
- /* If the queue is empty, the exchange operation is enough to acquire
- * the lock. Hence, the exchange operation requires acquire semantics.
- * The store to me->next above should complete before the node is
- * visible to other CPUs/threads. Hence, the exchange operation requires
- * release semantics as well.
- */
- prev = __atomic_exchange_n(msl, me, __ATOMIC_ACQ_REL);
- if (likely(prev == NULL)) {
- /* Queue was empty, no further action required,
- * proceed with lock taken.
- */
- return;
- }
- /* The store to me->next above should also complete before the node is
- * visible to predecessor thread releasing the lock. Hence, the store
- * prev->next also requires release semantics. Note that, for example,
- * on ARM, the release semantics in the exchange operation is not
- * strong as a release fence and is not sufficient to enforce the
- * desired order here.
- */
- __atomic_store_n(&prev->next, me, __ATOMIC_RELEASE);
-
- /* The while-load of me->locked should not move above the previous
- * store to prev->next. Otherwise it will cause a deadlock. Need a
- * store-load barrier.
- */
- __atomic_thread_fence(__ATOMIC_ACQ_REL);
- /* If the lock has already been acquired, it first atomically
- * places the node at the end of the queue and then proceeds
- * to spin on me->locked until the previous lock holder resets
- * the me->locked using mcslock_unlock().
- */
- rte_wait_until_equal_32((uint32_t *)&me->locked, 0, __ATOMIC_ACQUIRE);
-}
-
-/**
- * Release the MCS lock.
- *
- * @param msl
- * A pointer to the pointer of a MCS lock.
- * @param me
- * A pointer to the node of MCS lock passed in rte_mcslock_lock.
- */
-static inline void
-rte_mcslock_unlock(rte_mcslock_t **msl, rte_mcslock_t *me)
-{
- /* Check if there are more nodes in the queue. */
- if (likely(__atomic_load_n(&me->next, __ATOMIC_RELAXED) == NULL)) {
- /* No, last member in the queue. */
- rte_mcslock_t *save_me = __atomic_load_n(&me, __ATOMIC_RELAXED);
-
- /* Release the lock by setting it to NULL */
- if (likely(__atomic_compare_exchange_n(msl, &save_me, NULL, 0,
- __ATOMIC_RELEASE, __ATOMIC_RELAXED)))
- return;
-
- /* Speculative execution would be allowed to read in the
- * while-loop first. This has the potential to cause a
- * deadlock. Need a load barrier.
- */
- __atomic_thread_fence(__ATOMIC_ACQUIRE);
- /* More nodes added to the queue by other CPUs.
- * Wait until the next pointer is set.
- */
- uintptr_t *next;
- next = (uintptr_t *)&me->next;
- RTE_WAIT_UNTIL_MASKED(next, UINTPTR_MAX, !=, 0,
- __ATOMIC_RELAXED);
- }
-
- /* Pass lock to next waiter. */
- __atomic_store_n(&me->next->locked, 0, __ATOMIC_RELEASE);
-}
-
-/**
- * Try to take the lock.
- *
- * @param msl
- * A pointer to the pointer of a MCS lock.
- * @param me
- * A pointer to a new node of MCS lock.
- * @return
- * 1 if the lock is successfully taken; 0 otherwise.
- */
-static inline int
-rte_mcslock_trylock(rte_mcslock_t **msl, rte_mcslock_t *me)
-{
- /* Init me node */
- __atomic_store_n(&me->next, NULL, __ATOMIC_RELAXED);
-
- /* Try to lock */
- rte_mcslock_t *expected = NULL;
-
- /* The lock can be taken only when the queue is empty. Hence,
- * the compare-exchange operation requires acquire semantics.
- * The store to me->next above should complete before the node
- * is visible to other CPUs/threads. Hence, the compare-exchange
- * operation requires release semantics as well.
- */
- return __atomic_compare_exchange_n(msl, &expected, me, 0,
- __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
-}
-
-/**
- * Test if the lock is taken.
- *
- * @param msl
- * A pointer to a MCS lock node.
- * @return
- * 1 if the lock is currently taken; 0 otherwise.
- */
-static inline int
-rte_mcslock_is_locked(rte_mcslock_t *msl)
-{
- return (__atomic_load_n(&msl, __ATOMIC_RELAXED) != NULL);
-}
-
-#endif /* _RTE_MCSLOCK_H_ */
+++ /dev/null
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2021 Microsoft Corp.
- * All rights reserved.
- *
- * Derived from Concurrency Kit
- * Copyright 2011-2015 Samy Al Bahra.
- */
-
-#ifndef _RTE_PFLOCK_H_
-#define _RTE_PFLOCK_H_
-
-/**
- * @file
- *
- * Phase-fair locks
- *
- * This file defines an API for phase-fair reader writer locks,
- * which is a variant of typical reader-writer locks that prevent
- * starvation. In this type of lock, readers and writers alternate.
- * This significantly reduces the worst-case blocking for readers and writers.
- *
- * This is an implementation derived from FreeBSD
- * based on the work described in:
- * Brandenburg, B. and Anderson, J. 2010. Spin-Based
- * Reader-Writer Synchronization for Multiprocessor Real-Time Systems
- *
- * All locks must be initialised before use, and only initialised once.
- */
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <rte_common.h>
-#include <rte_pause.h>
-
-/**
- * The rte_pflock_t type.
- */
-struct rte_pflock {
- struct {
- uint16_t in;
- uint16_t out;
- } rd, wr;
-};
-typedef struct rte_pflock rte_pflock_t;
-
-/*
- * Allocation of bits to reader
- *
- * 15 4 3 2 1 0
- * +-------------------+---+-+-+
- * | rin: reads issued |x|x| | |
- * +-------------------+---+-+-+
- * ^ ^
- * | |
- * PRES: writer present ----/ |
- * PHID: writer phase id -----/
- *
- * 15 4 3 2 1 0
- * +------------------+------+
- * |rout:read complete|unused|
- * +------------------+------+
- *
- * The maximum number of readers is 4095
- */
-
-/* Constants used to map the bits in reader counter */
-#define RTE_PFLOCK_WBITS 0x3 /* Writer bits in reader. */
-#define RTE_PFLOCK_PRES 0x2 /* Writer present bit. */
-#define RTE_PFLOCK_PHID 0x1 /* Phase ID bit. */
-#define RTE_PFLOCK_LSB 0xFFF0 /* reader bits. */
-#define RTE_PFLOCK_RINC 0x10 /* Reader increment. */
-
-/**
- * A static pflock initializer.
- */
-#define RTE_PFLOCK_INITIALIZER { }
-
-/**
- * @warning
- * @b EXPERIMENTAL: this API may change without prior notice.
- *
- * Initialize the pflock to an unlocked state.
- *
- * @param pf
- * A pointer to the pflock.
- */
-__rte_experimental
-static inline void
-rte_pflock_init(struct rte_pflock *pf)
-{
- pf->rd.in = 0;
- pf->rd.out = 0;
- pf->wr.in = 0;
- pf->wr.out = 0;
-}
-
-/**
- * @warning
- * @b EXPERIMENTAL: this API may change without prior notice.
- *
- * Take a pflock for read.
- *
- * @param pf
- * A pointer to a pflock structure.
- */
-__rte_experimental
-static inline void
-rte_pflock_read_lock(rte_pflock_t *pf)
-{
- uint16_t w;
-
- /*
- * If no writer is present, then the operation has completed
- * successfully.
- */
- w = __atomic_fetch_add(&pf->rd.in, RTE_PFLOCK_RINC, __ATOMIC_ACQUIRE)
- & RTE_PFLOCK_WBITS;
- if (w == 0)
- return;
-
- /* Wait for current write phase to complete. */
- RTE_WAIT_UNTIL_MASKED(&pf->rd.in, RTE_PFLOCK_WBITS, !=, w,
- __ATOMIC_ACQUIRE);
-}
-
-/**
- * @warning
- * @b EXPERIMENTAL: this API may change without prior notice.
- *
- * Release a pflock locked for reading.
- *
- * @param pf
- * A pointer to the pflock structure.
- */
-__rte_experimental
-static inline void
-rte_pflock_read_unlock(rte_pflock_t *pf)
-{
- __atomic_fetch_add(&pf->rd.out, RTE_PFLOCK_RINC, __ATOMIC_RELEASE);
-}
-
-/**
- * @warning
- * @b EXPERIMENTAL: this API may change without prior notice.
- *
- * Take the pflock for write.
- *
- * @param pf
- * A pointer to the pflock structure.
- */
-__rte_experimental
-static inline void
-rte_pflock_write_lock(rte_pflock_t *pf)
-{
- uint16_t ticket, w;
-
- /* Acquire ownership of write-phase.
- * This is same as rte_ticketlock_lock().
- */
- ticket = __atomic_fetch_add(&pf->wr.in, 1, __ATOMIC_RELAXED);
- rte_wait_until_equal_16(&pf->wr.out, ticket, __ATOMIC_ACQUIRE);
-
- /*
- * Acquire ticket on read-side in order to allow them
- * to flush. Indicates to any incoming reader that a
- * write-phase is pending.
- *
- * The load of rd.out in wait loop could be executed
- * speculatively.
- */
- w = RTE_PFLOCK_PRES | (ticket & RTE_PFLOCK_PHID);
- ticket = __atomic_fetch_add(&pf->rd.in, w, __ATOMIC_RELAXED);
-
- /* Wait for any pending readers to flush. */
- rte_wait_until_equal_16(&pf->rd.out, ticket, __ATOMIC_ACQUIRE);
-}
-
-/**
- * @warning
- * @b EXPERIMENTAL: this API may change without prior notice.
- *
- * Release a pflock held for writing.
- *
- * @param pf
- * A pointer to a pflock structure.
- */
-__rte_experimental
-static inline void
-rte_pflock_write_unlock(rte_pflock_t *pf)
-{
- /* Migrate from write phase to read phase. */
- __atomic_fetch_and(&pf->rd.in, RTE_PFLOCK_LSB, __ATOMIC_RELEASE);
-
- /* Allow other writers to continue. */
- __atomic_fetch_add(&pf->wr.out, 1, __ATOMIC_RELEASE);
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* RTE_PFLOCK_H */
+++ /dev/null
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2019 Arm Limited
- */
-
-#ifndef _RTE_TICKETLOCK_H_
-#define _RTE_TICKETLOCK_H_
-
-/**
- * @file
- *
- * RTE ticket locks
- *
- * This file defines an API for ticket locks, which give each waiting
- * thread a ticket and take the lock one by one, first come, first
- * serviced.
- *
- * All locks must be initialised before use, and only initialised once.
- *
- */
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <rte_common.h>
-#include <rte_lcore.h>
-#include <rte_pause.h>
-
-/**
- * The rte_ticketlock_t type.
- */
-typedef union {
- uint32_t tickets;
- struct {
- uint16_t current;
- uint16_t next;
- } s;
-} rte_ticketlock_t;
-
-/**
- * A static ticketlock initializer.
- */
-#define RTE_TICKETLOCK_INITIALIZER { 0 }
-
-/**
- * Initialize the ticketlock to an unlocked state.
- *
- * @param tl
- * A pointer to the ticketlock.
- */
-static inline void
-rte_ticketlock_init(rte_ticketlock_t *tl)
-{
- __atomic_store_n(&tl->tickets, 0, __ATOMIC_RELAXED);
-}
-
-/**
- * Take the ticketlock.
- *
- * @param tl
- * A pointer to the ticketlock.
- */
-static inline void
-rte_ticketlock_lock(rte_ticketlock_t *tl)
-{
- uint16_t me = __atomic_fetch_add(&tl->s.next, 1, __ATOMIC_RELAXED);
- rte_wait_until_equal_16(&tl->s.current, me, __ATOMIC_ACQUIRE);
-}
-
-/**
- * Release the ticketlock.
- *
- * @param tl
- * A pointer to the ticketlock.
- */
-static inline void
-rte_ticketlock_unlock(rte_ticketlock_t *tl)
-{
- uint16_t i = __atomic_load_n(&tl->s.current, __ATOMIC_RELAXED);
- __atomic_store_n(&tl->s.current, i + 1, __ATOMIC_RELEASE);
-}
-
-/**
- * Try to take the lock.
- *
- * @param tl
- * A pointer to the ticketlock.
- * @return
- * 1 if the lock is successfully taken; 0 otherwise.
- */
-static inline int
-rte_ticketlock_trylock(rte_ticketlock_t *tl)
-{
- rte_ticketlock_t oldl, newl;
- oldl.tickets = __atomic_load_n(&tl->tickets, __ATOMIC_RELAXED);
- newl.tickets = oldl.tickets;
- newl.s.next++;
- if (oldl.s.next == oldl.s.current) {
- if (__atomic_compare_exchange_n(&tl->tickets, &oldl.tickets,
- newl.tickets, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED))
- return 1;
- }
-
- return 0;
-}
-
-/**
- * Test if the lock is taken.
- *
- * @param tl
- * A pointer to the ticketlock.
- * @return
- * 1 if the lock is currently taken; 0 otherwise.
- */
-static inline int
-rte_ticketlock_is_locked(rte_ticketlock_t *tl)
-{
- rte_ticketlock_t tic;
- tic.tickets = __atomic_load_n(&tl->tickets, __ATOMIC_ACQUIRE);
- return (tic.s.current != tic.s.next);
-}
-
-/**
- * The rte_ticketlock_recursive_t type.
- */
-#define TICKET_LOCK_INVALID_ID -1
-
-typedef struct {
- rte_ticketlock_t tl; /**< the actual ticketlock */
- int user; /**< core id using lock, TICKET_LOCK_INVALID_ID for unused */
- unsigned int count; /**< count of time this lock has been called */
-} rte_ticketlock_recursive_t;
-
-/**
- * A static recursive ticketlock initializer.
- */
-#define RTE_TICKETLOCK_RECURSIVE_INITIALIZER {RTE_TICKETLOCK_INITIALIZER, \
- TICKET_LOCK_INVALID_ID, 0}
-
-/**
- * Initialize the recursive ticketlock to an unlocked state.
- *
- * @param tlr
- * A pointer to the recursive ticketlock.
- */
-static inline void
-rte_ticketlock_recursive_init(rte_ticketlock_recursive_t *tlr)
-{
- rte_ticketlock_init(&tlr->tl);
- __atomic_store_n(&tlr->user, TICKET_LOCK_INVALID_ID, __ATOMIC_RELAXED);
- tlr->count = 0;
-}
-
-/**
- * Take the recursive ticketlock.
- *
- * @param tlr
- * A pointer to the recursive ticketlock.
- */
-static inline void
-rte_ticketlock_recursive_lock(rte_ticketlock_recursive_t *tlr)
-{
- int id = rte_gettid();
-
- if (__atomic_load_n(&tlr->user, __ATOMIC_RELAXED) != id) {
- rte_ticketlock_lock(&tlr->tl);
- __atomic_store_n(&tlr->user, id, __ATOMIC_RELAXED);
- }
- tlr->count++;
-}
-
-/**
- * Release the recursive ticketlock.
- *
- * @param tlr
- * A pointer to the recursive ticketlock.
- */
-static inline void
-rte_ticketlock_recursive_unlock(rte_ticketlock_recursive_t *tlr)
-{
- if (--(tlr->count) == 0) {
- __atomic_store_n(&tlr->user, TICKET_LOCK_INVALID_ID,
- __ATOMIC_RELAXED);
- rte_ticketlock_unlock(&tlr->tl);
- }
-}
-
-/**
- * Try to take the recursive lock.
- *
- * @param tlr
- * A pointer to the recursive ticketlock.
- * @return
- * 1 if the lock is successfully taken; 0 otherwise.
- */
-static inline int
-rte_ticketlock_recursive_trylock(rte_ticketlock_recursive_t *tlr)
-{
- int id = rte_gettid();
-
- if (__atomic_load_n(&tlr->user, __ATOMIC_RELAXED) != id) {
- if (rte_ticketlock_trylock(&tlr->tl) == 0)
- return 0;
- __atomic_store_n(&tlr->user, id, __ATOMIC_RELAXED);
- }
- tlr->count++;
- return 1;
-}
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_TICKETLOCK_H_ */
'rte_lcore.h',
'rte_log.h',
'rte_malloc.h',
+ 'rte_mcslock.h',
'rte_memory.h',
'rte_memzone.h',
'rte_pci_dev_feature_defs.h',
'rte_pci_dev_features.h',
'rte_per_lcore.h',
+ 'rte_pflock.h',
'rte_random.h',
'rte_reciprocal.h',
'rte_seqcount.h',
'rte_string_fns.h',
'rte_tailq.h',
'rte_thread.h',
+ 'rte_ticketlock.h',
'rte_time.h',
'rte_trace.h',
'rte_trace_point.h',
'generic/rte_cpuflags.h',
'generic/rte_cycles.h',
'generic/rte_io.h',
- 'generic/rte_mcslock.h',
'generic/rte_memcpy.h',
'generic/rte_pause.h',
- 'generic/rte_pflock.h',
'generic/rte_power_intrinsics.h',
'generic/rte_prefetch.h',
'generic/rte_rwlock.h',
'generic/rte_spinlock.h',
- 'generic/rte_ticketlock.h',
'generic/rte_vect.h',
)
install_headers(generic_headers, subdir: 'generic')
--- /dev/null
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019 Arm Limited
+ */
+
+#ifndef _RTE_MCSLOCK_H_
+#define _RTE_MCSLOCK_H_
+
+/**
+ * @file
+ *
+ * RTE MCS lock
+ *
+ * This file defines the main data structure and APIs for MCS queued lock.
+ *
+ * The MCS lock (proposed by John M. Mellor-Crummey and Michael L. Scott)
+ * provides scalability by spinning on a CPU/thread local variable which
+ * avoids expensive cache bouncings. It provides fairness by maintaining
+ * a list of acquirers and passing the lock to each CPU/thread in the order
+ * they acquired the lock.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <rte_lcore.h>
+#include <rte_common.h>
+#include <rte_pause.h>
+#include <rte_branch_prediction.h>
+
+/**
+ * The rte_mcslock_t type.
+ */
+typedef struct rte_mcslock {
+ struct rte_mcslock *next;
+ int locked; /* 1 if the queue locked, 0 otherwise */
+} rte_mcslock_t;
+
+/**
+ * Take the MCS lock.
+ *
+ * @param msl
+ * A pointer to the pointer of a MCS lock.
+ * When the lock is initialized or declared, the msl pointer should be
+ * set to NULL.
+ * @param me
+ * A pointer to a new node of MCS lock. Each CPU/thread acquiring the
+ * lock should use its 'own node'.
+ */
+static inline void
+rte_mcslock_lock(rte_mcslock_t **msl, rte_mcslock_t *me)
+{
+ rte_mcslock_t *prev;
+
+ /* Init me node */
+ __atomic_store_n(&me->locked, 1, __ATOMIC_RELAXED);
+ __atomic_store_n(&me->next, NULL, __ATOMIC_RELAXED);
+
+ /* If the queue is empty, the exchange operation is enough to acquire
+ * the lock. Hence, the exchange operation requires acquire semantics.
+ * The store to me->next above should complete before the node is
+ * visible to other CPUs/threads. Hence, the exchange operation requires
+ * release semantics as well.
+ */
+ prev = __atomic_exchange_n(msl, me, __ATOMIC_ACQ_REL);
+ if (likely(prev == NULL)) {
+ /* Queue was empty, no further action required,
+ * proceed with lock taken.
+ */
+ return;
+ }
+ /* The store to me->next above should also complete before the node is
+ * visible to predecessor thread releasing the lock. Hence, the store
+ * prev->next also requires release semantics. Note that, for example,
+ * on ARM, the release semantics in the exchange operation is not
+ * strong as a release fence and is not sufficient to enforce the
+ * desired order here.
+ */
+ __atomic_store_n(&prev->next, me, __ATOMIC_RELEASE);
+
+ /* The while-load of me->locked should not move above the previous
+ * store to prev->next. Otherwise it will cause a deadlock. Need a
+ * store-load barrier.
+ */
+ __atomic_thread_fence(__ATOMIC_ACQ_REL);
+ /* If the lock has already been acquired, it first atomically
+ * places the node at the end of the queue and then proceeds
+ * to spin on me->locked until the previous lock holder resets
+ * the me->locked using mcslock_unlock().
+ */
+ rte_wait_until_equal_32((uint32_t *)&me->locked, 0, __ATOMIC_ACQUIRE);
+}
+
+/**
+ * Release the MCS lock.
+ *
+ * @param msl
+ * A pointer to the pointer of a MCS lock.
+ * @param me
+ * A pointer to the node of MCS lock passed in rte_mcslock_lock.
+ */
+static inline void
+rte_mcslock_unlock(rte_mcslock_t **msl, rte_mcslock_t *me)
+{
+ /* Check if there are more nodes in the queue. */
+ if (likely(__atomic_load_n(&me->next, __ATOMIC_RELAXED) == NULL)) {
+ /* No, last member in the queue. */
+ rte_mcslock_t *save_me = __atomic_load_n(&me, __ATOMIC_RELAXED);
+
+ /* Release the lock by setting it to NULL */
+ if (likely(__atomic_compare_exchange_n(msl, &save_me, NULL, 0,
+ __ATOMIC_RELEASE, __ATOMIC_RELAXED)))
+ return;
+
+ /* Speculative execution would be allowed to read in the
+ * while-loop first. This has the potential to cause a
+ * deadlock. Need a load barrier.
+ */
+ __atomic_thread_fence(__ATOMIC_ACQUIRE);
+ /* More nodes added to the queue by other CPUs.
+ * Wait until the next pointer is set.
+ */
+ uintptr_t *next;
+ next = (uintptr_t *)&me->next;
+ RTE_WAIT_UNTIL_MASKED(next, UINTPTR_MAX, !=, 0,
+ __ATOMIC_RELAXED);
+ }
+
+ /* Pass lock to next waiter. */
+ __atomic_store_n(&me->next->locked, 0, __ATOMIC_RELEASE);
+}
+
+/**
+ * Try to take the lock.
+ *
+ * @param msl
+ * A pointer to the pointer of a MCS lock.
+ * @param me
+ * A pointer to a new node of MCS lock.
+ * @return
+ * 1 if the lock is successfully taken; 0 otherwise.
+ */
+static inline int
+rte_mcslock_trylock(rte_mcslock_t **msl, rte_mcslock_t *me)
+{
+ /* Init me node */
+ __atomic_store_n(&me->next, NULL, __ATOMIC_RELAXED);
+
+ /* Try to lock */
+ rte_mcslock_t *expected = NULL;
+
+ /* The lock can be taken only when the queue is empty. Hence,
+ * the compare-exchange operation requires acquire semantics.
+ * The store to me->next above should complete before the node
+ * is visible to other CPUs/threads. Hence, the compare-exchange
+ * operation requires release semantics as well.
+ */
+ return __atomic_compare_exchange_n(msl, &expected, me, 0,
+ __ATOMIC_ACQ_REL, __ATOMIC_RELAXED);
+}
+
+/**
+ * Test if the lock is taken.
+ *
+ * @param msl
+ * A pointer to a MCS lock node.
+ * @return
+ * 1 if the lock is currently taken; 0 otherwise.
+ */
+static inline int
+rte_mcslock_is_locked(rte_mcslock_t *msl)
+{
+ return (__atomic_load_n(&msl, __ATOMIC_RELAXED) != NULL);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_MCSLOCK_H_ */
--- /dev/null
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2021 Microsoft Corp.
+ * All rights reserved.
+ *
+ * Derived from Concurrency Kit
+ * Copyright 2011-2015 Samy Al Bahra.
+ */
+
+#ifndef _RTE_PFLOCK_H_
+#define _RTE_PFLOCK_H_
+
+/**
+ * @file
+ *
+ * Phase-fair locks
+ *
+ * This file defines an API for phase-fair reader writer locks,
+ * which is a variant of typical reader-writer locks that prevent
+ * starvation. In this type of lock, readers and writers alternate.
+ * This significantly reduces the worst-case blocking for readers and writers.
+ *
+ * This is an implementation derived from FreeBSD
+ * based on the work described in:
+ * Brandenburg, B. and Anderson, J. 2010. Spin-Based
+ * Reader-Writer Synchronization for Multiprocessor Real-Time Systems
+ *
+ * All locks must be initialised before use, and only initialised once.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <rte_common.h>
+#include <rte_pause.h>
+
+/**
+ * The rte_pflock_t type.
+ */
+struct rte_pflock {
+ struct {
+ uint16_t in;
+ uint16_t out;
+ } rd, wr;
+};
+typedef struct rte_pflock rte_pflock_t;
+
+/*
+ * Allocation of bits to reader
+ *
+ * 15 4 3 2 1 0
+ * +-------------------+---+-+-+
+ * | rin: reads issued |x|x| | |
+ * +-------------------+---+-+-+
+ * ^ ^
+ * | |
+ * PRES: writer present ----/ |
+ * PHID: writer phase id -----/
+ *
+ * 15 4 3 2 1 0
+ * +------------------+------+
+ * |rout:read complete|unused|
+ * +------------------+------+
+ *
+ * The maximum number of readers is 4095
+ */
+
+/* Constants used to map the bits in reader counter */
+#define RTE_PFLOCK_WBITS 0x3 /* Writer bits in reader. */
+#define RTE_PFLOCK_PRES 0x2 /* Writer present bit. */
+#define RTE_PFLOCK_PHID 0x1 /* Phase ID bit. */
+#define RTE_PFLOCK_LSB 0xFFF0 /* reader bits. */
+#define RTE_PFLOCK_RINC 0x10 /* Reader increment. */
+
+/**
+ * A static pflock initializer.
+ */
+#define RTE_PFLOCK_INITIALIZER { }
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Initialize the pflock to an unlocked state.
+ *
+ * @param pf
+ * A pointer to the pflock.
+ */
+__rte_experimental
+static inline void
+rte_pflock_init(struct rte_pflock *pf)
+{
+ pf->rd.in = 0;
+ pf->rd.out = 0;
+ pf->wr.in = 0;
+ pf->wr.out = 0;
+}
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Take a pflock for read.
+ *
+ * @param pf
+ * A pointer to a pflock structure.
+ */
+__rte_experimental
+static inline void
+rte_pflock_read_lock(rte_pflock_t *pf)
+{
+ uint16_t w;
+
+ /*
+ * If no writer is present, then the operation has completed
+ * successfully.
+ */
+ w = __atomic_fetch_add(&pf->rd.in, RTE_PFLOCK_RINC, __ATOMIC_ACQUIRE)
+ & RTE_PFLOCK_WBITS;
+ if (w == 0)
+ return;
+
+ /* Wait for current write phase to complete. */
+ RTE_WAIT_UNTIL_MASKED(&pf->rd.in, RTE_PFLOCK_WBITS, !=, w,
+ __ATOMIC_ACQUIRE);
+}
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Release a pflock locked for reading.
+ *
+ * @param pf
+ * A pointer to the pflock structure.
+ */
+__rte_experimental
+static inline void
+rte_pflock_read_unlock(rte_pflock_t *pf)
+{
+ __atomic_fetch_add(&pf->rd.out, RTE_PFLOCK_RINC, __ATOMIC_RELEASE);
+}
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Take the pflock for write.
+ *
+ * @param pf
+ * A pointer to the pflock structure.
+ */
+__rte_experimental
+static inline void
+rte_pflock_write_lock(rte_pflock_t *pf)
+{
+ uint16_t ticket, w;
+
+ /* Acquire ownership of write-phase.
+ * This is same as rte_ticketlock_lock().
+ */
+ ticket = __atomic_fetch_add(&pf->wr.in, 1, __ATOMIC_RELAXED);
+ rte_wait_until_equal_16(&pf->wr.out, ticket, __ATOMIC_ACQUIRE);
+
+ /*
+ * Acquire ticket on read-side in order to allow them
+ * to flush. Indicates to any incoming reader that a
+ * write-phase is pending.
+ *
+ * The load of rd.out in wait loop could be executed
+ * speculatively.
+ */
+ w = RTE_PFLOCK_PRES | (ticket & RTE_PFLOCK_PHID);
+ ticket = __atomic_fetch_add(&pf->rd.in, w, __ATOMIC_RELAXED);
+
+ /* Wait for any pending readers to flush. */
+ rte_wait_until_equal_16(&pf->rd.out, ticket, __ATOMIC_ACQUIRE);
+}
+
+/**
+ * @warning
+ * @b EXPERIMENTAL: this API may change without prior notice.
+ *
+ * Release a pflock held for writing.
+ *
+ * @param pf
+ * A pointer to a pflock structure.
+ */
+__rte_experimental
+static inline void
+rte_pflock_write_unlock(rte_pflock_t *pf)
+{
+ /* Migrate from write phase to read phase. */
+ __atomic_fetch_and(&pf->rd.in, RTE_PFLOCK_LSB, __ATOMIC_RELEASE);
+
+ /* Allow other writers to continue. */
+ __atomic_fetch_add(&pf->wr.out, 1, __ATOMIC_RELEASE);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* RTE_PFLOCK_H */
--- /dev/null
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2019 Arm Limited
+ */
+
+#ifndef _RTE_TICKETLOCK_H_
+#define _RTE_TICKETLOCK_H_
+
+/**
+ * @file
+ *
+ * RTE ticket locks
+ *
+ * This file defines an API for ticket locks, which give each waiting
+ * thread a ticket and take the lock one by one, first come, first
+ * serviced.
+ *
+ * All locks must be initialised before use, and only initialised once.
+ *
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <rte_common.h>
+#include <rte_lcore.h>
+#include <rte_pause.h>
+
+/**
+ * The rte_ticketlock_t type.
+ */
+typedef union {
+ uint32_t tickets;
+ struct {
+ uint16_t current;
+ uint16_t next;
+ } s;
+} rte_ticketlock_t;
+
+/**
+ * A static ticketlock initializer.
+ */
+#define RTE_TICKETLOCK_INITIALIZER { 0 }
+
+/**
+ * Initialize the ticketlock to an unlocked state.
+ *
+ * @param tl
+ * A pointer to the ticketlock.
+ */
+static inline void
+rte_ticketlock_init(rte_ticketlock_t *tl)
+{
+ __atomic_store_n(&tl->tickets, 0, __ATOMIC_RELAXED);
+}
+
+/**
+ * Take the ticketlock.
+ *
+ * @param tl
+ * A pointer to the ticketlock.
+ */
+static inline void
+rte_ticketlock_lock(rte_ticketlock_t *tl)
+{
+ uint16_t me = __atomic_fetch_add(&tl->s.next, 1, __ATOMIC_RELAXED);
+ rte_wait_until_equal_16(&tl->s.current, me, __ATOMIC_ACQUIRE);
+}
+
+/**
+ * Release the ticketlock.
+ *
+ * @param tl
+ * A pointer to the ticketlock.
+ */
+static inline void
+rte_ticketlock_unlock(rte_ticketlock_t *tl)
+{
+ uint16_t i = __atomic_load_n(&tl->s.current, __ATOMIC_RELAXED);
+ __atomic_store_n(&tl->s.current, i + 1, __ATOMIC_RELEASE);
+}
+
+/**
+ * Try to take the lock.
+ *
+ * @param tl
+ * A pointer to the ticketlock.
+ * @return
+ * 1 if the lock is successfully taken; 0 otherwise.
+ */
+static inline int
+rte_ticketlock_trylock(rte_ticketlock_t *tl)
+{
+ rte_ticketlock_t oldl, newl;
+ oldl.tickets = __atomic_load_n(&tl->tickets, __ATOMIC_RELAXED);
+ newl.tickets = oldl.tickets;
+ newl.s.next++;
+ if (oldl.s.next == oldl.s.current) {
+ if (__atomic_compare_exchange_n(&tl->tickets, &oldl.tickets,
+ newl.tickets, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED))
+ return 1;
+ }
+
+ return 0;
+}
+
+/**
+ * Test if the lock is taken.
+ *
+ * @param tl
+ * A pointer to the ticketlock.
+ * @return
+ * 1 if the lock is currently taken; 0 otherwise.
+ */
+static inline int
+rte_ticketlock_is_locked(rte_ticketlock_t *tl)
+{
+ rte_ticketlock_t tic;
+ tic.tickets = __atomic_load_n(&tl->tickets, __ATOMIC_ACQUIRE);
+ return (tic.s.current != tic.s.next);
+}
+
+/**
+ * The rte_ticketlock_recursive_t type.
+ */
+#define TICKET_LOCK_INVALID_ID -1
+
+typedef struct {
+ rte_ticketlock_t tl; /**< the actual ticketlock */
+ int user; /**< core id using lock, TICKET_LOCK_INVALID_ID for unused */
+ unsigned int count; /**< count of time this lock has been called */
+} rte_ticketlock_recursive_t;
+
+/**
+ * A static recursive ticketlock initializer.
+ */
+#define RTE_TICKETLOCK_RECURSIVE_INITIALIZER {RTE_TICKETLOCK_INITIALIZER, \
+ TICKET_LOCK_INVALID_ID, 0}
+
+/**
+ * Initialize the recursive ticketlock to an unlocked state.
+ *
+ * @param tlr
+ * A pointer to the recursive ticketlock.
+ */
+static inline void
+rte_ticketlock_recursive_init(rte_ticketlock_recursive_t *tlr)
+{
+ rte_ticketlock_init(&tlr->tl);
+ __atomic_store_n(&tlr->user, TICKET_LOCK_INVALID_ID, __ATOMIC_RELAXED);
+ tlr->count = 0;
+}
+
+/**
+ * Take the recursive ticketlock.
+ *
+ * @param tlr
+ * A pointer to the recursive ticketlock.
+ */
+static inline void
+rte_ticketlock_recursive_lock(rte_ticketlock_recursive_t *tlr)
+{
+ int id = rte_gettid();
+
+ if (__atomic_load_n(&tlr->user, __ATOMIC_RELAXED) != id) {
+ rte_ticketlock_lock(&tlr->tl);
+ __atomic_store_n(&tlr->user, id, __ATOMIC_RELAXED);
+ }
+ tlr->count++;
+}
+
+/**
+ * Release the recursive ticketlock.
+ *
+ * @param tlr
+ * A pointer to the recursive ticketlock.
+ */
+static inline void
+rte_ticketlock_recursive_unlock(rte_ticketlock_recursive_t *tlr)
+{
+ if (--(tlr->count) == 0) {
+ __atomic_store_n(&tlr->user, TICKET_LOCK_INVALID_ID,
+ __ATOMIC_RELAXED);
+ rte_ticketlock_unlock(&tlr->tl);
+ }
+}
+
+/**
+ * Try to take the recursive lock.
+ *
+ * @param tlr
+ * A pointer to the recursive ticketlock.
+ * @return
+ * 1 if the lock is successfully taken; 0 otherwise.
+ */
+static inline int
+rte_ticketlock_recursive_trylock(rte_ticketlock_recursive_t *tlr)
+{
+ int id = rte_gettid();
+
+ if (__atomic_load_n(&tlr->user, __ATOMIC_RELAXED) != id) {
+ if (rte_ticketlock_trylock(&tlr->tl) == 0)
+ return 0;
+ __atomic_store_n(&tlr->user, id, __ATOMIC_RELAXED);
+ }
+ tlr->count++;
+ return 1;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _RTE_TICKETLOCK_H_ */
'rte_cpuflags.h',
'rte_cycles.h',
'rte_io.h',
- 'rte_mcslock.h',
'rte_memcpy.h',
'rte_pause.h',
- 'rte_pflock.h',
'rte_power_intrinsics.h',
'rte_prefetch.h',
'rte_rwlock.h',
'rte_spinlock.h',
- 'rte_ticketlock.h',
'rte_vect.h',
)
install_headers(arch_headers, subdir: get_option('include_subdir_arch'))
+++ /dev/null
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2019 Arm Limited
- */
-
-#ifndef _RTE_MCSLOCK_PPC_64_H_
-#define _RTE_MCSLOCK_PPC_64_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "generic/rte_mcslock.h"
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_MCSLOCK_PPC_64_H_ */
+++ /dev/null
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2021 Microsoft Corporation
- */
-#ifndef _RTE_PFLOCK_PPC_64_H_
-#define _RTE_PFLOCK_PPC_64_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "generic/rte_pflock.h"
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_PFLOCK_PPC_64_H_ */
+++ /dev/null
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2019 Arm Limited
- */
-
-#ifndef _RTE_TICKETLOCK_PPC_64_H_
-#define _RTE_TICKETLOCK_PPC_64_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "generic/rte_ticketlock.h"
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_TICKETLOCK_PPC_64_H_ */
'rte_cpuflags.h',
'rte_cycles.h',
'rte_io.h',
- 'rte_mcslock.h',
'rte_memcpy.h',
'rte_pause.h',
- 'rte_pflock.h',
'rte_power_intrinsics.h',
'rte_prefetch.h',
'rte_rwlock.h',
'rte_spinlock.h',
- 'rte_ticketlock.h',
'rte_vect.h',
)
install_headers(arch_headers, subdir: get_option('include_subdir_arch'))
+++ /dev/null
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2019 Arm Limited
- */
-
-#ifndef RTE_MCSLOCK_RISCV_H
-#define RTE_MCSLOCK_RISCV_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "generic/rte_mcslock.h"
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* RTE_MCSLOCK_RISCV_H */
+++ /dev/null
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2021 Microsoft Corporation
- */
-#ifndef RTE_PFLOCK_RISCV_H
-#define RTE_PFLOCK_RISCV_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "generic/rte_pflock.h"
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* RTE_PFLOCK_RISCV_H */
+++ /dev/null
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2019 Arm Limited
- * Copyright(c) 2022 StarFive
- * Copyright(c) 2022 SiFive
- * Copyright(c) 2022 Semihalf
- */
-
-#ifndef RTE_TICKETLOCK_RISCV_H
-#define RTE_TICKETLOCK_RISCV_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "generic/rte_ticketlock.h"
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* RTE_TICKETLOCK_RISCV_H */
'rte_cpuflags.h',
'rte_cycles.h',
'rte_io.h',
- 'rte_mcslock.h',
'rte_memcpy.h',
'rte_pause.h',
- 'rte_pflock.h',
'rte_power_intrinsics.h',
'rte_prefetch.h',
'rte_rtm.h',
'rte_rwlock.h',
'rte_spinlock.h',
- 'rte_ticketlock.h',
'rte_vect.h',
)
arch_indirect_headers = files(
+++ /dev/null
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2019 Arm Limited
- */
-
-#ifndef _RTE_MCSLOCK_X86_64_H_
-#define _RTE_MCSLOCK_X86_64_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "generic/rte_mcslock.h"
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_MCSLOCK_X86_64_H_ */
+++ /dev/null
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2021 Microsoft Corporation
- */
-
-#ifndef _RTE_PFLOCK_X86_64_H_
-#define _RTE_PFLOCK_X86_64_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "generic/rte_pflock.h"
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_PFLOCK_X86_64_H_ */
+++ /dev/null
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2019 Arm Limited
- */
-
-#ifndef _RTE_TICKETLOCK_X86_64_H_
-#define _RTE_TICKETLOCK_X86_64_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "generic/rte_ticketlock.h"
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_TICKETLOCK_X86_64_H_ */
git.droids-corp.org / dpdk.git / commitdiff