#include <rte_event_timer_adapter.h>
#include <rte_event_timer_adapter_pmd.h>
+#include <rte_reciprocal.h>
#include "otx2_dev.h"
#define OTX2_TIM_MAX_CHUNK_SLOTS (0x1FFE)
#define OTX2_TIM_MIN_TMO_TKS (256)
+#define OTX2_TIM_SP 0x1
+#define OTX2_TIM_MP 0x2
+#define OTX2_TIM_BKT_AND 0x4
+#define OTX2_TIM_BKT_MOD 0x8
+#define OTX2_TIM_ENA_FB 0x10
+#define OTX2_TIM_ENA_DFB 0x20
+
enum otx2_tim_clk_src {
OTX2_TIM_CLK_SRC_10NS = RTE_EVENT_TIMER_ADAPTER_CPU_CLK,
OTX2_TIM_CLK_SRC_GPIO = RTE_EVENT_TIMER_ADAPTER_EXT_CLK0,
uint64_t pad;
} __rte_packed __rte_aligned(32);
+struct otx2_tim_ent {
+ uint64_t w0;
+ uint64_t wqe;
+} __rte_packed;
+
struct otx2_tim_evdev {
struct rte_pci_device *pci_dev;
struct rte_eventdev *event_dev;
struct otx2_tim_ring {
uintptr_t base;
+ struct rte_reciprocal_u64 fast_div;
uint16_t nb_chunk_slots;
uint32_t nb_bkts;
+ uint64_t ring_start_cyc;
struct otx2_tim_bkt *bkt;
struct rte_mempool *chunk_pool;
uint64_t tck_int;
return mz->addr;
}
+#define TIM_ARM_FASTPATH_MODES \
+FP(mod_sp, 0, 0, 0, OTX2_TIM_BKT_MOD | OTX2_TIM_ENA_DFB | OTX2_TIM_SP) \
+FP(mod_mp, 0, 0, 1, OTX2_TIM_BKT_MOD | OTX2_TIM_ENA_DFB | OTX2_TIM_MP) \
+FP(mod_fb_sp, 0, 1, 0, OTX2_TIM_BKT_MOD | OTX2_TIM_ENA_FB | OTX2_TIM_SP) \
+FP(mod_fb_mp, 0, 1, 1, OTX2_TIM_BKT_MOD | OTX2_TIM_ENA_FB | OTX2_TIM_MP) \
+FP(and_sp, 1, 0, 0, OTX2_TIM_BKT_AND | OTX2_TIM_ENA_DFB | OTX2_TIM_SP) \
+FP(and_mp, 1, 0, 1, OTX2_TIM_BKT_AND | OTX2_TIM_ENA_DFB | OTX2_TIM_MP) \
+FP(and_fb_sp, 1, 1, 0, OTX2_TIM_BKT_AND | OTX2_TIM_ENA_FB | OTX2_TIM_SP) \
+FP(and_fb_mp, 1, 1, 1, OTX2_TIM_BKT_AND | OTX2_TIM_ENA_FB | OTX2_TIM_MP) \
+
+#define FP(_name, _f3, _f2, _f1, flags) \
+uint16_t otx2_tim_arm_burst_ ## _name( \
+ const struct rte_event_timer_adapter *adptr, \
+ struct rte_event_timer **tim, \
+ const uint16_t nb_timers);
+TIM_ARM_FASTPATH_MODES
+#undef FP
+
int otx2_tim_caps_get(const struct rte_eventdev *dev, uint64_t flags,
uint32_t *caps,
const struct rte_event_timer_adapter_ops **ops);
#include "otx2_tim_evdev.h"
#include "otx2_tim_worker.h"
+static inline int
+tim_arm_checks(const struct otx2_tim_ring * const tim_ring,
+ struct rte_event_timer * const tim)
+{
+ if (unlikely(tim->state)) {
+ tim->state = RTE_EVENT_TIMER_ERROR;
+ rte_errno = EALREADY;
+ goto fail;
+ }
+
+ if (unlikely(!tim->timeout_ticks ||
+ tim->timeout_ticks >= tim_ring->nb_bkts)) {
+ tim->state = tim->timeout_ticks ? RTE_EVENT_TIMER_ERROR_TOOLATE
+ : RTE_EVENT_TIMER_ERROR_TOOEARLY;
+ rte_errno = EINVAL;
+ goto fail;
+ }
+
+ return 0;
+
+fail:
+ return -EINVAL;
+}
+
+static inline void
+tim_format_event(const struct rte_event_timer * const tim,
+ struct otx2_tim_ent * const entry)
+{
+ entry->w0 = (tim->ev.event & 0xFFC000000000) >> 6 |
+ (tim->ev.event & 0xFFFFFFFFF);
+ entry->wqe = tim->ev.u64;
+}
+
+static __rte_always_inline uint16_t
+tim_timer_arm_burst(const struct rte_event_timer_adapter *adptr,
+ struct rte_event_timer **tim,
+ const uint16_t nb_timers,
+ const uint8_t flags)
+{
+ struct otx2_tim_ring *tim_ring = adptr->data->adapter_priv;
+ struct otx2_tim_ent entry;
+ uint16_t index;
+ int ret;
+
+ for (index = 0; index < nb_timers; index++) {
+ if (tim_arm_checks(tim_ring, tim[index]))
+ break;
+
+ tim_format_event(tim[index], &entry);
+ if (flags & OTX2_TIM_SP)
+ ret = tim_add_entry_sp(tim_ring,
+ tim[index]->timeout_ticks,
+ tim[index], &entry, flags);
+ if (flags & OTX2_TIM_MP)
+ ret = tim_add_entry_mp(tim_ring,
+ tim[index]->timeout_ticks,
+ tim[index], &entry, flags);
+
+ if (unlikely(ret)) {
+ rte_errno = -ret;
+ break;
+ }
+ }
+
+ return index;
+}
+
+#define FP(_name, _f3, _f2, _f1, _flags) \
+uint16_t __rte_noinline \
+otx2_tim_arm_burst_ ## _name(const struct rte_event_timer_adapter *adptr, \
+ struct rte_event_timer **tim, \
+ const uint16_t nb_timers) \
+{ \
+ return tim_timer_arm_burst(adptr, tim, nb_timers, _flags); \
+}
+TIM_ARM_FASTPATH_MODES
+#undef FP
return __atomic_and_fetch(&bktp->w1, v, __ATOMIC_ACQ_REL);
}
+static __rte_always_inline struct otx2_tim_bkt *
+tim_get_target_bucket(struct otx2_tim_ring * const tim_ring,
+ const uint32_t rel_bkt, const uint8_t flag)
+{
+ const uint64_t bkt_cyc = rte_rdtsc() - tim_ring->ring_start_cyc;
+ uint32_t bucket = rte_reciprocal_divide_u64(bkt_cyc,
+ &tim_ring->fast_div) + rel_bkt;
+
+ if (flag & OTX2_TIM_BKT_MOD)
+ bucket = bucket % tim_ring->nb_bkts;
+ if (flag & OTX2_TIM_BKT_AND)
+ bucket = bucket & (tim_ring->nb_bkts - 1);
+
+ return &tim_ring->bkt[bucket];
+}
+
+static struct otx2_tim_ent *
+tim_clr_bkt(struct otx2_tim_ring * const tim_ring,
+ struct otx2_tim_bkt * const bkt)
+{
+ struct otx2_tim_ent *chunk;
+ struct otx2_tim_ent *pnext;
+
+ chunk = ((struct otx2_tim_ent *)(uintptr_t)bkt->first_chunk);
+ chunk = (struct otx2_tim_ent *)(uintptr_t)(chunk +
+ tim_ring->nb_chunk_slots)->w0;
+ while (chunk) {
+ pnext = (struct otx2_tim_ent *)(uintptr_t)
+ ((chunk + tim_ring->nb_chunk_slots)->w0);
+ rte_mempool_put(tim_ring->chunk_pool, chunk);
+ chunk = pnext;
+ }
+
+ return (struct otx2_tim_ent *)(uintptr_t)bkt->first_chunk;
+}
+
+static struct otx2_tim_ent *
+tim_refill_chunk(struct otx2_tim_bkt * const bkt,
+ struct otx2_tim_ring * const tim_ring)
+{
+ struct otx2_tim_ent *chunk;
+
+ if (bkt->nb_entry || !bkt->first_chunk) {
+ if (unlikely(rte_mempool_get(tim_ring->chunk_pool,
+ (void **)&chunk)))
+ return NULL;
+ if (bkt->nb_entry) {
+ *(uint64_t *)(((struct otx2_tim_ent *)(uintptr_t)
+ bkt->current_chunk) +
+ tim_ring->nb_chunk_slots) =
+ (uintptr_t)chunk;
+ } else {
+ bkt->first_chunk = (uintptr_t)chunk;
+ }
+ } else {
+ chunk = tim_clr_bkt(tim_ring, bkt);
+ bkt->first_chunk = (uintptr_t)chunk;
+ }
+ *(uint64_t *)(chunk + tim_ring->nb_chunk_slots) = 0;
+
+ return chunk;
+}
+
+static struct otx2_tim_ent *
+tim_insert_chunk(struct otx2_tim_bkt * const bkt,
+ struct otx2_tim_ring * const tim_ring)
+{
+ struct otx2_tim_ent *chunk;
+
+ if (unlikely(rte_mempool_get(tim_ring->chunk_pool, (void **)&chunk)))
+ return NULL;
+
+ *(uint64_t *)(chunk + tim_ring->nb_chunk_slots) = 0;
+ if (bkt->nb_entry) {
+ *(uint64_t *)(((struct otx2_tim_ent *)(uintptr_t)
+ bkt->current_chunk) +
+ tim_ring->nb_chunk_slots) = (uintptr_t)chunk;
+ } else {
+ bkt->first_chunk = (uintptr_t)chunk;
+ }
+
+ return chunk;
+}
+
+static __rte_always_inline int
+tim_add_entry_sp(struct otx2_tim_ring * const tim_ring,
+ const uint32_t rel_bkt,
+ struct rte_event_timer * const tim,
+ const struct otx2_tim_ent * const pent,
+ const uint8_t flags)
+{
+ struct otx2_tim_ent *chunk;
+ struct otx2_tim_bkt *bkt;
+ uint64_t lock_sema;
+ int16_t rem;
+
+ bkt = tim_get_target_bucket(tim_ring, rel_bkt, flags);
+
+__retry:
+ /* Get Bucket sema*/
+ lock_sema = tim_bkt_fetch_sema(bkt);
+
+ /* Bucket related checks. */
+ if (unlikely(tim_bkt_get_hbt(lock_sema)))
+ goto __retry;
+
+ /* Insert the work. */
+ rem = tim_bkt_fetch_rem(lock_sema);
+
+ if (!rem) {
+ if (flags & OTX2_TIM_ENA_FB)
+ chunk = tim_refill_chunk(bkt, tim_ring);
+ if (flags & OTX2_TIM_ENA_DFB)
+ chunk = tim_insert_chunk(bkt, tim_ring);
+
+ if (unlikely(chunk == NULL)) {
+ tim_bkt_set_rem(bkt, 0);
+ tim->impl_opaque[0] = 0;
+ tim->impl_opaque[1] = 0;
+ tim->state = RTE_EVENT_TIMER_ERROR;
+ return -ENOMEM;
+ }
+ bkt->current_chunk = (uintptr_t)chunk;
+ tim_bkt_set_rem(bkt, tim_ring->nb_chunk_slots - 1);
+ } else {
+ chunk = (struct otx2_tim_ent *)(uintptr_t)bkt->current_chunk;
+ chunk += tim_ring->nb_chunk_slots - rem;
+ }
+
+ /* Copy work entry. */
+ *chunk = *pent;
+
+ tim_bkt_inc_nent(bkt);
+
+ tim->impl_opaque[0] = (uintptr_t)chunk;
+ tim->impl_opaque[1] = (uintptr_t)bkt;
+ tim->state = RTE_EVENT_TIMER_ARMED;
+
+ return 0;
+}
+
+static __rte_always_inline int
+tim_add_entry_mp(struct otx2_tim_ring * const tim_ring,
+ const uint32_t rel_bkt,
+ struct rte_event_timer * const tim,
+ const struct otx2_tim_ent * const pent,
+ const uint8_t flags)
+{
+ struct otx2_tim_ent *chunk;
+ struct otx2_tim_bkt *bkt;
+ uint64_t lock_sema;
+ int16_t rem;
+
+__retry:
+ bkt = tim_get_target_bucket(tim_ring, rel_bkt, flags);
+
+ /* Get Bucket sema*/
+ lock_sema = tim_bkt_fetch_sema_lock(bkt);
+
+ /* Bucket related checks. */
+ if (unlikely(tim_bkt_get_hbt(lock_sema))) {
+ tim_bkt_dec_lock(bkt);
+ goto __retry;
+ }
+
+ rem = tim_bkt_fetch_rem(lock_sema);
+
+ if (rem < 0) {
+ /* Goto diff bucket. */
+ tim_bkt_dec_lock(bkt);
+ goto __retry;
+ } else if (!rem) {
+ /* Only one thread can be here*/
+ if (flags & OTX2_TIM_ENA_FB)
+ chunk = tim_refill_chunk(bkt, tim_ring);
+ if (flags & OTX2_TIM_ENA_DFB)
+ chunk = tim_insert_chunk(bkt, tim_ring);
+
+ if (unlikely(chunk == NULL)) {
+ tim_bkt_set_rem(bkt, 0);
+ tim_bkt_dec_lock(bkt);
+ tim->impl_opaque[0] = 0;
+ tim->impl_opaque[1] = 0;
+ tim->state = RTE_EVENT_TIMER_ERROR;
+ return -ENOMEM;
+ }
+ bkt->current_chunk = (uintptr_t)chunk;
+ tim_bkt_set_rem(bkt, tim_ring->nb_chunk_slots - 1);
+ } else {
+ chunk = (struct otx2_tim_ent *)(uintptr_t)bkt->current_chunk;
+ chunk += tim_ring->nb_chunk_slots - rem;
+ }
+
+ /* Copy work entry. */
+ *chunk = *pent;
+ tim_bkt_dec_lock(bkt);
+ tim_bkt_inc_nent(bkt);
+ tim->impl_opaque[0] = (uintptr_t)chunk;
+ tim->impl_opaque[1] = (uintptr_t)bkt;
+ tim->state = RTE_EVENT_TIMER_ARMED;
+
+ return 0;
+}
+
#endif /* __OTX2_TIM_WORKER_H__ */
git.droids-corp.org / dpdk.git / commitdiff