From b6d814d846779bf96dd244c4b01119823b1ace2c Mon Sep 17 00:00:00 2001 From: Pavan Nikhilesh Date: Tue, 10 Apr 2018 02:30:30 +0530 Subject: [PATCH] event/octeontx: add multiproducer timer arm and cancel Signed-off-by: Pavan Nikhilesh Acked-by: Jerin Jacob --- drivers/event/octeontx/Makefile | 5 + drivers/event/octeontx/meson.build | 1 + drivers/event/octeontx/timvf_evdev.c | 9 +- drivers/event/octeontx/timvf_evdev.h | 35 ++++ drivers/event/octeontx/timvf_worker.c | 107 ++++++++++ drivers/event/octeontx/timvf_worker.h | 279 ++++++++++++++++++++++++++ 6 files changed, 435 insertions(+), 1 deletion(-) create mode 100644 drivers/event/octeontx/timvf_worker.c create mode 100644 drivers/event/octeontx/timvf_worker.h diff --git a/drivers/event/octeontx/Makefile b/drivers/event/octeontx/Makefile index 8dbfd88c57..90ad2217f3 100644 --- a/drivers/event/octeontx/Makefile +++ b/drivers/event/octeontx/Makefile @@ -30,20 +30,25 @@ SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF) += ssovf_worker.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF) += ssovf_evdev.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF) += ssovf_evdev_selftest.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF) += ssovf_probe.c +SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF) += timvf_worker.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF) += timvf_evdev.c SRCS-$(CONFIG_RTE_LIBRTE_PMD_OCTEONTX_SSOVF) += timvf_probe.c ifeq ($(CONFIG_RTE_TOOLCHAIN_GCC),y) CFLAGS_ssovf_worker.o += -fno-prefetch-loop-arrays +CFLAGS_timvf_worker.o += -fno-prefetch-loop-arrays ifeq ($(shell test $(GCC_VERSION) -ge 46 && echo 1), 1) CFLAGS_ssovf_worker.o += -Ofast +CFLAGS_timvf_worker.o += -Ofast else CFLAGS_ssovf_worker.o += -O3 -ffast-math +CFLAGS_timvf_worker.o += -O3 -ffast-math endif else CFLAGS_ssovf_worker.o += -Ofast +CFLAGS_timvf_worker.o += -Ofast endif include $(RTE_SDK)/mk/rte.lib.mk diff --git a/drivers/event/octeontx/meson.build b/drivers/event/octeontx/meson.build index 90af4fd1a4..0418553308 100644 --- a/drivers/event/octeontx/meson.build +++ b/drivers/event/octeontx/meson.build @@ -5,6 +5,7 @@ sources = files('ssovf_worker.c', 'ssovf_evdev.c', 'ssovf_evdev_selftest.c', 'ssovf_probe.c', + 'timvf_worker.c', 'timvf_evdev.c', 'timvf_probe.c' ) diff --git a/drivers/event/octeontx/timvf_evdev.c b/drivers/event/octeontx/timvf_evdev.c index c66db437eb..5ffb460a8b 100644 --- a/drivers/event/octeontx/timvf_evdev.c +++ b/drivers/event/octeontx/timvf_evdev.c @@ -130,6 +130,7 @@ timvf_ring_start(const struct rte_event_timer_adapter *adptr) timvf_write64((uintptr_t)timr->bkt, (uint8_t *)timr->vbar0 + TIM_VRING_BASE); + timvf_set_chunk_refill(timr); if (timvf_ring_conf_set(&rctrl, timr->tim_ring_id)) { ret = -EACCES; goto error; @@ -312,7 +313,13 @@ timvf_timer_adapter_caps_get(const struct rte_eventdev *dev, uint64_t flags, timvf_ops.stats_reset = timvf_stats_reset; } + if (enable_stats) + timvf_ops.arm_burst = timvf_timer_arm_burst_mp_stats; + else + timvf_ops.arm_burst = timvf_timer_arm_burst_mp; + + timvf_ops.cancel_burst = timvf_timer_cancel_burst; *caps = RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT; *ops = &timvf_ops; - return -EINVAL; + return 0; } diff --git a/drivers/event/octeontx/timvf_evdev.h b/drivers/event/octeontx/timvf_evdev.h index e89a43531a..cae949c73d 100644 --- a/drivers/event/octeontx/timvf_evdev.h +++ b/drivers/event/octeontx/timvf_evdev.h @@ -75,6 +75,33 @@ #define TIM_VRING_AURA (0x108) #define TIM_VRING_REL (0x110) +#define TIM_CTL1_W0_S_BUCKET 20 +#define TIM_CTL1_W0_M_BUCKET ((1ull << (40 - 20)) - 1) + +#define TIM_BUCKET_W1_S_NUM_ENTRIES (0) /*Shift*/ +#define TIM_BUCKET_W1_M_NUM_ENTRIES ((1ull << (32 - 0)) - 1) +#define TIM_BUCKET_W1_S_SBT (32) +#define TIM_BUCKET_W1_M_SBT ((1ull << (33 - 32)) - 1) +#define TIM_BUCKET_W1_S_HBT (33) +#define TIM_BUCKET_W1_M_HBT ((1ull << (34 - 33)) - 1) +#define TIM_BUCKET_W1_S_BSK (34) +#define TIM_BUCKET_W1_M_BSK ((1ull << (35 - 34)) - 1) +#define TIM_BUCKET_W1_S_LOCK (40) +#define TIM_BUCKET_W1_M_LOCK ((1ull << (48 - 40)) - 1) +#define TIM_BUCKET_W1_S_CHUNK_REMAINDER (48) +#define TIM_BUCKET_W1_M_CHUNK_REMAINDER ((1ull << (64 - 48)) - 1) + +#define TIM_BUCKET_SEMA \ + (TIM_BUCKET_CHUNK_REMAIN) + +#define TIM_BUCKET_CHUNK_REMAIN \ + (TIM_BUCKET_W1_M_CHUNK_REMAINDER << TIM_BUCKET_W1_S_CHUNK_REMAINDER) + +#define TIM_BUCKET_LOCK \ + (TIM_BUCKET_W1_M_LOCK << TIM_BUCKET_W1_S_LOCK) + +#define TIM_BUCKET_SEMA_WLOCK \ + (TIM_BUCKET_CHUNK_REMAIN | (1ull << TIM_BUCKET_W1_S_LOCK)) #define NSEC_PER_SEC 1E9 #define NSEC2CLK(__ns, __freq) (((__ns) * (__freq)) / NSEC_PER_SEC) @@ -168,5 +195,13 @@ void *timvf_bar(uint8_t id, uint8_t bar); int timvf_timer_adapter_caps_get(const struct rte_eventdev *dev, uint64_t flags, uint32_t *caps, const struct rte_event_timer_adapter_ops **ops, uint8_t enable_stats); +uint16_t timvf_timer_cancel_burst(const struct rte_event_timer_adapter *adptr, + struct rte_event_timer **tim, const uint16_t nb_timers); +uint16_t timvf_timer_arm_burst_mp(const struct rte_event_timer_adapter *adptr, + struct rte_event_timer **tim, const uint16_t nb_timers); +uint16_t timvf_timer_arm_burst_mp_stats( + const struct rte_event_timer_adapter *adptr, + struct rte_event_timer **tim, const uint16_t nb_timers); +void timvf_set_chunk_refill(struct timvf_ring * const timr); #endif /* __TIMVF_EVDEV_H__ */ diff --git a/drivers/event/octeontx/timvf_worker.c b/drivers/event/octeontx/timvf_worker.c new file mode 100644 index 0000000000..38b16d1ced --- /dev/null +++ b/drivers/event/octeontx/timvf_worker.c @@ -0,0 +1,107 @@ +/* + * SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Cavium, Inc + */ + +#include "timvf_worker.h" + +static inline int +timvf_timer_reg_checks(const struct timvf_ring * const timr, + struct rte_event_timer * const tim) +{ + if (unlikely(tim->state)) { + tim->state = RTE_EVENT_TIMER_ERROR; + rte_errno = EALREADY; + goto fail; + } + + if (unlikely(!tim->timeout_ticks || + tim->timeout_ticks >= timr->nb_bkts)) { + tim->state = tim->timeout_ticks ? RTE_EVENT_TIMER_ERROR_TOOLATE + : RTE_EVENT_TIMER_ERROR_TOOEARLY; + rte_errno = EINVAL; + goto fail; + } + + return 0; +fail: + return -EINVAL; +} + +static inline void +timvf_format_event(const struct rte_event_timer * const tim, + struct tim_mem_entry * const entry) +{ + entry->w0 = (tim->ev.event & 0xFFC000000000) >> 6 | + (tim->ev.event & 0xFFFFFFFFF); + entry->wqe = tim->ev.u64; +} + +uint16_t +timvf_timer_cancel_burst(const struct rte_event_timer_adapter *adptr, + struct rte_event_timer **tim, const uint16_t nb_timers) +{ + RTE_SET_USED(adptr); + int ret; + uint16_t index; + + for (index = 0; index < nb_timers; index++) { + if (tim[index]->state == RTE_EVENT_TIMER_CANCELED) { + rte_errno = EALREADY; + break; + } + + if (tim[index]->state != RTE_EVENT_TIMER_ARMED) { + rte_errno = EINVAL; + break; + } + ret = timvf_rem_entry(tim[index]); + if (ret) { + rte_errno = -ret; + break; + } + } + return index; +} + +uint16_t +timvf_timer_arm_burst_mp(const struct rte_event_timer_adapter *adptr, + struct rte_event_timer **tim, const uint16_t nb_timers) +{ + int ret; + uint16_t index; + struct tim_mem_entry entry; + struct timvf_ring *timr = adptr->data->adapter_priv; + for (index = 0; index < nb_timers; index++) { + if (timvf_timer_reg_checks(timr, tim[index])) + break; + timvf_format_event(tim[index], &entry); + ret = timvf_add_entry_mp(timr, tim[index]->timeout_ticks, + tim[index], &entry); + if (unlikely(ret)) { + rte_errno = -ret; + break; + } + } + + return index; +} + +uint16_t +timvf_timer_arm_burst_mp_stats(const struct rte_event_timer_adapter *adptr, + struct rte_event_timer **tim, const uint16_t nb_timers) +{ + uint16_t ret; + struct timvf_ring *timr = adptr->data->adapter_priv; + + ret = timvf_timer_arm_burst_mp(adptr, tim, nb_timers); + timr->tim_arm_cnt += ret; + + return ret; +} + +void +timvf_set_chunk_refill(struct timvf_ring * const timr) +{ + timr->refill_chunk = timvf_refill_chunk_generic; +} diff --git a/drivers/event/octeontx/timvf_worker.h b/drivers/event/octeontx/timvf_worker.h new file mode 100644 index 0000000000..c20d4d0cdd --- /dev/null +++ b/drivers/event/octeontx/timvf_worker.h @@ -0,0 +1,279 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * Copyright(c) 2017 Cavium, Inc + */ + +#include +#include + +#include "timvf_evdev.h" + +static inline int16_t +timr_bkt_fetch_rem(uint64_t w1) +{ + return (w1 >> TIM_BUCKET_W1_S_CHUNK_REMAINDER) & + TIM_BUCKET_W1_M_CHUNK_REMAINDER; +} + +static inline int16_t +timr_bkt_get_rem(struct tim_mem_bucket *bktp) +{ + return __atomic_load_n(&bktp->chunk_remainder, + __ATOMIC_ACQUIRE); +} + +static inline void +timr_bkt_set_rem(struct tim_mem_bucket *bktp, uint16_t v) +{ + __atomic_store_n(&bktp->chunk_remainder, v, + __ATOMIC_RELEASE); +} + +static inline void +timr_bkt_sub_rem(struct tim_mem_bucket *bktp, uint16_t v) +{ + __atomic_fetch_sub(&bktp->chunk_remainder, v, + __ATOMIC_RELEASE); +} + +static inline uint8_t +timr_bkt_get_sbt(uint64_t w1) +{ + return (w1 >> TIM_BUCKET_W1_S_SBT) & TIM_BUCKET_W1_M_SBT; +} + +static inline uint64_t +timr_bkt_set_sbt(struct tim_mem_bucket *bktp) +{ + const uint64_t v = TIM_BUCKET_W1_M_SBT << TIM_BUCKET_W1_S_SBT; + return __atomic_fetch_or(&bktp->w1, v, __ATOMIC_ACQ_REL); +} + +static inline uint64_t +timr_bkt_clr_sbt(struct tim_mem_bucket *bktp) +{ + const uint64_t v = ~(TIM_BUCKET_W1_M_SBT << TIM_BUCKET_W1_S_SBT); + return __atomic_fetch_and(&bktp->w1, v, __ATOMIC_ACQ_REL); +} + +static inline uint8_t +timr_bkt_get_shbt(uint64_t w1) +{ + return ((w1 >> TIM_BUCKET_W1_S_HBT) & TIM_BUCKET_W1_M_HBT) | + ((w1 >> TIM_BUCKET_W1_S_SBT) & TIM_BUCKET_W1_M_SBT); +} + +static inline uint8_t +timr_bkt_get_hbt(uint64_t w1) +{ + return (w1 >> TIM_BUCKET_W1_S_HBT) & TIM_BUCKET_W1_M_HBT; +} + +static inline uint8_t +timr_bkt_get_bsk(uint64_t w1) +{ + return (w1 >> TIM_BUCKET_W1_S_BSK) & TIM_BUCKET_W1_M_BSK; +} + +static inline uint64_t +timr_bkt_clr_bsk(struct tim_mem_bucket *bktp) +{ + /*Clear everything except lock. */ + const uint64_t v = TIM_BUCKET_W1_M_LOCK << TIM_BUCKET_W1_S_LOCK; + return __atomic_fetch_and(&bktp->w1, v, __ATOMIC_ACQ_REL); +} + +static inline uint64_t +timr_bkt_fetch_sema_lock(struct tim_mem_bucket *bktp) +{ + return __atomic_fetch_add(&bktp->w1, TIM_BUCKET_SEMA_WLOCK, + __ATOMIC_ACQ_REL); +} + +static inline uint64_t +timr_bkt_fetch_sema(struct tim_mem_bucket *bktp) +{ + return __atomic_fetch_add(&bktp->w1, TIM_BUCKET_SEMA, + __ATOMIC_RELAXED); +} + +static inline uint64_t +timr_bkt_inc_lock(struct tim_mem_bucket *bktp) +{ + const uint64_t v = 1ull << TIM_BUCKET_W1_S_LOCK; + return __atomic_fetch_add(&bktp->w1, v, __ATOMIC_ACQ_REL); +} + +static inline void +timr_bkt_dec_lock(struct tim_mem_bucket *bktp) +{ + __atomic_add_fetch(&bktp->lock, 0xff, __ATOMIC_ACQ_REL); +} + +static inline uint32_t +timr_bkt_get_nent(uint64_t w1) +{ + return (w1 >> TIM_BUCKET_W1_S_NUM_ENTRIES) & + TIM_BUCKET_W1_M_NUM_ENTRIES; +} + +static inline void +timr_bkt_inc_nent(struct tim_mem_bucket *bktp) +{ + __atomic_add_fetch(&bktp->nb_entry, 1, __ATOMIC_RELAXED); +} + +static inline void +timr_bkt_add_nent(struct tim_mem_bucket *bktp, uint32_t v) +{ + __atomic_add_fetch(&bktp->nb_entry, v, __ATOMIC_RELAXED); +} + +static inline uint64_t +timr_bkt_clr_nent(struct tim_mem_bucket *bktp) +{ + const uint64_t v = ~(TIM_BUCKET_W1_M_NUM_ENTRIES << + TIM_BUCKET_W1_S_NUM_ENTRIES); + return __atomic_and_fetch(&bktp->w1, v, __ATOMIC_ACQ_REL); +} + +static inline struct tim_mem_entry * +timr_clr_bkt(struct timvf_ring * const timr, struct tim_mem_bucket * const bkt) +{ + struct tim_mem_entry *chunk; + struct tim_mem_entry *pnext; + chunk = ((struct tim_mem_entry *)(uintptr_t)bkt->first_chunk); + chunk = (struct tim_mem_entry *)(uintptr_t)(chunk + nb_chunk_slots)->w0; + + while (chunk) { + pnext = (struct tim_mem_entry *)(uintptr_t) + ((chunk + nb_chunk_slots)->w0); + rte_mempool_put(timr->chunk_pool, chunk); + chunk = pnext; + } + return (struct tim_mem_entry *)(uintptr_t)bkt->first_chunk; +} + +static inline int +timvf_rem_entry(struct rte_event_timer *tim) +{ + uint64_t lock_sema; + struct tim_mem_entry *entry; + struct tim_mem_bucket *bkt; + if (tim->impl_opaque[1] == 0 || + tim->impl_opaque[0] == 0) + return -ENOENT; + + entry = (struct tim_mem_entry *)(uintptr_t)tim->impl_opaque[0]; + if (entry->wqe != tim->ev.u64) { + tim->impl_opaque[1] = tim->impl_opaque[0] = 0; + return -ENOENT; + } + bkt = (struct tim_mem_bucket *)(uintptr_t)tim->impl_opaque[1]; + lock_sema = timr_bkt_inc_lock(bkt); + if (timr_bkt_get_shbt(lock_sema) + || !timr_bkt_get_nent(lock_sema)) { + timr_bkt_dec_lock(bkt); + tim->impl_opaque[1] = tim->impl_opaque[0] = 0; + return -ENOENT; + } + + entry->w0 = entry->wqe = 0; + timr_bkt_dec_lock(bkt); + + tim->state = RTE_EVENT_TIMER_CANCELED; + tim->impl_opaque[1] = tim->impl_opaque[0] = 0; + return 0; +} + +static inline struct tim_mem_entry * +timvf_refill_chunk_generic(struct tim_mem_bucket * const bkt, + struct timvf_ring * const timr) +{ + struct tim_mem_entry *chunk; + + if (bkt->nb_entry || !bkt->first_chunk) { + if (unlikely(rte_mempool_get(timr->chunk_pool, + (void **)&chunk))) { + return NULL; + } + if (bkt->nb_entry) { + *(uint64_t *)(((struct tim_mem_entry *)(uintptr_t) + bkt->current_chunk) + + nb_chunk_slots) = + (uintptr_t) chunk; + } else { + bkt->first_chunk = (uintptr_t) chunk; + } + } else { + chunk = timr_clr_bkt(timr, bkt); + bkt->first_chunk = (uintptr_t)chunk; + } + *(uint64_t *)(chunk + nb_chunk_slots) = 0; + + return chunk; +} + +static inline struct tim_mem_bucket * +timvf_get_target_bucket(struct timvf_ring * const timr, const uint32_t rel_bkt) +{ + const uint64_t bkt_cyc = rte_rdtsc() - timr->ring_start_cyc; + const uint32_t bucket = rte_reciprocal_divide_u64(bkt_cyc, + &timr->fast_div) + rel_bkt; + const uint32_t tbkt_id = timr->get_target_bkt(bucket, + timr->nb_bkts); + return &timr->bkt[tbkt_id]; +} + +/* Multi producer functions. */ +static inline int +timvf_add_entry_mp(struct timvf_ring * const timr, const uint32_t rel_bkt, + struct rte_event_timer * const tim, + const struct tim_mem_entry * const pent) +{ + int16_t rem; + uint64_t lock_sema; + struct tim_mem_bucket *bkt; + struct tim_mem_entry *chunk; + +__retry: + bkt = timvf_get_target_bucket(timr, rel_bkt); + /* Bucket related checks. */ + /*Get Bucket sema*/ + lock_sema = timr_bkt_fetch_sema_lock(bkt); + if (unlikely(timr_bkt_get_shbt(lock_sema))) { + timr_bkt_dec_lock(bkt); + goto __retry; + } + + rem = timr_bkt_fetch_rem(lock_sema); + + if (rem < 0) { + /* goto diff bucket. */ + timr_bkt_dec_lock(bkt); + goto __retry; + } else if (!rem) { + /*Only one thread can be here*/ + chunk = timr->refill_chunk(bkt, timr); + if (unlikely(chunk == NULL)) { + timr_bkt_set_rem(bkt, 0); + timr_bkt_dec_lock(bkt); + tim->impl_opaque[0] = tim->impl_opaque[1] = 0; + tim->state = RTE_EVENT_TIMER_ERROR; + return -ENOMEM; + } + bkt->current_chunk = (uintptr_t) chunk; + timr_bkt_set_rem(bkt, nb_chunk_slots - 1); + } else { + chunk = (struct tim_mem_entry *)(uintptr_t)bkt->current_chunk; + chunk += nb_chunk_slots - rem; + } + /* Copy work entry. */ + *chunk = *pent; + timr_bkt_inc_nent(bkt); + timr_bkt_dec_lock(bkt); + + tim->impl_opaque[0] = (uintptr_t)chunk; + tim->impl_opaque[1] = (uintptr_t)bkt; + tim->state = RTE_EVENT_TIMER_ARMED; + return 0; +} -- 2.20.1