From: Pavan Nikhilesh Date: Tue, 4 May 2021 00:27:21 +0000 (+0530) Subject: event/cnxk: add timer arm timeout burst X-Git-Url: http://git.droids-corp.org/?a=commitdiff_plain;h=5f644e1bd14c14b0aa2fd4bc8e3b437f69a9ba49;p=dpdk.git event/cnxk: add timer arm timeout burst Add event timer arm timeout burst function. All the timers requested to be armed have the same timeout. Signed-off-by: Pavan Nikhilesh Signed-off-by: Shijith Thotton --- diff --git a/drivers/event/cnxk/cnxk_tim_evdev.c b/drivers/event/cnxk/cnxk_tim_evdev.c index ecc952a6a7..68c3b30499 100644 --- a/drivers/event/cnxk/cnxk_tim_evdev.c +++ b/drivers/event/cnxk/cnxk_tim_evdev.c @@ -88,7 +88,14 @@ cnxk_tim_set_fp_ops(struct cnxk_tim_ring *tim_ring) #undef FP }; + const rte_event_timer_arm_tmo_tick_burst_t arm_tmo_burst[2] = { +#define FP(_name, _f1, flags) [_f1] = cnxk_tim_arm_tmo_tick_burst_##_name, + TIM_ARM_TMO_FASTPATH_MODES +#undef FP + }; + cnxk_tim_ops.arm_burst = arm_burst[tim_ring->ena_dfb][prod_flag]; + cnxk_tim_ops.arm_tmo_tick_burst = arm_tmo_burst[tim_ring->ena_dfb]; } static void diff --git a/drivers/event/cnxk/cnxk_tim_evdev.h b/drivers/event/cnxk/cnxk_tim_evdev.h index 1f2aad17a7..b66aac17c6 100644 --- a/drivers/event/cnxk/cnxk_tim_evdev.h +++ b/drivers/event/cnxk/cnxk_tim_evdev.h @@ -217,6 +217,10 @@ cnxk_tim_cntfrq(void) FP(fb_sp, 1, 0, CNXK_TIM_ENA_FB | CNXK_TIM_SP) \ FP(fb_mp, 1, 1, CNXK_TIM_ENA_FB | CNXK_TIM_MP) +#define TIM_ARM_TMO_FASTPATH_MODES \ + FP(dfb, 0, CNXK_TIM_ENA_DFB) \ + FP(fb, 1, CNXK_TIM_ENA_FB) + #define FP(_name, _f2, _f1, flags) \ uint16_t cnxk_tim_arm_burst_##_name( \ const struct rte_event_timer_adapter *adptr, \ @@ -224,6 +228,14 @@ cnxk_tim_cntfrq(void) TIM_ARM_FASTPATH_MODES #undef FP +#define FP(_name, _f1, flags) \ + uint16_t cnxk_tim_arm_tmo_tick_burst_##_name( \ + const struct rte_event_timer_adapter *adptr, \ + struct rte_event_timer **tim, const uint64_t timeout_tick, \ + const uint16_t nb_timers); +TIM_ARM_TMO_FASTPATH_MODES +#undef FP + int cnxk_tim_caps_get(const struct rte_eventdev *dev, uint64_t flags, uint32_t *caps, const struct rte_event_timer_adapter_ops **ops); diff --git a/drivers/event/cnxk/cnxk_tim_worker.c b/drivers/event/cnxk/cnxk_tim_worker.c index 268f845c89..717c53fb73 100644 --- a/drivers/event/cnxk/cnxk_tim_worker.c +++ b/drivers/event/cnxk/cnxk_tim_worker.c @@ -99,3 +99,56 @@ cnxk_tim_timer_arm_burst(const struct rte_event_timer_adapter *adptr, } TIM_ARM_FASTPATH_MODES #undef FP + +static __rte_always_inline uint16_t +cnxk_tim_timer_arm_tmo_brst(const struct rte_event_timer_adapter *adptr, + struct rte_event_timer **tim, + const uint64_t timeout_tick, + const uint16_t nb_timers, const uint8_t flags) +{ + struct cnxk_tim_ent entry[CNXK_TIM_MAX_BURST] __rte_cache_aligned; + struct cnxk_tim_ring *tim_ring = adptr->data->adapter_priv; + uint16_t set_timers = 0; + uint16_t arr_idx = 0; + uint16_t idx; + int ret; + + if (unlikely(!timeout_tick || timeout_tick > tim_ring->nb_bkts)) { + const enum rte_event_timer_state state = + timeout_tick ? RTE_EVENT_TIMER_ERROR_TOOLATE : + RTE_EVENT_TIMER_ERROR_TOOEARLY; + for (idx = 0; idx < nb_timers; idx++) + tim[idx]->state = state; + + rte_errno = EINVAL; + return 0; + } + + cnxk_tim_sync_start_cyc(tim_ring); + while (arr_idx < nb_timers) { + for (idx = 0; idx < CNXK_TIM_MAX_BURST && (arr_idx < nb_timers); + idx++, arr_idx++) { + cnxk_tim_format_event(tim[arr_idx], &entry[idx]); + } + ret = cnxk_tim_add_entry_brst(tim_ring, timeout_tick, + &tim[set_timers], entry, idx, + flags); + set_timers += ret; + if (ret != idx) + break; + } + + return set_timers; +} + +#define FP(_name, _f1, _flags) \ + uint16_t __rte_noinline cnxk_tim_arm_tmo_tick_burst_##_name( \ + const struct rte_event_timer_adapter *adptr, \ + struct rte_event_timer **tim, const uint64_t timeout_tick, \ + const uint16_t nb_timers) \ + { \ + return cnxk_tim_timer_arm_tmo_brst(adptr, tim, timeout_tick, \ + nb_timers, _flags); \ + } +TIM_ARM_TMO_FASTPATH_MODES +#undef FP diff --git a/drivers/event/cnxk/cnxk_tim_worker.h b/drivers/event/cnxk/cnxk_tim_worker.h index de8464e336..56cb4cdd93 100644 --- a/drivers/event/cnxk/cnxk_tim_worker.h +++ b/drivers/event/cnxk/cnxk_tim_worker.h @@ -420,4 +420,145 @@ __retry: return 0; } +static inline uint16_t +cnxk_tim_cpy_wrk(uint16_t index, uint16_t cpy_lmt, struct cnxk_tim_ent *chunk, + struct rte_event_timer **const tim, + const struct cnxk_tim_ent *const ents, + const struct cnxk_tim_bkt *const bkt) +{ + for (; index < cpy_lmt; index++) { + *chunk = *(ents + index); + tim[index]->impl_opaque[0] = (uintptr_t)chunk++; + tim[index]->impl_opaque[1] = (uintptr_t)bkt; + tim[index]->state = RTE_EVENT_TIMER_ARMED; + } + + return index; +} + +/* Burst mode functions */ +static inline int +cnxk_tim_add_entry_brst(struct cnxk_tim_ring *const tim_ring, + const uint16_t rel_bkt, + struct rte_event_timer **const tim, + const struct cnxk_tim_ent *ents, + const uint16_t nb_timers, const uint8_t flags) +{ + struct cnxk_tim_ent *chunk = NULL; + struct cnxk_tim_bkt *mirr_bkt; + struct cnxk_tim_bkt *bkt; + uint16_t chunk_remainder; + uint16_t index = 0; + uint64_t lock_sema; + int16_t rem, crem; + uint8_t lock_cnt; + +__retry: + cnxk_tim_get_target_bucket(tim_ring, rel_bkt, &bkt, &mirr_bkt); + + /* Only one thread beyond this. */ + lock_sema = cnxk_tim_bkt_inc_lock(bkt); + lock_cnt = (uint8_t)((lock_sema >> TIM_BUCKET_W1_S_LOCK) & + TIM_BUCKET_W1_M_LOCK); + + if (lock_cnt) { + cnxk_tim_bkt_dec_lock(bkt); +#ifdef RTE_ARCH_ARM64 + asm volatile(PLT_CPU_FEATURE_PREAMBLE + " ldxrb %w[lock_cnt], [%[lock]] \n" + " tst %w[lock_cnt], 255 \n" + " beq dne%= \n" + " sevl \n" + "rty%=: wfe \n" + " ldxrb %w[lock_cnt], [%[lock]] \n" + " tst %w[lock_cnt], 255 \n" + " bne rty%= \n" + "dne%=: \n" + : [lock_cnt] "=&r"(lock_cnt) + : [lock] "r"(&bkt->lock) + : "memory"); +#else + while (__atomic_load_n(&bkt->lock, __ATOMIC_RELAXED)) + ; +#endif + goto __retry; + } + + /* Bucket related checks. */ + if (unlikely(cnxk_tim_bkt_get_hbt(lock_sema))) { + if (cnxk_tim_bkt_get_nent(lock_sema) != 0) { + uint64_t hbt_state; +#ifdef RTE_ARCH_ARM64 + asm volatile(PLT_CPU_FEATURE_PREAMBLE + " ldxr %[hbt], [%[w1]] \n" + " tbz %[hbt], 33, dne%= \n" + " sevl \n" + "rty%=: wfe \n" + " ldxr %[hbt], [%[w1]] \n" + " tbnz %[hbt], 33, rty%= \n" + "dne%=: \n" + : [hbt] "=&r"(hbt_state) + : [w1] "r"((&bkt->w1)) + : "memory"); +#else + do { + hbt_state = __atomic_load_n(&bkt->w1, + __ATOMIC_RELAXED); + } while (hbt_state & BIT_ULL(33)); +#endif + + if (!(hbt_state & BIT_ULL(34))) { + cnxk_tim_bkt_dec_lock(bkt); + goto __retry; + } + } + } + + chunk_remainder = cnxk_tim_bkt_fetch_rem(lock_sema); + rem = chunk_remainder - nb_timers; + if (rem < 0) { + crem = tim_ring->nb_chunk_slots - chunk_remainder; + if (chunk_remainder && crem) { + chunk = ((struct cnxk_tim_ent *) + mirr_bkt->current_chunk) + + crem; + + index = cnxk_tim_cpy_wrk(index, chunk_remainder, chunk, + tim, ents, bkt); + cnxk_tim_bkt_sub_rem(bkt, chunk_remainder); + cnxk_tim_bkt_add_nent(bkt, chunk_remainder); + } + + if (flags & CNXK_TIM_ENA_FB) + chunk = cnxk_tim_refill_chunk(bkt, mirr_bkt, tim_ring); + if (flags & CNXK_TIM_ENA_DFB) + chunk = cnxk_tim_insert_chunk(bkt, mirr_bkt, tim_ring); + + if (unlikely(chunk == NULL)) { + cnxk_tim_bkt_dec_lock(bkt); + rte_errno = ENOMEM; + tim[index]->state = RTE_EVENT_TIMER_ERROR; + return crem; + } + *(uint64_t *)(chunk + tim_ring->nb_chunk_slots) = 0; + mirr_bkt->current_chunk = (uintptr_t)chunk; + cnxk_tim_cpy_wrk(index, nb_timers, chunk, tim, ents, bkt); + + rem = nb_timers - chunk_remainder; + cnxk_tim_bkt_set_rem(bkt, tim_ring->nb_chunk_slots - rem); + cnxk_tim_bkt_add_nent(bkt, rem); + } else { + chunk = (struct cnxk_tim_ent *)mirr_bkt->current_chunk; + chunk += (tim_ring->nb_chunk_slots - chunk_remainder); + + cnxk_tim_cpy_wrk(index, nb_timers, chunk, tim, ents, bkt); + cnxk_tim_bkt_sub_rem(bkt, nb_timers); + cnxk_tim_bkt_add_nent(bkt, nb_timers); + } + + cnxk_tim_bkt_dec_lock(bkt); + + return nb_timers; +} + #endif /* __CNXK_TIM_WORKER_H__ */