1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2019 Marvell International Ltd.
5 #ifndef __OTX2_TIM_WORKER_H__
6 #define __OTX2_TIM_WORKER_H__
8 #include "otx2_tim_evdev.h"
11 tim_bkt_fetch_rem(uint64_t w1)
13 return (w1 >> TIM_BUCKET_W1_S_CHUNK_REMAINDER) &
14 TIM_BUCKET_W1_M_CHUNK_REMAINDER;
18 tim_bkt_get_rem(struct otx2_tim_bkt *bktp)
20 return __atomic_load_n(&bktp->chunk_remainder, __ATOMIC_ACQUIRE);
24 tim_bkt_set_rem(struct otx2_tim_bkt *bktp, uint16_t v)
26 __atomic_store_n(&bktp->chunk_remainder, v, __ATOMIC_RELAXED);
30 tim_bkt_sub_rem(struct otx2_tim_bkt *bktp, uint16_t v)
32 __atomic_fetch_sub(&bktp->chunk_remainder, v, __ATOMIC_RELAXED);
36 tim_bkt_get_hbt(uint64_t w1)
38 return (w1 >> TIM_BUCKET_W1_S_HBT) & TIM_BUCKET_W1_M_HBT;
42 tim_bkt_get_bsk(uint64_t w1)
44 return (w1 >> TIM_BUCKET_W1_S_BSK) & TIM_BUCKET_W1_M_BSK;
47 static inline uint64_t
48 tim_bkt_clr_bsk(struct otx2_tim_bkt *bktp)
50 /* Clear everything except lock. */
51 const uint64_t v = TIM_BUCKET_W1_M_LOCK << TIM_BUCKET_W1_S_LOCK;
53 return __atomic_fetch_and(&bktp->w1, v, __ATOMIC_ACQ_REL);
56 static inline uint64_t
57 tim_bkt_fetch_sema_lock(struct otx2_tim_bkt *bktp)
59 return __atomic_fetch_add(&bktp->w1, TIM_BUCKET_SEMA_WLOCK,
63 static inline uint64_t
64 tim_bkt_fetch_sema(struct otx2_tim_bkt *bktp)
66 return __atomic_fetch_add(&bktp->w1, TIM_BUCKET_SEMA, __ATOMIC_RELAXED);
69 static inline uint64_t
70 tim_bkt_inc_lock(struct otx2_tim_bkt *bktp)
72 const uint64_t v = 1ull << TIM_BUCKET_W1_S_LOCK;
74 return __atomic_fetch_add(&bktp->w1, v, __ATOMIC_ACQUIRE);
78 tim_bkt_dec_lock(struct otx2_tim_bkt *bktp)
80 __atomic_add_fetch(&bktp->lock, 0xff, __ATOMIC_RELEASE);
83 static inline uint32_t
84 tim_bkt_get_nent(uint64_t w1)
86 return (w1 >> TIM_BUCKET_W1_S_NUM_ENTRIES) &
87 TIM_BUCKET_W1_M_NUM_ENTRIES;
91 tim_bkt_inc_nent(struct otx2_tim_bkt *bktp)
93 __atomic_add_fetch(&bktp->nb_entry, 1, __ATOMIC_RELAXED);
97 tim_bkt_add_nent(struct otx2_tim_bkt *bktp, uint32_t v)
99 __atomic_add_fetch(&bktp->nb_entry, v, __ATOMIC_RELAXED);
102 static inline uint64_t
103 tim_bkt_clr_nent(struct otx2_tim_bkt *bktp)
105 const uint64_t v = ~(TIM_BUCKET_W1_M_NUM_ENTRIES <<
106 TIM_BUCKET_W1_S_NUM_ENTRIES);
108 return __atomic_and_fetch(&bktp->w1, v, __ATOMIC_ACQ_REL);
111 static __rte_always_inline struct otx2_tim_bkt *
112 tim_get_target_bucket(struct otx2_tim_ring * const tim_ring,
113 const uint32_t rel_bkt, const uint8_t flag)
115 const uint64_t bkt_cyc = rte_rdtsc() - tim_ring->ring_start_cyc;
116 uint32_t bucket = rte_reciprocal_divide_u64(bkt_cyc,
117 &tim_ring->fast_div) + rel_bkt;
119 if (flag & OTX2_TIM_BKT_MOD)
120 bucket = bucket % tim_ring->nb_bkts;
121 if (flag & OTX2_TIM_BKT_AND)
122 bucket = bucket & (tim_ring->nb_bkts - 1);
124 return &tim_ring->bkt[bucket];
127 static struct otx2_tim_ent *
128 tim_clr_bkt(struct otx2_tim_ring * const tim_ring,
129 struct otx2_tim_bkt * const bkt)
131 struct otx2_tim_ent *chunk;
132 struct otx2_tim_ent *pnext;
134 chunk = ((struct otx2_tim_ent *)(uintptr_t)bkt->first_chunk);
135 chunk = (struct otx2_tim_ent *)(uintptr_t)(chunk +
136 tim_ring->nb_chunk_slots)->w0;
138 pnext = (struct otx2_tim_ent *)(uintptr_t)
139 ((chunk + tim_ring->nb_chunk_slots)->w0);
140 rte_mempool_put(tim_ring->chunk_pool, chunk);
144 return (struct otx2_tim_ent *)(uintptr_t)bkt->first_chunk;
147 static struct otx2_tim_ent *
148 tim_refill_chunk(struct otx2_tim_bkt * const bkt,
149 struct otx2_tim_ring * const tim_ring)
151 struct otx2_tim_ent *chunk;
153 if (bkt->nb_entry || !bkt->first_chunk) {
154 if (unlikely(rte_mempool_get(tim_ring->chunk_pool,
158 *(uint64_t *)(((struct otx2_tim_ent *)(uintptr_t)
159 bkt->current_chunk) +
160 tim_ring->nb_chunk_slots) =
163 bkt->first_chunk = (uintptr_t)chunk;
166 chunk = tim_clr_bkt(tim_ring, bkt);
167 bkt->first_chunk = (uintptr_t)chunk;
169 *(uint64_t *)(chunk + tim_ring->nb_chunk_slots) = 0;
174 static struct otx2_tim_ent *
175 tim_insert_chunk(struct otx2_tim_bkt * const bkt,
176 struct otx2_tim_ring * const tim_ring)
178 struct otx2_tim_ent *chunk;
180 if (unlikely(rte_mempool_get(tim_ring->chunk_pool, (void **)&chunk)))
183 *(uint64_t *)(chunk + tim_ring->nb_chunk_slots) = 0;
185 *(uint64_t *)(((struct otx2_tim_ent *)(uintptr_t)
186 bkt->current_chunk) +
187 tim_ring->nb_chunk_slots) = (uintptr_t)chunk;
189 bkt->first_chunk = (uintptr_t)chunk;
195 static __rte_always_inline int
196 tim_add_entry_sp(struct otx2_tim_ring * const tim_ring,
197 const uint32_t rel_bkt,
198 struct rte_event_timer * const tim,
199 const struct otx2_tim_ent * const pent,
202 struct otx2_tim_ent *chunk;
203 struct otx2_tim_bkt *bkt;
207 bkt = tim_get_target_bucket(tim_ring, rel_bkt, flags);
211 lock_sema = tim_bkt_fetch_sema(bkt);
213 /* Bucket related checks. */
214 if (unlikely(tim_bkt_get_hbt(lock_sema)))
217 /* Insert the work. */
218 rem = tim_bkt_fetch_rem(lock_sema);
221 if (flags & OTX2_TIM_ENA_FB)
222 chunk = tim_refill_chunk(bkt, tim_ring);
223 if (flags & OTX2_TIM_ENA_DFB)
224 chunk = tim_insert_chunk(bkt, tim_ring);
226 if (unlikely(chunk == NULL)) {
227 tim_bkt_set_rem(bkt, 0);
228 tim->impl_opaque[0] = 0;
229 tim->impl_opaque[1] = 0;
230 tim->state = RTE_EVENT_TIMER_ERROR;
233 bkt->current_chunk = (uintptr_t)chunk;
234 tim_bkt_set_rem(bkt, tim_ring->nb_chunk_slots - 1);
236 chunk = (struct otx2_tim_ent *)(uintptr_t)bkt->current_chunk;
237 chunk += tim_ring->nb_chunk_slots - rem;
240 /* Copy work entry. */
243 tim_bkt_inc_nent(bkt);
245 tim->impl_opaque[0] = (uintptr_t)chunk;
246 tim->impl_opaque[1] = (uintptr_t)bkt;
247 tim->state = RTE_EVENT_TIMER_ARMED;
252 static __rte_always_inline int
253 tim_add_entry_mp(struct otx2_tim_ring * const tim_ring,
254 const uint32_t rel_bkt,
255 struct rte_event_timer * const tim,
256 const struct otx2_tim_ent * const pent,
259 struct otx2_tim_ent *chunk;
260 struct otx2_tim_bkt *bkt;
265 bkt = tim_get_target_bucket(tim_ring, rel_bkt, flags);
268 lock_sema = tim_bkt_fetch_sema_lock(bkt);
270 /* Bucket related checks. */
271 if (unlikely(tim_bkt_get_hbt(lock_sema))) {
272 tim_bkt_dec_lock(bkt);
276 rem = tim_bkt_fetch_rem(lock_sema);
279 /* Goto diff bucket. */
280 tim_bkt_dec_lock(bkt);
283 /* Only one thread can be here*/
284 if (flags & OTX2_TIM_ENA_FB)
285 chunk = tim_refill_chunk(bkt, tim_ring);
286 if (flags & OTX2_TIM_ENA_DFB)
287 chunk = tim_insert_chunk(bkt, tim_ring);
289 if (unlikely(chunk == NULL)) {
290 tim_bkt_set_rem(bkt, 0);
291 tim_bkt_dec_lock(bkt);
292 tim->impl_opaque[0] = 0;
293 tim->impl_opaque[1] = 0;
294 tim->state = RTE_EVENT_TIMER_ERROR;
297 bkt->current_chunk = (uintptr_t)chunk;
298 tim_bkt_set_rem(bkt, tim_ring->nb_chunk_slots - 1);
300 chunk = (struct otx2_tim_ent *)(uintptr_t)bkt->current_chunk;
301 chunk += tim_ring->nb_chunk_slots - rem;
304 /* Copy work entry. */
306 tim_bkt_dec_lock(bkt);
307 tim_bkt_inc_nent(bkt);
308 tim->impl_opaque[0] = (uintptr_t)chunk;
309 tim->impl_opaque[1] = (uintptr_t)bkt;
310 tim->state = RTE_EVENT_TIMER_ARMED;
315 static inline uint16_t
316 tim_cpy_wrk(uint16_t index, uint16_t cpy_lmt,
317 struct otx2_tim_ent *chunk,
318 struct rte_event_timer ** const tim,
319 const struct otx2_tim_ent * const ents,
320 const struct otx2_tim_bkt * const bkt)
322 for (; index < cpy_lmt; index++) {
323 *chunk = *(ents + index);
324 tim[index]->impl_opaque[0] = (uintptr_t)chunk++;
325 tim[index]->impl_opaque[1] = (uintptr_t)bkt;
326 tim[index]->state = RTE_EVENT_TIMER_ARMED;
332 /* Burst mode functions */
334 tim_add_entry_brst(struct otx2_tim_ring * const tim_ring,
335 const uint16_t rel_bkt,
336 struct rte_event_timer ** const tim,
337 const struct otx2_tim_ent *ents,
338 const uint16_t nb_timers, const uint8_t flags)
340 struct otx2_tim_ent *chunk;
341 struct otx2_tim_bkt *bkt;
342 uint16_t chunk_remainder;
349 bkt = tim_get_target_bucket(tim_ring, rel_bkt, flags);
351 /* Only one thread beyond this. */
352 lock_sema = tim_bkt_inc_lock(bkt);
354 ((lock_sema >> TIM_BUCKET_W1_S_LOCK) & TIM_BUCKET_W1_M_LOCK);
357 tim_bkt_dec_lock(bkt);
361 /* Bucket related checks. */
362 if (unlikely(tim_bkt_get_hbt(lock_sema))) {
363 tim_bkt_dec_lock(bkt);
367 chunk_remainder = tim_bkt_fetch_rem(lock_sema);
368 rem = chunk_remainder - nb_timers;
370 crem = tim_ring->nb_chunk_slots - chunk_remainder;
371 if (chunk_remainder && crem) {
372 chunk = ((struct otx2_tim_ent *)
373 (uintptr_t)bkt->current_chunk) + crem;
375 index = tim_cpy_wrk(index, chunk_remainder, chunk, tim,
377 tim_bkt_sub_rem(bkt, chunk_remainder);
378 tim_bkt_add_nent(bkt, chunk_remainder);
381 if (flags & OTX2_TIM_ENA_FB)
382 chunk = tim_refill_chunk(bkt, tim_ring);
383 if (flags & OTX2_TIM_ENA_DFB)
384 chunk = tim_insert_chunk(bkt, tim_ring);
386 if (unlikely(chunk == NULL)) {
387 tim_bkt_dec_lock(bkt);
389 tim[index]->state = RTE_EVENT_TIMER_ERROR;
392 *(uint64_t *)(chunk + tim_ring->nb_chunk_slots) = 0;
393 bkt->current_chunk = (uintptr_t)chunk;
394 tim_cpy_wrk(index, nb_timers, chunk, tim, ents, bkt);
396 rem = nb_timers - chunk_remainder;
397 tim_bkt_set_rem(bkt, tim_ring->nb_chunk_slots - rem);
398 tim_bkt_add_nent(bkt, rem);
400 chunk = (struct otx2_tim_ent *)(uintptr_t)bkt->current_chunk;
401 chunk += (tim_ring->nb_chunk_slots - chunk_remainder);
403 tim_cpy_wrk(index, nb_timers, chunk, tim, ents, bkt);
404 tim_bkt_sub_rem(bkt, nb_timers);
405 tim_bkt_add_nent(bkt, nb_timers);
408 tim_bkt_dec_lock(bkt);
414 tim_rm_entry(struct rte_event_timer *tim)
416 struct otx2_tim_ent *entry;
417 struct otx2_tim_bkt *bkt;
420 if (tim->impl_opaque[1] == 0 || tim->impl_opaque[0] == 0)
423 entry = (struct otx2_tim_ent *)(uintptr_t)tim->impl_opaque[0];
424 if (entry->wqe != tim->ev.u64) {
425 tim->impl_opaque[0] = 0;
426 tim->impl_opaque[1] = 0;
430 bkt = (struct otx2_tim_bkt *)(uintptr_t)tim->impl_opaque[1];
431 lock_sema = tim_bkt_inc_lock(bkt);
432 if (tim_bkt_get_hbt(lock_sema) || !tim_bkt_get_nent(lock_sema)) {
433 tim_bkt_dec_lock(bkt);
434 tim->impl_opaque[0] = 0;
435 tim->impl_opaque[1] = 0;
441 tim_bkt_dec_lock(bkt);
443 tim->state = RTE_EVENT_TIMER_CANCELED;
444 tim->impl_opaque[0] = 0;
445 tim->impl_opaque[1] = 0;
450 #endif /* __OTX2_TIM_WORKER_H__ */