drivers/event/octeontx2/otx2_tim_worker.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #ifndef __OTX2_TIM_WORKER_H__
   6 #define __OTX2_TIM_WORKER_H__
   7
   8 #include "otx2_tim_evdev.h"
   9
  10 static inline int16_t
  11 tim_bkt_fetch_rem(uint64_t w1)
  12 {
  13         return (w1 >> TIM_BUCKET_W1_S_CHUNK_REMAINDER) &
  14                 TIM_BUCKET_W1_M_CHUNK_REMAINDER;
  15 }
  16
  17 static inline int16_t
  18 tim_bkt_get_rem(struct otx2_tim_bkt *bktp)
  19 {
  20         return __atomic_load_n(&bktp->chunk_remainder, __ATOMIC_ACQUIRE);
  21 }
  22
  23 static inline void
  24 tim_bkt_set_rem(struct otx2_tim_bkt *bktp, uint16_t v)
  25 {
  26         __atomic_store_n(&bktp->chunk_remainder, v, __ATOMIC_RELAXED);
  27 }
  28
  29 static inline void
  30 tim_bkt_sub_rem(struct otx2_tim_bkt *bktp, uint16_t v)
  31 {
  32         __atomic_fetch_sub(&bktp->chunk_remainder, v, __ATOMIC_RELAXED);
  33 }
  34
  35 static inline uint8_t
  36 tim_bkt_get_hbt(uint64_t w1)
  37 {
  38         return (w1 >> TIM_BUCKET_W1_S_HBT) & TIM_BUCKET_W1_M_HBT;
  39 }
  40
  41 static inline uint8_t
  42 tim_bkt_get_bsk(uint64_t w1)
  43 {
  44         return (w1 >> TIM_BUCKET_W1_S_BSK) & TIM_BUCKET_W1_M_BSK;
  45 }
  46
  47 static inline uint64_t
  48 tim_bkt_clr_bsk(struct otx2_tim_bkt *bktp)
  49 {
  50         /* Clear everything except lock. */
  51         const uint64_t v = TIM_BUCKET_W1_M_LOCK << TIM_BUCKET_W1_S_LOCK;
  52
  53         return __atomic_fetch_and(&bktp->w1, v, __ATOMIC_ACQ_REL);
  54 }
  55
  56 static inline uint64_t
  57 tim_bkt_fetch_sema_lock(struct otx2_tim_bkt *bktp)
  58 {
  59         return __atomic_fetch_add(&bktp->w1, TIM_BUCKET_SEMA_WLOCK,
  60                         __ATOMIC_ACQUIRE);
  61 }
  62
  63 static inline uint64_t
  64 tim_bkt_fetch_sema(struct otx2_tim_bkt *bktp)
  65 {
  66         return __atomic_fetch_add(&bktp->w1, TIM_BUCKET_SEMA, __ATOMIC_RELAXED);
  67 }
  68
  69 static inline uint64_t
  70 tim_bkt_inc_lock(struct otx2_tim_bkt *bktp)
  71 {
  72         const uint64_t v = 1ull << TIM_BUCKET_W1_S_LOCK;
  73
  74         return __atomic_fetch_add(&bktp->w1, v, __ATOMIC_ACQUIRE);
  75 }
  76
  77 static inline void
  78 tim_bkt_dec_lock(struct otx2_tim_bkt *bktp)
  79 {
  80         __atomic_add_fetch(&bktp->lock, 0xff, __ATOMIC_RELEASE);
  81 }
  82
  83 static inline uint32_t
  84 tim_bkt_get_nent(uint64_t w1)
  85 {
  86         return (w1 >> TIM_BUCKET_W1_S_NUM_ENTRIES) &
  87                 TIM_BUCKET_W1_M_NUM_ENTRIES;
  88 }
  89
  90 static inline void
  91 tim_bkt_inc_nent(struct otx2_tim_bkt *bktp)
  92 {
  93         __atomic_add_fetch(&bktp->nb_entry, 1, __ATOMIC_RELAXED);
  94 }
  95
  96 static inline void
  97 tim_bkt_add_nent(struct otx2_tim_bkt *bktp, uint32_t v)
  98 {
  99         __atomic_add_fetch(&bktp->nb_entry, v, __ATOMIC_RELAXED);
 100 }
 101
 102 static inline uint64_t
 103 tim_bkt_clr_nent(struct otx2_tim_bkt *bktp)
 104 {
 105         const uint64_t v = ~(TIM_BUCKET_W1_M_NUM_ENTRIES <<
 106                         TIM_BUCKET_W1_S_NUM_ENTRIES);
 107
 108         return __atomic_and_fetch(&bktp->w1, v, __ATOMIC_ACQ_REL);
 109 }
 110
 111 static __rte_always_inline struct otx2_tim_bkt *
 112 tim_get_target_bucket(struct otx2_tim_ring * const tim_ring,
 113                       const uint32_t rel_bkt, const uint8_t flag)
 114 {
 115         const uint64_t bkt_cyc = rte_rdtsc() - tim_ring->ring_start_cyc;
 116         uint32_t bucket = rte_reciprocal_divide_u64(bkt_cyc,
 117                         &tim_ring->fast_div) + rel_bkt;
 118
 119         if (flag & OTX2_TIM_BKT_MOD)
 120                 bucket = bucket % tim_ring->nb_bkts;
 121         if (flag & OTX2_TIM_BKT_AND)
 122                 bucket = bucket & (tim_ring->nb_bkts - 1);
 123
 124         return &tim_ring->bkt[bucket];
 125 }
 126
 127 static struct otx2_tim_ent *
 128 tim_clr_bkt(struct otx2_tim_ring * const tim_ring,
 129             struct otx2_tim_bkt * const bkt)
 130 {
 131         struct otx2_tim_ent *chunk;
 132         struct otx2_tim_ent *pnext;
 133
 134         chunk = ((struct otx2_tim_ent *)(uintptr_t)bkt->first_chunk);
 135         chunk = (struct otx2_tim_ent *)(uintptr_t)(chunk +
 136                         tim_ring->nb_chunk_slots)->w0;
 137         while (chunk) {
 138                 pnext = (struct otx2_tim_ent *)(uintptr_t)
 139                         ((chunk + tim_ring->nb_chunk_slots)->w0);
 140                 rte_mempool_put(tim_ring->chunk_pool, chunk);
 141                 chunk = pnext;
 142         }
 143
 144         return (struct otx2_tim_ent *)(uintptr_t)bkt->first_chunk;
 145 }
 146
 147 static struct otx2_tim_ent *
 148 tim_refill_chunk(struct otx2_tim_bkt * const bkt,
 149                  struct otx2_tim_ring * const tim_ring)
 150 {
 151         struct otx2_tim_ent *chunk;
 152
 153         if (bkt->nb_entry || !bkt->first_chunk) {
 154                 if (unlikely(rte_mempool_get(tim_ring->chunk_pool,
 155                                              (void **)&chunk)))
 156                         return NULL;
 157                 if (bkt->nb_entry) {
 158                         *(uint64_t *)(((struct otx2_tim_ent *)(uintptr_t)
 159                                                 bkt->current_chunk) +
 160                                         tim_ring->nb_chunk_slots) =
 161                                 (uintptr_t)chunk;
 162                 } else {
 163                         bkt->first_chunk = (uintptr_t)chunk;
 164                 }
 165         } else {
 166                 chunk = tim_clr_bkt(tim_ring, bkt);
 167                 bkt->first_chunk = (uintptr_t)chunk;
 168         }
 169         *(uint64_t *)(chunk + tim_ring->nb_chunk_slots) = 0;
 170
 171         return chunk;
 172 }
 173
 174 static struct otx2_tim_ent *
 175 tim_insert_chunk(struct otx2_tim_bkt * const bkt,
 176                  struct otx2_tim_ring * const tim_ring)
 177 {
 178         struct otx2_tim_ent *chunk;
 179
 180         if (unlikely(rte_mempool_get(tim_ring->chunk_pool, (void **)&chunk)))
 181                 return NULL;
 182
 183         *(uint64_t *)(chunk + tim_ring->nb_chunk_slots) = 0;
 184         if (bkt->nb_entry) {
 185                 *(uint64_t *)(((struct otx2_tim_ent *)(uintptr_t)
 186                                         bkt->current_chunk) +
 187                                 tim_ring->nb_chunk_slots) = (uintptr_t)chunk;
 188         } else {
 189                 bkt->first_chunk = (uintptr_t)chunk;
 190         }
 191
 192         return chunk;
 193 }
 194
 195 static __rte_always_inline int
 196 tim_add_entry_sp(struct otx2_tim_ring * const tim_ring,
 197                  const uint32_t rel_bkt,
 198                  struct rte_event_timer * const tim,
 199                  const struct otx2_tim_ent * const pent,
 200                  const uint8_t flags)
 201 {
 202         struct otx2_tim_ent *chunk;
 203         struct otx2_tim_bkt *bkt;
 204         uint64_t lock_sema;
 205         int16_t rem;
 206
 207         bkt = tim_get_target_bucket(tim_ring, rel_bkt, flags);
 208
 209 __retry:
 210         /* Get Bucket sema*/
 211         lock_sema = tim_bkt_fetch_sema(bkt);
 212
 213         /* Bucket related checks. */
 214         if (unlikely(tim_bkt_get_hbt(lock_sema)))
 215                 goto __retry;
 216
 217         /* Insert the work. */
 218         rem = tim_bkt_fetch_rem(lock_sema);
 219
 220         if (!rem) {
 221                 if (flags & OTX2_TIM_ENA_FB)
 222                         chunk = tim_refill_chunk(bkt, tim_ring);
 223                 if (flags & OTX2_TIM_ENA_DFB)
 224                         chunk = tim_insert_chunk(bkt, tim_ring);
 225
 226                 if (unlikely(chunk == NULL)) {
 227                         tim_bkt_set_rem(bkt, 0);
 228                         tim->impl_opaque[0] = 0;
 229                         tim->impl_opaque[1] = 0;
 230                         tim->state = RTE_EVENT_TIMER_ERROR;
 231                         return -ENOMEM;
 232                 }
 233                 bkt->current_chunk = (uintptr_t)chunk;
 234                 tim_bkt_set_rem(bkt, tim_ring->nb_chunk_slots - 1);
 235         } else {
 236                 chunk = (struct otx2_tim_ent *)(uintptr_t)bkt->current_chunk;
 237                 chunk += tim_ring->nb_chunk_slots - rem;
 238         }
 239
 240         /* Copy work entry. */
 241         *chunk = *pent;
 242
 243         tim_bkt_inc_nent(bkt);
 244
 245         tim->impl_opaque[0] = (uintptr_t)chunk;
 246         tim->impl_opaque[1] = (uintptr_t)bkt;
 247         tim->state = RTE_EVENT_TIMER_ARMED;
 248
 249         return 0;
 250 }
 251
 252 static __rte_always_inline int
 253 tim_add_entry_mp(struct otx2_tim_ring * const tim_ring,
 254                  const uint32_t rel_bkt,
 255                  struct rte_event_timer * const tim,
 256                  const struct otx2_tim_ent * const pent,
 257                  const uint8_t flags)
 258 {
 259         struct otx2_tim_ent *chunk;
 260         struct otx2_tim_bkt *bkt;
 261         uint64_t lock_sema;
 262         int16_t rem;
 263
 264 __retry:
 265         bkt = tim_get_target_bucket(tim_ring, rel_bkt, flags);
 266
 267         /* Get Bucket sema*/
 268         lock_sema = tim_bkt_fetch_sema_lock(bkt);
 269
 270         /* Bucket related checks. */
 271         if (unlikely(tim_bkt_get_hbt(lock_sema))) {
 272                 tim_bkt_dec_lock(bkt);
 273                 goto __retry;
 274         }
 275
 276         rem = tim_bkt_fetch_rem(lock_sema);
 277
 278         if (rem < 0) {
 279                 /* Goto diff bucket. */
 280                 tim_bkt_dec_lock(bkt);
 281                 goto __retry;
 282         } else if (!rem) {
 283                 /* Only one thread can be here*/
 284                 if (flags & OTX2_TIM_ENA_FB)
 285                         chunk = tim_refill_chunk(bkt, tim_ring);
 286                 if (flags & OTX2_TIM_ENA_DFB)
 287                         chunk = tim_insert_chunk(bkt, tim_ring);
 288
 289                 if (unlikely(chunk == NULL)) {
 290                         tim_bkt_set_rem(bkt, 0);
 291                         tim_bkt_dec_lock(bkt);
 292                         tim->impl_opaque[0] = 0;
 293                         tim->impl_opaque[1] = 0;
 294                         tim->state = RTE_EVENT_TIMER_ERROR;
 295                         return -ENOMEM;
 296                 }
 297                 bkt->current_chunk = (uintptr_t)chunk;
 298                 tim_bkt_set_rem(bkt, tim_ring->nb_chunk_slots - 1);
 299         } else {
 300                 chunk = (struct otx2_tim_ent *)(uintptr_t)bkt->current_chunk;
 301                 chunk += tim_ring->nb_chunk_slots - rem;
 302         }
 303
 304         /* Copy work entry. */
 305         *chunk = *pent;
 306         tim_bkt_dec_lock(bkt);
 307         tim_bkt_inc_nent(bkt);
 308         tim->impl_opaque[0] = (uintptr_t)chunk;
 309         tim->impl_opaque[1] = (uintptr_t)bkt;
 310         tim->state = RTE_EVENT_TIMER_ARMED;
 311
 312         return 0;
 313 }
 314
 315 #endif /* __OTX2_TIM_WORKER_H__ */