drivers/event/octeontx/timvf_worker.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Cavium, Inc
   3  */
   4
   5 #include <rte_common.h>
   6 #include <rte_branch_prediction.h>
   7
   8 #include "timvf_evdev.h"
   9
  10 static inline int16_t
  11 timr_bkt_fetch_rem(uint64_t w1)
  12 {
  13         return (w1 >> TIM_BUCKET_W1_S_CHUNK_REMAINDER) &
  14                 TIM_BUCKET_W1_M_CHUNK_REMAINDER;
  15 }
  16
  17 static inline int16_t
  18 timr_bkt_get_rem(struct tim_mem_bucket *bktp)
  19 {
  20         return __atomic_load_n(&bktp->chunk_remainder,
  21                         __ATOMIC_ACQUIRE);
  22 }
  23
  24 static inline void
  25 timr_bkt_set_rem(struct tim_mem_bucket *bktp, uint16_t v)
  26 {
  27         __atomic_store_n(&bktp->chunk_remainder, v,
  28                         __ATOMIC_RELEASE);
  29 }
  30
  31 static inline void
  32 timr_bkt_sub_rem(struct tim_mem_bucket *bktp, uint16_t v)
  33 {
  34         __atomic_fetch_sub(&bktp->chunk_remainder, v,
  35                         __ATOMIC_RELEASE);
  36 }
  37
  38 static inline uint8_t
  39 timr_bkt_get_sbt(uint64_t w1)
  40 {
  41         return (w1 >> TIM_BUCKET_W1_S_SBT) & TIM_BUCKET_W1_M_SBT;
  42 }
  43
  44 static inline uint64_t
  45 timr_bkt_set_sbt(struct tim_mem_bucket *bktp)
  46 {
  47         const uint64_t v = TIM_BUCKET_W1_M_SBT << TIM_BUCKET_W1_S_SBT;
  48         return __atomic_fetch_or(&bktp->w1, v, __ATOMIC_ACQ_REL);
  49 }
  50
  51 static inline uint64_t
  52 timr_bkt_clr_sbt(struct tim_mem_bucket *bktp)
  53 {
  54         const uint64_t v = ~(TIM_BUCKET_W1_M_SBT << TIM_BUCKET_W1_S_SBT);
  55         return __atomic_fetch_and(&bktp->w1, v, __ATOMIC_ACQ_REL);
  56 }
  57
  58 static inline uint8_t
  59 timr_bkt_get_shbt(uint64_t w1)
  60 {
  61         return ((w1 >> TIM_BUCKET_W1_S_HBT) & TIM_BUCKET_W1_M_HBT) |
  62                 ((w1 >> TIM_BUCKET_W1_S_SBT) & TIM_BUCKET_W1_M_SBT);
  63 }
  64
  65 static inline uint8_t
  66 timr_bkt_get_hbt(uint64_t w1)
  67 {
  68         return (w1 >> TIM_BUCKET_W1_S_HBT) & TIM_BUCKET_W1_M_HBT;
  69 }
  70
  71 static inline uint8_t
  72 timr_bkt_get_bsk(uint64_t w1)
  73 {
  74         return (w1 >> TIM_BUCKET_W1_S_BSK) & TIM_BUCKET_W1_M_BSK;
  75 }
  76
  77 static inline uint64_t
  78 timr_bkt_clr_bsk(struct tim_mem_bucket *bktp)
  79 {
  80         /*Clear everything except lock. */
  81         const uint64_t v = TIM_BUCKET_W1_M_LOCK << TIM_BUCKET_W1_S_LOCK;
  82         return __atomic_fetch_and(&bktp->w1, v, __ATOMIC_ACQ_REL);
  83 }
  84
  85 static inline uint64_t
  86 timr_bkt_fetch_sema_lock(struct tim_mem_bucket *bktp)
  87 {
  88         return __atomic_fetch_add(&bktp->w1, TIM_BUCKET_SEMA_WLOCK,
  89                         __ATOMIC_ACQ_REL);
  90 }
  91
  92 static inline uint64_t
  93 timr_bkt_fetch_sema(struct tim_mem_bucket *bktp)
  94 {
  95         return __atomic_fetch_add(&bktp->w1, TIM_BUCKET_SEMA,
  96                         __ATOMIC_RELAXED);
  97 }
  98
  99 static inline uint64_t
 100 timr_bkt_inc_lock(struct tim_mem_bucket *bktp)
 101 {
 102         const uint64_t v = 1ull << TIM_BUCKET_W1_S_LOCK;
 103         return __atomic_fetch_add(&bktp->w1, v, __ATOMIC_ACQ_REL);
 104 }
 105
 106 static inline void
 107 timr_bkt_dec_lock(struct tim_mem_bucket *bktp)
 108 {
 109         __atomic_add_fetch(&bktp->lock, 0xff, __ATOMIC_ACQ_REL);
 110 }
 111
 112 static inline uint32_t
 113 timr_bkt_get_nent(uint64_t w1)
 114 {
 115         return (w1 >> TIM_BUCKET_W1_S_NUM_ENTRIES) &
 116                 TIM_BUCKET_W1_M_NUM_ENTRIES;
 117 }
 118
 119 static inline void
 120 timr_bkt_inc_nent(struct tim_mem_bucket *bktp)
 121 {
 122         __atomic_add_fetch(&bktp->nb_entry, 1, __ATOMIC_RELAXED);
 123 }
 124
 125 static inline void
 126 timr_bkt_add_nent(struct tim_mem_bucket *bktp, uint32_t v)
 127 {
 128         __atomic_add_fetch(&bktp->nb_entry, v, __ATOMIC_RELAXED);
 129 }
 130
 131 static inline uint64_t
 132 timr_bkt_clr_nent(struct tim_mem_bucket *bktp)
 133 {
 134         const uint64_t v = ~(TIM_BUCKET_W1_M_NUM_ENTRIES <<
 135                         TIM_BUCKET_W1_S_NUM_ENTRIES);
 136         return __atomic_and_fetch(&bktp->w1, v, __ATOMIC_ACQ_REL);
 137 }
 138
 139 static inline struct tim_mem_entry *
 140 timr_clr_bkt(struct timvf_ring * const timr, struct tim_mem_bucket * const bkt)
 141 {
 142         struct tim_mem_entry *chunk;
 143         struct tim_mem_entry *pnext;
 144         chunk = ((struct tim_mem_entry *)(uintptr_t)bkt->first_chunk);
 145         chunk = (struct tim_mem_entry *)(uintptr_t)(chunk + nb_chunk_slots)->w0;
 146
 147         while (chunk) {
 148                 pnext = (struct tim_mem_entry *)(uintptr_t)
 149                         ((chunk + nb_chunk_slots)->w0);
 150                 rte_mempool_put(timr->chunk_pool, chunk);
 151                 chunk = pnext;
 152         }
 153         return (struct tim_mem_entry *)(uintptr_t)bkt->first_chunk;
 154 }
 155
 156 static inline int
 157 timvf_rem_entry(struct rte_event_timer *tim)
 158 {
 159         uint64_t lock_sema;
 160         struct tim_mem_entry *entry;
 161         struct tim_mem_bucket *bkt;
 162         if (tim->impl_opaque[1] == 0 ||
 163                         tim->impl_opaque[0] == 0)
 164                 return -ENOENT;
 165
 166         entry = (struct tim_mem_entry *)(uintptr_t)tim->impl_opaque[0];
 167         if (entry->wqe != tim->ev.u64) {
 168                 tim->impl_opaque[1] = tim->impl_opaque[0] = 0;
 169                 return -ENOENT;
 170         }
 171         bkt = (struct tim_mem_bucket *)(uintptr_t)tim->impl_opaque[1];
 172         lock_sema = timr_bkt_inc_lock(bkt);
 173         if (timr_bkt_get_shbt(lock_sema)
 174                         || !timr_bkt_get_nent(lock_sema)) {
 175                 timr_bkt_dec_lock(bkt);
 176                 tim->impl_opaque[1] = tim->impl_opaque[0] = 0;
 177                 return -ENOENT;
 178         }
 179
 180         entry->w0 = entry->wqe = 0;
 181         timr_bkt_dec_lock(bkt);
 182
 183         tim->state = RTE_EVENT_TIMER_CANCELED;
 184         tim->impl_opaque[1] = tim->impl_opaque[0] = 0;
 185         return 0;
 186 }
 187
 188 static inline struct tim_mem_entry *
 189 timvf_refill_chunk_generic(struct tim_mem_bucket * const bkt,
 190                 struct timvf_ring * const timr)
 191 {
 192         struct tim_mem_entry *chunk;
 193
 194         if (bkt->nb_entry || !bkt->first_chunk) {
 195                 if (unlikely(rte_mempool_get(timr->chunk_pool,
 196                                                 (void **)&chunk))) {
 197                         return NULL;
 198                 }
 199                 if (bkt->nb_entry) {
 200                         *(uint64_t *)(((struct tim_mem_entry *)(uintptr_t)
 201                                         bkt->current_chunk) +
 202                                         nb_chunk_slots) =
 203                                 (uintptr_t) chunk;
 204                 } else {
 205                         bkt->first_chunk = (uintptr_t) chunk;
 206                 }
 207         } else {
 208                 chunk = timr_clr_bkt(timr, bkt);
 209                 bkt->first_chunk = (uintptr_t)chunk;
 210         }
 211         *(uint64_t *)(chunk + nb_chunk_slots) = 0;
 212
 213         return chunk;
 214 }
 215
 216 static inline struct tim_mem_entry *
 217 timvf_refill_chunk_fpa(struct tim_mem_bucket * const bkt,
 218                 struct timvf_ring * const timr)
 219 {
 220         struct tim_mem_entry *chunk;
 221
 222         if (unlikely(rte_mempool_get(timr->chunk_pool, (void **)&chunk)))
 223                 return NULL;
 224
 225         *(uint64_t *)(chunk + nb_chunk_slots) = 0;
 226         if (bkt->nb_entry) {
 227                 *(uint64_t *)(((struct tim_mem_entry *)(uintptr_t)
 228                                 bkt->current_chunk) +
 229                                 nb_chunk_slots) =
 230                         (uintptr_t) chunk;
 231         } else {
 232                 bkt->first_chunk = (uintptr_t) chunk;
 233         }
 234
 235         return chunk;
 236 }
 237
 238 static inline struct tim_mem_bucket *
 239 timvf_get_target_bucket(struct timvf_ring * const timr, const uint32_t rel_bkt)
 240 {
 241         const uint64_t bkt_cyc = rte_rdtsc() - timr->ring_start_cyc;
 242         const uint32_t bucket = rte_reciprocal_divide_u64(bkt_cyc,
 243                         &timr->fast_div) + rel_bkt;
 244         const uint32_t tbkt_id = timr->get_target_bkt(bucket,
 245                         timr->nb_bkts);
 246         return &timr->bkt[tbkt_id];
 247 }
 248
 249 /* Single producer functions. */
 250 static inline int
 251 timvf_add_entry_sp(struct timvf_ring * const timr, const uint32_t rel_bkt,
 252                 struct rte_event_timer * const tim,
 253                 const struct tim_mem_entry * const pent)
 254 {
 255         int16_t rem;
 256         uint64_t lock_sema;
 257         struct tim_mem_bucket *bkt;
 258         struct tim_mem_entry *chunk;
 259
 260
 261         bkt = timvf_get_target_bucket(timr, rel_bkt);
 262 __retry:
 263         /*Get Bucket sema*/
 264         lock_sema = timr_bkt_fetch_sema(bkt);
 265         /* Bucket related checks. */
 266         if (unlikely(timr_bkt_get_hbt(lock_sema)))
 267                 goto __retry;
 268
 269         /* Insert the work. */
 270         rem = timr_bkt_fetch_rem(lock_sema);
 271
 272         if (!rem) {
 273                 chunk = timr->refill_chunk(bkt, timr);
 274                 if (unlikely(chunk == NULL)) {
 275                         timr_bkt_set_rem(bkt, 0);
 276                         tim->impl_opaque[0] = tim->impl_opaque[1] = 0;
 277                         tim->state = RTE_EVENT_TIMER_ERROR;
 278                         return -ENOMEM;
 279                 }
 280                 bkt->current_chunk = (uintptr_t) chunk;
 281                 timr_bkt_set_rem(bkt, nb_chunk_slots - 1);
 282         } else {
 283                 chunk = (struct tim_mem_entry *)(uintptr_t)bkt->current_chunk;
 284                 chunk += nb_chunk_slots - rem;
 285         }
 286         /* Copy work entry. */
 287         *chunk = *pent;
 288         timr_bkt_inc_nent(bkt);
 289
 290         tim->impl_opaque[0] = (uintptr_t)chunk;
 291         tim->impl_opaque[1] = (uintptr_t)bkt;
 292         tim->state = RTE_EVENT_TIMER_ARMED;
 293         return 0;
 294 }
 295
 296 /* Multi producer functions. */
 297 static inline int
 298 timvf_add_entry_mp(struct timvf_ring * const timr, const uint32_t rel_bkt,
 299                 struct rte_event_timer * const tim,
 300                 const struct tim_mem_entry * const pent)
 301 {
 302         int16_t rem;
 303         uint64_t lock_sema;
 304         struct tim_mem_bucket *bkt;
 305         struct tim_mem_entry *chunk;
 306
 307 __retry:
 308         bkt = timvf_get_target_bucket(timr, rel_bkt);
 309         /* Bucket related checks. */
 310         /*Get Bucket sema*/
 311         lock_sema = timr_bkt_fetch_sema_lock(bkt);
 312         if (unlikely(timr_bkt_get_shbt(lock_sema))) {
 313                 timr_bkt_dec_lock(bkt);
 314                 goto __retry;
 315         }
 316
 317         rem = timr_bkt_fetch_rem(lock_sema);
 318
 319         if (rem < 0) {
 320                 /* goto diff bucket. */
 321                 timr_bkt_dec_lock(bkt);
 322                 goto __retry;
 323         } else if (!rem) {
 324                 /*Only one thread can be here*/
 325                 chunk = timr->refill_chunk(bkt, timr);
 326                 if (unlikely(chunk == NULL)) {
 327                         timr_bkt_set_rem(bkt, 0);
 328                         timr_bkt_dec_lock(bkt);
 329                         tim->impl_opaque[0] = tim->impl_opaque[1] = 0;
 330                         tim->state = RTE_EVENT_TIMER_ERROR;
 331                         return -ENOMEM;
 332                 }
 333                 bkt->current_chunk = (uintptr_t) chunk;
 334                 timr_bkt_set_rem(bkt, nb_chunk_slots - 1);
 335         } else {
 336                 chunk = (struct tim_mem_entry *)(uintptr_t)bkt->current_chunk;
 337                 chunk += nb_chunk_slots - rem;
 338         }
 339         /* Copy work entry. */
 340         *chunk = *pent;
 341         timr_bkt_inc_nent(bkt);
 342         timr_bkt_dec_lock(bkt);
 343
 344         tim->impl_opaque[0] = (uintptr_t)chunk;
 345         tim->impl_opaque[1] = (uintptr_t)bkt;
 346         tim->state = RTE_EVENT_TIMER_ARMED;
 347         return 0;
 348 }
 349
 350 static inline uint16_t
 351 timvf_cpy_wrk(uint16_t index, uint16_t cpy_lmt,
 352                 struct tim_mem_entry *chunk,
 353                 struct rte_event_timer ** const tim,
 354                 const struct tim_mem_entry * const ents,
 355                 const struct tim_mem_bucket * const bkt)
 356 {
 357         for (; index < cpy_lmt; index++) {
 358                 *chunk = *(ents + index);
 359                 tim[index]->impl_opaque[0] = (uintptr_t)chunk++;
 360                 tim[index]->impl_opaque[1] = (uintptr_t)bkt;
 361                 tim[index]->state = RTE_EVENT_TIMER_ARMED;
 362         }
 363
 364         return index;
 365 }
 366
 367 /* Burst mode functions */
 368 static inline int
 369 timvf_add_entry_brst(struct timvf_ring * const timr, const uint16_t rel_bkt,
 370                 struct rte_event_timer ** const tim,
 371                 const struct tim_mem_entry *ents,
 372                 const uint16_t nb_timers)
 373 {
 374         int16_t rem;
 375         int16_t crem;
 376         uint8_t lock_cnt;
 377         uint16_t index = 0;
 378         uint16_t chunk_remainder;
 379         uint64_t lock_sema;
 380         struct tim_mem_bucket *bkt;
 381         struct tim_mem_entry *chunk;
 382
 383 __retry:
 384         bkt = timvf_get_target_bucket(timr, rel_bkt);
 385
 386         /* Only one thread beyond this. */
 387         lock_sema = timr_bkt_inc_lock(bkt);
 388         lock_cnt = (uint8_t)
 389                 ((lock_sema >> TIM_BUCKET_W1_S_LOCK) & TIM_BUCKET_W1_M_LOCK);
 390
 391         if (lock_cnt) {
 392                 timr_bkt_dec_lock(bkt);
 393                 goto __retry;
 394         }
 395
 396         /* Bucket related checks. */
 397         if (unlikely(timr_bkt_get_hbt(lock_sema))) {
 398                 timr_bkt_dec_lock(bkt);
 399                 goto __retry;
 400         }
 401
 402         chunk_remainder = timr_bkt_fetch_rem(lock_sema);
 403         rem = chunk_remainder - nb_timers;
 404         if (rem < 0) {
 405                 crem = nb_chunk_slots - chunk_remainder;
 406                 if (chunk_remainder && crem) {
 407                         chunk = ((struct tim_mem_entry *)
 408                                         (uintptr_t)bkt->current_chunk) + crem;
 409
 410                         index = timvf_cpy_wrk(index, chunk_remainder,
 411                                         chunk, tim, ents, bkt);
 412                         timr_bkt_sub_rem(bkt, chunk_remainder);
 413                         timr_bkt_add_nent(bkt, chunk_remainder);
 414                 }
 415                 rem = nb_timers - chunk_remainder;
 416                 ents = ents + chunk_remainder;
 417
 418                 chunk = timr->refill_chunk(bkt, timr);
 419                 if (unlikely(chunk == NULL)) {
 420                         timr_bkt_dec_lock(bkt);
 421                         rte_errno = ENOMEM;
 422                         tim[index]->state = RTE_EVENT_TIMER_ERROR;
 423                         return crem;
 424                 }
 425                 *(uint64_t *)(chunk + nb_chunk_slots) = 0;
 426                 bkt->current_chunk = (uintptr_t) chunk;
 427
 428                 index = timvf_cpy_wrk(index, nb_timers, chunk, tim, ents, bkt);
 429                 timr_bkt_set_rem(bkt, nb_chunk_slots - rem);
 430                 timr_bkt_add_nent(bkt, rem);
 431         } else {
 432                 chunk = (struct tim_mem_entry *)(uintptr_t)bkt->current_chunk;
 433                 chunk += (nb_chunk_slots - chunk_remainder);
 434
 435                 index = timvf_cpy_wrk(index, nb_timers,
 436                                 chunk, tim, ents, bkt);
 437                 timr_bkt_sub_rem(bkt, nb_timers);
 438                 timr_bkt_add_nent(bkt, nb_timers);
 439         }
 440
 441         timr_bkt_dec_lock(bkt);
 442         return nb_timers;
 443 }