lib/librte_rcu/rte_rcu_qsbr.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (c) 2018 Arm Limited
   4  */
   5
   6 #include <stdio.h>
   7 #include <string.h>
   8 #include <stdint.h>
   9 #include <inttypes.h>
  10 #include <errno.h>
  11
  12 #include <rte_common.h>
  13 #include <rte_log.h>
  14 #include <rte_memory.h>
  15 #include <rte_malloc.h>
  16 #include <rte_eal.h>
  17 #include <rte_eal_memconfig.h>
  18 #include <rte_atomic.h>
  19 #include <rte_per_lcore.h>
  20 #include <rte_lcore.h>
  21 #include <rte_errno.h>
  22
  23 #include "rte_rcu_qsbr.h"
  24
  25 /* Get the memory size of QSBR variable */
  26 size_t __rte_experimental
  27 rte_rcu_qsbr_get_memsize(uint32_t max_threads)
  28 {
  29         size_t sz;
  30
  31         if (max_threads == 0) {
  32                 rte_log(RTE_LOG_ERR, rte_rcu_log_type,
  33                         "%s(): Invalid max_threads %u\n",
  34                         __func__, max_threads);
  35                 rte_errno = EINVAL;
  36
  37                 return 1;
  38         }
  39
  40         sz = sizeof(struct rte_rcu_qsbr);
  41
  42         /* Add the size of quiescent state counter array */
  43         sz += sizeof(struct rte_rcu_qsbr_cnt) * max_threads;
  44
  45         /* Add the size of the registered thread ID bitmap array */
  46         sz += __RTE_QSBR_THRID_ARRAY_SIZE(max_threads);
  47
  48         return sz;
  49 }
  50
  51 /* Initialize a quiescent state variable */
  52 int __rte_experimental
  53 rte_rcu_qsbr_init(struct rte_rcu_qsbr *v, uint32_t max_threads)
  54 {
  55         size_t sz;
  56
  57         if (v == NULL) {
  58                 rte_log(RTE_LOG_ERR, rte_rcu_log_type,
  59                         "%s(): Invalid input parameter\n", __func__);
  60                 rte_errno = EINVAL;
  61
  62                 return 1;
  63         }
  64
  65         sz = rte_rcu_qsbr_get_memsize(max_threads);
  66         if (sz == 1)
  67                 return 1;
  68
  69         /* Set all the threads to offline */
  70         memset(v, 0, sz);
  71         v->max_threads = max_threads;
  72         v->num_elems = RTE_ALIGN_MUL_CEIL(max_threads,
  73                         __RTE_QSBR_THRID_ARRAY_ELM_SIZE) /
  74                         __RTE_QSBR_THRID_ARRAY_ELM_SIZE;
  75         v->token = __RTE_QSBR_CNT_INIT;
  76
  77         return 0;
  78 }
  79
  80 /* Register a reader thread to report its quiescent state
  81  * on a QS variable.
  82  */
  83 int __rte_experimental
  84 rte_rcu_qsbr_thread_register(struct rte_rcu_qsbr *v, unsigned int thread_id)
  85 {
  86         unsigned int i, id, success;
  87         uint64_t old_bmap, new_bmap;
  88
  89         if (v == NULL || thread_id >= v->max_threads) {
  90                 rte_log(RTE_LOG_ERR, rte_rcu_log_type,
  91                         "%s(): Invalid input parameter\n", __func__);
  92                 rte_errno = EINVAL;
  93
  94                 return 1;
  95         }
  96
  97         __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, ERR, "Lock counter %u\n",
  98                                 v->qsbr_cnt[thread_id].lock_cnt);
  99
 100         id = thread_id & __RTE_QSBR_THRID_MASK;
 101         i = thread_id >> __RTE_QSBR_THRID_INDEX_SHIFT;
 102
 103         /* Make sure that the counter for registered threads does not
 104          * go out of sync. Hence, additional checks are required.
 105          */
 106         /* Check if the thread is already registered */
 107         old_bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
 108                                         __ATOMIC_RELAXED);
 109         if (old_bmap & 1UL << id)
 110                 return 0;
 111
 112         do {
 113                 new_bmap = old_bmap | (1UL << id);
 114                 success = __atomic_compare_exchange(
 115                                         __RTE_QSBR_THRID_ARRAY_ELM(v, i),
 116                                         &old_bmap, &new_bmap, 0,
 117                                         __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 118
 119                 if (success)
 120                         __atomic_fetch_add(&v->num_threads,
 121                                                 1, __ATOMIC_RELAXED);
 122                 else if (old_bmap & (1UL << id))
 123                         /* Someone else registered this thread.
 124                          * Counter should not be incremented.
 125                          */
 126                         return 0;
 127         } while (success == 0);
 128
 129         return 0;
 130 }
 131
 132 /* Remove a reader thread, from the list of threads reporting their
 133  * quiescent state on a QS variable.
 134  */
 135 int __rte_experimental
 136 rte_rcu_qsbr_thread_unregister(struct rte_rcu_qsbr *v, unsigned int thread_id)
 137 {
 138         unsigned int i, id, success;
 139         uint64_t old_bmap, new_bmap;
 140
 141         if (v == NULL || thread_id >= v->max_threads) {
 142                 rte_log(RTE_LOG_ERR, rte_rcu_log_type,
 143                         "%s(): Invalid input parameter\n", __func__);
 144                 rte_errno = EINVAL;
 145
 146                 return 1;
 147         }
 148
 149         __RTE_RCU_IS_LOCK_CNT_ZERO(v, thread_id, ERR, "Lock counter %u\n",
 150                                 v->qsbr_cnt[thread_id].lock_cnt);
 151
 152         id = thread_id & __RTE_QSBR_THRID_MASK;
 153         i = thread_id >> __RTE_QSBR_THRID_INDEX_SHIFT;
 154
 155         /* Make sure that the counter for registered threads does not
 156          * go out of sync. Hence, additional checks are required.
 157          */
 158         /* Check if the thread is already unregistered */
 159         old_bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
 160                                         __ATOMIC_RELAXED);
 161         if (old_bmap & ~(1UL << id))
 162                 return 0;
 163
 164         do {
 165                 new_bmap = old_bmap & ~(1UL << id);
 166                 /* Make sure any loads of the shared data structure are
 167                  * completed before removal of the thread from the list of
 168                  * reporting threads.
 169                  */
 170                 success = __atomic_compare_exchange(
 171                                         __RTE_QSBR_THRID_ARRAY_ELM(v, i),
 172                                         &old_bmap, &new_bmap, 0,
 173                                         __ATOMIC_RELEASE, __ATOMIC_RELAXED);
 174
 175                 if (success)
 176                         __atomic_fetch_sub(&v->num_threads,
 177                                                 1, __ATOMIC_RELAXED);
 178                 else if (old_bmap & ~(1UL << id))
 179                         /* Someone else unregistered this thread.
 180                          * Counter should not be incremented.
 181                          */
 182                         return 0;
 183         } while (success == 0);
 184
 185         return 0;
 186 }
 187
 188 /* Wait till the reader threads have entered quiescent state. */
 189 void __rte_experimental
 190 rte_rcu_qsbr_synchronize(struct rte_rcu_qsbr *v, unsigned int thread_id)
 191 {
 192         uint64_t t;
 193
 194         RTE_ASSERT(v != NULL);
 195
 196         t = rte_rcu_qsbr_start(v);
 197
 198         /* If the current thread has readside critical section,
 199          * update its quiescent state status.
 200          */
 201         if (thread_id != RTE_QSBR_THRID_INVALID)
 202                 rte_rcu_qsbr_quiescent(v, thread_id);
 203
 204         /* Wait for other readers to enter quiescent state */
 205         rte_rcu_qsbr_check(v, t, true);
 206 }
 207
 208 /* Dump the details of a single quiescent state variable to a file. */
 209 int __rte_experimental
 210 rte_rcu_qsbr_dump(FILE *f, struct rte_rcu_qsbr *v)
 211 {
 212         uint64_t bmap;
 213         uint32_t i, t, id;
 214
 215         if (v == NULL || f == NULL) {
 216                 rte_log(RTE_LOG_ERR, rte_rcu_log_type,
 217                         "%s(): Invalid input parameter\n", __func__);
 218                 rte_errno = EINVAL;
 219
 220                 return 1;
 221         }
 222
 223         fprintf(f, "\nQuiescent State Variable @%p\n", v);
 224
 225         fprintf(f, "  QS variable memory size = %zu\n",
 226                                 rte_rcu_qsbr_get_memsize(v->max_threads));
 227         fprintf(f, "  Given # max threads = %u\n", v->max_threads);
 228         fprintf(f, "  Current # threads = %u\n", v->num_threads);
 229
 230         fprintf(f, "  Registered thread IDs = ");
 231         for (i = 0; i < v->num_elems; i++) {
 232                 bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
 233                                         __ATOMIC_ACQUIRE);
 234                 id = i << __RTE_QSBR_THRID_INDEX_SHIFT;
 235                 while (bmap) {
 236                         t = __builtin_ctzl(bmap);
 237                         fprintf(f, "%u ", id + t);
 238
 239                         bmap &= ~(1UL << t);
 240                 }
 241         }
 242
 243         fprintf(f, "\n");
 244
 245         fprintf(f, "  Token = %"PRIu64"\n",
 246                         __atomic_load_n(&v->token, __ATOMIC_ACQUIRE));
 247
 248         fprintf(f, "Quiescent State Counts for readers:\n");
 249         for (i = 0; i < v->num_elems; i++) {
 250                 bmap = __atomic_load_n(__RTE_QSBR_THRID_ARRAY_ELM(v, i),
 251                                         __ATOMIC_ACQUIRE);
 252                 id = i << __RTE_QSBR_THRID_INDEX_SHIFT;
 253                 while (bmap) {
 254                         t = __builtin_ctzl(bmap);
 255                         fprintf(f, "thread ID = %u, count = %"PRIu64", lock count = %u\n",
 256                                 id + t,
 257                                 __atomic_load_n(
 258                                         &v->qsbr_cnt[id + t].cnt,
 259                                         __ATOMIC_RELAXED),
 260                                 __atomic_load_n(
 261                                         &v->qsbr_cnt[id + t].lock_cnt,
 262                                         __ATOMIC_RELAXED));
 263                         bmap &= ~(1UL << t);
 264                 }
 265         }
 266
 267         return 0;
 268 }
 269
 270 int rte_rcu_log_type;
 271
 272 RTE_INIT(rte_rcu_register)
 273 {
 274         rte_rcu_log_type = rte_log_register("lib.rcu");
 275         if (rte_rcu_log_type >= 0)
 276                 rte_log_set_level(rte_rcu_log_type, RTE_LOG_ERR);
 277 }