drivers/vdpa/mlx5/mlx5_vdpa_cthread.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright (c) 2022 NVIDIA Corporation & Affiliates
   3  */
   4 #include <string.h>
   5 #include <unistd.h>
   6 #include <sys/eventfd.h>
   7
   8 #include <rte_malloc.h>
   9 #include <rte_errno.h>
  10 #include <rte_io.h>
  11 #include <rte_alarm.h>
  12 #include <rte_tailq.h>
  13 #include <rte_ring_elem.h>
  14 #include <rte_ring_peek.h>
  15
  16 #include <mlx5_common.h>
  17
  18 #include "mlx5_vdpa_utils.h"
  19 #include "mlx5_vdpa.h"
  20
  21 static inline uint32_t
  22 mlx5_vdpa_c_thrd_ring_dequeue_bulk(struct rte_ring *r,
  23         void **obj, uint32_t n, uint32_t *avail)
  24 {
  25         uint32_t m;
  26
  27         m = rte_ring_dequeue_bulk_elem_start(r, obj,
  28                 sizeof(struct mlx5_vdpa_task), n, avail);
  29         n = (m == n) ? n : 0;
  30         rte_ring_dequeue_elem_finish(r, n);
  31         return n;
  32 }
  33
  34 static inline uint32_t
  35 mlx5_vdpa_c_thrd_ring_enqueue_bulk(struct rte_ring *r,
  36         void * const *obj, uint32_t n, uint32_t *free)
  37 {
  38         uint32_t m;
  39
  40         m = rte_ring_enqueue_bulk_elem_start(r, n, free);
  41         n = (m == n) ? n : 0;
  42         rte_ring_enqueue_elem_finish(r, obj,
  43                 sizeof(struct mlx5_vdpa_task), n);
  44         return n;
  45 }
  46
  47 bool
  48 mlx5_vdpa_task_add(struct mlx5_vdpa_priv *priv,
  49                 uint32_t thrd_idx,
  50                 uint32_t num)
  51 {
  52         struct rte_ring *rng = conf_thread_mng.cthrd[thrd_idx].rng;
  53         struct mlx5_vdpa_task task[MLX5_VDPA_TASKS_PER_DEV];
  54         uint32_t i;
  55
  56         MLX5_ASSERT(num <= MLX5_VDPA_TASKS_PER_DEV);
  57         for (i = 0 ; i < num; i++) {
  58                 task[i].priv = priv;
  59                 /* To be added later. */
  60         }
  61         if (!mlx5_vdpa_c_thrd_ring_enqueue_bulk(rng, (void **)&task, num, NULL))
  62                 return -1;
  63         for (i = 0 ; i < num; i++)
  64                 if (task[i].remaining_cnt)
  65                         __atomic_fetch_add(task[i].remaining_cnt, 1,
  66                                 __ATOMIC_RELAXED);
  67         /* wake up conf thread. */
  68         pthread_mutex_lock(&conf_thread_mng.cthrd_lock);
  69         pthread_cond_signal(&conf_thread_mng.cthrd[thrd_idx].c_cond);
  70         pthread_mutex_unlock(&conf_thread_mng.cthrd_lock);
  71         return 0;
  72 }
  73
  74 static void *
  75 mlx5_vdpa_c_thread_handle(void *arg)
  76 {
  77         struct mlx5_vdpa_conf_thread_mng *multhrd = arg;
  78         pthread_t thread_id = pthread_self();
  79         struct mlx5_vdpa_priv *priv;
  80         struct mlx5_vdpa_task task;
  81         struct rte_ring *rng;
  82         uint32_t thrd_idx;
  83         uint32_t task_num;
  84
  85         for (thrd_idx = 0; thrd_idx < multhrd->max_thrds;
  86                 thrd_idx++)
  87                 if (multhrd->cthrd[thrd_idx].tid == thread_id)
  88                         break;
  89         if (thrd_idx >= multhrd->max_thrds)
  90                 return NULL;
  91         rng = multhrd->cthrd[thrd_idx].rng;
  92         while (1) {
  93                 task_num = mlx5_vdpa_c_thrd_ring_dequeue_bulk(rng,
  94                         (void **)&task, 1, NULL);
  95                 if (!task_num) {
  96                         /* No task and condition wait. */
  97                         pthread_mutex_lock(&multhrd->cthrd_lock);
  98                         pthread_cond_wait(
  99                                 &multhrd->cthrd[thrd_idx].c_cond,
 100                                 &multhrd->cthrd_lock);
 101                         pthread_mutex_unlock(&multhrd->cthrd_lock);
 102                 }
 103                 priv = task.priv;
 104                 if (priv == NULL)
 105                         continue;
 106                 __atomic_fetch_sub(task.remaining_cnt,
 107                         1, __ATOMIC_RELAXED);
 108                 /* To be added later. */
 109         }
 110         return NULL;
 111 }
 112
 113 static void
 114 mlx5_vdpa_c_thread_destroy(uint32_t thrd_idx, bool need_unlock)
 115 {
 116         if (conf_thread_mng.cthrd[thrd_idx].tid) {
 117                 pthread_cancel(conf_thread_mng.cthrd[thrd_idx].tid);
 118                 pthread_join(conf_thread_mng.cthrd[thrd_idx].tid, NULL);
 119                 conf_thread_mng.cthrd[thrd_idx].tid = 0;
 120                 if (need_unlock)
 121                         pthread_mutex_init(&conf_thread_mng.cthrd_lock, NULL);
 122         }
 123         if (conf_thread_mng.cthrd[thrd_idx].rng) {
 124                 rte_ring_free(conf_thread_mng.cthrd[thrd_idx].rng);
 125                 conf_thread_mng.cthrd[thrd_idx].rng = NULL;
 126         }
 127 }
 128
 129 static int
 130 mlx5_vdpa_c_thread_create(int cpu_core)
 131 {
 132         const struct sched_param sp = {
 133                 .sched_priority = sched_get_priority_max(SCHED_RR),
 134         };
 135         rte_cpuset_t cpuset;
 136         pthread_attr_t attr;
 137         uint32_t thrd_idx;
 138         uint32_t ring_num;
 139         char name[32];
 140         int ret;
 141
 142         pthread_mutex_lock(&conf_thread_mng.cthrd_lock);
 143         pthread_attr_init(&attr);
 144         ret = pthread_attr_setschedpolicy(&attr, SCHED_RR);
 145         if (ret) {
 146                 DRV_LOG(ERR, "Failed to set thread sched policy = RR.");
 147                 goto c_thread_err;
 148         }
 149         ret = pthread_attr_setschedparam(&attr, &sp);
 150         if (ret) {
 151                 DRV_LOG(ERR, "Failed to set thread priority.");
 152                 goto c_thread_err;
 153         }
 154         ring_num = MLX5_VDPA_MAX_TASKS_PER_THRD / conf_thread_mng.max_thrds;
 155         if (!ring_num) {
 156                 DRV_LOG(ERR, "Invalid ring number for thread.");
 157                 goto c_thread_err;
 158         }
 159         for (thrd_idx = 0; thrd_idx < conf_thread_mng.max_thrds;
 160                 thrd_idx++) {
 161                 snprintf(name, sizeof(name), "vDPA-mthread-ring-%d",
 162                         thrd_idx);
 163                 conf_thread_mng.cthrd[thrd_idx].rng = rte_ring_create_elem(name,
 164                         sizeof(struct mlx5_vdpa_task), ring_num,
 165                         rte_socket_id(),
 166                         RING_F_MP_HTS_ENQ | RING_F_MC_HTS_DEQ |
 167                         RING_F_EXACT_SZ);
 168                 if (!conf_thread_mng.cthrd[thrd_idx].rng) {
 169                         DRV_LOG(ERR,
 170                         "Failed to create vdpa multi-threads %d ring.",
 171                         thrd_idx);
 172                         goto c_thread_err;
 173                 }
 174                 ret = pthread_create(&conf_thread_mng.cthrd[thrd_idx].tid,
 175                                 &attr, mlx5_vdpa_c_thread_handle,
 176                                 (void *)&conf_thread_mng);
 177                 if (ret) {
 178                         DRV_LOG(ERR, "Failed to create vdpa multi-threads %d.",
 179                                         thrd_idx);
 180                         goto c_thread_err;
 181                 }
 182                 CPU_ZERO(&cpuset);
 183                 if (cpu_core != -1)
 184                         CPU_SET(cpu_core, &cpuset);
 185                 else
 186                         cpuset = rte_lcore_cpuset(rte_get_main_lcore());
 187                 ret = pthread_setaffinity_np(
 188                                 conf_thread_mng.cthrd[thrd_idx].tid,
 189                                 sizeof(cpuset), &cpuset);
 190                 if (ret) {
 191                         DRV_LOG(ERR, "Failed to set thread affinity for "
 192                         "vdpa multi-threads %d.", thrd_idx);
 193                         goto c_thread_err;
 194                 }
 195                 snprintf(name, sizeof(name), "vDPA-mthread-%d", thrd_idx);
 196                 ret = pthread_setname_np(
 197                                 conf_thread_mng.cthrd[thrd_idx].tid, name);
 198                 if (ret)
 199                         DRV_LOG(ERR, "Failed to set vdpa multi-threads name %s.",
 200                                         name);
 201                 else
 202                         DRV_LOG(DEBUG, "Thread name: %s.", name);
 203                 pthread_cond_init(&conf_thread_mng.cthrd[thrd_idx].c_cond,
 204                         NULL);
 205         }
 206         pthread_mutex_unlock(&conf_thread_mng.cthrd_lock);
 207         return 0;
 208 c_thread_err:
 209         for (thrd_idx = 0; thrd_idx < conf_thread_mng.max_thrds;
 210                 thrd_idx++)
 211                 mlx5_vdpa_c_thread_destroy(thrd_idx, false);
 212         pthread_mutex_unlock(&conf_thread_mng.cthrd_lock);
 213         return -1;
 214 }
 215
 216 int
 217 mlx5_vdpa_mult_threads_create(int cpu_core)
 218 {
 219         pthread_mutex_init(&conf_thread_mng.cthrd_lock, NULL);
 220         if (mlx5_vdpa_c_thread_create(cpu_core)) {
 221                 DRV_LOG(ERR, "Cannot create vDPA configuration threads.");
 222                 mlx5_vdpa_mult_threads_destroy(false);
 223                 return -1;
 224         }
 225         return 0;
 226 }
 227
 228 void
 229 mlx5_vdpa_mult_threads_destroy(bool need_unlock)
 230 {
 231         uint32_t thrd_idx;
 232
 233         if (!conf_thread_mng.initializer_priv)
 234                 return;
 235         for (thrd_idx = 0; thrd_idx < conf_thread_mng.max_thrds;
 236                 thrd_idx++)
 237                 mlx5_vdpa_c_thread_destroy(thrd_idx, need_unlock);
 238         pthread_mutex_destroy(&conf_thread_mng.cthrd_lock);
 239         memset(&conf_thread_mng, 0, sizeof(struct mlx5_vdpa_conf_thread_mng));
 240 }