drivers/event/octeontx/timvf_evdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Cavium, Inc
   3  */
   4
   5 #include "ssovf_evdev.h"
   6 #include "timvf_evdev.h"
   7
   8 RTE_LOG_REGISTER_SUFFIX(otx_logtype_timvf, timer, NOTICE);
   9
  10 static struct rte_eventdev *event_dev;
  11
  12 struct __rte_packed timvf_mbox_dev_info {
  13         uint64_t ring_active[4];
  14         uint64_t clk_freq;
  15 };
  16
  17 /* Response messages */
  18 enum {
  19         MBOX_RET_SUCCESS,
  20         MBOX_RET_INVALID,
  21         MBOX_RET_INTERNAL_ERR,
  22 };
  23
  24 static int
  25 timvf_mbox_dev_info_get(struct timvf_mbox_dev_info *info)
  26 {
  27         struct octeontx_mbox_hdr hdr = {0};
  28         uint16_t len = sizeof(struct timvf_mbox_dev_info);
  29
  30         hdr.coproc = TIM_COPROC;
  31         hdr.msg = TIM_GET_DEV_INFO;
  32         hdr.vfid = 0; /* TIM DEV is always 0. TIM RING ID changes. */
  33
  34         memset(info, 0, len);
  35         return octeontx_mbox_send(&hdr, NULL, 0, info, len);
  36 }
  37
  38 static void
  39 timvf_ring_info_get(const struct rte_event_timer_adapter *adptr,
  40                 struct rte_event_timer_adapter_info *adptr_info)
  41 {
  42         struct timvf_ring *timr = adptr->data->adapter_priv;
  43         adptr_info->max_tmo_ns = timr->max_tout;
  44         adptr_info->min_resolution_ns = timr->tck_nsec;
  45         rte_memcpy(&adptr_info->conf, &adptr->data->conf,
  46                         sizeof(struct rte_event_timer_adapter_conf));
  47 }
  48
  49 static int
  50 timvf_ring_conf_set(struct timvf_ctrl_reg *rctl, uint8_t ring_id)
  51 {
  52         struct octeontx_mbox_hdr hdr = {0};
  53         uint16_t len = sizeof(struct timvf_ctrl_reg);
  54         int ret;
  55
  56         hdr.coproc = TIM_COPROC;
  57         hdr.msg = TIM_SET_RING_INFO;
  58         hdr.vfid = ring_id;
  59
  60         ret = octeontx_mbox_send(&hdr, rctl, len, NULL, 0);
  61         if (ret < 0 || hdr.res_code != MBOX_RET_SUCCESS)
  62                 return -EACCES;
  63         return 0;
  64 }
  65
  66 static int
  67 timvf_get_start_cyc(uint64_t *now, uint8_t ring_id)
  68 {
  69         struct octeontx_mbox_hdr hdr = {0};
  70
  71         hdr.coproc = TIM_COPROC;
  72         hdr.msg = TIM_RING_START_CYC_GET;
  73         hdr.vfid = ring_id;
  74         *now = 0;
  75         return octeontx_mbox_send(&hdr, NULL, 0, now, sizeof(uint64_t));
  76 }
  77
  78 static int
  79 optimize_bucket_parameters(struct timvf_ring *timr)
  80 {
  81         uint32_t hbkts;
  82         uint32_t lbkts;
  83         uint64_t tck_nsec;
  84
  85         hbkts = rte_align32pow2(timr->nb_bkts);
  86         tck_nsec = RTE_ALIGN_MUL_CEIL(timr->max_tout / (hbkts - 1), 10);
  87
  88         if ((tck_nsec < 1000 || hbkts > TIM_MAX_BUCKETS))
  89                 hbkts = 0;
  90
  91         lbkts = rte_align32prevpow2(timr->nb_bkts);
  92         tck_nsec = RTE_ALIGN_MUL_CEIL((timr->max_tout / (lbkts - 1)), 10);
  93
  94         if ((tck_nsec < 1000 || hbkts > TIM_MAX_BUCKETS))
  95                 lbkts = 0;
  96
  97         if (!hbkts && !lbkts)
  98                 return 0;
  99
 100         if (!hbkts) {
 101                 timr->nb_bkts = lbkts;
 102                 goto end;
 103         } else if (!lbkts) {
 104                 timr->nb_bkts = hbkts;
 105                 goto end;
 106         }
 107
 108         timr->nb_bkts = (hbkts - timr->nb_bkts) <
 109                 (timr->nb_bkts - lbkts) ? hbkts : lbkts;
 110 end:
 111         timr->get_target_bkt = bkt_and;
 112         timr->tck_nsec = RTE_ALIGN_MUL_CEIL((timr->max_tout /
 113                                 (timr->nb_bkts - 1)), 10);
 114         return 1;
 115 }
 116
 117 static int
 118 timvf_ring_start(const struct rte_event_timer_adapter *adptr)
 119 {
 120         int ret;
 121         uint8_t use_fpa = 0;
 122         uint64_t interval;
 123         uintptr_t pool;
 124         struct timvf_ctrl_reg rctrl;
 125         struct timvf_mbox_dev_info dinfo;
 126         struct timvf_ring *timr = adptr->data->adapter_priv;
 127
 128         ret = timvf_mbox_dev_info_get(&dinfo);
 129         if (ret < 0 || ret != sizeof(struct timvf_mbox_dev_info))
 130                 return -EINVAL;
 131
 132         /* Calculate the interval cycles according to clock source. */
 133         switch (timr->clk_src) {
 134         case TIM_CLK_SRC_SCLK:
 135                 interval = NSEC2CLK(timr->tck_nsec, dinfo.clk_freq);
 136                 break;
 137         case TIM_CLK_SRC_GPIO:
 138                 /* GPIO doesn't work on tck_nsec. */
 139                 interval = 0;
 140                 break;
 141         case TIM_CLK_SRC_GTI:
 142                 interval = NSEC2CLK(timr->tck_nsec, dinfo.clk_freq);
 143                 break;
 144         case TIM_CLK_SRC_PTP:
 145                 interval = NSEC2CLK(timr->tck_nsec, dinfo.clk_freq);
 146                 break;
 147         default:
 148                 timvf_log_err("Unsupported clock source configured %d",
 149                                 timr->clk_src);
 150                 return -EINVAL;
 151         }
 152
 153         if (!strcmp(rte_mbuf_best_mempool_ops(), "octeontx_fpavf"))
 154                 use_fpa = 1;
 155
 156         /*CTRL0 register.*/
 157         rctrl.rctrl0 = interval;
 158
 159         /*CTRL1 register.*/
 160         rctrl.rctrl1 =  (uint64_t)(timr->clk_src) << 51 |
 161                 1ull << 48 /* LOCK_EN (Enable hw bucket lock mechanism) */ |
 162                 1ull << 47 /* ENA */ |
 163                 1ull << 44 /* ENA_LDWB */ |
 164                 (timr->nb_bkts - 1);
 165
 166         rctrl.rctrl2 = (uint64_t)(TIM_CHUNK_SIZE / 16) << 40;
 167
 168         if (use_fpa) {
 169                 pool = (uintptr_t)((struct rte_mempool *)
 170                                 timr->chunk_pool)->pool_id;
 171                 ret = octeontx_fpa_bufpool_gaura(pool);
 172                 if (ret < 0) {
 173                         timvf_log_dbg("Unable to get gaura id");
 174                         ret = -ENOMEM;
 175                         goto error;
 176                 }
 177                 timvf_write64((uint64_t)ret,
 178                                 (uint8_t *)timr->vbar0 + TIM_VRING_AURA);
 179         } else {
 180                 rctrl.rctrl1 |= 1ull << 43 /* ENA_DFB (Enable don't free) */;
 181         }
 182
 183         timvf_write64((uintptr_t)timr->bkt,
 184                         (uint8_t *)timr->vbar0 + TIM_VRING_BASE);
 185         timvf_set_chunk_refill(timr, use_fpa);
 186         if (timvf_ring_conf_set(&rctrl, timr->tim_ring_id)) {
 187                 ret = -EACCES;
 188                 goto error;
 189         }
 190
 191         if (timvf_get_start_cyc(&timr->ring_start_cyc,
 192                                 timr->tim_ring_id) < 0) {
 193                 ret = -EACCES;
 194                 goto error;
 195         }
 196         timr->tck_int = NSEC2CLK(timr->tck_nsec, rte_get_timer_hz());
 197         timr->fast_div = rte_reciprocal_value_u64(timr->tck_int);
 198         timvf_log_info("nb_bkts %d min_ns %"PRIu64" min_cyc %"PRIu64""
 199                         " maxtmo %"PRIu64"\n",
 200                         timr->nb_bkts, timr->tck_nsec, interval,
 201                         timr->max_tout);
 202
 203         return 0;
 204 error:
 205         rte_free(timr->bkt);
 206         rte_mempool_free(timr->chunk_pool);
 207         return ret;
 208 }
 209
 210 static int
 211 timvf_ring_stop(const struct rte_event_timer_adapter *adptr)
 212 {
 213         struct timvf_ring *timr = adptr->data->adapter_priv;
 214         struct timvf_ctrl_reg rctrl = {0};
 215         rctrl.rctrl0 = timvf_read64((uint8_t *)timr->vbar0 + TIM_VRING_CTL0);
 216         rctrl.rctrl1 = timvf_read64((uint8_t *)timr->vbar0 + TIM_VRING_CTL1);
 217         rctrl.rctrl1 &= ~(1ull << 47); /* Disable */
 218         rctrl.rctrl2 = timvf_read64((uint8_t *)timr->vbar0 + TIM_VRING_CTL2);
 219
 220         if (timvf_ring_conf_set(&rctrl, timr->tim_ring_id))
 221                 return -EACCES;
 222         return 0;
 223 }
 224
 225 static int
 226 timvf_ring_create(struct rte_event_timer_adapter *adptr)
 227 {
 228         struct rte_event_timer_adapter_conf *rcfg = &adptr->data->conf;
 229         uint16_t free_idx = UINT16_MAX;
 230         unsigned int mp_flags = 0;
 231         struct ssovf_evdev *edev;
 232         struct timvf_ring *timr;
 233         const char *mempool_ops;
 234         uint8_t tim_ring_id;
 235         char pool_name[25];
 236         int i, ret;
 237
 238         tim_ring_id = timvf_get_ring();
 239         if (tim_ring_id == UINT8_MAX)
 240                 return -ENODEV;
 241
 242         edev = ssovf_pmd_priv(event_dev);
 243         timr = rte_zmalloc("octeontx_timvf_priv",
 244                         sizeof(struct timvf_ring), 0);
 245         if (timr == NULL)
 246                 return -ENOMEM;
 247
 248         adptr->data->adapter_priv = timr;
 249         /* Check config parameters. */
 250         if ((rcfg->clk_src != RTE_EVENT_TIMER_ADAPTER_CPU_CLK) &&
 251                         (!rcfg->timer_tick_ns ||
 252                          rcfg->timer_tick_ns < TIM_MIN_INTERVAL)) {
 253                 timvf_log_err("Too low timer ticks");
 254                 goto cfg_err;
 255         }
 256
 257         timr->clk_src = (int) rcfg->clk_src;
 258         timr->tim_ring_id = tim_ring_id;
 259         timr->tck_nsec = RTE_ALIGN_MUL_CEIL(rcfg->timer_tick_ns, 10);
 260         timr->max_tout = rcfg->max_tmo_ns;
 261         timr->nb_bkts = (timr->max_tout / timr->tck_nsec);
 262         timr->vbar0 = timvf_bar(timr->tim_ring_id, 0);
 263         timr->bkt_pos = (uint8_t *)timr->vbar0 + TIM_VRING_REL;
 264         timr->nb_timers = rcfg->nb_timers;
 265         timr->get_target_bkt = bkt_mod;
 266
 267         if (edev->available_events < timr->nb_timers) {
 268                 timvf_log_err(
 269                         "Max available events %"PRIu32" requested timer events %"PRIu64"",
 270                         edev->available_events, timr->nb_timers);
 271                 return -ENOMEM;
 272         }
 273
 274         for (i = 0; i < edev->tim_ring_cnt; i++) {
 275                 if (edev->tim_ring_ids[i] == UINT16_MAX)
 276                         free_idx = i;
 277         }
 278
 279         if (free_idx == UINT16_MAX) {
 280                 void *old_ptr;
 281
 282                 edev->tim_ring_cnt++;
 283                 old_ptr = edev->tim_ring_ids;
 284                 edev->tim_ring_ids =
 285                         rte_realloc(edev->tim_ring_ids,
 286                                     sizeof(uint16_t) * edev->tim_ring_cnt, 0);
 287                 if (edev->tim_ring_ids == NULL) {
 288                         edev->tim_ring_ids = old_ptr;
 289                         edev->tim_ring_cnt--;
 290                         return -ENOMEM;
 291                 }
 292
 293                 edev->available_events -= timr->nb_timers;
 294         } else {
 295                 edev->tim_ring_ids[free_idx] = tim_ring_id;
 296                 edev->available_events -= timr->nb_timers;
 297         }
 298
 299         timr->nb_chunks = timr->nb_timers / nb_chunk_slots;
 300
 301         /* Try to optimize the bucket parameters. */
 302         if ((rcfg->flags & RTE_EVENT_TIMER_ADAPTER_F_ADJUST_RES)
 303                         && !rte_is_power_of_2(timr->nb_bkts)) {
 304                 if (optimize_bucket_parameters(timr)) {
 305                         timvf_log_info("Optimized configured values");
 306                         timvf_log_dbg("nb_bkts  : %"PRIu32"", timr->nb_bkts);
 307                         timvf_log_dbg("tck_nsec : %"PRIu64"", timr->tck_nsec);
 308                 } else
 309                         timvf_log_info("Failed to Optimize configured values");
 310         }
 311
 312         if (rcfg->flags & RTE_EVENT_TIMER_ADAPTER_F_SP_PUT) {
 313                 mp_flags = MEMPOOL_F_SP_PUT | MEMPOOL_F_SC_GET;
 314                 timvf_log_info("Using single producer mode");
 315         }
 316
 317         timr->bkt = rte_zmalloc("octeontx_timvf_bucket",
 318                         (timr->nb_bkts) * sizeof(struct tim_mem_bucket),
 319                         0);
 320         if (timr->bkt == NULL)
 321                 goto mem_err;
 322
 323         snprintf(pool_name, sizeof(pool_name), "timvf_chunk_pool%d",
 324                         timr->tim_ring_id);
 325         timr->chunk_pool = (void *)rte_mempool_create_empty(pool_name,
 326                         timr->nb_chunks, TIM_CHUNK_SIZE, 0, 0, rte_socket_id(),
 327                         mp_flags);
 328
 329         if (!timr->chunk_pool) {
 330                 rte_free(timr->bkt);
 331                 timvf_log_err("Unable to create chunkpool.");
 332                 return -ENOMEM;
 333         }
 334
 335         mempool_ops = rte_mbuf_best_mempool_ops();
 336         ret = rte_mempool_set_ops_byname(timr->chunk_pool,
 337                         mempool_ops, NULL);
 338
 339         if (ret != 0) {
 340                 timvf_log_err("Unable to set chunkpool ops.");
 341                 goto mem_err;
 342         }
 343
 344         ret = rte_mempool_populate_default(timr->chunk_pool);
 345         if (ret < 0) {
 346                 timvf_log_err("Unable to set populate chunkpool.");
 347                 goto mem_err;
 348         }
 349         timvf_write64(0, (uint8_t *)timr->vbar0 + TIM_VRING_BASE);
 350         timvf_write64(0, (uint8_t *)timr->vbar0 + TIM_VF_NRSPERR_INT);
 351         timvf_write64(0, (uint8_t *)timr->vbar0 + TIM_VF_NRSPERR_INT_W1S);
 352         timvf_write64(0x7, (uint8_t *)timr->vbar0 + TIM_VF_NRSPERR_ENA_W1C);
 353         timvf_write64(0x7, (uint8_t *)timr->vbar0 + TIM_VF_NRSPERR_ENA_W1S);
 354
 355         return 0;
 356 mem_err:
 357         rte_free(timr);
 358         return -ENOMEM;
 359 cfg_err:
 360         rte_free(timr);
 361         return -EINVAL;
 362 }
 363
 364 static int
 365 timvf_ring_free(struct rte_event_timer_adapter *adptr)
 366 {
 367         struct timvf_ring *timr = adptr->data->adapter_priv;
 368         struct ssovf_evdev *edev;
 369         int i;
 370
 371         edev = ssovf_pmd_priv(event_dev);
 372         for (i = 0; i < edev->tim_ring_cnt; i++) {
 373                 if (edev->tim_ring_ids[i] == timr->tim_ring_id) {
 374                         edev->available_events += timr->nb_timers;
 375                         edev->tim_ring_ids[i] = UINT16_MAX;
 376                         break;
 377                 }
 378         }
 379
 380         rte_mempool_free(timr->chunk_pool);
 381         rte_free(timr->bkt);
 382         timvf_release_ring(timr->tim_ring_id);
 383         rte_free(adptr->data->adapter_priv);
 384         return 0;
 385 }
 386
 387 static int
 388 timvf_stats_get(const struct rte_event_timer_adapter *adapter,
 389                 struct rte_event_timer_adapter_stats *stats)
 390 {
 391         struct timvf_ring *timr = adapter->data->adapter_priv;
 392         uint64_t bkt_cyc = rte_rdtsc() - timr->ring_start_cyc;
 393
 394         stats->evtim_exp_count = timr->tim_arm_cnt;
 395         stats->ev_enq_count = timr->tim_arm_cnt;
 396         stats->adapter_tick_count = rte_reciprocal_divide_u64(bkt_cyc,
 397                                 &timr->fast_div);
 398         return 0;
 399 }
 400
 401 static int
 402 timvf_stats_reset(const struct rte_event_timer_adapter *adapter)
 403 {
 404         struct timvf_ring *timr = adapter->data->adapter_priv;
 405
 406         timr->tim_arm_cnt = 0;
 407         return 0;
 408 }
 409
 410 static struct rte_event_timer_adapter_ops timvf_ops = {
 411         .init           = timvf_ring_create,
 412         .uninit         = timvf_ring_free,
 413         .start          = timvf_ring_start,
 414         .stop           = timvf_ring_stop,
 415         .get_info       = timvf_ring_info_get,
 416 };
 417
 418 int
 419 timvf_timer_adapter_caps_get(const struct rte_eventdev *dev, uint64_t flags,
 420                 uint32_t *caps, const struct rte_event_timer_adapter_ops **ops,
 421                 uint8_t enable_stats)
 422 {
 423         RTE_SET_USED(dev);
 424
 425         if (enable_stats) {
 426                 timvf_ops.stats_get   = timvf_stats_get;
 427                 timvf_ops.stats_reset = timvf_stats_reset;
 428         }
 429
 430         if (flags & RTE_EVENT_TIMER_ADAPTER_F_SP_PUT)
 431                 timvf_ops.arm_burst = enable_stats ?
 432                         timvf_timer_arm_burst_sp_stats :
 433                         timvf_timer_arm_burst_sp;
 434         else
 435                 timvf_ops.arm_burst = enable_stats ?
 436                         timvf_timer_arm_burst_mp_stats :
 437                         timvf_timer_arm_burst_mp;
 438
 439         timvf_ops.arm_tmo_tick_burst = enable_stats ?
 440                 timvf_timer_arm_tmo_brst_stats :
 441                 timvf_timer_arm_tmo_brst;
 442         timvf_ops.cancel_burst = timvf_timer_cancel_burst;
 443         *caps = RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT;
 444         *ops = &timvf_ops;
 445         return 0;
 446 }
 447
 448 void
 449 timvf_set_eventdevice(struct rte_eventdev *dev)
 450 {
 451         event_dev = dev;
 452 }