drivers/event/octeontx/timvf_evdev.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(c) 2017 Cavium, Inc
   3  */
   4
   5 #include "timvf_evdev.h"
   6
   7 int otx_logtype_timvf;
   8
   9 RTE_INIT(otx_timvf_init_log);
  10 static void
  11 otx_timvf_init_log(void)
  12 {
  13         otx_logtype_timvf = rte_log_register("pmd.event.octeontx.timer");
  14         if (otx_logtype_timvf >= 0)
  15                 rte_log_set_level(otx_logtype_timvf, RTE_LOG_NOTICE);
  16 }
  17
  18 struct __rte_packed timvf_mbox_dev_info {
  19         uint64_t ring_active[4];
  20         uint64_t clk_freq;
  21 };
  22
  23 /* Response messages */
  24 enum {
  25         MBOX_RET_SUCCESS,
  26         MBOX_RET_INVALID,
  27         MBOX_RET_INTERNAL_ERR,
  28 };
  29
  30 static int
  31 timvf_mbox_dev_info_get(struct timvf_mbox_dev_info *info)
  32 {
  33         struct octeontx_mbox_hdr hdr = {0};
  34         uint16_t len = sizeof(struct timvf_mbox_dev_info);
  35
  36         hdr.coproc = TIM_COPROC;
  37         hdr.msg = TIM_GET_DEV_INFO;
  38         hdr.vfid = 0; /* TIM DEV is always 0. TIM RING ID changes. */
  39
  40         memset(info, 0, len);
  41         return octeontx_mbox_send(&hdr, NULL, 0, info, len);
  42 }
  43
  44 static void
  45 timvf_ring_info_get(const struct rte_event_timer_adapter *adptr,
  46                 struct rte_event_timer_adapter_info *adptr_info)
  47 {
  48         struct timvf_ring *timr = adptr->data->adapter_priv;
  49         adptr_info->max_tmo_ns = timr->max_tout;
  50         adptr_info->min_resolution_ns = timr->tck_nsec;
  51         rte_memcpy(&adptr_info->conf, &adptr->data->conf,
  52                         sizeof(struct rte_event_timer_adapter_conf));
  53 }
  54
  55 static int
  56 timvf_ring_conf_set(struct timvf_ctrl_reg *rctl, uint8_t ring_id)
  57 {
  58         struct octeontx_mbox_hdr hdr = {0};
  59         uint16_t len = sizeof(struct timvf_ctrl_reg);
  60         int ret;
  61
  62         hdr.coproc = TIM_COPROC;
  63         hdr.msg = TIM_SET_RING_INFO;
  64         hdr.vfid = ring_id;
  65
  66         ret = octeontx_mbox_send(&hdr, rctl, len, NULL, 0);
  67         if (ret < 0 || hdr.res_code != MBOX_RET_SUCCESS)
  68                 return -EACCES;
  69         return 0;
  70 }
  71
  72 static int
  73 timvf_get_start_cyc(uint64_t *now, uint8_t ring_id)
  74 {
  75         struct octeontx_mbox_hdr hdr = {0};
  76
  77         hdr.coproc = TIM_COPROC;
  78         hdr.msg = TIM_RING_START_CYC_GET;
  79         hdr.vfid = ring_id;
  80         *now = 0;
  81         return octeontx_mbox_send(&hdr, NULL, 0, now, sizeof(uint64_t));
  82 }
  83
  84 static int
  85 optimize_bucket_parameters(struct timvf_ring *timr)
  86 {
  87         uint32_t hbkts;
  88         uint32_t lbkts;
  89         uint64_t tck_nsec;
  90
  91         hbkts = rte_align32pow2(timr->nb_bkts);
  92         tck_nsec = RTE_ALIGN_MUL_CEIL(timr->max_tout / (hbkts - 1), 10);
  93
  94         if ((tck_nsec < 1000 || hbkts > TIM_MAX_BUCKETS))
  95                 hbkts = 0;
  96
  97         lbkts = rte_align32prevpow2(timr->nb_bkts);
  98         tck_nsec = RTE_ALIGN_MUL_CEIL((timr->max_tout / (lbkts - 1)), 10);
  99
 100         if ((tck_nsec < 1000 || hbkts > TIM_MAX_BUCKETS))
 101                 lbkts = 0;
 102
 103         if (!hbkts && !lbkts)
 104                 return 0;
 105
 106         if (!hbkts) {
 107                 timr->nb_bkts = lbkts;
 108                 goto end;
 109         } else if (!lbkts) {
 110                 timr->nb_bkts = hbkts;
 111                 goto end;
 112         }
 113
 114         timr->nb_bkts = (hbkts - timr->nb_bkts) <
 115                 (timr->nb_bkts - lbkts) ? hbkts : lbkts;
 116 end:
 117         timr->get_target_bkt = bkt_and;
 118         timr->tck_nsec = RTE_ALIGN_MUL_CEIL((timr->max_tout /
 119                                 (timr->nb_bkts - 1)), 10);
 120         return 1;
 121 }
 122
 123 static int
 124 timvf_ring_start(const struct rte_event_timer_adapter *adptr)
 125 {
 126         int ret;
 127         uint8_t use_fpa = 0;
 128         uint64_t interval;
 129         uintptr_t pool;
 130         struct timvf_ctrl_reg rctrl;
 131         struct timvf_mbox_dev_info dinfo;
 132         struct timvf_ring *timr = adptr->data->adapter_priv;
 133
 134         ret = timvf_mbox_dev_info_get(&dinfo);
 135         if (ret < 0 || ret != sizeof(struct timvf_mbox_dev_info))
 136                 return -EINVAL;
 137
 138         /* Calculate the interval cycles according to clock source. */
 139         switch (timr->clk_src) {
 140         case TIM_CLK_SRC_SCLK:
 141                 interval = NSEC2CLK(timr->tck_nsec, dinfo.clk_freq);
 142                 break;
 143         case TIM_CLK_SRC_GPIO:
 144                 /* GPIO doesn't work on tck_nsec. */
 145                 interval = 0;
 146                 break;
 147         case TIM_CLK_SRC_GTI:
 148                 interval = NSEC2CLK(timr->tck_nsec, dinfo.clk_freq);
 149                 break;
 150         case TIM_CLK_SRC_PTP:
 151                 interval = NSEC2CLK(timr->tck_nsec, dinfo.clk_freq);
 152                 break;
 153         default:
 154                 timvf_log_err("Unsupported clock source configured %d",
 155                                 timr->clk_src);
 156                 return -EINVAL;
 157         }
 158
 159         if (!strcmp(rte_mbuf_best_mempool_ops(), "octeontx_fpavf"))
 160                 use_fpa = 1;
 161
 162         /*CTRL0 register.*/
 163         rctrl.rctrl0 = interval;
 164
 165         /*CTRL1 register.*/
 166         rctrl.rctrl1 =  (uint64_t)(timr->clk_src) << 51 |
 167                 1ull << 48 /* LOCK_EN (Enable hw bucket lock mechanism) */ |
 168                 1ull << 47 /* ENA */ |
 169                 1ull << 44 /* ENA_LDWB */ |
 170                 (timr->nb_bkts - 1);
 171
 172         rctrl.rctrl2 = (uint64_t)(TIM_CHUNK_SIZE / 16) << 40;
 173
 174         if (use_fpa) {
 175                 pool = (uintptr_t)((struct rte_mempool *)
 176                                 timr->chunk_pool)->pool_id;
 177                 ret = octeontx_fpa_bufpool_gpool(pool);
 178                 if (ret < 0) {
 179                         timvf_log_dbg("Unable to get gaura id");
 180                         ret = -ENOMEM;
 181                         goto error;
 182                 }
 183                 timvf_write64((uint64_t)ret,
 184                                 (uint8_t *)timr->vbar0 + TIM_VRING_AURA);
 185         } else {
 186                 rctrl.rctrl1 |= 1ull << 43 /* ENA_DFB (Enable don't free) */;
 187         }
 188
 189         timvf_write64((uintptr_t)timr->bkt,
 190                         (uint8_t *)timr->vbar0 + TIM_VRING_BASE);
 191         timvf_set_chunk_refill(timr, use_fpa);
 192         if (timvf_ring_conf_set(&rctrl, timr->tim_ring_id)) {
 193                 ret = -EACCES;
 194                 goto error;
 195         }
 196
 197         if (timvf_get_start_cyc(&timr->ring_start_cyc,
 198                                 timr->tim_ring_id) < 0) {
 199                 ret = -EACCES;
 200                 goto error;
 201         }
 202         timr->tck_int = NSEC2CLK(timr->tck_nsec, rte_get_timer_hz());
 203         timr->fast_div = rte_reciprocal_value_u64(timr->tck_int);
 204         timvf_log_info("nb_bkts %d min_ns %"PRIu64" min_cyc %"PRIu64""
 205                         " maxtmo %"PRIu64"\n",
 206                         timr->nb_bkts, timr->tck_nsec, interval,
 207                         timr->max_tout);
 208
 209         return 0;
 210 error:
 211         rte_free(timr->bkt);
 212         rte_mempool_free(timr->chunk_pool);
 213         return ret;
 214 }
 215
 216 static int
 217 timvf_ring_stop(const struct rte_event_timer_adapter *adptr)
 218 {
 219         struct timvf_ring *timr = adptr->data->adapter_priv;
 220         struct timvf_ctrl_reg rctrl = {0};
 221         rctrl.rctrl0 = timvf_read64((uint8_t *)timr->vbar0 + TIM_VRING_CTL0);
 222         rctrl.rctrl1 = timvf_read64((uint8_t *)timr->vbar0 + TIM_VRING_CTL1);
 223         rctrl.rctrl1 &= ~(1ull << 47); /* Disable */
 224         rctrl.rctrl2 = timvf_read64((uint8_t *)timr->vbar0 + TIM_VRING_CTL2);
 225
 226         if (timvf_ring_conf_set(&rctrl, timr->tim_ring_id))
 227                 return -EACCES;
 228         return 0;
 229 }
 230
 231 static int
 232 timvf_ring_create(struct rte_event_timer_adapter *adptr)
 233 {
 234         char pool_name[25];
 235         int ret;
 236         uint64_t nb_timers;
 237         struct rte_event_timer_adapter_conf *rcfg = &adptr->data->conf;
 238         struct timvf_ring *timr;
 239         struct timvf_info tinfo;
 240         const char *mempool_ops;
 241         unsigned int mp_flags = 0;
 242
 243         if (timvf_info(&tinfo) < 0)
 244                 return -ENODEV;
 245
 246         if (adptr->data->id >= tinfo.total_timvfs)
 247                 return -ENODEV;
 248
 249         timr = rte_zmalloc("octeontx_timvf_priv",
 250                         sizeof(struct timvf_ring), 0);
 251         if (timr == NULL)
 252                 return -ENOMEM;
 253
 254         adptr->data->adapter_priv = timr;
 255         /* Check config parameters. */
 256         if ((rcfg->clk_src != RTE_EVENT_TIMER_ADAPTER_CPU_CLK) &&
 257                         (!rcfg->timer_tick_ns ||
 258                          rcfg->timer_tick_ns < TIM_MIN_INTERVAL)) {
 259                 timvf_log_err("Too low timer ticks");
 260                 goto cfg_err;
 261         }
 262
 263         timr->clk_src = (int) rcfg->clk_src;
 264         timr->tim_ring_id = adptr->data->id;
 265         timr->tck_nsec = RTE_ALIGN_MUL_CEIL(rcfg->timer_tick_ns, 10);
 266         timr->max_tout = rcfg->max_tmo_ns;
 267         timr->nb_bkts = (timr->max_tout / timr->tck_nsec);
 268         timr->vbar0 = timvf_bar(timr->tim_ring_id, 0);
 269         timr->bkt_pos = (uint8_t *)timr->vbar0 + TIM_VRING_REL;
 270         nb_timers = rcfg->nb_timers;
 271         timr->get_target_bkt = bkt_mod;
 272
 273         timr->nb_chunks = nb_timers / nb_chunk_slots;
 274
 275         /* Try to optimize the bucket parameters. */
 276         if ((rcfg->flags & RTE_EVENT_TIMER_ADAPTER_F_ADJUST_RES)
 277                         && !rte_is_power_of_2(timr->nb_bkts)) {
 278                 if (optimize_bucket_parameters(timr)) {
 279                         timvf_log_info("Optimized configured values");
 280                         timvf_log_dbg("nb_bkts  : %"PRIu32"", timr->nb_bkts);
 281                         timvf_log_dbg("tck_nsec : %"PRIu64"", timr->tck_nsec);
 282                 } else
 283                         timvf_log_info("Failed to Optimize configured values");
 284         }
 285
 286         if (rcfg->flags & RTE_EVENT_TIMER_ADAPTER_F_SP_PUT) {
 287                 mp_flags = MEMPOOL_F_SP_PUT | MEMPOOL_F_SC_GET;
 288                 timvf_log_info("Using single producer mode");
 289         }
 290
 291         timr->bkt = rte_zmalloc("octeontx_timvf_bucket",
 292                         (timr->nb_bkts) * sizeof(struct tim_mem_bucket),
 293                         0);
 294         if (timr->bkt == NULL)
 295                 goto mem_err;
 296
 297         snprintf(pool_name, sizeof(pool_name), "timvf_chunk_pool%d",
 298                         timr->tim_ring_id);
 299         timr->chunk_pool = (void *)rte_mempool_create_empty(pool_name,
 300                         timr->nb_chunks, TIM_CHUNK_SIZE, 0, 0, rte_socket_id(),
 301                         mp_flags);
 302
 303         if (!timr->chunk_pool) {
 304                 rte_free(timr->bkt);
 305                 timvf_log_err("Unable to create chunkpool.");
 306                 return -ENOMEM;
 307         }
 308
 309         mempool_ops = rte_mbuf_best_mempool_ops();
 310         ret = rte_mempool_set_ops_byname(timr->chunk_pool,
 311                         mempool_ops, NULL);
 312
 313         if (ret != 0) {
 314                 timvf_log_err("Unable to set chunkpool ops.");
 315                 goto mem_err;
 316         }
 317
 318         ret = rte_mempool_populate_default(timr->chunk_pool);
 319         if (ret < 0) {
 320                 timvf_log_err("Unable to set populate chunkpool.");
 321                 goto mem_err;
 322         }
 323         timvf_write64(0, (uint8_t *)timr->vbar0 + TIM_VRING_BASE);
 324         timvf_write64(0, (uint8_t *)timr->vbar0 + TIM_VF_NRSPERR_INT);
 325         timvf_write64(0, (uint8_t *)timr->vbar0 + TIM_VF_NRSPERR_INT_W1S);
 326         timvf_write64(0x7, (uint8_t *)timr->vbar0 + TIM_VF_NRSPERR_ENA_W1C);
 327         timvf_write64(0x7, (uint8_t *)timr->vbar0 + TIM_VF_NRSPERR_ENA_W1S);
 328
 329         return 0;
 330 mem_err:
 331         rte_free(timr);
 332         return -ENOMEM;
 333 cfg_err:
 334         rte_free(timr);
 335         return -EINVAL;
 336 }
 337
 338 static int
 339 timvf_ring_free(struct rte_event_timer_adapter *adptr)
 340 {
 341         struct timvf_ring *timr = adptr->data->adapter_priv;
 342         rte_mempool_free(timr->chunk_pool);
 343         rte_free(timr->bkt);
 344         rte_free(adptr->data->adapter_priv);
 345         return 0;
 346 }
 347
 348 static int
 349 timvf_stats_get(const struct rte_event_timer_adapter *adapter,
 350                 struct rte_event_timer_adapter_stats *stats)
 351 {
 352         struct timvf_ring *timr = adapter->data->adapter_priv;
 353         uint64_t bkt_cyc = rte_rdtsc() - timr->ring_start_cyc;
 354
 355         stats->evtim_exp_count = timr->tim_arm_cnt;
 356         stats->ev_enq_count = timr->tim_arm_cnt;
 357         stats->adapter_tick_count = rte_reciprocal_divide_u64(bkt_cyc,
 358                                 &timr->fast_div);
 359         return 0;
 360 }
 361
 362 static int
 363 timvf_stats_reset(const struct rte_event_timer_adapter *adapter)
 364 {
 365         struct timvf_ring *timr = adapter->data->adapter_priv;
 366
 367         timr->tim_arm_cnt = 0;
 368         return 0;
 369 }
 370
 371 static struct rte_event_timer_adapter_ops timvf_ops = {
 372         .init           = timvf_ring_create,
 373         .uninit         = timvf_ring_free,
 374         .start          = timvf_ring_start,
 375         .stop           = timvf_ring_stop,
 376         .get_info       = timvf_ring_info_get,
 377 };
 378
 379 int
 380 timvf_timer_adapter_caps_get(const struct rte_eventdev *dev, uint64_t flags,
 381                 uint32_t *caps, const struct rte_event_timer_adapter_ops **ops,
 382                 uint8_t enable_stats)
 383 {
 384         RTE_SET_USED(dev);
 385
 386         if (enable_stats) {
 387                 timvf_ops.stats_get   = timvf_stats_get;
 388                 timvf_ops.stats_reset = timvf_stats_reset;
 389         }
 390
 391         if (flags & RTE_EVENT_TIMER_ADAPTER_F_SP_PUT)
 392                 timvf_ops.arm_burst = enable_stats ?
 393                         timvf_timer_arm_burst_sp_stats :
 394                         timvf_timer_arm_burst_sp;
 395         else
 396                 timvf_ops.arm_burst = enable_stats ?
 397                         timvf_timer_arm_burst_mp_stats :
 398                         timvf_timer_arm_burst_mp;
 399
 400         timvf_ops.arm_tmo_tick_burst = enable_stats ?
 401                 timvf_timer_arm_tmo_brst_stats :
 402                 timvf_timer_arm_tmo_brst;
 403         timvf_ops.cancel_burst = timvf_timer_cancel_burst;
 404         *caps = RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT;
 405         *ops = &timvf_ops;
 406         return 0;
 407 }