drivers/event/octeontx/timvf_evdev.c

   1 /*
   2  * SPDX-License-Identifier: BSD-3-Clause
   3  * Copyright(c) 2017 Cavium, Inc
   4  */
   5
   6 #include "timvf_evdev.h"
   7
   8 int otx_logtype_timvf;
   9
  10 RTE_INIT(otx_timvf_init_log);
  11 static void
  12 otx_timvf_init_log(void)
  13 {
  14         otx_logtype_timvf = rte_log_register("pmd.event.octeontx.timer");
  15         if (otx_logtype_timvf >= 0)
  16                 rte_log_set_level(otx_logtype_timvf, RTE_LOG_NOTICE);
  17 }
  18
  19 struct __rte_packed timvf_mbox_dev_info {
  20         uint64_t ring_active[4];
  21         uint64_t clk_freq;
  22 };
  23
  24 /* Response messages */
  25 enum {
  26         MBOX_RET_SUCCESS,
  27         MBOX_RET_INVALID,
  28         MBOX_RET_INTERNAL_ERR,
  29 };
  30
  31 static int
  32 timvf_mbox_dev_info_get(struct timvf_mbox_dev_info *info)
  33 {
  34         struct octeontx_mbox_hdr hdr = {0};
  35         uint16_t len = sizeof(struct timvf_mbox_dev_info);
  36
  37         hdr.coproc = TIM_COPROC;
  38         hdr.msg = TIM_GET_DEV_INFO;
  39         hdr.vfid = 0; /* TIM DEV is always 0. TIM RING ID changes. */
  40
  41         memset(info, 0, len);
  42         return octeontx_mbox_send(&hdr, NULL, 0, info, len);
  43 }
  44
  45 static void
  46 timvf_ring_info_get(const struct rte_event_timer_adapter *adptr,
  47                 struct rte_event_timer_adapter_info *adptr_info)
  48 {
  49         struct timvf_ring *timr = adptr->data->adapter_priv;
  50         adptr_info->max_tmo_ns = timr->max_tout;
  51         adptr_info->min_resolution_ns = timr->tck_nsec;
  52         rte_memcpy(&adptr_info->conf, &adptr->data->conf,
  53                         sizeof(struct rte_event_timer_adapter_conf));
  54 }
  55
  56 static int
  57 timvf_ring_conf_set(struct timvf_ctrl_reg *rctl, uint8_t ring_id)
  58 {
  59         struct octeontx_mbox_hdr hdr = {0};
  60         uint16_t len = sizeof(struct timvf_ctrl_reg);
  61         int ret;
  62
  63         hdr.coproc = TIM_COPROC;
  64         hdr.msg = TIM_SET_RING_INFO;
  65         hdr.vfid = ring_id;
  66
  67         ret = octeontx_mbox_send(&hdr, rctl, len, NULL, 0);
  68         if (ret < 0 || hdr.res_code != MBOX_RET_SUCCESS)
  69                 return -EACCES;
  70         return 0;
  71 }
  72
  73 static int
  74 timvf_get_start_cyc(uint64_t *now, uint8_t ring_id)
  75 {
  76         struct octeontx_mbox_hdr hdr = {0};
  77
  78         hdr.coproc = TIM_COPROC;
  79         hdr.msg = TIM_RING_START_CYC_GET;
  80         hdr.vfid = ring_id;
  81         *now = 0;
  82         return octeontx_mbox_send(&hdr, NULL, 0, now, sizeof(uint64_t));
  83 }
  84
  85 static int
  86 optimize_bucket_parameters(struct timvf_ring *timr)
  87 {
  88         uint32_t hbkts;
  89         uint32_t lbkts;
  90         uint64_t tck_nsec;
  91
  92         hbkts = rte_align32pow2(timr->nb_bkts);
  93         tck_nsec = RTE_ALIGN_MUL_CEIL(timr->max_tout / (hbkts - 1), 10);
  94
  95         if ((tck_nsec < 1000 || hbkts > TIM_MAX_BUCKETS))
  96                 hbkts = 0;
  97
  98         lbkts = rte_align32prevpow2(timr->nb_bkts);
  99         tck_nsec = RTE_ALIGN_MUL_CEIL((timr->max_tout / (lbkts - 1)), 10);
 100
 101         if ((tck_nsec < 1000 || hbkts > TIM_MAX_BUCKETS))
 102                 lbkts = 0;
 103
 104         if (!hbkts && !lbkts)
 105                 return 0;
 106
 107         if (!hbkts) {
 108                 timr->nb_bkts = lbkts;
 109                 goto end;
 110         } else if (!lbkts) {
 111                 timr->nb_bkts = hbkts;
 112                 goto end;
 113         }
 114
 115         timr->nb_bkts = (hbkts - timr->nb_bkts) <
 116                 (timr->nb_bkts - lbkts) ? hbkts : lbkts;
 117 end:
 118         timr->get_target_bkt = bkt_and;
 119         timr->tck_nsec = RTE_ALIGN_MUL_CEIL((timr->max_tout /
 120                                 (timr->nb_bkts - 1)), 10);
 121         return 1;
 122 }
 123
 124 static int
 125 timvf_ring_start(const struct rte_event_timer_adapter *adptr)
 126 {
 127         int ret;
 128         uint64_t interval;
 129         struct timvf_ctrl_reg rctrl;
 130         struct timvf_mbox_dev_info dinfo;
 131         struct timvf_ring *timr = adptr->data->adapter_priv;
 132
 133         ret = timvf_mbox_dev_info_get(&dinfo);
 134         if (ret < 0 || ret != sizeof(struct timvf_mbox_dev_info))
 135                 return -EINVAL;
 136
 137         /* Calculate the interval cycles according to clock source. */
 138         switch (timr->clk_src) {
 139         case TIM_CLK_SRC_SCLK:
 140                 interval = NSEC2CLK(timr->tck_nsec, dinfo.clk_freq);
 141                 break;
 142         case TIM_CLK_SRC_GPIO:
 143                 /* GPIO doesn't work on tck_nsec. */
 144                 interval = 0;
 145                 break;
 146         case TIM_CLK_SRC_GTI:
 147                 interval = NSEC2CLK(timr->tck_nsec, dinfo.clk_freq);
 148                 break;
 149         case TIM_CLK_SRC_PTP:
 150                 interval = NSEC2CLK(timr->tck_nsec, dinfo.clk_freq);
 151                 break;
 152         default:
 153                 timvf_log_err("Unsupported clock source configured %d",
 154                                 timr->clk_src);
 155                 return -EINVAL;
 156         }
 157
 158         /*CTRL0 register.*/
 159         rctrl.rctrl0 = interval;
 160
 161         /*CTRL1 register.*/
 162         rctrl.rctrl1 =  (uint64_t)(timr->clk_src) << 51 |
 163                 1ull << 48 /* LOCK_EN (Enable hw bucket lock mechanism) */ |
 164                 1ull << 47 /* ENA */ |
 165                 1ull << 44 /* ENA_LDWB */ |
 166                 (timr->nb_bkts - 1);
 167
 168         rctrl.rctrl2 = (uint64_t)(TIM_CHUNK_SIZE / 16) << 40;
 169
 170         timvf_write64((uintptr_t)timr->bkt,
 171                         (uint8_t *)timr->vbar0 + TIM_VRING_BASE);
 172         timvf_set_chunk_refill(timr);
 173         if (timvf_ring_conf_set(&rctrl, timr->tim_ring_id)) {
 174                 ret = -EACCES;
 175                 goto error;
 176         }
 177
 178         if (timvf_get_start_cyc(&timr->ring_start_cyc,
 179                                 timr->tim_ring_id) < 0) {
 180                 ret = -EACCES;
 181                 goto error;
 182         }
 183         timr->tck_int = NSEC2CLK(timr->tck_nsec, rte_get_timer_hz());
 184         timr->fast_div = rte_reciprocal_value_u64(timr->tck_int);
 185         timvf_log_info("nb_bkts %d min_ns %"PRIu64" min_cyc %"PRIu64""
 186                         " maxtmo %"PRIu64"\n",
 187                         timr->nb_bkts, timr->tck_nsec, interval,
 188                         timr->max_tout);
 189
 190         return 0;
 191 error:
 192         rte_free(timr->bkt);
 193         rte_mempool_free(timr->chunk_pool);
 194         return ret;
 195 }
 196
 197 static int
 198 timvf_ring_stop(const struct rte_event_timer_adapter *adptr)
 199 {
 200         struct timvf_ring *timr = adptr->data->adapter_priv;
 201         struct timvf_ctrl_reg rctrl = {0};
 202         rctrl.rctrl0 = timvf_read64((uint8_t *)timr->vbar0 + TIM_VRING_CTL0);
 203         rctrl.rctrl1 = timvf_read64((uint8_t *)timr->vbar0 + TIM_VRING_CTL1);
 204         rctrl.rctrl1 &= ~(1ull << 47); /* Disable */
 205         rctrl.rctrl2 = timvf_read64((uint8_t *)timr->vbar0 + TIM_VRING_CTL2);
 206
 207         if (timvf_ring_conf_set(&rctrl, timr->tim_ring_id))
 208                 return -EACCES;
 209         return 0;
 210 }
 211
 212 static int
 213 timvf_ring_create(struct rte_event_timer_adapter *adptr)
 214 {
 215         char pool_name[25];
 216         int ret;
 217         uint64_t nb_timers;
 218         struct rte_event_timer_adapter_conf *rcfg = &adptr->data->conf;
 219         struct timvf_ring *timr;
 220         struct timvf_info tinfo;
 221         const char *mempool_ops;
 222         unsigned int mp_flags = 0;
 223
 224         if (timvf_info(&tinfo) < 0)
 225                 return -ENODEV;
 226
 227         if (adptr->data->id >= tinfo.total_timvfs)
 228                 return -ENODEV;
 229
 230         timr = rte_zmalloc("octeontx_timvf_priv",
 231                         sizeof(struct timvf_ring), 0);
 232         if (timr == NULL)
 233                 return -ENOMEM;
 234
 235         adptr->data->adapter_priv = timr;
 236         /* Check config parameters. */
 237         if ((rcfg->clk_src != RTE_EVENT_TIMER_ADAPTER_CPU_CLK) &&
 238                         (!rcfg->timer_tick_ns ||
 239                          rcfg->timer_tick_ns < TIM_MIN_INTERVAL)) {
 240                 timvf_log_err("Too low timer ticks");
 241                 goto cfg_err;
 242         }
 243
 244         timr->clk_src = (int) rcfg->clk_src;
 245         timr->tim_ring_id = adptr->data->id;
 246         timr->tck_nsec = RTE_ALIGN_MUL_CEIL(rcfg->timer_tick_ns, 10);
 247         timr->max_tout = rcfg->max_tmo_ns;
 248         timr->nb_bkts = (timr->max_tout / timr->tck_nsec);
 249         timr->vbar0 = timvf_bar(timr->tim_ring_id, 0);
 250         timr->bkt_pos = (uint8_t *)timr->vbar0 + TIM_VRING_REL;
 251         nb_timers = rcfg->nb_timers;
 252         timr->get_target_bkt = bkt_mod;
 253
 254         timr->nb_chunks = nb_timers / nb_chunk_slots;
 255
 256         /* Try to optimize the bucket parameters. */
 257         if ((rcfg->flags & RTE_EVENT_TIMER_ADAPTER_F_ADJUST_RES)
 258                         && !rte_is_power_of_2(timr->nb_bkts)) {
 259                 if (optimize_bucket_parameters(timr)) {
 260                         timvf_log_info("Optimized configured values");
 261                         timvf_log_dbg("nb_bkts  : %"PRIu32"", timr->nb_bkts);
 262                         timvf_log_dbg("tck_nsec : %"PRIu64"", timr->tck_nsec);
 263                 } else
 264                         timvf_log_info("Failed to Optimize configured values");
 265         }
 266
 267         if (rcfg->flags & RTE_EVENT_TIMER_ADAPTER_F_SP_PUT) {
 268                 mp_flags = MEMPOOL_F_SP_PUT | MEMPOOL_F_SC_GET;
 269                 timvf_log_info("Using single producer mode");
 270         }
 271
 272         timr->bkt = rte_zmalloc("octeontx_timvf_bucket",
 273                         (timr->nb_bkts) * sizeof(struct tim_mem_bucket),
 274                         0);
 275         if (timr->bkt == NULL)
 276                 goto mem_err;
 277
 278         snprintf(pool_name, 30, "timvf_chunk_pool%d", timr->tim_ring_id);
 279         timr->chunk_pool = (void *)rte_mempool_create_empty(pool_name,
 280                         timr->nb_chunks, TIM_CHUNK_SIZE, 0, 0, rte_socket_id(),
 281                         mp_flags);
 282
 283         if (!timr->chunk_pool) {
 284                 rte_free(timr->bkt);
 285                 timvf_log_err("Unable to create chunkpool.");
 286                 return -ENOMEM;
 287         }
 288
 289         mempool_ops = rte_mbuf_best_mempool_ops();
 290         ret = rte_mempool_set_ops_byname(timr->chunk_pool,
 291                         mempool_ops, NULL);
 292
 293         if (ret != 0) {
 294                 timvf_log_err("Unable to set chunkpool ops.");
 295                 goto mem_err;
 296         }
 297
 298         ret = rte_mempool_populate_default(timr->chunk_pool);
 299         if (ret < 0) {
 300                 timvf_log_err("Unable to set populate chunkpool.");
 301                 goto mem_err;
 302         }
 303         timvf_write64(0, (uint8_t *)timr->vbar0 + TIM_VRING_BASE);
 304         timvf_write64(0, (uint8_t *)timr->vbar0 + TIM_VF_NRSPERR_INT);
 305         timvf_write64(0, (uint8_t *)timr->vbar0 + TIM_VF_NRSPERR_INT_W1S);
 306         timvf_write64(0x7, (uint8_t *)timr->vbar0 + TIM_VF_NRSPERR_ENA_W1C);
 307         timvf_write64(0x7, (uint8_t *)timr->vbar0 + TIM_VF_NRSPERR_ENA_W1S);
 308
 309         return 0;
 310 mem_err:
 311         rte_free(timr);
 312         return -ENOMEM;
 313 cfg_err:
 314         rte_free(timr);
 315         return -EINVAL;
 316 }
 317
 318 static int
 319 timvf_ring_free(struct rte_event_timer_adapter *adptr)
 320 {
 321         struct timvf_ring *timr = adptr->data->adapter_priv;
 322         rte_mempool_free(timr->chunk_pool);
 323         rte_free(timr->bkt);
 324         rte_free(adptr->data->adapter_priv);
 325         return 0;
 326 }
 327
 328 static int
 329 timvf_stats_get(const struct rte_event_timer_adapter *adapter,
 330                 struct rte_event_timer_adapter_stats *stats)
 331 {
 332         struct timvf_ring *timr = adapter->data->adapter_priv;
 333         uint64_t bkt_cyc = rte_rdtsc() - timr->ring_start_cyc;
 334
 335         stats->evtim_exp_count = timr->tim_arm_cnt;
 336         stats->ev_enq_count = timr->tim_arm_cnt;
 337         stats->adapter_tick_count = rte_reciprocal_divide_u64(bkt_cyc,
 338                                 &timr->fast_div);
 339         return 0;
 340 }
 341
 342 static int
 343 timvf_stats_reset(const struct rte_event_timer_adapter *adapter)
 344 {
 345         struct timvf_ring *timr = adapter->data->adapter_priv;
 346
 347         timr->tim_arm_cnt = 0;
 348         return 0;
 349 }
 350
 351 static struct rte_event_timer_adapter_ops timvf_ops = {
 352         .init           = timvf_ring_create,
 353         .uninit         = timvf_ring_free,
 354         .start          = timvf_ring_start,
 355         .stop           = timvf_ring_stop,
 356         .get_info       = timvf_ring_info_get,
 357 };
 358
 359 int
 360 timvf_timer_adapter_caps_get(const struct rte_eventdev *dev, uint64_t flags,
 361                 uint32_t *caps, const struct rte_event_timer_adapter_ops **ops,
 362                 uint8_t enable_stats)
 363 {
 364         RTE_SET_USED(dev);
 365
 366         if (enable_stats) {
 367                 timvf_ops.stats_get   = timvf_stats_get;
 368                 timvf_ops.stats_reset = timvf_stats_reset;
 369         }
 370
 371         if (flags & RTE_EVENT_TIMER_ADAPTER_F_SP_PUT)
 372                 timvf_ops.arm_burst = enable_stats ?
 373                         timvf_timer_arm_burst_sp_stats :
 374                         timvf_timer_arm_burst_sp;
 375         else
 376                 timvf_ops.arm_burst = enable_stats ?
 377                         timvf_timer_arm_burst_mp_stats :
 378                         timvf_timer_arm_burst_mp;
 379
 380         timvf_ops.arm_tmo_tick_burst = enable_stats ?
 381                 timvf_timer_arm_tmo_brst_stats :
 382                 timvf_timer_arm_tmo_brst;
 383         timvf_ops.cancel_burst = timvf_timer_cancel_burst;
 384         *caps = RTE_EVENT_TIMER_ADAPTER_CAP_INTERNAL_PORT;
 385         *ops = &timvf_ops;
 386         return 0;
 387 }