drivers/event/octeontx2/otx2_evdev.h

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright(C) 2019 Marvell International Ltd.
   3  */
   4
   5 #ifndef __OTX2_EVDEV_H__
   6 #define __OTX2_EVDEV_H__
   7
   8 #include <rte_eventdev.h>
   9 #include <eventdev_pmd.h>
  10 #include <rte_event_eth_rx_adapter.h>
  11 #include <rte_event_eth_tx_adapter.h>
  12
  13 #include "otx2_common.h"
  14 #include "otx2_dev.h"
  15 #include "otx2_ethdev.h"
  16 #include "otx2_mempool.h"
  17 #include "otx2_tim_evdev.h"
  18
  19 #define EVENTDEV_NAME_OCTEONTX2_PMD event_octeontx2
  20
  21 #define sso_func_trace otx2_sso_dbg
  22
  23 #define OTX2_SSO_MAX_VHGRP                  RTE_EVENT_MAX_QUEUES_PER_DEV
  24 #define OTX2_SSO_MAX_VHWS                   (UINT8_MAX)
  25 #define OTX2_SSO_FC_NAME                    "otx2_evdev_xaq_fc"
  26 #define OTX2_SSO_SQB_LIMIT                  (0x180)
  27 #define OTX2_SSO_XAQ_SLACK                  (8)
  28 #define OTX2_SSO_XAQ_CACHE_CNT              (0x7)
  29 #define OTX2_SSO_WQE_SG_PTR                 (9)
  30
  31 /* SSO LF register offsets (BAR2) */
  32 #define SSO_LF_GGRP_OP_ADD_WORK0            (0x0ull)
  33 #define SSO_LF_GGRP_OP_ADD_WORK1            (0x8ull)
  34
  35 #define SSO_LF_GGRP_QCTL                    (0x20ull)
  36 #define SSO_LF_GGRP_EXE_DIS                 (0x80ull)
  37 #define SSO_LF_GGRP_INT                     (0x100ull)
  38 #define SSO_LF_GGRP_INT_W1S                 (0x108ull)
  39 #define SSO_LF_GGRP_INT_ENA_W1S             (0x110ull)
  40 #define SSO_LF_GGRP_INT_ENA_W1C             (0x118ull)
  41 #define SSO_LF_GGRP_INT_THR                 (0x140ull)
  42 #define SSO_LF_GGRP_INT_CNT                 (0x180ull)
  43 #define SSO_LF_GGRP_XAQ_CNT                 (0x1b0ull)
  44 #define SSO_LF_GGRP_AQ_CNT                  (0x1c0ull)
  45 #define SSO_LF_GGRP_AQ_THR                  (0x1e0ull)
  46 #define SSO_LF_GGRP_MISC_CNT                (0x200ull)
  47
  48 /* SSOW LF register offsets (BAR2) */
  49 #define SSOW_LF_GWS_LINKS                   (0x10ull)
  50 #define SSOW_LF_GWS_PENDWQP                 (0x40ull)
  51 #define SSOW_LF_GWS_PENDSTATE               (0x50ull)
  52 #define SSOW_LF_GWS_NW_TIM                  (0x70ull)
  53 #define SSOW_LF_GWS_GRPMSK_CHG              (0x80ull)
  54 #define SSOW_LF_GWS_INT                     (0x100ull)
  55 #define SSOW_LF_GWS_INT_W1S                 (0x108ull)
  56 #define SSOW_LF_GWS_INT_ENA_W1S             (0x110ull)
  57 #define SSOW_LF_GWS_INT_ENA_W1C             (0x118ull)
  58 #define SSOW_LF_GWS_TAG                     (0x200ull)
  59 #define SSOW_LF_GWS_WQP                     (0x210ull)
  60 #define SSOW_LF_GWS_SWTP                    (0x220ull)
  61 #define SSOW_LF_GWS_PENDTAG                 (0x230ull)
  62 #define SSOW_LF_GWS_OP_ALLOC_WE             (0x400ull)
  63 #define SSOW_LF_GWS_OP_GET_WORK             (0x600ull)
  64 #define SSOW_LF_GWS_OP_SWTAG_FLUSH          (0x800ull)
  65 #define SSOW_LF_GWS_OP_SWTAG_UNTAG          (0x810ull)
  66 #define SSOW_LF_GWS_OP_SWTP_CLR             (0x820ull)
  67 #define SSOW_LF_GWS_OP_UPD_WQP_GRP0         (0x830ull)
  68 #define SSOW_LF_GWS_OP_UPD_WQP_GRP1         (0x838ull)
  69 #define SSOW_LF_GWS_OP_DESCHED              (0x880ull)
  70 #define SSOW_LF_GWS_OP_DESCHED_NOSCH        (0x8c0ull)
  71 #define SSOW_LF_GWS_OP_SWTAG_DESCHED        (0x980ull)
  72 #define SSOW_LF_GWS_OP_SWTAG_NOSCHED        (0x9c0ull)
  73 #define SSOW_LF_GWS_OP_CLR_NSCHED0          (0xa00ull)
  74 #define SSOW_LF_GWS_OP_CLR_NSCHED1          (0xa08ull)
  75 #define SSOW_LF_GWS_OP_SWTP_SET             (0xc00ull)
  76 #define SSOW_LF_GWS_OP_SWTAG_NORM           (0xc10ull)
  77 #define SSOW_LF_GWS_OP_SWTAG_FULL0          (0xc20ull)
  78 #define SSOW_LF_GWS_OP_SWTAG_FULL1          (0xc28ull)
  79 #define SSOW_LF_GWS_OP_GWC_INVAL            (0xe00ull)
  80
  81 #define OTX2_SSOW_GET_BASE_ADDR(_GW)        ((_GW) - SSOW_LF_GWS_OP_GET_WORK)
  82 #define OTX2_SSOW_TT_FROM_TAG(x)            (((x) >> 32) & SSO_TT_EMPTY)
  83 #define OTX2_SSOW_GRP_FROM_TAG(x)           (((x) >> 36) & 0x3ff)
  84
  85 #define NSEC2USEC(__ns)                 ((__ns) / 1E3)
  86 #define USEC2NSEC(__us)                 ((__us) * 1E3)
  87 #define NSEC2TICK(__ns, __freq)         (((__ns) * (__freq)) / 1E9)
  88 #define TICK2NSEC(__tck, __freq)        (((__tck) * 1E9) / (__freq))
  89
  90 enum otx2_sso_lf_type {
  91         SSO_LF_GGRP,
  92         SSO_LF_GWS
  93 };
  94
  95 union otx2_sso_event {
  96         uint64_t get_work0;
  97         struct {
  98                 uint32_t flow_id:20;
  99                 uint32_t sub_event_type:8;
 100                 uint32_t event_type:4;
 101                 uint8_t op:2;
 102                 uint8_t rsvd:4;
 103                 uint8_t sched_type:2;
 104                 uint8_t queue_id;
 105                 uint8_t priority;
 106                 uint8_t impl_opaque;
 107         };
 108 } __rte_aligned(64);
 109
 110 enum {
 111         SSO_SYNC_ORDERED,
 112         SSO_SYNC_ATOMIC,
 113         SSO_SYNC_UNTAGGED,
 114         SSO_SYNC_EMPTY
 115 };
 116
 117 struct otx2_sso_qos {
 118         uint8_t queue;
 119         uint8_t xaq_prcnt;
 120         uint8_t taq_prcnt;
 121         uint8_t iaq_prcnt;
 122 };
 123
 124 struct otx2_sso_evdev {
 125         OTX2_DEV; /* Base class */
 126         uint8_t max_event_queues;
 127         uint8_t max_event_ports;
 128         uint8_t is_timeout_deq;
 129         uint8_t nb_event_queues;
 130         uint8_t nb_event_ports;
 131         uint8_t configured;
 132         uint32_t deq_tmo_ns;
 133         uint32_t min_dequeue_timeout_ns;
 134         uint32_t max_dequeue_timeout_ns;
 135         int32_t max_num_events;
 136         uint64_t *fc_mem;
 137         uint64_t xaq_lmt;
 138         uint64_t nb_xaq_cfg;
 139         rte_iova_t fc_iova;
 140         struct rte_mempool *xaq_pool;
 141         uint64_t rx_offloads;
 142         uint64_t tx_offloads;
 143         uint64_t adptr_xae_cnt;
 144         uint16_t rx_adptr_pool_cnt;
 145         uint64_t *rx_adptr_pools;
 146         uint16_t max_port_id;
 147         uint16_t tim_adptr_ring_cnt;
 148         uint16_t *timer_adptr_rings;
 149         uint64_t *timer_adptr_sz;
 150         /* Dev args */
 151         uint8_t dual_ws;
 152         uint32_t xae_cnt;
 153         uint8_t qos_queue_cnt;
 154         uint8_t force_rx_bp;
 155         struct otx2_sso_qos *qos_parse_data;
 156         /* HW const */
 157         uint32_t xae_waes;
 158         uint32_t xaq_buf_size;
 159         uint32_t iue;
 160         /* MSIX offsets */
 161         uint16_t sso_msixoff[OTX2_SSO_MAX_VHGRP];
 162         uint16_t ssow_msixoff[OTX2_SSO_MAX_VHWS];
 163         /* PTP timestamp */
 164         struct otx2_timesync_info *tstamp;
 165 } __rte_cache_aligned;
 166
 167 #define OTX2_SSOGWS_OPS                                                        \
 168         /* WS ops */                                                           \
 169         uintptr_t getwrk_op;                                                   \
 170         uintptr_t tag_op;                                                      \
 171         uintptr_t wqp_op;                                                      \
 172         uintptr_t swtag_flush_op;                                              \
 173         uintptr_t swtag_norm_op;                                               \
 174         uintptr_t swtag_desched_op;
 175
 176 /* Event port aka GWS */
 177 struct otx2_ssogws {
 178         /* Get Work Fastpath data */
 179         OTX2_SSOGWS_OPS;
 180         /* PTP timestamp */
 181         struct otx2_timesync_info *tstamp;
 182         void *lookup_mem;
 183         uint8_t swtag_req;
 184         uint8_t port;
 185         /* Add Work Fastpath data */
 186         uint64_t xaq_lmt __rte_cache_aligned;
 187         uint64_t *fc_mem;
 188         uintptr_t grps_base[OTX2_SSO_MAX_VHGRP];
 189         /* Tx Fastpath data */
 190         uint64_t base __rte_cache_aligned;
 191         uint8_t tx_adptr_data[];
 192 } __rte_cache_aligned;
 193
 194 struct otx2_ssogws_state {
 195         OTX2_SSOGWS_OPS;
 196 };
 197
 198 struct otx2_ssogws_dual {
 199         /* Get Work Fastpath data */
 200         struct otx2_ssogws_state ws_state[2]; /* Ping and Pong */
 201         /* PTP timestamp */
 202         struct otx2_timesync_info *tstamp;
 203         void *lookup_mem;
 204         uint8_t swtag_req;
 205         uint8_t vws; /* Ping pong bit */
 206         uint8_t port;
 207         /* Add Work Fastpath data */
 208         uint64_t xaq_lmt __rte_cache_aligned;
 209         uint64_t *fc_mem;
 210         uintptr_t grps_base[OTX2_SSO_MAX_VHGRP];
 211         /* Tx Fastpath data */
 212         uint64_t base[2] __rte_cache_aligned;
 213         uint8_t tx_adptr_data[];
 214 } __rte_cache_aligned;
 215
 216 static inline struct otx2_sso_evdev *
 217 sso_pmd_priv(const struct rte_eventdev *event_dev)
 218 {
 219         return event_dev->data->dev_private;
 220 }
 221
 222 struct otx2_ssogws_cookie {
 223         const struct rte_eventdev *event_dev;
 224         bool configured;
 225 };
 226
 227 static inline struct otx2_ssogws_cookie *
 228 ssogws_get_cookie(void *ws)
 229 {
 230         return (struct otx2_ssogws_cookie *)
 231                 ((uint8_t *)ws - RTE_CACHE_LINE_SIZE);
 232 }
 233
 234 static const union mbuf_initializer mbuf_init = {
 235         .fields = {
 236                 .data_off = RTE_PKTMBUF_HEADROOM,
 237                 .refcnt = 1,
 238                 .nb_segs = 1,
 239                 .port = 0
 240         }
 241 };
 242
 243 static __rte_always_inline void
 244 otx2_wqe_to_mbuf(uint64_t get_work1, const uint64_t mbuf, uint8_t port_id,
 245                  const uint32_t tag, const uint32_t flags,
 246                  const void * const lookup_mem)
 247 {
 248         struct nix_wqe_hdr_s *wqe = (struct nix_wqe_hdr_s *)get_work1;
 249         uint64_t val = mbuf_init.value | (uint64_t)port_id << 48;
 250
 251         if (flags & NIX_RX_OFFLOAD_TSTAMP_F)
 252                 val |= NIX_TIMESYNC_RX_OFFSET;
 253
 254         otx2_nix_cqe_to_mbuf((struct nix_cqe_hdr_s *)wqe, tag,
 255                              (struct rte_mbuf *)mbuf, lookup_mem,
 256                               val, flags);
 257
 258 }
 259
 260 static inline int
 261 parse_kvargs_flag(const char *key, const char *value, void *opaque)
 262 {
 263         RTE_SET_USED(key);
 264
 265         *(uint8_t *)opaque = !!atoi(value);
 266         return 0;
 267 }
 268
 269 static inline int
 270 parse_kvargs_value(const char *key, const char *value, void *opaque)
 271 {
 272         RTE_SET_USED(key);
 273
 274         *(uint32_t *)opaque = (uint32_t)atoi(value);
 275         return 0;
 276 }
 277
 278 #define SSO_RX_ADPTR_ENQ_FASTPATH_FUNC  NIX_RX_FASTPATH_MODES
 279 #define SSO_TX_ADPTR_ENQ_FASTPATH_FUNC  NIX_TX_FASTPATH_MODES
 280
 281 /* Single WS API's */
 282 uint16_t otx2_ssogws_enq(void *port, const struct rte_event *ev);
 283 uint16_t otx2_ssogws_enq_burst(void *port, const struct rte_event ev[],
 284                                uint16_t nb_events);
 285 uint16_t otx2_ssogws_enq_new_burst(void *port, const struct rte_event ev[],
 286                                    uint16_t nb_events);
 287 uint16_t otx2_ssogws_enq_fwd_burst(void *port, const struct rte_event ev[],
 288                                    uint16_t nb_events);
 289
 290 /* Dual WS API's */
 291 uint16_t otx2_ssogws_dual_enq(void *port, const struct rte_event *ev);
 292 uint16_t otx2_ssogws_dual_enq_burst(void *port, const struct rte_event ev[],
 293                                     uint16_t nb_events);
 294 uint16_t otx2_ssogws_dual_enq_new_burst(void *port, const struct rte_event ev[],
 295                                         uint16_t nb_events);
 296 uint16_t otx2_ssogws_dual_enq_fwd_burst(void *port, const struct rte_event ev[],
 297                                         uint16_t nb_events);
 298
 299 /* Auto generated API's */
 300 #define R(name, f6, f5, f4, f3, f2, f1, f0, flags)                             \
 301 uint16_t otx2_ssogws_deq_ ##name(void *port, struct rte_event *ev,             \
 302                                  uint64_t timeout_ticks);                      \
 303 uint16_t otx2_ssogws_deq_burst_ ##name(void *port, struct rte_event ev[],      \
 304                                        uint16_t nb_events,                     \
 305                                        uint64_t timeout_ticks);                \
 306 uint16_t otx2_ssogws_deq_timeout_ ##name(void *port,                           \
 307                                          struct rte_event *ev,                 \
 308                                          uint64_t timeout_ticks);              \
 309 uint16_t otx2_ssogws_deq_timeout_burst_ ##name(void *port,                     \
 310                                                struct rte_event ev[],          \
 311                                                uint16_t nb_events,             \
 312                                                uint64_t timeout_ticks);        \
 313 uint16_t otx2_ssogws_deq_seg_ ##name(void *port, struct rte_event *ev,         \
 314                                      uint64_t timeout_ticks);                  \
 315 uint16_t otx2_ssogws_deq_seg_burst_ ##name(void *port,                         \
 316                                            struct rte_event ev[],              \
 317                                            uint16_t nb_events,                 \
 318                                            uint64_t timeout_ticks);            \
 319 uint16_t otx2_ssogws_deq_seg_timeout_ ##name(void *port,                       \
 320                                              struct rte_event *ev,             \
 321                                              uint64_t timeout_ticks);          \
 322 uint16_t otx2_ssogws_deq_seg_timeout_burst_ ##name(void *port,                 \
 323                                                    struct rte_event ev[],      \
 324                                                    uint16_t nb_events,         \
 325                                                    uint64_t timeout_ticks);    \
 326                                                                                \
 327 uint16_t otx2_ssogws_dual_deq_ ##name(void *port, struct rte_event *ev,        \
 328                                       uint64_t timeout_ticks);                 \
 329 uint16_t otx2_ssogws_dual_deq_burst_ ##name(void *port,                        \
 330                                             struct rte_event ev[],             \
 331                                             uint16_t nb_events,                \
 332                                             uint64_t timeout_ticks);           \
 333 uint16_t otx2_ssogws_dual_deq_timeout_ ##name(void *port,                      \
 334                                               struct rte_event *ev,            \
 335                                               uint64_t timeout_ticks);         \
 336 uint16_t otx2_ssogws_dual_deq_timeout_burst_ ##name(void *port,                \
 337                                                     struct rte_event ev[],     \
 338                                                     uint16_t nb_events,        \
 339                                                     uint64_t timeout_ticks);   \
 340 uint16_t otx2_ssogws_dual_deq_seg_ ##name(void *port, struct rte_event *ev,    \
 341                                           uint64_t timeout_ticks);             \
 342 uint16_t otx2_ssogws_dual_deq_seg_burst_ ##name(void *port,                    \
 343                                                 struct rte_event ev[],         \
 344                                                 uint16_t nb_events,            \
 345                                                 uint64_t timeout_ticks);       \
 346 uint16_t otx2_ssogws_dual_deq_seg_timeout_ ##name(void *port,                  \
 347                                                   struct rte_event *ev,        \
 348                                                   uint64_t timeout_ticks);     \
 349 uint16_t otx2_ssogws_dual_deq_seg_timeout_burst_ ##name(void *port,            \
 350                                                         struct rte_event ev[], \
 351                                                         uint16_t nb_events,    \
 352                                                        uint64_t timeout_ticks);\
 353
 354 SSO_RX_ADPTR_ENQ_FASTPATH_FUNC
 355 #undef R
 356
 357 #define T(name, f6, f5, f4, f3, f2, f1, f0, sz, flags)                       \
 358 uint16_t otx2_ssogws_tx_adptr_enq_ ## name(void *port, struct rte_event ev[],\
 359                                            uint16_t nb_events);              \
 360 uint16_t otx2_ssogws_tx_adptr_enq_seg_ ## name(void *port,                   \
 361                                                struct rte_event ev[],        \
 362                                                uint16_t nb_events);          \
 363 uint16_t otx2_ssogws_dual_tx_adptr_enq_ ## name(void *port,                  \
 364                                                 struct rte_event ev[],       \
 365                                                 uint16_t nb_events);         \
 366 uint16_t otx2_ssogws_dual_tx_adptr_enq_seg_ ## name(void *port,              \
 367                                                     struct rte_event ev[],   \
 368                                                     uint16_t nb_events);     \
 369
 370 SSO_TX_ADPTR_ENQ_FASTPATH_FUNC
 371 #undef T
 372
 373 void sso_updt_xae_cnt(struct otx2_sso_evdev *dev, void *data,
 374                       uint32_t event_type);
 375 int sso_xae_reconfigure(struct rte_eventdev *event_dev);
 376 void sso_fastpath_fns_set(struct rte_eventdev *event_dev);
 377
 378 int otx2_sso_rx_adapter_caps_get(const struct rte_eventdev *event_dev,
 379                                  const struct rte_eth_dev *eth_dev,
 380                                  uint32_t *caps);
 381 int otx2_sso_rx_adapter_queue_add(const struct rte_eventdev *event_dev,
 382                                   const struct rte_eth_dev *eth_dev,
 383                                   int32_t rx_queue_id,
 384                 const struct rte_event_eth_rx_adapter_queue_conf *queue_conf);
 385 int otx2_sso_rx_adapter_queue_del(const struct rte_eventdev *event_dev,
 386                                   const struct rte_eth_dev *eth_dev,
 387                                   int32_t rx_queue_id);
 388 int otx2_sso_rx_adapter_start(const struct rte_eventdev *event_dev,
 389                               const struct rte_eth_dev *eth_dev);
 390 int otx2_sso_rx_adapter_stop(const struct rte_eventdev *event_dev,
 391                              const struct rte_eth_dev *eth_dev);
 392 int otx2_sso_tx_adapter_caps_get(const struct rte_eventdev *dev,
 393                                  const struct rte_eth_dev *eth_dev,
 394                                  uint32_t *caps);
 395 int otx2_sso_tx_adapter_queue_add(uint8_t id,
 396                                   const struct rte_eventdev *event_dev,
 397                                   const struct rte_eth_dev *eth_dev,
 398                                   int32_t tx_queue_id);
 399
 400 int otx2_sso_tx_adapter_queue_del(uint8_t id,
 401                                   const struct rte_eventdev *event_dev,
 402                                   const struct rte_eth_dev *eth_dev,
 403                                   int32_t tx_queue_id);
 404
 405 /* Event crypto adapter API's */
 406 int otx2_ca_caps_get(const struct rte_eventdev *dev,
 407                      const struct rte_cryptodev *cdev, uint32_t *caps);
 408
 409 int otx2_ca_qp_add(const struct rte_eventdev *dev,
 410                    const struct rte_cryptodev *cdev, int32_t queue_pair_id,
 411                    const struct rte_event *event);
 412
 413 int otx2_ca_qp_del(const struct rte_eventdev *dev,
 414                    const struct rte_cryptodev *cdev, int32_t queue_pair_id);
 415
 416 /* Clean up API's */
 417 typedef void (*otx2_handle_event_t)(void *arg, struct rte_event ev);
 418 void ssogws_flush_events(struct otx2_ssogws *ws, uint8_t queue_id,
 419                          uintptr_t base, otx2_handle_event_t fn, void *arg);
 420 void ssogws_reset(struct otx2_ssogws *ws);
 421 /* Selftest */
 422 int otx2_sso_selftest(void);
 423 /* Init and Fini API's */
 424 int otx2_sso_init(struct rte_eventdev *event_dev);
 425 int otx2_sso_fini(struct rte_eventdev *event_dev);
 426 /* IRQ handlers */
 427 int sso_register_irqs(const struct rte_eventdev *event_dev);
 428 void sso_unregister_irqs(const struct rte_eventdev *event_dev);
 429
 430 #endif /* __OTX2_EVDEV_H__ */