1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016-2020 Intel Corporation
11 #include <rte_bus_pci.h>
12 #include <rte_eventdev.h>
13 #include <rte_eventdev_pmd.h>
14 #include <rte_eventdev_pmd_pci.h>
19 #include "rte_pmd_dlb.h"
21 #ifndef RTE_LIBRTE_PMD_DLB_QUELL_STATS
22 #define DLB_INC_STAT(_stat, _incr_val) ((_stat) += _incr_val)
24 #define DLB_INC_STAT(_stat, _incr_val)
27 #define EVDEV_DLB_NAME_PMD_STR "dlb_event"
29 /* command line arg strings */
30 #define NUMA_NODE_ARG "numa_node"
31 #define DLB_MAX_NUM_EVENTS "max_num_events"
32 #define DLB_NUM_DIR_CREDITS "num_dir_credits"
33 #define DEV_ID_ARG "dev_id"
34 #define DLB_DEFER_SCHED_ARG "defer_sched"
35 #define DLB_NUM_ATM_INFLIGHTS_ARG "atm_inflights"
37 /* Begin HW related defines and structs */
39 #define DLB_MAX_NUM_DOMAINS 32
40 #define DLB_MAX_NUM_VFS 16
41 #define DLB_MAX_NUM_LDB_QUEUES 128
42 #define DLB_MAX_NUM_LDB_PORTS 64
43 #define DLB_MAX_NUM_DIR_PORTS 128
44 #define DLB_MAX_NUM_DIR_QUEUES 128
45 #define DLB_MAX_NUM_FLOWS (64 * 1024)
46 #define DLB_MAX_NUM_LDB_CREDITS 16384
47 #define DLB_MAX_NUM_DIR_CREDITS 4096
48 #define DLB_MAX_NUM_LDB_CREDIT_POOLS 64
49 #define DLB_MAX_NUM_DIR_CREDIT_POOLS 64
50 #define DLB_MAX_NUM_HIST_LIST_ENTRIES 5120
51 #define DLB_MAX_NUM_ATM_INFLIGHTS 2048
52 #define DLB_MAX_NUM_QIDS_PER_LDB_CQ 8
53 #define DLB_QID_PRIORITIES 8
54 #define DLB_MAX_DEVICE_PATH 32
55 #define DLB_MIN_DEQUEUE_TIMEOUT_NS 1
56 #define DLB_NUM_SN_GROUPS 4
57 #define DLB_MAX_LDB_SN_ALLOC 1024
58 /* Note: "- 1" here to support the timeout range check in eventdev_autotest */
59 #define DLB_MAX_DEQUEUE_TIMEOUT_NS (UINT32_MAX - 1)
60 #define DLB_DEF_UNORDERED_QID_INFLIGHTS 2048
62 /* 5120 total hist list entries and 64 total ldb ports, which
63 * makes for 5120/64 == 80 hist list entries per port. However, CQ
64 * depth must be a power of 2 and must also be >= HIST LIST entries.
65 * As a result we just limit the maximum dequeue depth to 64.
67 #define DLB_MIN_LDB_CQ_DEPTH 1
68 #define DLB_MIN_DIR_CQ_DEPTH 8
69 #define DLB_MIN_HARDWARE_CQ_DEPTH 8
70 #define DLB_MAX_CQ_DEPTH 64
71 #define DLB_NUM_HIST_LIST_ENTRIES_PER_LDB_PORT \
74 /* Static per queue/port provisioning values */
75 #define DLB_NUM_ATOMIC_INFLIGHTS_PER_QUEUE 16
77 #define PP_BASE(is_dir) ((is_dir) ? DLB_DIR_PP_BASE : DLB_LDB_PP_BASE)
79 #define PAGE_SIZE (sysconf(_SC_PAGESIZE))
81 #define DLB_NUM_QES_PER_CACHE_LINE 4
83 #define DLB_MAX_ENQUEUE_DEPTH 64
84 #define DLB_MIN_ENQUEUE_DEPTH 4
86 #define DLB_NAME_SIZE 64
88 /* Use the upper 3 bits of the event priority to select the DLB priority */
89 #define EV_TO_DLB_PRIO(x) ((x) >> 5)
90 #define DLB_TO_EV_PRIO(x) ((x) << 5)
92 enum dlb_hw_port_type {
96 /* NUM_DLB_PORT_TYPES must be last */
100 #define PORT_TYPE(p) ((p)->is_directed ? DLB_DIR : DLB_LDB)
102 /* Do not change - must match hardware! */
103 enum dlb_hw_sched_type {
104 DLB_SCHED_ATOMIC = 0,
109 /* DLB_NUM_HW_SCHED_TYPES must be last */
110 DLB_NUM_HW_SCHED_TYPES
116 int num_dir_credits_override;
119 int num_atm_inflights;
122 struct dlb_hw_rsrcs {
123 int32_t nb_events_limit;
124 uint32_t num_queues; /* Total queues (ldb + dir) */
125 uint32_t num_ldb_queues; /* Number of available ldb queues */
126 uint32_t num_ldb_ports; /* Number of load balanced ports */
127 uint32_t num_dir_ports; /* Number of directed ports */
128 uint32_t num_ldb_credits; /* Number of load balanced credits */
129 uint32_t num_dir_credits; /* Number of directed credits */
130 uint32_t reorder_window_size; /* Size of reorder window */
133 struct dlb_hw_resource_info {
134 /**> Max resources that can be provided */
135 struct dlb_hw_rsrcs hw_rsrc_max;
136 int num_sched_domains;
138 /**> EAL flags passed to this DLB instance, allowing the application to
139 * identify the pmd backend indicating hardware or software.
141 const char *eal_flags;
144 /* hw-specific format - do not change */
146 struct dlb_event_type {
152 union dlb_opaque_data {
153 uint16_t opaque_data;
154 struct dlb_event_type event_type;
157 struct dlb_msg_info {
159 uint8_t sched_type:2;
164 #define DLB_NEW_CMD_BYTE 0x08
165 #define DLB_FWD_CMD_BYTE 0x0A
166 #define DLB_COMP_CMD_BYTE 0x02
167 #define DLB_NOOP_CMD_BYTE 0x00
168 #define DLB_POP_CMD_BYTE 0x01
170 /* hw-specific format - do not change */
171 struct dlb_enqueue_qe {
174 union dlb_opaque_data u;
176 uint8_t sched_type:2;
199 /* hw-specific format - do not change */
200 struct dlb_cq_pop_qe {
202 union dlb_opaque_data u;
204 uint8_t sched_type:2;
227 /* hw-specific format - do not change */
228 struct dlb_dequeue_qe {
230 union dlb_opaque_data u;
232 uint8_t sched_type:2;
245 enum dlb_port_state {
251 enum dlb_configuration_state {
252 /* The resource has not been configured */
254 /* The resource was configured, but the device was stopped */
256 /* The resource is currently configured */
264 uint16_t dir_credits;
265 uint32_t dequeue_depth;
266 enum dlb_token_pop_mode token_pop_mode;
268 uint16_t cached_ldb_credits;
269 uint16_t ldb_pushcount_at_credit_expiry;
270 uint16_t ldb_credits;
271 uint16_t cached_dir_credits;
272 uint16_t dir_pushcount_at_credit_expiry;
274 bool use_rsvd_token_scheme;
275 uint8_t cq_rsvd_token_deficit;
276 uint16_t owed_tokens;
277 int16_t issued_releases;
278 int16_t token_pop_thresh;
281 uint16_t cq_idx_unmasked;
282 uint16_t cq_depth_mask;
283 uint16_t gen_bit_shift;
284 enum dlb_port_state state;
285 enum dlb_configuration_state config_state;
287 uint8_t *qid_mappings;
288 struct dlb_enqueue_qe *qe4; /* Cache line's worth of QEs (4) */
289 struct dlb_cq_pop_qe *consume_qe;
290 struct dlb_eventdev *dlb; /* back ptr */
291 struct dlb_eventdev_port *ev_port; /* back ptr */
294 /* Per-process per-port mmio and memory pointers */
295 struct process_local_port_data {
297 uint16_t *ldb_popcount;
298 uint16_t *dir_popcount;
299 struct dlb_dequeue_qe *cq_base;
300 const struct rte_memzone *mz;
307 uint32_t ldb_credit_pool_id;
308 uint32_t dir_credit_pool_id;
309 uint32_t num_ldb_credits;
310 uint32_t num_dir_credits;
311 struct dlb_create_sched_domain_args resources;
315 struct dlb_config cfg;
316 struct dlb_hw_resource_info info;
317 void *pf_dev; /* opaque pointer to PF PMD dev (struct dlb_dev) */
321 rte_spinlock_t resource_lock; /* for MP support */
322 } __rte_cache_aligned;
324 /* End HW related defines and structs */
326 /* Begin DLB PMD Eventdev related defines and structs */
328 #define DLB_MAX_NUM_QUEUES \
329 (DLB_MAX_NUM_DIR_QUEUES + DLB_MAX_NUM_LDB_QUEUES)
331 #define DLB_MAX_NUM_PORTS (DLB_MAX_NUM_DIR_PORTS + DLB_MAX_NUM_LDB_PORTS)
332 #define DLB_MAX_INPUT_QUEUE_DEPTH 256
334 /** Structure to hold the queue to port link establishment attributes */
336 struct dlb_event_queue_link {
343 struct dlb_traffic_stats {
346 uint64_t rx_interrupt_wait;
347 uint64_t rx_umonitor_umwait;
349 uint64_t total_polls;
351 uint64_t tx_nospc_ldb_hw_credits;
352 uint64_t tx_nospc_dir_hw_credits;
353 uint64_t tx_nospc_inflight_max;
354 uint64_t tx_nospc_new_event_limit;
355 uint64_t tx_nospc_inflight_credits;
358 struct dlb_port_stats {
359 struct dlb_traffic_stats traffic;
360 uint64_t tx_op_cnt[4]; /* indexed by rte_event.op */
361 uint64_t tx_implicit_rel;
362 uint64_t tx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
364 uint64_t rx_sched_cnt[DLB_NUM_HW_SCHED_TYPES];
365 uint64_t rx_sched_invalid;
366 uint64_t enq_ok[DLB_MAX_NUM_QUEUES]; /* per-queue enq_ok */
369 struct dlb_eventdev_port {
370 struct dlb_port qm_port; /* hw specific data structure */
371 struct rte_event_port_conf conf; /* user-supplied configuration */
372 uint16_t inflight_credits; /* num credits this port has right now */
373 uint16_t credit_update_quanta;
374 struct dlb_eventdev *dlb; /* backlink optimization */
375 struct dlb_port_stats stats __rte_cache_aligned;
376 struct dlb_event_queue_link link[DLB_MAX_NUM_QIDS_PER_LDB_CQ];
379 /* num releases yet to be completed on this port.
380 * Only applies to load-balanced ports.
382 uint16_t outstanding_releases;
383 uint16_t inflight_max; /* app requested max inflights for this port */
384 /* setup_done is set when the event port is setup */
386 /* enq_configured is set when the qm port is created */
388 uint8_t implicit_release; /* release events before dequeueing */
389 } __rte_cache_aligned;
392 uint32_t num_qid_inflights; /* User config */
393 uint32_t num_atm_inflights; /* User config */
394 enum dlb_configuration_state config_state;
395 int sched_type; /* LB queue only */
400 struct dlb_eventdev_queue {
401 struct dlb_queue qm_queue;
402 struct rte_event_queue_conf conf; /* User config */
410 DLB_RUN_STATE_STOPPED = 0,
411 DLB_RUN_STATE_STOPPING,
412 DLB_RUN_STATE_STARTING,
413 DLB_RUN_STATE_STARTED
416 struct dlb_eventdev {
417 struct dlb_eventdev_port ev_ports[DLB_MAX_NUM_PORTS];
418 struct dlb_eventdev_queue ev_queues[DLB_MAX_NUM_QUEUES];
419 uint8_t qm_ldb_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
420 uint8_t qm_dir_to_ev_queue_id[DLB_MAX_NUM_QUEUES];
422 /* store num stats and offset of the stats for each queue */
423 uint16_t xstats_count_per_qid[DLB_MAX_NUM_QUEUES];
424 uint16_t xstats_offset_for_qid[DLB_MAX_NUM_QUEUES];
426 /* store num stats and offset of the stats for each port */
427 uint16_t xstats_count_per_port[DLB_MAX_NUM_PORTS];
428 uint16_t xstats_offset_for_port[DLB_MAX_NUM_PORTS];
429 struct dlb_get_num_resources_args hw_rsrc_query_results;
430 uint32_t xstats_count_mode_queue;
431 struct dlb_hw_dev qm_instance; /* strictly hw related */
432 uint64_t global_dequeue_wait_ticks;
433 struct dlb_xstats_entry *xstats;
434 struct rte_eventdev *event_dev; /* backlink to dev */
435 uint32_t xstats_count_mode_port;
436 uint32_t xstats_count_mode_dev;
437 uint32_t xstats_count;
438 uint32_t inflights; /* use __atomic builtins to access */
439 uint32_t new_event_limit;
440 int max_num_events_override;
441 int num_dir_credits_override;
442 volatile enum dlb_run_state run_state;
443 uint16_t num_dir_queues; /* total num of evdev dir queues requested */
444 uint16_t num_dir_credits;
445 uint16_t num_ldb_credits;
446 uint16_t num_queues; /* total queues */
447 uint16_t num_ldb_queues; /* total num of evdev ldb queues requested */
448 uint16_t num_ports; /* total num of evdev ports requested */
449 uint16_t num_ldb_ports; /* total num of ldb ports requested */
450 uint16_t num_dir_ports; /* total num of dir ports requested */
453 bool global_dequeue_wait; /* Not using per dequeue wait if true */
455 unsigned int num_atm_inflights_per_queue;
456 enum dlb_cq_poll_modes poll_mode;
461 /* End Eventdev related defines and structs */
465 extern struct process_local_port_data dlb_port[][NUM_DLB_PORT_TYPES];
467 /* Forwards for non-inlined functions */
469 void dlb_eventdev_dump(struct rte_eventdev *dev, FILE *f);
471 int dlb_xstats_init(struct dlb_eventdev *dlb);
473 void dlb_xstats_uninit(struct dlb_eventdev *dlb);
475 int dlb_eventdev_xstats_get(const struct rte_eventdev *dev,
476 enum rte_event_dev_xstats_mode mode,
477 uint8_t queue_port_id, const unsigned int ids[],
478 uint64_t values[], unsigned int n);
480 int dlb_eventdev_xstats_get_names(const struct rte_eventdev *dev,
481 enum rte_event_dev_xstats_mode mode,
482 uint8_t queue_port_id,
483 struct rte_event_dev_xstats_name *xstat_names,
484 unsigned int *ids, unsigned int size);
486 uint64_t dlb_eventdev_xstats_get_by_name(const struct rte_eventdev *dev,
487 const char *name, unsigned int *id);
489 int dlb_eventdev_xstats_reset(struct rte_eventdev *dev,
490 enum rte_event_dev_xstats_mode mode,
491 int16_t queue_port_id,
492 const uint32_t ids[],
495 int test_dlb_eventdev(void);
497 int dlb_primary_eventdev_probe(struct rte_eventdev *dev,
499 struct dlb_devargs *dlb_args);
501 int dlb_secondary_eventdev_probe(struct rte_eventdev *dev,
504 uint32_t dlb_get_queue_depth(struct dlb_eventdev *dlb,
505 struct dlb_eventdev_queue *queue);
507 int dlb_parse_params(const char *params,
509 struct dlb_devargs *dlb_args);
511 void dlb_entry_points_init(struct rte_eventdev *dev);
513 #endif /* _DLB_PRIV_H_ */