#include <stdbool.h>
#include <stdio.h>
+#include <rte_branch_prediction.h>
#include <rte_common.h>
#include <rte_compat.h>
+#include <rte_cycles.h>
+#include <rte_per_lcore.h>
+#include <rte_string_fns.h>
+#include <rte_uuid.h>
/** The tracepoint object. */
typedef uint64_t rte_trace_point_t;
-/** Macro to define the tracepoint. */
-#define RTE_TRACE_POINT_DEFINE(tp) \
-rte_trace_point_t __attribute__((section("__rte_trace_point"))) __##tp
-
/**
* Macro to define the tracepoint arguments in RTE_TRACE_POINT macro.
*
* @param tp
* Tracepoint object. Before using the tracepoint, an application needs to
- * define the tracepoint using RTE_TRACE_POINT_DEFINE macro.
+ * define the tracepoint using RTE_TRACE_POINT_REGISTER macro.
* @param args
* C function style input arguments to define the arguments to tracepoint
* function.
* Define the payload of trace function. The payload will be formed using
* rte_trace_point_emit_* macros. Use ";" delimiter between two payloads.
*
- * @see RTE_TRACE_POINT_ARGS, RTE_TRACE_POINT_DEFINE, rte_trace_point_emit_*
+ * @see RTE_TRACE_POINT_ARGS, RTE_TRACE_POINT_REGISTER, rte_trace_point_emit_*
*/
#define RTE_TRACE_POINT(tp, args, ...) \
__RTE_TRACE_POINT(generic, tp, args, __VA_ARGS__)
*
* @param tp
* Tracepoint object. Before using the tracepoint, an application needs to
- * define the tracepoint using RTE_TRACE_POINT_DEFINE macro.
+ * define the tracepoint using RTE_TRACE_POINT_REGISTER macro.
* @param args
* C function style input arguments to define the arguments to tracepoint.
* function.
#ifdef __DOXYGEN__
-/**
- * Macro to select rte_trace_point_emit_* definition for trace register function
- *
- * rte_trace_point_emit_* emits different definitions for trace function.
- * Application must define RTE_TRACE_POINT_REGISTER_SELECT before including
- * rte_trace_point.h in the C file where RTE_TRACE_POINT_REGISTER used.
- *
- * @see RTE_TRACE_POINT_REGISTER
- */
-#define RTE_TRACE_POINT_REGISTER_SELECT
-
/**
* Register a tracepoint.
*
* @param trace
- * The tracepoint object created using RTE_TRACE_POINT_DEFINE.
+ * The tracepoint object created using RTE_TRACE_POINT_REGISTER.
* @param name
* The name of the tracepoint object.
* @return
* - 0: Successfully registered the tracepoint.
* - <0: Failure to register the tracepoint.
- *
- * @see RTE_TRACE_POINT_REGISTER_SELECT
*/
#define RTE_TRACE_POINT_REGISTER(trace, name)
#define rte_trace_point_emit_int(val)
/** Tracepoint function payload for long datatype */
#define rte_trace_point_emit_long(val)
+/** Tracepoint function payload for size_t datatype */
+#define rte_trace_point_emit_size_t(val)
/** Tracepoint function payload for float datatype */
#define rte_trace_point_emit_float(val)
/** Tracepoint function payload for double datatype */
#endif
}
+/**
+ * @internal
+ *
+ * Allocate trace memory buffer per thread.
+ *
+ */
+__rte_experimental
+void __rte_trace_mem_per_thread_alloc(void);
+
+/**
+ * @internal
+ *
+ * Helper function to emit field.
+ *
+ * @param sz
+ * The tracepoint size.
+ * @param field
+ * The name of the trace event.
+ * @param type
+ * The datatype of the trace event as string.
+ * @return
+ * - 0: Success.
+ * - <0: Failure.
+ */
+__rte_experimental
+void __rte_trace_point_emit_field(size_t sz, const char *field,
+ const char *type);
+
+/**
+ * @internal
+ *
+ * Helper function to register a dynamic tracepoint.
+ * Use RTE_TRACE_POINT_REGISTER macro for tracepoint registration.
+ *
+ * @param trace
+ * The tracepoint object created using RTE_TRACE_POINT_REGISTER.
+ * @param name
+ * The name of the tracepoint object.
+ * @param register_fn
+ * Trace registration function.
+ * @return
+ * - 0: Successfully registered the tracepoint.
+ * - <0: Failure to register the tracepoint.
+ */
+__rte_experimental
+int __rte_trace_point_register(rte_trace_point_t *trace, const char *name,
+ void (*register_fn)(void));
+
+#ifndef __DOXYGEN__
+
+#ifndef _RTE_TRACE_POINT_REGISTER_H_
+#ifdef ALLOW_EXPERIMENTAL_API
+
+#define __RTE_TRACE_EVENT_HEADER_ID_SHIFT (48)
+
+#define __RTE_TRACE_FIELD_SIZE_SHIFT 0
+#define __RTE_TRACE_FIELD_SIZE_MASK (0xffffULL << __RTE_TRACE_FIELD_SIZE_SHIFT)
+#define __RTE_TRACE_FIELD_ID_SHIFT (16)
+#define __RTE_TRACE_FIELD_ID_MASK (0xffffULL << __RTE_TRACE_FIELD_ID_SHIFT)
+#define __RTE_TRACE_FIELD_ENABLE_MASK (1ULL << 63)
+#define __RTE_TRACE_FIELD_ENABLE_DISCARD (1ULL << 62)
+
+struct __rte_trace_stream_header {
+ uint32_t magic;
+ rte_uuid_t uuid;
+ uint32_t lcore_id;
+ char thread_name[__RTE_TRACE_EMIT_STRING_LEN_MAX];
+} __rte_packed;
+
+struct __rte_trace_header {
+ uint32_t offset;
+ uint32_t len;
+ struct __rte_trace_stream_header stream_header;
+ uint8_t mem[];
+};
+
+RTE_DECLARE_PER_LCORE(void *, trace_mem);
+
+static __rte_always_inline void *
+__rte_trace_mem_get(uint64_t in)
+{
+ struct __rte_trace_header *trace =
+ (struct __rte_trace_header *)(RTE_PER_LCORE(trace_mem));
+ const uint16_t sz = in & __RTE_TRACE_FIELD_SIZE_MASK;
+
+ /* Trace memory is not initialized for this thread */
+ if (unlikely(trace == NULL)) {
+ __rte_trace_mem_per_thread_alloc();
+ trace = (struct __rte_trace_header *)(RTE_PER_LCORE(trace_mem));
+ if (unlikely(trace == NULL))
+ return NULL;
+ }
+ /* Check the wrap around case */
+ uint32_t offset = trace->offset;
+ if (unlikely((offset + sz) >= trace->len)) {
+ /* Disable the trace event if it in DISCARD mode */
+ if (unlikely(in & __RTE_TRACE_FIELD_ENABLE_DISCARD))
+ return NULL;
+
+ offset = 0;
+ }
+ /* Align to event header size */
+ offset = RTE_ALIGN_CEIL(offset, __RTE_TRACE_EVENT_HEADER_SZ);
+ void *mem = RTE_PTR_ADD(&trace->mem[0], offset);
+ offset += sz;
+ trace->offset = offset;
+
+ return mem;
+}
+
+static __rte_always_inline void *
+__rte_trace_point_emit_ev_header(void *mem, uint64_t in)
+{
+ uint64_t val;
+
+ /* Event header [63:0] = id [63:48] | timestamp [47:0] */
+ val = rte_get_tsc_cycles() &
+ ~(0xffffULL << __RTE_TRACE_EVENT_HEADER_ID_SHIFT);
+ val |= ((in & __RTE_TRACE_FIELD_ID_MASK) <<
+ (__RTE_TRACE_EVENT_HEADER_ID_SHIFT -
+ __RTE_TRACE_FIELD_ID_SHIFT));
+
+ *(uint64_t *)mem = val;
+ return RTE_PTR_ADD(mem, __RTE_TRACE_EVENT_HEADER_SZ);
+}
+
+#define __rte_trace_point_emit_header_generic(t) \
+void *mem; \
+do { \
+ const uint64_t val = __atomic_load_n(t, __ATOMIC_ACQUIRE); \
+ if (likely(!(val & __RTE_TRACE_FIELD_ENABLE_MASK))) \
+ return; \
+ mem = __rte_trace_mem_get(val); \
+ if (unlikely(mem == NULL)) \
+ return; \
+ mem = __rte_trace_point_emit_ev_header(mem, val); \
+} while (0)
+
+#define __rte_trace_point_emit_header_fp(t) \
+ if (!__rte_trace_point_fp_is_enabled()) \
+ return; \
+ __rte_trace_point_emit_header_generic(t)
+
+#define __rte_trace_point_emit(in, type) \
+do { \
+ memcpy(mem, &(in), sizeof(in)); \
+ mem = RTE_PTR_ADD(mem, sizeof(in)); \
+} while (0)
+
+#define rte_trace_point_emit_string(in) \
+do { \
+ if (unlikely(in == NULL)) \
+ return; \
+ rte_strscpy(mem, in, __RTE_TRACE_EMIT_STRING_LEN_MAX); \
+ mem = RTE_PTR_ADD(mem, __RTE_TRACE_EMIT_STRING_LEN_MAX); \
+} while (0)
+
+#else
+
+#define __rte_trace_point_emit_header_generic(t) RTE_SET_USED(t)
+#define __rte_trace_point_emit_header_fp(t) RTE_SET_USED(t)
+#define __rte_trace_point_emit(in, type) RTE_SET_USED(in)
+#define rte_trace_point_emit_string(in) RTE_SET_USED(in)
+
+#endif /* ALLOW_EXPERIMENTAL_API */
+#endif /* _RTE_TRACE_POINT_REGISTER_H_ */
+
+#define rte_trace_point_emit_u64(in) __rte_trace_point_emit(in, uint64_t)
+#define rte_trace_point_emit_i64(in) __rte_trace_point_emit(in, int64_t)
+#define rte_trace_point_emit_u32(in) __rte_trace_point_emit(in, uint32_t)
+#define rte_trace_point_emit_i32(in) __rte_trace_point_emit(in, int32_t)
+#define rte_trace_point_emit_u16(in) __rte_trace_point_emit(in, uint16_t)
+#define rte_trace_point_emit_i16(in) __rte_trace_point_emit(in, int16_t)
+#define rte_trace_point_emit_u8(in) __rte_trace_point_emit(in, uint8_t)
+#define rte_trace_point_emit_i8(in) __rte_trace_point_emit(in, int8_t)
+#define rte_trace_point_emit_int(in) __rte_trace_point_emit(in, int32_t)
+#define rte_trace_point_emit_long(in) __rte_trace_point_emit(in, long)
+#define rte_trace_point_emit_size_t(in) __rte_trace_point_emit(in, size_t)
+#define rte_trace_point_emit_float(in) __rte_trace_point_emit(in, float)
+#define rte_trace_point_emit_double(in) __rte_trace_point_emit(in, double)
+#define rte_trace_point_emit_ptr(in) __rte_trace_point_emit(in, uintptr_t)
+
+#endif /* __DOXYGEN__ */
+
#ifdef __cplusplus
}
#endif