#include "ssovf_worker.h"
-static force_inline void
+static __rte_always_inline void
ssows_new_event(struct ssows *ws, const struct rte_event *ev)
{
const uint64_t event_ptr = ev->u64;
ssows_add_work(ws, event_ptr, tag, new_tt, grp);
}
-static force_inline void
+static __rte_always_inline void
ssows_fwd_swtag(struct ssows *ws, const struct rte_event *ev, const uint8_t grp)
{
const uint8_t cur_tt = ws->cur_tt;
#define OCT_EVENT_TYPE_GRP_FWD (RTE_EVENT_TYPE_MAX - 1)
-static force_inline void
+static __rte_always_inline void
ssows_fwd_group(struct ssows *ws, const struct rte_event *ev, const uint8_t grp)
{
const uint64_t event_ptr = ev->u64;
ssows_add_work(ws, event_ptr, tag, new_tt, grp);
}
-static force_inline void
+static __rte_always_inline void
ssows_forward_event(struct ssows *ws, const struct rte_event *ev)
{
const uint8_t grp = ev->queue_id;
ssows_fwd_group(ws, ev, grp);
}
-static force_inline void
+static __rte_always_inline void
ssows_release_event(struct ssows *ws)
{
if (likely(ws->cur_tt != SSO_SYNC_UNTAGGED))
ssows_swtag_untag(ws);
}
-force_inline uint16_t __hot
+__rte_always_inline uint16_t __hot
ssows_deq(void *port, struct rte_event *ev, uint64_t timeout_ticks)
{
struct ssows *ws = port;
RTE_SET_USED(timeout_ticks);
- ssows_swtag_wait(ws);
if (ws->swtag_req) {
ws->swtag_req = 0;
+ ssows_swtag_wait(ws);
return 1;
} else {
return ssows_get_work(ws, ev);
}
}
-force_inline uint16_t __hot
+__rte_always_inline uint16_t __hot
ssows_deq_timeout(void *port, struct rte_event *ev, uint64_t timeout_ticks)
{
struct ssows *ws = port;
uint64_t iter;
uint16_t ret = 1;
- ssows_swtag_wait(ws);
if (ws->swtag_req) {
ws->swtag_req = 0;
+ ssows_swtag_wait(ws);
} else {
ret = ssows_get_work(ws, ev);
for (iter = 1; iter < timeout_ticks && (ret == 0); iter++)
return ssows_deq_timeout(port, ev, timeout_ticks);
}
-force_inline uint16_t __hot
+__rte_always_inline uint16_t __hot
ssows_enq(void *port, const struct rte_event *ev)
{
struct ssows *ws = port;
switch (ev->op) {
case RTE_EVENT_OP_NEW:
+ rte_smp_wmb();
ssows_new_event(ws, ev);
break;
case RTE_EVENT_OP_FORWARD: