From 2acfae37f6b32b6ca245e9a571b9aa84ffd67dd2 Mon Sep 17 00:00:00 2001 From: Jerin Jacob Date: Tue, 3 Sep 2019 16:29:37 +0530 Subject: [PATCH] bpf/arm: add atomic-exchange-and-add operation Implement XADD eBPF instruction using STADD arm64 instruction. If the given platform does not have atomics support, use LDXR and STXR pair for critical section instead of STADD. Signed-off-by: Jerin Jacob Acked-by: Konstantin Ananyev --- lib/librte_bpf/bpf_jit_arm64.c | 85 +++++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/lib/librte_bpf/bpf_jit_arm64.c b/lib/librte_bpf/bpf_jit_arm64.c index c797c9c62d..62fa6a505a 100644 --- a/lib/librte_bpf/bpf_jit_arm64.c +++ b/lib/librte_bpf/bpf_jit_arm64.c @@ -837,6 +837,83 @@ emit_return_zero_if_src_zero(struct a64_jit_ctx *ctx, bool is64, uint8_t src) emit_b(ctx, jump_to_epilogue); } +static void +emit_stadd(struct a64_jit_ctx *ctx, bool is64, uint8_t rs, uint8_t rn) +{ + uint32_t insn; + + insn = 0xb820001f; + insn |= (!!is64) << 30; + insn |= rs << 16; + insn |= rn << 5; + + emit_insn(ctx, insn, check_reg(rs) || check_reg(rn)); +} + +static void +emit_ldxr(struct a64_jit_ctx *ctx, bool is64, uint8_t rt, uint8_t rn) +{ + uint32_t insn; + + insn = 0x885f7c00; + insn |= (!!is64) << 30; + insn |= rn << 5; + insn |= rt; + + emit_insn(ctx, insn, check_reg(rt) || check_reg(rn)); +} + +static void +emit_stxr(struct a64_jit_ctx *ctx, bool is64, uint8_t rs, uint8_t rt, + uint8_t rn) +{ + uint32_t insn; + + insn = 0x88007c00; + insn |= (!!is64) << 30; + insn |= rs << 16; + insn |= rn << 5; + insn |= rt; + + emit_insn(ctx, insn, check_reg(rs) || check_reg(rt) || check_reg(rn)); +} + +static int +has_atomics(void) +{ + int rc = 0; + +#if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS) + rc = 1; +#endif + return rc; +} + +static void +emit_xadd(struct a64_jit_ctx *ctx, uint8_t op, uint8_t tmp1, uint8_t tmp2, + uint8_t tmp3, uint8_t dst, int16_t off, uint8_t src) +{ + bool is64 = (BPF_SIZE(op) == EBPF_DW); + uint8_t rn; + + if (off) { + emit_mov_imm(ctx, 1, tmp1, off); + emit_add(ctx, 1, tmp1, dst); + rn = tmp1; + } else { + rn = dst; + } + + if (has_atomics()) { + emit_stadd(ctx, is64, src, rn); + } else { + emit_ldxr(ctx, is64, tmp2, rn); + emit_add(ctx, is64, tmp2, src); + emit_stxr(ctx, is64, tmp3, tmp2, rn); + emit_cbnz(ctx, is64, tmp3, -3); + } +} + static void check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf) { @@ -863,7 +940,7 @@ check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf) static int emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf) { - uint8_t op, dst, src, tmp1, tmp2; + uint8_t op, dst, src, tmp1, tmp2, tmp3; const struct ebpf_insn *ins; uint64_t u64; int16_t off; @@ -878,6 +955,7 @@ emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf) ctx->stack_sz = RTE_ALIGN_MUL_CEIL(bpf->stack_sz, 16); tmp1 = ebpf_to_a64_reg(ctx, TMP_REG_1); tmp2 = ebpf_to_a64_reg(ctx, TMP_REG_2); + tmp3 = ebpf_to_a64_reg(ctx, TMP_REG_3); emit_prologue(ctx); @@ -1067,6 +1145,11 @@ emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf) emit_mov_imm(ctx, 1, tmp2, off); emit_str(ctx, BPF_SIZE(op), tmp1, dst, tmp2); break; + /* STX XADD: lock *(size *)(dst + off) += src */ + case (BPF_STX | EBPF_XADD | BPF_W): + case (BPF_STX | EBPF_XADD | EBPF_DW): + emit_xadd(ctx, op, tmp1, tmp2, tmp3, dst, off, src); + break; /* Return r0 */ case (BPF_JMP | EBPF_EXIT): emit_epilogue(ctx); -- 2.20.1