bpf/arm: add atomic-exchange-and-add operation

Implement XADD eBPF instruction using STADD arm64 instruction.
If the given platform does not have atomics support,
use LDXR and STXR pair for critical section instead of STADD.

Signed-off-by: Jerin Jacob <jerinj@marvell.com>
Acked-by: Konstantin Ananyev <konstantin.ananyev@intel.com>

diff --git a/lib/librte_bpf/bpf_jit_arm64.c b/lib/librte_bpf/bpf_jit_arm64.c
index c797c9c..62fa6a5 100644 (file)
--- a/lib/librte_bpf/bpf_jit_arm64.c
+++ b/lib/librte_bpf/bpf_jit_arm64.c
@@ -837,6 +837,83 @@ emit_return_zero_if_src_zero(struct a64_jit_ctx *ctx, bool is64, uint8_t src)
        emit_b(ctx, jump_to_epilogue);
 }
 
+static void
+emit_stadd(struct a64_jit_ctx *ctx, bool is64, uint8_t rs, uint8_t rn)
+{
+       uint32_t insn;
+
+       insn = 0xb820001f;
+       insn |= (!!is64) << 30;
+       insn |= rs << 16;
+       insn |= rn << 5;
+
+       emit_insn(ctx, insn, check_reg(rs) || check_reg(rn));
+}
+
+static void
+emit_ldxr(struct a64_jit_ctx *ctx, bool is64, uint8_t rt, uint8_t rn)
+{
+       uint32_t insn;
+
+       insn = 0x885f7c00;
+       insn |= (!!is64) << 30;
+       insn |= rn << 5;
+       insn |= rt;
+
+       emit_insn(ctx, insn, check_reg(rt) || check_reg(rn));
+}
+
+static void
+emit_stxr(struct a64_jit_ctx *ctx, bool is64, uint8_t rs, uint8_t rt,
+         uint8_t rn)
+{
+       uint32_t insn;
+
+       insn = 0x88007c00;
+       insn |= (!!is64) << 30;
+       insn |= rs << 16;
+       insn |= rn << 5;
+       insn |= rt;
+
+       emit_insn(ctx, insn, check_reg(rs) || check_reg(rt) || check_reg(rn));
+}
+
+static int
+has_atomics(void)
+{
+       int rc = 0;
+
+#if defined(__ARM_FEATURE_ATOMICS) || defined(RTE_ARM_FEATURE_ATOMICS)
+       rc = 1;
+#endif
+       return rc;
+}
+
+static void
+emit_xadd(struct a64_jit_ctx *ctx, uint8_t op, uint8_t tmp1, uint8_t tmp2,
+         uint8_t tmp3, uint8_t dst, int16_t off, uint8_t src)
+{
+       bool is64 = (BPF_SIZE(op) == EBPF_DW);
+       uint8_t rn;
+
+       if (off) {
+               emit_mov_imm(ctx, 1, tmp1, off);
+               emit_add(ctx, 1, tmp1, dst);
+               rn = tmp1;
+       } else {
+               rn = dst;
+       }
+
+       if (has_atomics()) {
+               emit_stadd(ctx, is64, src, rn);
+       } else {
+               emit_ldxr(ctx, is64, tmp2, rn);
+               emit_add(ctx, is64, tmp2, src);
+               emit_stxr(ctx, is64, tmp3, tmp2, rn);
+               emit_cbnz(ctx, is64, tmp3, -3);
+       }
+}
+
 static void
 check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
 {
@@ -863,7 +940,7 @@ check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
 static int
 emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
 {
-       uint8_t op, dst, src, tmp1, tmp2;
+       uint8_t op, dst, src, tmp1, tmp2, tmp3;
        const struct ebpf_insn *ins;
        uint64_t u64;
        int16_t off;
@@ -878,6 +955,7 @@ emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
        ctx->stack_sz = RTE_ALIGN_MUL_CEIL(bpf->stack_sz, 16);
        tmp1 = ebpf_to_a64_reg(ctx, TMP_REG_1);
        tmp2 = ebpf_to_a64_reg(ctx, TMP_REG_2);
+       tmp3 = ebpf_to_a64_reg(ctx, TMP_REG_3);
 
        emit_prologue(ctx);
 
@@ -1067,6 +1145,11 @@ emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
                        emit_mov_imm(ctx, 1, tmp2, off);
                        emit_str(ctx, BPF_SIZE(op), tmp1, dst, tmp2);
                        break;
+               /* STX XADD: lock *(size *)(dst + off) += src */
+               case (BPF_STX | EBPF_XADD | BPF_W):
+               case (BPF_STX | EBPF_XADD | EBPF_DW):
+                       emit_xadd(ctx, op, tmp1, tmp2, tmp3, dst, off, src);
+                       break;
                /* Return r0 */
                case (BPF_JMP | EBPF_EXIT):
                        emit_epilogue(ctx);