1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(C) 2019 Marvell International Ltd.
8 #include <rte_common.h>
9 #include <rte_byteorder.h>
13 #define A64_REG_MASK(r) ((r) & 0x1f)
14 #define A64_INVALID_OP_CODE (0xffffffff)
16 #define TMP_REG_1 (EBPF_REG_10 + 1)
17 #define TMP_REG_2 (EBPF_REG_10 + 2)
18 #define TMP_REG_3 (EBPF_REG_10 + 3)
20 #define EBPF_FP (EBPF_REG_10)
21 #define EBPF_OP_GET(op) (BPF_OP(op) >> 4)
29 #define check_imm(n, val) (((val) >= 0) ? !!((val) >> (n)) : !!((~val) >> (n)))
30 #define mask_imm(n, val) ((val) & ((1 << (n)) - 1))
33 uint32_t off; /* eBPF to arm64 insn offset mapping for jump */
34 uint8_t off_to_b; /* Offset to branch instruction delta */
38 size_t stack_sz; /* Stack size */
39 uint32_t *ins; /* ARM64 instructions. NULL if first pass */
40 struct ebpf_a64_map *map; /* eBPF to arm64 insn mapping for jump */
41 uint32_t idx; /* Current instruction index */
42 uint32_t program_start; /* Program index, Just after prologue */
43 uint32_t program_sz; /* Program size. Found in first pass */
44 uint8_t foundcall; /* Found EBPF_CALL class code in eBPF pgm */
48 check_immr_imms(bool is64, uint8_t immr, uint8_t imms)
50 const unsigned int width = is64 ? 64 : 32;
52 if (immr >= width || imms >= width)
59 check_mov_hw(bool is64, const uint8_t val)
61 if (val == 16 || val == 0)
63 else if (is64 && val != 64 && val != 48 && val != 32)
70 check_ls_sz(uint8_t sz)
72 if (sz == BPF_B || sz == BPF_H || sz == BPF_W || sz == EBPF_DW)
81 return (r > 31) ? 1 : 0;
85 is_first_pass(struct a64_jit_ctx *ctx)
87 return (ctx->ins == NULL);
91 check_invalid_args(struct a64_jit_ctx *ctx, uint32_t limit)
95 if (is_first_pass(ctx))
98 for (idx = 0; idx < limit; idx++) {
99 if (rte_le_to_cpu_32(ctx->ins[idx]) == A64_INVALID_OP_CODE) {
101 "%s: invalid opcode at %u;\n", __func__, idx);
108 /* Emit an instruction */
110 emit_insn(struct a64_jit_ctx *ctx, uint32_t insn, int error)
113 insn = A64_INVALID_OP_CODE;
116 ctx->ins[ctx->idx] = rte_cpu_to_le_32(insn);
122 emit_ret(struct a64_jit_ctx *ctx)
124 emit_insn(ctx, 0xd65f03c0, 0);
128 emit_add_sub_imm(struct a64_jit_ctx *ctx, bool is64, bool sub, uint8_t rd,
129 uint8_t rn, int16_t imm12)
133 imm = mask_imm(12, imm12);
134 insn = (!!is64) << 31;
135 insn |= (!!sub) << 30;
142 check_reg(rd) || check_reg(rn) || check_imm(12, imm12));
146 emit_add_imm_64(struct a64_jit_ctx *ctx, uint8_t rd, uint8_t rn, uint16_t imm12)
148 emit_add_sub_imm(ctx, 1, 0, rd, rn, imm12);
152 emit_sub_imm_64(struct a64_jit_ctx *ctx, uint8_t rd, uint8_t rn, uint16_t imm12)
154 emit_add_sub_imm(ctx, 1, 1, rd, rn, imm12);
158 emit_mov(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rn)
160 emit_add_sub_imm(ctx, is64, 0, rd, rn, 0);
164 emit_mov_64(struct a64_jit_ctx *ctx, uint8_t rd, uint8_t rn)
166 emit_mov(ctx, 1, rd, rn);
170 emit_ls_pair_64(struct a64_jit_ctx *ctx, uint8_t rt, uint8_t rt2, uint8_t rn,
171 bool push, bool load, bool pre_index)
175 insn = (!!load) << 22;
176 insn |= (!!pre_index) << 24;
182 insn |= 0x7e << 15; /* 0x7e means -2 with imm7 */
186 emit_insn(ctx, insn, check_reg(rn) || check_reg(rt) || check_reg(rt2));
190 /* Emit stp rt, rt2, [sp, #-16]! */
192 emit_stack_push(struct a64_jit_ctx *ctx, uint8_t rt, uint8_t rt2)
194 emit_ls_pair_64(ctx, rt, rt2, A64_SP, 1, 0, 1);
197 /* Emit ldp rt, rt2, [sp, #16] */
199 emit_stack_pop(struct a64_jit_ctx *ctx, uint8_t rt, uint8_t rt2)
201 emit_ls_pair_64(ctx, rt, rt2, A64_SP, 0, 1, 0);
208 mov_imm(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t type,
209 uint16_t imm16, uint8_t shift)
213 insn = (!!is64) << 31;
216 insn |= (shift/16) << 21;
220 emit_insn(ctx, insn, check_reg(rd) || check_mov_hw(is64, shift));
224 emit_mov_imm32(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint32_t val)
226 uint16_t upper = val >> 16;
227 uint16_t lower = val & 0xffff;
229 /* Positive number */
230 if ((val & 1UL << 31) == 0) {
231 mov_imm(ctx, is64, rd, A64_MOVZ, lower, 0);
233 mov_imm(ctx, is64, rd, A64_MOVK, upper, 16);
234 } else { /* Negative number */
235 if (upper == 0xffff) {
236 mov_imm(ctx, is64, rd, A64_MOVN, ~lower, 0);
238 mov_imm(ctx, is64, rd, A64_MOVN, ~upper, 16);
240 mov_imm(ctx, is64, rd, A64_MOVK, lower, 0);
246 u16_blocks_weight(const uint64_t val, bool one)
248 return (((val >> 0) & 0xffff) == (one ? 0xffff : 0x0000)) +
249 (((val >> 16) & 0xffff) == (one ? 0xffff : 0x0000)) +
250 (((val >> 32) & 0xffff) == (one ? 0xffff : 0x0000)) +
251 (((val >> 48) & 0xffff) == (one ? 0xffff : 0x0000));
255 emit_mov_imm(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint64_t val)
257 uint64_t nval = ~val;
261 return emit_mov_imm32(ctx, 0, rd, (uint32_t)(val & 0xffffffff));
263 /* Find MOVN or MOVZ first */
264 movn = u16_blocks_weight(val, true) > u16_blocks_weight(val, false);
265 /* Find shift right value */
266 sr = movn ? rte_fls_u64(nval) - 1 : rte_fls_u64(val) - 1;
267 sr = RTE_ALIGN_FLOOR(sr, 16);
271 mov_imm(ctx, 1, rd, A64_MOVN, (nval >> sr) & 0xffff, sr);
273 mov_imm(ctx, 1, rd, A64_MOVZ, (val >> sr) & 0xffff, sr);
277 if (((val >> sr) & 0xffff) != (movn ? 0xffff : 0x0000))
278 mov_imm(ctx, 1, rd, A64_MOVK, (val >> sr) & 0xffff, sr);
284 emit_ls(struct a64_jit_ctx *ctx, uint8_t sz, uint8_t rt, uint8_t rn, uint8_t rm,
294 else if (sz == BPF_H)
296 else if (sz == BPF_W)
298 else if (sz == EBPF_DW)
302 insn |= 0x1a << 10; /* LSL and S = 0 */
306 emit_insn(ctx, insn, check_reg(rt) || check_reg(rn) || check_reg(rm) ||
311 emit_str(struct a64_jit_ctx *ctx, uint8_t sz, uint8_t rt, uint8_t rn,
314 emit_ls(ctx, sz, rt, rn, rm, 0);
318 emit_ldr(struct a64_jit_ctx *ctx, uint8_t sz, uint8_t rt, uint8_t rn,
321 emit_ls(ctx, sz, rt, rn, rm, 1);
325 #define A64_SUB 0x258
327 emit_add_sub(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rn,
328 uint8_t rm, uint16_t op)
332 insn = (!!is64) << 31;
333 insn |= op << 21; /* shift == 0 */
338 emit_insn(ctx, insn, check_reg(rd) || check_reg(rm));
342 emit_add(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm)
344 emit_add_sub(ctx, is64, rd, rd, rm, A64_ADD);
348 emit_sub(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm)
350 emit_add_sub(ctx, is64, rd, rd, rm, A64_SUB);
354 emit_neg(struct a64_jit_ctx *ctx, bool is64, uint8_t rd)
356 emit_add_sub(ctx, is64, rd, A64_ZR, rd, A64_SUB);
360 emit_mul(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm)
364 insn = (!!is64) << 31;
367 insn |= A64_ZR << 10;
371 emit_insn(ctx, insn, check_reg(rd) || check_reg(rm));
379 emit_data_process_two_src(struct a64_jit_ctx *ctx, bool is64, uint8_t rd,
380 uint8_t rn, uint8_t rm, uint16_t op)
385 insn = (!!is64) << 31;
392 emit_insn(ctx, insn, check_reg(rd) || check_reg(rm));
396 emit_div(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm)
398 emit_data_process_two_src(ctx, is64, rd, rd, rm, A64_UDIV);
402 emit_lslv(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm)
404 emit_data_process_two_src(ctx, is64, rd, rd, rm, A64_LSLV);
408 emit_lsrv(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm)
410 emit_data_process_two_src(ctx, is64, rd, rd, rm, A64_LSRV);
414 emit_asrv(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm)
416 emit_data_process_two_src(ctx, is64, rd, rd, rm, A64_ASRV);
422 emit_bitfield(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rn,
423 uint8_t immr, uint8_t imms, uint16_t op)
428 insn = (!!is64) << 31;
430 insn |= 1 << 22; /* Set N bit when is64 is set */
438 emit_insn(ctx, insn, check_reg(rd) || check_reg(rn) ||
439 check_immr_imms(is64, immr, imms));
442 emit_lsl(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t imm)
444 const unsigned int width = is64 ? 64 : 32;
447 immr = (width - imm) & (width - 1);
448 imms = width - 1 - imm;
450 emit_bitfield(ctx, is64, rd, rd, immr, imms, A64_UBFM);
454 emit_lsr(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t imm)
456 emit_bitfield(ctx, is64, rd, rd, imm, is64 ? 63 : 31, A64_UBFM);
460 emit_asr(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t imm)
462 emit_bitfield(ctx, is64, rd, rd, imm, is64 ? 63 : 31, A64_SBFM);
469 emit_logical(struct a64_jit_ctx *ctx, bool is64, uint8_t rd,
470 uint8_t rm, uint16_t op)
474 insn = (!!is64) << 31;
481 emit_insn(ctx, insn, check_reg(rd) || check_reg(rm));
485 emit_or(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm)
487 emit_logical(ctx, is64, rd, rm, A64_OR);
491 emit_and(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm)
493 emit_logical(ctx, is64, rd, rm, A64_AND);
497 emit_xor(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rm)
499 emit_logical(ctx, is64, rd, rm, A64_XOR);
503 emit_msub(struct a64_jit_ctx *ctx, bool is64, uint8_t rd, uint8_t rn,
504 uint8_t rm, uint8_t ra)
508 insn = (!!is64) << 31;
516 emit_insn(ctx, insn, check_reg(rd) || check_reg(rn) || check_reg(rm) ||
521 emit_mod(struct a64_jit_ctx *ctx, bool is64, uint8_t tmp, uint8_t rd,
524 emit_data_process_two_src(ctx, is64, tmp, rd, rm, A64_UDIV);
525 emit_msub(ctx, is64, rd, tmp, rm, rd);
529 emit_zero_extend(struct a64_jit_ctx *ctx, uint8_t rd, int32_t imm)
533 /* Zero-extend 16 bits into 64 bits */
534 emit_bitfield(ctx, 1, rd, rd, 0, 15, A64_UBFM);
537 /* Zero-extend 32 bits into 64 bits */
538 emit_bitfield(ctx, 1, rd, rd, 0, 31, A64_UBFM);
544 emit_insn(ctx, 0, 1);
549 emit_rev(struct a64_jit_ctx *ctx, uint8_t rd, int32_t imm)
560 emit_insn(ctx, insn, check_reg(rd));
561 emit_zero_extend(ctx, rd, 16);
565 emit_insn(ctx, insn, check_reg(rd));
566 /* Upper 32 bits already cleared */
570 emit_insn(ctx, insn, check_reg(rd));
574 emit_insn(ctx, insn, 1);
581 #if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
589 emit_be(struct a64_jit_ctx *ctx, uint8_t rd, int32_t imm)
592 emit_zero_extend(ctx, rd, imm);
594 emit_rev(ctx, rd, imm);
598 emit_le(struct a64_jit_ctx *ctx, uint8_t rd, int32_t imm)
601 emit_rev(ctx, rd, imm);
603 emit_zero_extend(ctx, rd, imm);
607 ebpf_to_a64_reg(struct a64_jit_ctx *ctx, uint8_t reg)
609 const uint32_t ebpf2a64_has_call[] = {
610 /* Map A64 R7 register as EBPF return register */
611 [EBPF_REG_0] = A64_R(7),
612 /* Map A64 arguments register as EBPF arguments register */
613 [EBPF_REG_1] = A64_R(0),
614 [EBPF_REG_2] = A64_R(1),
615 [EBPF_REG_3] = A64_R(2),
616 [EBPF_REG_4] = A64_R(3),
617 [EBPF_REG_5] = A64_R(4),
618 /* Map A64 callee save register as EBPF callee save register */
619 [EBPF_REG_6] = A64_R(19),
620 [EBPF_REG_7] = A64_R(20),
621 [EBPF_REG_8] = A64_R(21),
622 [EBPF_REG_9] = A64_R(22),
623 [EBPF_FP] = A64_R(25),
624 /* Map A64 scratch registers as temporary storage */
625 [TMP_REG_1] = A64_R(9),
626 [TMP_REG_2] = A64_R(10),
627 [TMP_REG_3] = A64_R(11),
630 const uint32_t ebpf2a64_no_call[] = {
631 /* Map A64 R7 register as EBPF return register */
632 [EBPF_REG_0] = A64_R(7),
633 /* Map A64 arguments register as EBPF arguments register */
634 [EBPF_REG_1] = A64_R(0),
635 [EBPF_REG_2] = A64_R(1),
636 [EBPF_REG_3] = A64_R(2),
637 [EBPF_REG_4] = A64_R(3),
638 [EBPF_REG_5] = A64_R(4),
640 * EBPF program does not have EBPF_CALL op code,
641 * Map A64 scratch registers as EBPF callee save registers.
643 [EBPF_REG_6] = A64_R(9),
644 [EBPF_REG_7] = A64_R(10),
645 [EBPF_REG_8] = A64_R(11),
646 [EBPF_REG_9] = A64_R(12),
647 /* Map A64 FP register as EBPF FP register */
649 /* Map remaining A64 scratch registers as temporary storage */
650 [TMP_REG_1] = A64_R(13),
651 [TMP_REG_2] = A64_R(14),
652 [TMP_REG_3] = A64_R(15),
656 return ebpf2a64_has_call[reg];
658 return ebpf2a64_no_call[reg];
662 * Procedure call standard for the arm64
663 * -------------------------------------
664 * R0..R7 - Parameter/result registers
665 * R8 - Indirect result location register
666 * R9..R15 - Scratch registers
667 * R15 - Platform Register
668 * R16 - First intra-procedure-call scratch register
669 * R17 - Second intra-procedure-call temporary register
670 * R19-R28 - Callee saved registers
671 * R29 - Frame pointer
672 * R30 - Link register
673 * R31 - Stack pointer
676 emit_prologue_has_call(struct a64_jit_ctx *ctx)
678 uint8_t r6, r7, r8, r9, fp;
680 r6 = ebpf_to_a64_reg(ctx, EBPF_REG_6);
681 r7 = ebpf_to_a64_reg(ctx, EBPF_REG_7);
682 r8 = ebpf_to_a64_reg(ctx, EBPF_REG_8);
683 r9 = ebpf_to_a64_reg(ctx, EBPF_REG_9);
684 fp = ebpf_to_a64_reg(ctx, EBPF_FP);
687 * eBPF prog stack layout
690 * eBPF prologue 0:+-----+ <= original A64_SP
692 * -16:+-----+ <= current A64_FP
693 * Callee saved registers | ... |
694 * EBPF_FP => -64:+-----+
696 * eBPF prog stack | ... |
698 * (EBPF_FP - bpf->stack_sz)=> +-----+
699 * Pad for A64_SP 16B alignment| PAD |
700 * (EBPF_FP - ctx->stack_sz)=> +-----+ <= current A64_SP
702 * | ... | Function call stack
707 emit_stack_push(ctx, A64_FP, A64_LR);
708 emit_mov_64(ctx, A64_FP, A64_SP);
709 emit_stack_push(ctx, r6, r7);
710 emit_stack_push(ctx, r8, r9);
712 * There is no requirement to save A64_R(28) in stack. Doing it here,
713 * because, A64_SP needs be to 16B aligned and STR vs STP
714 * takes same number of cycles(typically).
716 emit_stack_push(ctx, fp, A64_R(28));
717 emit_mov_64(ctx, fp, A64_SP);
719 emit_sub_imm_64(ctx, A64_SP, A64_SP, ctx->stack_sz);
723 emit_epilogue_has_call(struct a64_jit_ctx *ctx)
725 uint8_t r6, r7, r8, r9, fp, r0;
727 r6 = ebpf_to_a64_reg(ctx, EBPF_REG_6);
728 r7 = ebpf_to_a64_reg(ctx, EBPF_REG_7);
729 r8 = ebpf_to_a64_reg(ctx, EBPF_REG_8);
730 r9 = ebpf_to_a64_reg(ctx, EBPF_REG_9);
731 fp = ebpf_to_a64_reg(ctx, EBPF_FP);
732 r0 = ebpf_to_a64_reg(ctx, EBPF_REG_0);
735 emit_add_imm_64(ctx, A64_SP, A64_SP, ctx->stack_sz);
736 emit_stack_pop(ctx, fp, A64_R(28));
737 emit_stack_pop(ctx, r8, r9);
738 emit_stack_pop(ctx, r6, r7);
739 emit_stack_pop(ctx, A64_FP, A64_LR);
740 emit_mov_64(ctx, A64_R(0), r0);
745 emit_prologue_no_call(struct a64_jit_ctx *ctx)
748 * eBPF prog stack layout without EBPF_CALL opcode
751 * eBPF prologue(EBPF_FP) 0:+-----+ <= original A64_SP/current A64_FP
754 * eBPF prog stack | |
756 * (EBPF_FP - bpf->stack_sz)=> +-----+
757 * Pad for A64_SP 16B alignment| PAD |
758 * (EBPF_FP - ctx->stack_sz)=> +-----+ <= current A64_SP
760 * | ... | Function call stack
766 emit_mov_64(ctx, A64_FP, A64_SP);
767 emit_sub_imm_64(ctx, A64_SP, A64_SP, ctx->stack_sz);
772 emit_epilogue_no_call(struct a64_jit_ctx *ctx)
775 emit_add_imm_64(ctx, A64_SP, A64_SP, ctx->stack_sz);
776 emit_mov_64(ctx, A64_R(0), ebpf_to_a64_reg(ctx, EBPF_REG_0));
781 emit_prologue(struct a64_jit_ctx *ctx)
784 emit_prologue_has_call(ctx);
786 emit_prologue_no_call(ctx);
788 ctx->program_start = ctx->idx;
792 emit_epilogue(struct a64_jit_ctx *ctx)
794 ctx->program_sz = ctx->idx - ctx->program_start;
797 emit_epilogue_has_call(ctx);
799 emit_epilogue_no_call(ctx);
803 emit_cbnz(struct a64_jit_ctx *ctx, bool is64, uint8_t rt, int32_t imm19)
807 imm = mask_imm(19, imm19);
808 insn = (!!is64) << 31;
813 emit_insn(ctx, insn, check_reg(rt) || check_imm(19, imm19));
817 emit_b(struct a64_jit_ctx *ctx, int32_t imm26)
821 imm = mask_imm(26, imm26);
825 emit_insn(ctx, insn, check_imm(26, imm26));
829 emit_return_zero_if_src_zero(struct a64_jit_ctx *ctx, bool is64, uint8_t src)
831 uint8_t r0 = ebpf_to_a64_reg(ctx, EBPF_REG_0);
832 uint16_t jump_to_epilogue;
834 emit_cbnz(ctx, is64, src, 3);
835 emit_mov_imm(ctx, is64, r0, 0);
836 jump_to_epilogue = (ctx->program_start + ctx->program_sz) - ctx->idx;
837 emit_b(ctx, jump_to_epilogue);
841 check_program_has_call(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
843 const struct ebpf_insn *ins;
847 for (i = 0; i != bpf->prm.nb_ins; i++) {
848 ins = bpf->prm.ins + i;
853 case (BPF_JMP | EBPF_CALL):
861 * Walk through eBPF code and translate them to arm64 one.
864 emit(struct a64_jit_ctx *ctx, struct rte_bpf *bpf)
866 uint8_t op, dst, src, tmp1, tmp2;
867 const struct ebpf_insn *ins;
875 /* Reset context fields */
877 /* arm64 SP must be aligned to 16 */
878 ctx->stack_sz = RTE_ALIGN_MUL_CEIL(bpf->stack_sz, 16);
879 tmp1 = ebpf_to_a64_reg(ctx, TMP_REG_1);
880 tmp2 = ebpf_to_a64_reg(ctx, TMP_REG_2);
884 for (i = 0; i != bpf->prm.nb_ins; i++) {
886 ins = bpf->prm.ins + i;
891 dst = ebpf_to_a64_reg(ctx, ins->dst_reg);
892 src = ebpf_to_a64_reg(ctx, ins->src_reg);
893 is64 = (BPF_CLASS(op) == EBPF_ALU64);
897 case (BPF_ALU | EBPF_MOV | BPF_X):
898 case (EBPF_ALU64 | EBPF_MOV | BPF_X):
899 emit_mov(ctx, is64, dst, src);
902 case (BPF_ALU | EBPF_MOV | BPF_K):
903 case (EBPF_ALU64 | EBPF_MOV | BPF_K):
904 emit_mov_imm(ctx, is64, dst, imm);
907 case (BPF_ALU | BPF_ADD | BPF_X):
908 case (EBPF_ALU64 | BPF_ADD | BPF_X):
909 emit_add(ctx, is64, dst, src);
912 case (BPF_ALU | BPF_ADD | BPF_K):
913 case (EBPF_ALU64 | BPF_ADD | BPF_K):
914 emit_mov_imm(ctx, is64, tmp1, imm);
915 emit_add(ctx, is64, dst, tmp1);
918 case (BPF_ALU | BPF_SUB | BPF_X):
919 case (EBPF_ALU64 | BPF_SUB | BPF_X):
920 emit_sub(ctx, is64, dst, src);
923 case (BPF_ALU | BPF_SUB | BPF_K):
924 case (EBPF_ALU64 | BPF_SUB | BPF_K):
925 emit_mov_imm(ctx, is64, tmp1, imm);
926 emit_sub(ctx, is64, dst, tmp1);
929 case (BPF_ALU | BPF_MUL | BPF_X):
930 case (EBPF_ALU64 | BPF_MUL | BPF_X):
931 emit_mul(ctx, is64, dst, src);
934 case (BPF_ALU | BPF_MUL | BPF_K):
935 case (EBPF_ALU64 | BPF_MUL | BPF_K):
936 emit_mov_imm(ctx, is64, tmp1, imm);
937 emit_mul(ctx, is64, dst, tmp1);
940 case (BPF_ALU | BPF_DIV | BPF_X):
941 case (EBPF_ALU64 | BPF_DIV | BPF_X):
942 emit_return_zero_if_src_zero(ctx, is64, src);
943 emit_div(ctx, is64, dst, src);
946 case (BPF_ALU | BPF_DIV | BPF_K):
947 case (EBPF_ALU64 | BPF_DIV | BPF_K):
948 emit_mov_imm(ctx, is64, tmp1, imm);
949 emit_div(ctx, is64, dst, tmp1);
952 case (BPF_ALU | BPF_MOD | BPF_X):
953 case (EBPF_ALU64 | BPF_MOD | BPF_X):
954 emit_return_zero_if_src_zero(ctx, is64, src);
955 emit_mod(ctx, is64, tmp1, dst, src);
958 case (BPF_ALU | BPF_MOD | BPF_K):
959 case (EBPF_ALU64 | BPF_MOD | BPF_K):
960 emit_mov_imm(ctx, is64, tmp1, imm);
961 emit_mod(ctx, is64, tmp2, dst, tmp1);
964 case (BPF_ALU | BPF_OR | BPF_X):
965 case (EBPF_ALU64 | BPF_OR | BPF_X):
966 emit_or(ctx, is64, dst, src);
969 case (BPF_ALU | BPF_OR | BPF_K):
970 case (EBPF_ALU64 | BPF_OR | BPF_K):
971 emit_mov_imm(ctx, is64, tmp1, imm);
972 emit_or(ctx, is64, dst, tmp1);
975 case (BPF_ALU | BPF_AND | BPF_X):
976 case (EBPF_ALU64 | BPF_AND | BPF_X):
977 emit_and(ctx, is64, dst, src);
980 case (BPF_ALU | BPF_AND | BPF_K):
981 case (EBPF_ALU64 | BPF_AND | BPF_K):
982 emit_mov_imm(ctx, is64, tmp1, imm);
983 emit_and(ctx, is64, dst, tmp1);
986 case (BPF_ALU | BPF_XOR | BPF_X):
987 case (EBPF_ALU64 | BPF_XOR | BPF_X):
988 emit_xor(ctx, is64, dst, src);
991 case (BPF_ALU | BPF_XOR | BPF_K):
992 case (EBPF_ALU64 | BPF_XOR | BPF_K):
993 emit_mov_imm(ctx, is64, tmp1, imm);
994 emit_xor(ctx, is64, dst, tmp1);
997 case (BPF_ALU | BPF_NEG):
998 case (EBPF_ALU64 | BPF_NEG):
999 emit_neg(ctx, is64, dst);
1002 case BPF_ALU | BPF_LSH | BPF_X:
1003 case EBPF_ALU64 | BPF_LSH | BPF_X:
1004 emit_lslv(ctx, is64, dst, src);
1007 case BPF_ALU | BPF_LSH | BPF_K:
1008 case EBPF_ALU64 | BPF_LSH | BPF_K:
1009 emit_lsl(ctx, is64, dst, imm);
1012 case BPF_ALU | BPF_RSH | BPF_X:
1013 case EBPF_ALU64 | BPF_RSH | BPF_X:
1014 emit_lsrv(ctx, is64, dst, src);
1017 case BPF_ALU | BPF_RSH | BPF_K:
1018 case EBPF_ALU64 | BPF_RSH | BPF_K:
1019 emit_lsr(ctx, is64, dst, imm);
1021 /* dst >>= src (arithmetic) */
1022 case BPF_ALU | EBPF_ARSH | BPF_X:
1023 case EBPF_ALU64 | EBPF_ARSH | BPF_X:
1024 emit_asrv(ctx, is64, dst, src);
1026 /* dst >>= imm (arithmetic) */
1027 case BPF_ALU | EBPF_ARSH | BPF_K:
1028 case EBPF_ALU64 | EBPF_ARSH | BPF_K:
1029 emit_asr(ctx, is64, dst, imm);
1031 /* dst = be##imm(dst) */
1032 case (BPF_ALU | EBPF_END | EBPF_TO_BE):
1033 emit_be(ctx, dst, imm);
1035 /* dst = le##imm(dst) */
1036 case (BPF_ALU | EBPF_END | EBPF_TO_LE):
1037 emit_le(ctx, dst, imm);
1039 /* dst = *(size *) (src + off) */
1040 case (BPF_LDX | BPF_MEM | BPF_B):
1041 case (BPF_LDX | BPF_MEM | BPF_H):
1042 case (BPF_LDX | BPF_MEM | BPF_W):
1043 case (BPF_LDX | BPF_MEM | EBPF_DW):
1044 emit_mov_imm(ctx, 1, tmp1, off);
1045 emit_ldr(ctx, BPF_SIZE(op), dst, src, tmp1);
1048 case (BPF_LD | BPF_IMM | EBPF_DW):
1049 u64 = ((uint64_t)ins[1].imm << 32) | (uint32_t)imm;
1050 emit_mov_imm(ctx, 1, dst, u64);
1053 /* *(size *)(dst + off) = src */
1054 case (BPF_STX | BPF_MEM | BPF_B):
1055 case (BPF_STX | BPF_MEM | BPF_H):
1056 case (BPF_STX | BPF_MEM | BPF_W):
1057 case (BPF_STX | BPF_MEM | EBPF_DW):
1058 emit_mov_imm(ctx, 1, tmp1, off);
1059 emit_str(ctx, BPF_SIZE(op), src, dst, tmp1);
1061 /* *(size *)(dst + off) = imm */
1062 case (BPF_ST | BPF_MEM | BPF_B):
1063 case (BPF_ST | BPF_MEM | BPF_H):
1064 case (BPF_ST | BPF_MEM | BPF_W):
1065 case (BPF_ST | BPF_MEM | EBPF_DW):
1066 emit_mov_imm(ctx, 1, tmp1, imm);
1067 emit_mov_imm(ctx, 1, tmp2, off);
1068 emit_str(ctx, BPF_SIZE(op), tmp1, dst, tmp2);
1071 case (BPF_JMP | EBPF_EXIT):
1076 "%s(%p): invalid opcode %#x at pc: %u;\n",
1077 __func__, bpf, ins->code, i);
1081 rc = check_invalid_args(ctx, ctx->idx);
1087 * Produce a native ISA version of the given BPF code.
1090 bpf_jit_arm64(struct rte_bpf *bpf)
1092 struct a64_jit_ctx ctx;
1096 /* Init JIT context */
1097 memset(&ctx, 0, sizeof(ctx));
1099 /* Find eBPF program has call class or not */
1100 check_program_has_call(&ctx, bpf);
1102 /* First pass to calculate total code size and valid jump offsets */
1103 rc = emit(&ctx, bpf);
1107 size = ctx.idx * sizeof(uint32_t);
1108 /* Allocate JIT program memory */
1109 ctx.ins = mmap(NULL, size, PROT_READ | PROT_WRITE,
1110 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
1111 if (ctx.ins == MAP_FAILED) {
1116 /* Second pass to generate code */
1117 rc = emit(&ctx, bpf);
1121 rc = mprotect(ctx.ins, size, PROT_READ | PROT_EXEC) != 0;
1127 /* Flush the icache */
1128 __builtin___clear_cache(ctx.ins, ctx.ins + ctx.idx);
1130 bpf->jit.func = (void *)ctx.ins;
1136 munmap(ctx.ins, size);