1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
10 #include <rte_common.h>
12 #include <rte_debug.h>
13 #include <rte_memory.h>
15 #include <rte_byteorder.h>
19 #define GET_BPF_OP(op) (BPF_OP(op) >> 4)
22 RAX = 0, /* scratch, return value */
23 RCX = 1, /* scratch, 4th arg */
24 RDX = 2, /* scratch, 3rd arg */
25 RBX = 3, /* callee saved */
26 RSP = 4, /* stack pointer */
27 RBP = 5, /* frame pointer, callee saved */
28 RSI = 6, /* scratch, 2nd arg */
29 RDI = 7, /* scratch, 1st arg */
30 R8 = 8, /* scratch, 5th arg */
31 R9 = 9, /* scratch, 6th arg */
32 R10 = 10, /* scratch */
33 R11 = 11, /* scratch */
34 R12 = 12, /* callee saved */
35 R13 = 13, /* callee saved */
36 R14 = 14, /* callee saved */
37 R15 = 15, /* callee saved */
40 #define IS_EXT_REG(r) ((r) >= R8)
43 REX_PREFIX = 0x40, /* fixed value 0100 */
44 REX_W = 0x8, /* 64bit operand size */
45 REX_R = 0x4, /* extension of the ModRM.reg field */
46 REX_X = 0x2, /* extension of the SIB.index field */
47 REX_B = 0x1, /* extension of the ModRM.rm field */
65 * eBPF to x86_64 register mappings.
67 static const uint32_t ebpf2x86[] = {
82 * r10 and r11 are used as a scratch temporary registers.
90 /* LD_ABS/LD_IMM offsets */
92 LDMB_FSP_OFS, /* fast-path */
93 LDMB_SLP_OFS, /* slow-path */
94 LDMB_FIN_OFS, /* final part */
99 * callee saved registers list.
100 * keep RBP as the last one.
102 static const uint32_t save_regs[] = {RBX, R12, R13, R14, R15, RBP};
104 struct bpf_jit_state {
119 #define INUSE(v, r) (((v) >> (r)) & 1)
120 #define USED(v, r) ((v) |= 1 << (r))
128 * In many cases for imm8 we can produce shorter code.
134 return sizeof(int8_t);
135 return sizeof(int32_t);
139 emit_bytes(struct bpf_jit_state *st, const uint8_t ins[], uint32_t sz)
143 if (st->ins != NULL) {
144 for (i = 0; i != sz; i++)
145 st->ins[st->sz + i] = ins[i];
151 emit_imm(struct bpf_jit_state *st, const uint32_t imm, uint32_t sz)
156 emit_bytes(st, v.u8, sz);
163 emit_rex(struct bpf_jit_state *st, uint32_t op, uint32_t reg, uint32_t rm)
167 /* mark operand registers as used*/
168 USED(st->reguse, reg);
169 USED(st->reguse, rm);
172 if (BPF_CLASS(op) == EBPF_ALU64 ||
173 op == (BPF_ST | BPF_MEM | EBPF_DW) ||
174 op == (BPF_STX | BPF_MEM | EBPF_DW) ||
175 op == (BPF_STX | EBPF_XADD | EBPF_DW) ||
176 op == (BPF_LD | BPF_IMM | EBPF_DW) ||
177 (BPF_CLASS(op) == BPF_LDX &&
178 BPF_MODE(op) == BPF_MEM &&
179 BPF_SIZE(op) != BPF_W))
188 /* store using SIL, DIL */
189 if (op == (BPF_STX | BPF_MEM | BPF_B) && (reg == RDI || reg == RSI))
194 emit_bytes(st, &rex, sizeof(rex));
202 emit_modregrm(struct bpf_jit_state *st, uint32_t mod, uint32_t reg, uint32_t rm)
206 v = mod << 6 | (reg & 7) << 3 | (rm & 7);
207 emit_bytes(st, &v, sizeof(v));
214 emit_sib(struct bpf_jit_state *st, uint32_t scale, uint32_t idx, uint32_t base)
218 v = scale << 6 | (idx & 7) << 3 | (base & 7);
219 emit_bytes(st, &v, sizeof(v));
223 * emit OPCODE+REGIDX byte
226 emit_opcode(struct bpf_jit_state *st, uint8_t ops, uint32_t reg)
231 emit_bytes(st, &v, sizeof(v));
236 * emit xchg %<sreg>, %<dreg>
239 emit_xchg_reg(struct bpf_jit_state *st, uint32_t sreg, uint32_t dreg)
241 const uint8_t ops = 0x87;
243 emit_rex(st, EBPF_ALU64, sreg, dreg);
244 emit_bytes(st, &ops, sizeof(ops));
245 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
252 emit_neg(struct bpf_jit_state *st, uint32_t op, uint32_t dreg)
254 const uint8_t ops = 0xF7;
255 const uint8_t mods = 3;
257 emit_rex(st, op, 0, dreg);
258 emit_bytes(st, &ops, sizeof(ops));
259 emit_modregrm(st, MOD_DIRECT, mods, dreg);
263 * emit mov %<sreg>, %<dreg>
266 emit_mov_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
269 const uint8_t ops = 0x89;
271 /* if operands are 32-bit, then it can be used to clear upper 32-bit */
272 if (sreg != dreg || BPF_CLASS(op) == BPF_ALU) {
273 emit_rex(st, op, sreg, dreg);
274 emit_bytes(st, &ops, sizeof(ops));
275 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
280 * emit movzwl %<sreg>, %<dreg>
283 emit_movzwl(struct bpf_jit_state *st, uint32_t sreg, uint32_t dreg)
285 static const uint8_t ops[] = {0x0F, 0xB7};
287 emit_rex(st, BPF_ALU, sreg, dreg);
288 emit_bytes(st, ops, sizeof(ops));
289 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
293 * emit ror <imm8>, %<dreg>
296 emit_ror_imm(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
298 const uint8_t prfx = 0x66;
299 const uint8_t ops = 0xC1;
300 const uint8_t mods = 1;
302 emit_bytes(st, &prfx, sizeof(prfx));
303 emit_rex(st, BPF_ALU, 0, dreg);
304 emit_bytes(st, &ops, sizeof(ops));
305 emit_modregrm(st, MOD_DIRECT, mods, dreg);
306 emit_imm(st, imm, imm_size(imm));
313 emit_be2le_48(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
317 const uint8_t ops = 0x0F;
318 const uint8_t mods = 1;
320 rop = (imm == 64) ? EBPF_ALU64 : BPF_ALU;
321 emit_rex(st, rop, 0, dreg);
322 emit_bytes(st, &ops, sizeof(ops));
323 emit_modregrm(st, MOD_DIRECT, mods, dreg);
327 emit_be2le(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
330 emit_ror_imm(st, dreg, 8);
331 emit_movzwl(st, dreg, dreg);
333 emit_be2le_48(st, dreg, imm);
337 * In general it is NOP for x86.
338 * Just clear the upper bits.
341 emit_le2be(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
344 emit_movzwl(st, dreg, dreg);
346 emit_mov_reg(st, BPF_ALU | EBPF_MOV | BPF_X, dreg, dreg);
358 emit_alu_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
363 const uint8_t op8 = 0x83;
364 const uint8_t op32 = 0x81;
365 static const uint8_t mods[] = {
366 [GET_BPF_OP(BPF_ADD)] = 0,
367 [GET_BPF_OP(BPF_AND)] = 4,
368 [GET_BPF_OP(BPF_OR)] = 1,
369 [GET_BPF_OP(BPF_SUB)] = 5,
370 [GET_BPF_OP(BPF_XOR)] = 6,
373 bop = GET_BPF_OP(op);
376 imsz = imm_size(imm);
377 opcode = (imsz == 1) ? op8 : op32;
379 emit_rex(st, op, 0, dreg);
380 emit_bytes(st, &opcode, sizeof(opcode));
381 emit_modregrm(st, MOD_DIRECT, mod, dreg);
382 emit_imm(st, imm, imsz);
387 * add %<sreg>, %<dreg>
388 * and %<sreg>, %<dreg>
389 * or %<sreg>, %<dreg>
390 * sub %<sreg>, %<dreg>
391 * xor %<sreg>, %<dreg>
394 emit_alu_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
399 static const uint8_t ops[] = {
400 [GET_BPF_OP(BPF_ADD)] = 0x01,
401 [GET_BPF_OP(BPF_AND)] = 0x21,
402 [GET_BPF_OP(BPF_OR)] = 0x09,
403 [GET_BPF_OP(BPF_SUB)] = 0x29,
404 [GET_BPF_OP(BPF_XOR)] = 0x31,
407 bop = GET_BPF_OP(op);
409 emit_rex(st, op, sreg, dreg);
410 emit_bytes(st, &ops[bop], sizeof(ops[bop]));
411 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
415 emit_shift(struct bpf_jit_state *st, uint32_t op, uint32_t dreg)
420 static const uint8_t ops[] = {0xC1, 0xD3};
421 static const uint8_t mods[] = {
422 [GET_BPF_OP(BPF_LSH)] = 4,
423 [GET_BPF_OP(BPF_RSH)] = 5,
424 [GET_BPF_OP(EBPF_ARSH)] = 7,
427 bop = GET_BPF_OP(op);
429 opx = (BPF_SRC(op) == BPF_X);
431 emit_rex(st, op, 0, dreg);
432 emit_bytes(st, &ops[opx], sizeof(ops[opx]));
433 emit_modregrm(st, MOD_DIRECT, mod, dreg);
443 emit_shift_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg,
446 emit_shift(st, op, dreg);
447 emit_imm(st, imm, imm_size(imm));
455 * note that rcx is implicitly used as a source register, so few extra
456 * instructions for register spillage might be necessary.
459 emit_shift_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
463 emit_xchg_reg(st, RCX, sreg);
465 emit_shift(st, op, (dreg == RCX) ? sreg : dreg);
468 emit_xchg_reg(st, RCX, sreg);
472 * emit mov <imm>, %<dreg>
475 emit_mov_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
477 const uint8_t ops = 0xC7;
480 /* replace 'mov 0, %<dst>' with 'xor %<dst>, %<dst>' */
481 op = BPF_CLASS(op) | BPF_XOR | BPF_X;
482 emit_alu_reg(st, op, dreg, dreg);
486 emit_rex(st, op, 0, dreg);
487 emit_bytes(st, &ops, sizeof(ops));
488 emit_modregrm(st, MOD_DIRECT, 0, dreg);
489 emit_imm(st, imm, sizeof(imm));
493 * emit mov <imm64>, %<dreg>
496 emit_ld_imm64(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm0,
501 const uint8_t ops = 0xB8;
503 op = (imm1 == 0) ? BPF_ALU : EBPF_ALU64;
505 emit_rex(st, op, 0, dreg);
506 emit_opcode(st, ops, dreg);
508 emit_imm(st, imm0, sizeof(imm0));
510 emit_imm(st, imm1, sizeof(imm1));
514 * note that rax:rdx are implicitly used as source/destination registers,
515 * so some reg spillage is necessary.
530 emit_mul(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
533 const uint8_t ops = 0xF7;
534 const uint8_t mods = 4;
537 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, REG_TMP0);
538 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, REG_TMP1);
541 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, dreg, RAX);
543 if (BPF_SRC(op) == BPF_X)
545 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
546 sreg == RAX ? REG_TMP0 : sreg, RDX);
549 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, RDX, imm);
551 emit_rex(st, op, RAX, RDX);
552 emit_bytes(st, &ops, sizeof(ops));
553 emit_modregrm(st, MOD_DIRECT, mods, RDX);
557 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX);
561 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, dreg);
563 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP0, RAX);
568 * emit mov <ofs>(%<sreg>), %<dreg>
569 * note that for non 64-bit ops, higher bits have to be cleared.
572 emit_ld_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
576 const uint8_t op32 = 0x8B;
577 const uint8_t op16[] = {0x0F, 0xB7};
578 const uint8_t op8[] = {0x0F, 0xB6};
580 emit_rex(st, op, dreg, sreg);
584 emit_bytes(st, op8, sizeof(op8));
585 else if (opsz == BPF_H)
586 emit_bytes(st, op16, sizeof(op16));
588 emit_bytes(st, &op32, sizeof(op32));
590 mods = (imm_size(ofs) == 1) ? MOD_IDISP8 : MOD_IDISP32;
592 emit_modregrm(st, mods, dreg, sreg);
593 if (sreg == RSP || sreg == R12)
594 emit_sib(st, SIB_SCALE_1, sreg, sreg);
595 emit_imm(st, ofs, imm_size(ofs));
600 * mov %<sreg>, <ofs>(%<dreg>)
601 * mov <imm>, <ofs>(%<dreg>)
604 emit_st_common(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
605 uint32_t dreg, uint32_t imm, int32_t ofs)
607 uint32_t mods, imsz, opsz, opx;
608 const uint8_t prfx16 = 0x66;
610 /* 8 bit instruction opcodes */
611 static const uint8_t op8[] = {0xC6, 0x88};
613 /* 16/32/64 bit instruction opcodes */
614 static const uint8_t ops[] = {0xC7, 0x89};
616 /* is the instruction has immediate value or src reg? */
617 opx = (BPF_CLASS(op) == BPF_STX);
621 emit_bytes(st, &prfx16, sizeof(prfx16));
623 emit_rex(st, op, sreg, dreg);
626 emit_bytes(st, &op8[opx], sizeof(op8[opx]));
628 emit_bytes(st, &ops[opx], sizeof(ops[opx]));
630 imsz = imm_size(ofs);
631 mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32;
633 emit_modregrm(st, mods, sreg, dreg);
635 if (dreg == RSP || dreg == R12)
636 emit_sib(st, SIB_SCALE_1, dreg, dreg);
638 emit_imm(st, ofs, imsz);
641 imsz = RTE_MIN(bpf_size(opsz), sizeof(imm));
642 emit_imm(st, imm, imsz);
647 emit_st_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm,
650 emit_st_common(st, op, 0, dreg, imm, ofs);
654 emit_st_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
657 emit_st_common(st, op, sreg, dreg, 0, ofs);
661 * emit lock add %<sreg>, <ofs>(%<dreg>)
664 emit_st_xadd(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
665 uint32_t dreg, int32_t ofs)
669 const uint8_t lck = 0xF0; /* lock prefix */
670 const uint8_t ops = 0x01; /* add opcode */
672 imsz = imm_size(ofs);
673 mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32;
675 emit_bytes(st, &lck, sizeof(lck));
676 emit_rex(st, op, sreg, dreg);
677 emit_bytes(st, &ops, sizeof(ops));
678 emit_modregrm(st, mods, sreg, dreg);
679 emit_imm(st, ofs, imsz);
684 * mov <imm64>, (%rax)
688 emit_call(struct bpf_jit_state *st, uintptr_t trg)
690 const uint8_t ops = 0xFF;
691 const uint8_t mods = 2;
693 emit_ld_imm64(st, RAX, trg, trg >> 32);
694 emit_bytes(st, &ops, sizeof(ops));
695 emit_modregrm(st, MOD_DIRECT, mods, RAX);
700 * where 'ofs' is the target offset for the native code.
703 emit_abs_jmp(struct bpf_jit_state *st, int32_t ofs)
708 const uint8_t op8 = 0xEB;
709 const uint8_t op32 = 0xE9;
711 const int32_t sz8 = sizeof(op8) + sizeof(uint8_t);
712 const int32_t sz32 = sizeof(op32) + sizeof(uint32_t);
714 /* max possible jmp instruction size */
715 const int32_t iszm = RTE_MAX(sz8, sz32);
718 imsz = RTE_MAX(imm_size(joff), imm_size(joff + iszm));
721 emit_bytes(st, &op8, sizeof(op8));
724 emit_bytes(st, &op32, sizeof(op32));
728 emit_imm(st, joff, imsz);
733 * where 'ofs' is the target offset for the BPF bytecode.
736 emit_jmp(struct bpf_jit_state *st, int32_t ofs)
738 emit_abs_jmp(st, st->off[st->idx + ofs]);
743 * cmovz %<sreg>, <%dreg>
744 * cmovne %<sreg>, <%dreg>
745 * cmova %<sreg>, <%dreg>
746 * cmovb %<sreg>, <%dreg>
747 * cmovae %<sreg>, <%dreg>
748 * cmovbe %<sreg>, <%dreg>
749 * cmovg %<sreg>, <%dreg>
750 * cmovl %<sreg>, <%dreg>
751 * cmovge %<sreg>, <%dreg>
752 * cmovle %<sreg>, <%dreg>
755 emit_movcc_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
760 static const uint8_t ops[][2] = {
761 [GET_BPF_OP(BPF_JEQ)] = {0x0F, 0x44}, /* CMOVZ */
762 [GET_BPF_OP(EBPF_JNE)] = {0x0F, 0x45}, /* CMOVNE */
763 [GET_BPF_OP(BPF_JGT)] = {0x0F, 0x47}, /* CMOVA */
764 [GET_BPF_OP(EBPF_JLT)] = {0x0F, 0x42}, /* CMOVB */
765 [GET_BPF_OP(BPF_JGE)] = {0x0F, 0x43}, /* CMOVAE */
766 [GET_BPF_OP(EBPF_JLE)] = {0x0F, 0x46}, /* CMOVBE */
767 [GET_BPF_OP(EBPF_JSGT)] = {0x0F, 0x4F}, /* CMOVG */
768 [GET_BPF_OP(EBPF_JSLT)] = {0x0F, 0x4C}, /* CMOVL */
769 [GET_BPF_OP(EBPF_JSGE)] = {0x0F, 0x4D}, /* CMOVGE */
770 [GET_BPF_OP(EBPF_JSLE)] = {0x0F, 0x4E}, /* CMOVLE */
771 [GET_BPF_OP(BPF_JSET)] = {0x0F, 0x45}, /* CMOVNE */
774 bop = GET_BPF_OP(op);
776 emit_rex(st, op, dreg, sreg);
777 emit_bytes(st, ops[bop], sizeof(ops[bop]));
778 emit_modregrm(st, MOD_DIRECT, dreg, sreg);
793 * where 'ofs' is the target offset for the native code.
796 emit_abs_jcc(struct bpf_jit_state *st, uint32_t op, int32_t ofs)
801 static const uint8_t op8[] = {
802 [GET_BPF_OP(BPF_JEQ)] = 0x74, /* JE */
803 [GET_BPF_OP(EBPF_JNE)] = 0x75, /* JNE */
804 [GET_BPF_OP(BPF_JGT)] = 0x77, /* JA */
805 [GET_BPF_OP(EBPF_JLT)] = 0x72, /* JB */
806 [GET_BPF_OP(BPF_JGE)] = 0x73, /* JAE */
807 [GET_BPF_OP(EBPF_JLE)] = 0x76, /* JBE */
808 [GET_BPF_OP(EBPF_JSGT)] = 0x7F, /* JG */
809 [GET_BPF_OP(EBPF_JSLT)] = 0x7C, /* JL */
810 [GET_BPF_OP(EBPF_JSGE)] = 0x7D, /*JGE */
811 [GET_BPF_OP(EBPF_JSLE)] = 0x7E, /* JLE */
812 [GET_BPF_OP(BPF_JSET)] = 0x75, /*JNE */
815 static const uint8_t op32[][2] = {
816 [GET_BPF_OP(BPF_JEQ)] = {0x0F, 0x84}, /* JE */
817 [GET_BPF_OP(EBPF_JNE)] = {0x0F, 0x85}, /* JNE */
818 [GET_BPF_OP(BPF_JGT)] = {0x0F, 0x87}, /* JA */
819 [GET_BPF_OP(EBPF_JLT)] = {0x0F, 0x82}, /* JB */
820 [GET_BPF_OP(BPF_JGE)] = {0x0F, 0x83}, /* JAE */
821 [GET_BPF_OP(EBPF_JLE)] = {0x0F, 0x86}, /* JBE */
822 [GET_BPF_OP(EBPF_JSGT)] = {0x0F, 0x8F}, /* JG */
823 [GET_BPF_OP(EBPF_JSLT)] = {0x0F, 0x8C}, /* JL */
824 [GET_BPF_OP(EBPF_JSGE)] = {0x0F, 0x8D}, /*JGE */
825 [GET_BPF_OP(EBPF_JSLE)] = {0x0F, 0x8E}, /* JLE */
826 [GET_BPF_OP(BPF_JSET)] = {0x0F, 0x85}, /*JNE */
829 const int32_t sz8 = sizeof(op8[0]) + sizeof(uint8_t);
830 const int32_t sz32 = sizeof(op32[0]) + sizeof(uint32_t);
832 /* max possible jcc instruction size */
833 const int32_t iszm = RTE_MAX(sz8, sz32);
836 imsz = RTE_MAX(imm_size(joff), imm_size(joff + iszm));
838 bop = GET_BPF_OP(op);
841 emit_bytes(st, &op8[bop], sizeof(op8[bop]));
844 emit_bytes(st, op32[bop], sizeof(op32[bop]));
848 emit_imm(st, joff, imsz);
863 * where 'ofs' is the target offset for the BPF bytecode.
866 emit_jcc(struct bpf_jit_state *st, uint32_t op, int32_t ofs)
868 emit_abs_jcc(st, op, st->off[st->idx + ofs]);
873 * emit cmp <imm>, %<dreg>
876 emit_cmp_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
881 const uint8_t op8 = 0x83;
882 const uint8_t op32 = 0x81;
883 const uint8_t mods = 7;
885 imsz = imm_size(imm);
886 ops = (imsz == 1) ? op8 : op32;
888 emit_rex(st, op, 0, dreg);
889 emit_bytes(st, &ops, sizeof(ops));
890 emit_modregrm(st, MOD_DIRECT, mods, dreg);
891 emit_imm(st, imm, imsz);
895 * emit test <imm>, %<dreg>
898 emit_tst_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
900 const uint8_t ops = 0xF7;
901 const uint8_t mods = 0;
903 emit_rex(st, op, 0, dreg);
904 emit_bytes(st, &ops, sizeof(ops));
905 emit_modregrm(st, MOD_DIRECT, mods, dreg);
906 emit_imm(st, imm, imm_size(imm));
910 emit_jcc_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg,
911 uint32_t imm, int32_t ofs)
913 if (BPF_OP(op) == BPF_JSET)
914 emit_tst_imm(st, EBPF_ALU64, dreg, imm);
916 emit_cmp_imm(st, EBPF_ALU64, dreg, imm);
918 emit_jcc(st, op, ofs);
922 * emit test %<sreg>, %<dreg>
925 emit_tst_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
928 const uint8_t ops = 0x85;
930 emit_rex(st, op, sreg, dreg);
931 emit_bytes(st, &ops, sizeof(ops));
932 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
936 * emit cmp %<sreg>, %<dreg>
939 emit_cmp_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
942 const uint8_t ops = 0x39;
944 emit_rex(st, op, sreg, dreg);
945 emit_bytes(st, &ops, sizeof(ops));
946 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
951 emit_jcc_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
952 uint32_t dreg, int32_t ofs)
954 if (BPF_OP(op) == BPF_JSET)
955 emit_tst_reg(st, EBPF_ALU64, sreg, dreg);
957 emit_cmp_reg(st, EBPF_ALU64, sreg, dreg);
959 emit_jcc(st, op, ofs);
963 * note that rax:rdx are implicitly used as source/destination registers,
964 * so some reg spillage is necessary.
970 * for divisor as immediate value:
984 emit_div(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
989 const uint8_t ops = 0xF7;
990 const uint8_t mods = 6;
992 if (BPF_SRC(op) == BPF_X) {
994 /* check that src divisor is not zero */
995 emit_tst_reg(st, BPF_CLASS(op), sreg, sreg);
997 /* exit with return value zero */
998 emit_movcc_reg(st, BPF_CLASS(op) | BPF_JEQ | BPF_X, sreg, RAX);
999 emit_abs_jcc(st, BPF_JMP | BPF_JEQ | BPF_K, st->exit.off);
1002 /* save rax & rdx */
1004 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, REG_TMP0);
1006 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, REG_TMP1);
1008 /* fill rax & rdx */
1009 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, dreg, RAX);
1010 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, RDX, 0);
1012 if (BPF_SRC(op) == BPF_X) {
1020 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, sr, imm);
1023 emit_rex(st, op, 0, sr);
1024 emit_bytes(st, &ops, sizeof(ops));
1025 emit_modregrm(st, MOD_DIRECT, mods, sr);
1027 if (BPF_OP(op) == BPF_DIV)
1028 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, dreg);
1030 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, dreg);
1033 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP0, RAX);
1035 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX);
1039 * helper function, used by emit_ld_mbuf().
1040 * generates code for 'fast_path':
1041 * calculate load offset and check is it inside first packet segment.
1044 emit_ldmb_fast_path(struct bpf_jit_state *st, const uint32_t rg[EBPF_REG_7],
1045 uint32_t sreg, uint32_t mode, uint32_t sz, uint32_t imm,
1046 const int32_t ofs[LDMB_OFS_NUM])
1048 /* make R2 contain *off* value */
1050 if (sreg != rg[EBPF_REG_2]) {
1051 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K,
1052 rg[EBPF_REG_2], imm);
1053 if (mode == BPF_IND)
1054 emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
1055 sreg, rg[EBPF_REG_2]);
1057 /* BPF_IND with sreg == R2 */
1058 emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K,
1059 rg[EBPF_REG_2], imm);
1061 /* R3 = mbuf->data_len */
1062 emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H,
1063 rg[EBPF_REG_6], rg[EBPF_REG_3],
1064 offsetof(struct rte_mbuf, data_len));
1067 emit_alu_reg(st, EBPF_ALU64 | BPF_SUB | BPF_X,
1068 rg[EBPF_REG_2], rg[EBPF_REG_3]);
1070 /* JSLT R3, <sz> <slow_path> */
1071 emit_cmp_imm(st, EBPF_ALU64, rg[EBPF_REG_3], sz);
1072 emit_abs_jcc(st, BPF_JMP | EBPF_JSLT | BPF_K, ofs[LDMB_SLP_OFS]);
1074 /* R3 = mbuf->data_off */
1075 emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H,
1076 rg[EBPF_REG_6], rg[EBPF_REG_3],
1077 offsetof(struct rte_mbuf, data_off));
1079 /* R0 = mbuf->buf_addr */
1080 emit_ld_reg(st, BPF_LDX | BPF_MEM | EBPF_DW,
1081 rg[EBPF_REG_6], rg[EBPF_REG_0],
1082 offsetof(struct rte_mbuf, buf_addr));
1085 emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
1086 rg[EBPF_REG_3], rg[EBPF_REG_0]);
1089 emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
1090 rg[EBPF_REG_2], rg[EBPF_REG_0]);
1092 /* JMP <fin_part> */
1093 emit_abs_jmp(st, ofs[LDMB_FIN_OFS]);
1097 * helper function, used by emit_ld_mbuf().
1098 * generates code for 'slow_path':
1099 * call __rte_pktmbuf_read() and check return value.
1102 emit_ldmb_slow_path(struct bpf_jit_state *st, const uint32_t rg[EBPF_REG_7],
1105 /* make R3 contain *len* value (1/2/4) */
1107 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, rg[EBPF_REG_3], sz);
1109 /* make R4 contain (RBP - ldmb.stack_ofs) */
1111 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RBP, rg[EBPF_REG_4]);
1112 emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, rg[EBPF_REG_4],
1113 st->ldmb.stack_ofs);
1115 /* make R1 contain mbuf ptr */
1117 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
1118 rg[EBPF_REG_6], rg[EBPF_REG_1]);
1120 /* call rte_pktmbuf_read */
1121 emit_call(st, (uintptr_t)__rte_pktmbuf_read);
1123 /* check that return value (R0) is not zero */
1124 emit_tst_reg(st, EBPF_ALU64, rg[EBPF_REG_0], rg[EBPF_REG_0]);
1125 emit_abs_jcc(st, BPF_JMP | BPF_JEQ | BPF_K, st->exit.off);
1129 * helper function, used by emit_ld_mbuf().
1130 * generates final part of code for BPF_ABS/BPF_IND load:
1131 * perform data load and endianness conversion.
1132 * expects dreg to contain valid data pointer.
1135 emit_ldmb_fin(struct bpf_jit_state *st, uint32_t dreg, uint32_t opsz,
1138 emit_ld_reg(st, BPF_LDX | BPF_MEM | opsz, dreg, dreg, 0);
1139 if (sz != sizeof(uint8_t))
1140 emit_be2le(st, dreg, sz * CHAR_BIT);
1144 * emit code for BPF_ABS/BPF_IND load.
1145 * generates the following construction:
1147 * off = ins->sreg + ins->imm
1148 * if (mbuf->data_len - off < ins->opsz)
1150 * ptr = mbuf->buf_addr + mbuf->data_off + off;
1153 * typeof(ins->opsz) buf; //allocate space on the stack
1154 * ptr = __rte_pktmbuf_read(mbuf, off, ins->opsz, &buf);
1158 * res = *(typeof(ins->opsz))ptr;
1162 emit_ld_mbuf(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t imm)
1164 uint32_t i, mode, opsz, sz;
1165 uint32_t rg[EBPF_REG_7];
1166 int32_t ofs[LDMB_OFS_NUM];
1168 mode = BPF_MODE(op);
1169 opsz = BPF_SIZE(op);
1170 sz = bpf_size(opsz);
1172 for (i = 0; i != RTE_DIM(rg); i++)
1173 rg[i] = ebpf2x86[i];
1175 /* fill with fake offsets */
1176 for (i = 0; i != RTE_DIM(ofs); i++)
1177 ofs[i] = st->sz + INT8_MAX;
1179 /* dry run first to calculate jump offsets */
1181 ofs[LDMB_FSP_OFS] = st->sz;
1182 emit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs);
1183 ofs[LDMB_SLP_OFS] = st->sz;
1184 emit_ldmb_slow_path(st, rg, sz);
1185 ofs[LDMB_FIN_OFS] = st->sz;
1186 emit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz);
1188 RTE_VERIFY(ofs[LDMB_FIN_OFS] - ofs[LDMB_FSP_OFS] <= INT8_MAX);
1190 /* reset dry-run code and do a proper run */
1192 st->sz = ofs[LDMB_FSP_OFS];
1193 emit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs);
1194 emit_ldmb_slow_path(st, rg, sz);
1195 emit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz);
1199 emit_prolog(struct bpf_jit_state *st, int32_t stack_size)
1205 for (i = 0; i != RTE_DIM(save_regs); i++)
1206 spil += INUSE(st->reguse, save_regs[i]);
1208 /* we can avoid touching the stack at all */
1213 emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, RSP,
1214 spil * sizeof(uint64_t));
1217 for (i = 0; i != RTE_DIM(save_regs); i++) {
1218 if (INUSE(st->reguse, save_regs[i]) != 0) {
1219 emit_st_reg(st, BPF_STX | BPF_MEM | EBPF_DW,
1220 save_regs[i], RSP, ofs);
1221 ofs += sizeof(uint64_t);
1225 if (INUSE(st->reguse, RBP) != 0) {
1226 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RSP, RBP);
1227 emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, RSP, stack_size);
1235 emit_ret(struct bpf_jit_state *st)
1237 const uint8_t ops = 0xC3;
1239 emit_bytes(st, &ops, sizeof(ops));
1243 emit_epilog(struct bpf_jit_state *st)
1248 /* if we allready have an epilog generate a jump to it */
1249 if (st->exit.num++ != 0) {
1250 emit_abs_jmp(st, st->exit.off);
1254 /* store offset of epilog block */
1255 st->exit.off = st->sz;
1258 for (i = 0; i != RTE_DIM(save_regs); i++)
1259 spil += INUSE(st->reguse, save_regs[i]);
1263 if (INUSE(st->reguse, RBP) != 0)
1264 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
1268 for (i = 0; i != RTE_DIM(save_regs); i++) {
1269 if (INUSE(st->reguse, save_regs[i]) != 0) {
1270 emit_ld_reg(st, BPF_LDX | BPF_MEM | EBPF_DW,
1271 RSP, save_regs[i], ofs);
1272 ofs += sizeof(uint64_t);
1276 emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K, RSP,
1277 spil * sizeof(uint64_t));
1284 * walk through bpf code and translate them x86_64 one.
1287 emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)
1289 uint32_t i, dr, op, sr;
1290 const struct ebpf_insn *ins;
1292 /* reset state fields */
1295 st->ldmb.stack_ofs = bpf->stack_sz;
1297 emit_prolog(st, bpf->stack_sz);
1299 for (i = 0; i != bpf->prm.nb_ins; i++) {
1302 st->off[i] = st->sz;
1304 ins = bpf->prm.ins + i;
1306 dr = ebpf2x86[ins->dst_reg];
1307 sr = ebpf2x86[ins->src_reg];
1311 /* 32 bit ALU IMM operations */
1312 case (BPF_ALU | BPF_ADD | BPF_K):
1313 case (BPF_ALU | BPF_SUB | BPF_K):
1314 case (BPF_ALU | BPF_AND | BPF_K):
1315 case (BPF_ALU | BPF_OR | BPF_K):
1316 case (BPF_ALU | BPF_XOR | BPF_K):
1317 emit_alu_imm(st, op, dr, ins->imm);
1319 case (BPF_ALU | BPF_LSH | BPF_K):
1320 case (BPF_ALU | BPF_RSH | BPF_K):
1321 emit_shift_imm(st, op, dr, ins->imm);
1323 case (BPF_ALU | EBPF_MOV | BPF_K):
1324 emit_mov_imm(st, op, dr, ins->imm);
1326 /* 32 bit ALU REG operations */
1327 case (BPF_ALU | BPF_ADD | BPF_X):
1328 case (BPF_ALU | BPF_SUB | BPF_X):
1329 case (BPF_ALU | BPF_AND | BPF_X):
1330 case (BPF_ALU | BPF_OR | BPF_X):
1331 case (BPF_ALU | BPF_XOR | BPF_X):
1332 emit_alu_reg(st, op, sr, dr);
1334 case (BPF_ALU | BPF_LSH | BPF_X):
1335 case (BPF_ALU | BPF_RSH | BPF_X):
1336 emit_shift_reg(st, op, sr, dr);
1338 case (BPF_ALU | EBPF_MOV | BPF_X):
1339 emit_mov_reg(st, op, sr, dr);
1341 case (BPF_ALU | BPF_NEG):
1342 emit_neg(st, op, dr);
1344 case (BPF_ALU | EBPF_END | EBPF_TO_BE):
1345 emit_be2le(st, dr, ins->imm);
1347 case (BPF_ALU | EBPF_END | EBPF_TO_LE):
1348 emit_le2be(st, dr, ins->imm);
1350 /* 64 bit ALU IMM operations */
1351 case (EBPF_ALU64 | BPF_ADD | BPF_K):
1352 case (EBPF_ALU64 | BPF_SUB | BPF_K):
1353 case (EBPF_ALU64 | BPF_AND | BPF_K):
1354 case (EBPF_ALU64 | BPF_OR | BPF_K):
1355 case (EBPF_ALU64 | BPF_XOR | BPF_K):
1356 emit_alu_imm(st, op, dr, ins->imm);
1358 case (EBPF_ALU64 | BPF_LSH | BPF_K):
1359 case (EBPF_ALU64 | BPF_RSH | BPF_K):
1360 case (EBPF_ALU64 | EBPF_ARSH | BPF_K):
1361 emit_shift_imm(st, op, dr, ins->imm);
1363 case (EBPF_ALU64 | EBPF_MOV | BPF_K):
1364 emit_mov_imm(st, op, dr, ins->imm);
1366 /* 64 bit ALU REG operations */
1367 case (EBPF_ALU64 | BPF_ADD | BPF_X):
1368 case (EBPF_ALU64 | BPF_SUB | BPF_X):
1369 case (EBPF_ALU64 | BPF_AND | BPF_X):
1370 case (EBPF_ALU64 | BPF_OR | BPF_X):
1371 case (EBPF_ALU64 | BPF_XOR | BPF_X):
1372 emit_alu_reg(st, op, sr, dr);
1374 case (EBPF_ALU64 | BPF_LSH | BPF_X):
1375 case (EBPF_ALU64 | BPF_RSH | BPF_X):
1376 case (EBPF_ALU64 | EBPF_ARSH | BPF_X):
1377 emit_shift_reg(st, op, sr, dr);
1379 case (EBPF_ALU64 | EBPF_MOV | BPF_X):
1380 emit_mov_reg(st, op, sr, dr);
1382 case (EBPF_ALU64 | BPF_NEG):
1383 emit_neg(st, op, dr);
1385 /* multiply instructions */
1386 case (BPF_ALU | BPF_MUL | BPF_K):
1387 case (BPF_ALU | BPF_MUL | BPF_X):
1388 case (EBPF_ALU64 | BPF_MUL | BPF_K):
1389 case (EBPF_ALU64 | BPF_MUL | BPF_X):
1390 emit_mul(st, op, sr, dr, ins->imm);
1392 /* divide instructions */
1393 case (BPF_ALU | BPF_DIV | BPF_K):
1394 case (BPF_ALU | BPF_MOD | BPF_K):
1395 case (BPF_ALU | BPF_DIV | BPF_X):
1396 case (BPF_ALU | BPF_MOD | BPF_X):
1397 case (EBPF_ALU64 | BPF_DIV | BPF_K):
1398 case (EBPF_ALU64 | BPF_MOD | BPF_K):
1399 case (EBPF_ALU64 | BPF_DIV | BPF_X):
1400 case (EBPF_ALU64 | BPF_MOD | BPF_X):
1401 emit_div(st, op, sr, dr, ins->imm);
1403 /* load instructions */
1404 case (BPF_LDX | BPF_MEM | BPF_B):
1405 case (BPF_LDX | BPF_MEM | BPF_H):
1406 case (BPF_LDX | BPF_MEM | BPF_W):
1407 case (BPF_LDX | BPF_MEM | EBPF_DW):
1408 emit_ld_reg(st, op, sr, dr, ins->off);
1410 /* load 64 bit immediate value */
1411 case (BPF_LD | BPF_IMM | EBPF_DW):
1412 emit_ld_imm64(st, dr, ins[0].imm, ins[1].imm);
1415 /* load absolute/indirect instructions */
1416 case (BPF_LD | BPF_ABS | BPF_B):
1417 case (BPF_LD | BPF_ABS | BPF_H):
1418 case (BPF_LD | BPF_ABS | BPF_W):
1419 case (BPF_LD | BPF_IND | BPF_B):
1420 case (BPF_LD | BPF_IND | BPF_H):
1421 case (BPF_LD | BPF_IND | BPF_W):
1422 emit_ld_mbuf(st, op, sr, ins->imm);
1424 /* store instructions */
1425 case (BPF_STX | BPF_MEM | BPF_B):
1426 case (BPF_STX | BPF_MEM | BPF_H):
1427 case (BPF_STX | BPF_MEM | BPF_W):
1428 case (BPF_STX | BPF_MEM | EBPF_DW):
1429 emit_st_reg(st, op, sr, dr, ins->off);
1431 case (BPF_ST | BPF_MEM | BPF_B):
1432 case (BPF_ST | BPF_MEM | BPF_H):
1433 case (BPF_ST | BPF_MEM | BPF_W):
1434 case (BPF_ST | BPF_MEM | EBPF_DW):
1435 emit_st_imm(st, op, dr, ins->imm, ins->off);
1437 /* atomic add instructions */
1438 case (BPF_STX | EBPF_XADD | BPF_W):
1439 case (BPF_STX | EBPF_XADD | EBPF_DW):
1440 emit_st_xadd(st, op, sr, dr, ins->off);
1442 /* jump instructions */
1443 case (BPF_JMP | BPF_JA):
1444 emit_jmp(st, ins->off + 1);
1446 /* jump IMM instructions */
1447 case (BPF_JMP | BPF_JEQ | BPF_K):
1448 case (BPF_JMP | EBPF_JNE | BPF_K):
1449 case (BPF_JMP | BPF_JGT | BPF_K):
1450 case (BPF_JMP | EBPF_JLT | BPF_K):
1451 case (BPF_JMP | BPF_JGE | BPF_K):
1452 case (BPF_JMP | EBPF_JLE | BPF_K):
1453 case (BPF_JMP | EBPF_JSGT | BPF_K):
1454 case (BPF_JMP | EBPF_JSLT | BPF_K):
1455 case (BPF_JMP | EBPF_JSGE | BPF_K):
1456 case (BPF_JMP | EBPF_JSLE | BPF_K):
1457 case (BPF_JMP | BPF_JSET | BPF_K):
1458 emit_jcc_imm(st, op, dr, ins->imm, ins->off + 1);
1460 /* jump REG instructions */
1461 case (BPF_JMP | BPF_JEQ | BPF_X):
1462 case (BPF_JMP | EBPF_JNE | BPF_X):
1463 case (BPF_JMP | BPF_JGT | BPF_X):
1464 case (BPF_JMP | EBPF_JLT | BPF_X):
1465 case (BPF_JMP | BPF_JGE | BPF_X):
1466 case (BPF_JMP | EBPF_JLE | BPF_X):
1467 case (BPF_JMP | EBPF_JSGT | BPF_X):
1468 case (BPF_JMP | EBPF_JSLT | BPF_X):
1469 case (BPF_JMP | EBPF_JSGE | BPF_X):
1470 case (BPF_JMP | EBPF_JSLE | BPF_X):
1471 case (BPF_JMP | BPF_JSET | BPF_X):
1472 emit_jcc_reg(st, op, sr, dr, ins->off + 1);
1474 /* call instructions */
1475 case (BPF_JMP | EBPF_CALL):
1477 (uintptr_t)bpf->prm.xsym[ins->imm].func.val);
1479 /* return instruction */
1480 case (BPF_JMP | EBPF_EXIT):
1485 "%s(%p): invalid opcode %#x at pc: %u;\n",
1486 __func__, bpf, ins->code, i);
1495 * produce a native ISA version of the given BPF code.
1498 bpf_jit_x86(struct rte_bpf *bpf)
1503 struct bpf_jit_state st;
1506 memset(&st, 0, sizeof(st));
1507 st.off = malloc(bpf->prm.nb_ins * sizeof(st.off[0]));
1511 /* fill with fake offsets */
1512 st.exit.off = INT32_MAX;
1513 for (i = 0; i != bpf->prm.nb_ins; i++)
1514 st.off[i] = INT32_MAX;
1517 * dry runs, used to calculate total code size and valid jump offsets.
1518 * stop when we get minimal possible size
1522 rc = emit(&st, bpf);
1523 } while (rc == 0 && sz != st.sz);
1527 /* allocate memory needed */
1528 st.ins = mmap(NULL, st.sz, PROT_READ | PROT_WRITE,
1529 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
1530 if (st.ins == MAP_FAILED)
1534 rc = emit(&st, bpf);
1537 if (rc == 0 && mprotect(st.ins, st.sz, PROT_READ | PROT_EXEC) != 0)
1541 munmap(st.ins, st.sz);
1543 bpf->jit.func = (void *)st.ins;
1544 bpf->jit.sz = st.sz;