1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
10 #include <rte_common.h>
12 #include <rte_debug.h>
13 #include <rte_memory.h>
15 #include <rte_byteorder.h>
19 #define GET_BPF_OP(op) (BPF_OP(op) >> 4)
22 RAX = 0, /* scratch, return value */
23 RCX = 1, /* scratch, 4th arg */
24 RDX = 2, /* scratch, 3rd arg */
25 RBX = 3, /* callee saved */
26 RSP = 4, /* stack pointer */
27 RBP = 5, /* frame pointer, callee saved */
28 RSI = 6, /* scratch, 2nd arg */
29 RDI = 7, /* scratch, 1st arg */
30 R8 = 8, /* scratch, 5th arg */
31 R9 = 9, /* scratch, 6th arg */
32 R10 = 10, /* scratch */
33 R11 = 11, /* scratch */
34 R12 = 12, /* callee saved */
35 R13 = 13, /* callee saved */
36 R14 = 14, /* callee saved */
37 R15 = 15, /* callee saved */
40 #define IS_EXT_REG(r) ((r) >= R8)
43 REX_PREFIX = 0x40, /* fixed value 0100 */
44 REX_W = 0x8, /* 64bit operand size */
45 REX_R = 0x4, /* extension of the ModRM.reg field */
46 REX_X = 0x2, /* extension of the SIB.index field */
47 REX_B = 0x1, /* extension of the ModRM.rm field */
65 * eBPF to x86_64 register mappings.
67 static const uint32_t ebpf2x86[] = {
82 * r10 and r11 are used as a scratch temporary registers.
91 * callee saved registers list.
92 * keep RBP as the last one.
94 static const uint32_t save_regs[] = {RBX, R12, R13, R14, R15, RBP};
96 struct bpf_jit_state {
108 #define INUSE(v, r) (((v) >> (r)) & 1)
109 #define USED(v, r) ((v) |= 1 << (r))
117 bpf_size(uint32_t bpf_op_sz)
119 if (bpf_op_sz == BPF_B)
120 return sizeof(uint8_t);
121 else if (bpf_op_sz == BPF_H)
122 return sizeof(uint16_t);
123 else if (bpf_op_sz == BPF_W)
124 return sizeof(uint32_t);
125 else if (bpf_op_sz == EBPF_DW)
126 return sizeof(uint64_t);
131 * In many cases for imm8 we can produce shorter code.
137 return sizeof(int8_t);
138 return sizeof(int32_t);
142 emit_bytes(struct bpf_jit_state *st, const uint8_t ins[], uint32_t sz)
146 if (st->ins != NULL) {
147 for (i = 0; i != sz; i++)
148 st->ins[st->sz + i] = ins[i];
154 emit_imm(struct bpf_jit_state *st, const uint32_t imm, uint32_t sz)
159 emit_bytes(st, v.u8, sz);
166 emit_rex(struct bpf_jit_state *st, uint32_t op, uint32_t reg, uint32_t rm)
170 /* mark operand registers as used*/
171 USED(st->reguse, reg);
172 USED(st->reguse, rm);
175 if (BPF_CLASS(op) == EBPF_ALU64 ||
176 op == (BPF_ST | BPF_MEM | EBPF_DW) ||
177 op == (BPF_STX | BPF_MEM | EBPF_DW) ||
178 op == (BPF_STX | EBPF_XADD | EBPF_DW) ||
179 op == (BPF_LD | BPF_IMM | EBPF_DW) ||
180 (BPF_CLASS(op) == BPF_LDX &&
181 BPF_MODE(op) == BPF_MEM &&
182 BPF_SIZE(op) != BPF_W))
191 /* store using SIL, DIL */
192 if (op == (BPF_STX | BPF_MEM | BPF_B) && (reg == RDI || reg == RSI))
197 emit_bytes(st, &rex, sizeof(rex));
205 emit_modregrm(struct bpf_jit_state *st, uint32_t mod, uint32_t reg, uint32_t rm)
209 v = mod << 6 | (reg & 7) << 3 | (rm & 7);
210 emit_bytes(st, &v, sizeof(v));
217 emit_sib(struct bpf_jit_state *st, uint32_t scale, uint32_t idx, uint32_t base)
221 v = scale << 6 | (idx & 7) << 3 | (base & 7);
222 emit_bytes(st, &v, sizeof(v));
226 * emit xchg %<sreg>, %<dreg>
229 emit_xchg_reg(struct bpf_jit_state *st, uint32_t sreg, uint32_t dreg)
231 const uint8_t ops = 0x87;
233 emit_rex(st, EBPF_ALU64, sreg, dreg);
234 emit_bytes(st, &ops, sizeof(ops));
235 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
242 emit_neg(struct bpf_jit_state *st, uint32_t op, uint32_t dreg)
244 const uint8_t ops = 0xF7;
245 const uint8_t mods = 3;
247 emit_rex(st, op, 0, dreg);
248 emit_bytes(st, &ops, sizeof(ops));
249 emit_modregrm(st, MOD_DIRECT, mods, dreg);
253 * emit mov %<sreg>, %<dreg>
256 emit_mov_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
259 const uint8_t ops = 0x89;
261 /* if operands are 32-bit, then it can be used to clear upper 32-bit */
262 if (sreg != dreg || BPF_CLASS(op) == BPF_ALU) {
263 emit_rex(st, op, sreg, dreg);
264 emit_bytes(st, &ops, sizeof(ops));
265 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
270 * emit movzwl %<sreg>, %<dreg>
273 emit_movzwl(struct bpf_jit_state *st, uint32_t sreg, uint32_t dreg)
275 static const uint8_t ops[] = {0x0F, 0xB7};
277 emit_rex(st, BPF_ALU, sreg, dreg);
278 emit_bytes(st, ops, sizeof(ops));
279 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
283 * emit ror <imm8>, %<dreg>
286 emit_ror_imm(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
288 const uint8_t prfx = 0x66;
289 const uint8_t ops = 0xC1;
290 const uint8_t mods = 1;
292 emit_bytes(st, &prfx, sizeof(prfx));
293 emit_rex(st, BPF_ALU, 0, dreg);
294 emit_bytes(st, &ops, sizeof(ops));
295 emit_modregrm(st, MOD_DIRECT, mods, dreg);
296 emit_imm(st, imm, imm_size(imm));
303 emit_be2le_48(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
307 const uint8_t ops = 0x0F;
308 const uint8_t mods = 1;
310 rop = (imm == 64) ? EBPF_ALU64 : BPF_ALU;
311 emit_rex(st, rop, 0, dreg);
312 emit_bytes(st, &ops, sizeof(ops));
313 emit_modregrm(st, MOD_DIRECT, mods, dreg);
317 emit_be2le(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
320 emit_ror_imm(st, dreg, 8);
321 emit_movzwl(st, dreg, dreg);
323 emit_be2le_48(st, dreg, imm);
327 * In general it is NOP for x86.
328 * Just clear the upper bits.
331 emit_le2be(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
334 emit_movzwl(st, dreg, dreg);
336 emit_mov_reg(st, BPF_ALU | EBPF_MOV | BPF_X, dreg, dreg);
348 emit_alu_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
353 const uint8_t op8 = 0x83;
354 const uint8_t op32 = 0x81;
355 static const uint8_t mods[] = {
356 [GET_BPF_OP(BPF_ADD)] = 0,
357 [GET_BPF_OP(BPF_AND)] = 4,
358 [GET_BPF_OP(BPF_OR)] = 1,
359 [GET_BPF_OP(BPF_SUB)] = 5,
360 [GET_BPF_OP(BPF_XOR)] = 6,
363 bop = GET_BPF_OP(op);
366 imsz = imm_size(imm);
367 opcode = (imsz == 1) ? op8 : op32;
369 emit_rex(st, op, 0, dreg);
370 emit_bytes(st, &opcode, sizeof(opcode));
371 emit_modregrm(st, MOD_DIRECT, mod, dreg);
372 emit_imm(st, imm, imsz);
377 * add %<sreg>, %<dreg>
378 * and %<sreg>, %<dreg>
379 * or %<sreg>, %<dreg>
380 * sub %<sreg>, %<dreg>
381 * xor %<sreg>, %<dreg>
384 emit_alu_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
389 static const uint8_t ops[] = {
390 [GET_BPF_OP(BPF_ADD)] = 0x01,
391 [GET_BPF_OP(BPF_AND)] = 0x21,
392 [GET_BPF_OP(BPF_OR)] = 0x09,
393 [GET_BPF_OP(BPF_SUB)] = 0x29,
394 [GET_BPF_OP(BPF_XOR)] = 0x31,
397 bop = GET_BPF_OP(op);
399 emit_rex(st, op, sreg, dreg);
400 emit_bytes(st, &ops[bop], sizeof(ops[bop]));
401 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
405 emit_shift(struct bpf_jit_state *st, uint32_t op, uint32_t dreg)
410 static const uint8_t ops[] = {0xC1, 0xD3};
411 static const uint8_t mods[] = {
412 [GET_BPF_OP(BPF_LSH)] = 4,
413 [GET_BPF_OP(BPF_RSH)] = 5,
414 [GET_BPF_OP(EBPF_ARSH)] = 7,
417 bop = GET_BPF_OP(op);
419 opx = (BPF_SRC(op) == BPF_X);
421 emit_rex(st, op, 0, dreg);
422 emit_bytes(st, &ops[opx], sizeof(ops[opx]));
423 emit_modregrm(st, MOD_DIRECT, mod, dreg);
433 emit_shift_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg,
436 emit_shift(st, op, dreg);
437 emit_imm(st, imm, imm_size(imm));
445 * note that rcx is implicitly used as a source register, so few extra
446 * instructions for register spillage might be necessary.
449 emit_shift_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
453 emit_xchg_reg(st, RCX, sreg);
455 emit_shift(st, op, (dreg == RCX) ? sreg : dreg);
458 emit_xchg_reg(st, RCX, sreg);
462 * emit mov <imm>, %<dreg>
465 emit_mov_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
467 const uint8_t ops = 0xC7;
470 /* replace 'mov 0, %<dst>' with 'xor %<dst>, %<dst>' */
471 op = BPF_CLASS(op) | BPF_XOR | BPF_X;
472 emit_alu_reg(st, op, dreg, dreg);
476 emit_rex(st, op, 0, dreg);
477 emit_bytes(st, &ops, sizeof(ops));
478 emit_modregrm(st, MOD_DIRECT, 0, dreg);
479 emit_imm(st, imm, sizeof(imm));
483 * emit mov <imm64>, %<dreg>
486 emit_ld_imm64(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm0,
489 const uint8_t ops = 0xB8;
492 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, dreg, imm0);
496 emit_rex(st, EBPF_ALU64, 0, dreg);
497 emit_bytes(st, &ops, sizeof(ops));
498 emit_modregrm(st, MOD_DIRECT, 0, dreg);
500 emit_imm(st, imm0, sizeof(imm0));
501 emit_imm(st, imm1, sizeof(imm1));
505 * note that rax:rdx are implicitly used as source/destination registers,
506 * so some reg spillage is necessary.
521 emit_mul(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
524 const uint8_t ops = 0xF7;
525 const uint8_t mods = 4;
528 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, REG_TMP0);
529 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, REG_TMP1);
532 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, dreg, RAX);
534 if (BPF_SRC(op) == BPF_X)
536 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
537 sreg == RAX ? REG_TMP0 : sreg, RDX);
540 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, RDX, imm);
542 emit_rex(st, op, RAX, RDX);
543 emit_bytes(st, &ops, sizeof(ops));
544 emit_modregrm(st, MOD_DIRECT, mods, RDX);
548 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX);
552 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, dreg);
554 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP0, RAX);
559 * emit mov <ofs>(%<sreg>), %<dreg>
560 * note that for non 64-bit ops, higher bits have to be cleared.
563 emit_ld_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
567 const uint8_t op32 = 0x8B;
568 const uint8_t op16[] = {0x0F, 0xB7};
569 const uint8_t op8[] = {0x0F, 0xB6};
571 emit_rex(st, op, dreg, sreg);
575 emit_bytes(st, op8, sizeof(op8));
576 else if (opsz == BPF_H)
577 emit_bytes(st, op16, sizeof(op16));
579 emit_bytes(st, &op32, sizeof(op32));
581 mods = (imm_size(ofs) == 1) ? MOD_IDISP8 : MOD_IDISP32;
583 emit_modregrm(st, mods, dreg, sreg);
584 if (sreg == RSP || sreg == R12)
585 emit_sib(st, SIB_SCALE_1, sreg, sreg);
586 emit_imm(st, ofs, imm_size(ofs));
591 * mov %<sreg>, <ofs>(%<dreg>)
592 * mov <imm>, <ofs>(%<dreg>)
595 emit_st_common(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
596 uint32_t dreg, uint32_t imm, int32_t ofs)
598 uint32_t mods, imsz, opsz, opx;
599 const uint8_t prfx16 = 0x66;
601 /* 8 bit instruction opcodes */
602 static const uint8_t op8[] = {0xC6, 0x88};
604 /* 16/32/64 bit instruction opcodes */
605 static const uint8_t ops[] = {0xC7, 0x89};
607 /* is the instruction has immediate value or src reg? */
608 opx = (BPF_CLASS(op) == BPF_STX);
612 emit_bytes(st, &prfx16, sizeof(prfx16));
614 emit_rex(st, op, sreg, dreg);
617 emit_bytes(st, &op8[opx], sizeof(op8[opx]));
619 emit_bytes(st, &ops[opx], sizeof(ops[opx]));
621 imsz = imm_size(ofs);
622 mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32;
624 emit_modregrm(st, mods, sreg, dreg);
626 if (dreg == RSP || dreg == R12)
627 emit_sib(st, SIB_SCALE_1, dreg, dreg);
629 emit_imm(st, ofs, imsz);
632 imsz = RTE_MIN(bpf_size(opsz), sizeof(imm));
633 emit_imm(st, imm, imsz);
638 emit_st_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm,
641 emit_st_common(st, op, 0, dreg, imm, ofs);
645 emit_st_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
648 emit_st_common(st, op, sreg, dreg, 0, ofs);
652 * emit lock add %<sreg>, <ofs>(%<dreg>)
655 emit_st_xadd(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
656 uint32_t dreg, int32_t ofs)
660 const uint8_t lck = 0xF0; /* lock prefix */
661 const uint8_t ops = 0x01; /* add opcode */
663 imsz = imm_size(ofs);
664 mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32;
666 emit_bytes(st, &lck, sizeof(lck));
667 emit_rex(st, op, sreg, dreg);
668 emit_bytes(st, &ops, sizeof(ops));
669 emit_modregrm(st, mods, sreg, dreg);
670 emit_imm(st, ofs, imsz);
675 * mov <imm64>, (%rax)
679 emit_call(struct bpf_jit_state *st, uintptr_t trg)
681 const uint8_t ops = 0xFF;
682 const uint8_t mods = 2;
684 emit_ld_imm64(st, RAX, trg, trg >> 32);
685 emit_bytes(st, &ops, sizeof(ops));
686 emit_modregrm(st, MOD_DIRECT, mods, RAX);
691 * where 'ofs' is the target offset for the native code.
694 emit_abs_jmp(struct bpf_jit_state *st, int32_t ofs)
699 const uint8_t op8 = 0xEB;
700 const uint8_t op32 = 0xE9;
702 const int32_t sz8 = sizeof(op8) + sizeof(uint8_t);
703 const int32_t sz32 = sizeof(op32) + sizeof(uint32_t);
705 /* max possible jmp instruction size */
706 const int32_t iszm = RTE_MAX(sz8, sz32);
709 imsz = RTE_MAX(imm_size(joff), imm_size(joff + iszm));
712 emit_bytes(st, &op8, sizeof(op8));
715 emit_bytes(st, &op32, sizeof(op32));
719 emit_imm(st, joff, imsz);
724 * where 'ofs' is the target offset for the BPF bytecode.
727 emit_jmp(struct bpf_jit_state *st, int32_t ofs)
729 emit_abs_jmp(st, st->off[st->idx + ofs]);
734 * cmovz %<sreg>, <%dreg>
735 * cmovne %<sreg>, <%dreg>
736 * cmova %<sreg>, <%dreg>
737 * cmovb %<sreg>, <%dreg>
738 * cmovae %<sreg>, <%dreg>
739 * cmovbe %<sreg>, <%dreg>
740 * cmovg %<sreg>, <%dreg>
741 * cmovl %<sreg>, <%dreg>
742 * cmovge %<sreg>, <%dreg>
743 * cmovle %<sreg>, <%dreg>
746 emit_movcc_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
751 static const uint8_t ops[][2] = {
752 [GET_BPF_OP(BPF_JEQ)] = {0x0F, 0x44}, /* CMOVZ */
753 [GET_BPF_OP(EBPF_JNE)] = {0x0F, 0x45}, /* CMOVNE */
754 [GET_BPF_OP(BPF_JGT)] = {0x0F, 0x47}, /* CMOVA */
755 [GET_BPF_OP(EBPF_JLT)] = {0x0F, 0x42}, /* CMOVB */
756 [GET_BPF_OP(BPF_JGE)] = {0x0F, 0x43}, /* CMOVAE */
757 [GET_BPF_OP(EBPF_JLE)] = {0x0F, 0x46}, /* CMOVBE */
758 [GET_BPF_OP(EBPF_JSGT)] = {0x0F, 0x4F}, /* CMOVG */
759 [GET_BPF_OP(EBPF_JSLT)] = {0x0F, 0x4C}, /* CMOVL */
760 [GET_BPF_OP(EBPF_JSGE)] = {0x0F, 0x4D}, /* CMOVGE */
761 [GET_BPF_OP(EBPF_JSLE)] = {0x0F, 0x4E}, /* CMOVLE */
762 [GET_BPF_OP(BPF_JSET)] = {0x0F, 0x45}, /* CMOVNE */
765 bop = GET_BPF_OP(op);
767 emit_rex(st, op, dreg, sreg);
768 emit_bytes(st, ops[bop], sizeof(ops[bop]));
769 emit_modregrm(st, MOD_DIRECT, dreg, sreg);
784 * where 'ofs' is the target offset for the native code.
787 emit_abs_jcc(struct bpf_jit_state *st, uint32_t op, int32_t ofs)
792 static const uint8_t op8[] = {
793 [GET_BPF_OP(BPF_JEQ)] = 0x74, /* JE */
794 [GET_BPF_OP(EBPF_JNE)] = 0x75, /* JNE */
795 [GET_BPF_OP(BPF_JGT)] = 0x77, /* JA */
796 [GET_BPF_OP(EBPF_JLT)] = 0x72, /* JB */
797 [GET_BPF_OP(BPF_JGE)] = 0x73, /* JAE */
798 [GET_BPF_OP(EBPF_JLE)] = 0x76, /* JBE */
799 [GET_BPF_OP(EBPF_JSGT)] = 0x7F, /* JG */
800 [GET_BPF_OP(EBPF_JSLT)] = 0x7C, /* JL */
801 [GET_BPF_OP(EBPF_JSGE)] = 0x7D, /*JGE */
802 [GET_BPF_OP(EBPF_JSLE)] = 0x7E, /* JLE */
803 [GET_BPF_OP(BPF_JSET)] = 0x75, /*JNE */
806 static const uint8_t op32[][2] = {
807 [GET_BPF_OP(BPF_JEQ)] = {0x0F, 0x84}, /* JE */
808 [GET_BPF_OP(EBPF_JNE)] = {0x0F, 0x85}, /* JNE */
809 [GET_BPF_OP(BPF_JGT)] = {0x0F, 0x87}, /* JA */
810 [GET_BPF_OP(EBPF_JLT)] = {0x0F, 0x82}, /* JB */
811 [GET_BPF_OP(BPF_JGE)] = {0x0F, 0x83}, /* JAE */
812 [GET_BPF_OP(EBPF_JLE)] = {0x0F, 0x86}, /* JBE */
813 [GET_BPF_OP(EBPF_JSGT)] = {0x0F, 0x8F}, /* JG */
814 [GET_BPF_OP(EBPF_JSLT)] = {0x0F, 0x8C}, /* JL */
815 [GET_BPF_OP(EBPF_JSGE)] = {0x0F, 0x8D}, /*JGE */
816 [GET_BPF_OP(EBPF_JSLE)] = {0x0F, 0x8E}, /* JLE */
817 [GET_BPF_OP(BPF_JSET)] = {0x0F, 0x85}, /*JNE */
820 const int32_t sz8 = sizeof(op8[0]) + sizeof(uint8_t);
821 const int32_t sz32 = sizeof(op32[0]) + sizeof(uint32_t);
823 /* max possible jcc instruction size */
824 const int32_t iszm = RTE_MAX(sz8, sz32);
827 imsz = RTE_MAX(imm_size(joff), imm_size(joff + iszm));
829 bop = GET_BPF_OP(op);
832 emit_bytes(st, &op8[bop], sizeof(op8[bop]));
835 emit_bytes(st, op32[bop], sizeof(op32[bop]));
839 emit_imm(st, joff, imsz);
854 * where 'ofs' is the target offset for the BPF bytecode.
857 emit_jcc(struct bpf_jit_state *st, uint32_t op, int32_t ofs)
859 emit_abs_jcc(st, op, st->off[st->idx + ofs]);
864 * emit cmp <imm>, %<dreg>
867 emit_cmp_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
872 const uint8_t op8 = 0x83;
873 const uint8_t op32 = 0x81;
874 const uint8_t mods = 7;
876 imsz = imm_size(imm);
877 ops = (imsz == 1) ? op8 : op32;
879 emit_rex(st, op, 0, dreg);
880 emit_bytes(st, &ops, sizeof(ops));
881 emit_modregrm(st, MOD_DIRECT, mods, dreg);
882 emit_imm(st, imm, imsz);
886 * emit test <imm>, %<dreg>
889 emit_tst_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
891 const uint8_t ops = 0xF7;
892 const uint8_t mods = 0;
894 emit_rex(st, op, 0, dreg);
895 emit_bytes(st, &ops, sizeof(ops));
896 emit_modregrm(st, MOD_DIRECT, mods, dreg);
897 emit_imm(st, imm, imm_size(imm));
901 emit_jcc_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg,
902 uint32_t imm, int32_t ofs)
904 if (BPF_OP(op) == BPF_JSET)
905 emit_tst_imm(st, EBPF_ALU64, dreg, imm);
907 emit_cmp_imm(st, EBPF_ALU64, dreg, imm);
909 emit_jcc(st, op, ofs);
913 * emit test %<sreg>, %<dreg>
916 emit_tst_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
919 const uint8_t ops = 0x85;
921 emit_rex(st, op, sreg, dreg);
922 emit_bytes(st, &ops, sizeof(ops));
923 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
927 * emit cmp %<sreg>, %<dreg>
930 emit_cmp_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
933 const uint8_t ops = 0x39;
935 emit_rex(st, op, sreg, dreg);
936 emit_bytes(st, &ops, sizeof(ops));
937 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
942 emit_jcc_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
943 uint32_t dreg, int32_t ofs)
945 if (BPF_OP(op) == BPF_JSET)
946 emit_tst_reg(st, EBPF_ALU64, sreg, dreg);
948 emit_cmp_reg(st, EBPF_ALU64, sreg, dreg);
950 emit_jcc(st, op, ofs);
954 * note that rax:rdx are implicitly used as source/destination registers,
955 * so some reg spillage is necessary.
961 * for divisor as immediate value:
975 emit_div(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
980 const uint8_t ops = 0xF7;
981 const uint8_t mods = 6;
983 if (BPF_SRC(op) == BPF_X) {
985 /* check that src divisor is not zero */
986 emit_tst_reg(st, BPF_CLASS(op), sreg, sreg);
988 /* exit with return value zero */
989 emit_movcc_reg(st, BPF_CLASS(op) | BPF_JEQ | BPF_X, sreg, RAX);
990 emit_abs_jcc(st, BPF_JMP | BPF_JEQ | BPF_K, st->exit.off);
995 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, REG_TMP0);
997 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, REG_TMP1);
1000 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, dreg, RAX);
1001 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, RDX, 0);
1003 if (BPF_SRC(op) == BPF_X) {
1011 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, sr, imm);
1014 emit_rex(st, op, 0, sr);
1015 emit_bytes(st, &ops, sizeof(ops));
1016 emit_modregrm(st, MOD_DIRECT, mods, sr);
1018 if (BPF_OP(op) == BPF_DIV)
1019 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, dreg);
1021 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, dreg);
1024 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP0, RAX);
1026 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX);
1030 emit_prolog(struct bpf_jit_state *st, int32_t stack_size)
1036 for (i = 0; i != RTE_DIM(save_regs); i++)
1037 spil += INUSE(st->reguse, save_regs[i]);
1039 /* we can avoid touching the stack at all */
1044 emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, RSP,
1045 spil * sizeof(uint64_t));
1048 for (i = 0; i != RTE_DIM(save_regs); i++) {
1049 if (INUSE(st->reguse, save_regs[i]) != 0) {
1050 emit_st_reg(st, BPF_STX | BPF_MEM | EBPF_DW,
1051 save_regs[i], RSP, ofs);
1052 ofs += sizeof(uint64_t);
1056 if (INUSE(st->reguse, RBP) != 0) {
1057 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RSP, RBP);
1058 emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, RSP, stack_size);
1066 emit_ret(struct bpf_jit_state *st)
1068 const uint8_t ops = 0xC3;
1070 emit_bytes(st, &ops, sizeof(ops));
1074 emit_epilog(struct bpf_jit_state *st)
1079 /* if we allready have an epilog generate a jump to it */
1080 if (st->exit.num++ != 0) {
1081 emit_abs_jmp(st, st->exit.off);
1085 /* store offset of epilog block */
1086 st->exit.off = st->sz;
1089 for (i = 0; i != RTE_DIM(save_regs); i++)
1090 spil += INUSE(st->reguse, save_regs[i]);
1094 if (INUSE(st->reguse, RBP) != 0)
1095 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
1099 for (i = 0; i != RTE_DIM(save_regs); i++) {
1100 if (INUSE(st->reguse, save_regs[i]) != 0) {
1101 emit_ld_reg(st, BPF_LDX | BPF_MEM | EBPF_DW,
1102 RSP, save_regs[i], ofs);
1103 ofs += sizeof(uint64_t);
1107 emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K, RSP,
1108 spil * sizeof(uint64_t));
1115 * walk through bpf code and translate them x86_64 one.
1118 emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)
1120 uint32_t i, dr, op, sr;
1121 const struct ebpf_insn *ins;
1123 /* reset state fields */
1127 emit_prolog(st, bpf->stack_sz);
1129 for (i = 0; i != bpf->prm.nb_ins; i++) {
1132 st->off[i] = st->sz;
1134 ins = bpf->prm.ins + i;
1136 dr = ebpf2x86[ins->dst_reg];
1137 sr = ebpf2x86[ins->src_reg];
1141 /* 32 bit ALU IMM operations */
1142 case (BPF_ALU | BPF_ADD | BPF_K):
1143 case (BPF_ALU | BPF_SUB | BPF_K):
1144 case (BPF_ALU | BPF_AND | BPF_K):
1145 case (BPF_ALU | BPF_OR | BPF_K):
1146 case (BPF_ALU | BPF_XOR | BPF_K):
1147 emit_alu_imm(st, op, dr, ins->imm);
1149 case (BPF_ALU | BPF_LSH | BPF_K):
1150 case (BPF_ALU | BPF_RSH | BPF_K):
1151 emit_shift_imm(st, op, dr, ins->imm);
1153 case (BPF_ALU | EBPF_MOV | BPF_K):
1154 emit_mov_imm(st, op, dr, ins->imm);
1156 /* 32 bit ALU REG operations */
1157 case (BPF_ALU | BPF_ADD | BPF_X):
1158 case (BPF_ALU | BPF_SUB | BPF_X):
1159 case (BPF_ALU | BPF_AND | BPF_X):
1160 case (BPF_ALU | BPF_OR | BPF_X):
1161 case (BPF_ALU | BPF_XOR | BPF_X):
1162 emit_alu_reg(st, op, sr, dr);
1164 case (BPF_ALU | BPF_LSH | BPF_X):
1165 case (BPF_ALU | BPF_RSH | BPF_X):
1166 emit_shift_reg(st, op, sr, dr);
1168 case (BPF_ALU | EBPF_MOV | BPF_X):
1169 emit_mov_reg(st, op, sr, dr);
1171 case (BPF_ALU | BPF_NEG):
1172 emit_neg(st, op, dr);
1174 case (BPF_ALU | EBPF_END | EBPF_TO_BE):
1175 emit_be2le(st, dr, ins->imm);
1177 case (BPF_ALU | EBPF_END | EBPF_TO_LE):
1178 emit_le2be(st, dr, ins->imm);
1180 /* 64 bit ALU IMM operations */
1181 case (EBPF_ALU64 | BPF_ADD | BPF_K):
1182 case (EBPF_ALU64 | BPF_SUB | BPF_K):
1183 case (EBPF_ALU64 | BPF_AND | BPF_K):
1184 case (EBPF_ALU64 | BPF_OR | BPF_K):
1185 case (EBPF_ALU64 | BPF_XOR | BPF_K):
1186 emit_alu_imm(st, op, dr, ins->imm);
1188 case (EBPF_ALU64 | BPF_LSH | BPF_K):
1189 case (EBPF_ALU64 | BPF_RSH | BPF_K):
1190 case (EBPF_ALU64 | EBPF_ARSH | BPF_K):
1191 emit_shift_imm(st, op, dr, ins->imm);
1193 case (EBPF_ALU64 | EBPF_MOV | BPF_K):
1194 emit_mov_imm(st, op, dr, ins->imm);
1196 /* 64 bit ALU REG operations */
1197 case (EBPF_ALU64 | BPF_ADD | BPF_X):
1198 case (EBPF_ALU64 | BPF_SUB | BPF_X):
1199 case (EBPF_ALU64 | BPF_AND | BPF_X):
1200 case (EBPF_ALU64 | BPF_OR | BPF_X):
1201 case (EBPF_ALU64 | BPF_XOR | BPF_X):
1202 emit_alu_reg(st, op, sr, dr);
1204 case (EBPF_ALU64 | BPF_LSH | BPF_X):
1205 case (EBPF_ALU64 | BPF_RSH | BPF_X):
1206 case (EBPF_ALU64 | EBPF_ARSH | BPF_X):
1207 emit_shift_reg(st, op, sr, dr);
1209 case (EBPF_ALU64 | EBPF_MOV | BPF_X):
1210 emit_mov_reg(st, op, sr, dr);
1212 case (EBPF_ALU64 | BPF_NEG):
1213 emit_neg(st, op, dr);
1215 /* multiply instructions */
1216 case (BPF_ALU | BPF_MUL | BPF_K):
1217 case (BPF_ALU | BPF_MUL | BPF_X):
1218 case (EBPF_ALU64 | BPF_MUL | BPF_K):
1219 case (EBPF_ALU64 | BPF_MUL | BPF_X):
1220 emit_mul(st, op, sr, dr, ins->imm);
1222 /* divide instructions */
1223 case (BPF_ALU | BPF_DIV | BPF_K):
1224 case (BPF_ALU | BPF_MOD | BPF_K):
1225 case (BPF_ALU | BPF_DIV | BPF_X):
1226 case (BPF_ALU | BPF_MOD | BPF_X):
1227 case (EBPF_ALU64 | BPF_DIV | BPF_K):
1228 case (EBPF_ALU64 | BPF_MOD | BPF_K):
1229 case (EBPF_ALU64 | BPF_DIV | BPF_X):
1230 case (EBPF_ALU64 | BPF_MOD | BPF_X):
1231 emit_div(st, op, sr, dr, ins->imm);
1233 /* load instructions */
1234 case (BPF_LDX | BPF_MEM | BPF_B):
1235 case (BPF_LDX | BPF_MEM | BPF_H):
1236 case (BPF_LDX | BPF_MEM | BPF_W):
1237 case (BPF_LDX | BPF_MEM | EBPF_DW):
1238 emit_ld_reg(st, op, sr, dr, ins->off);
1240 /* load 64 bit immediate value */
1241 case (BPF_LD | BPF_IMM | EBPF_DW):
1242 emit_ld_imm64(st, dr, ins[0].imm, ins[1].imm);
1245 /* store instructions */
1246 case (BPF_STX | BPF_MEM | BPF_B):
1247 case (BPF_STX | BPF_MEM | BPF_H):
1248 case (BPF_STX | BPF_MEM | BPF_W):
1249 case (BPF_STX | BPF_MEM | EBPF_DW):
1250 emit_st_reg(st, op, sr, dr, ins->off);
1252 case (BPF_ST | BPF_MEM | BPF_B):
1253 case (BPF_ST | BPF_MEM | BPF_H):
1254 case (BPF_ST | BPF_MEM | BPF_W):
1255 case (BPF_ST | BPF_MEM | EBPF_DW):
1256 emit_st_imm(st, op, dr, ins->imm, ins->off);
1258 /* atomic add instructions */
1259 case (BPF_STX | EBPF_XADD | BPF_W):
1260 case (BPF_STX | EBPF_XADD | EBPF_DW):
1261 emit_st_xadd(st, op, sr, dr, ins->off);
1263 /* jump instructions */
1264 case (BPF_JMP | BPF_JA):
1265 emit_jmp(st, ins->off + 1);
1267 /* jump IMM instructions */
1268 case (BPF_JMP | BPF_JEQ | BPF_K):
1269 case (BPF_JMP | EBPF_JNE | BPF_K):
1270 case (BPF_JMP | BPF_JGT | BPF_K):
1271 case (BPF_JMP | EBPF_JLT | BPF_K):
1272 case (BPF_JMP | BPF_JGE | BPF_K):
1273 case (BPF_JMP | EBPF_JLE | BPF_K):
1274 case (BPF_JMP | EBPF_JSGT | BPF_K):
1275 case (BPF_JMP | EBPF_JSLT | BPF_K):
1276 case (BPF_JMP | EBPF_JSGE | BPF_K):
1277 case (BPF_JMP | EBPF_JSLE | BPF_K):
1278 case (BPF_JMP | BPF_JSET | BPF_K):
1279 emit_jcc_imm(st, op, dr, ins->imm, ins->off + 1);
1281 /* jump REG instructions */
1282 case (BPF_JMP | BPF_JEQ | BPF_X):
1283 case (BPF_JMP | EBPF_JNE | BPF_X):
1284 case (BPF_JMP | BPF_JGT | BPF_X):
1285 case (BPF_JMP | EBPF_JLT | BPF_X):
1286 case (BPF_JMP | BPF_JGE | BPF_X):
1287 case (BPF_JMP | EBPF_JLE | BPF_X):
1288 case (BPF_JMP | EBPF_JSGT | BPF_X):
1289 case (BPF_JMP | EBPF_JSLT | BPF_X):
1290 case (BPF_JMP | EBPF_JSGE | BPF_X):
1291 case (BPF_JMP | EBPF_JSLE | BPF_X):
1292 case (BPF_JMP | BPF_JSET | BPF_X):
1293 emit_jcc_reg(st, op, sr, dr, ins->off + 1);
1295 /* call instructions */
1296 case (BPF_JMP | EBPF_CALL):
1297 emit_call(st, (uintptr_t)bpf->prm.xsym[ins->imm].func);
1299 /* return instruction */
1300 case (BPF_JMP | EBPF_EXIT):
1305 "%s(%p): invalid opcode %#x at pc: %u;\n",
1306 __func__, bpf, ins->code, i);
1315 * produce a native ISA version of the given BPF code.
1318 bpf_jit_x86(struct rte_bpf *bpf)
1323 struct bpf_jit_state st;
1326 memset(&st, 0, sizeof(st));
1327 st.off = malloc(bpf->prm.nb_ins * sizeof(st.off[0]));
1331 /* fill with fake offsets */
1332 st.exit.off = INT32_MAX;
1333 for (i = 0; i != bpf->prm.nb_ins; i++)
1334 st.off[i] = INT32_MAX;
1337 * dry runs, used to calculate total code size and valid jump offsets.
1338 * stop when we get minimal possible size
1342 rc = emit(&st, bpf);
1343 } while (rc == 0 && sz != st.sz);
1347 /* allocate memory needed */
1348 st.ins = mmap(NULL, st.sz, PROT_READ | PROT_WRITE,
1349 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
1350 if (st.ins == MAP_FAILED)
1354 rc = emit(&st, bpf);
1357 if (rc == 0 && mprotect(st.ins, st.sz, PROT_READ | PROT_EXEC) != 0)
1361 munmap(st.ins, st.sz);
1363 bpf->jit.func = (void *)st.ins;
1364 bpf->jit.sz = st.sz;