1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
10 #include <rte_common.h>
12 #include <rte_debug.h>
13 #include <rte_memory.h>
15 #include <rte_byteorder.h>
19 #define GET_BPF_OP(op) (BPF_OP(op) >> 4)
22 RAX = 0, /* scratch, return value */
23 RCX = 1, /* scratch, 4th arg */
24 RDX = 2, /* scratch, 3rd arg */
25 RBX = 3, /* callee saved */
26 RSP = 4, /* stack pointer */
27 RBP = 5, /* frame pointer, callee saved */
28 RSI = 6, /* scratch, 2nd arg */
29 RDI = 7, /* scratch, 1st arg */
30 R8 = 8, /* scratch, 5th arg */
31 R9 = 9, /* scratch, 6th arg */
32 R10 = 10, /* scratch */
33 R11 = 11, /* scratch */
34 R12 = 12, /* callee saved */
35 R13 = 13, /* callee saved */
36 R14 = 14, /* callee saved */
37 R15 = 15, /* callee saved */
40 #define IS_EXT_REG(r) ((r) >= R8)
43 REX_PREFIX = 0x40, /* fixed value 0100 */
44 REX_W = 0x8, /* 64bit operand size */
45 REX_R = 0x4, /* extension of the ModRM.reg field */
46 REX_X = 0x2, /* extension of the SIB.index field */
47 REX_B = 0x1, /* extension of the ModRM.rm field */
65 * eBPF to x86_64 register mappings.
67 static const uint32_t ebpf2x86[] = {
82 * r10 and r11 are used as a scratch temporary registers.
91 * callee saved registers list.
92 * keep RBP as the last one.
94 static const uint32_t save_regs[] = {RBX, R12, R13, R14, R15, RBP};
96 struct bpf_jit_state {
108 #define INUSE(v, r) (((v) >> (r)) & 1)
109 #define USED(v, r) ((v) |= 1 << (r))
117 * In many cases for imm8 we can produce shorter code.
123 return sizeof(int8_t);
124 return sizeof(int32_t);
128 emit_bytes(struct bpf_jit_state *st, const uint8_t ins[], uint32_t sz)
132 if (st->ins != NULL) {
133 for (i = 0; i != sz; i++)
134 st->ins[st->sz + i] = ins[i];
140 emit_imm(struct bpf_jit_state *st, const uint32_t imm, uint32_t sz)
145 emit_bytes(st, v.u8, sz);
152 emit_rex(struct bpf_jit_state *st, uint32_t op, uint32_t reg, uint32_t rm)
156 /* mark operand registers as used*/
157 USED(st->reguse, reg);
158 USED(st->reguse, rm);
161 if (BPF_CLASS(op) == EBPF_ALU64 ||
162 op == (BPF_ST | BPF_MEM | EBPF_DW) ||
163 op == (BPF_STX | BPF_MEM | EBPF_DW) ||
164 op == (BPF_STX | EBPF_XADD | EBPF_DW) ||
165 op == (BPF_LD | BPF_IMM | EBPF_DW) ||
166 (BPF_CLASS(op) == BPF_LDX &&
167 BPF_MODE(op) == BPF_MEM &&
168 BPF_SIZE(op) != BPF_W))
177 /* store using SIL, DIL */
178 if (op == (BPF_STX | BPF_MEM | BPF_B) && (reg == RDI || reg == RSI))
183 emit_bytes(st, &rex, sizeof(rex));
191 emit_modregrm(struct bpf_jit_state *st, uint32_t mod, uint32_t reg, uint32_t rm)
195 v = mod << 6 | (reg & 7) << 3 | (rm & 7);
196 emit_bytes(st, &v, sizeof(v));
203 emit_sib(struct bpf_jit_state *st, uint32_t scale, uint32_t idx, uint32_t base)
207 v = scale << 6 | (idx & 7) << 3 | (base & 7);
208 emit_bytes(st, &v, sizeof(v));
212 * emit xchg %<sreg>, %<dreg>
215 emit_xchg_reg(struct bpf_jit_state *st, uint32_t sreg, uint32_t dreg)
217 const uint8_t ops = 0x87;
219 emit_rex(st, EBPF_ALU64, sreg, dreg);
220 emit_bytes(st, &ops, sizeof(ops));
221 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
228 emit_neg(struct bpf_jit_state *st, uint32_t op, uint32_t dreg)
230 const uint8_t ops = 0xF7;
231 const uint8_t mods = 3;
233 emit_rex(st, op, 0, dreg);
234 emit_bytes(st, &ops, sizeof(ops));
235 emit_modregrm(st, MOD_DIRECT, mods, dreg);
239 * emit mov %<sreg>, %<dreg>
242 emit_mov_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
245 const uint8_t ops = 0x89;
247 /* if operands are 32-bit, then it can be used to clear upper 32-bit */
248 if (sreg != dreg || BPF_CLASS(op) == BPF_ALU) {
249 emit_rex(st, op, sreg, dreg);
250 emit_bytes(st, &ops, sizeof(ops));
251 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
256 * emit movzwl %<sreg>, %<dreg>
259 emit_movzwl(struct bpf_jit_state *st, uint32_t sreg, uint32_t dreg)
261 static const uint8_t ops[] = {0x0F, 0xB7};
263 emit_rex(st, BPF_ALU, sreg, dreg);
264 emit_bytes(st, ops, sizeof(ops));
265 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
269 * emit ror <imm8>, %<dreg>
272 emit_ror_imm(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
274 const uint8_t prfx = 0x66;
275 const uint8_t ops = 0xC1;
276 const uint8_t mods = 1;
278 emit_bytes(st, &prfx, sizeof(prfx));
279 emit_rex(st, BPF_ALU, 0, dreg);
280 emit_bytes(st, &ops, sizeof(ops));
281 emit_modregrm(st, MOD_DIRECT, mods, dreg);
282 emit_imm(st, imm, imm_size(imm));
289 emit_be2le_48(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
293 const uint8_t ops = 0x0F;
294 const uint8_t mods = 1;
296 rop = (imm == 64) ? EBPF_ALU64 : BPF_ALU;
297 emit_rex(st, rop, 0, dreg);
298 emit_bytes(st, &ops, sizeof(ops));
299 emit_modregrm(st, MOD_DIRECT, mods, dreg);
303 emit_be2le(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
306 emit_ror_imm(st, dreg, 8);
307 emit_movzwl(st, dreg, dreg);
309 emit_be2le_48(st, dreg, imm);
313 * In general it is NOP for x86.
314 * Just clear the upper bits.
317 emit_le2be(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
320 emit_movzwl(st, dreg, dreg);
322 emit_mov_reg(st, BPF_ALU | EBPF_MOV | BPF_X, dreg, dreg);
334 emit_alu_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
339 const uint8_t op8 = 0x83;
340 const uint8_t op32 = 0x81;
341 static const uint8_t mods[] = {
342 [GET_BPF_OP(BPF_ADD)] = 0,
343 [GET_BPF_OP(BPF_AND)] = 4,
344 [GET_BPF_OP(BPF_OR)] = 1,
345 [GET_BPF_OP(BPF_SUB)] = 5,
346 [GET_BPF_OP(BPF_XOR)] = 6,
349 bop = GET_BPF_OP(op);
352 imsz = imm_size(imm);
353 opcode = (imsz == 1) ? op8 : op32;
355 emit_rex(st, op, 0, dreg);
356 emit_bytes(st, &opcode, sizeof(opcode));
357 emit_modregrm(st, MOD_DIRECT, mod, dreg);
358 emit_imm(st, imm, imsz);
363 * add %<sreg>, %<dreg>
364 * and %<sreg>, %<dreg>
365 * or %<sreg>, %<dreg>
366 * sub %<sreg>, %<dreg>
367 * xor %<sreg>, %<dreg>
370 emit_alu_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
375 static const uint8_t ops[] = {
376 [GET_BPF_OP(BPF_ADD)] = 0x01,
377 [GET_BPF_OP(BPF_AND)] = 0x21,
378 [GET_BPF_OP(BPF_OR)] = 0x09,
379 [GET_BPF_OP(BPF_SUB)] = 0x29,
380 [GET_BPF_OP(BPF_XOR)] = 0x31,
383 bop = GET_BPF_OP(op);
385 emit_rex(st, op, sreg, dreg);
386 emit_bytes(st, &ops[bop], sizeof(ops[bop]));
387 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
391 emit_shift(struct bpf_jit_state *st, uint32_t op, uint32_t dreg)
396 static const uint8_t ops[] = {0xC1, 0xD3};
397 static const uint8_t mods[] = {
398 [GET_BPF_OP(BPF_LSH)] = 4,
399 [GET_BPF_OP(BPF_RSH)] = 5,
400 [GET_BPF_OP(EBPF_ARSH)] = 7,
403 bop = GET_BPF_OP(op);
405 opx = (BPF_SRC(op) == BPF_X);
407 emit_rex(st, op, 0, dreg);
408 emit_bytes(st, &ops[opx], sizeof(ops[opx]));
409 emit_modregrm(st, MOD_DIRECT, mod, dreg);
419 emit_shift_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg,
422 emit_shift(st, op, dreg);
423 emit_imm(st, imm, imm_size(imm));
431 * note that rcx is implicitly used as a source register, so few extra
432 * instructions for register spillage might be necessary.
435 emit_shift_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
439 emit_xchg_reg(st, RCX, sreg);
441 emit_shift(st, op, (dreg == RCX) ? sreg : dreg);
444 emit_xchg_reg(st, RCX, sreg);
448 * emit mov <imm>, %<dreg>
451 emit_mov_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
453 const uint8_t ops = 0xC7;
456 /* replace 'mov 0, %<dst>' with 'xor %<dst>, %<dst>' */
457 op = BPF_CLASS(op) | BPF_XOR | BPF_X;
458 emit_alu_reg(st, op, dreg, dreg);
462 emit_rex(st, op, 0, dreg);
463 emit_bytes(st, &ops, sizeof(ops));
464 emit_modregrm(st, MOD_DIRECT, 0, dreg);
465 emit_imm(st, imm, sizeof(imm));
469 * emit mov <imm64>, %<dreg>
472 emit_ld_imm64(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm0,
475 const uint8_t ops = 0xB8;
478 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, dreg, imm0);
482 emit_rex(st, EBPF_ALU64, 0, dreg);
483 emit_bytes(st, &ops, sizeof(ops));
484 emit_modregrm(st, MOD_DIRECT, 0, dreg);
486 emit_imm(st, imm0, sizeof(imm0));
487 emit_imm(st, imm1, sizeof(imm1));
491 * note that rax:rdx are implicitly used as source/destination registers,
492 * so some reg spillage is necessary.
507 emit_mul(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
510 const uint8_t ops = 0xF7;
511 const uint8_t mods = 4;
514 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, REG_TMP0);
515 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, REG_TMP1);
518 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, dreg, RAX);
520 if (BPF_SRC(op) == BPF_X)
522 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
523 sreg == RAX ? REG_TMP0 : sreg, RDX);
526 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, RDX, imm);
528 emit_rex(st, op, RAX, RDX);
529 emit_bytes(st, &ops, sizeof(ops));
530 emit_modregrm(st, MOD_DIRECT, mods, RDX);
534 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX);
538 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, dreg);
540 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP0, RAX);
545 * emit mov <ofs>(%<sreg>), %<dreg>
546 * note that for non 64-bit ops, higher bits have to be cleared.
549 emit_ld_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
553 const uint8_t op32 = 0x8B;
554 const uint8_t op16[] = {0x0F, 0xB7};
555 const uint8_t op8[] = {0x0F, 0xB6};
557 emit_rex(st, op, dreg, sreg);
561 emit_bytes(st, op8, sizeof(op8));
562 else if (opsz == BPF_H)
563 emit_bytes(st, op16, sizeof(op16));
565 emit_bytes(st, &op32, sizeof(op32));
567 mods = (imm_size(ofs) == 1) ? MOD_IDISP8 : MOD_IDISP32;
569 emit_modregrm(st, mods, dreg, sreg);
570 if (sreg == RSP || sreg == R12)
571 emit_sib(st, SIB_SCALE_1, sreg, sreg);
572 emit_imm(st, ofs, imm_size(ofs));
577 * mov %<sreg>, <ofs>(%<dreg>)
578 * mov <imm>, <ofs>(%<dreg>)
581 emit_st_common(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
582 uint32_t dreg, uint32_t imm, int32_t ofs)
584 uint32_t mods, imsz, opsz, opx;
585 const uint8_t prfx16 = 0x66;
587 /* 8 bit instruction opcodes */
588 static const uint8_t op8[] = {0xC6, 0x88};
590 /* 16/32/64 bit instruction opcodes */
591 static const uint8_t ops[] = {0xC7, 0x89};
593 /* is the instruction has immediate value or src reg? */
594 opx = (BPF_CLASS(op) == BPF_STX);
598 emit_bytes(st, &prfx16, sizeof(prfx16));
600 emit_rex(st, op, sreg, dreg);
603 emit_bytes(st, &op8[opx], sizeof(op8[opx]));
605 emit_bytes(st, &ops[opx], sizeof(ops[opx]));
607 imsz = imm_size(ofs);
608 mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32;
610 emit_modregrm(st, mods, sreg, dreg);
612 if (dreg == RSP || dreg == R12)
613 emit_sib(st, SIB_SCALE_1, dreg, dreg);
615 emit_imm(st, ofs, imsz);
618 imsz = RTE_MIN(bpf_size(opsz), sizeof(imm));
619 emit_imm(st, imm, imsz);
624 emit_st_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm,
627 emit_st_common(st, op, 0, dreg, imm, ofs);
631 emit_st_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
634 emit_st_common(st, op, sreg, dreg, 0, ofs);
638 * emit lock add %<sreg>, <ofs>(%<dreg>)
641 emit_st_xadd(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
642 uint32_t dreg, int32_t ofs)
646 const uint8_t lck = 0xF0; /* lock prefix */
647 const uint8_t ops = 0x01; /* add opcode */
649 imsz = imm_size(ofs);
650 mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32;
652 emit_bytes(st, &lck, sizeof(lck));
653 emit_rex(st, op, sreg, dreg);
654 emit_bytes(st, &ops, sizeof(ops));
655 emit_modregrm(st, mods, sreg, dreg);
656 emit_imm(st, ofs, imsz);
661 * mov <imm64>, (%rax)
665 emit_call(struct bpf_jit_state *st, uintptr_t trg)
667 const uint8_t ops = 0xFF;
668 const uint8_t mods = 2;
670 emit_ld_imm64(st, RAX, trg, trg >> 32);
671 emit_bytes(st, &ops, sizeof(ops));
672 emit_modregrm(st, MOD_DIRECT, mods, RAX);
677 * where 'ofs' is the target offset for the native code.
680 emit_abs_jmp(struct bpf_jit_state *st, int32_t ofs)
685 const uint8_t op8 = 0xEB;
686 const uint8_t op32 = 0xE9;
688 const int32_t sz8 = sizeof(op8) + sizeof(uint8_t);
689 const int32_t sz32 = sizeof(op32) + sizeof(uint32_t);
691 /* max possible jmp instruction size */
692 const int32_t iszm = RTE_MAX(sz8, sz32);
695 imsz = RTE_MAX(imm_size(joff), imm_size(joff + iszm));
698 emit_bytes(st, &op8, sizeof(op8));
701 emit_bytes(st, &op32, sizeof(op32));
705 emit_imm(st, joff, imsz);
710 * where 'ofs' is the target offset for the BPF bytecode.
713 emit_jmp(struct bpf_jit_state *st, int32_t ofs)
715 emit_abs_jmp(st, st->off[st->idx + ofs]);
720 * cmovz %<sreg>, <%dreg>
721 * cmovne %<sreg>, <%dreg>
722 * cmova %<sreg>, <%dreg>
723 * cmovb %<sreg>, <%dreg>
724 * cmovae %<sreg>, <%dreg>
725 * cmovbe %<sreg>, <%dreg>
726 * cmovg %<sreg>, <%dreg>
727 * cmovl %<sreg>, <%dreg>
728 * cmovge %<sreg>, <%dreg>
729 * cmovle %<sreg>, <%dreg>
732 emit_movcc_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
737 static const uint8_t ops[][2] = {
738 [GET_BPF_OP(BPF_JEQ)] = {0x0F, 0x44}, /* CMOVZ */
739 [GET_BPF_OP(EBPF_JNE)] = {0x0F, 0x45}, /* CMOVNE */
740 [GET_BPF_OP(BPF_JGT)] = {0x0F, 0x47}, /* CMOVA */
741 [GET_BPF_OP(EBPF_JLT)] = {0x0F, 0x42}, /* CMOVB */
742 [GET_BPF_OP(BPF_JGE)] = {0x0F, 0x43}, /* CMOVAE */
743 [GET_BPF_OP(EBPF_JLE)] = {0x0F, 0x46}, /* CMOVBE */
744 [GET_BPF_OP(EBPF_JSGT)] = {0x0F, 0x4F}, /* CMOVG */
745 [GET_BPF_OP(EBPF_JSLT)] = {0x0F, 0x4C}, /* CMOVL */
746 [GET_BPF_OP(EBPF_JSGE)] = {0x0F, 0x4D}, /* CMOVGE */
747 [GET_BPF_OP(EBPF_JSLE)] = {0x0F, 0x4E}, /* CMOVLE */
748 [GET_BPF_OP(BPF_JSET)] = {0x0F, 0x45}, /* CMOVNE */
751 bop = GET_BPF_OP(op);
753 emit_rex(st, op, dreg, sreg);
754 emit_bytes(st, ops[bop], sizeof(ops[bop]));
755 emit_modregrm(st, MOD_DIRECT, dreg, sreg);
770 * where 'ofs' is the target offset for the native code.
773 emit_abs_jcc(struct bpf_jit_state *st, uint32_t op, int32_t ofs)
778 static const uint8_t op8[] = {
779 [GET_BPF_OP(BPF_JEQ)] = 0x74, /* JE */
780 [GET_BPF_OP(EBPF_JNE)] = 0x75, /* JNE */
781 [GET_BPF_OP(BPF_JGT)] = 0x77, /* JA */
782 [GET_BPF_OP(EBPF_JLT)] = 0x72, /* JB */
783 [GET_BPF_OP(BPF_JGE)] = 0x73, /* JAE */
784 [GET_BPF_OP(EBPF_JLE)] = 0x76, /* JBE */
785 [GET_BPF_OP(EBPF_JSGT)] = 0x7F, /* JG */
786 [GET_BPF_OP(EBPF_JSLT)] = 0x7C, /* JL */
787 [GET_BPF_OP(EBPF_JSGE)] = 0x7D, /*JGE */
788 [GET_BPF_OP(EBPF_JSLE)] = 0x7E, /* JLE */
789 [GET_BPF_OP(BPF_JSET)] = 0x75, /*JNE */
792 static const uint8_t op32[][2] = {
793 [GET_BPF_OP(BPF_JEQ)] = {0x0F, 0x84}, /* JE */
794 [GET_BPF_OP(EBPF_JNE)] = {0x0F, 0x85}, /* JNE */
795 [GET_BPF_OP(BPF_JGT)] = {0x0F, 0x87}, /* JA */
796 [GET_BPF_OP(EBPF_JLT)] = {0x0F, 0x82}, /* JB */
797 [GET_BPF_OP(BPF_JGE)] = {0x0F, 0x83}, /* JAE */
798 [GET_BPF_OP(EBPF_JLE)] = {0x0F, 0x86}, /* JBE */
799 [GET_BPF_OP(EBPF_JSGT)] = {0x0F, 0x8F}, /* JG */
800 [GET_BPF_OP(EBPF_JSLT)] = {0x0F, 0x8C}, /* JL */
801 [GET_BPF_OP(EBPF_JSGE)] = {0x0F, 0x8D}, /*JGE */
802 [GET_BPF_OP(EBPF_JSLE)] = {0x0F, 0x8E}, /* JLE */
803 [GET_BPF_OP(BPF_JSET)] = {0x0F, 0x85}, /*JNE */
806 const int32_t sz8 = sizeof(op8[0]) + sizeof(uint8_t);
807 const int32_t sz32 = sizeof(op32[0]) + sizeof(uint32_t);
809 /* max possible jcc instruction size */
810 const int32_t iszm = RTE_MAX(sz8, sz32);
813 imsz = RTE_MAX(imm_size(joff), imm_size(joff + iszm));
815 bop = GET_BPF_OP(op);
818 emit_bytes(st, &op8[bop], sizeof(op8[bop]));
821 emit_bytes(st, op32[bop], sizeof(op32[bop]));
825 emit_imm(st, joff, imsz);
840 * where 'ofs' is the target offset for the BPF bytecode.
843 emit_jcc(struct bpf_jit_state *st, uint32_t op, int32_t ofs)
845 emit_abs_jcc(st, op, st->off[st->idx + ofs]);
850 * emit cmp <imm>, %<dreg>
853 emit_cmp_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
858 const uint8_t op8 = 0x83;
859 const uint8_t op32 = 0x81;
860 const uint8_t mods = 7;
862 imsz = imm_size(imm);
863 ops = (imsz == 1) ? op8 : op32;
865 emit_rex(st, op, 0, dreg);
866 emit_bytes(st, &ops, sizeof(ops));
867 emit_modregrm(st, MOD_DIRECT, mods, dreg);
868 emit_imm(st, imm, imsz);
872 * emit test <imm>, %<dreg>
875 emit_tst_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
877 const uint8_t ops = 0xF7;
878 const uint8_t mods = 0;
880 emit_rex(st, op, 0, dreg);
881 emit_bytes(st, &ops, sizeof(ops));
882 emit_modregrm(st, MOD_DIRECT, mods, dreg);
883 emit_imm(st, imm, imm_size(imm));
887 emit_jcc_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg,
888 uint32_t imm, int32_t ofs)
890 if (BPF_OP(op) == BPF_JSET)
891 emit_tst_imm(st, EBPF_ALU64, dreg, imm);
893 emit_cmp_imm(st, EBPF_ALU64, dreg, imm);
895 emit_jcc(st, op, ofs);
899 * emit test %<sreg>, %<dreg>
902 emit_tst_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
905 const uint8_t ops = 0x85;
907 emit_rex(st, op, sreg, dreg);
908 emit_bytes(st, &ops, sizeof(ops));
909 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
913 * emit cmp %<sreg>, %<dreg>
916 emit_cmp_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
919 const uint8_t ops = 0x39;
921 emit_rex(st, op, sreg, dreg);
922 emit_bytes(st, &ops, sizeof(ops));
923 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
928 emit_jcc_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
929 uint32_t dreg, int32_t ofs)
931 if (BPF_OP(op) == BPF_JSET)
932 emit_tst_reg(st, EBPF_ALU64, sreg, dreg);
934 emit_cmp_reg(st, EBPF_ALU64, sreg, dreg);
936 emit_jcc(st, op, ofs);
940 * note that rax:rdx are implicitly used as source/destination registers,
941 * so some reg spillage is necessary.
947 * for divisor as immediate value:
961 emit_div(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
966 const uint8_t ops = 0xF7;
967 const uint8_t mods = 6;
969 if (BPF_SRC(op) == BPF_X) {
971 /* check that src divisor is not zero */
972 emit_tst_reg(st, BPF_CLASS(op), sreg, sreg);
974 /* exit with return value zero */
975 emit_movcc_reg(st, BPF_CLASS(op) | BPF_JEQ | BPF_X, sreg, RAX);
976 emit_abs_jcc(st, BPF_JMP | BPF_JEQ | BPF_K, st->exit.off);
981 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, REG_TMP0);
983 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, REG_TMP1);
986 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, dreg, RAX);
987 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, RDX, 0);
989 if (BPF_SRC(op) == BPF_X) {
997 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, sr, imm);
1000 emit_rex(st, op, 0, sr);
1001 emit_bytes(st, &ops, sizeof(ops));
1002 emit_modregrm(st, MOD_DIRECT, mods, sr);
1004 if (BPF_OP(op) == BPF_DIV)
1005 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, dreg);
1007 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, dreg);
1010 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP0, RAX);
1012 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX);
1016 emit_prolog(struct bpf_jit_state *st, int32_t stack_size)
1022 for (i = 0; i != RTE_DIM(save_regs); i++)
1023 spil += INUSE(st->reguse, save_regs[i]);
1025 /* we can avoid touching the stack at all */
1030 emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, RSP,
1031 spil * sizeof(uint64_t));
1034 for (i = 0; i != RTE_DIM(save_regs); i++) {
1035 if (INUSE(st->reguse, save_regs[i]) != 0) {
1036 emit_st_reg(st, BPF_STX | BPF_MEM | EBPF_DW,
1037 save_regs[i], RSP, ofs);
1038 ofs += sizeof(uint64_t);
1042 if (INUSE(st->reguse, RBP) != 0) {
1043 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RSP, RBP);
1044 emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, RSP, stack_size);
1052 emit_ret(struct bpf_jit_state *st)
1054 const uint8_t ops = 0xC3;
1056 emit_bytes(st, &ops, sizeof(ops));
1060 emit_epilog(struct bpf_jit_state *st)
1065 /* if we allready have an epilog generate a jump to it */
1066 if (st->exit.num++ != 0) {
1067 emit_abs_jmp(st, st->exit.off);
1071 /* store offset of epilog block */
1072 st->exit.off = st->sz;
1075 for (i = 0; i != RTE_DIM(save_regs); i++)
1076 spil += INUSE(st->reguse, save_regs[i]);
1080 if (INUSE(st->reguse, RBP) != 0)
1081 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
1085 for (i = 0; i != RTE_DIM(save_regs); i++) {
1086 if (INUSE(st->reguse, save_regs[i]) != 0) {
1087 emit_ld_reg(st, BPF_LDX | BPF_MEM | EBPF_DW,
1088 RSP, save_regs[i], ofs);
1089 ofs += sizeof(uint64_t);
1093 emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K, RSP,
1094 spil * sizeof(uint64_t));
1101 * walk through bpf code and translate them x86_64 one.
1104 emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)
1106 uint32_t i, dr, op, sr;
1107 const struct ebpf_insn *ins;
1109 /* reset state fields */
1113 emit_prolog(st, bpf->stack_sz);
1115 for (i = 0; i != bpf->prm.nb_ins; i++) {
1118 st->off[i] = st->sz;
1120 ins = bpf->prm.ins + i;
1122 dr = ebpf2x86[ins->dst_reg];
1123 sr = ebpf2x86[ins->src_reg];
1127 /* 32 bit ALU IMM operations */
1128 case (BPF_ALU | BPF_ADD | BPF_K):
1129 case (BPF_ALU | BPF_SUB | BPF_K):
1130 case (BPF_ALU | BPF_AND | BPF_K):
1131 case (BPF_ALU | BPF_OR | BPF_K):
1132 case (BPF_ALU | BPF_XOR | BPF_K):
1133 emit_alu_imm(st, op, dr, ins->imm);
1135 case (BPF_ALU | BPF_LSH | BPF_K):
1136 case (BPF_ALU | BPF_RSH | BPF_K):
1137 emit_shift_imm(st, op, dr, ins->imm);
1139 case (BPF_ALU | EBPF_MOV | BPF_K):
1140 emit_mov_imm(st, op, dr, ins->imm);
1142 /* 32 bit ALU REG operations */
1143 case (BPF_ALU | BPF_ADD | BPF_X):
1144 case (BPF_ALU | BPF_SUB | BPF_X):
1145 case (BPF_ALU | BPF_AND | BPF_X):
1146 case (BPF_ALU | BPF_OR | BPF_X):
1147 case (BPF_ALU | BPF_XOR | BPF_X):
1148 emit_alu_reg(st, op, sr, dr);
1150 case (BPF_ALU | BPF_LSH | BPF_X):
1151 case (BPF_ALU | BPF_RSH | BPF_X):
1152 emit_shift_reg(st, op, sr, dr);
1154 case (BPF_ALU | EBPF_MOV | BPF_X):
1155 emit_mov_reg(st, op, sr, dr);
1157 case (BPF_ALU | BPF_NEG):
1158 emit_neg(st, op, dr);
1160 case (BPF_ALU | EBPF_END | EBPF_TO_BE):
1161 emit_be2le(st, dr, ins->imm);
1163 case (BPF_ALU | EBPF_END | EBPF_TO_LE):
1164 emit_le2be(st, dr, ins->imm);
1166 /* 64 bit ALU IMM operations */
1167 case (EBPF_ALU64 | BPF_ADD | BPF_K):
1168 case (EBPF_ALU64 | BPF_SUB | BPF_K):
1169 case (EBPF_ALU64 | BPF_AND | BPF_K):
1170 case (EBPF_ALU64 | BPF_OR | BPF_K):
1171 case (EBPF_ALU64 | BPF_XOR | BPF_K):
1172 emit_alu_imm(st, op, dr, ins->imm);
1174 case (EBPF_ALU64 | BPF_LSH | BPF_K):
1175 case (EBPF_ALU64 | BPF_RSH | BPF_K):
1176 case (EBPF_ALU64 | EBPF_ARSH | BPF_K):
1177 emit_shift_imm(st, op, dr, ins->imm);
1179 case (EBPF_ALU64 | EBPF_MOV | BPF_K):
1180 emit_mov_imm(st, op, dr, ins->imm);
1182 /* 64 bit ALU REG operations */
1183 case (EBPF_ALU64 | BPF_ADD | BPF_X):
1184 case (EBPF_ALU64 | BPF_SUB | BPF_X):
1185 case (EBPF_ALU64 | BPF_AND | BPF_X):
1186 case (EBPF_ALU64 | BPF_OR | BPF_X):
1187 case (EBPF_ALU64 | BPF_XOR | BPF_X):
1188 emit_alu_reg(st, op, sr, dr);
1190 case (EBPF_ALU64 | BPF_LSH | BPF_X):
1191 case (EBPF_ALU64 | BPF_RSH | BPF_X):
1192 case (EBPF_ALU64 | EBPF_ARSH | BPF_X):
1193 emit_shift_reg(st, op, sr, dr);
1195 case (EBPF_ALU64 | EBPF_MOV | BPF_X):
1196 emit_mov_reg(st, op, sr, dr);
1198 case (EBPF_ALU64 | BPF_NEG):
1199 emit_neg(st, op, dr);
1201 /* multiply instructions */
1202 case (BPF_ALU | BPF_MUL | BPF_K):
1203 case (BPF_ALU | BPF_MUL | BPF_X):
1204 case (EBPF_ALU64 | BPF_MUL | BPF_K):
1205 case (EBPF_ALU64 | BPF_MUL | BPF_X):
1206 emit_mul(st, op, sr, dr, ins->imm);
1208 /* divide instructions */
1209 case (BPF_ALU | BPF_DIV | BPF_K):
1210 case (BPF_ALU | BPF_MOD | BPF_K):
1211 case (BPF_ALU | BPF_DIV | BPF_X):
1212 case (BPF_ALU | BPF_MOD | BPF_X):
1213 case (EBPF_ALU64 | BPF_DIV | BPF_K):
1214 case (EBPF_ALU64 | BPF_MOD | BPF_K):
1215 case (EBPF_ALU64 | BPF_DIV | BPF_X):
1216 case (EBPF_ALU64 | BPF_MOD | BPF_X):
1217 emit_div(st, op, sr, dr, ins->imm);
1219 /* load instructions */
1220 case (BPF_LDX | BPF_MEM | BPF_B):
1221 case (BPF_LDX | BPF_MEM | BPF_H):
1222 case (BPF_LDX | BPF_MEM | BPF_W):
1223 case (BPF_LDX | BPF_MEM | EBPF_DW):
1224 emit_ld_reg(st, op, sr, dr, ins->off);
1226 /* load 64 bit immediate value */
1227 case (BPF_LD | BPF_IMM | EBPF_DW):
1228 emit_ld_imm64(st, dr, ins[0].imm, ins[1].imm);
1231 /* store instructions */
1232 case (BPF_STX | BPF_MEM | BPF_B):
1233 case (BPF_STX | BPF_MEM | BPF_H):
1234 case (BPF_STX | BPF_MEM | BPF_W):
1235 case (BPF_STX | BPF_MEM | EBPF_DW):
1236 emit_st_reg(st, op, sr, dr, ins->off);
1238 case (BPF_ST | BPF_MEM | BPF_B):
1239 case (BPF_ST | BPF_MEM | BPF_H):
1240 case (BPF_ST | BPF_MEM | BPF_W):
1241 case (BPF_ST | BPF_MEM | EBPF_DW):
1242 emit_st_imm(st, op, dr, ins->imm, ins->off);
1244 /* atomic add instructions */
1245 case (BPF_STX | EBPF_XADD | BPF_W):
1246 case (BPF_STX | EBPF_XADD | EBPF_DW):
1247 emit_st_xadd(st, op, sr, dr, ins->off);
1249 /* jump instructions */
1250 case (BPF_JMP | BPF_JA):
1251 emit_jmp(st, ins->off + 1);
1253 /* jump IMM instructions */
1254 case (BPF_JMP | BPF_JEQ | BPF_K):
1255 case (BPF_JMP | EBPF_JNE | BPF_K):
1256 case (BPF_JMP | BPF_JGT | BPF_K):
1257 case (BPF_JMP | EBPF_JLT | BPF_K):
1258 case (BPF_JMP | BPF_JGE | BPF_K):
1259 case (BPF_JMP | EBPF_JLE | BPF_K):
1260 case (BPF_JMP | EBPF_JSGT | BPF_K):
1261 case (BPF_JMP | EBPF_JSLT | BPF_K):
1262 case (BPF_JMP | EBPF_JSGE | BPF_K):
1263 case (BPF_JMP | EBPF_JSLE | BPF_K):
1264 case (BPF_JMP | BPF_JSET | BPF_K):
1265 emit_jcc_imm(st, op, dr, ins->imm, ins->off + 1);
1267 /* jump REG instructions */
1268 case (BPF_JMP | BPF_JEQ | BPF_X):
1269 case (BPF_JMP | EBPF_JNE | BPF_X):
1270 case (BPF_JMP | BPF_JGT | BPF_X):
1271 case (BPF_JMP | EBPF_JLT | BPF_X):
1272 case (BPF_JMP | BPF_JGE | BPF_X):
1273 case (BPF_JMP | EBPF_JLE | BPF_X):
1274 case (BPF_JMP | EBPF_JSGT | BPF_X):
1275 case (BPF_JMP | EBPF_JSLT | BPF_X):
1276 case (BPF_JMP | EBPF_JSGE | BPF_X):
1277 case (BPF_JMP | EBPF_JSLE | BPF_X):
1278 case (BPF_JMP | BPF_JSET | BPF_X):
1279 emit_jcc_reg(st, op, sr, dr, ins->off + 1);
1281 /* call instructions */
1282 case (BPF_JMP | EBPF_CALL):
1284 (uintptr_t)bpf->prm.xsym[ins->imm].func.val);
1286 /* return instruction */
1287 case (BPF_JMP | EBPF_EXIT):
1292 "%s(%p): invalid opcode %#x at pc: %u;\n",
1293 __func__, bpf, ins->code, i);
1302 * produce a native ISA version of the given BPF code.
1305 bpf_jit_x86(struct rte_bpf *bpf)
1310 struct bpf_jit_state st;
1313 memset(&st, 0, sizeof(st));
1314 st.off = malloc(bpf->prm.nb_ins * sizeof(st.off[0]));
1318 /* fill with fake offsets */
1319 st.exit.off = INT32_MAX;
1320 for (i = 0; i != bpf->prm.nb_ins; i++)
1321 st.off[i] = INT32_MAX;
1324 * dry runs, used to calculate total code size and valid jump offsets.
1325 * stop when we get minimal possible size
1329 rc = emit(&st, bpf);
1330 } while (rc == 0 && sz != st.sz);
1334 /* allocate memory needed */
1335 st.ins = mmap(NULL, st.sz, PROT_READ | PROT_WRITE,
1336 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
1337 if (st.ins == MAP_FAILED)
1341 rc = emit(&st, bpf);
1344 if (rc == 0 && mprotect(st.ins, st.sz, PROT_READ | PROT_EXEC) != 0)
1348 munmap(st.ins, st.sz);
1350 bpf->jit.func = (void *)st.ins;
1351 bpf->jit.sz = st.sz;