1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
8 #include <rte_common.h>
10 #include <rte_debug.h>
14 #define GET_BPF_OP(op) (BPF_OP(op) >> 4)
17 RAX = 0, /* scratch, return value */
18 RCX = 1, /* scratch, 4th arg */
19 RDX = 2, /* scratch, 3rd arg */
20 RBX = 3, /* callee saved */
21 RSP = 4, /* stack pointer */
22 RBP = 5, /* frame pointer, callee saved */
23 RSI = 6, /* scratch, 2nd arg */
24 RDI = 7, /* scratch, 1st arg */
25 R8 = 8, /* scratch, 5th arg */
26 R9 = 9, /* scratch, 6th arg */
27 R10 = 10, /* scratch */
28 R11 = 11, /* scratch */
29 R12 = 12, /* callee saved */
30 R13 = 13, /* callee saved */
31 R14 = 14, /* callee saved */
32 R15 = 15, /* callee saved */
35 #define IS_EXT_REG(r) ((r) >= R8)
38 REX_PREFIX = 0x40, /* fixed value 0100 */
39 REX_W = 0x8, /* 64bit operand size */
40 REX_R = 0x4, /* extension of the ModRM.reg field */
41 REX_X = 0x2, /* extension of the SIB.index field */
42 REX_B = 0x1, /* extension of the ModRM.rm field */
60 * eBPF to x86_64 register mappings.
62 static const uint32_t ebpf2x86[] = {
77 * r10 and r11 are used as a scratch temporary registers.
85 /* LD_ABS/LD_IMM offsets */
87 LDMB_FSP_OFS, /* fast-path */
88 LDMB_SLP_OFS, /* slow-path */
89 LDMB_FIN_OFS, /* final part */
94 * callee saved registers list.
95 * keep RBP as the last one.
97 static const uint32_t save_regs[] = {RBX, R12, R13, R14, R15, RBP};
99 struct bpf_jit_state {
114 #define INUSE(v, r) (((v) >> (r)) & 1)
115 #define USED(v, r) ((v) |= 1 << (r))
123 * In many cases for imm8 we can produce shorter code.
129 return sizeof(int8_t);
130 return sizeof(int32_t);
134 emit_bytes(struct bpf_jit_state *st, const uint8_t ins[], uint32_t sz)
138 if (st->ins != NULL) {
139 for (i = 0; i != sz; i++)
140 st->ins[st->sz + i] = ins[i];
146 emit_imm(struct bpf_jit_state *st, const uint32_t imm, uint32_t sz)
151 emit_bytes(st, v.u8, sz);
158 emit_rex(struct bpf_jit_state *st, uint32_t op, uint32_t reg, uint32_t rm)
162 /* mark operand registers as used*/
163 USED(st->reguse, reg);
164 USED(st->reguse, rm);
167 if (BPF_CLASS(op) == EBPF_ALU64 ||
168 op == (BPF_ST | BPF_MEM | EBPF_DW) ||
169 op == (BPF_STX | BPF_MEM | EBPF_DW) ||
170 op == (BPF_STX | EBPF_XADD | EBPF_DW) ||
171 op == (BPF_LD | BPF_IMM | EBPF_DW) ||
172 (BPF_CLASS(op) == BPF_LDX &&
173 BPF_MODE(op) == BPF_MEM &&
174 BPF_SIZE(op) != BPF_W))
183 /* store using SIL, DIL */
184 if (op == (BPF_STX | BPF_MEM | BPF_B) && (reg == RDI || reg == RSI))
189 emit_bytes(st, &rex, sizeof(rex));
197 emit_modregrm(struct bpf_jit_state *st, uint32_t mod, uint32_t reg, uint32_t rm)
201 v = mod << 6 | (reg & 7) << 3 | (rm & 7);
202 emit_bytes(st, &v, sizeof(v));
209 emit_sib(struct bpf_jit_state *st, uint32_t scale, uint32_t idx, uint32_t base)
213 v = scale << 6 | (idx & 7) << 3 | (base & 7);
214 emit_bytes(st, &v, sizeof(v));
218 * emit OPCODE+REGIDX byte
221 emit_opcode(struct bpf_jit_state *st, uint8_t ops, uint32_t reg)
226 emit_bytes(st, &v, sizeof(v));
231 * emit xchg %<sreg>, %<dreg>
234 emit_xchg_reg(struct bpf_jit_state *st, uint32_t sreg, uint32_t dreg)
236 const uint8_t ops = 0x87;
238 emit_rex(st, EBPF_ALU64, sreg, dreg);
239 emit_bytes(st, &ops, sizeof(ops));
240 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
247 emit_neg(struct bpf_jit_state *st, uint32_t op, uint32_t dreg)
249 const uint8_t ops = 0xF7;
250 const uint8_t mods = 3;
252 emit_rex(st, op, 0, dreg);
253 emit_bytes(st, &ops, sizeof(ops));
254 emit_modregrm(st, MOD_DIRECT, mods, dreg);
258 * emit mov %<sreg>, %<dreg>
261 emit_mov_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
264 const uint8_t ops = 0x89;
266 /* if operands are 32-bit, then it can be used to clear upper 32-bit */
267 if (sreg != dreg || BPF_CLASS(op) == BPF_ALU) {
268 emit_rex(st, op, sreg, dreg);
269 emit_bytes(st, &ops, sizeof(ops));
270 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
275 * emit movzwl %<sreg>, %<dreg>
278 emit_movzwl(struct bpf_jit_state *st, uint32_t sreg, uint32_t dreg)
280 static const uint8_t ops[] = {0x0F, 0xB7};
282 emit_rex(st, BPF_ALU, sreg, dreg);
283 emit_bytes(st, ops, sizeof(ops));
284 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
288 * emit ror <imm8>, %<dreg>
291 emit_ror_imm(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
293 const uint8_t prfx = 0x66;
294 const uint8_t ops = 0xC1;
295 const uint8_t mods = 1;
297 emit_bytes(st, &prfx, sizeof(prfx));
298 emit_rex(st, BPF_ALU, 0, dreg);
299 emit_bytes(st, &ops, sizeof(ops));
300 emit_modregrm(st, MOD_DIRECT, mods, dreg);
301 emit_imm(st, imm, imm_size(imm));
308 emit_be2le_48(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
312 const uint8_t ops = 0x0F;
313 const uint8_t mods = 1;
315 rop = (imm == 64) ? EBPF_ALU64 : BPF_ALU;
316 emit_rex(st, rop, 0, dreg);
317 emit_bytes(st, &ops, sizeof(ops));
318 emit_modregrm(st, MOD_DIRECT, mods, dreg);
322 emit_be2le(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
325 emit_ror_imm(st, dreg, 8);
326 emit_movzwl(st, dreg, dreg);
328 emit_be2le_48(st, dreg, imm);
332 * In general it is NOP for x86.
333 * Just clear the upper bits.
336 emit_le2be(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
339 emit_movzwl(st, dreg, dreg);
341 emit_mov_reg(st, BPF_ALU | EBPF_MOV | BPF_X, dreg, dreg);
353 emit_alu_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
358 const uint8_t op8 = 0x83;
359 const uint8_t op32 = 0x81;
360 static const uint8_t mods[] = {
361 [GET_BPF_OP(BPF_ADD)] = 0,
362 [GET_BPF_OP(BPF_AND)] = 4,
363 [GET_BPF_OP(BPF_OR)] = 1,
364 [GET_BPF_OP(BPF_SUB)] = 5,
365 [GET_BPF_OP(BPF_XOR)] = 6,
368 bop = GET_BPF_OP(op);
371 imsz = imm_size(imm);
372 opcode = (imsz == 1) ? op8 : op32;
374 emit_rex(st, op, 0, dreg);
375 emit_bytes(st, &opcode, sizeof(opcode));
376 emit_modregrm(st, MOD_DIRECT, mod, dreg);
377 emit_imm(st, imm, imsz);
382 * add %<sreg>, %<dreg>
383 * and %<sreg>, %<dreg>
384 * or %<sreg>, %<dreg>
385 * sub %<sreg>, %<dreg>
386 * xor %<sreg>, %<dreg>
389 emit_alu_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
394 static const uint8_t ops[] = {
395 [GET_BPF_OP(BPF_ADD)] = 0x01,
396 [GET_BPF_OP(BPF_AND)] = 0x21,
397 [GET_BPF_OP(BPF_OR)] = 0x09,
398 [GET_BPF_OP(BPF_SUB)] = 0x29,
399 [GET_BPF_OP(BPF_XOR)] = 0x31,
402 bop = GET_BPF_OP(op);
404 emit_rex(st, op, sreg, dreg);
405 emit_bytes(st, &ops[bop], sizeof(ops[bop]));
406 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
410 emit_shift(struct bpf_jit_state *st, uint32_t op, uint32_t dreg)
415 static const uint8_t ops[] = {0xC1, 0xD3};
416 static const uint8_t mods[] = {
417 [GET_BPF_OP(BPF_LSH)] = 4,
418 [GET_BPF_OP(BPF_RSH)] = 5,
419 [GET_BPF_OP(EBPF_ARSH)] = 7,
422 bop = GET_BPF_OP(op);
424 opx = (BPF_SRC(op) == BPF_X);
426 emit_rex(st, op, 0, dreg);
427 emit_bytes(st, &ops[opx], sizeof(ops[opx]));
428 emit_modregrm(st, MOD_DIRECT, mod, dreg);
438 emit_shift_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg,
441 emit_shift(st, op, dreg);
442 emit_imm(st, imm, imm_size(imm));
450 * note that rcx is implicitly used as a source register, so few extra
451 * instructions for register spillage might be necessary.
454 emit_shift_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
458 emit_xchg_reg(st, RCX, sreg);
460 emit_shift(st, op, (dreg == RCX) ? sreg : dreg);
463 emit_xchg_reg(st, RCX, sreg);
467 * emit mov <imm>, %<dreg>
470 emit_mov_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
472 const uint8_t ops = 0xC7;
475 /* replace 'mov 0, %<dst>' with 'xor %<dst>, %<dst>' */
476 op = BPF_CLASS(op) | BPF_XOR | BPF_X;
477 emit_alu_reg(st, op, dreg, dreg);
481 emit_rex(st, op, 0, dreg);
482 emit_bytes(st, &ops, sizeof(ops));
483 emit_modregrm(st, MOD_DIRECT, 0, dreg);
484 emit_imm(st, imm, sizeof(imm));
488 * emit mov <imm64>, %<dreg>
491 emit_ld_imm64(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm0,
496 const uint8_t ops = 0xB8;
498 op = (imm1 == 0) ? BPF_ALU : EBPF_ALU64;
500 emit_rex(st, op, 0, dreg);
501 emit_opcode(st, ops, dreg);
503 emit_imm(st, imm0, sizeof(imm0));
505 emit_imm(st, imm1, sizeof(imm1));
509 * note that rax:rdx are implicitly used as source/destination registers,
510 * so some reg spillage is necessary.
525 emit_mul(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
528 const uint8_t ops = 0xF7;
529 const uint8_t mods = 4;
532 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, REG_TMP0);
533 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, REG_TMP1);
536 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, dreg, RAX);
538 if (BPF_SRC(op) == BPF_X)
540 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
541 sreg == RAX ? REG_TMP0 : sreg, RDX);
544 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, RDX, imm);
546 emit_rex(st, op, RAX, RDX);
547 emit_bytes(st, &ops, sizeof(ops));
548 emit_modregrm(st, MOD_DIRECT, mods, RDX);
552 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX);
556 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, dreg);
558 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP0, RAX);
563 * emit mov <ofs>(%<sreg>), %<dreg>
564 * note that for non 64-bit ops, higher bits have to be cleared.
567 emit_ld_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
571 const uint8_t op32 = 0x8B;
572 const uint8_t op16[] = {0x0F, 0xB7};
573 const uint8_t op8[] = {0x0F, 0xB6};
575 emit_rex(st, op, dreg, sreg);
579 emit_bytes(st, op8, sizeof(op8));
580 else if (opsz == BPF_H)
581 emit_bytes(st, op16, sizeof(op16));
583 emit_bytes(st, &op32, sizeof(op32));
585 mods = (imm_size(ofs) == 1) ? MOD_IDISP8 : MOD_IDISP32;
587 emit_modregrm(st, mods, dreg, sreg);
588 if (sreg == RSP || sreg == R12)
589 emit_sib(st, SIB_SCALE_1, sreg, sreg);
590 emit_imm(st, ofs, imm_size(ofs));
595 * mov %<sreg>, <ofs>(%<dreg>)
596 * mov <imm>, <ofs>(%<dreg>)
599 emit_st_common(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
600 uint32_t dreg, uint32_t imm, int32_t ofs)
602 uint32_t mods, imsz, opsz, opx;
603 const uint8_t prfx16 = 0x66;
605 /* 8 bit instruction opcodes */
606 static const uint8_t op8[] = {0xC6, 0x88};
608 /* 16/32/64 bit instruction opcodes */
609 static const uint8_t ops[] = {0xC7, 0x89};
611 /* is the instruction has immediate value or src reg? */
612 opx = (BPF_CLASS(op) == BPF_STX);
616 emit_bytes(st, &prfx16, sizeof(prfx16));
618 emit_rex(st, op, sreg, dreg);
621 emit_bytes(st, &op8[opx], sizeof(op8[opx]));
623 emit_bytes(st, &ops[opx], sizeof(ops[opx]));
625 imsz = imm_size(ofs);
626 mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32;
628 emit_modregrm(st, mods, sreg, dreg);
630 if (dreg == RSP || dreg == R12)
631 emit_sib(st, SIB_SCALE_1, dreg, dreg);
633 emit_imm(st, ofs, imsz);
636 imsz = RTE_MIN(bpf_size(opsz), sizeof(imm));
637 emit_imm(st, imm, imsz);
642 emit_st_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm,
645 emit_st_common(st, op, 0, dreg, imm, ofs);
649 emit_st_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
652 emit_st_common(st, op, sreg, dreg, 0, ofs);
656 * emit lock add %<sreg>, <ofs>(%<dreg>)
659 emit_st_xadd(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
660 uint32_t dreg, int32_t ofs)
664 const uint8_t lck = 0xF0; /* lock prefix */
665 const uint8_t ops = 0x01; /* add opcode */
667 imsz = imm_size(ofs);
668 mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32;
670 emit_bytes(st, &lck, sizeof(lck));
671 emit_rex(st, op, sreg, dreg);
672 emit_bytes(st, &ops, sizeof(ops));
673 emit_modregrm(st, mods, sreg, dreg);
674 emit_imm(st, ofs, imsz);
679 * mov <imm64>, (%rax)
683 emit_call(struct bpf_jit_state *st, uintptr_t trg)
685 const uint8_t ops = 0xFF;
686 const uint8_t mods = 2;
688 emit_ld_imm64(st, RAX, trg, trg >> 32);
689 emit_bytes(st, &ops, sizeof(ops));
690 emit_modregrm(st, MOD_DIRECT, mods, RAX);
695 * where 'ofs' is the target offset for the native code.
698 emit_abs_jmp(struct bpf_jit_state *st, int32_t ofs)
703 const uint8_t op8 = 0xEB;
704 const uint8_t op32 = 0xE9;
706 const int32_t sz8 = sizeof(op8) + sizeof(uint8_t);
707 const int32_t sz32 = sizeof(op32) + sizeof(uint32_t);
709 /* max possible jmp instruction size */
710 const int32_t iszm = RTE_MAX(sz8, sz32);
713 imsz = RTE_MAX(imm_size(joff), imm_size(joff + iszm));
716 emit_bytes(st, &op8, sizeof(op8));
719 emit_bytes(st, &op32, sizeof(op32));
723 emit_imm(st, joff, imsz);
728 * where 'ofs' is the target offset for the BPF bytecode.
731 emit_jmp(struct bpf_jit_state *st, int32_t ofs)
733 emit_abs_jmp(st, st->off[st->idx + ofs]);
738 * cmovz %<sreg>, <%dreg>
739 * cmovne %<sreg>, <%dreg>
740 * cmova %<sreg>, <%dreg>
741 * cmovb %<sreg>, <%dreg>
742 * cmovae %<sreg>, <%dreg>
743 * cmovbe %<sreg>, <%dreg>
744 * cmovg %<sreg>, <%dreg>
745 * cmovl %<sreg>, <%dreg>
746 * cmovge %<sreg>, <%dreg>
747 * cmovle %<sreg>, <%dreg>
750 emit_movcc_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
755 static const uint8_t ops[][2] = {
756 [GET_BPF_OP(BPF_JEQ)] = {0x0F, 0x44}, /* CMOVZ */
757 [GET_BPF_OP(EBPF_JNE)] = {0x0F, 0x45}, /* CMOVNE */
758 [GET_BPF_OP(BPF_JGT)] = {0x0F, 0x47}, /* CMOVA */
759 [GET_BPF_OP(EBPF_JLT)] = {0x0F, 0x42}, /* CMOVB */
760 [GET_BPF_OP(BPF_JGE)] = {0x0F, 0x43}, /* CMOVAE */
761 [GET_BPF_OP(EBPF_JLE)] = {0x0F, 0x46}, /* CMOVBE */
762 [GET_BPF_OP(EBPF_JSGT)] = {0x0F, 0x4F}, /* CMOVG */
763 [GET_BPF_OP(EBPF_JSLT)] = {0x0F, 0x4C}, /* CMOVL */
764 [GET_BPF_OP(EBPF_JSGE)] = {0x0F, 0x4D}, /* CMOVGE */
765 [GET_BPF_OP(EBPF_JSLE)] = {0x0F, 0x4E}, /* CMOVLE */
766 [GET_BPF_OP(BPF_JSET)] = {0x0F, 0x45}, /* CMOVNE */
769 bop = GET_BPF_OP(op);
771 emit_rex(st, op, dreg, sreg);
772 emit_bytes(st, ops[bop], sizeof(ops[bop]));
773 emit_modregrm(st, MOD_DIRECT, dreg, sreg);
788 * where 'ofs' is the target offset for the native code.
791 emit_abs_jcc(struct bpf_jit_state *st, uint32_t op, int32_t ofs)
796 static const uint8_t op8[] = {
797 [GET_BPF_OP(BPF_JEQ)] = 0x74, /* JE */
798 [GET_BPF_OP(EBPF_JNE)] = 0x75, /* JNE */
799 [GET_BPF_OP(BPF_JGT)] = 0x77, /* JA */
800 [GET_BPF_OP(EBPF_JLT)] = 0x72, /* JB */
801 [GET_BPF_OP(BPF_JGE)] = 0x73, /* JAE */
802 [GET_BPF_OP(EBPF_JLE)] = 0x76, /* JBE */
803 [GET_BPF_OP(EBPF_JSGT)] = 0x7F, /* JG */
804 [GET_BPF_OP(EBPF_JSLT)] = 0x7C, /* JL */
805 [GET_BPF_OP(EBPF_JSGE)] = 0x7D, /*JGE */
806 [GET_BPF_OP(EBPF_JSLE)] = 0x7E, /* JLE */
807 [GET_BPF_OP(BPF_JSET)] = 0x75, /*JNE */
810 static const uint8_t op32[][2] = {
811 [GET_BPF_OP(BPF_JEQ)] = {0x0F, 0x84}, /* JE */
812 [GET_BPF_OP(EBPF_JNE)] = {0x0F, 0x85}, /* JNE */
813 [GET_BPF_OP(BPF_JGT)] = {0x0F, 0x87}, /* JA */
814 [GET_BPF_OP(EBPF_JLT)] = {0x0F, 0x82}, /* JB */
815 [GET_BPF_OP(BPF_JGE)] = {0x0F, 0x83}, /* JAE */
816 [GET_BPF_OP(EBPF_JLE)] = {0x0F, 0x86}, /* JBE */
817 [GET_BPF_OP(EBPF_JSGT)] = {0x0F, 0x8F}, /* JG */
818 [GET_BPF_OP(EBPF_JSLT)] = {0x0F, 0x8C}, /* JL */
819 [GET_BPF_OP(EBPF_JSGE)] = {0x0F, 0x8D}, /*JGE */
820 [GET_BPF_OP(EBPF_JSLE)] = {0x0F, 0x8E}, /* JLE */
821 [GET_BPF_OP(BPF_JSET)] = {0x0F, 0x85}, /*JNE */
824 const int32_t sz8 = sizeof(op8[0]) + sizeof(uint8_t);
825 const int32_t sz32 = sizeof(op32[0]) + sizeof(uint32_t);
827 /* max possible jcc instruction size */
828 const int32_t iszm = RTE_MAX(sz8, sz32);
831 imsz = RTE_MAX(imm_size(joff), imm_size(joff + iszm));
833 bop = GET_BPF_OP(op);
836 emit_bytes(st, &op8[bop], sizeof(op8[bop]));
839 emit_bytes(st, op32[bop], sizeof(op32[bop]));
843 emit_imm(st, joff, imsz);
858 * where 'ofs' is the target offset for the BPF bytecode.
861 emit_jcc(struct bpf_jit_state *st, uint32_t op, int32_t ofs)
863 emit_abs_jcc(st, op, st->off[st->idx + ofs]);
868 * emit cmp <imm>, %<dreg>
871 emit_cmp_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
876 const uint8_t op8 = 0x83;
877 const uint8_t op32 = 0x81;
878 const uint8_t mods = 7;
880 imsz = imm_size(imm);
881 ops = (imsz == 1) ? op8 : op32;
883 emit_rex(st, op, 0, dreg);
884 emit_bytes(st, &ops, sizeof(ops));
885 emit_modregrm(st, MOD_DIRECT, mods, dreg);
886 emit_imm(st, imm, imsz);
890 * emit test <imm>, %<dreg>
893 emit_tst_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
895 const uint8_t ops = 0xF7;
896 const uint8_t mods = 0;
898 emit_rex(st, op, 0, dreg);
899 emit_bytes(st, &ops, sizeof(ops));
900 emit_modregrm(st, MOD_DIRECT, mods, dreg);
901 emit_imm(st, imm, imm_size(imm));
905 emit_jcc_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg,
906 uint32_t imm, int32_t ofs)
908 if (BPF_OP(op) == BPF_JSET)
909 emit_tst_imm(st, EBPF_ALU64, dreg, imm);
911 emit_cmp_imm(st, EBPF_ALU64, dreg, imm);
913 emit_jcc(st, op, ofs);
917 * emit test %<sreg>, %<dreg>
920 emit_tst_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
923 const uint8_t ops = 0x85;
925 emit_rex(st, op, sreg, dreg);
926 emit_bytes(st, &ops, sizeof(ops));
927 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
931 * emit cmp %<sreg>, %<dreg>
934 emit_cmp_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
937 const uint8_t ops = 0x39;
939 emit_rex(st, op, sreg, dreg);
940 emit_bytes(st, &ops, sizeof(ops));
941 emit_modregrm(st, MOD_DIRECT, sreg, dreg);
946 emit_jcc_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
947 uint32_t dreg, int32_t ofs)
949 if (BPF_OP(op) == BPF_JSET)
950 emit_tst_reg(st, EBPF_ALU64, sreg, dreg);
952 emit_cmp_reg(st, EBPF_ALU64, sreg, dreg);
954 emit_jcc(st, op, ofs);
958 * note that rax:rdx are implicitly used as source/destination registers,
959 * so some reg spillage is necessary.
965 * for divisor as immediate value:
979 emit_div(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
984 const uint8_t ops = 0xF7;
985 const uint8_t mods = 6;
987 if (BPF_SRC(op) == BPF_X) {
989 /* check that src divisor is not zero */
990 emit_tst_reg(st, BPF_CLASS(op), sreg, sreg);
992 /* exit with return value zero */
993 emit_movcc_reg(st, BPF_CLASS(op) | BPF_JEQ | BPF_X, sreg, RAX);
994 emit_abs_jcc(st, BPF_JMP | BPF_JEQ | BPF_K, st->exit.off);
999 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, REG_TMP0);
1001 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, REG_TMP1);
1003 /* fill rax & rdx */
1004 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, dreg, RAX);
1005 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, RDX, 0);
1007 if (BPF_SRC(op) == BPF_X) {
1015 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, sr, imm);
1018 emit_rex(st, op, 0, sr);
1019 emit_bytes(st, &ops, sizeof(ops));
1020 emit_modregrm(st, MOD_DIRECT, mods, sr);
1022 if (BPF_OP(op) == BPF_DIV)
1023 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, dreg);
1025 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, dreg);
1028 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP0, RAX);
1030 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX);
1034 * helper function, used by emit_ld_mbuf().
1035 * generates code for 'fast_path':
1036 * calculate load offset and check is it inside first packet segment.
1039 emit_ldmb_fast_path(struct bpf_jit_state *st, const uint32_t rg[EBPF_REG_7],
1040 uint32_t sreg, uint32_t mode, uint32_t sz, uint32_t imm,
1041 const int32_t ofs[LDMB_OFS_NUM])
1043 /* make R2 contain *off* value */
1045 if (sreg != rg[EBPF_REG_2]) {
1046 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K,
1047 rg[EBPF_REG_2], imm);
1048 if (mode == BPF_IND)
1049 emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
1050 sreg, rg[EBPF_REG_2]);
1052 /* BPF_IND with sreg == R2 */
1053 emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K,
1054 rg[EBPF_REG_2], imm);
1056 /* R3 = mbuf->data_len */
1057 emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H,
1058 rg[EBPF_REG_6], rg[EBPF_REG_3],
1059 offsetof(struct rte_mbuf, data_len));
1062 emit_alu_reg(st, EBPF_ALU64 | BPF_SUB | BPF_X,
1063 rg[EBPF_REG_2], rg[EBPF_REG_3]);
1065 /* JSLT R3, <sz> <slow_path> */
1066 emit_cmp_imm(st, EBPF_ALU64, rg[EBPF_REG_3], sz);
1067 emit_abs_jcc(st, BPF_JMP | EBPF_JSLT | BPF_K, ofs[LDMB_SLP_OFS]);
1069 /* R3 = mbuf->data_off */
1070 emit_ld_reg(st, BPF_LDX | BPF_MEM | BPF_H,
1071 rg[EBPF_REG_6], rg[EBPF_REG_3],
1072 offsetof(struct rte_mbuf, data_off));
1074 /* R0 = mbuf->buf_addr */
1075 emit_ld_reg(st, BPF_LDX | BPF_MEM | EBPF_DW,
1076 rg[EBPF_REG_6], rg[EBPF_REG_0],
1077 offsetof(struct rte_mbuf, buf_addr));
1080 emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
1081 rg[EBPF_REG_3], rg[EBPF_REG_0]);
1084 emit_alu_reg(st, EBPF_ALU64 | BPF_ADD | BPF_X,
1085 rg[EBPF_REG_2], rg[EBPF_REG_0]);
1087 /* JMP <fin_part> */
1088 emit_abs_jmp(st, ofs[LDMB_FIN_OFS]);
1092 * helper function, used by emit_ld_mbuf().
1093 * generates code for 'slow_path':
1094 * call __rte_pktmbuf_read() and check return value.
1097 emit_ldmb_slow_path(struct bpf_jit_state *st, const uint32_t rg[EBPF_REG_7],
1100 /* make R3 contain *len* value (1/2/4) */
1102 emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, rg[EBPF_REG_3], sz);
1104 /* make R4 contain (RBP - ldmb.stack_ofs) */
1106 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RBP, rg[EBPF_REG_4]);
1107 emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, rg[EBPF_REG_4],
1108 st->ldmb.stack_ofs);
1110 /* make R1 contain mbuf ptr */
1112 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
1113 rg[EBPF_REG_6], rg[EBPF_REG_1]);
1115 /* call rte_pktmbuf_read */
1116 emit_call(st, (uintptr_t)__rte_pktmbuf_read);
1118 /* check that return value (R0) is not zero */
1119 emit_tst_reg(st, EBPF_ALU64, rg[EBPF_REG_0], rg[EBPF_REG_0]);
1120 emit_abs_jcc(st, BPF_JMP | BPF_JEQ | BPF_K, st->exit.off);
1124 * helper function, used by emit_ld_mbuf().
1125 * generates final part of code for BPF_ABS/BPF_IND load:
1126 * perform data load and endianness conversion.
1127 * expects dreg to contain valid data pointer.
1130 emit_ldmb_fin(struct bpf_jit_state *st, uint32_t dreg, uint32_t opsz,
1133 emit_ld_reg(st, BPF_LDX | BPF_MEM | opsz, dreg, dreg, 0);
1134 if (sz != sizeof(uint8_t))
1135 emit_be2le(st, dreg, sz * CHAR_BIT);
1139 * emit code for BPF_ABS/BPF_IND load.
1140 * generates the following construction:
1142 * off = ins->sreg + ins->imm
1143 * if (mbuf->data_len - off < ins->opsz)
1145 * ptr = mbuf->buf_addr + mbuf->data_off + off;
1148 * typeof(ins->opsz) buf; //allocate space on the stack
1149 * ptr = __rte_pktmbuf_read(mbuf, off, ins->opsz, &buf);
1153 * res = *(typeof(ins->opsz))ptr;
1157 emit_ld_mbuf(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t imm)
1159 uint32_t i, mode, opsz, sz;
1160 uint32_t rg[EBPF_REG_7];
1161 int32_t ofs[LDMB_OFS_NUM];
1163 mode = BPF_MODE(op);
1164 opsz = BPF_SIZE(op);
1165 sz = bpf_size(opsz);
1167 for (i = 0; i != RTE_DIM(rg); i++)
1168 rg[i] = ebpf2x86[i];
1170 /* fill with fake offsets */
1171 for (i = 0; i != RTE_DIM(ofs); i++)
1172 ofs[i] = st->sz + INT8_MAX;
1174 /* dry run first to calculate jump offsets */
1176 ofs[LDMB_FSP_OFS] = st->sz;
1177 emit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs);
1178 ofs[LDMB_SLP_OFS] = st->sz;
1179 emit_ldmb_slow_path(st, rg, sz);
1180 ofs[LDMB_FIN_OFS] = st->sz;
1181 emit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz);
1183 RTE_VERIFY(ofs[LDMB_FIN_OFS] - ofs[LDMB_FSP_OFS] <= INT8_MAX);
1185 /* reset dry-run code and do a proper run */
1187 st->sz = ofs[LDMB_FSP_OFS];
1188 emit_ldmb_fast_path(st, rg, sreg, mode, sz, imm, ofs);
1189 emit_ldmb_slow_path(st, rg, sz);
1190 emit_ldmb_fin(st, rg[EBPF_REG_0], opsz, sz);
1194 emit_prolog(struct bpf_jit_state *st, int32_t stack_size)
1200 for (i = 0; i != RTE_DIM(save_regs); i++)
1201 spil += INUSE(st->reguse, save_regs[i]);
1203 /* we can avoid touching the stack at all */
1208 emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, RSP,
1209 spil * sizeof(uint64_t));
1212 for (i = 0; i != RTE_DIM(save_regs); i++) {
1213 if (INUSE(st->reguse, save_regs[i]) != 0) {
1214 emit_st_reg(st, BPF_STX | BPF_MEM | EBPF_DW,
1215 save_regs[i], RSP, ofs);
1216 ofs += sizeof(uint64_t);
1220 if (INUSE(st->reguse, RBP) != 0) {
1221 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RSP, RBP);
1222 emit_alu_imm(st, EBPF_ALU64 | BPF_SUB | BPF_K, RSP, stack_size);
1230 emit_ret(struct bpf_jit_state *st)
1232 const uint8_t ops = 0xC3;
1234 emit_bytes(st, &ops, sizeof(ops));
1238 emit_epilog(struct bpf_jit_state *st)
1243 /* if we already have an epilog generate a jump to it */
1244 if (st->exit.num++ != 0) {
1245 emit_abs_jmp(st, st->exit.off);
1249 /* store offset of epilog block */
1250 st->exit.off = st->sz;
1253 for (i = 0; i != RTE_DIM(save_regs); i++)
1254 spil += INUSE(st->reguse, save_regs[i]);
1258 if (INUSE(st->reguse, RBP) != 0)
1259 emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
1263 for (i = 0; i != RTE_DIM(save_regs); i++) {
1264 if (INUSE(st->reguse, save_regs[i]) != 0) {
1265 emit_ld_reg(st, BPF_LDX | BPF_MEM | EBPF_DW,
1266 RSP, save_regs[i], ofs);
1267 ofs += sizeof(uint64_t);
1271 emit_alu_imm(st, EBPF_ALU64 | BPF_ADD | BPF_K, RSP,
1272 spil * sizeof(uint64_t));
1279 * walk through bpf code and translate them x86_64 one.
1282 emit(struct bpf_jit_state *st, const struct rte_bpf *bpf)
1284 uint32_t i, dr, op, sr;
1285 const struct ebpf_insn *ins;
1287 /* reset state fields */
1290 st->ldmb.stack_ofs = bpf->stack_sz;
1292 emit_prolog(st, bpf->stack_sz);
1294 for (i = 0; i != bpf->prm.nb_ins; i++) {
1297 st->off[i] = st->sz;
1299 ins = bpf->prm.ins + i;
1301 dr = ebpf2x86[ins->dst_reg];
1302 sr = ebpf2x86[ins->src_reg];
1306 /* 32 bit ALU IMM operations */
1307 case (BPF_ALU | BPF_ADD | BPF_K):
1308 case (BPF_ALU | BPF_SUB | BPF_K):
1309 case (BPF_ALU | BPF_AND | BPF_K):
1310 case (BPF_ALU | BPF_OR | BPF_K):
1311 case (BPF_ALU | BPF_XOR | BPF_K):
1312 emit_alu_imm(st, op, dr, ins->imm);
1314 case (BPF_ALU | BPF_LSH | BPF_K):
1315 case (BPF_ALU | BPF_RSH | BPF_K):
1316 emit_shift_imm(st, op, dr, ins->imm);
1318 case (BPF_ALU | EBPF_MOV | BPF_K):
1319 emit_mov_imm(st, op, dr, ins->imm);
1321 /* 32 bit ALU REG operations */
1322 case (BPF_ALU | BPF_ADD | BPF_X):
1323 case (BPF_ALU | BPF_SUB | BPF_X):
1324 case (BPF_ALU | BPF_AND | BPF_X):
1325 case (BPF_ALU | BPF_OR | BPF_X):
1326 case (BPF_ALU | BPF_XOR | BPF_X):
1327 emit_alu_reg(st, op, sr, dr);
1329 case (BPF_ALU | BPF_LSH | BPF_X):
1330 case (BPF_ALU | BPF_RSH | BPF_X):
1331 emit_shift_reg(st, op, sr, dr);
1333 case (BPF_ALU | EBPF_MOV | BPF_X):
1334 emit_mov_reg(st, op, sr, dr);
1336 case (BPF_ALU | BPF_NEG):
1337 emit_neg(st, op, dr);
1339 case (BPF_ALU | EBPF_END | EBPF_TO_BE):
1340 emit_be2le(st, dr, ins->imm);
1342 case (BPF_ALU | EBPF_END | EBPF_TO_LE):
1343 emit_le2be(st, dr, ins->imm);
1345 /* 64 bit ALU IMM operations */
1346 case (EBPF_ALU64 | BPF_ADD | BPF_K):
1347 case (EBPF_ALU64 | BPF_SUB | BPF_K):
1348 case (EBPF_ALU64 | BPF_AND | BPF_K):
1349 case (EBPF_ALU64 | BPF_OR | BPF_K):
1350 case (EBPF_ALU64 | BPF_XOR | BPF_K):
1351 emit_alu_imm(st, op, dr, ins->imm);
1353 case (EBPF_ALU64 | BPF_LSH | BPF_K):
1354 case (EBPF_ALU64 | BPF_RSH | BPF_K):
1355 case (EBPF_ALU64 | EBPF_ARSH | BPF_K):
1356 emit_shift_imm(st, op, dr, ins->imm);
1358 case (EBPF_ALU64 | EBPF_MOV | BPF_K):
1359 emit_mov_imm(st, op, dr, ins->imm);
1361 /* 64 bit ALU REG operations */
1362 case (EBPF_ALU64 | BPF_ADD | BPF_X):
1363 case (EBPF_ALU64 | BPF_SUB | BPF_X):
1364 case (EBPF_ALU64 | BPF_AND | BPF_X):
1365 case (EBPF_ALU64 | BPF_OR | BPF_X):
1366 case (EBPF_ALU64 | BPF_XOR | BPF_X):
1367 emit_alu_reg(st, op, sr, dr);
1369 case (EBPF_ALU64 | BPF_LSH | BPF_X):
1370 case (EBPF_ALU64 | BPF_RSH | BPF_X):
1371 case (EBPF_ALU64 | EBPF_ARSH | BPF_X):
1372 emit_shift_reg(st, op, sr, dr);
1374 case (EBPF_ALU64 | EBPF_MOV | BPF_X):
1375 emit_mov_reg(st, op, sr, dr);
1377 case (EBPF_ALU64 | BPF_NEG):
1378 emit_neg(st, op, dr);
1380 /* multiply instructions */
1381 case (BPF_ALU | BPF_MUL | BPF_K):
1382 case (BPF_ALU | BPF_MUL | BPF_X):
1383 case (EBPF_ALU64 | BPF_MUL | BPF_K):
1384 case (EBPF_ALU64 | BPF_MUL | BPF_X):
1385 emit_mul(st, op, sr, dr, ins->imm);
1387 /* divide instructions */
1388 case (BPF_ALU | BPF_DIV | BPF_K):
1389 case (BPF_ALU | BPF_MOD | BPF_K):
1390 case (BPF_ALU | BPF_DIV | BPF_X):
1391 case (BPF_ALU | BPF_MOD | BPF_X):
1392 case (EBPF_ALU64 | BPF_DIV | BPF_K):
1393 case (EBPF_ALU64 | BPF_MOD | BPF_K):
1394 case (EBPF_ALU64 | BPF_DIV | BPF_X):
1395 case (EBPF_ALU64 | BPF_MOD | BPF_X):
1396 emit_div(st, op, sr, dr, ins->imm);
1398 /* load instructions */
1399 case (BPF_LDX | BPF_MEM | BPF_B):
1400 case (BPF_LDX | BPF_MEM | BPF_H):
1401 case (BPF_LDX | BPF_MEM | BPF_W):
1402 case (BPF_LDX | BPF_MEM | EBPF_DW):
1403 emit_ld_reg(st, op, sr, dr, ins->off);
1405 /* load 64 bit immediate value */
1406 case (BPF_LD | BPF_IMM | EBPF_DW):
1407 emit_ld_imm64(st, dr, ins[0].imm, ins[1].imm);
1410 /* load absolute/indirect instructions */
1411 case (BPF_LD | BPF_ABS | BPF_B):
1412 case (BPF_LD | BPF_ABS | BPF_H):
1413 case (BPF_LD | BPF_ABS | BPF_W):
1414 case (BPF_LD | BPF_IND | BPF_B):
1415 case (BPF_LD | BPF_IND | BPF_H):
1416 case (BPF_LD | BPF_IND | BPF_W):
1417 emit_ld_mbuf(st, op, sr, ins->imm);
1419 /* store instructions */
1420 case (BPF_STX | BPF_MEM | BPF_B):
1421 case (BPF_STX | BPF_MEM | BPF_H):
1422 case (BPF_STX | BPF_MEM | BPF_W):
1423 case (BPF_STX | BPF_MEM | EBPF_DW):
1424 emit_st_reg(st, op, sr, dr, ins->off);
1426 case (BPF_ST | BPF_MEM | BPF_B):
1427 case (BPF_ST | BPF_MEM | BPF_H):
1428 case (BPF_ST | BPF_MEM | BPF_W):
1429 case (BPF_ST | BPF_MEM | EBPF_DW):
1430 emit_st_imm(st, op, dr, ins->imm, ins->off);
1432 /* atomic add instructions */
1433 case (BPF_STX | EBPF_XADD | BPF_W):
1434 case (BPF_STX | EBPF_XADD | EBPF_DW):
1435 emit_st_xadd(st, op, sr, dr, ins->off);
1437 /* jump instructions */
1438 case (BPF_JMP | BPF_JA):
1439 emit_jmp(st, ins->off + 1);
1441 /* jump IMM instructions */
1442 case (BPF_JMP | BPF_JEQ | BPF_K):
1443 case (BPF_JMP | EBPF_JNE | BPF_K):
1444 case (BPF_JMP | BPF_JGT | BPF_K):
1445 case (BPF_JMP | EBPF_JLT | BPF_K):
1446 case (BPF_JMP | BPF_JGE | BPF_K):
1447 case (BPF_JMP | EBPF_JLE | BPF_K):
1448 case (BPF_JMP | EBPF_JSGT | BPF_K):
1449 case (BPF_JMP | EBPF_JSLT | BPF_K):
1450 case (BPF_JMP | EBPF_JSGE | BPF_K):
1451 case (BPF_JMP | EBPF_JSLE | BPF_K):
1452 case (BPF_JMP | BPF_JSET | BPF_K):
1453 emit_jcc_imm(st, op, dr, ins->imm, ins->off + 1);
1455 /* jump REG instructions */
1456 case (BPF_JMP | BPF_JEQ | BPF_X):
1457 case (BPF_JMP | EBPF_JNE | BPF_X):
1458 case (BPF_JMP | BPF_JGT | BPF_X):
1459 case (BPF_JMP | EBPF_JLT | BPF_X):
1460 case (BPF_JMP | BPF_JGE | BPF_X):
1461 case (BPF_JMP | EBPF_JLE | BPF_X):
1462 case (BPF_JMP | EBPF_JSGT | BPF_X):
1463 case (BPF_JMP | EBPF_JSLT | BPF_X):
1464 case (BPF_JMP | EBPF_JSGE | BPF_X):
1465 case (BPF_JMP | EBPF_JSLE | BPF_X):
1466 case (BPF_JMP | BPF_JSET | BPF_X):
1467 emit_jcc_reg(st, op, sr, dr, ins->off + 1);
1469 /* call instructions */
1470 case (BPF_JMP | EBPF_CALL):
1472 (uintptr_t)bpf->prm.xsym[ins->imm].func.val);
1474 /* return instruction */
1475 case (BPF_JMP | EBPF_EXIT):
1480 "%s(%p): invalid opcode %#x at pc: %u;\n",
1481 __func__, bpf, ins->code, i);
1490 * produce a native ISA version of the given BPF code.
1493 bpf_jit_x86(struct rte_bpf *bpf)
1498 struct bpf_jit_state st;
1501 memset(&st, 0, sizeof(st));
1502 st.off = malloc(bpf->prm.nb_ins * sizeof(st.off[0]));
1506 /* fill with fake offsets */
1507 st.exit.off = INT32_MAX;
1508 for (i = 0; i != bpf->prm.nb_ins; i++)
1509 st.off[i] = INT32_MAX;
1512 * dry runs, used to calculate total code size and valid jump offsets.
1513 * stop when we get minimal possible size
1517 rc = emit(&st, bpf);
1518 } while (rc == 0 && sz != st.sz);
1522 /* allocate memory needed */
1523 st.ins = mmap(NULL, st.sz, PROT_READ | PROT_WRITE,
1524 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
1525 if (st.ins == MAP_FAILED)
1529 rc = emit(&st, bpf);
1532 if (rc == 0 && mprotect(st.ins, st.sz, PROT_READ | PROT_EXEC) != 0)
1536 munmap(st.ins, st.sz);
1538 bpf->jit.func = (void *)st.ins;
1539 bpf->jit.sz = st.sz;