1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
8 #include <rte_common.h>
10 #include <rte_debug.h>
11 #include <rte_byteorder.h>
15 #define BPF_JMP_UNC(ins) ((ins) += (ins)->off)
17 #define BPF_JMP_CND_REG(reg, ins, op, type) \
19 ((type)(reg)[(ins)->dst_reg] op (type)(reg)[(ins)->src_reg]) ? \
22 #define BPF_JMP_CND_IMM(reg, ins, op, type) \
24 ((type)(reg)[(ins)->dst_reg] op (type)(ins)->imm) ? \
27 #define BPF_NEG_ALU(reg, ins, type) \
28 ((reg)[(ins)->dst_reg] = (type)(-(reg)[(ins)->dst_reg]))
30 #define EBPF_MOV_ALU_REG(reg, ins, type) \
31 ((reg)[(ins)->dst_reg] = (type)(reg)[(ins)->src_reg])
33 #define BPF_OP_ALU_REG(reg, ins, op, type) \
34 ((reg)[(ins)->dst_reg] = \
35 (type)(reg)[(ins)->dst_reg] op (type)(reg)[(ins)->src_reg])
37 #define EBPF_MOV_ALU_IMM(reg, ins, type) \
38 ((reg)[(ins)->dst_reg] = (type)(ins)->imm)
40 #define BPF_OP_ALU_IMM(reg, ins, op, type) \
41 ((reg)[(ins)->dst_reg] = \
42 (type)(reg)[(ins)->dst_reg] op (type)(ins)->imm)
44 #define BPF_DIV_ZERO_CHECK(bpf, reg, ins, type) do { \
45 if ((type)(reg)[(ins)->src_reg] == 0) { \
47 "%s(%p): division by 0 at pc: %#zx;\n", \
49 (uintptr_t)(ins) - (uintptr_t)(bpf)->prm.ins); \
54 #define BPF_LD_REG(reg, ins, type) \
55 ((reg)[(ins)->dst_reg] = \
56 *(type *)(uintptr_t)((reg)[(ins)->src_reg] + (ins)->off))
58 #define BPF_ST_IMM(reg, ins, type) \
59 (*(type *)(uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off) = \
62 #define BPF_ST_REG(reg, ins, type) \
63 (*(type *)(uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off) = \
64 (type)(reg)[(ins)->src_reg])
66 #define BPF_ST_XADD_REG(reg, ins, tp) \
67 (rte_atomic##tp##_add((rte_atomic##tp##_t *) \
68 (uintptr_t)((reg)[(ins)->dst_reg] + (ins)->off), \
71 /* BPF_LD | BPF_ABS/BPF_IND */
75 #define BPF_LD_ABS(bpf, reg, ins, type, op) do { \
76 const type *p = bpf_ld_mbuf(bpf, reg, ins, (ins)->imm, sizeof(type)); \
79 reg[EBPF_REG_0] = op(p[0]); \
82 #define BPF_LD_IND(bpf, reg, ins, type, op) do { \
83 uint32_t ofs = reg[ins->src_reg] + (ins)->imm; \
84 const type *p = bpf_ld_mbuf(bpf, reg, ins, ofs, sizeof(type)); \
87 reg[EBPF_REG_0] = op(p[0]); \
92 bpf_alu_be(uint64_t reg[EBPF_REG_NUM], const struct ebpf_insn *ins)
96 v = reg + ins->dst_reg;
99 *v = rte_cpu_to_be_16(*v);
102 *v = rte_cpu_to_be_32(*v);
105 *v = rte_cpu_to_be_64(*v);
111 bpf_alu_le(uint64_t reg[EBPF_REG_NUM], const struct ebpf_insn *ins)
115 v = reg + ins->dst_reg;
118 *v = rte_cpu_to_le_16(*v);
121 *v = rte_cpu_to_le_32(*v);
124 *v = rte_cpu_to_le_64(*v);
129 static inline const void *
130 bpf_ld_mbuf(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM],
131 const struct ebpf_insn *ins, uint32_t off, uint32_t len)
133 const struct rte_mbuf *mb;
136 mb = (const struct rte_mbuf *)(uintptr_t)reg[EBPF_REG_6];
137 p = rte_pktmbuf_read(mb, off, len, reg + EBPF_REG_0);
139 RTE_BPF_LOG(DEBUG, "%s(bpf=%p, mbuf=%p, ofs=%u, len=%u): "
140 "load beyond packet boundary at pc: %#zx;\n",
141 __func__, bpf, mb, off, len,
142 (uintptr_t)(ins) - (uintptr_t)(bpf)->prm.ins);
146 static inline uint64_t
147 bpf_exec(const struct rte_bpf *bpf, uint64_t reg[EBPF_REG_NUM])
149 const struct ebpf_insn *ins;
151 for (ins = bpf->prm.ins; ; ins++) {
153 /* 32 bit ALU IMM operations */
154 case (BPF_ALU | BPF_ADD | BPF_K):
155 BPF_OP_ALU_IMM(reg, ins, +, uint32_t);
157 case (BPF_ALU | BPF_SUB | BPF_K):
158 BPF_OP_ALU_IMM(reg, ins, -, uint32_t);
160 case (BPF_ALU | BPF_AND | BPF_K):
161 BPF_OP_ALU_IMM(reg, ins, &, uint32_t);
163 case (BPF_ALU | BPF_OR | BPF_K):
164 BPF_OP_ALU_IMM(reg, ins, |, uint32_t);
166 case (BPF_ALU | BPF_LSH | BPF_K):
167 BPF_OP_ALU_IMM(reg, ins, <<, uint32_t);
169 case (BPF_ALU | BPF_RSH | BPF_K):
170 BPF_OP_ALU_IMM(reg, ins, >>, uint32_t);
172 case (BPF_ALU | BPF_XOR | BPF_K):
173 BPF_OP_ALU_IMM(reg, ins, ^, uint32_t);
175 case (BPF_ALU | BPF_MUL | BPF_K):
176 BPF_OP_ALU_IMM(reg, ins, *, uint32_t);
178 case (BPF_ALU | BPF_DIV | BPF_K):
179 BPF_OP_ALU_IMM(reg, ins, /, uint32_t);
181 case (BPF_ALU | BPF_MOD | BPF_K):
182 BPF_OP_ALU_IMM(reg, ins, %, uint32_t);
184 case (BPF_ALU | EBPF_MOV | BPF_K):
185 EBPF_MOV_ALU_IMM(reg, ins, uint32_t);
187 /* 32 bit ALU REG operations */
188 case (BPF_ALU | BPF_ADD | BPF_X):
189 BPF_OP_ALU_REG(reg, ins, +, uint32_t);
191 case (BPF_ALU | BPF_SUB | BPF_X):
192 BPF_OP_ALU_REG(reg, ins, -, uint32_t);
194 case (BPF_ALU | BPF_AND | BPF_X):
195 BPF_OP_ALU_REG(reg, ins, &, uint32_t);
197 case (BPF_ALU | BPF_OR | BPF_X):
198 BPF_OP_ALU_REG(reg, ins, |, uint32_t);
200 case (BPF_ALU | BPF_LSH | BPF_X):
201 BPF_OP_ALU_REG(reg, ins, <<, uint32_t);
203 case (BPF_ALU | BPF_RSH | BPF_X):
204 BPF_OP_ALU_REG(reg, ins, >>, uint32_t);
206 case (BPF_ALU | BPF_XOR | BPF_X):
207 BPF_OP_ALU_REG(reg, ins, ^, uint32_t);
209 case (BPF_ALU | BPF_MUL | BPF_X):
210 BPF_OP_ALU_REG(reg, ins, *, uint32_t);
212 case (BPF_ALU | BPF_DIV | BPF_X):
213 BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint32_t);
214 BPF_OP_ALU_REG(reg, ins, /, uint32_t);
216 case (BPF_ALU | BPF_MOD | BPF_X):
217 BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint32_t);
218 BPF_OP_ALU_REG(reg, ins, %, uint32_t);
220 case (BPF_ALU | EBPF_MOV | BPF_X):
221 EBPF_MOV_ALU_REG(reg, ins, uint32_t);
223 case (BPF_ALU | BPF_NEG):
224 BPF_NEG_ALU(reg, ins, uint32_t);
226 case (BPF_ALU | EBPF_END | EBPF_TO_BE):
227 bpf_alu_be(reg, ins);
229 case (BPF_ALU | EBPF_END | EBPF_TO_LE):
230 bpf_alu_le(reg, ins);
232 /* 64 bit ALU IMM operations */
233 case (EBPF_ALU64 | BPF_ADD | BPF_K):
234 BPF_OP_ALU_IMM(reg, ins, +, uint64_t);
236 case (EBPF_ALU64 | BPF_SUB | BPF_K):
237 BPF_OP_ALU_IMM(reg, ins, -, uint64_t);
239 case (EBPF_ALU64 | BPF_AND | BPF_K):
240 BPF_OP_ALU_IMM(reg, ins, &, uint64_t);
242 case (EBPF_ALU64 | BPF_OR | BPF_K):
243 BPF_OP_ALU_IMM(reg, ins, |, uint64_t);
245 case (EBPF_ALU64 | BPF_LSH | BPF_K):
246 BPF_OP_ALU_IMM(reg, ins, <<, uint64_t);
248 case (EBPF_ALU64 | BPF_RSH | BPF_K):
249 BPF_OP_ALU_IMM(reg, ins, >>, uint64_t);
251 case (EBPF_ALU64 | EBPF_ARSH | BPF_K):
252 BPF_OP_ALU_IMM(reg, ins, >>, int64_t);
254 case (EBPF_ALU64 | BPF_XOR | BPF_K):
255 BPF_OP_ALU_IMM(reg, ins, ^, uint64_t);
257 case (EBPF_ALU64 | BPF_MUL | BPF_K):
258 BPF_OP_ALU_IMM(reg, ins, *, uint64_t);
260 case (EBPF_ALU64 | BPF_DIV | BPF_K):
261 BPF_OP_ALU_IMM(reg, ins, /, uint64_t);
263 case (EBPF_ALU64 | BPF_MOD | BPF_K):
264 BPF_OP_ALU_IMM(reg, ins, %, uint64_t);
266 case (EBPF_ALU64 | EBPF_MOV | BPF_K):
267 EBPF_MOV_ALU_IMM(reg, ins, uint64_t);
269 /* 64 bit ALU REG operations */
270 case (EBPF_ALU64 | BPF_ADD | BPF_X):
271 BPF_OP_ALU_REG(reg, ins, +, uint64_t);
273 case (EBPF_ALU64 | BPF_SUB | BPF_X):
274 BPF_OP_ALU_REG(reg, ins, -, uint64_t);
276 case (EBPF_ALU64 | BPF_AND | BPF_X):
277 BPF_OP_ALU_REG(reg, ins, &, uint64_t);
279 case (EBPF_ALU64 | BPF_OR | BPF_X):
280 BPF_OP_ALU_REG(reg, ins, |, uint64_t);
282 case (EBPF_ALU64 | BPF_LSH | BPF_X):
283 BPF_OP_ALU_REG(reg, ins, <<, uint64_t);
285 case (EBPF_ALU64 | BPF_RSH | BPF_X):
286 BPF_OP_ALU_REG(reg, ins, >>, uint64_t);
288 case (EBPF_ALU64 | EBPF_ARSH | BPF_X):
289 BPF_OP_ALU_REG(reg, ins, >>, int64_t);
291 case (EBPF_ALU64 | BPF_XOR | BPF_X):
292 BPF_OP_ALU_REG(reg, ins, ^, uint64_t);
294 case (EBPF_ALU64 | BPF_MUL | BPF_X):
295 BPF_OP_ALU_REG(reg, ins, *, uint64_t);
297 case (EBPF_ALU64 | BPF_DIV | BPF_X):
298 BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint64_t);
299 BPF_OP_ALU_REG(reg, ins, /, uint64_t);
301 case (EBPF_ALU64 | BPF_MOD | BPF_X):
302 BPF_DIV_ZERO_CHECK(bpf, reg, ins, uint64_t);
303 BPF_OP_ALU_REG(reg, ins, %, uint64_t);
305 case (EBPF_ALU64 | EBPF_MOV | BPF_X):
306 EBPF_MOV_ALU_REG(reg, ins, uint64_t);
308 case (EBPF_ALU64 | BPF_NEG):
309 BPF_NEG_ALU(reg, ins, uint64_t);
311 /* load instructions */
312 case (BPF_LDX | BPF_MEM | BPF_B):
313 BPF_LD_REG(reg, ins, uint8_t);
315 case (BPF_LDX | BPF_MEM | BPF_H):
316 BPF_LD_REG(reg, ins, uint16_t);
318 case (BPF_LDX | BPF_MEM | BPF_W):
319 BPF_LD_REG(reg, ins, uint32_t);
321 case (BPF_LDX | BPF_MEM | EBPF_DW):
322 BPF_LD_REG(reg, ins, uint64_t);
324 /* load 64 bit immediate value */
325 case (BPF_LD | BPF_IMM | EBPF_DW):
326 reg[ins->dst_reg] = (uint32_t)ins[0].imm |
327 (uint64_t)(uint32_t)ins[1].imm << 32;
330 /* load absolute instructions */
331 case (BPF_LD | BPF_ABS | BPF_B):
332 BPF_LD_ABS(bpf, reg, ins, uint8_t, NOP);
334 case (BPF_LD | BPF_ABS | BPF_H):
335 BPF_LD_ABS(bpf, reg, ins, uint16_t, rte_be_to_cpu_16);
337 case (BPF_LD | BPF_ABS | BPF_W):
338 BPF_LD_ABS(bpf, reg, ins, uint32_t, rte_be_to_cpu_32);
340 /* load indirect instructions */
341 case (BPF_LD | BPF_IND | BPF_B):
342 BPF_LD_IND(bpf, reg, ins, uint8_t, NOP);
344 case (BPF_LD | BPF_IND | BPF_H):
345 BPF_LD_IND(bpf, reg, ins, uint16_t, rte_be_to_cpu_16);
347 case (BPF_LD | BPF_IND | BPF_W):
348 BPF_LD_IND(bpf, reg, ins, uint32_t, rte_be_to_cpu_32);
350 /* store instructions */
351 case (BPF_STX | BPF_MEM | BPF_B):
352 BPF_ST_REG(reg, ins, uint8_t);
354 case (BPF_STX | BPF_MEM | BPF_H):
355 BPF_ST_REG(reg, ins, uint16_t);
357 case (BPF_STX | BPF_MEM | BPF_W):
358 BPF_ST_REG(reg, ins, uint32_t);
360 case (BPF_STX | BPF_MEM | EBPF_DW):
361 BPF_ST_REG(reg, ins, uint64_t);
363 case (BPF_ST | BPF_MEM | BPF_B):
364 BPF_ST_IMM(reg, ins, uint8_t);
366 case (BPF_ST | BPF_MEM | BPF_H):
367 BPF_ST_IMM(reg, ins, uint16_t);
369 case (BPF_ST | BPF_MEM | BPF_W):
370 BPF_ST_IMM(reg, ins, uint32_t);
372 case (BPF_ST | BPF_MEM | EBPF_DW):
373 BPF_ST_IMM(reg, ins, uint64_t);
375 /* atomic add instructions */
376 case (BPF_STX | EBPF_XADD | BPF_W):
377 BPF_ST_XADD_REG(reg, ins, 32);
379 case (BPF_STX | EBPF_XADD | EBPF_DW):
380 BPF_ST_XADD_REG(reg, ins, 64);
382 /* jump instructions */
383 case (BPF_JMP | BPF_JA):
386 /* jump IMM instructions */
387 case (BPF_JMP | BPF_JEQ | BPF_K):
388 BPF_JMP_CND_IMM(reg, ins, ==, uint64_t);
390 case (BPF_JMP | EBPF_JNE | BPF_K):
391 BPF_JMP_CND_IMM(reg, ins, !=, uint64_t);
393 case (BPF_JMP | BPF_JGT | BPF_K):
394 BPF_JMP_CND_IMM(reg, ins, >, uint64_t);
396 case (BPF_JMP | EBPF_JLT | BPF_K):
397 BPF_JMP_CND_IMM(reg, ins, <, uint64_t);
399 case (BPF_JMP | BPF_JGE | BPF_K):
400 BPF_JMP_CND_IMM(reg, ins, >=, uint64_t);
402 case (BPF_JMP | EBPF_JLE | BPF_K):
403 BPF_JMP_CND_IMM(reg, ins, <=, uint64_t);
405 case (BPF_JMP | EBPF_JSGT | BPF_K):
406 BPF_JMP_CND_IMM(reg, ins, >, int64_t);
408 case (BPF_JMP | EBPF_JSLT | BPF_K):
409 BPF_JMP_CND_IMM(reg, ins, <, int64_t);
411 case (BPF_JMP | EBPF_JSGE | BPF_K):
412 BPF_JMP_CND_IMM(reg, ins, >=, int64_t);
414 case (BPF_JMP | EBPF_JSLE | BPF_K):
415 BPF_JMP_CND_IMM(reg, ins, <=, int64_t);
417 case (BPF_JMP | BPF_JSET | BPF_K):
418 BPF_JMP_CND_IMM(reg, ins, &, uint64_t);
420 /* jump REG instructions */
421 case (BPF_JMP | BPF_JEQ | BPF_X):
422 BPF_JMP_CND_REG(reg, ins, ==, uint64_t);
424 case (BPF_JMP | EBPF_JNE | BPF_X):
425 BPF_JMP_CND_REG(reg, ins, !=, uint64_t);
427 case (BPF_JMP | BPF_JGT | BPF_X):
428 BPF_JMP_CND_REG(reg, ins, >, uint64_t);
430 case (BPF_JMP | EBPF_JLT | BPF_X):
431 BPF_JMP_CND_REG(reg, ins, <, uint64_t);
433 case (BPF_JMP | BPF_JGE | BPF_X):
434 BPF_JMP_CND_REG(reg, ins, >=, uint64_t);
436 case (BPF_JMP | EBPF_JLE | BPF_X):
437 BPF_JMP_CND_REG(reg, ins, <=, uint64_t);
439 case (BPF_JMP | EBPF_JSGT | BPF_X):
440 BPF_JMP_CND_REG(reg, ins, >, int64_t);
442 case (BPF_JMP | EBPF_JSLT | BPF_X):
443 BPF_JMP_CND_REG(reg, ins, <, int64_t);
445 case (BPF_JMP | EBPF_JSGE | BPF_X):
446 BPF_JMP_CND_REG(reg, ins, >=, int64_t);
448 case (BPF_JMP | EBPF_JSLE | BPF_X):
449 BPF_JMP_CND_REG(reg, ins, <=, int64_t);
451 case (BPF_JMP | BPF_JSET | BPF_X):
452 BPF_JMP_CND_REG(reg, ins, &, uint64_t);
454 /* call instructions */
455 case (BPF_JMP | EBPF_CALL):
456 reg[EBPF_REG_0] = bpf->prm.xsym[ins->imm].func.val(
457 reg[EBPF_REG_1], reg[EBPF_REG_2],
458 reg[EBPF_REG_3], reg[EBPF_REG_4],
461 /* return instruction */
462 case (BPF_JMP | EBPF_EXIT):
463 return reg[EBPF_REG_0];
466 "%s(%p): invalid opcode %#x at pc: %#zx;\n",
467 __func__, bpf, ins->code,
468 (uintptr_t)ins - (uintptr_t)bpf->prm.ins);
473 /* should never be reached */
479 rte_bpf_exec_burst(const struct rte_bpf *bpf, void *ctx[], uint64_t rc[],
483 uint64_t reg[EBPF_REG_NUM];
484 uint64_t stack[MAX_BPF_STACK_SIZE / sizeof(uint64_t)];
486 for (i = 0; i != num; i++) {
488 reg[EBPF_REG_1] = (uintptr_t)ctx[i];
489 reg[EBPF_REG_10] = (uintptr_t)(stack + RTE_DIM(stack));
491 rc[i] = bpf_exec(bpf, reg);
498 rte_bpf_exec(const struct rte_bpf *bpf, void *ctx)
502 rte_bpf_exec_burst(bpf, &ctx, &rc, 1);