1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
11 #include <rte_common.h>
15 #define BPF_ARG_PTR_STACK RTE_BPF_ARG_RESERVED
30 struct bpf_eval_state {
31 struct bpf_reg_val rv[EBPF_REG_NUM];
32 struct bpf_reg_val sv[MAX_BPF_STACK_SIZE / sizeof(uint64_t)];
35 /* possible instruction node colour */
43 /* possible edge types */
58 uint8_t edge_type[MAX_EDGES];
59 uint32_t edge_dest[MAX_EDGES];
61 struct bpf_eval_state *evst;
65 const struct rte_bpf_prm *prm;
69 uint32_t nb_jcc_nodes;
70 uint32_t nb_ldmb_nodes;
71 uint32_t node_colour[MAX_NODE_COLOUR];
72 uint32_t edge_type[MAX_EDGE_TYPE];
73 struct bpf_eval_state *evst;
74 struct inst_node *evin;
78 struct bpf_eval_state *ent;
82 struct bpf_ins_check {
95 const char * (*check)(const struct ebpf_insn *);
96 const char * (*eval)(struct bpf_verifier *, const struct ebpf_insn *);
99 #define ALL_REGS RTE_LEN2MASK(EBPF_REG_NUM, uint16_t)
100 #define WRT_REGS RTE_LEN2MASK(EBPF_REG_10, uint16_t)
101 #define ZERO_REG RTE_LEN2MASK(EBPF_REG_1, uint16_t)
103 /* For LD_IND R6 is an implicit CTX register. */
104 #define IND_SRC_REGS (WRT_REGS ^ 1 << EBPF_REG_6)
107 * check and evaluate functions for particular instruction types.
111 check_alu_bele(const struct ebpf_insn *ins)
113 if (ins->imm != 16 && ins->imm != 32 && ins->imm != 64)
114 return "invalid imm field";
119 eval_exit(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
122 if (bvf->evst->rv[EBPF_REG_0].v.type == RTE_BPF_ARG_UNDEF)
123 return "undefined return value";
127 /* setup max possible with this mask bounds */
129 eval_umax_bound(struct bpf_reg_val *rv, uint64_t mask)
136 eval_smax_bound(struct bpf_reg_val *rv, uint64_t mask)
138 rv->s.max = mask >> 1;
139 rv->s.min = rv->s.max ^ UINT64_MAX;
143 eval_max_bound(struct bpf_reg_val *rv, uint64_t mask)
145 eval_umax_bound(rv, mask);
146 eval_smax_bound(rv, mask);
150 eval_fill_max_bound(struct bpf_reg_val *rv, uint64_t mask)
152 eval_max_bound(rv, mask);
153 rv->v.type = RTE_BPF_ARG_RAW;
158 eval_fill_imm64(struct bpf_reg_val *rv, uint64_t mask, uint64_t val)
168 eval_fill_imm(struct bpf_reg_val *rv, uint64_t mask, int32_t imm)
172 v = (uint64_t)imm & mask;
174 rv->v.type = RTE_BPF_ARG_RAW;
175 eval_fill_imm64(rv, mask, v);
179 eval_ld_imm64(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
183 struct bpf_reg_val *rd;
185 val = (uint32_t)ins[0].imm | (uint64_t)(uint32_t)ins[1].imm << 32;
187 rd = bvf->evst->rv + ins->dst_reg;
188 rd->v.type = RTE_BPF_ARG_RAW;
189 eval_fill_imm64(rd, UINT64_MAX, val);
191 for (i = 0; i != bvf->prm->nb_xsym; i++) {
193 /* load of external variable */
194 if (bvf->prm->xsym[i].type == RTE_BPF_XTYPE_VAR &&
195 (uintptr_t)bvf->prm->xsym[i].var.val == val) {
196 rd->v = bvf->prm->xsym[i].var.desc;
197 eval_fill_imm64(rd, UINT64_MAX, 0);
206 eval_apply_mask(struct bpf_reg_val *rv, uint64_t mask)
208 struct bpf_reg_val rt;
210 rt.u.min = rv->u.min & mask;
211 rt.u.max = rv->u.max & mask;
212 if (rt.u.min != rv->u.min || rt.u.max != rv->u.max) {
213 rv->u.max = RTE_MAX(rt.u.max, mask);
217 eval_smax_bound(&rt, mask);
218 rv->s.max = RTE_MIN(rt.s.max, rv->s.max);
219 rv->s.min = RTE_MAX(rt.s.min, rv->s.min);
225 eval_add(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, uint64_t msk)
227 struct bpf_reg_val rv;
229 rv.u.min = (rd->u.min + rs->u.min) & msk;
230 rv.u.max = (rd->u.max + rs->u.max) & msk;
231 rv.s.min = (rd->s.min + rs->s.min) & msk;
232 rv.s.max = (rd->s.max + rs->s.max) & msk;
235 * if at least one of the operands is not constant,
236 * then check for overflow
238 if ((rd->u.min != rd->u.max || rs->u.min != rs->u.max) &&
239 (rv.u.min < rd->u.min || rv.u.max < rd->u.max))
240 eval_umax_bound(&rv, msk);
242 if ((rd->s.min != rd->s.max || rs->s.min != rs->s.max) &&
243 (((rs->s.min < 0 && rv.s.min > rd->s.min) ||
244 rv.s.min < rd->s.min) ||
245 ((rs->s.max < 0 && rv.s.max > rd->s.max) ||
246 rv.s.max < rd->s.max)))
247 eval_smax_bound(&rv, msk);
254 eval_sub(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, uint64_t msk)
256 struct bpf_reg_val rv;
258 rv.u.min = (rd->u.min - rs->u.max) & msk;
259 rv.u.max = (rd->u.max - rs->u.min) & msk;
260 rv.s.min = (rd->s.min - rs->s.max) & msk;
261 rv.s.max = (rd->s.max - rs->s.min) & msk;
264 * if at least one of the operands is not constant,
265 * then check for overflow
267 if ((rd->u.min != rd->u.max || rs->u.min != rs->u.max) &&
268 (rv.u.min > rd->u.min || rv.u.max > rd->u.max))
269 eval_umax_bound(&rv, msk);
271 if ((rd->s.min != rd->s.max || rs->s.min != rs->s.max) &&
272 (((rs->s.min < 0 && rv.s.min < rd->s.min) ||
273 rv.s.min > rd->s.min) ||
274 ((rs->s.max < 0 && rv.s.max < rd->s.max) ||
275 rv.s.max > rd->s.max)))
276 eval_smax_bound(&rv, msk);
283 eval_lsh(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, size_t opsz,
286 /* check if shift value is less then max result bits */
287 if (rs->u.max >= opsz) {
288 eval_max_bound(rd, msk);
292 /* check for overflow */
293 if (rd->u.max > RTE_LEN2MASK(opsz - rs->u.max, uint64_t))
294 eval_umax_bound(rd, msk);
296 rd->u.max <<= rs->u.max;
297 rd->u.min <<= rs->u.min;
300 /* check that dreg values are and would remain always positive */
301 if ((uint64_t)rd->s.min >> (opsz - 1) != 0 || rd->s.max >=
302 RTE_LEN2MASK(opsz - rs->u.max - 1, int64_t))
303 eval_smax_bound(rd, msk);
305 rd->s.max <<= rs->u.max;
306 rd->s.min <<= rs->u.min;
311 eval_rsh(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, size_t opsz,
314 /* check if shift value is less then max result bits */
315 if (rs->u.max >= opsz) {
316 eval_max_bound(rd, msk);
320 rd->u.max >>= rs->u.min;
321 rd->u.min >>= rs->u.max;
323 /* check that dreg values are always positive */
324 if ((uint64_t)rd->s.min >> (opsz - 1) != 0)
325 eval_smax_bound(rd, msk);
327 rd->s.max >>= rs->u.min;
328 rd->s.min >>= rs->u.max;
333 eval_arsh(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, size_t opsz,
338 /* check if shift value is less then max result bits */
339 if (rs->u.max >= opsz) {
340 eval_max_bound(rd, msk);
344 rd->u.max = (int64_t)rd->u.max >> rs->u.min;
345 rd->u.min = (int64_t)rd->u.min >> rs->u.max;
347 /* if we have 32-bit values - extend them to 64-bit */
348 if (opsz == sizeof(uint32_t) * CHAR_BIT) {
356 rd->s.min = (rd->s.min >> (rs->u.min + shv)) & msk;
358 rd->s.min = (rd->s.min >> (rs->u.max + shv)) & msk;
361 rd->s.max = (rd->s.max >> (rs->u.max + shv)) & msk;
363 rd->s.max = (rd->s.max >> (rs->u.min + shv)) & msk;
367 eval_umax_bits(uint64_t v, size_t opsz)
372 v = __builtin_clzll(v);
373 return RTE_LEN2MASK(opsz - v, uint64_t);
376 /* estimate max possible value for (v1 & v2) */
378 eval_uand_max(uint64_t v1, uint64_t v2, size_t opsz)
380 v1 = eval_umax_bits(v1, opsz);
381 v2 = eval_umax_bits(v2, opsz);
385 /* estimate max possible value for (v1 | v2) */
387 eval_uor_max(uint64_t v1, uint64_t v2, size_t opsz)
389 v1 = eval_umax_bits(v1, opsz);
390 v2 = eval_umax_bits(v2, opsz);
395 eval_and(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, size_t opsz,
398 /* both operands are constants */
399 if (rd->u.min == rd->u.max && rs->u.min == rs->u.max) {
400 rd->u.min &= rs->u.min;
401 rd->u.max &= rs->u.max;
403 rd->u.max = eval_uand_max(rd->u.max, rs->u.max, opsz);
404 rd->u.min &= rs->u.min;
407 /* both operands are constants */
408 if (rd->s.min == rd->s.max && rs->s.min == rs->s.max) {
409 rd->s.min &= rs->s.min;
410 rd->s.max &= rs->s.max;
411 /* at least one of operand is non-negative */
412 } else if (rd->s.min >= 0 || rs->s.min >= 0) {
413 rd->s.max = eval_uand_max(rd->s.max & (msk >> 1),
414 rs->s.max & (msk >> 1), opsz);
415 rd->s.min &= rs->s.min;
417 eval_smax_bound(rd, msk);
421 eval_or(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, size_t opsz,
424 /* both operands are constants */
425 if (rd->u.min == rd->u.max && rs->u.min == rs->u.max) {
426 rd->u.min |= rs->u.min;
427 rd->u.max |= rs->u.max;
429 rd->u.max = eval_uor_max(rd->u.max, rs->u.max, opsz);
430 rd->u.min |= rs->u.min;
433 /* both operands are constants */
434 if (rd->s.min == rd->s.max && rs->s.min == rs->s.max) {
435 rd->s.min |= rs->s.min;
436 rd->s.max |= rs->s.max;
438 /* both operands are non-negative */
439 } else if (rd->s.min >= 0 || rs->s.min >= 0) {
440 rd->s.max = eval_uor_max(rd->s.max, rs->s.max, opsz);
441 rd->s.min |= rs->s.min;
443 eval_smax_bound(rd, msk);
447 eval_xor(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, size_t opsz,
450 /* both operands are constants */
451 if (rd->u.min == rd->u.max && rs->u.min == rs->u.max) {
452 rd->u.min ^= rs->u.min;
453 rd->u.max ^= rs->u.max;
455 rd->u.max = eval_uor_max(rd->u.max, rs->u.max, opsz);
459 /* both operands are constants */
460 if (rd->s.min == rd->s.max && rs->s.min == rs->s.max) {
461 rd->s.min ^= rs->s.min;
462 rd->s.max ^= rs->s.max;
464 /* both operands are non-negative */
465 } else if (rd->s.min >= 0 || rs->s.min >= 0) {
466 rd->s.max = eval_uor_max(rd->s.max, rs->s.max, opsz);
469 eval_smax_bound(rd, msk);
473 eval_mul(struct bpf_reg_val *rd, const struct bpf_reg_val *rs, size_t opsz,
476 /* both operands are constants */
477 if (rd->u.min == rd->u.max && rs->u.min == rs->u.max) {
478 rd->u.min = (rd->u.min * rs->u.min) & msk;
479 rd->u.max = (rd->u.max * rs->u.max) & msk;
480 /* check for overflow */
481 } else if (rd->u.max <= msk >> opsz / 2 && rs->u.max <= msk >> opsz) {
482 rd->u.max *= rs->u.max;
483 rd->u.min *= rd->u.min;
485 eval_umax_bound(rd, msk);
487 /* both operands are constants */
488 if (rd->s.min == rd->s.max && rs->s.min == rs->s.max) {
489 rd->s.min = (rd->s.min * rs->s.min) & msk;
490 rd->s.max = (rd->s.max * rs->s.max) & msk;
491 /* check that both operands are positive and no overflow */
492 } else if (rd->s.min >= 0 && rs->s.min >= 0) {
493 rd->s.max *= rs->s.max;
494 rd->s.min *= rd->s.min;
496 eval_smax_bound(rd, msk);
500 eval_divmod(uint32_t op, struct bpf_reg_val *rd, struct bpf_reg_val *rs,
501 size_t opsz, uint64_t msk)
503 /* both operands are constants */
504 if (rd->u.min == rd->u.max && rs->u.min == rs->u.max) {
506 return "division by 0";
508 rd->u.min /= rs->u.min;
509 rd->u.max /= rs->u.max;
511 rd->u.min %= rs->u.min;
512 rd->u.max %= rs->u.max;
516 rd->u.max = RTE_MIN(rd->u.max, rs->u.max - 1);
518 rd->u.max = rd->u.max;
522 /* if we have 32-bit values - extend them to 64-bit */
523 if (opsz == sizeof(uint32_t) * CHAR_BIT) {
524 rd->s.min = (int32_t)rd->s.min;
525 rd->s.max = (int32_t)rd->s.max;
526 rs->s.min = (int32_t)rs->s.min;
527 rs->s.max = (int32_t)rs->s.max;
530 /* both operands are constants */
531 if (rd->s.min == rd->s.max && rs->s.min == rs->s.max) {
533 return "division by 0";
535 rd->s.min /= rs->s.min;
536 rd->s.max /= rs->s.max;
538 rd->s.min %= rs->s.min;
539 rd->s.max %= rs->s.max;
541 } else if (op == BPF_MOD) {
542 rd->s.min = RTE_MAX(rd->s.max, 0);
543 rd->s.min = RTE_MIN(rd->s.min, 0);
545 eval_smax_bound(rd, msk);
554 eval_neg(struct bpf_reg_val *rd, size_t opsz, uint64_t msk)
559 /* if we have 32-bit values - extend them to 64-bit */
560 if (opsz == sizeof(uint32_t) * CHAR_BIT) {
561 rd->u.min = (int32_t)rd->u.min;
562 rd->u.max = (int32_t)rd->u.max;
565 ux = -(int64_t)rd->u.min & msk;
566 uy = -(int64_t)rd->u.max & msk;
568 rd->u.max = RTE_MAX(ux, uy);
569 rd->u.min = RTE_MIN(ux, uy);
571 /* if we have 32-bit values - extend them to 64-bit */
572 if (opsz == sizeof(uint32_t) * CHAR_BIT) {
573 rd->s.min = (int32_t)rd->s.min;
574 rd->s.max = (int32_t)rd->s.max;
577 sx = -rd->s.min & msk;
578 sy = -rd->s.max & msk;
580 rd->s.max = RTE_MAX(sx, sy);
581 rd->s.min = RTE_MIN(sx, sy);
585 eval_ld_mbuf(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
588 struct bpf_reg_val *rv, ri, rs;
590 mode = BPF_MODE(ins->code);
592 /* R6 is an implicit input that must contain pointer to mbuf */
593 if (bvf->evst->rv[EBPF_REG_6].v.type != RTE_BPF_ARG_PTR_MBUF)
594 return "invalid type for implicit ctx register";
596 if (mode == BPF_IND) {
597 rs = bvf->evst->rv[ins->src_reg];
598 if (rs.v.type != RTE_BPF_ARG_RAW)
599 return "unexpected type for src register";
601 eval_fill_imm(&ri, UINT64_MAX, ins->imm);
602 eval_add(&rs, &ri, UINT64_MAX);
604 if (rs.s.max < 0 || rs.u.min > UINT32_MAX)
605 return "mbuf boundary violation";
608 /* R1-R5 scratch registers */
609 for (i = EBPF_REG_1; i != EBPF_REG_6; i++)
610 bvf->evst->rv[i].v.type = RTE_BPF_ARG_UNDEF;
612 /* R0 is an implicit output, contains data fetched from the packet */
613 rv = bvf->evst->rv + EBPF_REG_0;
614 rv->v.size = bpf_size(BPF_SIZE(ins->code));
615 eval_fill_max_bound(rv, RTE_LEN2MASK(rv->v.size * CHAR_BIT, uint64_t));
621 * check that destination and source operand are in defined state.
624 eval_defined(const struct bpf_reg_val *dst, const struct bpf_reg_val *src)
626 if (dst != NULL && dst->v.type == RTE_BPF_ARG_UNDEF)
627 return "dest reg value is undefined";
628 if (src != NULL && src->v.type == RTE_BPF_ARG_UNDEF)
629 return "src reg value is undefined";
634 eval_alu(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
640 struct bpf_eval_state *st;
641 struct bpf_reg_val *rd, rs;
643 opsz = (BPF_CLASS(ins->code) == BPF_ALU) ?
644 sizeof(uint32_t) : sizeof(uint64_t);
645 opsz = opsz * CHAR_BIT;
646 msk = RTE_LEN2MASK(opsz, uint64_t);
649 rd = st->rv + ins->dst_reg;
651 if (BPF_SRC(ins->code) == BPF_X) {
652 rs = st->rv[ins->src_reg];
653 eval_apply_mask(&rs, msk);
655 eval_fill_imm(&rs, msk, ins->imm);
657 eval_apply_mask(rd, msk);
659 op = BPF_OP(ins->code);
661 /* Allow self-xor as way to zero register */
662 if (op == BPF_XOR && BPF_SRC(ins->code) == BPF_X &&
663 ins->src_reg == ins->dst_reg) {
664 eval_fill_imm(&rs, UINT64_MAX, 0);
665 eval_fill_imm(rd, UINT64_MAX, 0);
668 err = eval_defined((op != EBPF_MOV) ? rd : NULL,
669 (op != BPF_NEG) ? &rs : NULL);
674 eval_add(rd, &rs, msk);
675 else if (op == BPF_SUB)
676 eval_sub(rd, &rs, msk);
677 else if (op == BPF_LSH)
678 eval_lsh(rd, &rs, opsz, msk);
679 else if (op == BPF_RSH)
680 eval_rsh(rd, &rs, opsz, msk);
681 else if (op == EBPF_ARSH)
682 eval_arsh(rd, &rs, opsz, msk);
683 else if (op == BPF_AND)
684 eval_and(rd, &rs, opsz, msk);
685 else if (op == BPF_OR)
686 eval_or(rd, &rs, opsz, msk);
687 else if (op == BPF_XOR)
688 eval_xor(rd, &rs, opsz, msk);
689 else if (op == BPF_MUL)
690 eval_mul(rd, &rs, opsz, msk);
691 else if (op == BPF_DIV || op == BPF_MOD)
692 err = eval_divmod(op, rd, &rs, opsz, msk);
693 else if (op == BPF_NEG)
694 eval_neg(rd, opsz, msk);
695 else if (op == EBPF_MOV)
698 eval_max_bound(rd, msk);
704 eval_bele(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
707 struct bpf_eval_state *st;
708 struct bpf_reg_val *rd;
711 msk = RTE_LEN2MASK(ins->imm, uint64_t);
714 rd = st->rv + ins->dst_reg;
716 err = eval_defined(rd, NULL);
720 #if RTE_BYTE_ORDER == RTE_LITTLE_ENDIAN
721 if (ins->code == (BPF_ALU | EBPF_END | EBPF_TO_BE))
722 eval_max_bound(rd, msk);
724 eval_apply_mask(rd, msk);
726 if (ins->code == (BPF_ALU | EBPF_END | EBPF_TO_LE))
727 eval_max_bound(rd, msk);
729 eval_apply_mask(rd, msk);
736 eval_ptr(struct bpf_verifier *bvf, struct bpf_reg_val *rm, uint32_t opsz,
737 uint32_t align, int16_t off)
739 struct bpf_reg_val rv;
741 /* calculate reg + offset */
742 eval_fill_imm(&rv, rm->mask, off);
743 eval_add(rm, &rv, rm->mask);
745 if (RTE_BPF_ARG_PTR_TYPE(rm->v.type) == 0)
746 return "destination is not a pointer";
748 if (rm->mask != UINT64_MAX)
749 return "pointer truncation";
751 if (rm->u.max + opsz > rm->v.size ||
752 (uint64_t)rm->s.max + opsz > rm->v.size ||
754 return "memory boundary violation";
756 if (rm->u.max % align != 0)
757 return "unaligned memory access";
759 if (rm->v.type == BPF_ARG_PTR_STACK) {
761 if (rm->u.max != rm->u.min || rm->s.max != rm->s.min ||
762 rm->u.max != (uint64_t)rm->s.max)
763 return "stack access with variable offset";
765 bvf->stack_sz = RTE_MAX(bvf->stack_sz, rm->v.size - rm->u.max);
767 /* pointer to mbuf */
768 } else if (rm->v.type == RTE_BPF_ARG_PTR_MBUF) {
770 if (rm->u.max != rm->u.min || rm->s.max != rm->s.min ||
771 rm->u.max != (uint64_t)rm->s.max)
772 return "mbuf access with variable offset";
779 eval_max_load(struct bpf_reg_val *rv, uint64_t mask)
781 eval_umax_bound(rv, mask);
783 /* full 64-bit load */
784 if (mask == UINT64_MAX)
785 eval_smax_bound(rv, mask);
787 /* zero-extend load */
788 rv->s.min = rv->u.min;
789 rv->s.max = rv->u.max;
794 eval_load(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
799 struct bpf_eval_state *st;
800 struct bpf_reg_val *rd, rs;
801 const struct bpf_reg_val *sv;
804 rd = st->rv + ins->dst_reg;
805 rs = st->rv[ins->src_reg];
806 opsz = bpf_size(BPF_SIZE(ins->code));
807 msk = RTE_LEN2MASK(opsz * CHAR_BIT, uint64_t);
809 err = eval_ptr(bvf, &rs, opsz, 1, ins->off);
813 if (rs.v.type == BPF_ARG_PTR_STACK) {
815 sv = st->sv + rs.u.max / sizeof(uint64_t);
816 if (sv->v.type == RTE_BPF_ARG_UNDEF || sv->mask < msk)
817 return "undefined value on the stack";
821 /* pointer to mbuf */
822 } else if (rs.v.type == RTE_BPF_ARG_PTR_MBUF) {
824 if (rs.u.max == offsetof(struct rte_mbuf, next)) {
825 eval_fill_imm(rd, msk, 0);
827 } else if (rs.u.max == offsetof(struct rte_mbuf, buf_addr)) {
828 eval_fill_imm(rd, msk, 0);
829 rd->v.type = RTE_BPF_ARG_PTR;
830 rd->v.size = rs.v.buf_size;
831 } else if (rs.u.max == offsetof(struct rte_mbuf, data_off)) {
832 eval_fill_imm(rd, msk, RTE_PKTMBUF_HEADROOM);
833 rd->v.type = RTE_BPF_ARG_RAW;
835 eval_max_load(rd, msk);
836 rd->v.type = RTE_BPF_ARG_RAW;
839 /* pointer to raw data */
841 eval_max_load(rd, msk);
842 rd->v.type = RTE_BPF_ARG_RAW;
849 eval_mbuf_store(const struct bpf_reg_val *rv, uint32_t opsz)
853 static const struct {
856 } mbuf_ro_fileds[] = {
857 { .off = offsetof(struct rte_mbuf, buf_addr), },
858 { .off = offsetof(struct rte_mbuf, refcnt), },
859 { .off = offsetof(struct rte_mbuf, nb_segs), },
860 { .off = offsetof(struct rte_mbuf, buf_len), },
861 { .off = offsetof(struct rte_mbuf, pool), },
862 { .off = offsetof(struct rte_mbuf, next), },
863 { .off = offsetof(struct rte_mbuf, priv_size), },
866 for (i = 0; i != RTE_DIM(mbuf_ro_fileds) &&
867 (mbuf_ro_fileds[i].off + mbuf_ro_fileds[i].sz <=
868 rv->u.max || rv->u.max + opsz <= mbuf_ro_fileds[i].off);
872 if (i != RTE_DIM(mbuf_ro_fileds))
873 return "store to the read-only mbuf field";
880 eval_store(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
885 struct bpf_eval_state *st;
886 struct bpf_reg_val rd, rs, *sv;
888 opsz = bpf_size(BPF_SIZE(ins->code));
889 msk = RTE_LEN2MASK(opsz * CHAR_BIT, uint64_t);
892 rd = st->rv[ins->dst_reg];
894 if (BPF_CLASS(ins->code) == BPF_STX) {
895 rs = st->rv[ins->src_reg];
896 eval_apply_mask(&rs, msk);
898 eval_fill_imm(&rs, msk, ins->imm);
900 err = eval_defined(NULL, &rs);
904 err = eval_ptr(bvf, &rd, opsz, 1, ins->off);
908 if (rd.v.type == BPF_ARG_PTR_STACK) {
910 sv = st->sv + rd.u.max / sizeof(uint64_t);
911 if (BPF_CLASS(ins->code) == BPF_STX &&
912 BPF_MODE(ins->code) == EBPF_XADD)
913 eval_max_bound(sv, msk);
917 /* pointer to mbuf */
918 } else if (rd.v.type == RTE_BPF_ARG_PTR_MBUF) {
919 err = eval_mbuf_store(&rd, opsz);
928 eval_func_arg(struct bpf_verifier *bvf, const struct rte_bpf_arg *arg,
929 struct bpf_reg_val *rv)
932 struct bpf_eval_state *st;
937 if (rv->v.type == RTE_BPF_ARG_UNDEF)
938 return "Undefined argument type";
940 if (arg->type != rv->v.type &&
941 arg->type != RTE_BPF_ARG_RAW &&
942 (arg->type != RTE_BPF_ARG_PTR ||
943 RTE_BPF_ARG_PTR_TYPE(rv->v.type) == 0))
944 return "Invalid argument type";
948 /* argument is a pointer */
949 if (RTE_BPF_ARG_PTR_TYPE(arg->type) != 0) {
951 err = eval_ptr(bvf, rv, arg->size, 1, 0);
954 * pointer to the variable on the stack is passed
955 * as an argument, mark stack space it occupies as initialized.
957 if (err == NULL && rv->v.type == BPF_ARG_PTR_STACK) {
959 i = rv->u.max / sizeof(uint64_t);
960 n = i + arg->size / sizeof(uint64_t);
962 eval_fill_max_bound(st->sv + i, UINT64_MAX);
972 eval_call(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
975 struct bpf_reg_val *rv;
976 const struct rte_bpf_xsym *xsym;
981 if (idx >= bvf->prm->nb_xsym ||
982 bvf->prm->xsym[idx].type != RTE_BPF_XTYPE_FUNC)
983 return "invalid external function index";
985 /* for now don't support function calls on 32 bit platform */
986 if (sizeof(uint64_t) != sizeof(uintptr_t))
987 return "function calls are supported only for 64 bit apps";
989 xsym = bvf->prm->xsym + idx;
991 /* evaluate function arguments */
993 for (i = 0; i != xsym->func.nb_args && err == NULL; i++) {
994 err = eval_func_arg(bvf, xsym->func.args + i,
995 bvf->evst->rv + EBPF_REG_1 + i);
998 /* R1-R5 argument/scratch registers */
999 for (i = EBPF_REG_1; i != EBPF_REG_6; i++)
1000 bvf->evst->rv[i].v.type = RTE_BPF_ARG_UNDEF;
1002 /* update return value */
1004 rv = bvf->evst->rv + EBPF_REG_0;
1005 rv->v = xsym->func.ret;
1006 if (rv->v.type == RTE_BPF_ARG_RAW)
1007 eval_fill_max_bound(rv,
1008 RTE_LEN2MASK(rv->v.size * CHAR_BIT, uint64_t));
1009 else if (RTE_BPF_ARG_PTR_TYPE(rv->v.type) != 0)
1010 eval_fill_imm64(rv, UINTPTR_MAX, 0);
1016 eval_jeq_jne(struct bpf_reg_val *trd, struct bpf_reg_val *trs)
1018 /* sreg is constant */
1019 if (trs->u.min == trs->u.max) {
1021 /* dreg is constant */
1022 } else if (trd->u.min == trd->u.max) {
1025 trd->u.max = RTE_MIN(trd->u.max, trs->u.max);
1026 trd->u.min = RTE_MAX(trd->u.min, trs->u.min);
1030 /* sreg is constant */
1031 if (trs->s.min == trs->s.max) {
1033 /* dreg is constant */
1034 } else if (trd->s.min == trd->s.max) {
1037 trd->s.max = RTE_MIN(trd->s.max, trs->s.max);
1038 trd->s.min = RTE_MAX(trd->s.min, trs->s.min);
1044 eval_jgt_jle(struct bpf_reg_val *trd, struct bpf_reg_val *trs,
1045 struct bpf_reg_val *frd, struct bpf_reg_val *frs)
1047 frd->u.max = RTE_MIN(frd->u.max, frs->u.min);
1048 trd->u.min = RTE_MAX(trd->u.min, trs->u.min + 1);
1052 eval_jlt_jge(struct bpf_reg_val *trd, struct bpf_reg_val *trs,
1053 struct bpf_reg_val *frd, struct bpf_reg_val *frs)
1055 frd->u.min = RTE_MAX(frd->u.min, frs->u.min);
1056 trd->u.max = RTE_MIN(trd->u.max, trs->u.max - 1);
1060 eval_jsgt_jsle(struct bpf_reg_val *trd, struct bpf_reg_val *trs,
1061 struct bpf_reg_val *frd, struct bpf_reg_val *frs)
1063 frd->s.max = RTE_MIN(frd->s.max, frs->s.min);
1064 trd->s.min = RTE_MAX(trd->s.min, trs->s.min + 1);
1068 eval_jslt_jsge(struct bpf_reg_val *trd, struct bpf_reg_val *trs,
1069 struct bpf_reg_val *frd, struct bpf_reg_val *frs)
1071 frd->s.min = RTE_MAX(frd->s.min, frs->s.min);
1072 trd->s.max = RTE_MIN(trd->s.max, trs->s.max - 1);
1076 eval_jcc(struct bpf_verifier *bvf, const struct ebpf_insn *ins)
1080 struct bpf_eval_state *fst, *tst;
1081 struct bpf_reg_val *frd, *frs, *trd, *trs;
1082 struct bpf_reg_val rvf, rvt;
1085 fst = bvf->evin->evst;
1087 frd = fst->rv + ins->dst_reg;
1088 trd = tst->rv + ins->dst_reg;
1090 if (BPF_SRC(ins->code) == BPF_X) {
1091 frs = fst->rv + ins->src_reg;
1092 trs = tst->rv + ins->src_reg;
1096 eval_fill_imm(frs, UINT64_MAX, ins->imm);
1097 eval_fill_imm(trs, UINT64_MAX, ins->imm);
1100 err = eval_defined(trd, trs);
1104 op = BPF_OP(ins->code);
1107 eval_jeq_jne(trd, trs);
1108 else if (op == EBPF_JNE)
1109 eval_jeq_jne(frd, frs);
1110 else if (op == BPF_JGT)
1111 eval_jgt_jle(trd, trs, frd, frs);
1112 else if (op == EBPF_JLE)
1113 eval_jgt_jle(frd, frs, trd, trs);
1114 else if (op == EBPF_JLT)
1115 eval_jlt_jge(trd, trs, frd, frs);
1116 else if (op == BPF_JGE)
1117 eval_jlt_jge(frd, frs, trd, trs);
1118 else if (op == EBPF_JSGT)
1119 eval_jsgt_jsle(trd, trs, frd, frs);
1120 else if (op == EBPF_JSLE)
1121 eval_jsgt_jsle(frd, frs, trd, trs);
1122 else if (op == EBPF_JSLT)
1123 eval_jslt_jsge(trd, trs, frd, frs);
1124 else if (op == EBPF_JSGE)
1125 eval_jslt_jsge(frd, frs, trd, trs);
1131 * validate parameters for each instruction type.
1133 static const struct bpf_ins_check ins_chk[UINT8_MAX + 1] = {
1134 /* ALU IMM 32-bit instructions */
1135 [(BPF_ALU | BPF_ADD | BPF_K)] = {
1136 .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
1137 .off = { .min = 0, .max = 0},
1138 .imm = { .min = 0, .max = UINT32_MAX,},
1141 [(BPF_ALU | BPF_SUB | BPF_K)] = {
1142 .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
1143 .off = { .min = 0, .max = 0},
1144 .imm = { .min = 0, .max = UINT32_MAX,},
1147 [(BPF_ALU | BPF_AND | BPF_K)] = {
1148 .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
1149 .off = { .min = 0, .max = 0},
1150 .imm = { .min = 0, .max = UINT32_MAX,},
1153 [(BPF_ALU | BPF_OR | BPF_K)] = {
1154 .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
1155 .off = { .min = 0, .max = 0},
1156 .imm = { .min = 0, .max = UINT32_MAX,},
1159 [(BPF_ALU | BPF_LSH | BPF_K)] = {
1160 .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
1161 .off = { .min = 0, .max = 0},
1162 .imm = { .min = 0, .max = UINT32_MAX,},
1165 [(BPF_ALU | BPF_RSH | BPF_K)] = {
1166 .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
1167 .off = { .min = 0, .max = 0},
1168 .imm = { .min = 0, .max = UINT32_MAX,},
1171 [(BPF_ALU | BPF_XOR | BPF_K)] = {
1172 .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
1173 .off = { .min = 0, .max = 0},
1174 .imm = { .min = 0, .max = UINT32_MAX,},
1177 [(BPF_ALU | BPF_MUL | BPF_K)] = {
1178 .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
1179 .off = { .min = 0, .max = 0},
1180 .imm = { .min = 0, .max = UINT32_MAX,},
1183 [(BPF_ALU | EBPF_MOV | BPF_K)] = {
1184 .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
1185 .off = { .min = 0, .max = 0},
1186 .imm = { .min = 0, .max = UINT32_MAX,},
1189 [(BPF_ALU | BPF_DIV | BPF_K)] = {
1190 .mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
1191 .off = { .min = 0, .max = 0},
1192 .imm = { .min = 1, .max = UINT32_MAX},
1195 [(BPF_ALU | BPF_MOD | BPF_K)] = {
1196 .mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
1197 .off = { .min = 0, .max = 0},
1198 .imm = { .min = 1, .max = UINT32_MAX},
1201 /* ALU IMM 64-bit instructions */
1202 [(EBPF_ALU64 | BPF_ADD | BPF_K)] = {
1203 .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
1204 .off = { .min = 0, .max = 0},
1205 .imm = { .min = 0, .max = UINT32_MAX,},
1208 [(EBPF_ALU64 | BPF_SUB | BPF_K)] = {
1209 .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
1210 .off = { .min = 0, .max = 0},
1211 .imm = { .min = 0, .max = UINT32_MAX,},
1214 [(EBPF_ALU64 | BPF_AND | BPF_K)] = {
1215 .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
1216 .off = { .min = 0, .max = 0},
1217 .imm = { .min = 0, .max = UINT32_MAX,},
1220 [(EBPF_ALU64 | BPF_OR | BPF_K)] = {
1221 .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
1222 .off = { .min = 0, .max = 0},
1223 .imm = { .min = 0, .max = UINT32_MAX,},
1226 [(EBPF_ALU64 | BPF_LSH | BPF_K)] = {
1227 .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
1228 .off = { .min = 0, .max = 0},
1229 .imm = { .min = 0, .max = UINT32_MAX,},
1232 [(EBPF_ALU64 | BPF_RSH | BPF_K)] = {
1233 .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
1234 .off = { .min = 0, .max = 0},
1235 .imm = { .min = 0, .max = UINT32_MAX,},
1238 [(EBPF_ALU64 | EBPF_ARSH | BPF_K)] = {
1239 .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
1240 .off = { .min = 0, .max = 0},
1241 .imm = { .min = 0, .max = UINT32_MAX,},
1244 [(EBPF_ALU64 | BPF_XOR | BPF_K)] = {
1245 .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
1246 .off = { .min = 0, .max = 0},
1247 .imm = { .min = 0, .max = UINT32_MAX,},
1250 [(EBPF_ALU64 | BPF_MUL | BPF_K)] = {
1251 .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
1252 .off = { .min = 0, .max = 0},
1253 .imm = { .min = 0, .max = UINT32_MAX,},
1256 [(EBPF_ALU64 | EBPF_MOV | BPF_K)] = {
1257 .mask = {.dreg = WRT_REGS, .sreg = ZERO_REG},
1258 .off = { .min = 0, .max = 0},
1259 .imm = { .min = 0, .max = UINT32_MAX,},
1262 [(EBPF_ALU64 | BPF_DIV | BPF_K)] = {
1263 .mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
1264 .off = { .min = 0, .max = 0},
1265 .imm = { .min = 1, .max = UINT32_MAX},
1268 [(EBPF_ALU64 | BPF_MOD | BPF_K)] = {
1269 .mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
1270 .off = { .min = 0, .max = 0},
1271 .imm = { .min = 1, .max = UINT32_MAX},
1274 /* ALU REG 32-bit instructions */
1275 [(BPF_ALU | BPF_ADD | BPF_X)] = {
1276 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1277 .off = { .min = 0, .max = 0},
1278 .imm = { .min = 0, .max = 0},
1281 [(BPF_ALU | BPF_SUB | BPF_X)] = {
1282 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1283 .off = { .min = 0, .max = 0},
1284 .imm = { .min = 0, .max = 0},
1287 [(BPF_ALU | BPF_AND | BPF_X)] = {
1288 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1289 .off = { .min = 0, .max = 0},
1290 .imm = { .min = 0, .max = 0},
1293 [(BPF_ALU | BPF_OR | BPF_X)] = {
1294 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1295 .off = { .min = 0, .max = 0},
1296 .imm = { .min = 0, .max = 0},
1299 [(BPF_ALU | BPF_LSH | BPF_X)] = {
1300 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1301 .off = { .min = 0, .max = 0},
1302 .imm = { .min = 0, .max = 0},
1305 [(BPF_ALU | BPF_RSH | BPF_X)] = {
1306 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1307 .off = { .min = 0, .max = 0},
1308 .imm = { .min = 0, .max = 0},
1311 [(BPF_ALU | BPF_XOR | BPF_X)] = {
1312 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1313 .off = { .min = 0, .max = 0},
1314 .imm = { .min = 0, .max = 0},
1317 [(BPF_ALU | BPF_MUL | BPF_X)] = {
1318 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1319 .off = { .min = 0, .max = 0},
1320 .imm = { .min = 0, .max = 0},
1323 [(BPF_ALU | BPF_DIV | BPF_X)] = {
1324 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1325 .off = { .min = 0, .max = 0},
1326 .imm = { .min = 0, .max = 0},
1329 [(BPF_ALU | BPF_MOD | BPF_X)] = {
1330 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1331 .off = { .min = 0, .max = 0},
1332 .imm = { .min = 0, .max = 0},
1335 [(BPF_ALU | EBPF_MOV | BPF_X)] = {
1336 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1337 .off = { .min = 0, .max = 0},
1338 .imm = { .min = 0, .max = 0},
1341 [(BPF_ALU | BPF_NEG)] = {
1342 .mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
1343 .off = { .min = 0, .max = 0},
1344 .imm = { .min = 0, .max = 0},
1347 [(BPF_ALU | EBPF_END | EBPF_TO_BE)] = {
1348 .mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
1349 .off = { .min = 0, .max = 0},
1350 .imm = { .min = 16, .max = 64},
1351 .check = check_alu_bele,
1354 [(BPF_ALU | EBPF_END | EBPF_TO_LE)] = {
1355 .mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
1356 .off = { .min = 0, .max = 0},
1357 .imm = { .min = 16, .max = 64},
1358 .check = check_alu_bele,
1361 /* ALU REG 64-bit instructions */
1362 [(EBPF_ALU64 | BPF_ADD | BPF_X)] = {
1363 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1364 .off = { .min = 0, .max = 0},
1365 .imm = { .min = 0, .max = 0},
1368 [(EBPF_ALU64 | BPF_SUB | BPF_X)] = {
1369 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1370 .off = { .min = 0, .max = 0},
1371 .imm = { .min = 0, .max = 0},
1374 [(EBPF_ALU64 | BPF_AND | BPF_X)] = {
1375 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1376 .off = { .min = 0, .max = 0},
1377 .imm = { .min = 0, .max = 0},
1380 [(EBPF_ALU64 | BPF_OR | BPF_X)] = {
1381 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1382 .off = { .min = 0, .max = 0},
1383 .imm = { .min = 0, .max = 0},
1386 [(EBPF_ALU64 | BPF_LSH | BPF_X)] = {
1387 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1388 .off = { .min = 0, .max = 0},
1389 .imm = { .min = 0, .max = 0},
1392 [(EBPF_ALU64 | BPF_RSH | BPF_X)] = {
1393 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1394 .off = { .min = 0, .max = 0},
1395 .imm = { .min = 0, .max = 0},
1398 [(EBPF_ALU64 | EBPF_ARSH | BPF_X)] = {
1399 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1400 .off = { .min = 0, .max = 0},
1401 .imm = { .min = 0, .max = 0},
1404 [(EBPF_ALU64 | BPF_XOR | BPF_X)] = {
1405 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1406 .off = { .min = 0, .max = 0},
1407 .imm = { .min = 0, .max = 0},
1410 [(EBPF_ALU64 | BPF_MUL | BPF_X)] = {
1411 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1412 .off = { .min = 0, .max = 0},
1413 .imm = { .min = 0, .max = 0},
1416 [(EBPF_ALU64 | BPF_DIV | BPF_X)] = {
1417 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1418 .off = { .min = 0, .max = 0},
1419 .imm = { .min = 0, .max = 0},
1422 [(EBPF_ALU64 | BPF_MOD | BPF_X)] = {
1423 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1424 .off = { .min = 0, .max = 0},
1425 .imm = { .min = 0, .max = 0},
1428 [(EBPF_ALU64 | EBPF_MOV | BPF_X)] = {
1429 .mask = { .dreg = WRT_REGS, .sreg = ALL_REGS},
1430 .off = { .min = 0, .max = 0},
1431 .imm = { .min = 0, .max = 0},
1434 [(EBPF_ALU64 | BPF_NEG)] = {
1435 .mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
1436 .off = { .min = 0, .max = 0},
1437 .imm = { .min = 0, .max = 0},
1440 /* load instructions */
1441 [(BPF_LDX | BPF_MEM | BPF_B)] = {
1442 .mask = {. dreg = WRT_REGS, .sreg = ALL_REGS},
1443 .off = { .min = 0, .max = UINT16_MAX},
1444 .imm = { .min = 0, .max = 0},
1447 [(BPF_LDX | BPF_MEM | BPF_H)] = {
1448 .mask = {. dreg = WRT_REGS, .sreg = ALL_REGS},
1449 .off = { .min = 0, .max = UINT16_MAX},
1450 .imm = { .min = 0, .max = 0},
1453 [(BPF_LDX | BPF_MEM | BPF_W)] = {
1454 .mask = {. dreg = WRT_REGS, .sreg = ALL_REGS},
1455 .off = { .min = 0, .max = UINT16_MAX},
1456 .imm = { .min = 0, .max = 0},
1459 [(BPF_LDX | BPF_MEM | EBPF_DW)] = {
1460 .mask = {. dreg = WRT_REGS, .sreg = ALL_REGS},
1461 .off = { .min = 0, .max = UINT16_MAX},
1462 .imm = { .min = 0, .max = 0},
1465 /* load 64 bit immediate value */
1466 [(BPF_LD | BPF_IMM | EBPF_DW)] = {
1467 .mask = { .dreg = WRT_REGS, .sreg = ZERO_REG},
1468 .off = { .min = 0, .max = 0},
1469 .imm = { .min = 0, .max = UINT32_MAX},
1470 .eval = eval_ld_imm64,
1472 /* load absolute instructions */
1473 [(BPF_LD | BPF_ABS | BPF_B)] = {
1474 .mask = {. dreg = ZERO_REG, .sreg = ZERO_REG},
1475 .off = { .min = 0, .max = 0},
1476 .imm = { .min = 0, .max = INT32_MAX},
1477 .eval = eval_ld_mbuf,
1479 [(BPF_LD | BPF_ABS | BPF_H)] = {
1480 .mask = {. dreg = ZERO_REG, .sreg = ZERO_REG},
1481 .off = { .min = 0, .max = 0},
1482 .imm = { .min = 0, .max = INT32_MAX},
1483 .eval = eval_ld_mbuf,
1485 [(BPF_LD | BPF_ABS | BPF_W)] = {
1486 .mask = {. dreg = ZERO_REG, .sreg = ZERO_REG},
1487 .off = { .min = 0, .max = 0},
1488 .imm = { .min = 0, .max = INT32_MAX},
1489 .eval = eval_ld_mbuf,
1491 /* load indirect instructions */
1492 [(BPF_LD | BPF_IND | BPF_B)] = {
1493 .mask = {. dreg = ZERO_REG, .sreg = IND_SRC_REGS},
1494 .off = { .min = 0, .max = 0},
1495 .imm = { .min = 0, .max = UINT32_MAX},
1496 .eval = eval_ld_mbuf,
1498 [(BPF_LD | BPF_IND | BPF_H)] = {
1499 .mask = {. dreg = ZERO_REG, .sreg = IND_SRC_REGS},
1500 .off = { .min = 0, .max = 0},
1501 .imm = { .min = 0, .max = UINT32_MAX},
1502 .eval = eval_ld_mbuf,
1504 [(BPF_LD | BPF_IND | BPF_W)] = {
1505 .mask = {. dreg = ZERO_REG, .sreg = IND_SRC_REGS},
1506 .off = { .min = 0, .max = 0},
1507 .imm = { .min = 0, .max = UINT32_MAX},
1508 .eval = eval_ld_mbuf,
1510 /* store REG instructions */
1511 [(BPF_STX | BPF_MEM | BPF_B)] = {
1512 .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
1513 .off = { .min = 0, .max = UINT16_MAX},
1514 .imm = { .min = 0, .max = 0},
1517 [(BPF_STX | BPF_MEM | BPF_H)] = {
1518 .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
1519 .off = { .min = 0, .max = UINT16_MAX},
1520 .imm = { .min = 0, .max = 0},
1523 [(BPF_STX | BPF_MEM | BPF_W)] = {
1524 .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
1525 .off = { .min = 0, .max = UINT16_MAX},
1526 .imm = { .min = 0, .max = 0},
1529 [(BPF_STX | BPF_MEM | EBPF_DW)] = {
1530 .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
1531 .off = { .min = 0, .max = UINT16_MAX},
1532 .imm = { .min = 0, .max = 0},
1535 /* atomic add instructions */
1536 [(BPF_STX | EBPF_XADD | BPF_W)] = {
1537 .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
1538 .off = { .min = 0, .max = UINT16_MAX},
1539 .imm = { .min = 0, .max = 0},
1542 [(BPF_STX | EBPF_XADD | EBPF_DW)] = {
1543 .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
1544 .off = { .min = 0, .max = UINT16_MAX},
1545 .imm = { .min = 0, .max = 0},
1548 /* store IMM instructions */
1549 [(BPF_ST | BPF_MEM | BPF_B)] = {
1550 .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
1551 .off = { .min = 0, .max = UINT16_MAX},
1552 .imm = { .min = 0, .max = UINT32_MAX},
1555 [(BPF_ST | BPF_MEM | BPF_H)] = {
1556 .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
1557 .off = { .min = 0, .max = UINT16_MAX},
1558 .imm = { .min = 0, .max = UINT32_MAX},
1561 [(BPF_ST | BPF_MEM | BPF_W)] = {
1562 .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
1563 .off = { .min = 0, .max = UINT16_MAX},
1564 .imm = { .min = 0, .max = UINT32_MAX},
1567 [(BPF_ST | BPF_MEM | EBPF_DW)] = {
1568 .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
1569 .off = { .min = 0, .max = UINT16_MAX},
1570 .imm = { .min = 0, .max = UINT32_MAX},
1573 /* jump instruction */
1574 [(BPF_JMP | BPF_JA)] = {
1575 .mask = { .dreg = ZERO_REG, .sreg = ZERO_REG},
1576 .off = { .min = 0, .max = UINT16_MAX},
1577 .imm = { .min = 0, .max = 0},
1579 /* jcc IMM instructions */
1580 [(BPF_JMP | BPF_JEQ | BPF_K)] = {
1581 .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
1582 .off = { .min = 0, .max = UINT16_MAX},
1583 .imm = { .min = 0, .max = UINT32_MAX},
1586 [(BPF_JMP | EBPF_JNE | BPF_K)] = {
1587 .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
1588 .off = { .min = 0, .max = UINT16_MAX},
1589 .imm = { .min = 0, .max = UINT32_MAX},
1592 [(BPF_JMP | BPF_JGT | BPF_K)] = {
1593 .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
1594 .off = { .min = 0, .max = UINT16_MAX},
1595 .imm = { .min = 0, .max = UINT32_MAX},
1598 [(BPF_JMP | EBPF_JLT | BPF_K)] = {
1599 .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
1600 .off = { .min = 0, .max = UINT16_MAX},
1601 .imm = { .min = 0, .max = UINT32_MAX},
1604 [(BPF_JMP | BPF_JGE | BPF_K)] = {
1605 .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
1606 .off = { .min = 0, .max = UINT16_MAX},
1607 .imm = { .min = 0, .max = UINT32_MAX},
1610 [(BPF_JMP | EBPF_JLE | BPF_K)] = {
1611 .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
1612 .off = { .min = 0, .max = UINT16_MAX},
1613 .imm = { .min = 0, .max = UINT32_MAX},
1616 [(BPF_JMP | EBPF_JSGT | BPF_K)] = {
1617 .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
1618 .off = { .min = 0, .max = UINT16_MAX},
1619 .imm = { .min = 0, .max = UINT32_MAX},
1622 [(BPF_JMP | EBPF_JSLT | BPF_K)] = {
1623 .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
1624 .off = { .min = 0, .max = UINT16_MAX},
1625 .imm = { .min = 0, .max = UINT32_MAX},
1628 [(BPF_JMP | EBPF_JSGE | BPF_K)] = {
1629 .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
1630 .off = { .min = 0, .max = UINT16_MAX},
1631 .imm = { .min = 0, .max = UINT32_MAX},
1634 [(BPF_JMP | EBPF_JSLE | BPF_K)] = {
1635 .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
1636 .off = { .min = 0, .max = UINT16_MAX},
1637 .imm = { .min = 0, .max = UINT32_MAX},
1640 [(BPF_JMP | BPF_JSET | BPF_K)] = {
1641 .mask = { .dreg = ALL_REGS, .sreg = ZERO_REG},
1642 .off = { .min = 0, .max = UINT16_MAX},
1643 .imm = { .min = 0, .max = UINT32_MAX},
1646 /* jcc REG instructions */
1647 [(BPF_JMP | BPF_JEQ | BPF_X)] = {
1648 .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
1649 .off = { .min = 0, .max = UINT16_MAX},
1650 .imm = { .min = 0, .max = 0},
1653 [(BPF_JMP | EBPF_JNE | BPF_X)] = {
1654 .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
1655 .off = { .min = 0, .max = UINT16_MAX},
1656 .imm = { .min = 0, .max = 0},
1659 [(BPF_JMP | BPF_JGT | BPF_X)] = {
1660 .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
1661 .off = { .min = 0, .max = UINT16_MAX},
1662 .imm = { .min = 0, .max = 0},
1665 [(BPF_JMP | EBPF_JLT | BPF_X)] = {
1666 .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
1667 .off = { .min = 0, .max = UINT16_MAX},
1668 .imm = { .min = 0, .max = 0},
1671 [(BPF_JMP | BPF_JGE | BPF_X)] = {
1672 .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
1673 .off = { .min = 0, .max = UINT16_MAX},
1674 .imm = { .min = 0, .max = 0},
1677 [(BPF_JMP | EBPF_JLE | BPF_X)] = {
1678 .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
1679 .off = { .min = 0, .max = UINT16_MAX},
1680 .imm = { .min = 0, .max = 0},
1683 [(BPF_JMP | EBPF_JSGT | BPF_X)] = {
1684 .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
1685 .off = { .min = 0, .max = UINT16_MAX},
1686 .imm = { .min = 0, .max = 0},
1689 [(BPF_JMP | EBPF_JSLT | BPF_X)] = {
1690 .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
1691 .off = { .min = 0, .max = UINT16_MAX},
1692 .imm = { .min = 0, .max = 0},
1694 [(BPF_JMP | EBPF_JSGE | BPF_X)] = {
1695 .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
1696 .off = { .min = 0, .max = UINT16_MAX},
1697 .imm = { .min = 0, .max = 0},
1700 [(BPF_JMP | EBPF_JSLE | BPF_X)] = {
1701 .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
1702 .off = { .min = 0, .max = UINT16_MAX},
1703 .imm = { .min = 0, .max = 0},
1706 [(BPF_JMP | BPF_JSET | BPF_X)] = {
1707 .mask = { .dreg = ALL_REGS, .sreg = ALL_REGS},
1708 .off = { .min = 0, .max = UINT16_MAX},
1709 .imm = { .min = 0, .max = 0},
1712 /* call instruction */
1713 [(BPF_JMP | EBPF_CALL)] = {
1714 .mask = { .dreg = ZERO_REG, .sreg = ZERO_REG},
1715 .off = { .min = 0, .max = 0},
1716 .imm = { .min = 0, .max = UINT32_MAX},
1719 /* ret instruction */
1720 [(BPF_JMP | EBPF_EXIT)] = {
1721 .mask = { .dreg = ZERO_REG, .sreg = ZERO_REG},
1722 .off = { .min = 0, .max = 0},
1723 .imm = { .min = 0, .max = 0},
1729 * make sure that instruction syntax is valid,
1730 * and its fields don't violate particular instruction type restrictions.
1733 check_syntax(const struct ebpf_insn *ins)
1742 if (ins_chk[op].mask.dreg == 0)
1743 return "invalid opcode";
1745 if ((ins_chk[op].mask.dreg & 1 << ins->dst_reg) == 0)
1746 return "invalid dst-reg field";
1748 if ((ins_chk[op].mask.sreg & 1 << ins->src_reg) == 0)
1749 return "invalid src-reg field";
1752 if (ins_chk[op].off.min > off || ins_chk[op].off.max < off)
1753 return "invalid off field";
1756 if (ins_chk[op].imm.min > imm || ins_chk[op].imm.max < imm)
1757 return "invalid imm field";
1759 if (ins_chk[op].check != NULL)
1760 return ins_chk[op].check(ins);
1766 * helper function, return instruction index for the given node.
1769 get_node_idx(const struct bpf_verifier *bvf, const struct inst_node *node)
1771 return node - bvf->in;
1775 * helper function, used to walk through constructed CFG.
1777 static struct inst_node *
1778 get_next_node(struct bpf_verifier *bvf, struct inst_node *node)
1780 uint32_t ce, ne, dst;
1783 ce = node->cur_edge;
1788 dst = node->edge_dest[ce];
1789 return bvf->in + dst;
1793 set_node_colour(struct bpf_verifier *bvf, struct inst_node *node,
1798 prev = node->colour;
1801 bvf->node_colour[prev]--;
1802 bvf->node_colour[new]++;
1806 * helper function, add new edge between two nodes.
1809 add_edge(struct bpf_verifier *bvf, struct inst_node *node, uint32_t nidx)
1813 if (nidx > bvf->prm->nb_ins) {
1814 RTE_BPF_LOG(ERR, "%s: program boundary violation at pc: %u, "
1816 __func__, get_node_idx(bvf, node), nidx);
1821 if (ne >= RTE_DIM(node->edge_dest)) {
1822 RTE_BPF_LOG(ERR, "%s: internal error at pc: %u\n",
1823 __func__, get_node_idx(bvf, node));
1827 node->edge_dest[ne] = nidx;
1828 node->nb_edge = ne + 1;
1833 * helper function, determine type of edge between two nodes.
1836 set_edge_type(struct bpf_verifier *bvf, struct inst_node *node,
1837 const struct inst_node *next)
1839 uint32_t ce, clr, type;
1841 ce = node->cur_edge - 1;
1844 type = UNKNOWN_EDGE;
1848 else if (clr == GREY)
1850 else if (clr == BLACK)
1852 * in fact it could be either direct or cross edge,
1853 * but for now, we don't need to distinguish between them.
1857 node->edge_type[ce] = type;
1858 bvf->edge_type[type]++;
1861 static struct inst_node *
1862 get_prev_node(struct bpf_verifier *bvf, struct inst_node *node)
1864 return bvf->in + node->prev_node;
1868 * Depth-First Search (DFS) through previously constructed
1869 * Control Flow Graph (CFG).
1870 * Information collected at this path would be used later
1871 * to determine is there any loops, and/or unreachable instructions.
1874 dfs(struct bpf_verifier *bvf)
1876 struct inst_node *next, *node;
1879 while (node != NULL) {
1881 if (node->colour == WHITE)
1882 set_node_colour(bvf, node, GREY);
1884 if (node->colour == GREY) {
1886 /* find next unprocessed child node */
1888 next = get_next_node(bvf, node);
1891 set_edge_type(bvf, node, next);
1892 } while (next->colour != WHITE);
1895 /* proceed with next child */
1896 next->prev_node = get_node_idx(bvf, node);
1900 * finished with current node and all it's kids,
1901 * proceed with parent
1903 set_node_colour(bvf, node, BLACK);
1905 node = get_prev_node(bvf, node);
1913 * report unreachable instructions.
1916 log_unreachable(const struct bpf_verifier *bvf)
1919 struct inst_node *node;
1920 const struct ebpf_insn *ins;
1922 for (i = 0; i != bvf->prm->nb_ins; i++) {
1925 ins = bvf->prm->ins + i;
1927 if (node->colour == WHITE &&
1928 ins->code != (BPF_LD | BPF_IMM | EBPF_DW))
1929 RTE_BPF_LOG(ERR, "unreachable code at pc: %u;\n", i);
1934 * report loops detected.
1937 log_loop(const struct bpf_verifier *bvf)
1940 struct inst_node *node;
1942 for (i = 0; i != bvf->prm->nb_ins; i++) {
1945 if (node->colour != BLACK)
1948 for (j = 0; j != node->nb_edge; j++) {
1949 if (node->edge_type[j] == BACK_EDGE)
1951 "loop at pc:%u --> pc:%u;\n",
1952 i, node->edge_dest[j]);
1958 * First pass goes though all instructions in the set, checks that each
1959 * instruction is a valid one (correct syntax, valid field values, etc.)
1960 * and constructs control flow graph (CFG).
1961 * Then depth-first search is performed over the constructed graph.
1962 * Programs with unreachable instructions and/or loops will be rejected.
1965 validate(struct bpf_verifier *bvf)
1969 struct inst_node *node;
1970 const struct ebpf_insn *ins;
1974 for (i = 0; i < bvf->prm->nb_ins; i++) {
1976 ins = bvf->prm->ins + i;
1979 err = check_syntax(ins);
1981 RTE_BPF_LOG(ERR, "%s: %s at pc: %u\n",
1987 * construct CFG, jcc nodes have to outgoing edges,
1988 * 'exit' nodes - none, all other nodes have exactly one
1991 switch (ins->code) {
1992 case (BPF_JMP | EBPF_EXIT):
1994 case (BPF_JMP | BPF_JEQ | BPF_K):
1995 case (BPF_JMP | EBPF_JNE | BPF_K):
1996 case (BPF_JMP | BPF_JGT | BPF_K):
1997 case (BPF_JMP | EBPF_JLT | BPF_K):
1998 case (BPF_JMP | BPF_JGE | BPF_K):
1999 case (BPF_JMP | EBPF_JLE | BPF_K):
2000 case (BPF_JMP | EBPF_JSGT | BPF_K):
2001 case (BPF_JMP | EBPF_JSLT | BPF_K):
2002 case (BPF_JMP | EBPF_JSGE | BPF_K):
2003 case (BPF_JMP | EBPF_JSLE | BPF_K):
2004 case (BPF_JMP | BPF_JSET | BPF_K):
2005 case (BPF_JMP | BPF_JEQ | BPF_X):
2006 case (BPF_JMP | EBPF_JNE | BPF_X):
2007 case (BPF_JMP | BPF_JGT | BPF_X):
2008 case (BPF_JMP | EBPF_JLT | BPF_X):
2009 case (BPF_JMP | BPF_JGE | BPF_X):
2010 case (BPF_JMP | EBPF_JLE | BPF_X):
2011 case (BPF_JMP | EBPF_JSGT | BPF_X):
2012 case (BPF_JMP | EBPF_JSLT | BPF_X):
2013 case (BPF_JMP | EBPF_JSGE | BPF_X):
2014 case (BPF_JMP | EBPF_JSLE | BPF_X):
2015 case (BPF_JMP | BPF_JSET | BPF_X):
2016 rc |= add_edge(bvf, node, i + ins->off + 1);
2017 rc |= add_edge(bvf, node, i + 1);
2018 bvf->nb_jcc_nodes++;
2020 case (BPF_JMP | BPF_JA):
2021 rc |= add_edge(bvf, node, i + ins->off + 1);
2023 /* load 64 bit immediate value */
2024 case (BPF_LD | BPF_IMM | EBPF_DW):
2025 rc |= add_edge(bvf, node, i + 2);
2028 case (BPF_LD | BPF_ABS | BPF_B):
2029 case (BPF_LD | BPF_ABS | BPF_H):
2030 case (BPF_LD | BPF_ABS | BPF_W):
2031 case (BPF_LD | BPF_IND | BPF_B):
2032 case (BPF_LD | BPF_IND | BPF_H):
2033 case (BPF_LD | BPF_IND | BPF_W):
2034 bvf->nb_ldmb_nodes++;
2037 rc |= add_edge(bvf, node, i + 1);
2042 bvf->node_colour[WHITE]++;
2050 RTE_BPF_LOG(DEBUG, "%s(%p) stats:\n"
2052 "nb_jcc_nodes=%u;\n"
2053 "node_color={[WHITE]=%u, [GREY]=%u,, [BLACK]=%u};\n"
2054 "edge_type={[UNKNOWN]=%u, [TREE]=%u, [BACK]=%u, [CROSS]=%u};\n",
2058 bvf->node_colour[WHITE], bvf->node_colour[GREY],
2059 bvf->node_colour[BLACK],
2060 bvf->edge_type[UNKNOWN_EDGE], bvf->edge_type[TREE_EDGE],
2061 bvf->edge_type[BACK_EDGE], bvf->edge_type[CROSS_EDGE]);
2063 if (bvf->node_colour[BLACK] != bvf->nb_nodes) {
2064 RTE_BPF_LOG(ERR, "%s(%p) unreachable instructions;\n",
2066 log_unreachable(bvf);
2070 if (bvf->node_colour[GREY] != 0 || bvf->node_colour[WHITE] != 0 ||
2071 bvf->edge_type[UNKNOWN_EDGE] != 0) {
2072 RTE_BPF_LOG(ERR, "%s(%p) DFS internal error;\n",
2077 if (bvf->edge_type[BACK_EDGE] != 0) {
2078 RTE_BPF_LOG(ERR, "%s(%p) loops detected;\n",
2088 * helper functions get/free eval states.
2090 static struct bpf_eval_state *
2091 pull_eval_state(struct bpf_verifier *bvf)
2095 n = bvf->evst_pool.cur;
2096 if (n == bvf->evst_pool.num)
2099 bvf->evst_pool.cur = n + 1;
2100 return bvf->evst_pool.ent + n;
2104 push_eval_state(struct bpf_verifier *bvf)
2106 bvf->evst_pool.cur--;
2110 evst_pool_fini(struct bpf_verifier *bvf)
2113 free(bvf->evst_pool.ent);
2114 memset(&bvf->evst_pool, 0, sizeof(bvf->evst_pool));
2118 evst_pool_init(struct bpf_verifier *bvf)
2122 n = bvf->nb_jcc_nodes + 1;
2124 bvf->evst_pool.ent = calloc(n, sizeof(bvf->evst_pool.ent[0]));
2125 if (bvf->evst_pool.ent == NULL)
2128 bvf->evst_pool.num = n;
2129 bvf->evst_pool.cur = 0;
2131 bvf->evst = pull_eval_state(bvf);
2136 * Save current eval state.
2139 save_eval_state(struct bpf_verifier *bvf, struct inst_node *node)
2141 struct bpf_eval_state *st;
2143 /* get new eval_state for this node */
2144 st = pull_eval_state(bvf);
2147 "%s: internal error (out of space) at pc: %u\n",
2148 __func__, get_node_idx(bvf, node));
2152 /* make a copy of current state */
2153 memcpy(st, bvf->evst, sizeof(*st));
2155 /* swap current state with new one */
2156 node->evst = bvf->evst;
2159 RTE_BPF_LOG(DEBUG, "%s(bvf=%p,node=%u) old/new states: %p/%p;\n",
2160 __func__, bvf, get_node_idx(bvf, node), node->evst, bvf->evst);
2166 * Restore previous eval state and mark current eval state as free.
2169 restore_eval_state(struct bpf_verifier *bvf, struct inst_node *node)
2171 RTE_BPF_LOG(DEBUG, "%s(bvf=%p,node=%u) old/new states: %p/%p;\n",
2172 __func__, bvf, get_node_idx(bvf, node), bvf->evst, node->evst);
2174 bvf->evst = node->evst;
2176 push_eval_state(bvf);
2180 log_eval_state(const struct bpf_verifier *bvf, const struct ebpf_insn *ins,
2181 uint32_t pc, int32_t loglvl)
2183 const struct bpf_eval_state *st;
2184 const struct bpf_reg_val *rv;
2186 rte_log(loglvl, rte_bpf_logtype, "%s(pc=%u):\n", __func__, pc);
2189 rv = st->rv + ins->dst_reg;
2191 rte_log(loglvl, rte_bpf_logtype,
2193 "\tv={type=%u, size=%zu},\n"
2194 "\tmask=0x%" PRIx64 ",\n"
2195 "\tu={min=0x%" PRIx64 ", max=0x%" PRIx64 "},\n"
2196 "\ts={min=%" PRId64 ", max=%" PRId64 "},\n"
2199 rv->v.type, rv->v.size,
2201 rv->u.min, rv->u.max,
2202 rv->s.min, rv->s.max);
2206 * Do second pass through CFG and try to evaluate instructions
2207 * via each possible path.
2208 * Right now evaluation functionality is quite limited.
2209 * Still need to add extra checks for:
2210 * - use/return uninitialized registers.
2211 * - use uninitialized data from the stack.
2212 * - memory boundaries violation.
2215 evaluate(struct bpf_verifier *bvf)
2220 const struct ebpf_insn *ins;
2221 struct inst_node *next, *node;
2223 /* initial state of frame pointer */
2224 static const struct bpf_reg_val rvfp = {
2226 .type = BPF_ARG_PTR_STACK,
2227 .size = MAX_BPF_STACK_SIZE,
2230 .u = {.min = MAX_BPF_STACK_SIZE, .max = MAX_BPF_STACK_SIZE},
2231 .s = {.min = MAX_BPF_STACK_SIZE, .max = MAX_BPF_STACK_SIZE},
2234 bvf->evst->rv[EBPF_REG_1].v = bvf->prm->prog_arg;
2235 bvf->evst->rv[EBPF_REG_1].mask = UINT64_MAX;
2236 if (bvf->prm->prog_arg.type == RTE_BPF_ARG_RAW)
2237 eval_max_bound(bvf->evst->rv + EBPF_REG_1, UINT64_MAX);
2239 bvf->evst->rv[EBPF_REG_10] = rvfp;
2241 ins = bvf->prm->ins;
2246 while (node != NULL && rc == 0) {
2249 * current node evaluation, make sure we evaluate
2250 * each node only once.
2255 idx = get_node_idx(bvf, node);
2258 /* for jcc node make a copy of evaluation state */
2259 if (node->nb_edge > 1)
2260 rc |= save_eval_state(bvf, node);
2262 if (ins_chk[op].eval != NULL && rc == 0) {
2263 err = ins_chk[op].eval(bvf, ins + idx);
2265 RTE_BPF_LOG(ERR, "%s: %s at pc: %u\n",
2266 __func__, err, idx);
2271 log_eval_state(bvf, ins + idx, idx, RTE_LOG_DEBUG);
2275 /* proceed through CFG */
2276 next = get_next_node(bvf, node);
2279 /* proceed with next child */
2280 if (node->cur_edge == node->nb_edge &&
2282 restore_eval_state(bvf, node);
2284 next->prev_node = get_node_idx(bvf, node);
2288 * finished with current node and all it's kids,
2289 * proceed with parent
2292 node = get_prev_node(bvf, node);
2295 if (node == bvf->in)
2304 bpf_validate(struct rte_bpf *bpf)
2307 struct bpf_verifier bvf;
2309 /* check input argument type, don't allow mbuf ptr on 32-bit */
2310 if (bpf->prm.prog_arg.type != RTE_BPF_ARG_RAW &&
2311 bpf->prm.prog_arg.type != RTE_BPF_ARG_PTR &&
2312 (sizeof(uint64_t) != sizeof(uintptr_t) ||
2313 bpf->prm.prog_arg.type != RTE_BPF_ARG_PTR_MBUF)) {
2314 RTE_BPF_LOG(ERR, "%s: unsupported argument type\n", __func__);
2318 memset(&bvf, 0, sizeof(bvf));
2319 bvf.prm = &bpf->prm;
2320 bvf.in = calloc(bpf->prm.nb_ins, sizeof(bvf.in[0]));
2324 rc = validate(&bvf);
2327 rc = evst_pool_init(&bvf);
2329 rc = evaluate(&bvf);
2330 evst_pool_fini(&bvf);
2335 /* copy collected info */
2337 bpf->stack_sz = bvf.stack_sz;
2339 /* for LD_ABS/LD_IND, we'll need extra space on the stack */
2340 if (bvf.nb_ldmb_nodes != 0)
2341 bpf->stack_sz = RTE_ALIGN_CEIL(bpf->stack_sz +
2342 sizeof(uint64_t), sizeof(uint64_t));