1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
13 #include <sys/types.h>
16 #include <sys/queue.h>
18 #include <rte_common.h>
19 #include <rte_byteorder.h>
20 #include <rte_malloc.h>
22 #include <rte_debug.h>
23 #include <rte_cycles.h>
25 #include <rte_per_lcore.h>
26 #include <rte_lcore.h>
27 #include <rte_atomic.h>
29 #include <rte_ethdev.h>
31 #include <rte_bpf_ethdev.h>
35 * information about installed BPF rx/tx callback
39 /* used by both data & control path */
40 uint32_t use; /*usage counter */
41 const struct rte_eth_rxtx_callback *cb; /* callback handle */
43 struct rte_bpf_jit jit;
44 /* used by control path only */
45 LIST_ENTRY(bpf_eth_cbi) link;
48 } __rte_cache_aligned;
51 * Odd number means that callback is used by datapath.
52 * Even number means that callback is not used by datapath.
54 #define BPF_ETH_CBI_INUSE 1
57 * List to manage RX/TX installed callbacks.
59 LIST_HEAD(bpf_eth_cbi_list, bpf_eth_cbi);
68 * information about all installed BPF rx/tx callbacks
72 struct bpf_eth_cbi_list list;
76 static struct bpf_eth_cbh rx_cbh = {
77 .lock = RTE_SPINLOCK_INITIALIZER,
78 .list = LIST_HEAD_INITIALIZER(list),
82 static struct bpf_eth_cbh tx_cbh = {
83 .lock = RTE_SPINLOCK_INITIALIZER,
84 .list = LIST_HEAD_INITIALIZER(list),
89 * Marks given callback as used by datapath.
91 static __rte_always_inline void
92 bpf_eth_cbi_inuse(struct bpf_eth_cbi *cbi)
95 /* make sure no store/load reordering could happen */
100 * Marks given callback list as not used by datapath.
102 static __rte_always_inline void
103 bpf_eth_cbi_unuse(struct bpf_eth_cbi *cbi)
105 /* make sure all previous loads are completed */
111 * Waits till datapath finished using given callback.
114 bpf_eth_cbi_wait(const struct bpf_eth_cbi *cbi)
118 /* make sure all previous loads and stores are completed */
123 /* in use, busy wait till current RX/TX iteration is finished */
124 if ((puse & BPF_ETH_CBI_INUSE) != 0) {
125 RTE_WAIT_UNTIL_MASKED((uint32_t *)(uintptr_t)&cbi->use,
126 UINT32_MAX, !=, puse, __ATOMIC_RELAXED);
131 bpf_eth_cbi_cleanup(struct bpf_eth_cbi *bc)
134 memset(&bc->jit, 0, sizeof(bc->jit));
137 static struct bpf_eth_cbi *
138 bpf_eth_cbh_find(struct bpf_eth_cbh *cbh, uint16_t port, uint16_t queue)
140 struct bpf_eth_cbi *cbi;
142 LIST_FOREACH(cbi, &cbh->list, link) {
143 if (cbi->port == port && cbi->queue == queue)
149 static struct bpf_eth_cbi *
150 bpf_eth_cbh_add(struct bpf_eth_cbh *cbh, uint16_t port, uint16_t queue)
152 struct bpf_eth_cbi *cbi;
154 /* return an existing one */
155 cbi = bpf_eth_cbh_find(cbh, port, queue);
159 cbi = rte_zmalloc(NULL, sizeof(*cbi), RTE_CACHE_LINE_SIZE);
163 LIST_INSERT_HEAD(&cbh->list, cbi, link);
169 * BPF packet processing routines.
172 static inline uint32_t
173 apply_filter(struct rte_mbuf *mb[], const uint64_t rc[], uint32_t num,
177 struct rte_mbuf *dr[num];
179 for (i = 0, j = 0, k = 0; i != num; i++) {
190 /* free filtered out mbufs */
191 for (i = 0; i != k; i++)
192 rte_pktmbuf_free(dr[i]);
194 /* copy filtered out mbufs beyond good ones */
195 for (i = 0; i != k; i++)
202 static inline uint32_t
203 pkt_filter_vm(const struct rte_bpf *bpf, struct rte_mbuf *mb[], uint32_t num,
210 for (i = 0; i != num; i++)
211 dp[i] = rte_pktmbuf_mtod(mb[i], void *);
213 rte_bpf_exec_burst(bpf, dp, rc, num);
214 return apply_filter(mb, rc, num, drop);
217 static inline uint32_t
218 pkt_filter_jit(const struct rte_bpf_jit *jit, struct rte_mbuf *mb[],
219 uint32_t num, uint32_t drop)
226 for (i = 0; i != num; i++) {
227 dp = rte_pktmbuf_mtod(mb[i], void *);
228 rc[i] = jit->func(dp);
233 num = apply_filter(mb, rc, num, drop);
238 static inline uint32_t
239 pkt_filter_mb_vm(const struct rte_bpf *bpf, struct rte_mbuf *mb[], uint32_t num,
244 rte_bpf_exec_burst(bpf, (void **)mb, rc, num);
245 return apply_filter(mb, rc, num, drop);
248 static inline uint32_t
249 pkt_filter_mb_jit(const struct rte_bpf_jit *jit, struct rte_mbuf *mb[],
250 uint32_t num, uint32_t drop)
256 for (i = 0; i != num; i++) {
257 rc[i] = jit->func(mb[i]);
262 num = apply_filter(mb, rc, num, drop);
268 * RX/TX callbacks for raw data bpf.
272 bpf_rx_callback_vm(__rte_unused uint16_t port, __rte_unused uint16_t queue,
273 struct rte_mbuf *pkt[], uint16_t nb_pkts,
274 __rte_unused uint16_t max_pkts, void *user_param)
276 struct bpf_eth_cbi *cbi;
281 bpf_eth_cbi_inuse(cbi);
282 rc = (cbi->cb != NULL) ?
283 pkt_filter_vm(cbi->bpf, pkt, nb_pkts, 1) :
285 bpf_eth_cbi_unuse(cbi);
290 bpf_rx_callback_jit(__rte_unused uint16_t port, __rte_unused uint16_t queue,
291 struct rte_mbuf *pkt[], uint16_t nb_pkts,
292 __rte_unused uint16_t max_pkts, void *user_param)
294 struct bpf_eth_cbi *cbi;
298 bpf_eth_cbi_inuse(cbi);
299 rc = (cbi->cb != NULL) ?
300 pkt_filter_jit(&cbi->jit, pkt, nb_pkts, 1) :
302 bpf_eth_cbi_unuse(cbi);
307 bpf_tx_callback_vm(__rte_unused uint16_t port, __rte_unused uint16_t queue,
308 struct rte_mbuf *pkt[], uint16_t nb_pkts, void *user_param)
310 struct bpf_eth_cbi *cbi;
314 bpf_eth_cbi_inuse(cbi);
315 rc = (cbi->cb != NULL) ?
316 pkt_filter_vm(cbi->bpf, pkt, nb_pkts, 0) :
318 bpf_eth_cbi_unuse(cbi);
323 bpf_tx_callback_jit(__rte_unused uint16_t port, __rte_unused uint16_t queue,
324 struct rte_mbuf *pkt[], uint16_t nb_pkts, void *user_param)
326 struct bpf_eth_cbi *cbi;
330 bpf_eth_cbi_inuse(cbi);
331 rc = (cbi->cb != NULL) ?
332 pkt_filter_jit(&cbi->jit, pkt, nb_pkts, 0) :
334 bpf_eth_cbi_unuse(cbi);
339 * RX/TX callbacks for mbuf.
343 bpf_rx_callback_mb_vm(__rte_unused uint16_t port, __rte_unused uint16_t queue,
344 struct rte_mbuf *pkt[], uint16_t nb_pkts,
345 __rte_unused uint16_t max_pkts, void *user_param)
347 struct bpf_eth_cbi *cbi;
351 bpf_eth_cbi_inuse(cbi);
352 rc = (cbi->cb != NULL) ?
353 pkt_filter_mb_vm(cbi->bpf, pkt, nb_pkts, 1) :
355 bpf_eth_cbi_unuse(cbi);
360 bpf_rx_callback_mb_jit(__rte_unused uint16_t port, __rte_unused uint16_t queue,
361 struct rte_mbuf *pkt[], uint16_t nb_pkts,
362 __rte_unused uint16_t max_pkts, void *user_param)
364 struct bpf_eth_cbi *cbi;
368 bpf_eth_cbi_inuse(cbi);
369 rc = (cbi->cb != NULL) ?
370 pkt_filter_mb_jit(&cbi->jit, pkt, nb_pkts, 1) :
372 bpf_eth_cbi_unuse(cbi);
377 bpf_tx_callback_mb_vm(__rte_unused uint16_t port, __rte_unused uint16_t queue,
378 struct rte_mbuf *pkt[], uint16_t nb_pkts, void *user_param)
380 struct bpf_eth_cbi *cbi;
384 bpf_eth_cbi_inuse(cbi);
385 rc = (cbi->cb != NULL) ?
386 pkt_filter_mb_vm(cbi->bpf, pkt, nb_pkts, 0) :
388 bpf_eth_cbi_unuse(cbi);
393 bpf_tx_callback_mb_jit(__rte_unused uint16_t port, __rte_unused uint16_t queue,
394 struct rte_mbuf *pkt[], uint16_t nb_pkts, void *user_param)
396 struct bpf_eth_cbi *cbi;
400 bpf_eth_cbi_inuse(cbi);
401 rc = (cbi->cb != NULL) ?
402 pkt_filter_mb_jit(&cbi->jit, pkt, nb_pkts, 0) :
404 bpf_eth_cbi_unuse(cbi);
408 static rte_rx_callback_fn
409 select_rx_callback(enum rte_bpf_arg_type type, uint32_t flags)
411 if (flags & RTE_BPF_ETH_F_JIT) {
412 if (type == RTE_BPF_ARG_PTR)
413 return bpf_rx_callback_jit;
414 else if (type == RTE_BPF_ARG_PTR_MBUF)
415 return bpf_rx_callback_mb_jit;
416 } else if (type == RTE_BPF_ARG_PTR)
417 return bpf_rx_callback_vm;
418 else if (type == RTE_BPF_ARG_PTR_MBUF)
419 return bpf_rx_callback_mb_vm;
424 static rte_tx_callback_fn
425 select_tx_callback(enum rte_bpf_arg_type type, uint32_t flags)
427 if (flags & RTE_BPF_ETH_F_JIT) {
428 if (type == RTE_BPF_ARG_PTR)
429 return bpf_tx_callback_jit;
430 else if (type == RTE_BPF_ARG_PTR_MBUF)
431 return bpf_tx_callback_mb_jit;
432 } else if (type == RTE_BPF_ARG_PTR)
433 return bpf_tx_callback_vm;
434 else if (type == RTE_BPF_ARG_PTR_MBUF)
435 return bpf_tx_callback_mb_vm;
441 * helper function to perform BPF unload for given port/queue.
442 * have to introduce extra complexity (and possible slowdown) here,
443 * as right now there is no safe generic way to remove RX/TX callback
444 * while IO is active.
445 * Still don't free memory allocated for callback handle itself,
446 * again right now there is no safe way to do that without stopping RX/TX
447 * on given port/queue first.
450 bpf_eth_cbi_unload(struct bpf_eth_cbi *bc)
452 /* mark this cbi as empty */
456 /* make sure datapath doesn't use bpf anymore, then destroy bpf */
457 bpf_eth_cbi_wait(bc);
458 rte_bpf_destroy(bc->bpf);
459 bpf_eth_cbi_cleanup(bc);
463 bpf_eth_unload(struct bpf_eth_cbh *cbh, uint16_t port, uint16_t queue)
465 struct bpf_eth_cbi *bc;
467 bc = bpf_eth_cbh_find(cbh, port, queue);
468 if (bc == NULL || bc->cb == NULL)
471 if (cbh->type == BPF_ETH_RX)
472 rte_eth_remove_rx_callback(port, queue, bc->cb);
474 rte_eth_remove_tx_callback(port, queue, bc->cb);
476 bpf_eth_cbi_unload(bc);
481 rte_bpf_eth_rx_unload(uint16_t port, uint16_t queue)
483 struct bpf_eth_cbh *cbh;
486 rte_spinlock_lock(&cbh->lock);
487 bpf_eth_unload(cbh, port, queue);
488 rte_spinlock_unlock(&cbh->lock);
492 rte_bpf_eth_tx_unload(uint16_t port, uint16_t queue)
494 struct bpf_eth_cbh *cbh;
497 rte_spinlock_lock(&cbh->lock);
498 bpf_eth_unload(cbh, port, queue);
499 rte_spinlock_unlock(&cbh->lock);
503 bpf_eth_elf_load(struct bpf_eth_cbh *cbh, uint16_t port, uint16_t queue,
504 const struct rte_bpf_prm *prm, const char *fname, const char *sname,
508 struct bpf_eth_cbi *bc;
510 rte_rx_callback_fn frx;
511 rte_tx_callback_fn ftx;
512 struct rte_bpf_jit jit;
517 if (prm == NULL || rte_eth_dev_is_valid_port(port) == 0 ||
518 queue >= RTE_MAX_QUEUES_PER_PORT)
521 if (cbh->type == BPF_ETH_RX)
522 frx = select_rx_callback(prm->prog_arg.type, flags);
524 ftx = select_tx_callback(prm->prog_arg.type, flags);
526 if (frx == NULL && ftx == NULL) {
527 RTE_BPF_LOG(ERR, "%s(%u, %u): no callback selected;\n",
528 __func__, port, queue);
532 bpf = rte_bpf_elf_load(prm, fname, sname);
536 rte_bpf_get_jit(bpf, &jit);
538 if ((flags & RTE_BPF_ETH_F_JIT) != 0 && jit.func == NULL) {
539 RTE_BPF_LOG(ERR, "%s(%u, %u): no JIT generated;\n",
540 __func__, port, queue);
541 rte_bpf_destroy(bpf);
545 /* setup/update global callback info */
546 bc = bpf_eth_cbh_add(cbh, port, queue);
550 /* remove old one, if any */
552 bpf_eth_unload(cbh, port, queue);
557 if (cbh->type == BPF_ETH_RX)
558 bc->cb = rte_eth_add_rx_callback(port, queue, frx, bc);
560 bc->cb = rte_eth_add_tx_callback(port, queue, ftx, bc);
562 if (bc->cb == NULL) {
564 rte_bpf_destroy(bpf);
565 bpf_eth_cbi_cleanup(bc);
573 rte_bpf_eth_rx_elf_load(uint16_t port, uint16_t queue,
574 const struct rte_bpf_prm *prm, const char *fname, const char *sname,
578 struct bpf_eth_cbh *cbh;
581 rte_spinlock_lock(&cbh->lock);
582 rc = bpf_eth_elf_load(cbh, port, queue, prm, fname, sname, flags);
583 rte_spinlock_unlock(&cbh->lock);
589 rte_bpf_eth_tx_elf_load(uint16_t port, uint16_t queue,
590 const struct rte_bpf_prm *prm, const char *fname, const char *sname,
594 struct bpf_eth_cbh *cbh;
597 rte_spinlock_lock(&cbh->lock);
598 rc = bpf_eth_elf_load(cbh, port, queue, prm, fname, sname, flags);
599 rte_spinlock_unlock(&cbh->lock);