1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2019 6WIND S.A.
9 #include <rte_common.h>
11 #include <rte_eal_memconfig.h>
12 #include <rte_tailq.h>
13 #include <rte_errno.h>
14 #include <rte_malloc.h>
15 #include <rte_string_fns.h>
16 #include <rte_bitops.h>
18 #include <rte_mbuf_dyn.h>
20 #define RTE_MBUF_DYN_MZNAME "rte_mbuf_dyn"
22 struct mbuf_dynfield_elt {
23 struct rte_mbuf_dynfield params;
26 TAILQ_HEAD(mbuf_dynfield_list, rte_tailq_entry);
28 static struct rte_tailq_elem mbuf_dynfield_tailq = {
29 .name = "RTE_MBUF_DYNFIELD",
31 EAL_REGISTER_TAILQ(mbuf_dynfield_tailq);
33 struct mbuf_dynflag_elt {
34 struct rte_mbuf_dynflag params;
37 TAILQ_HEAD(mbuf_dynflag_list, rte_tailq_entry);
39 static struct rte_tailq_elem mbuf_dynflag_tailq = {
40 .name = "RTE_MBUF_DYNFLAG",
42 EAL_REGISTER_TAILQ(mbuf_dynflag_tailq);
46 * For each mbuf byte, free_space[i] != 0 if space is free.
47 * The value is the size of the biggest aligned element that
48 * can fit in the zone.
50 uint8_t free_space[sizeof(struct rte_mbuf)];
51 /** Bitfield of available flags. */
54 static struct mbuf_dyn_shm *shm;
56 /* Set the value of free_space[] according to the size and alignment of
57 * the free areas. This helps to select the best place when reserving a
58 * dynamic field. Assume tailq is locked.
63 size_t off, align, size, i;
65 /* first, erase previous info */
66 for (i = 0; i < sizeof(struct rte_mbuf); i++) {
67 if (shm->free_space[i])
68 shm->free_space[i] = 1;
72 while (off < sizeof(struct rte_mbuf)) {
73 /* get the size of the free zone */
74 for (size = 0; (off + size) < sizeof(struct rte_mbuf) &&
75 shm->free_space[off + size]; size++)
82 /* get the alignment of biggest object that can fit in
83 * the zone at this offset.
86 (off % (align << 1)) == 0 && (align << 1) <= size;
90 /* save it in free_space[] */
91 for (i = off; i < off + align; i++)
92 shm->free_space[i] = RTE_MAX(align, shm->free_space[i]);
98 /* Mark the area occupied by a mbuf field as available in the shm. */
99 #define mark_free(field) \
100 memset(&shm->free_space[offsetof(struct rte_mbuf, field)], \
101 1, sizeof(((struct rte_mbuf *)0)->field))
103 /* Allocate and initialize the shared memory. Assume tailq is locked */
105 init_shared_mem(void)
107 const struct rte_memzone *mz;
110 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
111 mz = rte_memzone_reserve_aligned(RTE_MBUF_DYN_MZNAME,
112 sizeof(struct mbuf_dyn_shm),
114 RTE_CACHE_LINE_SIZE);
116 mz = rte_memzone_lookup(RTE_MBUF_DYN_MZNAME);
123 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
124 /* init free_space, keep it sync'd with
125 * rte_mbuf_dynfield_copy().
127 memset(shm, 0, sizeof(*shm));
128 mark_free(dynfield0);
129 mark_free(dynfield1);
131 /* init free_flags */
132 for (mask = PKT_FIRST_FREE; mask <= PKT_LAST_FREE; mask <<= 1)
133 shm->free_flags |= mask;
141 /* check if this offset can be used */
143 check_offset(size_t offset, size_t size, size_t align)
147 if ((offset & (align - 1)) != 0)
149 if (offset + size > sizeof(struct rte_mbuf))
152 for (i = 0; i < size; i++) {
153 if (!shm->free_space[i + offset])
160 /* assume tailq is locked */
161 static struct mbuf_dynfield_elt *
162 __mbuf_dynfield_lookup(const char *name)
164 struct mbuf_dynfield_list *mbuf_dynfield_list;
165 struct mbuf_dynfield_elt *mbuf_dynfield;
166 struct rte_tailq_entry *te;
168 mbuf_dynfield_list = RTE_TAILQ_CAST(
169 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
171 TAILQ_FOREACH(te, mbuf_dynfield_list, next) {
172 mbuf_dynfield = (struct mbuf_dynfield_elt *)te->data;
173 if (strcmp(name, mbuf_dynfield->params.name) == 0)
177 if (te == NULL || mbuf_dynfield == NULL) {
182 return mbuf_dynfield;
186 rte_mbuf_dynfield_lookup(const char *name, struct rte_mbuf_dynfield *params)
188 struct mbuf_dynfield_elt *mbuf_dynfield;
190 rte_mcfg_tailq_read_lock();
191 if (shm == NULL && init_shared_mem() < 0)
192 mbuf_dynfield = NULL;
194 mbuf_dynfield = __mbuf_dynfield_lookup(name);
195 rte_mcfg_tailq_read_unlock();
197 if (mbuf_dynfield == NULL)
201 memcpy(params, &mbuf_dynfield->params, sizeof(*params));
203 return mbuf_dynfield->offset;
206 static int mbuf_dynfield_cmp(const struct rte_mbuf_dynfield *params1,
207 const struct rte_mbuf_dynfield *params2)
209 if (strcmp(params1->name, params2->name))
211 if (params1->size != params2->size)
213 if (params1->align != params2->align)
215 if (params1->flags != params2->flags)
220 /* assume tailq is locked */
222 __rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params,
225 struct mbuf_dynfield_list *mbuf_dynfield_list;
226 struct mbuf_dynfield_elt *mbuf_dynfield = NULL;
227 struct rte_tailq_entry *te = NULL;
228 unsigned int best_zone = UINT_MAX;
232 if (shm == NULL && init_shared_mem() < 0)
235 mbuf_dynfield = __mbuf_dynfield_lookup(params->name);
236 if (mbuf_dynfield != NULL) {
237 if (req != SIZE_MAX && req != mbuf_dynfield->offset) {
241 if (mbuf_dynfield_cmp(params, &mbuf_dynfield->params) < 0) {
245 return mbuf_dynfield->offset;
248 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
253 if (req == SIZE_MAX) {
254 /* Find the best place to put this field: we search the
255 * lowest value of shm->free_space[offset]: the zones
256 * containing room for larger fields are kept for later.
259 offset < sizeof(struct rte_mbuf);
261 if (check_offset(offset, params->size,
262 params->align) == 0 &&
263 shm->free_space[offset] < best_zone) {
264 best_zone = shm->free_space[offset];
268 if (req == SIZE_MAX) {
273 if (check_offset(req, params->size, params->align) < 0) {
280 mbuf_dynfield_list = RTE_TAILQ_CAST(
281 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
283 te = rte_zmalloc("MBUF_DYNFIELD_TAILQ_ENTRY", sizeof(*te), 0);
289 mbuf_dynfield = rte_zmalloc("mbuf_dynfield", sizeof(*mbuf_dynfield), 0);
290 if (mbuf_dynfield == NULL) {
296 ret = strlcpy(mbuf_dynfield->params.name, params->name,
297 sizeof(mbuf_dynfield->params.name));
298 if (ret < 0 || ret >= (int)sizeof(mbuf_dynfield->params.name)) {
299 rte_errno = ENAMETOOLONG;
300 rte_free(mbuf_dynfield);
304 memcpy(&mbuf_dynfield->params, params, sizeof(mbuf_dynfield->params));
305 mbuf_dynfield->offset = offset;
306 te->data = mbuf_dynfield;
308 TAILQ_INSERT_TAIL(mbuf_dynfield_list, te, next);
310 for (i = offset; i < offset + params->size; i++)
311 shm->free_space[i] = 0;
314 RTE_LOG(DEBUG, MBUF, "Registered dynamic field %s (sz=%zu, al=%zu, fl=0x%x) -> %zd\n",
315 params->name, params->size, params->align, params->flags,
322 rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params,
327 if (params->size >= sizeof(struct rte_mbuf)) {
331 if (!rte_is_power_of_2(params->align)) {
335 if (params->flags != 0) {
340 rte_mcfg_tailq_write_lock();
341 ret = __rte_mbuf_dynfield_register_offset(params, req);
342 rte_mcfg_tailq_write_unlock();
348 rte_mbuf_dynfield_register(const struct rte_mbuf_dynfield *params)
350 return rte_mbuf_dynfield_register_offset(params, SIZE_MAX);
353 /* assume tailq is locked */
354 static struct mbuf_dynflag_elt *
355 __mbuf_dynflag_lookup(const char *name)
357 struct mbuf_dynflag_list *mbuf_dynflag_list;
358 struct mbuf_dynflag_elt *mbuf_dynflag;
359 struct rte_tailq_entry *te;
361 mbuf_dynflag_list = RTE_TAILQ_CAST(
362 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
364 TAILQ_FOREACH(te, mbuf_dynflag_list, next) {
365 mbuf_dynflag = (struct mbuf_dynflag_elt *)te->data;
366 if (strncmp(name, mbuf_dynflag->params.name,
367 RTE_MBUF_DYN_NAMESIZE) == 0)
380 rte_mbuf_dynflag_lookup(const char *name,
381 struct rte_mbuf_dynflag *params)
383 struct mbuf_dynflag_elt *mbuf_dynflag;
385 rte_mcfg_tailq_read_lock();
386 if (shm == NULL && init_shared_mem() < 0)
389 mbuf_dynflag = __mbuf_dynflag_lookup(name);
390 rte_mcfg_tailq_read_unlock();
392 if (mbuf_dynflag == NULL)
396 memcpy(params, &mbuf_dynflag->params, sizeof(*params));
398 return mbuf_dynflag->bitnum;
401 static int mbuf_dynflag_cmp(const struct rte_mbuf_dynflag *params1,
402 const struct rte_mbuf_dynflag *params2)
404 if (strcmp(params1->name, params2->name))
406 if (params1->flags != params2->flags)
411 /* assume tailq is locked */
413 __rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params,
416 struct mbuf_dynflag_list *mbuf_dynflag_list;
417 struct mbuf_dynflag_elt *mbuf_dynflag = NULL;
418 struct rte_tailq_entry *te = NULL;
422 if (shm == NULL && init_shared_mem() < 0)
425 mbuf_dynflag = __mbuf_dynflag_lookup(params->name);
426 if (mbuf_dynflag != NULL) {
427 if (req != UINT_MAX && req != mbuf_dynflag->bitnum) {
431 if (mbuf_dynflag_cmp(params, &mbuf_dynflag->params) < 0) {
435 return mbuf_dynflag->bitnum;
438 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
443 if (req == UINT_MAX) {
444 if (shm->free_flags == 0) {
448 bitnum = rte_bsf64(shm->free_flags);
450 if ((shm->free_flags & (1ULL << req)) == 0) {
457 mbuf_dynflag_list = RTE_TAILQ_CAST(
458 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
460 te = rte_zmalloc("MBUF_DYNFLAG_TAILQ_ENTRY", sizeof(*te), 0);
466 mbuf_dynflag = rte_zmalloc("mbuf_dynflag", sizeof(*mbuf_dynflag), 0);
467 if (mbuf_dynflag == NULL) {
473 ret = strlcpy(mbuf_dynflag->params.name, params->name,
474 sizeof(mbuf_dynflag->params.name));
475 if (ret < 0 || ret >= (int)sizeof(mbuf_dynflag->params.name)) {
476 rte_free(mbuf_dynflag);
478 rte_errno = ENAMETOOLONG;
481 mbuf_dynflag->bitnum = bitnum;
482 te->data = mbuf_dynflag;
484 TAILQ_INSERT_TAIL(mbuf_dynflag_list, te, next);
486 shm->free_flags &= ~(1ULL << bitnum);
488 RTE_LOG(DEBUG, MBUF, "Registered dynamic flag %s (fl=0x%x) -> %u\n",
489 params->name, params->flags, bitnum);
495 rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params,
500 if (req >= RTE_SIZEOF_FIELD(struct rte_mbuf, ol_flags) * CHAR_BIT &&
506 rte_mcfg_tailq_write_lock();
507 ret = __rte_mbuf_dynflag_register_bitnum(params, req);
508 rte_mcfg_tailq_write_unlock();
514 rte_mbuf_dynflag_register(const struct rte_mbuf_dynflag *params)
516 return rte_mbuf_dynflag_register_bitnum(params, UINT_MAX);
519 void rte_mbuf_dyn_dump(FILE *out)
521 struct mbuf_dynfield_list *mbuf_dynfield_list;
522 struct mbuf_dynfield_elt *dynfield;
523 struct mbuf_dynflag_list *mbuf_dynflag_list;
524 struct mbuf_dynflag_elt *dynflag;
525 struct rte_tailq_entry *te;
528 rte_mcfg_tailq_write_lock();
530 fprintf(out, "Reserved fields:\n");
531 mbuf_dynfield_list = RTE_TAILQ_CAST(
532 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
533 TAILQ_FOREACH(te, mbuf_dynfield_list, next) {
534 dynfield = (struct mbuf_dynfield_elt *)te->data;
535 fprintf(out, " name=%s offset=%zd size=%zd align=%zd flags=%x\n",
536 dynfield->params.name, dynfield->offset,
537 dynfield->params.size, dynfield->params.align,
538 dynfield->params.flags);
540 fprintf(out, "Reserved flags:\n");
541 mbuf_dynflag_list = RTE_TAILQ_CAST(
542 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
543 TAILQ_FOREACH(te, mbuf_dynflag_list, next) {
544 dynflag = (struct mbuf_dynflag_elt *)te->data;
545 fprintf(out, " name=%s bitnum=%u flags=%x\n",
546 dynflag->params.name, dynflag->bitnum,
547 dynflag->params.flags);
549 fprintf(out, "Free space in mbuf (0 = occupied, value = free zone alignment):\n");
550 for (i = 0; i < sizeof(struct rte_mbuf); i++) {
552 fprintf(out, " %4.4zx: ", i);
553 fprintf(out, "%2.2x%s", shm->free_space[i],
554 (i % 8 != 7) ? " " : "\n");
556 fprintf(out, "Free bit in mbuf->ol_flags (0 = occupied, 1 = free):\n");
557 for (i = 0; i < sizeof(uint64_t) * CHAR_BIT; i++) {
559 fprintf(out, " %4.4zx: ", i);
560 fprintf(out, "%1.1x%s", (shm->free_flags & (1ULL << i)) ? 1 : 0,
561 (i % 8 != 7) ? " " : "\n");
564 rte_mcfg_tailq_write_unlock();
568 rte_mbuf_dyn_timestamp_register(int *field_offset, uint64_t *flag,
569 const char *direction, const char *flag_name)
571 static const struct rte_mbuf_dynfield field_desc = {
572 .name = RTE_MBUF_DYNFIELD_TIMESTAMP_NAME,
573 .size = sizeof(rte_mbuf_timestamp_t),
574 .align = __alignof__(rte_mbuf_timestamp_t),
576 struct rte_mbuf_dynflag flag_desc = {};
579 offset = rte_mbuf_dynfield_register(&field_desc);
582 "Failed to register mbuf field for timestamp\n");
585 if (field_offset != NULL)
586 *field_offset = offset;
588 strlcpy(flag_desc.name, flag_name, sizeof(flag_desc.name));
589 offset = rte_mbuf_dynflag_register(&flag_desc);
592 "Failed to register mbuf flag for %s timestamp\n",
597 *flag = RTE_BIT64(offset);
603 rte_mbuf_dyn_rx_timestamp_register(int *field_offset, uint64_t *rx_flag)
605 return rte_mbuf_dyn_timestamp_register(field_offset, rx_flag,
606 "Rx", RTE_MBUF_DYNFLAG_RX_TIMESTAMP_NAME);
610 rte_mbuf_dyn_tx_timestamp_register(int *field_offset, uint64_t *tx_flag)
612 return rte_mbuf_dyn_timestamp_register(field_offset, tx_flag,
613 "Tx", RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME);