1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2019 6WIND S.A.
9 #include <rte_common.h>
11 #include <rte_eal_memconfig.h>
12 #include <rte_tailq.h>
13 #include <rte_errno.h>
14 #include <rte_malloc.h>
15 #include <rte_string_fns.h>
17 #include <rte_mbuf_dyn.h>
19 #define RTE_MBUF_DYN_MZNAME "rte_mbuf_dyn"
21 struct mbuf_dynfield_elt {
22 struct rte_mbuf_dynfield params;
25 TAILQ_HEAD(mbuf_dynfield_list, rte_tailq_entry);
27 static struct rte_tailq_elem mbuf_dynfield_tailq = {
28 .name = "RTE_MBUF_DYNFIELD",
30 EAL_REGISTER_TAILQ(mbuf_dynfield_tailq);
32 struct mbuf_dynflag_elt {
33 struct rte_mbuf_dynflag params;
36 TAILQ_HEAD(mbuf_dynflag_list, rte_tailq_entry);
38 static struct rte_tailq_elem mbuf_dynflag_tailq = {
39 .name = "RTE_MBUF_DYNFLAG",
41 EAL_REGISTER_TAILQ(mbuf_dynflag_tailq);
45 * For each mbuf byte, free_space[i] != 0 if space is free.
46 * The value is the size of the biggest aligned element that
47 * can fit in the zone.
49 uint8_t free_space[sizeof(struct rte_mbuf)];
50 /** Bitfield of available flags. */
53 static struct mbuf_dyn_shm *shm;
55 /* Set the value of free_space[] according to the size and alignment of
56 * the free areas. This helps to select the best place when reserving a
57 * dynamic field. Assume tailq is locked.
62 size_t off, align, size, i;
64 /* first, erase previous info */
65 for (i = 0; i < sizeof(struct rte_mbuf); i++) {
66 if (shm->free_space[i])
67 shm->free_space[i] = 1;
70 for (off = 0; off < sizeof(struct rte_mbuf); off++) {
71 /* get the size of the free zone */
72 for (size = 0; (off + size) < sizeof(struct rte_mbuf) &&
73 shm->free_space[off + size]; size++)
78 /* get the alignment of biggest object that can fit in
79 * the zone at this offset.
82 (off % (align << 1)) == 0 && (align << 1) <= size;
86 /* save it in free_space[] */
87 for (i = off; i < off + size; i++)
88 shm->free_space[i] = RTE_MAX(align, shm->free_space[i]);
92 /* Mark the area occupied by a mbuf field as available in the shm. */
93 #define mark_free(field) \
94 memset(&shm->free_space[offsetof(struct rte_mbuf, field)], \
95 1, sizeof(((struct rte_mbuf *)0)->field))
97 /* Allocate and initialize the shared memory. Assume tailq is locked */
101 const struct rte_memzone *mz;
104 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
105 mz = rte_memzone_reserve_aligned(RTE_MBUF_DYN_MZNAME,
106 sizeof(struct mbuf_dyn_shm),
108 RTE_CACHE_LINE_SIZE);
110 mz = rte_memzone_lookup(RTE_MBUF_DYN_MZNAME);
117 if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
118 /* init free_space, keep it sync'd with
119 * rte_mbuf_dynfield_copy().
121 memset(shm, 0, sizeof(*shm));
122 mark_free(dynfield1);
124 /* init free_flags */
125 for (mask = PKT_FIRST_FREE; mask <= PKT_LAST_FREE; mask <<= 1)
126 shm->free_flags |= mask;
134 /* check if this offset can be used */
136 check_offset(size_t offset, size_t size, size_t align)
140 if ((offset & (align - 1)) != 0)
142 if (offset + size > sizeof(struct rte_mbuf))
145 for (i = 0; i < size; i++) {
146 if (!shm->free_space[i + offset])
153 /* assume tailq is locked */
154 static struct mbuf_dynfield_elt *
155 __mbuf_dynfield_lookup(const char *name)
157 struct mbuf_dynfield_list *mbuf_dynfield_list;
158 struct mbuf_dynfield_elt *mbuf_dynfield;
159 struct rte_tailq_entry *te;
161 mbuf_dynfield_list = RTE_TAILQ_CAST(
162 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
164 TAILQ_FOREACH(te, mbuf_dynfield_list, next) {
165 mbuf_dynfield = (struct mbuf_dynfield_elt *)te->data;
166 if (strcmp(name, mbuf_dynfield->params.name) == 0)
175 return mbuf_dynfield;
179 rte_mbuf_dynfield_lookup(const char *name, struct rte_mbuf_dynfield *params)
181 struct mbuf_dynfield_elt *mbuf_dynfield;
188 rte_mcfg_tailq_read_lock();
189 mbuf_dynfield = __mbuf_dynfield_lookup(name);
190 rte_mcfg_tailq_read_unlock();
192 if (mbuf_dynfield == NULL) {
198 memcpy(params, &mbuf_dynfield->params, sizeof(*params));
200 return mbuf_dynfield->offset;
203 static int mbuf_dynfield_cmp(const struct rte_mbuf_dynfield *params1,
204 const struct rte_mbuf_dynfield *params2)
206 if (strcmp(params1->name, params2->name))
208 if (params1->size != params2->size)
210 if (params1->align != params2->align)
212 if (params1->flags != params2->flags)
217 /* assume tailq is locked */
219 __rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params,
222 struct mbuf_dynfield_list *mbuf_dynfield_list;
223 struct mbuf_dynfield_elt *mbuf_dynfield = NULL;
224 struct rte_tailq_entry *te = NULL;
225 unsigned int best_zone = UINT_MAX;
229 if (shm == NULL && init_shared_mem() < 0)
232 mbuf_dynfield = __mbuf_dynfield_lookup(params->name);
233 if (mbuf_dynfield != NULL) {
234 if (req != SIZE_MAX && req != mbuf_dynfield->offset) {
238 if (mbuf_dynfield_cmp(params, &mbuf_dynfield->params) < 0) {
242 return mbuf_dynfield->offset;
245 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
250 if (req == SIZE_MAX) {
251 /* Find the best place to put this field: we search the
252 * lowest value of shm->free_space[offset]: the zones
253 * containing room for larger fields are kept for later.
256 offset < sizeof(struct rte_mbuf);
258 if (check_offset(offset, params->size,
259 params->align) == 0 &&
260 shm->free_space[offset] < best_zone) {
261 best_zone = shm->free_space[offset];
265 if (req == SIZE_MAX) {
270 if (check_offset(req, params->size, params->align) < 0) {
277 mbuf_dynfield_list = RTE_TAILQ_CAST(
278 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
280 te = rte_zmalloc("MBUF_DYNFIELD_TAILQ_ENTRY", sizeof(*te), 0);
284 mbuf_dynfield = rte_zmalloc("mbuf_dynfield", sizeof(*mbuf_dynfield), 0);
285 if (mbuf_dynfield == NULL) {
290 ret = strlcpy(mbuf_dynfield->params.name, params->name,
291 sizeof(mbuf_dynfield->params.name));
292 if (ret < 0 || ret >= (int)sizeof(mbuf_dynfield->params.name)) {
293 rte_errno = ENAMETOOLONG;
294 rte_free(mbuf_dynfield);
298 memcpy(&mbuf_dynfield->params, params, sizeof(mbuf_dynfield->params));
299 mbuf_dynfield->offset = offset;
300 te->data = mbuf_dynfield;
302 TAILQ_INSERT_TAIL(mbuf_dynfield_list, te, next);
304 for (i = offset; i < offset + params->size; i++)
305 shm->free_space[i] = 0;
308 RTE_LOG(DEBUG, MBUF, "Registered dynamic field %s (sz=%zu, al=%zu, fl=0x%x) -> %zd\n",
309 params->name, params->size, params->align, params->flags,
316 rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params,
321 if (params->size >= sizeof(struct rte_mbuf)) {
325 if (!rte_is_power_of_2(params->align)) {
329 if (params->flags != 0) {
334 rte_mcfg_tailq_write_lock();
335 ret = __rte_mbuf_dynfield_register_offset(params, req);
336 rte_mcfg_tailq_write_unlock();
342 rte_mbuf_dynfield_register(const struct rte_mbuf_dynfield *params)
344 return rte_mbuf_dynfield_register_offset(params, SIZE_MAX);
347 /* assume tailq is locked */
348 static struct mbuf_dynflag_elt *
349 __mbuf_dynflag_lookup(const char *name)
351 struct mbuf_dynflag_list *mbuf_dynflag_list;
352 struct mbuf_dynflag_elt *mbuf_dynflag;
353 struct rte_tailq_entry *te;
355 mbuf_dynflag_list = RTE_TAILQ_CAST(
356 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
358 TAILQ_FOREACH(te, mbuf_dynflag_list, next) {
359 mbuf_dynflag = (struct mbuf_dynflag_elt *)te->data;
360 if (strncmp(name, mbuf_dynflag->params.name,
361 RTE_MBUF_DYN_NAMESIZE) == 0)
374 rte_mbuf_dynflag_lookup(const char *name,
375 struct rte_mbuf_dynflag *params)
377 struct mbuf_dynflag_elt *mbuf_dynflag;
384 rte_mcfg_tailq_read_lock();
385 mbuf_dynflag = __mbuf_dynflag_lookup(name);
386 rte_mcfg_tailq_read_unlock();
388 if (mbuf_dynflag == NULL) {
394 memcpy(params, &mbuf_dynflag->params, sizeof(*params));
396 return mbuf_dynflag->bitnum;
399 static int mbuf_dynflag_cmp(const struct rte_mbuf_dynflag *params1,
400 const struct rte_mbuf_dynflag *params2)
402 if (strcmp(params1->name, params2->name))
404 if (params1->flags != params2->flags)
409 /* assume tailq is locked */
411 __rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params,
414 struct mbuf_dynflag_list *mbuf_dynflag_list;
415 struct mbuf_dynflag_elt *mbuf_dynflag = NULL;
416 struct rte_tailq_entry *te = NULL;
420 if (shm == NULL && init_shared_mem() < 0)
423 mbuf_dynflag = __mbuf_dynflag_lookup(params->name);
424 if (mbuf_dynflag != NULL) {
425 if (req != UINT_MAX && req != mbuf_dynflag->bitnum) {
429 if (mbuf_dynflag_cmp(params, &mbuf_dynflag->params) < 0) {
433 return mbuf_dynflag->bitnum;
436 if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
441 if (req == UINT_MAX) {
442 if (shm->free_flags == 0) {
446 bitnum = rte_bsf64(shm->free_flags);
448 if ((shm->free_flags & (1ULL << req)) == 0) {
455 mbuf_dynflag_list = RTE_TAILQ_CAST(
456 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
458 te = rte_zmalloc("MBUF_DYNFLAG_TAILQ_ENTRY", sizeof(*te), 0);
462 mbuf_dynflag = rte_zmalloc("mbuf_dynflag", sizeof(*mbuf_dynflag), 0);
463 if (mbuf_dynflag == NULL) {
468 ret = strlcpy(mbuf_dynflag->params.name, params->name,
469 sizeof(mbuf_dynflag->params.name));
470 if (ret < 0 || ret >= (int)sizeof(mbuf_dynflag->params.name)) {
471 rte_free(mbuf_dynflag);
473 rte_errno = ENAMETOOLONG;
476 mbuf_dynflag->bitnum = bitnum;
477 te->data = mbuf_dynflag;
479 TAILQ_INSERT_TAIL(mbuf_dynflag_list, te, next);
481 shm->free_flags &= ~(1ULL << bitnum);
483 RTE_LOG(DEBUG, MBUF, "Registered dynamic flag %s (fl=0x%x) -> %u\n",
484 params->name, params->flags, bitnum);
490 rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params,
495 if (req >= RTE_SIZEOF_FIELD(struct rte_mbuf, ol_flags) * CHAR_BIT &&
501 rte_mcfg_tailq_write_lock();
502 ret = __rte_mbuf_dynflag_register_bitnum(params, req);
503 rte_mcfg_tailq_write_unlock();
509 rte_mbuf_dynflag_register(const struct rte_mbuf_dynflag *params)
511 return rte_mbuf_dynflag_register_bitnum(params, UINT_MAX);
514 void rte_mbuf_dyn_dump(FILE *out)
516 struct mbuf_dynfield_list *mbuf_dynfield_list;
517 struct mbuf_dynfield_elt *dynfield;
518 struct mbuf_dynflag_list *mbuf_dynflag_list;
519 struct mbuf_dynflag_elt *dynflag;
520 struct rte_tailq_entry *te;
523 rte_mcfg_tailq_write_lock();
525 fprintf(out, "Reserved fields:\n");
526 mbuf_dynfield_list = RTE_TAILQ_CAST(
527 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
528 TAILQ_FOREACH(te, mbuf_dynfield_list, next) {
529 dynfield = (struct mbuf_dynfield_elt *)te->data;
530 fprintf(out, " name=%s offset=%zd size=%zd align=%zd flags=%x\n",
531 dynfield->params.name, dynfield->offset,
532 dynfield->params.size, dynfield->params.align,
533 dynfield->params.flags);
535 fprintf(out, "Reserved flags:\n");
536 mbuf_dynflag_list = RTE_TAILQ_CAST(
537 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
538 TAILQ_FOREACH(te, mbuf_dynflag_list, next) {
539 dynflag = (struct mbuf_dynflag_elt *)te->data;
540 fprintf(out, " name=%s bitnum=%u flags=%x\n",
541 dynflag->params.name, dynflag->bitnum,
542 dynflag->params.flags);
544 fprintf(out, "Free space in mbuf (0 = free, value = zone alignment):\n");
545 for (i = 0; i < sizeof(struct rte_mbuf); i++) {
547 fprintf(out, " %4.4zx: ", i);
548 fprintf(out, "%2.2x%s", shm->free_space[i],
549 (i % 8 != 7) ? " " : "\n");
551 rte_mcfg_tailq_write_unlock();