examples/ipsec-secgw: extend inline session to non AES-GCM
[dpdk.git] / lib / librte_mbuf / rte_mbuf_dyn.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2  * Copyright 2019 6WIND S.A.
3  */
4
5 #include <sys/queue.h>
6 #include <stdint.h>
7 #include <limits.h>
8
9 #include <rte_common.h>
10 #include <rte_eal.h>
11 #include <rte_eal_memconfig.h>
12 #include <rte_tailq.h>
13 #include <rte_errno.h>
14 #include <rte_malloc.h>
15 #include <rte_string_fns.h>
16 #include <rte_mbuf.h>
17 #include <rte_mbuf_dyn.h>
18
19 #define RTE_MBUF_DYN_MZNAME "rte_mbuf_dyn"
20
21 struct mbuf_dynfield_elt {
22         TAILQ_ENTRY(mbuf_dynfield_elt) next;
23         struct rte_mbuf_dynfield params;
24         size_t offset;
25 };
26 TAILQ_HEAD(mbuf_dynfield_list, rte_tailq_entry);
27
28 static struct rte_tailq_elem mbuf_dynfield_tailq = {
29         .name = "RTE_MBUF_DYNFIELD",
30 };
31 EAL_REGISTER_TAILQ(mbuf_dynfield_tailq);
32
33 struct mbuf_dynflag_elt {
34         TAILQ_ENTRY(mbuf_dynflag_elt) next;
35         struct rte_mbuf_dynflag params;
36         unsigned int bitnum;
37 };
38 TAILQ_HEAD(mbuf_dynflag_list, rte_tailq_entry);
39
40 static struct rte_tailq_elem mbuf_dynflag_tailq = {
41         .name = "RTE_MBUF_DYNFLAG",
42 };
43 EAL_REGISTER_TAILQ(mbuf_dynflag_tailq);
44
45 struct mbuf_dyn_shm {
46         /**
47          * For each mbuf byte, free_space[i] != 0 if space is free.
48          * The value is the size of the biggest aligned element that
49          * can fit in the zone.
50          */
51         uint8_t free_space[sizeof(struct rte_mbuf)];
52         /** Bitfield of available flags. */
53         uint64_t free_flags;
54 };
55 static struct mbuf_dyn_shm *shm;
56
57 /* Set the value of free_space[] according to the size and alignment of
58  * the free areas. This helps to select the best place when reserving a
59  * dynamic field. Assume tailq is locked.
60  */
61 static void
62 process_score(void)
63 {
64         size_t off, align, size, i;
65
66         /* first, erase previous info */
67         for (i = 0; i < sizeof(struct rte_mbuf); i++) {
68                 if (shm->free_space[i])
69                         shm->free_space[i] = 1;
70         }
71
72         for (off = 0; off < sizeof(struct rte_mbuf); off++) {
73                 /* get the size of the free zone */
74                 for (size = 0; shm->free_space[off + size]; size++)
75                         ;
76                 if (size == 0)
77                         continue;
78
79                 /* get the alignment of biggest object that can fit in
80                  * the zone at this offset.
81                  */
82                 for (align = 1;
83                      (off % (align << 1)) == 0 && (align << 1) <= size;
84                      align <<= 1)
85                         ;
86
87                 /* save it in free_space[] */
88                 for (i = off; i < off + size; i++)
89                         shm->free_space[i] = RTE_MAX(align, shm->free_space[i]);
90         }
91 }
92
93 /* Mark the area occupied by a mbuf field as available in the shm. */
94 #define mark_free(field)                                                \
95         memset(&shm->free_space[offsetof(struct rte_mbuf, field)],      \
96                 1, sizeof(((struct rte_mbuf *)0)->field))
97
98 /* Allocate and initialize the shared memory. Assume tailq is locked */
99 static int
100 init_shared_mem(void)
101 {
102         const struct rte_memzone *mz;
103         uint64_t mask;
104
105         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
106                 mz = rte_memzone_reserve_aligned(RTE_MBUF_DYN_MZNAME,
107                                                 sizeof(struct mbuf_dyn_shm),
108                                                 SOCKET_ID_ANY, 0,
109                                                 RTE_CACHE_LINE_SIZE);
110         } else {
111                 mz = rte_memzone_lookup(RTE_MBUF_DYN_MZNAME);
112         }
113         if (mz == NULL)
114                 return -1;
115
116         shm = mz->addr;
117
118         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
119                 /* init free_space, keep it sync'd with
120                  * rte_mbuf_dynfield_copy().
121                  */
122                 memset(shm, 0, sizeof(*shm));
123                 mark_free(dynfield1);
124
125                 /* init free_flags */
126                 for (mask = PKT_FIRST_FREE; mask <= PKT_LAST_FREE; mask <<= 1)
127                         shm->free_flags |= mask;
128
129                 process_score();
130         }
131
132         return 0;
133 }
134
135 /* check if this offset can be used */
136 static int
137 check_offset(size_t offset, size_t size, size_t align)
138 {
139         size_t i;
140
141         if ((offset & (align - 1)) != 0)
142                 return -1;
143         if (offset + size > sizeof(struct rte_mbuf))
144                 return -1;
145
146         for (i = 0; i < size; i++) {
147                 if (!shm->free_space[i + offset])
148                         return -1;
149         }
150
151         return 0;
152 }
153
154 /* assume tailq is locked */
155 static struct mbuf_dynfield_elt *
156 __mbuf_dynfield_lookup(const char *name)
157 {
158         struct mbuf_dynfield_list *mbuf_dynfield_list;
159         struct mbuf_dynfield_elt *mbuf_dynfield;
160         struct rte_tailq_entry *te;
161
162         mbuf_dynfield_list = RTE_TAILQ_CAST(
163                 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
164
165         TAILQ_FOREACH(te, mbuf_dynfield_list, next) {
166                 mbuf_dynfield = (struct mbuf_dynfield_elt *)te->data;
167                 if (strcmp(name, mbuf_dynfield->params.name) == 0)
168                         break;
169         }
170
171         if (te == NULL) {
172                 rte_errno = ENOENT;
173                 return NULL;
174         }
175
176         return mbuf_dynfield;
177 }
178
179 int
180 rte_mbuf_dynfield_lookup(const char *name, struct rte_mbuf_dynfield *params)
181 {
182         struct mbuf_dynfield_elt *mbuf_dynfield;
183
184         if (shm == NULL) {
185                 rte_errno = ENOENT;
186                 return -1;
187         }
188
189         rte_mcfg_tailq_read_lock();
190         mbuf_dynfield = __mbuf_dynfield_lookup(name);
191         rte_mcfg_tailq_read_unlock();
192
193         if (mbuf_dynfield == NULL) {
194                 rte_errno = ENOENT;
195                 return -1;
196         }
197
198         if (params != NULL)
199                 memcpy(params, &mbuf_dynfield->params, sizeof(*params));
200
201         return mbuf_dynfield->offset;
202 }
203
204 static int mbuf_dynfield_cmp(const struct rte_mbuf_dynfield *params1,
205                 const struct rte_mbuf_dynfield *params2)
206 {
207         if (strcmp(params1->name, params2->name))
208                 return -1;
209         if (params1->size != params2->size)
210                 return -1;
211         if (params1->align != params2->align)
212                 return -1;
213         if (params1->flags != params2->flags)
214                 return -1;
215         return 0;
216 }
217
218 /* assume tailq is locked */
219 static int
220 __rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params,
221                                 size_t req)
222 {
223         struct mbuf_dynfield_list *mbuf_dynfield_list;
224         struct mbuf_dynfield_elt *mbuf_dynfield = NULL;
225         struct rte_tailq_entry *te = NULL;
226         unsigned int best_zone = UINT_MAX;
227         size_t i, offset;
228         int ret;
229
230         if (shm == NULL && init_shared_mem() < 0)
231                 return -1;
232
233         mbuf_dynfield = __mbuf_dynfield_lookup(params->name);
234         if (mbuf_dynfield != NULL) {
235                 if (req != SIZE_MAX && req != mbuf_dynfield->offset) {
236                         rte_errno = EEXIST;
237                         return -1;
238                 }
239                 if (mbuf_dynfield_cmp(params, &mbuf_dynfield->params) < 0) {
240                         rte_errno = EEXIST;
241                         return -1;
242                 }
243                 return mbuf_dynfield->offset;
244         }
245
246         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
247                 rte_errno = EPERM;
248                 return -1;
249         }
250
251         if (req == SIZE_MAX) {
252                 /* Find the best place to put this field: we search the
253                  * lowest value of shm->free_space[offset]: the zones
254                  * containing room for larger fields are kept for later.
255                  */
256                 for (offset = 0;
257                      offset < sizeof(struct rte_mbuf);
258                      offset++) {
259                         if (check_offset(offset, params->size,
260                                                 params->align) == 0 &&
261                                         shm->free_space[offset] < best_zone) {
262                                 best_zone = shm->free_space[offset];
263                                 req = offset;
264                         }
265                 }
266                 if (req == SIZE_MAX) {
267                         rte_errno = ENOENT;
268                         return -1;
269                 }
270         } else {
271                 if (check_offset(req, params->size, params->align) < 0) {
272                         rte_errno = EBUSY;
273                         return -1;
274                 }
275         }
276
277         offset = req;
278         mbuf_dynfield_list = RTE_TAILQ_CAST(
279                 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
280
281         te = rte_zmalloc("MBUF_DYNFIELD_TAILQ_ENTRY", sizeof(*te), 0);
282         if (te == NULL)
283                 return -1;
284
285         mbuf_dynfield = rte_zmalloc("mbuf_dynfield", sizeof(*mbuf_dynfield), 0);
286         if (mbuf_dynfield == NULL) {
287                 rte_free(te);
288                 return -1;
289         }
290
291         ret = strlcpy(mbuf_dynfield->params.name, params->name,
292                 sizeof(mbuf_dynfield->params.name));
293         if (ret < 0 || ret >= (int)sizeof(mbuf_dynfield->params.name)) {
294                 rte_errno = ENAMETOOLONG;
295                 rte_free(mbuf_dynfield);
296                 rte_free(te);
297                 return -1;
298         }
299         memcpy(&mbuf_dynfield->params, params, sizeof(mbuf_dynfield->params));
300         mbuf_dynfield->offset = offset;
301         te->data = mbuf_dynfield;
302
303         TAILQ_INSERT_TAIL(mbuf_dynfield_list, te, next);
304
305         for (i = offset; i < offset + params->size; i++)
306                 shm->free_space[i] = 0;
307         process_score();
308
309         RTE_LOG(DEBUG, MBUF, "Registered dynamic field %s (sz=%zu, al=%zu, fl=0x%x) -> %zd\n",
310                 params->name, params->size, params->align, params->flags,
311                 offset);
312
313         return offset;
314 }
315
316 int
317 rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params,
318                                 size_t req)
319 {
320         int ret;
321
322         if (params->size >= sizeof(struct rte_mbuf)) {
323                 rte_errno = EINVAL;
324                 return -1;
325         }
326         if (!rte_is_power_of_2(params->align)) {
327                 rte_errno = EINVAL;
328                 return -1;
329         }
330         if (params->flags != 0) {
331                 rte_errno = EINVAL;
332                 return -1;
333         }
334
335         rte_mcfg_tailq_write_lock();
336         ret = __rte_mbuf_dynfield_register_offset(params, req);
337         rte_mcfg_tailq_write_unlock();
338
339         return ret;
340 }
341
342 int
343 rte_mbuf_dynfield_register(const struct rte_mbuf_dynfield *params)
344 {
345         return rte_mbuf_dynfield_register_offset(params, SIZE_MAX);
346 }
347
348 /* assume tailq is locked */
349 static struct mbuf_dynflag_elt *
350 __mbuf_dynflag_lookup(const char *name)
351 {
352         struct mbuf_dynflag_list *mbuf_dynflag_list;
353         struct mbuf_dynflag_elt *mbuf_dynflag;
354         struct rte_tailq_entry *te;
355
356         mbuf_dynflag_list = RTE_TAILQ_CAST(
357                 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
358
359         TAILQ_FOREACH(te, mbuf_dynflag_list, next) {
360                 mbuf_dynflag = (struct mbuf_dynflag_elt *)te->data;
361                 if (strncmp(name, mbuf_dynflag->params.name,
362                                 RTE_MBUF_DYN_NAMESIZE) == 0)
363                         break;
364         }
365
366         if (te == NULL) {
367                 rte_errno = ENOENT;
368                 return NULL;
369         }
370
371         return mbuf_dynflag;
372 }
373
374 int
375 rte_mbuf_dynflag_lookup(const char *name,
376                         struct rte_mbuf_dynflag *params)
377 {
378         struct mbuf_dynflag_elt *mbuf_dynflag;
379
380         if (shm == NULL) {
381                 rte_errno = ENOENT;
382                 return -1;
383         }
384
385         rte_mcfg_tailq_read_lock();
386         mbuf_dynflag = __mbuf_dynflag_lookup(name);
387         rte_mcfg_tailq_read_unlock();
388
389         if (mbuf_dynflag == NULL) {
390                 rte_errno = ENOENT;
391                 return -1;
392         }
393
394         if (params != NULL)
395                 memcpy(params, &mbuf_dynflag->params, sizeof(*params));
396
397         return mbuf_dynflag->bitnum;
398 }
399
400 static int mbuf_dynflag_cmp(const struct rte_mbuf_dynflag *params1,
401                 const struct rte_mbuf_dynflag *params2)
402 {
403         if (strcmp(params1->name, params2->name))
404                 return -1;
405         if (params1->flags != params2->flags)
406                 return -1;
407         return 0;
408 }
409
410 /* assume tailq is locked */
411 static int
412 __rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params,
413                                 unsigned int req)
414 {
415         struct mbuf_dynflag_list *mbuf_dynflag_list;
416         struct mbuf_dynflag_elt *mbuf_dynflag = NULL;
417         struct rte_tailq_entry *te = NULL;
418         unsigned int bitnum;
419         int ret;
420
421         if (shm == NULL && init_shared_mem() < 0)
422                 return -1;
423
424         mbuf_dynflag = __mbuf_dynflag_lookup(params->name);
425         if (mbuf_dynflag != NULL) {
426                 if (req != UINT_MAX && req != mbuf_dynflag->bitnum) {
427                         rte_errno = EEXIST;
428                         return -1;
429                 }
430                 if (mbuf_dynflag_cmp(params, &mbuf_dynflag->params) < 0) {
431                         rte_errno = EEXIST;
432                         return -1;
433                 }
434                 return mbuf_dynflag->bitnum;
435         }
436
437         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
438                 rte_errno = EPERM;
439                 return -1;
440         }
441
442         if (req == UINT_MAX) {
443                 if (shm->free_flags == 0) {
444                         rte_errno = ENOENT;
445                         return -1;
446                 }
447                 bitnum = rte_bsf64(shm->free_flags);
448         } else {
449                 if ((shm->free_flags & (1ULL << req)) == 0) {
450                         rte_errno = EBUSY;
451                         return -1;
452                 }
453                 bitnum = req;
454         }
455
456         mbuf_dynflag_list = RTE_TAILQ_CAST(
457                 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
458
459         te = rte_zmalloc("MBUF_DYNFLAG_TAILQ_ENTRY", sizeof(*te), 0);
460         if (te == NULL)
461                 return -1;
462
463         mbuf_dynflag = rte_zmalloc("mbuf_dynflag", sizeof(*mbuf_dynflag), 0);
464         if (mbuf_dynflag == NULL) {
465                 rte_free(te);
466                 return -1;
467         }
468
469         ret = strlcpy(mbuf_dynflag->params.name, params->name,
470                 sizeof(mbuf_dynflag->params.name));
471         if (ret < 0 || ret >= (int)sizeof(mbuf_dynflag->params.name)) {
472                 rte_free(mbuf_dynflag);
473                 rte_free(te);
474                 rte_errno = ENAMETOOLONG;
475                 return -1;
476         }
477         mbuf_dynflag->bitnum = bitnum;
478         te->data = mbuf_dynflag;
479
480         TAILQ_INSERT_TAIL(mbuf_dynflag_list, te, next);
481
482         shm->free_flags &= ~(1ULL << bitnum);
483
484         RTE_LOG(DEBUG, MBUF, "Registered dynamic flag %s (fl=0x%x) -> %u\n",
485                 params->name, params->flags, bitnum);
486
487         return bitnum;
488 }
489
490 int
491 rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params,
492                                 unsigned int req)
493 {
494         int ret;
495
496         if (req >= RTE_SIZEOF_FIELD(struct rte_mbuf, ol_flags) * CHAR_BIT &&
497                         req != UINT_MAX) {
498                 rte_errno = EINVAL;
499                 return -1;
500         }
501
502         rte_mcfg_tailq_write_lock();
503         ret = __rte_mbuf_dynflag_register_bitnum(params, req);
504         rte_mcfg_tailq_write_unlock();
505
506         return ret;
507 }
508
509 int
510 rte_mbuf_dynflag_register(const struct rte_mbuf_dynflag *params)
511 {
512         return rte_mbuf_dynflag_register_bitnum(params, UINT_MAX);
513 }
514
515 void rte_mbuf_dyn_dump(FILE *out)
516 {
517         struct mbuf_dynfield_list *mbuf_dynfield_list;
518         struct mbuf_dynfield_elt *dynfield;
519         struct mbuf_dynflag_list *mbuf_dynflag_list;
520         struct mbuf_dynflag_elt *dynflag;
521         struct rte_tailq_entry *te;
522         size_t i;
523
524         rte_mcfg_tailq_write_lock();
525         init_shared_mem();
526         fprintf(out, "Reserved fields:\n");
527         mbuf_dynfield_list = RTE_TAILQ_CAST(
528                 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
529         TAILQ_FOREACH(te, mbuf_dynfield_list, next) {
530                 dynfield = (struct mbuf_dynfield_elt *)te->data;
531                 fprintf(out, "  name=%s offset=%zd size=%zd align=%zd flags=%x\n",
532                         dynfield->params.name, dynfield->offset,
533                         dynfield->params.size, dynfield->params.align,
534                         dynfield->params.flags);
535         }
536         fprintf(out, "Reserved flags:\n");
537         mbuf_dynflag_list = RTE_TAILQ_CAST(
538                 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
539         TAILQ_FOREACH(te, mbuf_dynflag_list, next) {
540                 dynflag = (struct mbuf_dynflag_elt *)te->data;
541                 fprintf(out, "  name=%s bitnum=%u flags=%x\n",
542                         dynflag->params.name, dynflag->bitnum,
543                         dynflag->params.flags);
544         }
545         fprintf(out, "Free space in mbuf (0 = free, value = zone alignment):\n");
546         for (i = 0; i < sizeof(struct rte_mbuf); i++) {
547                 if ((i % 8) == 0)
548                         fprintf(out, "  %4.4zx: ", i);
549                 fprintf(out, "%2.2x%s", shm->free_space[i],
550                         (i % 8 != 7) ? " " : "\n");
551         }
552         rte_mcfg_tailq_write_unlock();
553 }