lib/librte_mbuf/rte_mbuf_dyn.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 6WIND S.A.
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdint.h>
   7 #include <limits.h>
   8
   9 #include <rte_common.h>
  10 #include <rte_eal.h>
  11 #include <rte_eal_memconfig.h>
  12 #include <rte_tailq.h>
  13 #include <rte_errno.h>
  14 #include <rte_malloc.h>
  15 #include <rte_string_fns.h>
  16 #include <rte_mbuf.h>
  17 #include <rte_mbuf_dyn.h>
  18
  19 #define RTE_MBUF_DYN_MZNAME "rte_mbuf_dyn"
  20
  21 struct mbuf_dynfield_elt {
  22         struct rte_mbuf_dynfield params;
  23         size_t offset;
  24 };
  25 TAILQ_HEAD(mbuf_dynfield_list, rte_tailq_entry);
  26
  27 static struct rte_tailq_elem mbuf_dynfield_tailq = {
  28         .name = "RTE_MBUF_DYNFIELD",
  29 };
  30 EAL_REGISTER_TAILQ(mbuf_dynfield_tailq);
  31
  32 struct mbuf_dynflag_elt {
  33         struct rte_mbuf_dynflag params;
  34         unsigned int bitnum;
  35 };
  36 TAILQ_HEAD(mbuf_dynflag_list, rte_tailq_entry);
  37
  38 static struct rte_tailq_elem mbuf_dynflag_tailq = {
  39         .name = "RTE_MBUF_DYNFLAG",
  40 };
  41 EAL_REGISTER_TAILQ(mbuf_dynflag_tailq);
  42
  43 struct mbuf_dyn_shm {
  44         /**
  45          * For each mbuf byte, free_space[i] != 0 if space is free.
  46          * The value is the size of the biggest aligned element that
  47          * can fit in the zone.
  48          */
  49         uint8_t free_space[sizeof(struct rte_mbuf)];
  50         /** Bitfield of available flags. */
  51         uint64_t free_flags;
  52 };
  53 static struct mbuf_dyn_shm *shm;
  54
  55 /* Set the value of free_space[] according to the size and alignment of
  56  * the free areas. This helps to select the best place when reserving a
  57  * dynamic field. Assume tailq is locked.
  58  */
  59 static void
  60 process_score(void)
  61 {
  62         size_t off, align, size, i;
  63
  64         /* first, erase previous info */
  65         for (i = 0; i < sizeof(struct rte_mbuf); i++) {
  66                 if (shm->free_space[i])
  67                         shm->free_space[i] = 1;
  68         }
  69
  70         for (off = 0; off < sizeof(struct rte_mbuf); off++) {
  71                 /* get the size of the free zone */
  72                 for (size = 0; shm->free_space[off + size]; size++)
  73                         ;
  74                 if (size == 0)
  75                         continue;
  76
  77                 /* get the alignment of biggest object that can fit in
  78                  * the zone at this offset.
  79                  */
  80                 for (align = 1;
  81                      (off % (align << 1)) == 0 && (align << 1) <= size;
  82                      align <<= 1)
  83                         ;
  84
  85                 /* save it in free_space[] */
  86                 for (i = off; i < off + size; i++)
  87                         shm->free_space[i] = RTE_MAX(align, shm->free_space[i]);
  88         }
  89 }
  90
  91 /* Mark the area occupied by a mbuf field as available in the shm. */
  92 #define mark_free(field)                                                \
  93         memset(&shm->free_space[offsetof(struct rte_mbuf, field)],      \
  94                 1, sizeof(((struct rte_mbuf *)0)->field))
  95
  96 /* Allocate and initialize the shared memory. Assume tailq is locked */
  97 static int
  98 init_shared_mem(void)
  99 {
 100         const struct rte_memzone *mz;
 101         uint64_t mask;
 102
 103         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 104                 mz = rte_memzone_reserve_aligned(RTE_MBUF_DYN_MZNAME,
 105                                                 sizeof(struct mbuf_dyn_shm),
 106                                                 SOCKET_ID_ANY, 0,
 107                                                 RTE_CACHE_LINE_SIZE);
 108         } else {
 109                 mz = rte_memzone_lookup(RTE_MBUF_DYN_MZNAME);
 110         }
 111         if (mz == NULL)
 112                 return -1;
 113
 114         shm = mz->addr;
 115
 116         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 117                 /* init free_space, keep it sync'd with
 118                  * rte_mbuf_dynfield_copy().
 119                  */
 120                 memset(shm, 0, sizeof(*shm));
 121                 mark_free(dynfield1);
 122
 123                 /* init free_flags */
 124                 for (mask = PKT_FIRST_FREE; mask <= PKT_LAST_FREE; mask <<= 1)
 125                         shm->free_flags |= mask;
 126
 127                 process_score();
 128         }
 129
 130         return 0;
 131 }
 132
 133 /* check if this offset can be used */
 134 static int
 135 check_offset(size_t offset, size_t size, size_t align)
 136 {
 137         size_t i;
 138
 139         if ((offset & (align - 1)) != 0)
 140                 return -1;
 141         if (offset + size > sizeof(struct rte_mbuf))
 142                 return -1;
 143
 144         for (i = 0; i < size; i++) {
 145                 if (!shm->free_space[i + offset])
 146                         return -1;
 147         }
 148
 149         return 0;
 150 }
 151
 152 /* assume tailq is locked */
 153 static struct mbuf_dynfield_elt *
 154 __mbuf_dynfield_lookup(const char *name)
 155 {
 156         struct mbuf_dynfield_list *mbuf_dynfield_list;
 157         struct mbuf_dynfield_elt *mbuf_dynfield;
 158         struct rte_tailq_entry *te;
 159
 160         mbuf_dynfield_list = RTE_TAILQ_CAST(
 161                 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
 162
 163         TAILQ_FOREACH(te, mbuf_dynfield_list, next) {
 164                 mbuf_dynfield = (struct mbuf_dynfield_elt *)te->data;
 165                 if (strcmp(name, mbuf_dynfield->params.name) == 0)
 166                         break;
 167         }
 168
 169         if (te == NULL) {
 170                 rte_errno = ENOENT;
 171                 return NULL;
 172         }
 173
 174         return mbuf_dynfield;
 175 }
 176
 177 int
 178 rte_mbuf_dynfield_lookup(const char *name, struct rte_mbuf_dynfield *params)
 179 {
 180         struct mbuf_dynfield_elt *mbuf_dynfield;
 181
 182         if (shm == NULL) {
 183                 rte_errno = ENOENT;
 184                 return -1;
 185         }
 186
 187         rte_mcfg_tailq_read_lock();
 188         mbuf_dynfield = __mbuf_dynfield_lookup(name);
 189         rte_mcfg_tailq_read_unlock();
 190
 191         if (mbuf_dynfield == NULL) {
 192                 rte_errno = ENOENT;
 193                 return -1;
 194         }
 195
 196         if (params != NULL)
 197                 memcpy(params, &mbuf_dynfield->params, sizeof(*params));
 198
 199         return mbuf_dynfield->offset;
 200 }
 201
 202 static int mbuf_dynfield_cmp(const struct rte_mbuf_dynfield *params1,
 203                 const struct rte_mbuf_dynfield *params2)
 204 {
 205         if (strcmp(params1->name, params2->name))
 206                 return -1;
 207         if (params1->size != params2->size)
 208                 return -1;
 209         if (params1->align != params2->align)
 210                 return -1;
 211         if (params1->flags != params2->flags)
 212                 return -1;
 213         return 0;
 214 }
 215
 216 /* assume tailq is locked */
 217 static int
 218 __rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params,
 219                                 size_t req)
 220 {
 221         struct mbuf_dynfield_list *mbuf_dynfield_list;
 222         struct mbuf_dynfield_elt *mbuf_dynfield = NULL;
 223         struct rte_tailq_entry *te = NULL;
 224         unsigned int best_zone = UINT_MAX;
 225         size_t i, offset;
 226         int ret;
 227
 228         if (shm == NULL && init_shared_mem() < 0)
 229                 return -1;
 230
 231         mbuf_dynfield = __mbuf_dynfield_lookup(params->name);
 232         if (mbuf_dynfield != NULL) {
 233                 if (req != SIZE_MAX && req != mbuf_dynfield->offset) {
 234                         rte_errno = EEXIST;
 235                         return -1;
 236                 }
 237                 if (mbuf_dynfield_cmp(params, &mbuf_dynfield->params) < 0) {
 238                         rte_errno = EEXIST;
 239                         return -1;
 240                 }
 241                 return mbuf_dynfield->offset;
 242         }
 243
 244         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 245                 rte_errno = EPERM;
 246                 return -1;
 247         }
 248
 249         if (req == SIZE_MAX) {
 250                 /* Find the best place to put this field: we search the
 251                  * lowest value of shm->free_space[offset]: the zones
 252                  * containing room for larger fields are kept for later.
 253                  */
 254                 for (offset = 0;
 255                      offset < sizeof(struct rte_mbuf);
 256                      offset++) {
 257                         if (check_offset(offset, params->size,
 258                                                 params->align) == 0 &&
 259                                         shm->free_space[offset] < best_zone) {
 260                                 best_zone = shm->free_space[offset];
 261                                 req = offset;
 262                         }
 263                 }
 264                 if (req == SIZE_MAX) {
 265                         rte_errno = ENOENT;
 266                         return -1;
 267                 }
 268         } else {
 269                 if (check_offset(req, params->size, params->align) < 0) {
 270                         rte_errno = EBUSY;
 271                         return -1;
 272                 }
 273         }
 274
 275         offset = req;
 276         mbuf_dynfield_list = RTE_TAILQ_CAST(
 277                 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
 278
 279         te = rte_zmalloc("MBUF_DYNFIELD_TAILQ_ENTRY", sizeof(*te), 0);
 280         if (te == NULL)
 281                 return -1;
 282
 283         mbuf_dynfield = rte_zmalloc("mbuf_dynfield", sizeof(*mbuf_dynfield), 0);
 284         if (mbuf_dynfield == NULL) {
 285                 rte_free(te);
 286                 return -1;
 287         }
 288
 289         ret = strlcpy(mbuf_dynfield->params.name, params->name,
 290                 sizeof(mbuf_dynfield->params.name));
 291         if (ret < 0 || ret >= (int)sizeof(mbuf_dynfield->params.name)) {
 292                 rte_errno = ENAMETOOLONG;
 293                 rte_free(mbuf_dynfield);
 294                 rte_free(te);
 295                 return -1;
 296         }
 297         memcpy(&mbuf_dynfield->params, params, sizeof(mbuf_dynfield->params));
 298         mbuf_dynfield->offset = offset;
 299         te->data = mbuf_dynfield;
 300
 301         TAILQ_INSERT_TAIL(mbuf_dynfield_list, te, next);
 302
 303         for (i = offset; i < offset + params->size; i++)
 304                 shm->free_space[i] = 0;
 305         process_score();
 306
 307         RTE_LOG(DEBUG, MBUF, "Registered dynamic field %s (sz=%zu, al=%zu, fl=0x%x) -> %zd\n",
 308                 params->name, params->size, params->align, params->flags,
 309                 offset);
 310
 311         return offset;
 312 }
 313
 314 int
 315 rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params,
 316                                 size_t req)
 317 {
 318         int ret;
 319
 320         if (params->size >= sizeof(struct rte_mbuf)) {
 321                 rte_errno = EINVAL;
 322                 return -1;
 323         }
 324         if (!rte_is_power_of_2(params->align)) {
 325                 rte_errno = EINVAL;
 326                 return -1;
 327         }
 328         if (params->flags != 0) {
 329                 rte_errno = EINVAL;
 330                 return -1;
 331         }
 332
 333         rte_mcfg_tailq_write_lock();
 334         ret = __rte_mbuf_dynfield_register_offset(params, req);
 335         rte_mcfg_tailq_write_unlock();
 336
 337         return ret;
 338 }
 339
 340 int
 341 rte_mbuf_dynfield_register(const struct rte_mbuf_dynfield *params)
 342 {
 343         return rte_mbuf_dynfield_register_offset(params, SIZE_MAX);
 344 }
 345
 346 /* assume tailq is locked */
 347 static struct mbuf_dynflag_elt *
 348 __mbuf_dynflag_lookup(const char *name)
 349 {
 350         struct mbuf_dynflag_list *mbuf_dynflag_list;
 351         struct mbuf_dynflag_elt *mbuf_dynflag;
 352         struct rte_tailq_entry *te;
 353
 354         mbuf_dynflag_list = RTE_TAILQ_CAST(
 355                 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
 356
 357         TAILQ_FOREACH(te, mbuf_dynflag_list, next) {
 358                 mbuf_dynflag = (struct mbuf_dynflag_elt *)te->data;
 359                 if (strncmp(name, mbuf_dynflag->params.name,
 360                                 RTE_MBUF_DYN_NAMESIZE) == 0)
 361                         break;
 362         }
 363
 364         if (te == NULL) {
 365                 rte_errno = ENOENT;
 366                 return NULL;
 367         }
 368
 369         return mbuf_dynflag;
 370 }
 371
 372 int
 373 rte_mbuf_dynflag_lookup(const char *name,
 374                         struct rte_mbuf_dynflag *params)
 375 {
 376         struct mbuf_dynflag_elt *mbuf_dynflag;
 377
 378         if (shm == NULL) {
 379                 rte_errno = ENOENT;
 380                 return -1;
 381         }
 382
 383         rte_mcfg_tailq_read_lock();
 384         mbuf_dynflag = __mbuf_dynflag_lookup(name);
 385         rte_mcfg_tailq_read_unlock();
 386
 387         if (mbuf_dynflag == NULL) {
 388                 rte_errno = ENOENT;
 389                 return -1;
 390         }
 391
 392         if (params != NULL)
 393                 memcpy(params, &mbuf_dynflag->params, sizeof(*params));
 394
 395         return mbuf_dynflag->bitnum;
 396 }
 397
 398 static int mbuf_dynflag_cmp(const struct rte_mbuf_dynflag *params1,
 399                 const struct rte_mbuf_dynflag *params2)
 400 {
 401         if (strcmp(params1->name, params2->name))
 402                 return -1;
 403         if (params1->flags != params2->flags)
 404                 return -1;
 405         return 0;
 406 }
 407
 408 /* assume tailq is locked */
 409 static int
 410 __rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params,
 411                                 unsigned int req)
 412 {
 413         struct mbuf_dynflag_list *mbuf_dynflag_list;
 414         struct mbuf_dynflag_elt *mbuf_dynflag = NULL;
 415         struct rte_tailq_entry *te = NULL;
 416         unsigned int bitnum;
 417         int ret;
 418
 419         if (shm == NULL && init_shared_mem() < 0)
 420                 return -1;
 421
 422         mbuf_dynflag = __mbuf_dynflag_lookup(params->name);
 423         if (mbuf_dynflag != NULL) {
 424                 if (req != UINT_MAX && req != mbuf_dynflag->bitnum) {
 425                         rte_errno = EEXIST;
 426                         return -1;
 427                 }
 428                 if (mbuf_dynflag_cmp(params, &mbuf_dynflag->params) < 0) {
 429                         rte_errno = EEXIST;
 430                         return -1;
 431                 }
 432                 return mbuf_dynflag->bitnum;
 433         }
 434
 435         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 436                 rte_errno = EPERM;
 437                 return -1;
 438         }
 439
 440         if (req == UINT_MAX) {
 441                 if (shm->free_flags == 0) {
 442                         rte_errno = ENOENT;
 443                         return -1;
 444                 }
 445                 bitnum = rte_bsf64(shm->free_flags);
 446         } else {
 447                 if ((shm->free_flags & (1ULL << req)) == 0) {
 448                         rte_errno = EBUSY;
 449                         return -1;
 450                 }
 451                 bitnum = req;
 452         }
 453
 454         mbuf_dynflag_list = RTE_TAILQ_CAST(
 455                 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
 456
 457         te = rte_zmalloc("MBUF_DYNFLAG_TAILQ_ENTRY", sizeof(*te), 0);
 458         if (te == NULL)
 459                 return -1;
 460
 461         mbuf_dynflag = rte_zmalloc("mbuf_dynflag", sizeof(*mbuf_dynflag), 0);
 462         if (mbuf_dynflag == NULL) {
 463                 rte_free(te);
 464                 return -1;
 465         }
 466
 467         ret = strlcpy(mbuf_dynflag->params.name, params->name,
 468                 sizeof(mbuf_dynflag->params.name));
 469         if (ret < 0 || ret >= (int)sizeof(mbuf_dynflag->params.name)) {
 470                 rte_free(mbuf_dynflag);
 471                 rte_free(te);
 472                 rte_errno = ENAMETOOLONG;
 473                 return -1;
 474         }
 475         mbuf_dynflag->bitnum = bitnum;
 476         te->data = mbuf_dynflag;
 477
 478         TAILQ_INSERT_TAIL(mbuf_dynflag_list, te, next);
 479
 480         shm->free_flags &= ~(1ULL << bitnum);
 481
 482         RTE_LOG(DEBUG, MBUF, "Registered dynamic flag %s (fl=0x%x) -> %u\n",
 483                 params->name, params->flags, bitnum);
 484
 485         return bitnum;
 486 }
 487
 488 int
 489 rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params,
 490                                 unsigned int req)
 491 {
 492         int ret;
 493
 494         if (req >= RTE_SIZEOF_FIELD(struct rte_mbuf, ol_flags) * CHAR_BIT &&
 495                         req != UINT_MAX) {
 496                 rte_errno = EINVAL;
 497                 return -1;
 498         }
 499
 500         rte_mcfg_tailq_write_lock();
 501         ret = __rte_mbuf_dynflag_register_bitnum(params, req);
 502         rte_mcfg_tailq_write_unlock();
 503
 504         return ret;
 505 }
 506
 507 int
 508 rte_mbuf_dynflag_register(const struct rte_mbuf_dynflag *params)
 509 {
 510         return rte_mbuf_dynflag_register_bitnum(params, UINT_MAX);
 511 }
 512
 513 void rte_mbuf_dyn_dump(FILE *out)
 514 {
 515         struct mbuf_dynfield_list *mbuf_dynfield_list;
 516         struct mbuf_dynfield_elt *dynfield;
 517         struct mbuf_dynflag_list *mbuf_dynflag_list;
 518         struct mbuf_dynflag_elt *dynflag;
 519         struct rte_tailq_entry *te;
 520         size_t i;
 521
 522         rte_mcfg_tailq_write_lock();
 523         init_shared_mem();
 524         fprintf(out, "Reserved fields:\n");
 525         mbuf_dynfield_list = RTE_TAILQ_CAST(
 526                 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
 527         TAILQ_FOREACH(te, mbuf_dynfield_list, next) {
 528                 dynfield = (struct mbuf_dynfield_elt *)te->data;
 529                 fprintf(out, "  name=%s offset=%zd size=%zd align=%zd flags=%x\n",
 530                         dynfield->params.name, dynfield->offset,
 531                         dynfield->params.size, dynfield->params.align,
 532                         dynfield->params.flags);
 533         }
 534         fprintf(out, "Reserved flags:\n");
 535         mbuf_dynflag_list = RTE_TAILQ_CAST(
 536                 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
 537         TAILQ_FOREACH(te, mbuf_dynflag_list, next) {
 538                 dynflag = (struct mbuf_dynflag_elt *)te->data;
 539                 fprintf(out, "  name=%s bitnum=%u flags=%x\n",
 540                         dynflag->params.name, dynflag->bitnum,
 541                         dynflag->params.flags);
 542         }
 543         fprintf(out, "Free space in mbuf (0 = free, value = zone alignment):\n");
 544         for (i = 0; i < sizeof(struct rte_mbuf); i++) {
 545                 if ((i % 8) == 0)
 546                         fprintf(out, "  %4.4zx: ", i);
 547                 fprintf(out, "%2.2x%s", shm->free_space[i],
 548                         (i % 8 != 7) ? " " : "\n");
 549         }
 550         rte_mcfg_tailq_write_unlock();
 551 }