lib/librte_mbuf/rte_mbuf_dyn.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 6WIND S.A.
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdint.h>
   7 #include <limits.h>
   8
   9 #include <rte_common.h>
  10 #include <rte_eal.h>
  11 #include <rte_eal_memconfig.h>
  12 #include <rte_tailq.h>
  13 #include <rte_errno.h>
  14 #include <rte_malloc.h>
  15 #include <rte_string_fns.h>
  16 #include <rte_mbuf.h>
  17 #include <rte_mbuf_dyn.h>
  18
  19 #define RTE_MBUF_DYN_MZNAME "rte_mbuf_dyn"
  20
  21 struct mbuf_dynfield_elt {
  22         struct rte_mbuf_dynfield params;
  23         size_t offset;
  24 };
  25 TAILQ_HEAD(mbuf_dynfield_list, rte_tailq_entry);
  26
  27 static struct rte_tailq_elem mbuf_dynfield_tailq = {
  28         .name = "RTE_MBUF_DYNFIELD",
  29 };
  30 EAL_REGISTER_TAILQ(mbuf_dynfield_tailq);
  31
  32 struct mbuf_dynflag_elt {
  33         struct rte_mbuf_dynflag params;
  34         unsigned int bitnum;
  35 };
  36 TAILQ_HEAD(mbuf_dynflag_list, rte_tailq_entry);
  37
  38 static struct rte_tailq_elem mbuf_dynflag_tailq = {
  39         .name = "RTE_MBUF_DYNFLAG",
  40 };
  41 EAL_REGISTER_TAILQ(mbuf_dynflag_tailq);
  42
  43 struct mbuf_dyn_shm {
  44         /**
  45          * For each mbuf byte, free_space[i] != 0 if space is free.
  46          * The value is the size of the biggest aligned element that
  47          * can fit in the zone.
  48          */
  49         uint8_t free_space[sizeof(struct rte_mbuf)];
  50         /** Bitfield of available flags. */
  51         uint64_t free_flags;
  52 };
  53 static struct mbuf_dyn_shm *shm;
  54
  55 /* Set the value of free_space[] according to the size and alignment of
  56  * the free areas. This helps to select the best place when reserving a
  57  * dynamic field. Assume tailq is locked.
  58  */
  59 static void
  60 process_score(void)
  61 {
  62         size_t off, align, size, i;
  63
  64         /* first, erase previous info */
  65         for (i = 0; i < sizeof(struct rte_mbuf); i++) {
  66                 if (shm->free_space[i])
  67                         shm->free_space[i] = 1;
  68         }
  69
  70         for (off = 0; off < sizeof(struct rte_mbuf); off++) {
  71                 /* get the size of the free zone */
  72                 for (size = 0; (off + size) < sizeof(struct rte_mbuf) &&
  73                              shm->free_space[off + size]; size++)
  74                         ;
  75                 if (size == 0)
  76                         continue;
  77
  78                 /* get the alignment of biggest object that can fit in
  79                  * the zone at this offset.
  80                  */
  81                 for (align = 1;
  82                      (off % (align << 1)) == 0 && (align << 1) <= size;
  83                      align <<= 1)
  84                         ;
  85
  86                 /* save it in free_space[] */
  87                 for (i = off; i < off + size; i++)
  88                         shm->free_space[i] = RTE_MAX(align, shm->free_space[i]);
  89         }
  90 }
  91
  92 /* Mark the area occupied by a mbuf field as available in the shm. */
  93 #define mark_free(field)                                                \
  94         memset(&shm->free_space[offsetof(struct rte_mbuf, field)],      \
  95                 1, sizeof(((struct rte_mbuf *)0)->field))
  96
  97 /* Allocate and initialize the shared memory. Assume tailq is locked */
  98 static int
  99 init_shared_mem(void)
 100 {
 101         const struct rte_memzone *mz;
 102         uint64_t mask;
 103
 104         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 105                 mz = rte_memzone_reserve_aligned(RTE_MBUF_DYN_MZNAME,
 106                                                 sizeof(struct mbuf_dyn_shm),
 107                                                 SOCKET_ID_ANY, 0,
 108                                                 RTE_CACHE_LINE_SIZE);
 109         } else {
 110                 mz = rte_memzone_lookup(RTE_MBUF_DYN_MZNAME);
 111         }
 112         if (mz == NULL)
 113                 return -1;
 114
 115         shm = mz->addr;
 116
 117         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 118                 /* init free_space, keep it sync'd with
 119                  * rte_mbuf_dynfield_copy().
 120                  */
 121                 memset(shm, 0, sizeof(*shm));
 122                 mark_free(dynfield1);
 123
 124                 /* init free_flags */
 125                 for (mask = PKT_FIRST_FREE; mask <= PKT_LAST_FREE; mask <<= 1)
 126                         shm->free_flags |= mask;
 127
 128                 process_score();
 129         }
 130
 131         return 0;
 132 }
 133
 134 /* check if this offset can be used */
 135 static int
 136 check_offset(size_t offset, size_t size, size_t align)
 137 {
 138         size_t i;
 139
 140         if ((offset & (align - 1)) != 0)
 141                 return -1;
 142         if (offset + size > sizeof(struct rte_mbuf))
 143                 return -1;
 144
 145         for (i = 0; i < size; i++) {
 146                 if (!shm->free_space[i + offset])
 147                         return -1;
 148         }
 149
 150         return 0;
 151 }
 152
 153 /* assume tailq is locked */
 154 static struct mbuf_dynfield_elt *
 155 __mbuf_dynfield_lookup(const char *name)
 156 {
 157         struct mbuf_dynfield_list *mbuf_dynfield_list;
 158         struct mbuf_dynfield_elt *mbuf_dynfield;
 159         struct rte_tailq_entry *te;
 160
 161         mbuf_dynfield_list = RTE_TAILQ_CAST(
 162                 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
 163
 164         TAILQ_FOREACH(te, mbuf_dynfield_list, next) {
 165                 mbuf_dynfield = (struct mbuf_dynfield_elt *)te->data;
 166                 if (strcmp(name, mbuf_dynfield->params.name) == 0)
 167                         break;
 168         }
 169
 170         if (te == NULL) {
 171                 rte_errno = ENOENT;
 172                 return NULL;
 173         }
 174
 175         return mbuf_dynfield;
 176 }
 177
 178 int
 179 rte_mbuf_dynfield_lookup(const char *name, struct rte_mbuf_dynfield *params)
 180 {
 181         struct mbuf_dynfield_elt *mbuf_dynfield;
 182
 183         if (shm == NULL) {
 184                 rte_errno = ENOENT;
 185                 return -1;
 186         }
 187
 188         rte_mcfg_tailq_read_lock();
 189         mbuf_dynfield = __mbuf_dynfield_lookup(name);
 190         rte_mcfg_tailq_read_unlock();
 191
 192         if (mbuf_dynfield == NULL) {
 193                 rte_errno = ENOENT;
 194                 return -1;
 195         }
 196
 197         if (params != NULL)
 198                 memcpy(params, &mbuf_dynfield->params, sizeof(*params));
 199
 200         return mbuf_dynfield->offset;
 201 }
 202
 203 static int mbuf_dynfield_cmp(const struct rte_mbuf_dynfield *params1,
 204                 const struct rte_mbuf_dynfield *params2)
 205 {
 206         if (strcmp(params1->name, params2->name))
 207                 return -1;
 208         if (params1->size != params2->size)
 209                 return -1;
 210         if (params1->align != params2->align)
 211                 return -1;
 212         if (params1->flags != params2->flags)
 213                 return -1;
 214         return 0;
 215 }
 216
 217 /* assume tailq is locked */
 218 static int
 219 __rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params,
 220                                 size_t req)
 221 {
 222         struct mbuf_dynfield_list *mbuf_dynfield_list;
 223         struct mbuf_dynfield_elt *mbuf_dynfield = NULL;
 224         struct rte_tailq_entry *te = NULL;
 225         unsigned int best_zone = UINT_MAX;
 226         size_t i, offset;
 227         int ret;
 228
 229         if (shm == NULL && init_shared_mem() < 0)
 230                 return -1;
 231
 232         mbuf_dynfield = __mbuf_dynfield_lookup(params->name);
 233         if (mbuf_dynfield != NULL) {
 234                 if (req != SIZE_MAX && req != mbuf_dynfield->offset) {
 235                         rte_errno = EEXIST;
 236                         return -1;
 237                 }
 238                 if (mbuf_dynfield_cmp(params, &mbuf_dynfield->params) < 0) {
 239                         rte_errno = EEXIST;
 240                         return -1;
 241                 }
 242                 return mbuf_dynfield->offset;
 243         }
 244
 245         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 246                 rte_errno = EPERM;
 247                 return -1;
 248         }
 249
 250         if (req == SIZE_MAX) {
 251                 /* Find the best place to put this field: we search the
 252                  * lowest value of shm->free_space[offset]: the zones
 253                  * containing room for larger fields are kept for later.
 254                  */
 255                 for (offset = 0;
 256                      offset < sizeof(struct rte_mbuf);
 257                      offset++) {
 258                         if (check_offset(offset, params->size,
 259                                                 params->align) == 0 &&
 260                                         shm->free_space[offset] < best_zone) {
 261                                 best_zone = shm->free_space[offset];
 262                                 req = offset;
 263                         }
 264                 }
 265                 if (req == SIZE_MAX) {
 266                         rte_errno = ENOENT;
 267                         return -1;
 268                 }
 269         } else {
 270                 if (check_offset(req, params->size, params->align) < 0) {
 271                         rte_errno = EBUSY;
 272                         return -1;
 273                 }
 274         }
 275
 276         offset = req;
 277         mbuf_dynfield_list = RTE_TAILQ_CAST(
 278                 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
 279
 280         te = rte_zmalloc("MBUF_DYNFIELD_TAILQ_ENTRY", sizeof(*te), 0);
 281         if (te == NULL) {
 282                 rte_errno = ENOMEM;
 283                 return -1;
 284         }
 285
 286         mbuf_dynfield = rte_zmalloc("mbuf_dynfield", sizeof(*mbuf_dynfield), 0);
 287         if (mbuf_dynfield == NULL) {
 288                 rte_free(te);
 289                 rte_errno = ENOMEM;
 290                 return -1;
 291         }
 292
 293         ret = strlcpy(mbuf_dynfield->params.name, params->name,
 294                 sizeof(mbuf_dynfield->params.name));
 295         if (ret < 0 || ret >= (int)sizeof(mbuf_dynfield->params.name)) {
 296                 rte_errno = ENAMETOOLONG;
 297                 rte_free(mbuf_dynfield);
 298                 rte_free(te);
 299                 return -1;
 300         }
 301         memcpy(&mbuf_dynfield->params, params, sizeof(mbuf_dynfield->params));
 302         mbuf_dynfield->offset = offset;
 303         te->data = mbuf_dynfield;
 304
 305         TAILQ_INSERT_TAIL(mbuf_dynfield_list, te, next);
 306
 307         for (i = offset; i < offset + params->size; i++)
 308                 shm->free_space[i] = 0;
 309         process_score();
 310
 311         RTE_LOG(DEBUG, MBUF, "Registered dynamic field %s (sz=%zu, al=%zu, fl=0x%x) -> %zd\n",
 312                 params->name, params->size, params->align, params->flags,
 313                 offset);
 314
 315         return offset;
 316 }
 317
 318 int
 319 rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params,
 320                                 size_t req)
 321 {
 322         int ret;
 323
 324         if (params->size >= sizeof(struct rte_mbuf)) {
 325                 rte_errno = EINVAL;
 326                 return -1;
 327         }
 328         if (!rte_is_power_of_2(params->align)) {
 329                 rte_errno = EINVAL;
 330                 return -1;
 331         }
 332         if (params->flags != 0) {
 333                 rte_errno = EINVAL;
 334                 return -1;
 335         }
 336
 337         rte_mcfg_tailq_write_lock();
 338         ret = __rte_mbuf_dynfield_register_offset(params, req);
 339         rte_mcfg_tailq_write_unlock();
 340
 341         return ret;
 342 }
 343
 344 int
 345 rte_mbuf_dynfield_register(const struct rte_mbuf_dynfield *params)
 346 {
 347         return rte_mbuf_dynfield_register_offset(params, SIZE_MAX);
 348 }
 349
 350 /* assume tailq is locked */
 351 static struct mbuf_dynflag_elt *
 352 __mbuf_dynflag_lookup(const char *name)
 353 {
 354         struct mbuf_dynflag_list *mbuf_dynflag_list;
 355         struct mbuf_dynflag_elt *mbuf_dynflag;
 356         struct rte_tailq_entry *te;
 357
 358         mbuf_dynflag_list = RTE_TAILQ_CAST(
 359                 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
 360
 361         TAILQ_FOREACH(te, mbuf_dynflag_list, next) {
 362                 mbuf_dynflag = (struct mbuf_dynflag_elt *)te->data;
 363                 if (strncmp(name, mbuf_dynflag->params.name,
 364                                 RTE_MBUF_DYN_NAMESIZE) == 0)
 365                         break;
 366         }
 367
 368         if (te == NULL) {
 369                 rte_errno = ENOENT;
 370                 return NULL;
 371         }
 372
 373         return mbuf_dynflag;
 374 }
 375
 376 int
 377 rte_mbuf_dynflag_lookup(const char *name,
 378                         struct rte_mbuf_dynflag *params)
 379 {
 380         struct mbuf_dynflag_elt *mbuf_dynflag;
 381
 382         if (shm == NULL) {
 383                 rte_errno = ENOENT;
 384                 return -1;
 385         }
 386
 387         rte_mcfg_tailq_read_lock();
 388         mbuf_dynflag = __mbuf_dynflag_lookup(name);
 389         rte_mcfg_tailq_read_unlock();
 390
 391         if (mbuf_dynflag == NULL) {
 392                 rte_errno = ENOENT;
 393                 return -1;
 394         }
 395
 396         if (params != NULL)
 397                 memcpy(params, &mbuf_dynflag->params, sizeof(*params));
 398
 399         return mbuf_dynflag->bitnum;
 400 }
 401
 402 static int mbuf_dynflag_cmp(const struct rte_mbuf_dynflag *params1,
 403                 const struct rte_mbuf_dynflag *params2)
 404 {
 405         if (strcmp(params1->name, params2->name))
 406                 return -1;
 407         if (params1->flags != params2->flags)
 408                 return -1;
 409         return 0;
 410 }
 411
 412 /* assume tailq is locked */
 413 static int
 414 __rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params,
 415                                 unsigned int req)
 416 {
 417         struct mbuf_dynflag_list *mbuf_dynflag_list;
 418         struct mbuf_dynflag_elt *mbuf_dynflag = NULL;
 419         struct rte_tailq_entry *te = NULL;
 420         unsigned int bitnum;
 421         int ret;
 422
 423         if (shm == NULL && init_shared_mem() < 0)
 424                 return -1;
 425
 426         mbuf_dynflag = __mbuf_dynflag_lookup(params->name);
 427         if (mbuf_dynflag != NULL) {
 428                 if (req != UINT_MAX && req != mbuf_dynflag->bitnum) {
 429                         rte_errno = EEXIST;
 430                         return -1;
 431                 }
 432                 if (mbuf_dynflag_cmp(params, &mbuf_dynflag->params) < 0) {
 433                         rte_errno = EEXIST;
 434                         return -1;
 435                 }
 436                 return mbuf_dynflag->bitnum;
 437         }
 438
 439         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 440                 rte_errno = EPERM;
 441                 return -1;
 442         }
 443
 444         if (req == UINT_MAX) {
 445                 if (shm->free_flags == 0) {
 446                         rte_errno = ENOENT;
 447                         return -1;
 448                 }
 449                 bitnum = rte_bsf64(shm->free_flags);
 450         } else {
 451                 if ((shm->free_flags & (1ULL << req)) == 0) {
 452                         rte_errno = EBUSY;
 453                         return -1;
 454                 }
 455                 bitnum = req;
 456         }
 457
 458         mbuf_dynflag_list = RTE_TAILQ_CAST(
 459                 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
 460
 461         te = rte_zmalloc("MBUF_DYNFLAG_TAILQ_ENTRY", sizeof(*te), 0);
 462         if (te == NULL) {
 463                 rte_errno = ENOMEM;
 464                 return -1;
 465         }
 466
 467         mbuf_dynflag = rte_zmalloc("mbuf_dynflag", sizeof(*mbuf_dynflag), 0);
 468         if (mbuf_dynflag == NULL) {
 469                 rte_free(te);
 470                 rte_errno = ENOMEM;
 471                 return -1;
 472         }
 473
 474         ret = strlcpy(mbuf_dynflag->params.name, params->name,
 475                 sizeof(mbuf_dynflag->params.name));
 476         if (ret < 0 || ret >= (int)sizeof(mbuf_dynflag->params.name)) {
 477                 rte_free(mbuf_dynflag);
 478                 rte_free(te);
 479                 rte_errno = ENAMETOOLONG;
 480                 return -1;
 481         }
 482         mbuf_dynflag->bitnum = bitnum;
 483         te->data = mbuf_dynflag;
 484
 485         TAILQ_INSERT_TAIL(mbuf_dynflag_list, te, next);
 486
 487         shm->free_flags &= ~(1ULL << bitnum);
 488
 489         RTE_LOG(DEBUG, MBUF, "Registered dynamic flag %s (fl=0x%x) -> %u\n",
 490                 params->name, params->flags, bitnum);
 491
 492         return bitnum;
 493 }
 494
 495 int
 496 rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params,
 497                                 unsigned int req)
 498 {
 499         int ret;
 500
 501         if (req >= RTE_SIZEOF_FIELD(struct rte_mbuf, ol_flags) * CHAR_BIT &&
 502                         req != UINT_MAX) {
 503                 rte_errno = EINVAL;
 504                 return -1;
 505         }
 506
 507         rte_mcfg_tailq_write_lock();
 508         ret = __rte_mbuf_dynflag_register_bitnum(params, req);
 509         rte_mcfg_tailq_write_unlock();
 510
 511         return ret;
 512 }
 513
 514 int
 515 rte_mbuf_dynflag_register(const struct rte_mbuf_dynflag *params)
 516 {
 517         return rte_mbuf_dynflag_register_bitnum(params, UINT_MAX);
 518 }
 519
 520 void rte_mbuf_dyn_dump(FILE *out)
 521 {
 522         struct mbuf_dynfield_list *mbuf_dynfield_list;
 523         struct mbuf_dynfield_elt *dynfield;
 524         struct mbuf_dynflag_list *mbuf_dynflag_list;
 525         struct mbuf_dynflag_elt *dynflag;
 526         struct rte_tailq_entry *te;
 527         size_t i;
 528
 529         rte_mcfg_tailq_write_lock();
 530         init_shared_mem();
 531         fprintf(out, "Reserved fields:\n");
 532         mbuf_dynfield_list = RTE_TAILQ_CAST(
 533                 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
 534         TAILQ_FOREACH(te, mbuf_dynfield_list, next) {
 535                 dynfield = (struct mbuf_dynfield_elt *)te->data;
 536                 fprintf(out, "  name=%s offset=%zd size=%zd align=%zd flags=%x\n",
 537                         dynfield->params.name, dynfield->offset,
 538                         dynfield->params.size, dynfield->params.align,
 539                         dynfield->params.flags);
 540         }
 541         fprintf(out, "Reserved flags:\n");
 542         mbuf_dynflag_list = RTE_TAILQ_CAST(
 543                 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
 544         TAILQ_FOREACH(te, mbuf_dynflag_list, next) {
 545                 dynflag = (struct mbuf_dynflag_elt *)te->data;
 546                 fprintf(out, "  name=%s bitnum=%u flags=%x\n",
 547                         dynflag->params.name, dynflag->bitnum,
 548                         dynflag->params.flags);
 549         }
 550         fprintf(out, "Free space in mbuf (0 = free, value = zone alignment):\n");
 551         for (i = 0; i < sizeof(struct rte_mbuf); i++) {
 552                 if ((i % 8) == 0)
 553                         fprintf(out, "  %4.4zx: ", i);
 554                 fprintf(out, "%2.2x%s", shm->free_space[i],
 555                         (i % 8 != 7) ? " " : "\n");
 556         }
 557         rte_mcfg_tailq_write_unlock();
 558 }