lib/mbuf/rte_mbuf_dyn.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  * Copyright 2019 6WIND S.A.
   3  */
   4
   5 #include <sys/queue.h>
   6 #include <stdint.h>
   7 #include <limits.h>
   8
   9 #include <rte_common.h>
  10 #include <rte_eal.h>
  11 #include <rte_eal_memconfig.h>
  12 #include <rte_tailq.h>
  13 #include <rte_errno.h>
  14 #include <rte_malloc.h>
  15 #include <rte_string_fns.h>
  16 #include <rte_bitops.h>
  17 #include <rte_mbuf.h>
  18 #include <rte_mbuf_dyn.h>
  19
  20 #define RTE_MBUF_DYN_MZNAME "rte_mbuf_dyn"
  21
  22 struct mbuf_dynfield_elt {
  23         struct rte_mbuf_dynfield params;
  24         size_t offset;
  25 };
  26 TAILQ_HEAD(mbuf_dynfield_list, rte_tailq_entry);
  27
  28 static struct rte_tailq_elem mbuf_dynfield_tailq = {
  29         .name = "RTE_MBUF_DYNFIELD",
  30 };
  31 EAL_REGISTER_TAILQ(mbuf_dynfield_tailq);
  32
  33 struct mbuf_dynflag_elt {
  34         struct rte_mbuf_dynflag params;
  35         unsigned int bitnum;
  36 };
  37 TAILQ_HEAD(mbuf_dynflag_list, rte_tailq_entry);
  38
  39 static struct rte_tailq_elem mbuf_dynflag_tailq = {
  40         .name = "RTE_MBUF_DYNFLAG",
  41 };
  42 EAL_REGISTER_TAILQ(mbuf_dynflag_tailq);
  43
  44 struct mbuf_dyn_shm {
  45         /**
  46          * For each mbuf byte, free_space[i] != 0 if space is free.
  47          * The value is the size of the biggest aligned element that
  48          * can fit in the zone.
  49          */
  50         uint8_t free_space[sizeof(struct rte_mbuf)];
  51         /** Bitfield of available flags. */
  52         uint64_t free_flags;
  53 };
  54 static struct mbuf_dyn_shm *shm;
  55
  56 /* Set the value of free_space[] according to the size and alignment of
  57  * the free areas. This helps to select the best place when reserving a
  58  * dynamic field. Assume tailq is locked.
  59  */
  60 static void
  61 process_score(void)
  62 {
  63         size_t off, align, size, i;
  64
  65         /* first, erase previous info */
  66         for (i = 0; i < sizeof(struct rte_mbuf); i++) {
  67                 if (shm->free_space[i])
  68                         shm->free_space[i] = 1;
  69         }
  70
  71         off = 0;
  72         while (off < sizeof(struct rte_mbuf)) {
  73                 /* get the size of the free zone */
  74                 for (size = 0; (off + size) < sizeof(struct rte_mbuf) &&
  75                              shm->free_space[off + size]; size++)
  76                         ;
  77                 if (size == 0) {
  78                         off++;
  79                         continue;
  80                 }
  81
  82                 /* get the alignment of biggest object that can fit in
  83                  * the zone at this offset.
  84                  */
  85                 for (align = 1;
  86                      (off % (align << 1)) == 0 && (align << 1) <= size;
  87                      align <<= 1)
  88                         ;
  89
  90                 /* save it in free_space[] */
  91                 for (i = off; i < off + align; i++)
  92                         shm->free_space[i] = RTE_MAX(align, shm->free_space[i]);
  93
  94                 off += align;
  95         }
  96 }
  97
  98 /* Mark the area occupied by a mbuf field as available in the shm. */
  99 #define mark_free(field)                                                \
 100         memset(&shm->free_space[offsetof(struct rte_mbuf, field)],      \
 101                 1, sizeof(((struct rte_mbuf *)0)->field))
 102
 103 /* Allocate and initialize the shared memory. Assume tailq is locked */
 104 static int
 105 init_shared_mem(void)
 106 {
 107         const struct rte_memzone *mz;
 108         uint64_t mask;
 109
 110         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 111                 mz = rte_memzone_reserve_aligned(RTE_MBUF_DYN_MZNAME,
 112                                                 sizeof(struct mbuf_dyn_shm),
 113                                                 SOCKET_ID_ANY, 0,
 114                                                 RTE_CACHE_LINE_SIZE);
 115         } else {
 116                 mz = rte_memzone_lookup(RTE_MBUF_DYN_MZNAME);
 117         }
 118         if (mz == NULL) {
 119                 RTE_LOG(ERR, MBUF, "Failed to get mbuf dyn shared memory\n");
 120                 return -1;
 121         }
 122
 123         shm = mz->addr;
 124
 125         if (rte_eal_process_type() == RTE_PROC_PRIMARY) {
 126                 /* init free_space, keep it sync'd with
 127                  * rte_mbuf_dynfield_copy().
 128                  */
 129                 memset(shm, 0, sizeof(*shm));
 130                 mark_free(dynfield1);
 131
 132                 /* init free_flags */
 133                 for (mask = RTE_MBUF_F_FIRST_FREE; mask <= RTE_MBUF_F_LAST_FREE; mask <<= 1)
 134                         shm->free_flags |= mask;
 135
 136                 process_score();
 137         }
 138
 139         return 0;
 140 }
 141
 142 /* check if this offset can be used */
 143 static int
 144 check_offset(size_t offset, size_t size, size_t align)
 145 {
 146         size_t i;
 147
 148         if ((offset & (align - 1)) != 0)
 149                 return -1;
 150         if (offset + size > sizeof(struct rte_mbuf))
 151                 return -1;
 152
 153         for (i = 0; i < size; i++) {
 154                 if (!shm->free_space[i + offset])
 155                         return -1;
 156         }
 157
 158         return 0;
 159 }
 160
 161 /* assume tailq is locked */
 162 static struct mbuf_dynfield_elt *
 163 __mbuf_dynfield_lookup(const char *name)
 164 {
 165         struct mbuf_dynfield_list *mbuf_dynfield_list;
 166         struct mbuf_dynfield_elt *mbuf_dynfield;
 167         struct rte_tailq_entry *te;
 168
 169         mbuf_dynfield_list = RTE_TAILQ_CAST(
 170                 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
 171
 172         TAILQ_FOREACH(te, mbuf_dynfield_list, next) {
 173                 mbuf_dynfield = (struct mbuf_dynfield_elt *)te->data;
 174                 if (strcmp(name, mbuf_dynfield->params.name) == 0)
 175                         break;
 176         }
 177
 178         if (te == NULL || mbuf_dynfield == NULL) {
 179                 rte_errno = ENOENT;
 180                 return NULL;
 181         }
 182
 183         return mbuf_dynfield;
 184 }
 185
 186 int
 187 rte_mbuf_dynfield_lookup(const char *name, struct rte_mbuf_dynfield *params)
 188 {
 189         struct mbuf_dynfield_elt *mbuf_dynfield;
 190
 191         rte_mcfg_tailq_read_lock();
 192         if (shm == NULL && init_shared_mem() < 0)
 193                 mbuf_dynfield = NULL;
 194         else
 195                 mbuf_dynfield = __mbuf_dynfield_lookup(name);
 196         rte_mcfg_tailq_read_unlock();
 197
 198         if (mbuf_dynfield == NULL)
 199                 return -1;
 200
 201         if (params != NULL)
 202                 memcpy(params, &mbuf_dynfield->params, sizeof(*params));
 203
 204         return mbuf_dynfield->offset;
 205 }
 206
 207 static int mbuf_dynfield_cmp(const struct rte_mbuf_dynfield *params1,
 208                 const struct rte_mbuf_dynfield *params2)
 209 {
 210         if (strcmp(params1->name, params2->name))
 211                 return -1;
 212         if (params1->size != params2->size)
 213                 return -1;
 214         if (params1->align != params2->align)
 215                 return -1;
 216         if (params1->flags != params2->flags)
 217                 return -1;
 218         return 0;
 219 }
 220
 221 /* assume tailq is locked */
 222 static int
 223 __rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params,
 224                                 size_t req)
 225 {
 226         struct mbuf_dynfield_list *mbuf_dynfield_list;
 227         struct mbuf_dynfield_elt *mbuf_dynfield = NULL;
 228         struct rte_tailq_entry *te = NULL;
 229         unsigned int best_zone = UINT_MAX;
 230         size_t i, offset;
 231         int ret;
 232
 233         if (shm == NULL && init_shared_mem() < 0)
 234                 return -1;
 235
 236         mbuf_dynfield = __mbuf_dynfield_lookup(params->name);
 237         if (mbuf_dynfield != NULL) {
 238                 if (req != SIZE_MAX && req != mbuf_dynfield->offset) {
 239                         rte_errno = EEXIST;
 240                         return -1;
 241                 }
 242                 if (mbuf_dynfield_cmp(params, &mbuf_dynfield->params) < 0) {
 243                         rte_errno = EEXIST;
 244                         return -1;
 245                 }
 246                 return mbuf_dynfield->offset;
 247         }
 248
 249         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 250                 rte_errno = EPERM;
 251                 return -1;
 252         }
 253
 254         if (req == SIZE_MAX) {
 255                 /* Find the best place to put this field: we search the
 256                  * lowest value of shm->free_space[offset]: the zones
 257                  * containing room for larger fields are kept for later.
 258                  */
 259                 for (offset = 0;
 260                      offset < sizeof(struct rte_mbuf);
 261                      offset++) {
 262                         if (check_offset(offset, params->size,
 263                                                 params->align) == 0 &&
 264                                         shm->free_space[offset] < best_zone) {
 265                                 best_zone = shm->free_space[offset];
 266                                 req = offset;
 267                         }
 268                 }
 269                 if (req == SIZE_MAX) {
 270                         rte_errno = ENOENT;
 271                         return -1;
 272                 }
 273         } else {
 274                 if (check_offset(req, params->size, params->align) < 0) {
 275                         rte_errno = EBUSY;
 276                         return -1;
 277                 }
 278         }
 279
 280         offset = req;
 281         mbuf_dynfield_list = RTE_TAILQ_CAST(
 282                 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
 283
 284         te = rte_zmalloc("MBUF_DYNFIELD_TAILQ_ENTRY", sizeof(*te), 0);
 285         if (te == NULL) {
 286                 rte_errno = ENOMEM;
 287                 return -1;
 288         }
 289
 290         mbuf_dynfield = rte_zmalloc("mbuf_dynfield", sizeof(*mbuf_dynfield), 0);
 291         if (mbuf_dynfield == NULL) {
 292                 rte_free(te);
 293                 rte_errno = ENOMEM;
 294                 return -1;
 295         }
 296
 297         ret = strlcpy(mbuf_dynfield->params.name, params->name,
 298                 sizeof(mbuf_dynfield->params.name));
 299         if (ret < 0 || ret >= (int)sizeof(mbuf_dynfield->params.name)) {
 300                 rte_errno = ENAMETOOLONG;
 301                 rte_free(mbuf_dynfield);
 302                 rte_free(te);
 303                 return -1;
 304         }
 305         memcpy(&mbuf_dynfield->params, params, sizeof(mbuf_dynfield->params));
 306         mbuf_dynfield->offset = offset;
 307         te->data = mbuf_dynfield;
 308
 309         TAILQ_INSERT_TAIL(mbuf_dynfield_list, te, next);
 310
 311         for (i = offset; i < offset + params->size; i++)
 312                 shm->free_space[i] = 0;
 313         process_score();
 314
 315         RTE_LOG(DEBUG, MBUF, "Registered dynamic field %s (sz=%zu, al=%zu, fl=0x%x) -> %zd\n",
 316                 params->name, params->size, params->align, params->flags,
 317                 offset);
 318
 319         return offset;
 320 }
 321
 322 int
 323 rte_mbuf_dynfield_register_offset(const struct rte_mbuf_dynfield *params,
 324                                 size_t req)
 325 {
 326         int ret;
 327
 328         if (params->size >= sizeof(struct rte_mbuf)) {
 329                 rte_errno = EINVAL;
 330                 return -1;
 331         }
 332         if (!rte_is_power_of_2(params->align)) {
 333                 rte_errno = EINVAL;
 334                 return -1;
 335         }
 336         if (params->flags != 0) {
 337                 rte_errno = EINVAL;
 338                 return -1;
 339         }
 340
 341         rte_mcfg_tailq_write_lock();
 342         ret = __rte_mbuf_dynfield_register_offset(params, req);
 343         rte_mcfg_tailq_write_unlock();
 344
 345         return ret;
 346 }
 347
 348 int
 349 rte_mbuf_dynfield_register(const struct rte_mbuf_dynfield *params)
 350 {
 351         return rte_mbuf_dynfield_register_offset(params, SIZE_MAX);
 352 }
 353
 354 /* assume tailq is locked */
 355 static struct mbuf_dynflag_elt *
 356 __mbuf_dynflag_lookup(const char *name)
 357 {
 358         struct mbuf_dynflag_list *mbuf_dynflag_list;
 359         struct mbuf_dynflag_elt *mbuf_dynflag;
 360         struct rte_tailq_entry *te;
 361
 362         mbuf_dynflag_list = RTE_TAILQ_CAST(
 363                 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
 364
 365         TAILQ_FOREACH(te, mbuf_dynflag_list, next) {
 366                 mbuf_dynflag = (struct mbuf_dynflag_elt *)te->data;
 367                 if (strncmp(name, mbuf_dynflag->params.name,
 368                                 RTE_MBUF_DYN_NAMESIZE) == 0)
 369                         break;
 370         }
 371
 372         if (te == NULL) {
 373                 rte_errno = ENOENT;
 374                 return NULL;
 375         }
 376
 377         return mbuf_dynflag;
 378 }
 379
 380 int
 381 rte_mbuf_dynflag_lookup(const char *name,
 382                         struct rte_mbuf_dynflag *params)
 383 {
 384         struct mbuf_dynflag_elt *mbuf_dynflag;
 385
 386         rte_mcfg_tailq_read_lock();
 387         if (shm == NULL && init_shared_mem() < 0)
 388                 mbuf_dynflag = NULL;
 389         else
 390                 mbuf_dynflag = __mbuf_dynflag_lookup(name);
 391         rte_mcfg_tailq_read_unlock();
 392
 393         if (mbuf_dynflag == NULL)
 394                 return -1;
 395
 396         if (params != NULL)
 397                 memcpy(params, &mbuf_dynflag->params, sizeof(*params));
 398
 399         return mbuf_dynflag->bitnum;
 400 }
 401
 402 static int mbuf_dynflag_cmp(const struct rte_mbuf_dynflag *params1,
 403                 const struct rte_mbuf_dynflag *params2)
 404 {
 405         if (strcmp(params1->name, params2->name))
 406                 return -1;
 407         if (params1->flags != params2->flags)
 408                 return -1;
 409         return 0;
 410 }
 411
 412 /* assume tailq is locked */
 413 static int
 414 __rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params,
 415                                 unsigned int req)
 416 {
 417         struct mbuf_dynflag_list *mbuf_dynflag_list;
 418         struct mbuf_dynflag_elt *mbuf_dynflag = NULL;
 419         struct rte_tailq_entry *te = NULL;
 420         unsigned int bitnum;
 421         int ret;
 422
 423         if (shm == NULL && init_shared_mem() < 0)
 424                 return -1;
 425
 426         mbuf_dynflag = __mbuf_dynflag_lookup(params->name);
 427         if (mbuf_dynflag != NULL) {
 428                 if (req != UINT_MAX && req != mbuf_dynflag->bitnum) {
 429                         rte_errno = EEXIST;
 430                         return -1;
 431                 }
 432                 if (mbuf_dynflag_cmp(params, &mbuf_dynflag->params) < 0) {
 433                         rte_errno = EEXIST;
 434                         return -1;
 435                 }
 436                 return mbuf_dynflag->bitnum;
 437         }
 438
 439         if (rte_eal_process_type() != RTE_PROC_PRIMARY) {
 440                 rte_errno = EPERM;
 441                 return -1;
 442         }
 443
 444         if (req == UINT_MAX) {
 445                 if (shm->free_flags == 0) {
 446                         rte_errno = ENOENT;
 447                         return -1;
 448                 }
 449                 bitnum = rte_bsf64(shm->free_flags);
 450         } else {
 451                 if ((shm->free_flags & (1ULL << req)) == 0) {
 452                         rte_errno = EBUSY;
 453                         return -1;
 454                 }
 455                 bitnum = req;
 456         }
 457
 458         mbuf_dynflag_list = RTE_TAILQ_CAST(
 459                 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
 460
 461         te = rte_zmalloc("MBUF_DYNFLAG_TAILQ_ENTRY", sizeof(*te), 0);
 462         if (te == NULL) {
 463                 rte_errno = ENOMEM;
 464                 return -1;
 465         }
 466
 467         mbuf_dynflag = rte_zmalloc("mbuf_dynflag", sizeof(*mbuf_dynflag), 0);
 468         if (mbuf_dynflag == NULL) {
 469                 rte_free(te);
 470                 rte_errno = ENOMEM;
 471                 return -1;
 472         }
 473
 474         ret = strlcpy(mbuf_dynflag->params.name, params->name,
 475                 sizeof(mbuf_dynflag->params.name));
 476         if (ret < 0 || ret >= (int)sizeof(mbuf_dynflag->params.name)) {
 477                 rte_free(mbuf_dynflag);
 478                 rte_free(te);
 479                 rte_errno = ENAMETOOLONG;
 480                 return -1;
 481         }
 482         mbuf_dynflag->bitnum = bitnum;
 483         te->data = mbuf_dynflag;
 484
 485         TAILQ_INSERT_TAIL(mbuf_dynflag_list, te, next);
 486
 487         shm->free_flags &= ~(1ULL << bitnum);
 488
 489         RTE_LOG(DEBUG, MBUF, "Registered dynamic flag %s (fl=0x%x) -> %u\n",
 490                 params->name, params->flags, bitnum);
 491
 492         return bitnum;
 493 }
 494
 495 int
 496 rte_mbuf_dynflag_register_bitnum(const struct rte_mbuf_dynflag *params,
 497                                 unsigned int req)
 498 {
 499         int ret;
 500
 501         if (params->flags != 0) {
 502                 rte_errno = EINVAL;
 503                 return -1;
 504         }
 505         if (req >= RTE_SIZEOF_FIELD(struct rte_mbuf, ol_flags) * CHAR_BIT &&
 506                         req != UINT_MAX) {
 507                 rte_errno = EINVAL;
 508                 return -1;
 509         }
 510
 511         rte_mcfg_tailq_write_lock();
 512         ret = __rte_mbuf_dynflag_register_bitnum(params, req);
 513         rte_mcfg_tailq_write_unlock();
 514
 515         return ret;
 516 }
 517
 518 int
 519 rte_mbuf_dynflag_register(const struct rte_mbuf_dynflag *params)
 520 {
 521         return rte_mbuf_dynflag_register_bitnum(params, UINT_MAX);
 522 }
 523
 524 void rte_mbuf_dyn_dump(FILE *out)
 525 {
 526         struct mbuf_dynfield_list *mbuf_dynfield_list;
 527         struct mbuf_dynfield_elt *dynfield;
 528         struct mbuf_dynflag_list *mbuf_dynflag_list;
 529         struct mbuf_dynflag_elt *dynflag;
 530         struct rte_tailq_entry *te;
 531         size_t i;
 532
 533         rte_mcfg_tailq_write_lock();
 534         if (shm == NULL && init_shared_mem() < 0) {
 535                 rte_mcfg_tailq_write_unlock();
 536                 return;
 537         }
 538
 539         fprintf(out, "Reserved fields:\n");
 540         mbuf_dynfield_list = RTE_TAILQ_CAST(
 541                 mbuf_dynfield_tailq.head, mbuf_dynfield_list);
 542         TAILQ_FOREACH(te, mbuf_dynfield_list, next) {
 543                 dynfield = (struct mbuf_dynfield_elt *)te->data;
 544                 fprintf(out, "  name=%s offset=%zd size=%zd align=%zd flags=%x\n",
 545                         dynfield->params.name, dynfield->offset,
 546                         dynfield->params.size, dynfield->params.align,
 547                         dynfield->params.flags);
 548         }
 549         fprintf(out, "Reserved flags:\n");
 550         mbuf_dynflag_list = RTE_TAILQ_CAST(
 551                 mbuf_dynflag_tailq.head, mbuf_dynflag_list);
 552         TAILQ_FOREACH(te, mbuf_dynflag_list, next) {
 553                 dynflag = (struct mbuf_dynflag_elt *)te->data;
 554                 fprintf(out, "  name=%s bitnum=%u flags=%x\n",
 555                         dynflag->params.name, dynflag->bitnum,
 556                         dynflag->params.flags);
 557         }
 558         fprintf(out, "Free space in mbuf (0 = occupied, value = free zone alignment):\n");
 559         for (i = 0; i < sizeof(struct rte_mbuf); i++) {
 560                 if ((i % 8) == 0)
 561                         fprintf(out, "  %4.4zx: ", i);
 562                 fprintf(out, "%2.2x%s", shm->free_space[i],
 563                         (i % 8 != 7) ? " " : "\n");
 564         }
 565         fprintf(out, "Free bit in mbuf->ol_flags (0 = occupied, 1 = free):\n");
 566         for (i = 0; i < sizeof(uint64_t) * CHAR_BIT; i++) {
 567                 if ((i % 8) == 0)
 568                         fprintf(out, "  %4.4zx: ", i);
 569                 fprintf(out, "%1.1x%s", (shm->free_flags & (1ULL << i)) ? 1 : 0,
 570                         (i % 8 != 7) ? " " : "\n");
 571         }
 572
 573         rte_mcfg_tailq_write_unlock();
 574 }
 575
 576 static int
 577 rte_mbuf_dyn_timestamp_register(int *field_offset, uint64_t *flag,
 578                 const char *direction, const char *flag_name)
 579 {
 580         static const struct rte_mbuf_dynfield field_desc = {
 581                 .name = RTE_MBUF_DYNFIELD_TIMESTAMP_NAME,
 582                 .size = sizeof(rte_mbuf_timestamp_t),
 583                 .align = __alignof__(rte_mbuf_timestamp_t),
 584         };
 585         struct rte_mbuf_dynflag flag_desc = {};
 586         int offset;
 587
 588         offset = rte_mbuf_dynfield_register(&field_desc);
 589         if (offset < 0) {
 590                 RTE_LOG(ERR, MBUF,
 591                         "Failed to register mbuf field for timestamp\n");
 592                 return -1;
 593         }
 594         if (field_offset != NULL)
 595                 *field_offset = offset;
 596
 597         strlcpy(flag_desc.name, flag_name, sizeof(flag_desc.name));
 598         offset = rte_mbuf_dynflag_register(&flag_desc);
 599         if (offset < 0) {
 600                 RTE_LOG(ERR, MBUF,
 601                         "Failed to register mbuf flag for %s timestamp\n",
 602                         direction);
 603                 return -1;
 604         }
 605         if (flag != NULL)
 606                 *flag = RTE_BIT64(offset);
 607
 608         return 0;
 609 }
 610
 611 int
 612 rte_mbuf_dyn_rx_timestamp_register(int *field_offset, uint64_t *rx_flag)
 613 {
 614         return rte_mbuf_dyn_timestamp_register(field_offset, rx_flag,
 615                         "Rx", RTE_MBUF_DYNFLAG_RX_TIMESTAMP_NAME);
 616 }
 617
 618 int
 619 rte_mbuf_dyn_tx_timestamp_register(int *field_offset, uint64_t *tx_flag)
 620 {
 621         return rte_mbuf_dyn_timestamp_register(field_offset, tx_flag,
 622                         "Tx", RTE_MBUF_DYNFLAG_TX_TIMESTAMP_NAME);
 623 }