drivers/net/sfc/sfc_mae_counter.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright(c) 2020-2021 Xilinx, Inc.
   4  */
   5
   6 #include <rte_common.h>
   7 #include <rte_service_component.h>
   8
   9 #include "efx.h"
  10 #include "efx_regs_counters_pkt_format.h"
  11
  12 #include "sfc_ev.h"
  13 #include "sfc.h"
  14 #include "sfc_rx.h"
  15 #include "sfc_mae_counter.h"
  16 #include "sfc_service.h"
  17
  18 static uint32_t
  19 sfc_mae_counter_get_service_lcore(struct sfc_adapter *sa)
  20 {
  21         uint32_t cid;
  22
  23         cid = sfc_get_service_lcore(sa->socket_id);
  24         if (cid != RTE_MAX_LCORE)
  25                 return cid;
  26
  27         if (sa->socket_id != SOCKET_ID_ANY)
  28                 cid = sfc_get_service_lcore(SOCKET_ID_ANY);
  29
  30         if (cid == RTE_MAX_LCORE) {
  31                 sfc_warn(sa, "failed to get service lcore for counter service");
  32         } else if (sa->socket_id != SOCKET_ID_ANY) {
  33                 sfc_warn(sa,
  34                         "failed to get service lcore for counter service at socket %d, but got at socket %u",
  35                         sa->socket_id, rte_lcore_to_socket_id(cid));
  36         }
  37         return cid;
  38 }
  39
  40 bool
  41 sfc_mae_counter_rxq_required(struct sfc_adapter *sa)
  42 {
  43         const efx_nic_cfg_t *encp = efx_nic_cfg_get(sa->nic);
  44
  45         if (encp->enc_mae_supported == B_FALSE)
  46                 return false;
  47
  48         if (sfc_mae_counter_get_service_lcore(sa) == RTE_MAX_LCORE)
  49                 return false;
  50
  51         return true;
  52 }
  53
  54 int
  55 sfc_mae_counter_enable(struct sfc_adapter *sa,
  56                        struct sfc_mae_counter_id *counterp)
  57 {
  58         struct sfc_mae_counter_registry *reg = &sa->mae.counter_registry;
  59         struct sfc_mae_counters *counters = &reg->counters;
  60         struct sfc_mae_counter *p;
  61         efx_counter_t mae_counter;
  62         uint32_t generation_count;
  63         uint32_t unused;
  64         int rc;
  65
  66         /*
  67          * The actual count of counters allocated is ignored since a failure
  68          * to allocate a single counter is indicated by non-zero return code.
  69          */
  70         rc = efx_mae_counters_alloc(sa->nic, 1, &unused, &mae_counter,
  71                                     &generation_count);
  72         if (rc != 0) {
  73                 sfc_err(sa, "failed to alloc MAE counter: %s",
  74                         rte_strerror(rc));
  75                 goto fail_mae_counter_alloc;
  76         }
  77
  78         if (mae_counter.id >= counters->n_mae_counters) {
  79                 /*
  80                  * ID of a counter is expected to be within the range
  81                  * between 0 and the maximum count of counters to always
  82                  * fit into a pre-allocated array size of maximum counter ID.
  83                  */
  84                 sfc_err(sa, "MAE counter ID is out of expected range");
  85                 rc = EFAULT;
  86                 goto fail_counter_id_range;
  87         }
  88
  89         counterp->mae_id = mae_counter;
  90
  91         p = &counters->mae_counters[mae_counter.id];
  92
  93         /*
  94          * Ordering is relaxed since it is the only operation on counter value.
  95          * And it does not depend on different stores/loads in other threads.
  96          * Paired with relaxed ordering in counter increment.
  97          */
  98         __atomic_store(&p->reset.pkts_bytes.int128,
  99                        &p->value.pkts_bytes.int128, __ATOMIC_RELAXED);
 100         p->generation_count = generation_count;
 101
 102         /*
 103          * The flag is set at the very end of add operation and reset
 104          * at the beginning of delete operation. Release ordering is
 105          * paired with acquire ordering on load in counter increment operation.
 106          */
 107         __atomic_store_n(&p->inuse, true, __ATOMIC_RELEASE);
 108
 109         sfc_info(sa, "enabled MAE counter #%u with reset pkts=%" PRIu64
 110                  " bytes=%" PRIu64, mae_counter.id,
 111                  p->reset.pkts, p->reset.bytes);
 112
 113         return 0;
 114
 115 fail_counter_id_range:
 116         (void)efx_mae_counters_free(sa->nic, 1, &unused, &mae_counter, NULL);
 117
 118 fail_mae_counter_alloc:
 119         sfc_log_init(sa, "failed: %s", rte_strerror(rc));
 120         return rc;
 121 }
 122
 123 int
 124 sfc_mae_counter_disable(struct sfc_adapter *sa,
 125                         struct sfc_mae_counter_id *counter)
 126 {
 127         struct sfc_mae_counter_registry *reg = &sa->mae.counter_registry;
 128         struct sfc_mae_counters *counters = &reg->counters;
 129         struct sfc_mae_counter *p;
 130         uint32_t unused;
 131         int rc;
 132
 133         if (counter->mae_id.id == EFX_MAE_RSRC_ID_INVALID)
 134                 return 0;
 135
 136         SFC_ASSERT(counter->mae_id.id < counters->n_mae_counters);
 137         /*
 138          * The flag is set at the very end of add operation and reset
 139          * at the beginning of delete operation. Release ordering is
 140          * paired with acquire ordering on load in counter increment operation.
 141          */
 142         p = &counters->mae_counters[counter->mae_id.id];
 143         __atomic_store_n(&p->inuse, false, __ATOMIC_RELEASE);
 144
 145         rc = efx_mae_counters_free(sa->nic, 1, &unused, &counter->mae_id, NULL);
 146         if (rc != 0)
 147                 sfc_err(sa, "failed to free MAE counter %u: %s",
 148                         counter->mae_id.id, rte_strerror(rc));
 149
 150         sfc_info(sa, "disabled MAE counter #%u with reset pkts=%" PRIu64
 151                  " bytes=%" PRIu64, counter->mae_id.id,
 152                  p->reset.pkts, p->reset.bytes);
 153
 154         /*
 155          * Do this regardless of what efx_mae_counters_free() return value is.
 156          * If there's some error, the resulting resource leakage is bad, but
 157          * nothing sensible can be done in this case.
 158          */
 159         counter->mae_id.id = EFX_MAE_RSRC_ID_INVALID;
 160
 161         return rc;
 162 }
 163
 164 static void
 165 sfc_mae_counter_increment(struct sfc_adapter *sa,
 166                           struct sfc_mae_counters *counters,
 167                           uint32_t mae_counter_id,
 168                           uint32_t generation_count,
 169                           uint64_t pkts, uint64_t bytes)
 170 {
 171         struct sfc_mae_counter *p = &counters->mae_counters[mae_counter_id];
 172         struct sfc_mae_counters_xstats *xstats = &counters->xstats;
 173         union sfc_pkts_bytes cnt_val;
 174         bool inuse;
 175
 176         /*
 177          * Acquire ordering is paired with release ordering in counter add
 178          * and delete operations.
 179          */
 180         __atomic_load(&p->inuse, &inuse, __ATOMIC_ACQUIRE);
 181         if (!inuse) {
 182                 /*
 183                  * Two possible cases include:
 184                  * 1) Counter is just allocated. Too early counter update
 185                  *    cannot be processed properly.
 186                  * 2) Stale update of freed and not reallocated counter.
 187                  *    There is no point in processing that update.
 188                  */
 189                 xstats->not_inuse_update++;
 190                 return;
 191         }
 192
 193         if (unlikely(generation_count < p->generation_count)) {
 194                 /*
 195                  * It is a stale update for the reallocated counter
 196                  * (i.e., freed and the same ID allocated again).
 197                  */
 198                 xstats->realloc_update++;
 199                 return;
 200         }
 201
 202         cnt_val.pkts = p->value.pkts + pkts;
 203         cnt_val.bytes = p->value.bytes + bytes;
 204
 205         /*
 206          * Ordering is relaxed since it is the only operation on counter value.
 207          * And it does not depend on different stores/loads in other threads.
 208          * Paired with relaxed ordering on counter reset.
 209          */
 210         __atomic_store(&p->value.pkts_bytes,
 211                        &cnt_val.pkts_bytes, __ATOMIC_RELAXED);
 212
 213         sfc_info(sa, "update MAE counter #%u: pkts+%" PRIu64 "=%" PRIu64
 214                  ", bytes+%" PRIu64 "=%" PRIu64, mae_counter_id,
 215                  pkts, cnt_val.pkts, bytes, cnt_val.bytes);
 216 }
 217
 218 static void
 219 sfc_mae_parse_counter_packet(struct sfc_adapter *sa,
 220                              struct sfc_mae_counter_registry *counter_registry,
 221                              const struct rte_mbuf *m)
 222 {
 223         uint32_t generation_count;
 224         const efx_xword_t *hdr;
 225         const efx_oword_t *counters_data;
 226         unsigned int version;
 227         unsigned int id;
 228         unsigned int header_offset;
 229         unsigned int payload_offset;
 230         unsigned int counter_count;
 231         unsigned int required_len;
 232         unsigned int i;
 233
 234         if (unlikely(m->nb_segs != 1)) {
 235                 sfc_err(sa, "unexpectedly scattered MAE counters packet (%u segments)",
 236                         m->nb_segs);
 237                 return;
 238         }
 239
 240         if (unlikely(m->data_len < ER_RX_SL_PACKETISER_HEADER_WORD_SIZE)) {
 241                 sfc_err(sa, "too short MAE counters packet (%u bytes)",
 242                         m->data_len);
 243                 return;
 244         }
 245
 246         /*
 247          * The generation count is located in the Rx prefix in the USER_MARK
 248          * field which is written into hash.fdir.hi field of an mbuf. See
 249          * SF-123581-TC SmartNIC Datapath Offloads section 4.7.5 Counters.
 250          */
 251         generation_count = m->hash.fdir.hi;
 252
 253         hdr = rte_pktmbuf_mtod(m, const efx_xword_t *);
 254
 255         version = EFX_XWORD_FIELD(*hdr, ERF_SC_PACKETISER_HEADER_VERSION);
 256         if (unlikely(version != ERF_SC_PACKETISER_HEADER_VERSION_2)) {
 257                 sfc_err(sa, "unexpected MAE counters packet version %u",
 258                         version);
 259                 return;
 260         }
 261
 262         id = EFX_XWORD_FIELD(*hdr, ERF_SC_PACKETISER_HEADER_IDENTIFIER);
 263         if (unlikely(id != ERF_SC_PACKETISER_HEADER_IDENTIFIER_AR)) {
 264                 sfc_err(sa, "unexpected MAE counters source identifier %u", id);
 265                 return;
 266         }
 267
 268         /* Packet layout definitions assume fixed header offset in fact */
 269         header_offset =
 270                 EFX_XWORD_FIELD(*hdr, ERF_SC_PACKETISER_HEADER_HEADER_OFFSET);
 271         if (unlikely(header_offset !=
 272                      ERF_SC_PACKETISER_HEADER_HEADER_OFFSET_DEFAULT)) {
 273                 sfc_err(sa, "unexpected MAE counters packet header offset %u",
 274                         header_offset);
 275                 return;
 276         }
 277
 278         payload_offset =
 279                 EFX_XWORD_FIELD(*hdr, ERF_SC_PACKETISER_HEADER_PAYLOAD_OFFSET);
 280
 281         counter_count = EFX_XWORD_FIELD(*hdr, ERF_SC_PACKETISER_HEADER_COUNT);
 282
 283         required_len = payload_offset +
 284                         counter_count * sizeof(counters_data[0]);
 285         if (unlikely(required_len > m->data_len)) {
 286                 sfc_err(sa, "truncated MAE counters packet: %u counters, packet length is %u vs %u required",
 287                         counter_count, m->data_len, required_len);
 288                 /*
 289                  * In theory it is possible process available counters data,
 290                  * but such condition is really unexpected and it is
 291                  * better to treat entire packet as corrupted.
 292                  */
 293                 return;
 294         }
 295
 296         /* Ensure that counters data is 32-bit aligned */
 297         if (unlikely(payload_offset % sizeof(uint32_t) != 0)) {
 298                 sfc_err(sa, "unsupported MAE counters payload offset %u, must be 32-bit aligned",
 299                         payload_offset);
 300                 return;
 301         }
 302         RTE_BUILD_BUG_ON(sizeof(counters_data[0]) !=
 303                         ER_RX_SL_PACKETISER_PAYLOAD_WORD_SIZE);
 304
 305         counters_data =
 306                 rte_pktmbuf_mtod_offset(m, const efx_oword_t *, payload_offset);
 307
 308         sfc_info(sa, "update %u MAE counters with gc=%u",
 309                  counter_count, generation_count);
 310
 311         for (i = 0; i < counter_count; ++i) {
 312                 uint32_t packet_count_lo;
 313                 uint32_t packet_count_hi;
 314                 uint32_t byte_count_lo;
 315                 uint32_t byte_count_hi;
 316
 317                 /*
 318                  * Use 32-bit field accessors below since counters data
 319                  * is not 64-bit aligned.
 320                  * 32-bit alignment is checked above taking into account
 321                  * that start of packet data is 32-bit aligned
 322                  * (cache-line size aligned in fact).
 323                  */
 324                 packet_count_lo =
 325                         EFX_OWORD_FIELD32(counters_data[i],
 326                                 ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_LO);
 327                 packet_count_hi =
 328                         EFX_OWORD_FIELD32(counters_data[i],
 329                                 ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_HI);
 330                 byte_count_lo =
 331                         EFX_OWORD_FIELD32(counters_data[i],
 332                                 ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_LO);
 333                 byte_count_hi =
 334                         EFX_OWORD_FIELD32(counters_data[i],
 335                                 ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_HI);
 336                 sfc_mae_counter_increment(sa,
 337                         &counter_registry->counters,
 338                         EFX_OWORD_FIELD32(counters_data[i],
 339                                 ERF_SC_PACKETISER_PAYLOAD_COUNTER_INDEX),
 340                         generation_count,
 341                         (uint64_t)packet_count_lo |
 342                         ((uint64_t)packet_count_hi <<
 343                          ERF_SC_PACKETISER_PAYLOAD_PACKET_COUNT_LO_WIDTH),
 344                         (uint64_t)byte_count_lo |
 345                         ((uint64_t)byte_count_hi <<
 346                          ERF_SC_PACKETISER_PAYLOAD_BYTE_COUNT_LO_WIDTH));
 347         }
 348 }
 349
 350 static int32_t
 351 sfc_mae_counter_routine(void *arg)
 352 {
 353         struct sfc_adapter *sa = arg;
 354         struct sfc_mae_counter_registry *counter_registry =
 355                 &sa->mae.counter_registry;
 356         struct rte_mbuf *mbufs[SFC_MAE_COUNTER_RX_BURST];
 357         unsigned int pushed_diff;
 358         unsigned int pushed;
 359         unsigned int i;
 360         uint16_t n;
 361         int rc;
 362
 363         n = counter_registry->rx_pkt_burst(counter_registry->rx_dp, mbufs,
 364                                            SFC_MAE_COUNTER_RX_BURST);
 365
 366         for (i = 0; i < n; i++)
 367                 sfc_mae_parse_counter_packet(sa, counter_registry, mbufs[i]);
 368
 369         rte_pktmbuf_free_bulk(mbufs, n);
 370
 371         if (!counter_registry->use_credits)
 372                 return 0;
 373
 374         pushed = sfc_rx_get_pushed(sa, counter_registry->rx_dp);
 375         pushed_diff = pushed - counter_registry->pushed_n_buffers;
 376
 377         if (pushed_diff >= SFC_COUNTER_RXQ_REFILL_LEVEL) {
 378                 rc = efx_mae_counters_stream_give_credits(sa->nic, pushed_diff);
 379                 if (rc == 0) {
 380                         counter_registry->pushed_n_buffers = pushed;
 381                 } else {
 382                         /*
 383                          * FIXME: counters might be important for the
 384                          * application. Handle the error in order to recover
 385                          * from the failure
 386                          */
 387                         SFC_GENERIC_LOG(DEBUG, "Give credits failed: %s",
 388                                         rte_strerror(rc));
 389                 }
 390         }
 391
 392         return 0;
 393 }
 394
 395 static void
 396 sfc_mae_counter_service_unregister(struct sfc_adapter *sa)
 397 {
 398         struct sfc_mae_counter_registry *registry =
 399                 &sa->mae.counter_registry;
 400         const unsigned int wait_ms = 10000;
 401         unsigned int i;
 402
 403         rte_service_runstate_set(registry->service_id, 0);
 404         rte_service_component_runstate_set(registry->service_id, 0);
 405
 406         /*
 407          * Wait for the counter routine to finish the last iteration.
 408          * Give up on timeout.
 409          */
 410         for (i = 0; i < wait_ms; i++) {
 411                 if (rte_service_may_be_active(registry->service_id) == 0)
 412                         break;
 413
 414                 rte_delay_ms(1);
 415         }
 416         if (i == wait_ms)
 417                 sfc_warn(sa, "failed to wait for counter service to stop");
 418
 419         rte_service_map_lcore_set(registry->service_id,
 420                                   registry->service_core_id, 0);
 421
 422         rte_service_component_unregister(registry->service_id);
 423 }
 424
 425 static struct sfc_rxq_info *
 426 sfc_counter_rxq_info_get(struct sfc_adapter *sa)
 427 {
 428         return &sfc_sa2shared(sa)->rxq_info[sa->counter_rxq.sw_index];
 429 }
 430
 431 static int
 432 sfc_mae_counter_service_register(struct sfc_adapter *sa,
 433                                  uint32_t counter_stream_flags)
 434 {
 435         struct rte_service_spec service;
 436         char counter_service_name[sizeof(service.name)] = "counter_service";
 437         struct sfc_mae_counter_registry *counter_registry =
 438                 &sa->mae.counter_registry;
 439         uint32_t cid;
 440         uint32_t sid;
 441         int rc;
 442
 443         sfc_log_init(sa, "entry");
 444
 445         /* Prepare service info */
 446         memset(&service, 0, sizeof(service));
 447         rte_strscpy(service.name, counter_service_name, sizeof(service.name));
 448         service.socket_id = sa->socket_id;
 449         service.callback = sfc_mae_counter_routine;
 450         service.callback_userdata = sa;
 451         counter_registry->rx_pkt_burst = sa->eth_dev->rx_pkt_burst;
 452         counter_registry->rx_dp = sfc_counter_rxq_info_get(sa)->dp;
 453         counter_registry->pushed_n_buffers = 0;
 454         counter_registry->use_credits = counter_stream_flags &
 455                 EFX_MAE_COUNTERS_STREAM_OUT_USES_CREDITS;
 456
 457         cid = sfc_get_service_lcore(sa->socket_id);
 458         if (cid == RTE_MAX_LCORE && sa->socket_id != SOCKET_ID_ANY) {
 459                 /* Warn and try to allocate on any NUMA node */
 460                 sfc_warn(sa,
 461                         "failed to get service lcore for counter service at socket %d",
 462                         sa->socket_id);
 463
 464                 cid = sfc_get_service_lcore(SOCKET_ID_ANY);
 465         }
 466         if (cid == RTE_MAX_LCORE) {
 467                 rc = ENOTSUP;
 468                 sfc_err(sa, "failed to get service lcore for counter service");
 469                 goto fail_get_service_lcore;
 470         }
 471
 472         /* Service core may be in "stopped" state, start it */
 473         rc = rte_service_lcore_start(cid);
 474         if (rc != 0 && rc != -EALREADY) {
 475                 sfc_err(sa, "failed to start service core for counter service: %s",
 476                         rte_strerror(-rc));
 477                 rc = ENOTSUP;
 478                 goto fail_start_core;
 479         }
 480
 481         /* Register counter service */
 482         rc = rte_service_component_register(&service, &sid);
 483         if (rc != 0) {
 484                 rc = ENOEXEC;
 485                 sfc_err(sa, "failed to register counter service component");
 486                 goto fail_register;
 487         }
 488
 489         /* Map the service with the service core */
 490         rc = rte_service_map_lcore_set(sid, cid, 1);
 491         if (rc != 0) {
 492                 rc = -rc;
 493                 sfc_err(sa, "failed to map lcore for counter service: %s",
 494                         rte_strerror(rc));
 495                 goto fail_map_lcore;
 496         }
 497
 498         /* Run the service */
 499         rc = rte_service_component_runstate_set(sid, 1);
 500         if (rc < 0) {
 501                 rc = -rc;
 502                 sfc_err(sa, "failed to run counter service component: %s",
 503                         rte_strerror(rc));
 504                 goto fail_component_runstate_set;
 505         }
 506         rc = rte_service_runstate_set(sid, 1);
 507         if (rc < 0) {
 508                 rc = -rc;
 509                 sfc_err(sa, "failed to run counter service");
 510                 goto fail_runstate_set;
 511         }
 512
 513         counter_registry->service_core_id = cid;
 514         counter_registry->service_id = sid;
 515
 516         sfc_log_init(sa, "done");
 517
 518         return 0;
 519
 520 fail_runstate_set:
 521         rte_service_component_runstate_set(sid, 0);
 522
 523 fail_component_runstate_set:
 524         rte_service_map_lcore_set(sid, cid, 0);
 525
 526 fail_map_lcore:
 527         rte_service_component_unregister(sid);
 528
 529 fail_register:
 530 fail_start_core:
 531 fail_get_service_lcore:
 532         sfc_log_init(sa, "failed: %s", rte_strerror(rc));
 533
 534         return rc;
 535 }
 536
 537 int
 538 sfc_mae_counters_init(struct sfc_mae_counters *counters,
 539                       uint32_t nb_counters_max)
 540 {
 541         int rc;
 542
 543         SFC_GENERIC_LOG(DEBUG, "%s: entry", __func__);
 544
 545         counters->mae_counters = rte_zmalloc("sfc_mae_counters",
 546                 sizeof(*counters->mae_counters) * nb_counters_max, 0);
 547         if (counters->mae_counters == NULL) {
 548                 rc = ENOMEM;
 549                 SFC_GENERIC_LOG(ERR, "%s: failed: %s", __func__,
 550                                 rte_strerror(rc));
 551                 return rc;
 552         }
 553
 554         counters->n_mae_counters = nb_counters_max;
 555
 556         SFC_GENERIC_LOG(DEBUG, "%s: done", __func__);
 557
 558         return 0;
 559 }
 560
 561 void
 562 sfc_mae_counters_fini(struct sfc_mae_counters *counters)
 563 {
 564         rte_free(counters->mae_counters);
 565         counters->mae_counters = NULL;
 566 }
 567
 568 int
 569 sfc_mae_counter_rxq_attach(struct sfc_adapter *sa)
 570 {
 571         struct sfc_adapter_shared * const sas = sfc_sa2shared(sa);
 572         char name[RTE_MEMPOOL_NAMESIZE];
 573         struct rte_mempool *mp;
 574         unsigned int n_elements;
 575         unsigned int cache_size;
 576         /* The mempool is internal and private area is not required */
 577         const uint16_t priv_size = 0;
 578         const uint16_t data_room_size = RTE_PKTMBUF_HEADROOM +
 579                 SFC_MAE_COUNTER_STREAM_PACKET_SIZE;
 580         int rc;
 581
 582         sfc_log_init(sa, "entry");
 583
 584         if (!sas->counters_rxq_allocated) {
 585                 sfc_log_init(sa, "counter queue is not supported - skip");
 586                 return 0;
 587         }
 588
 589         /*
 590          * At least one element in the ring is always unused to distinguish
 591          * between empty and full ring cases.
 592          */
 593         n_elements = SFC_COUNTER_RXQ_RX_DESC_COUNT - 1;
 594
 595         /*
 596          * The cache must have sufficient space to put received buckets
 597          * before they're reused on refill.
 598          */
 599         cache_size = rte_align32pow2(SFC_COUNTER_RXQ_REFILL_LEVEL +
 600                                      SFC_MAE_COUNTER_RX_BURST - 1);
 601
 602         if (snprintf(name, sizeof(name), "counter_rxq-pool-%u", sas->port_id) >=
 603             (int)sizeof(name)) {
 604                 sfc_err(sa, "failed: counter RxQ mempool name is too long");
 605                 rc = ENAMETOOLONG;
 606                 goto fail_long_name;
 607         }
 608
 609         /*
 610          * It could be single-producer single-consumer ring mempool which
 611          * requires minimal barriers. However, cache size and refill/burst
 612          * policy are aligned, therefore it does not matter which
 613          * mempool backend is chosen since backend is unused.
 614          */
 615         mp = rte_pktmbuf_pool_create(name, n_elements, cache_size,
 616                                      priv_size, data_room_size, sa->socket_id);
 617         if (mp == NULL) {
 618                 sfc_err(sa, "failed to create counter RxQ mempool");
 619                 rc = rte_errno;
 620                 goto fail_mp_create;
 621         }
 622
 623         sa->counter_rxq.sw_index = sfc_counters_rxq_sw_index(sas);
 624         sa->counter_rxq.mp = mp;
 625         sa->counter_rxq.state |= SFC_COUNTER_RXQ_ATTACHED;
 626
 627         sfc_log_init(sa, "done");
 628
 629         return 0;
 630
 631 fail_mp_create:
 632 fail_long_name:
 633         sfc_log_init(sa, "failed: %s", rte_strerror(rc));
 634
 635         return rc;
 636 }
 637
 638 void
 639 sfc_mae_counter_rxq_detach(struct sfc_adapter *sa)
 640 {
 641         struct sfc_adapter_shared * const sas = sfc_sa2shared(sa);
 642
 643         sfc_log_init(sa, "entry");
 644
 645         if (!sas->counters_rxq_allocated) {
 646                 sfc_log_init(sa, "counter queue is not supported - skip");
 647                 return;
 648         }
 649
 650         if ((sa->counter_rxq.state & SFC_COUNTER_RXQ_ATTACHED) == 0) {
 651                 sfc_log_init(sa, "counter queue is not attached - skip");
 652                 return;
 653         }
 654
 655         rte_mempool_free(sa->counter_rxq.mp);
 656         sa->counter_rxq.mp = NULL;
 657         sa->counter_rxq.state &= ~SFC_COUNTER_RXQ_ATTACHED;
 658
 659         sfc_log_init(sa, "done");
 660 }
 661
 662 int
 663 sfc_mae_counter_rxq_init(struct sfc_adapter *sa)
 664 {
 665         struct sfc_adapter_shared * const sas = sfc_sa2shared(sa);
 666         const struct rte_eth_rxconf rxconf = {
 667                 .rx_free_thresh = SFC_COUNTER_RXQ_REFILL_LEVEL,
 668                 .rx_drop_en = 1,
 669         };
 670         uint16_t nb_rx_desc = SFC_COUNTER_RXQ_RX_DESC_COUNT;
 671         int rc;
 672
 673         sfc_log_init(sa, "entry");
 674
 675         if (!sas->counters_rxq_allocated) {
 676                 sfc_log_init(sa, "counter queue is not supported - skip");
 677                 return 0;
 678         }
 679
 680         if ((sa->counter_rxq.state & SFC_COUNTER_RXQ_ATTACHED) == 0) {
 681                 sfc_log_init(sa, "counter queue is not attached - skip");
 682                 return 0;
 683         }
 684
 685         nb_rx_desc = RTE_MIN(nb_rx_desc, sa->rxq_max_entries);
 686         nb_rx_desc = RTE_MAX(nb_rx_desc, sa->rxq_min_entries);
 687
 688         rc = sfc_rx_qinit_info(sa, sa->counter_rxq.sw_index,
 689                                EFX_RXQ_FLAG_USER_MARK);
 690         if (rc != 0)
 691                 goto fail_counter_rxq_init_info;
 692
 693         rc = sfc_rx_qinit(sa, sa->counter_rxq.sw_index, nb_rx_desc,
 694                           sa->socket_id, &rxconf, sa->counter_rxq.mp);
 695         if (rc != 0) {
 696                 sfc_err(sa, "failed to init counter RxQ");
 697                 goto fail_counter_rxq_init;
 698         }
 699
 700         sa->counter_rxq.state |= SFC_COUNTER_RXQ_INITIALIZED;
 701
 702         sfc_log_init(sa, "done");
 703
 704         return 0;
 705
 706 fail_counter_rxq_init:
 707 fail_counter_rxq_init_info:
 708         sfc_log_init(sa, "failed: %s", rte_strerror(rc));
 709
 710         return rc;
 711 }
 712
 713 void
 714 sfc_mae_counter_rxq_fini(struct sfc_adapter *sa)
 715 {
 716         struct sfc_adapter_shared * const sas = sfc_sa2shared(sa);
 717
 718         sfc_log_init(sa, "entry");
 719
 720         if (!sas->counters_rxq_allocated) {
 721                 sfc_log_init(sa, "counter queue is not supported - skip");
 722                 return;
 723         }
 724
 725         if ((sa->counter_rxq.state & SFC_COUNTER_RXQ_INITIALIZED) == 0) {
 726                 sfc_log_init(sa, "counter queue is not initialized - skip");
 727                 return;
 728         }
 729
 730         sfc_rx_qfini(sa, sa->counter_rxq.sw_index);
 731
 732         sfc_log_init(sa, "done");
 733 }
 734
 735 void
 736 sfc_mae_counter_stop(struct sfc_adapter *sa)
 737 {
 738         struct sfc_mae *mae = &sa->mae;
 739
 740         sfc_log_init(sa, "entry");
 741
 742         if (!mae->counter_rxq_running) {
 743                 sfc_log_init(sa, "counter queue is not running - skip");
 744                 return;
 745         }
 746
 747         sfc_mae_counter_service_unregister(sa);
 748         efx_mae_counters_stream_stop(sa->nic, sa->counter_rxq.sw_index, NULL);
 749
 750         mae->counter_rxq_running = false;
 751
 752         sfc_log_init(sa, "done");
 753 }
 754
 755 int
 756 sfc_mae_counter_start(struct sfc_adapter *sa)
 757 {
 758         struct sfc_mae *mae = &sa->mae;
 759         uint32_t flags;
 760         int rc;
 761
 762         SFC_ASSERT(sa->counter_rxq.state & SFC_COUNTER_RXQ_ATTACHED);
 763
 764         if (mae->counter_rxq_running)
 765                 return 0;
 766
 767         sfc_log_init(sa, "entry");
 768
 769         rc = efx_mae_counters_stream_start(sa->nic, sa->counter_rxq.sw_index,
 770                                            SFC_MAE_COUNTER_STREAM_PACKET_SIZE,
 771                                            0 /* No flags required */, &flags);
 772         if (rc != 0) {
 773                 sfc_err(sa, "failed to start MAE counters stream: %s",
 774                         rte_strerror(rc));
 775                 goto fail_counter_stream;
 776         }
 777
 778         sfc_log_init(sa, "stream start flags: 0x%x", flags);
 779
 780         rc = sfc_mae_counter_service_register(sa, flags);
 781         if (rc != 0)
 782                 goto fail_service_register;
 783
 784         mae->counter_rxq_running = true;
 785
 786         return 0;
 787
 788 fail_service_register:
 789         efx_mae_counters_stream_stop(sa->nic, sa->counter_rxq.sw_index, NULL);
 790
 791 fail_counter_stream:
 792         sfc_log_init(sa, "failed: %s", rte_strerror(rc));
 793
 794         return rc;
 795 }
 796
 797 int
 798 sfc_mae_counter_get(struct sfc_mae_counters *counters,
 799                     const struct sfc_mae_counter_id *counter,
 800                     struct rte_flow_query_count *data)
 801 {
 802         struct sfc_mae_counter *p;
 803         union sfc_pkts_bytes value;
 804
 805         SFC_ASSERT(counter->mae_id.id < counters->n_mae_counters);
 806         p = &counters->mae_counters[counter->mae_id.id];
 807
 808         /*
 809          * Ordering is relaxed since it is the only operation on counter value.
 810          * And it does not depend on different stores/loads in other threads.
 811          * Paired with relaxed ordering in counter increment.
 812          */
 813         value.pkts_bytes.int128 = __atomic_load_n(&p->value.pkts_bytes.int128,
 814                                                   __ATOMIC_RELAXED);
 815
 816         data->hits_set = 1;
 817         data->bytes_set = 1;
 818         data->hits = value.pkts - p->reset.pkts;
 819         data->bytes = value.bytes - p->reset.bytes;
 820
 821         if (data->reset != 0) {
 822                 p->reset.pkts = value.pkts;
 823                 p->reset.bytes = value.bytes;
 824         }
 825
 826         return 0;
 827 }