drivers/net/sfc/sfc_ef10_rx.c

   1 /* SPDX-License-Identifier: BSD-3-Clause
   2  *
   3  * Copyright (c) 2016-2018 Solarflare Communications Inc.
   4  * All rights reserved.
   5  *
   6  * This software was jointly developed between OKTET Labs (under contract
   7  * for Solarflare) and Solarflare Communications, Inc.
   8  */
   9
  10 /* EF10 native datapath implementation */
  11
  12 #include <stdbool.h>
  13
  14 #include <rte_byteorder.h>
  15 #include <rte_mbuf_ptype.h>
  16 #include <rte_mbuf.h>
  17 #include <rte_io.h>
  18
  19 #include "efx.h"
  20 #include "efx_types.h"
  21 #include "efx_regs.h"
  22 #include "efx_regs_ef10.h"
  23
  24 #include "sfc_tweak.h"
  25 #include "sfc_dp_rx.h"
  26 #include "sfc_kvargs.h"
  27 #include "sfc_ef10.h"
  28
  29 #define SFC_EF10_RX_EV_ENCAP_SUPPORT    1
  30 #include "sfc_ef10_rx_ev.h"
  31
  32 #define sfc_ef10_rx_err(dpq, ...) \
  33         SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10, ERR, dpq, __VA_ARGS__)
  34
  35 /**
  36  * Maximum number of descriptors/buffers in the Rx ring.
  37  * It should guarantee that corresponding event queue never overfill.
  38  * EF10 native datapath uses event queue of the same size as Rx queue.
  39  * Maximum number of events on datapath can be estimated as number of
  40  * Rx queue entries (one event per Rx buffer in the worst case) plus
  41  * Rx error and flush events.
  42  */
  43 #define SFC_EF10_RXQ_LIMIT(_ndesc) \
  44         ((_ndesc) - 1 /* head must not step on tail */ - \
  45          (SFC_EF10_EV_PER_CACHE_LINE - 1) /* max unused EvQ entries */ - \
  46          1 /* Rx error */ - 1 /* flush */)
  47
  48 struct sfc_ef10_rx_sw_desc {
  49         struct rte_mbuf                 *mbuf;
  50 };
  51
  52 struct sfc_ef10_rxq {
  53         /* Used on data path */
  54         unsigned int                    flags;
  55 #define SFC_EF10_RXQ_STARTED            0x1
  56 #define SFC_EF10_RXQ_NOT_RUNNING        0x2
  57 #define SFC_EF10_RXQ_EXCEPTION          0x4
  58 #define SFC_EF10_RXQ_RSS_HASH           0x8
  59         unsigned int                    ptr_mask;
  60         unsigned int                    prepared;
  61         unsigned int                    completed;
  62         unsigned int                    evq_read_ptr;
  63         efx_qword_t                     *evq_hw_ring;
  64         struct sfc_ef10_rx_sw_desc      *sw_ring;
  65         uint64_t                        rearm_data;
  66         uint16_t                        prefix_size;
  67
  68         /* Used on refill */
  69         uint16_t                        buf_size;
  70         unsigned int                    added;
  71         unsigned int                    max_fill_level;
  72         unsigned int                    refill_threshold;
  73         struct rte_mempool              *refill_mb_pool;
  74         efx_qword_t                     *rxq_hw_ring;
  75         volatile void                   *doorbell;
  76
  77         /* Datapath receive queue anchor */
  78         struct sfc_dp_rxq               dp;
  79 };
  80
  81 static inline struct sfc_ef10_rxq *
  82 sfc_ef10_rxq_by_dp_rxq(struct sfc_dp_rxq *dp_rxq)
  83 {
  84         return container_of(dp_rxq, struct sfc_ef10_rxq, dp);
  85 }
  86
  87 static void
  88 sfc_ef10_rx_qrefill(struct sfc_ef10_rxq *rxq)
  89 {
  90         const unsigned int ptr_mask = rxq->ptr_mask;
  91         const uint32_t buf_size = rxq->buf_size;
  92         unsigned int free_space;
  93         unsigned int bulks;
  94         void *objs[SFC_RX_REFILL_BULK];
  95         unsigned int added = rxq->added;
  96
  97         RTE_BUILD_BUG_ON(SFC_RX_REFILL_BULK % SFC_EF10_RX_WPTR_ALIGN != 0);
  98
  99         free_space = rxq->max_fill_level - (added - rxq->completed);
 100
 101         if (free_space < rxq->refill_threshold)
 102                 return;
 103
 104         bulks = free_space / RTE_DIM(objs);
 105         /* refill_threshold guarantees that bulks is positive */
 106         SFC_ASSERT(bulks > 0);
 107
 108         do {
 109                 unsigned int id;
 110                 unsigned int i;
 111
 112                 if (unlikely(rte_mempool_get_bulk(rxq->refill_mb_pool, objs,
 113                                                   RTE_DIM(objs)) < 0)) {
 114                         struct rte_eth_dev_data *dev_data =
 115                                 rte_eth_devices[rxq->dp.dpq.port_id].data;
 116
 117                         /*
 118                          * It is hardly a safe way to increment counter
 119                          * from different contexts, but all PMDs do it.
 120                          */
 121                         dev_data->rx_mbuf_alloc_failed += RTE_DIM(objs);
 122                         /* Return if we have posted nothing yet */
 123                         if (added == rxq->added)
 124                                 return;
 125                         /* Push posted */
 126                         break;
 127                 }
 128
 129                 for (i = 0, id = added & ptr_mask;
 130                      i < RTE_DIM(objs);
 131                      ++i, ++id) {
 132                         struct rte_mbuf *m = objs[i];
 133                         struct sfc_ef10_rx_sw_desc *rxd;
 134                         rte_iova_t phys_addr;
 135
 136                         MBUF_RAW_ALLOC_CHECK(m);
 137
 138                         SFC_ASSERT((id & ~ptr_mask) == 0);
 139                         rxd = &rxq->sw_ring[id];
 140                         rxd->mbuf = m;
 141
 142                         /*
 143                          * Avoid writing to mbuf. It is cheaper to do it
 144                          * when we receive packet and fill in nearby
 145                          * structure members.
 146                          */
 147
 148                         phys_addr = rte_mbuf_data_iova_default(m);
 149                         EFX_POPULATE_QWORD_2(rxq->rxq_hw_ring[id],
 150                             ESF_DZ_RX_KER_BYTE_CNT, buf_size,
 151                             ESF_DZ_RX_KER_BUF_ADDR, phys_addr);
 152                 }
 153
 154                 added += RTE_DIM(objs);
 155         } while (--bulks > 0);
 156
 157         SFC_ASSERT(rxq->added != added);
 158         rxq->added = added;
 159         sfc_ef10_rx_qpush(rxq->doorbell, added, ptr_mask);
 160 }
 161
 162 static void
 163 sfc_ef10_rx_prefetch_next(struct sfc_ef10_rxq *rxq, unsigned int next_id)
 164 {
 165         struct rte_mbuf *next_mbuf;
 166
 167         /* Prefetch next bunch of software descriptors */
 168         if ((next_id % (RTE_CACHE_LINE_SIZE / sizeof(rxq->sw_ring[0]))) == 0)
 169                 rte_prefetch0(&rxq->sw_ring[next_id]);
 170
 171         /*
 172          * It looks strange to prefetch depending on previous prefetch
 173          * data, but measurements show that it is really efficient and
 174          * increases packet rate.
 175          */
 176         next_mbuf = rxq->sw_ring[next_id].mbuf;
 177         if (likely(next_mbuf != NULL)) {
 178                 /* Prefetch the next mbuf structure */
 179                 rte_mbuf_prefetch_part1(next_mbuf);
 180
 181                 /* Prefetch pseudo header of the next packet */
 182                 /* data_off is not filled in yet */
 183                 /* Yes, data could be not ready yet, but we hope */
 184                 rte_prefetch0((uint8_t *)next_mbuf->buf_addr +
 185                               RTE_PKTMBUF_HEADROOM);
 186         }
 187 }
 188
 189 static uint16_t
 190 sfc_ef10_rx_prepared(struct sfc_ef10_rxq *rxq, struct rte_mbuf **rx_pkts,
 191                      uint16_t nb_pkts)
 192 {
 193         uint16_t n_rx_pkts = RTE_MIN(nb_pkts, rxq->prepared);
 194         unsigned int completed = rxq->completed;
 195         unsigned int i;
 196
 197         rxq->prepared -= n_rx_pkts;
 198         rxq->completed = completed + n_rx_pkts;
 199
 200         for (i = 0; i < n_rx_pkts; ++i, ++completed)
 201                 rx_pkts[i] = rxq->sw_ring[completed & rxq->ptr_mask].mbuf;
 202
 203         return n_rx_pkts;
 204 }
 205
 206 static uint16_t
 207 sfc_ef10_rx_pseudo_hdr_get_len(const uint8_t *pseudo_hdr)
 208 {
 209         return rte_le_to_cpu_16(*(const uint16_t *)&pseudo_hdr[8]);
 210 }
 211
 212 static uint32_t
 213 sfc_ef10_rx_pseudo_hdr_get_hash(const uint8_t *pseudo_hdr)
 214 {
 215         return rte_le_to_cpu_32(*(const uint32_t *)pseudo_hdr);
 216 }
 217
 218 static uint16_t
 219 sfc_ef10_rx_process_event(struct sfc_ef10_rxq *rxq, efx_qword_t rx_ev,
 220                           struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 221 {
 222         const unsigned int ptr_mask = rxq->ptr_mask;
 223         unsigned int completed = rxq->completed;
 224         unsigned int ready;
 225         struct sfc_ef10_rx_sw_desc *rxd;
 226         struct rte_mbuf *m;
 227         struct rte_mbuf *m0;
 228         uint16_t n_rx_pkts;
 229         const uint8_t *pseudo_hdr;
 230         uint16_t pkt_len;
 231
 232         ready = (EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_DSC_PTR_LBITS) - completed) &
 233                 EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS);
 234         SFC_ASSERT(ready > 0);
 235
 236         if (rx_ev.eq_u64[0] &
 237             rte_cpu_to_le_64((1ull << ESF_DZ_RX_ECC_ERR_LBN) |
 238                              (1ull << ESF_DZ_RX_ECRC_ERR_LBN))) {
 239                 SFC_ASSERT(rxq->prepared == 0);
 240                 rxq->completed += ready;
 241                 while (ready-- > 0) {
 242                         rxd = &rxq->sw_ring[completed++ & ptr_mask];
 243                         rte_mbuf_raw_free(rxd->mbuf);
 244                 }
 245                 return 0;
 246         }
 247
 248         n_rx_pkts = RTE_MIN(ready, nb_pkts);
 249         rxq->prepared = ready - n_rx_pkts;
 250         rxq->completed += n_rx_pkts;
 251
 252         rxd = &rxq->sw_ring[completed++ & ptr_mask];
 253
 254         sfc_ef10_rx_prefetch_next(rxq, completed & ptr_mask);
 255
 256         m = rxd->mbuf;
 257
 258         *rx_pkts++ = m;
 259
 260         RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) != sizeof(rxq->rearm_data));
 261         m->rearm_data[0] = rxq->rearm_data;
 262
 263         /* Classify packet based on Rx event */
 264         /* Mask RSS hash offload flag if RSS is not enabled */
 265         sfc_ef10_rx_ev_to_offloads(rx_ev, m,
 266                                    (rxq->flags & SFC_EF10_RXQ_RSS_HASH) ?
 267                                    ~0ull : ~PKT_RX_RSS_HASH);
 268
 269         /* data_off already moved past pseudo header */
 270         pseudo_hdr = (uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM;
 271
 272         /*
 273          * Always get RSS hash from pseudo header to avoid
 274          * condition/branching. If it is valid or not depends on
 275          * PKT_RX_RSS_HASH in m->ol_flags.
 276          */
 277         m->hash.rss = sfc_ef10_rx_pseudo_hdr_get_hash(pseudo_hdr);
 278
 279         if (ready == 1)
 280                 pkt_len = EFX_QWORD_FIELD(rx_ev, ESF_DZ_RX_BYTES) -
 281                         rxq->prefix_size;
 282         else
 283                 pkt_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr);
 284         SFC_ASSERT(pkt_len > 0);
 285         rte_pktmbuf_data_len(m) = pkt_len;
 286         rte_pktmbuf_pkt_len(m) = pkt_len;
 287
 288         SFC_ASSERT(m->next == NULL);
 289
 290         /* Remember mbuf to copy offload flags and packet type from */
 291         m0 = m;
 292         for (--ready; ready > 0; --ready) {
 293                 rxd = &rxq->sw_ring[completed++ & ptr_mask];
 294
 295                 sfc_ef10_rx_prefetch_next(rxq, completed & ptr_mask);
 296
 297                 m = rxd->mbuf;
 298
 299                 if (ready > rxq->prepared)
 300                         *rx_pkts++ = m;
 301
 302                 RTE_BUILD_BUG_ON(sizeof(m->rearm_data[0]) !=
 303                                  sizeof(rxq->rearm_data));
 304                 m->rearm_data[0] = rxq->rearm_data;
 305
 306                 /* Event-dependent information is the same */
 307                 m->ol_flags = m0->ol_flags;
 308                 m->packet_type = m0->packet_type;
 309
 310                 /* data_off already moved past pseudo header */
 311                 pseudo_hdr = (uint8_t *)m->buf_addr + RTE_PKTMBUF_HEADROOM;
 312
 313                 /*
 314                  * Always get RSS hash from pseudo header to avoid
 315                  * condition/branching. If it is valid or not depends on
 316                  * PKT_RX_RSS_HASH in m->ol_flags.
 317                  */
 318                 m->hash.rss = sfc_ef10_rx_pseudo_hdr_get_hash(pseudo_hdr);
 319
 320                 pkt_len = sfc_ef10_rx_pseudo_hdr_get_len(pseudo_hdr);
 321                 SFC_ASSERT(pkt_len > 0);
 322                 rte_pktmbuf_data_len(m) = pkt_len;
 323                 rte_pktmbuf_pkt_len(m) = pkt_len;
 324
 325                 SFC_ASSERT(m->next == NULL);
 326         }
 327
 328         return n_rx_pkts;
 329 }
 330
 331 static bool
 332 sfc_ef10_rx_get_event(struct sfc_ef10_rxq *rxq, efx_qword_t *rx_ev)
 333 {
 334         *rx_ev = rxq->evq_hw_ring[rxq->evq_read_ptr & rxq->ptr_mask];
 335
 336         if (!sfc_ef10_ev_present(*rx_ev))
 337                 return false;
 338
 339         if (unlikely(EFX_QWORD_FIELD(*rx_ev, FSF_AZ_EV_CODE) !=
 340                      FSE_AZ_EV_CODE_RX_EV)) {
 341                 /*
 342                  * Do not move read_ptr to keep the event for exception
 343                  * handling by the control path.
 344                  */
 345                 rxq->flags |= SFC_EF10_RXQ_EXCEPTION;
 346                 sfc_ef10_rx_err(&rxq->dp.dpq,
 347                                 "RxQ exception at EvQ read ptr %#x",
 348                                 rxq->evq_read_ptr);
 349                 return false;
 350         }
 351
 352         rxq->evq_read_ptr++;
 353         return true;
 354 }
 355
 356 static uint16_t
 357 sfc_ef10_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)
 358 {
 359         struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(rx_queue);
 360         unsigned int evq_old_read_ptr;
 361         uint16_t n_rx_pkts;
 362         efx_qword_t rx_ev;
 363
 364         n_rx_pkts = sfc_ef10_rx_prepared(rxq, rx_pkts, nb_pkts);
 365
 366         if (unlikely(rxq->flags &
 367                      (SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION)))
 368                 goto done;
 369
 370         evq_old_read_ptr = rxq->evq_read_ptr;
 371         while (n_rx_pkts != nb_pkts && sfc_ef10_rx_get_event(rxq, &rx_ev)) {
 372                 /*
 373                  * DROP_EVENT is an internal to the NIC, software should
 374                  * never see it and, therefore, may ignore it.
 375                  */
 376
 377                 n_rx_pkts += sfc_ef10_rx_process_event(rxq, rx_ev,
 378                                                        rx_pkts + n_rx_pkts,
 379                                                        nb_pkts - n_rx_pkts);
 380         }
 381
 382         sfc_ef10_ev_qclear(rxq->evq_hw_ring, rxq->ptr_mask, evq_old_read_ptr,
 383                            rxq->evq_read_ptr);
 384
 385         /* It is not a problem if we refill in the case of exception */
 386         sfc_ef10_rx_qrefill(rxq);
 387
 388 done:
 389         return n_rx_pkts;
 390 }
 391
 392 const uint32_t *
 393 sfc_ef10_supported_ptypes_get(uint32_t tunnel_encaps)
 394 {
 395         static const uint32_t ef10_native_ptypes[] = {
 396                 RTE_PTYPE_L2_ETHER,
 397                 RTE_PTYPE_L2_ETHER_ARP,
 398                 RTE_PTYPE_L2_ETHER_VLAN,
 399                 RTE_PTYPE_L2_ETHER_QINQ,
 400                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
 401                 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
 402                 RTE_PTYPE_L4_FRAG,
 403                 RTE_PTYPE_L4_TCP,
 404                 RTE_PTYPE_L4_UDP,
 405                 RTE_PTYPE_UNKNOWN
 406         };
 407         static const uint32_t ef10_overlay_ptypes[] = {
 408                 RTE_PTYPE_L2_ETHER,
 409                 RTE_PTYPE_L2_ETHER_ARP,
 410                 RTE_PTYPE_L2_ETHER_VLAN,
 411                 RTE_PTYPE_L2_ETHER_QINQ,
 412                 RTE_PTYPE_L3_IPV4_EXT_UNKNOWN,
 413                 RTE_PTYPE_L3_IPV6_EXT_UNKNOWN,
 414                 RTE_PTYPE_L4_FRAG,
 415                 RTE_PTYPE_L4_TCP,
 416                 RTE_PTYPE_L4_UDP,
 417                 RTE_PTYPE_TUNNEL_VXLAN,
 418                 RTE_PTYPE_TUNNEL_NVGRE,
 419                 RTE_PTYPE_INNER_L2_ETHER,
 420                 RTE_PTYPE_INNER_L2_ETHER_VLAN,
 421                 RTE_PTYPE_INNER_L2_ETHER_QINQ,
 422                 RTE_PTYPE_INNER_L3_IPV4_EXT_UNKNOWN,
 423                 RTE_PTYPE_INNER_L3_IPV6_EXT_UNKNOWN,
 424                 RTE_PTYPE_INNER_L4_FRAG,
 425                 RTE_PTYPE_INNER_L4_TCP,
 426                 RTE_PTYPE_INNER_L4_UDP,
 427                 RTE_PTYPE_UNKNOWN
 428         };
 429
 430         /*
 431          * The function returns static set of supported packet types,
 432          * so we can't build it dynamically based on supported tunnel
 433          * encapsulations and should limit to known sets.
 434          */
 435         switch (tunnel_encaps) {
 436         case (1u << EFX_TUNNEL_PROTOCOL_VXLAN |
 437               1u << EFX_TUNNEL_PROTOCOL_GENEVE |
 438               1u << EFX_TUNNEL_PROTOCOL_NVGRE):
 439                 return ef10_overlay_ptypes;
 440         default:
 441                 SFC_GENERIC_LOG(ERR,
 442                         "Unexpected set of supported tunnel encapsulations: %#x",
 443                         tunnel_encaps);
 444                 /* FALLTHROUGH */
 445         case 0:
 446                 return ef10_native_ptypes;
 447         }
 448 }
 449
 450 static sfc_dp_rx_qdesc_npending_t sfc_ef10_rx_qdesc_npending;
 451 static unsigned int
 452 sfc_ef10_rx_qdesc_npending(__rte_unused struct sfc_dp_rxq *dp_rxq)
 453 {
 454         /*
 455          * Correct implementation requires EvQ polling and events
 456          * processing (keeping all ready mbufs in prepared).
 457          */
 458         return -ENOTSUP;
 459 }
 460
 461 static sfc_dp_rx_qdesc_status_t sfc_ef10_rx_qdesc_status;
 462 static int
 463 sfc_ef10_rx_qdesc_status(__rte_unused struct sfc_dp_rxq *dp_rxq,
 464                          __rte_unused uint16_t offset)
 465 {
 466         return -ENOTSUP;
 467 }
 468
 469
 470 static sfc_dp_rx_get_dev_info_t sfc_ef10_rx_get_dev_info;
 471 static void
 472 sfc_ef10_rx_get_dev_info(struct rte_eth_dev_info *dev_info)
 473 {
 474         /*
 475          * Number of descriptors just defines maximum number of pushed
 476          * descriptors (fill level).
 477          */
 478         dev_info->rx_desc_lim.nb_min = SFC_RX_REFILL_BULK;
 479         dev_info->rx_desc_lim.nb_align = SFC_RX_REFILL_BULK;
 480 }
 481
 482
 483 static sfc_dp_rx_qsize_up_rings_t sfc_ef10_rx_qsize_up_rings;
 484 static int
 485 sfc_ef10_rx_qsize_up_rings(uint16_t nb_rx_desc,
 486                            __rte_unused struct rte_mempool *mb_pool,
 487                            unsigned int *rxq_entries,
 488                            unsigned int *evq_entries,
 489                            unsigned int *rxq_max_fill_level)
 490 {
 491         /*
 492          * rte_ethdev API guarantees that the number meets min, max and
 493          * alignment requirements.
 494          */
 495         if (nb_rx_desc <= EFX_RXQ_MINNDESCS)
 496                 *rxq_entries = EFX_RXQ_MINNDESCS;
 497         else
 498                 *rxq_entries = rte_align32pow2(nb_rx_desc);
 499
 500         *evq_entries = *rxq_entries;
 501
 502         *rxq_max_fill_level = RTE_MIN(nb_rx_desc,
 503                                       SFC_EF10_RXQ_LIMIT(*evq_entries));
 504         return 0;
 505 }
 506
 507
 508 static uint64_t
 509 sfc_ef10_mk_mbuf_rearm_data(uint16_t port_id, uint16_t prefix_size)
 510 {
 511         struct rte_mbuf m;
 512
 513         memset(&m, 0, sizeof(m));
 514
 515         rte_mbuf_refcnt_set(&m, 1);
 516         m.data_off = RTE_PKTMBUF_HEADROOM + prefix_size;
 517         m.nb_segs = 1;
 518         m.port = port_id;
 519
 520         /* rearm_data covers structure members filled in above */
 521         rte_compiler_barrier();
 522         RTE_BUILD_BUG_ON(sizeof(m.rearm_data[0]) != sizeof(uint64_t));
 523         return m.rearm_data[0];
 524 }
 525
 526 static sfc_dp_rx_qcreate_t sfc_ef10_rx_qcreate;
 527 static int
 528 sfc_ef10_rx_qcreate(uint16_t port_id, uint16_t queue_id,
 529                     const struct rte_pci_addr *pci_addr, int socket_id,
 530                     const struct sfc_dp_rx_qcreate_info *info,
 531                     struct sfc_dp_rxq **dp_rxqp)
 532 {
 533         struct sfc_ef10_rxq *rxq;
 534         int rc;
 535
 536         rc = EINVAL;
 537         if (info->rxq_entries != info->evq_entries)
 538                 goto fail_rxq_args;
 539
 540         rc = ENOMEM;
 541         rxq = rte_zmalloc_socket("sfc-ef10-rxq", sizeof(*rxq),
 542                                  RTE_CACHE_LINE_SIZE, socket_id);
 543         if (rxq == NULL)
 544                 goto fail_rxq_alloc;
 545
 546         sfc_dp_queue_init(&rxq->dp.dpq, port_id, queue_id, pci_addr);
 547
 548         rc = ENOMEM;
 549         rxq->sw_ring = rte_calloc_socket("sfc-ef10-rxq-sw_ring",
 550                                          info->rxq_entries,
 551                                          sizeof(*rxq->sw_ring),
 552                                          RTE_CACHE_LINE_SIZE, socket_id);
 553         if (rxq->sw_ring == NULL)
 554                 goto fail_desc_alloc;
 555
 556         rxq->flags |= SFC_EF10_RXQ_NOT_RUNNING;
 557         if (info->flags & SFC_RXQ_FLAG_RSS_HASH)
 558                 rxq->flags |= SFC_EF10_RXQ_RSS_HASH;
 559         rxq->ptr_mask = info->rxq_entries - 1;
 560         rxq->evq_hw_ring = info->evq_hw_ring;
 561         rxq->max_fill_level = info->max_fill_level;
 562         rxq->refill_threshold = info->refill_threshold;
 563         rxq->rearm_data =
 564                 sfc_ef10_mk_mbuf_rearm_data(port_id, info->prefix_size);
 565         rxq->prefix_size = info->prefix_size;
 566         rxq->buf_size = info->buf_size;
 567         rxq->refill_mb_pool = info->refill_mb_pool;
 568         rxq->rxq_hw_ring = info->rxq_hw_ring;
 569         rxq->doorbell = (volatile uint8_t *)info->mem_bar +
 570                         ER_DZ_RX_DESC_UPD_REG_OFST +
 571                         (info->hw_index << info->vi_window_shift);
 572
 573         *dp_rxqp = &rxq->dp;
 574         return 0;
 575
 576 fail_desc_alloc:
 577         rte_free(rxq);
 578
 579 fail_rxq_alloc:
 580 fail_rxq_args:
 581         return rc;
 582 }
 583
 584 static sfc_dp_rx_qdestroy_t sfc_ef10_rx_qdestroy;
 585 static void
 586 sfc_ef10_rx_qdestroy(struct sfc_dp_rxq *dp_rxq)
 587 {
 588         struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq);
 589
 590         rte_free(rxq->sw_ring);
 591         rte_free(rxq);
 592 }
 593
 594 static sfc_dp_rx_qstart_t sfc_ef10_rx_qstart;
 595 static int
 596 sfc_ef10_rx_qstart(struct sfc_dp_rxq *dp_rxq, unsigned int evq_read_ptr)
 597 {
 598         struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq);
 599
 600         SFC_ASSERT(rxq->prepared == 0);
 601         SFC_ASSERT(rxq->completed == 0);
 602         SFC_ASSERT(rxq->added == 0);
 603
 604         sfc_ef10_rx_qrefill(rxq);
 605
 606         rxq->evq_read_ptr = evq_read_ptr;
 607
 608         rxq->flags |= SFC_EF10_RXQ_STARTED;
 609         rxq->flags &= ~(SFC_EF10_RXQ_NOT_RUNNING | SFC_EF10_RXQ_EXCEPTION);
 610
 611         return 0;
 612 }
 613
 614 static sfc_dp_rx_qstop_t sfc_ef10_rx_qstop;
 615 static void
 616 sfc_ef10_rx_qstop(struct sfc_dp_rxq *dp_rxq, unsigned int *evq_read_ptr)
 617 {
 618         struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq);
 619
 620         rxq->flags |= SFC_EF10_RXQ_NOT_RUNNING;
 621
 622         *evq_read_ptr = rxq->evq_read_ptr;
 623 }
 624
 625 static sfc_dp_rx_qrx_ev_t sfc_ef10_rx_qrx_ev;
 626 static bool
 627 sfc_ef10_rx_qrx_ev(struct sfc_dp_rxq *dp_rxq, __rte_unused unsigned int id)
 628 {
 629         __rte_unused struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq);
 630
 631         SFC_ASSERT(rxq->flags & SFC_EF10_RXQ_NOT_RUNNING);
 632
 633         /*
 634          * It is safe to ignore Rx event since we free all mbufs on
 635          * queue purge anyway.
 636          */
 637
 638         return false;
 639 }
 640
 641 static sfc_dp_rx_qpurge_t sfc_ef10_rx_qpurge;
 642 static void
 643 sfc_ef10_rx_qpurge(struct sfc_dp_rxq *dp_rxq)
 644 {
 645         struct sfc_ef10_rxq *rxq = sfc_ef10_rxq_by_dp_rxq(dp_rxq);
 646         unsigned int i;
 647         struct sfc_ef10_rx_sw_desc *rxd;
 648
 649         rxq->prepared = 0;
 650
 651         for (i = rxq->completed; i != rxq->added; ++i) {
 652                 rxd = &rxq->sw_ring[i & rxq->ptr_mask];
 653                 rte_mbuf_raw_free(rxd->mbuf);
 654                 rxd->mbuf = NULL;
 655         }
 656
 657         rxq->completed = rxq->added = 0;
 658
 659         rxq->flags &= ~SFC_EF10_RXQ_STARTED;
 660 }
 661
 662 struct sfc_dp_rx sfc_ef10_rx = {
 663         .dp = {
 664                 .name           = SFC_KVARG_DATAPATH_EF10,
 665                 .type           = SFC_DP_RX,
 666                 .hw_fw_caps     = SFC_DP_HW_FW_CAP_EF10,
 667         },
 668         .features               = SFC_DP_RX_FEAT_MULTI_PROCESS |
 669                                   SFC_DP_RX_FEAT_TUNNELS |
 670                                   SFC_DP_RX_FEAT_CHECKSUM,
 671         .get_dev_info           = sfc_ef10_rx_get_dev_info,
 672         .qsize_up_rings         = sfc_ef10_rx_qsize_up_rings,
 673         .qcreate                = sfc_ef10_rx_qcreate,
 674         .qdestroy               = sfc_ef10_rx_qdestroy,
 675         .qstart                 = sfc_ef10_rx_qstart,
 676         .qstop                  = sfc_ef10_rx_qstop,
 677         .qrx_ev                 = sfc_ef10_rx_qrx_ev,
 678         .qpurge                 = sfc_ef10_rx_qpurge,
 679         .supported_ptypes_get   = sfc_ef10_supported_ptypes_get,
 680         .qdesc_npending         = sfc_ef10_rx_qdesc_npending,
 681         .qdesc_status           = sfc_ef10_rx_qdesc_status,
 682         .pkt_burst              = sfc_ef10_recv_pkts,
 683 };